From 64e31fad5474ce7f6e6cc53746e898c9c4b84a49 Mon Sep 17 00:00:00 2001 From: Glax Date: Mon, 19 May 2025 17:36:32 +0200 Subject: [PATCH] Job-Cycle match JobTypes and MangaConnectors on running and waiting Jobs --- API/Tranga.cs | 139 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 84 insertions(+), 55 deletions(-) diff --git a/API/Tranga.cs b/API/Tranga.cs index 3b0e49a..76903ae 100644 --- a/API/Tranga.cs +++ b/API/Tranga.cs @@ -125,39 +125,28 @@ public static class Tranga using IServiceScope scope = serviceProvider.CreateScope(); PgsqlContext cycleContext = scope.ServiceProvider.GetRequiredService(); - List runningJobs = cycleContext.Jobs.Where(j => j.state == JobState.Running).ToList(); + //Get Running Jobs + List runningJobs = cycleContext.Jobs.GetRunningJobs(); DateTime filterStart = DateTime.UtcNow; Log.Debug("Filtering Jobs..."); - List busyConnectors = GetBusyConnectors(runningJobs); - List waitingJobs = cycleContext.Jobs.Where(j => j.state == JobState.CompletedWaiting || j.state == JobState.FirstExecution).ToList(); - List dueJobs = FilterDueJobs(waitingJobs); - List jobsWithoutBusyConnectors = FilterJobWithBusyConnectors(dueJobs, busyConnectors); - List jobsWithoutMissingDependencies = FilterJobDependencies(jobsWithoutBusyConnectors); + List waitingJobs = cycleContext.Jobs.GetWaitingJobs(); + List dueJobs = waitingJobs.FilterDueJobs(); + List jobsWithoutDependencies = dueJobs.FilterJobDependencies(); - List jobsWithoutDownloading = - jobsWithoutMissingDependencies - .Where(j => j.JobType != JobType.DownloadSingleChapterJob) - .DistinctBy(j => j.JobType) - .ToList(); - List firstChapterPerConnector = - jobsWithoutMissingDependencies - .Where(j => j.JobType == JobType.DownloadSingleChapterJob) - .AsEnumerable() - .OrderBy(j => - { - DownloadSingleChapterJob dscj = (DownloadSingleChapterJob)j; - return dscj.Chapter; - }) - .DistinctBy(j => - { - DownloadSingleChapterJob dscj = (DownloadSingleChapterJob)j; - return dscj.Chapter.ParentManga.MangaConnector; - }) - .ToList(); + List jobsWithoutDownloading = jobsWithoutDependencies.Where(j => GetJobConnector(j) is null).ToList(); + + //Match running and waiting jobs per Connector + Dictionary>> runningJobsPerConnector = + runningJobs.GetJobsPerJobTypeAndConnector(); + Dictionary>> waitingJobsPerConnector = + jobsWithoutDependencies.GetJobsPerJobTypeAndConnector(); + List jobsNotHeldBackByConnector = + MatchJobsRunningAndWaiting(runningJobsPerConnector, waitingJobsPerConnector); + - List startJobs = jobsWithoutDownloading.Concat(firstChapterPerConnector).ToList(); + List startJobs = jobsWithoutDownloading.Concat(jobsNotHeldBackByConnector).ToList(); Log.Debug($"Jobs Filtered! (took {DateTime.UtcNow.Subtract(filterStart).TotalMilliseconds}ms)"); @@ -178,11 +167,11 @@ public static class Tranga while(!running) Thread.Sleep(10); } - Log.Debug($"Running: {runningJobs.Count()}\n" + - $"Waiting: {waitingJobs.Count()}\n" + - $"\tof which Due: {dueJobs.Count()}\n" + - $"\t\tof which can be started: {jobsWithoutMissingDependencies.Count()}\n" + - $"\t\t\tof which started: {startJobs.Count()}"); + Log.Debug($"Running: {runningJobs.Count} Waiting: {waitingJobs.Count} Due: {dueJobs.Count} of which \n" + + $"{jobsWithoutDependencies.Count} without missing dependencies, of which\n" + + $"\t{jobsWithoutDownloading.Count} without downloading\n" + + $"\t{jobsNotHeldBackByConnector.Count} not held back by Connector\n" + + $"{startJobs.Count} were started."); (Thread, Job)[] removeFromThreadsList = RunningJobs.Where(t => !t.Key.IsAlive) .Select(t => (t.Key, t.Value)).ToArray(); @@ -204,36 +193,76 @@ public static class Tranga Thread.Sleep(TrangaSettings.startNewJobTimeoutMs); } } + + private static List GetRunningJobs(this IQueryable jobs) => + jobs.Where(j => j.state == JobState.Running).ToList(); + + private static List GetWaitingJobs(this IQueryable jobs) => + jobs.Where(j => j.state == JobState.CompletedWaiting || j.state == JobState.FirstExecution) + .ToList(); - private static List GetBusyConnectors(List runningJobs) + private static List FilterDueJobs(this List jobs) => + jobs.Where(j => j.NextExecution < DateTime.UtcNow) + .ToList(); + + private static List FilterJobDependencies(this List jobs) => + jobs.Where(job => job.DependsOnJobs.All(j => j.IsCompleted)) + .ToList(); + + private static Dictionary>> GetJobsPerJobTypeAndConnector(this List jobs) { - HashSet busyConnectors = new(); - foreach (Job runningJob in runningJobs) + Dictionary>> ret = new(); + foreach (Job job in jobs) { - if(GetJobConnector(runningJob) is { } mangaConnector) - busyConnectors.Add(mangaConnector); + if(GetJobConnector(job) is not { } connector) + continue; + if (!ret.ContainsKey(connector)) + ret.Add(connector, new()); + if (!ret[connector].ContainsKey(job.JobType)) + ret[connector].Add(job.JobType, new()); + ret[connector][job.JobType].Add(job); } - return busyConnectors.ToList(); + return ret; } - private static List FilterDueJobs(List jobs) => - jobs.ToList() - .Where(j => j.NextExecution < DateTime.UtcNow) - .ToList(); + private static List MatchJobsRunningAndWaiting(Dictionary>> running, + Dictionary>> waiting) + { + List ret = new(); + foreach ((MangaConnector connector, Dictionary> jobTypeJobsWaiting) in waiting) + { + if (running.TryGetValue(connector, out Dictionary>? jobTypeJobsRunning)) + { //MangaConnector has running Jobs + //Match per JobType + foreach ((JobType jobType, List jobsWaiting) in jobTypeJobsWaiting) + { + if(jobTypeJobsRunning.ContainsKey(jobType)) + //Already a job of Type running on MangaConnector + continue; + if (jobType is not JobType.DownloadSingleChapterJob) + //If it is not a DownloadSingleChapterJob, just add the first + ret.Add(jobsWaiting.First()); + else + //Add the Job with the lowest Chapternumber + ret.Add(jobsWaiting.OrderBy(j => ((DownloadSingleChapterJob)j).Chapter).First()); + } + } + else + { //MangaConnector has no running Jobs + foreach ((JobType jobType, List jobsWaiting) in jobTypeJobsWaiting) + { + if (jobType is not JobType.DownloadSingleChapterJob) + //If it is not a DownloadSingleChapterJob, just add the first + ret.Add(jobsWaiting.First()); + else + //Add the Job with the lowest Chapternumber + ret.Add(jobsWaiting.OrderBy(j => ((DownloadSingleChapterJob)j).Chapter).First()); + } + } + } - private static List FilterJobDependencies(List jobs) => - jobs - .Where(job => job.DependsOnJobs.All(j => j.IsCompleted)) - .ToList(); - - private static List FilterJobWithBusyConnectors(List jobs, List busyConnectors) => - jobs.Where(j => - { - //Filter jobs with busy connectors - if (GetJobConnector(j) is { } mangaConnector) - return busyConnectors.Contains(mangaConnector) == false; - return true; - }).ToList(); + return ret; + } private static MangaConnector? GetJobConnector(Job job) {