Start-Job endpoint: Add option to start Jobs that our job is dependent on

API/Controllers/JobController.cs

@@ -5,6 +5,7 @@ using API.Schema.Jobs;
 using Asp.Versioning;
 using log4net;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc.ModelBinding;
 using static Microsoft.AspNetCore.Http.StatusCodes;
 // ReSharper disable InconsistentNaming
 
@@ -326,6 +327,7 @@ public class JobController(PgsqlContext context, ILog Log) : Controller
     /// Starts the Job with the requested ID
     /// </summary>
     /// <param name="JobId">Job-ID</param>
+    /// <param name="startDependencies">Start Jobs necessary for execution</param>
     /// <response code="202">Job started</response>
     /// <response code="404">Job with ID not found</response>
     /// <response code="409">Job was already running</response>
@@ -335,16 +337,22 @@ public class JobController(PgsqlContext context, ILog Log) : Controller
     [ProducesResponseType(Status404NotFound)]
     [ProducesResponseType(Status409Conflict)]
     [ProducesResponseType<string>(Status500InternalServerError, "text/plain")]
-    public IActionResult StartJob(string JobId)
+    public IActionResult StartJob(string JobId, [FromBody(EmptyBodyBehavior = EmptyBodyBehavior.Allow)]bool startDependencies = false)
     {
         Job? ret = context.Jobs.Find(JobId);
         if (ret is null)
             return NotFound();
+        List<Job> dependencies = startDependencies ? ret.GetDependenciesAndSelf() : [ret];
+        
         try
         {
-            if (ret.state >= JobState.Running && ret.state < JobState.Completed)
+            if(dependencies.Any(d => d.state >= JobState.Running && d.state < JobState.Completed))
                 return new ConflictResult();
-            ret.LastExecution = DateTime.UnixEpoch;
+            dependencies.ForEach(d =>
+            {
+                d.LastExecution = DateTime.UnixEpoch;
+                d.state = JobState.CompletedWaiting;
+            });
             context.SaveChanges();
             return Accepted();
         }

API/Schema/Jobs/Job.cs

@@ -118,6 +118,17 @@ public abstract class Job
     
     protected abstract IEnumerable<Job> RunInternal(PgsqlContext context);
 
+    public List<Job> GetDependenciesAndSelf()
+    {
+        List<Job> ret = new ();
+        foreach (Job job in DependsOnJobs)
+        {
+            ret.AddRange(job.GetDependenciesAndSelf());
+        }
+        ret.Add(this);
+        return ret;
+    }
+
     public override string ToString()
     {
         return $"{JobId}";

API/Tranga.cs

@@ -125,39 +125,28 @@ public static class Tranga
             using IServiceScope scope = serviceProvider.CreateScope();
             PgsqlContext cycleContext = scope.ServiceProvider.GetRequiredService<PgsqlContext>();
 
-            List<Job> runningJobs = cycleContext.Jobs.Where(j => j.state == JobState.Running).ToList();
+            //Get Running Jobs
+            List<Job> runningJobs = cycleContext.Jobs.GetRunningJobs();
             
             DateTime filterStart = DateTime.UtcNow;
             Log.Debug("Filtering Jobs...");
-            List<MangaConnector> busyConnectors = GetBusyConnectors(runningJobs);
 
-            List<Job> waitingJobs = cycleContext.Jobs.Where(j => j.state == JobState.CompletedWaiting || j.state == JobState.FirstExecution).ToList();
+            List<Job> waitingJobs = cycleContext.Jobs.GetWaitingJobs();
-            List<Job> dueJobs = FilterDueJobs(waitingJobs);
+            List<Job> dueJobs = waitingJobs.FilterDueJobs();
-            List<Job> jobsWithoutBusyConnectors = FilterJobWithBusyConnectors(dueJobs, busyConnectors);
+            List<Job> jobsWithoutDependencies = dueJobs.FilterJobDependencies();
-            List<Job> jobsWithoutMissingDependencies = FilterJobDependencies(jobsWithoutBusyConnectors);
 
-            List<Job> jobsWithoutDownloading =
+            List<Job> jobsWithoutDownloading = jobsWithoutDependencies.Where(j => GetJobConnector(j) is null).ToList();
-                jobsWithoutMissingDependencies
+            
-                    .Where(j => j.JobType != JobType.DownloadSingleChapterJob)
+            //Match running and waiting jobs per Connector
-                    .DistinctBy(j => j.JobType)
+            Dictionary<MangaConnector, Dictionary<JobType, List<Job>>> runningJobsPerConnector =
-                    .ToList();
+                runningJobs.GetJobsPerJobTypeAndConnector();
-            List<Job> firstChapterPerConnector =
+            Dictionary<MangaConnector, Dictionary<JobType, List<Job>>> waitingJobsPerConnector =
-                jobsWithoutMissingDependencies
+                jobsWithoutDependencies.GetJobsPerJobTypeAndConnector();
-                    .Where(j => j.JobType == JobType.DownloadSingleChapterJob)
+            List<Job> jobsNotHeldBackByConnector =
-                    .AsEnumerable()
+                MatchJobsRunningAndWaiting(runningJobsPerConnector, waitingJobsPerConnector);
-                    .OrderBy(j =>
+            
-                    {
-                        DownloadSingleChapterJob dscj = (DownloadSingleChapterJob)j;
-                        return dscj.Chapter;
-                    })
-                    .DistinctBy(j =>
-                    {
-                        DownloadSingleChapterJob dscj = (DownloadSingleChapterJob)j;
-                        return dscj.Chapter.ParentManga.MangaConnector;
-                    })
-                    .ToList();
 
-            List<Job> startJobs = jobsWithoutDownloading.Concat(firstChapterPerConnector).ToList();
+            List<Job> startJobs = jobsWithoutDownloading.Concat(jobsNotHeldBackByConnector).ToList();
             Log.Debug($"Jobs Filtered! (took {DateTime.UtcNow.Subtract(filterStart).TotalMilliseconds}ms)");
             
             
@@ -178,11 +167,15 @@ public static class Tranga
                 while(!running)
                     Thread.Sleep(10);
             }
-            Log.Debug($"Running: {runningJobs.Count()}\n" +
+            Log.Debug($"Running: {runningJobs.Count} Waiting: {waitingJobs.Count} Due: {dueJobs.Count} of which \n" +
-                      $"Waiting: {waitingJobs.Count()}\n" +
+                      $"{jobsWithoutDependencies.Count} without missing dependencies, of which\n" +
-                      $"\tof which Due: {dueJobs.Count()}\n" +
+                      $"\t{jobsWithoutDownloading.Count} without downloading\n" +
-                      $"\t\tof which can be started: {jobsWithoutMissingDependencies.Count()}\n" +
+                      $"\t{jobsNotHeldBackByConnector.Count} not held back by Connector\n" +
-                      $"\t\t\tof which started: {startJobs.Count()}");
+                      $"{startJobs.Count} were started.");
+
+            if (Log.IsDebugEnabled && dueJobs.Count < 1)
+                if(waitingJobs.MinBy(j => j.NextExecution) is { } nextJob)
+                    Log.Debug($"Next job in {nextJob.NextExecution.Subtract(DateTime.UtcNow)} (at {nextJob.NextExecution}): {nextJob.JobId}");
 
             (Thread, Job)[] removeFromThreadsList = RunningJobs.Where(t => !t.Key.IsAlive)
                 .Select(t => (t.Key, t.Value)).ToArray();
@@ -204,36 +197,76 @@ public static class Tranga
             Thread.Sleep(TrangaSettings.startNewJobTimeoutMs);
         }
     }
+    
+    private static List<Job> GetRunningJobs(this IQueryable<Job> jobs) =>
+        jobs.Where(j => j.state == JobState.Running).ToList();
+    
+    private static List<Job> GetWaitingJobs(this IQueryable<Job> jobs) =>
+        jobs.Where(j => j.state == JobState.CompletedWaiting || j.state == JobState.FirstExecution)
+            .ToList();
 
-    private static List<MangaConnector> GetBusyConnectors(List<Job> runningJobs)
+    private static List<Job> FilterDueJobs(this List<Job> jobs) =>
+        jobs.Where(j => j.NextExecution < DateTime.UtcNow)
+            .ToList();
+
+    private static List<Job> FilterJobDependencies(this List<Job> jobs) =>
+        jobs.Where(job => job.DependsOnJobs.All(j => j.IsCompleted))
+            .ToList();
+
+    private static Dictionary<MangaConnector, Dictionary<JobType, List<Job>>> GetJobsPerJobTypeAndConnector(this List<Job> jobs)
     {
-        HashSet<MangaConnector> busyConnectors = new();
+        Dictionary<MangaConnector, Dictionary<JobType, List<Job>>> ret = new();
-        foreach (Job runningJob in runningJobs)
+        foreach (Job job in jobs)
         {
-            if(GetJobConnector(runningJob) is { } mangaConnector)
+            if(GetJobConnector(job) is not { } connector)
-                busyConnectors.Add(mangaConnector);
+                continue;
+            if (!ret.ContainsKey(connector))
+                ret.Add(connector, new());
+            if (!ret[connector].ContainsKey(job.JobType))
+                ret[connector].Add(job.JobType, new());
+            ret[connector][job.JobType].Add(job);
         }
-        return busyConnectors.ToList();
+        return ret;
     }
 
-    private static List<Job> FilterDueJobs(List<Job> jobs) =>
+    private static List<Job> MatchJobsRunningAndWaiting(Dictionary<MangaConnector, Dictionary<JobType, List<Job>>> running,
-        jobs.ToList()
+        Dictionary<MangaConnector, Dictionary<JobType, List<Job>>> waiting)
-            .Where(j => j.NextExecution < DateTime.UtcNow)
+    {
-            .ToList();
+        List<Job> ret = new();
+        foreach ((MangaConnector connector, Dictionary<JobType, List<Job>> jobTypeJobsWaiting) in waiting)
+        {
+            if (running.TryGetValue(connector, out Dictionary<JobType, List<Job>>? jobTypeJobsRunning))
+            {   //MangaConnector has running Jobs
+                //Match per JobType
+                foreach ((JobType jobType, List<Job> jobsWaiting) in jobTypeJobsWaiting)
+                {
+                    if(jobTypeJobsRunning.ContainsKey(jobType))
+                        //Already a job of Type running on MangaConnector
+                        continue;
+                    if (jobType is not JobType.DownloadSingleChapterJob)
+                        //If it is not a DownloadSingleChapterJob, just add the first
+                        ret.Add(jobsWaiting.First());
+                    else
+                        //Add the Job with the lowest Chapternumber
+                        ret.Add(jobsWaiting.OrderBy(j => ((DownloadSingleChapterJob)j).Chapter).First());
+                }
+            }
+            else
+            {   //MangaConnector has no running Jobs
+                foreach ((JobType jobType, List<Job> jobsWaiting) in jobTypeJobsWaiting)
+                {
+                    if (jobType is not JobType.DownloadSingleChapterJob)
+                        //If it is not a DownloadSingleChapterJob, just add the first
+                        ret.Add(jobsWaiting.First());
+                    else
+                        //Add the Job with the lowest Chapternumber
+                        ret.Add(jobsWaiting.OrderBy(j => ((DownloadSingleChapterJob)j).Chapter).First());
+                }
+            }
+        }
 
-    private static List<Job> FilterJobDependencies(List<Job> jobs) =>
+        return ret;
-        jobs
+    }
-            .Where(job => job.DependsOnJobs.All(j => j.IsCompleted))
-            .ToList();
-
-    private static List<Job> FilterJobWithBusyConnectors(List<Job> jobs, List<MangaConnector> busyConnectors) =>
-        jobs.Where(j =>
-            {
-                //Filter jobs with busy connectors
-                if (GetJobConnector(j) is { } mangaConnector)
-                    return busyConnectors.Contains(mangaConnector) == false;
-                return true;
-            }).ToList();
 
     private static MangaConnector? GetJobConnector(Job job)
     {

API/TrangaSettings.cs

@@ -35,7 +35,7 @@ public static class TrangaSettings
     [JsonIgnore]
     public static string coverImageCache => Path.Join(workingDirectory, "imageCache");
     public static bool aprilFoolsMode { get; private set; } = true;
-    public static int startNewJobTimeoutMs { get; private set; } = 1000;
+    public static int startNewJobTimeoutMs { get; private set; } = 5000;
     [JsonIgnore]
     internal static readonly Dictionary<RequestType, int> DefaultRequestLimits = new ()
     {