From a4aa5718705be7ee36e366b43c964ca99b054b0d Mon Sep 17 00:00:00 2001 From: glax Date: Fri, 4 Aug 2023 14:51:40 +0200 Subject: [PATCH] Added Jobs and ProgressToken --- Tranga/GlobalBase.cs | 2 +- Tranga/Jobs/DownloadChapter.cs | 23 ++++++ Tranga/Jobs/DownloadNewChapters.cs | 27 +++++++ Tranga/Jobs/Job.cs | 73 +++++++++++++++++++ Tranga/Jobs/JobBoss.cs | 70 ++++++++++++++++++ Tranga/Jobs/ProgressToken.cs | 44 +++++++++++ .../{Connector.cs => MangaConnector.cs} | 46 +++++------- Tranga/MangaConnectors/MangaDex.cs | 11 +-- Tranga/MangaConnectors/MangaKatana.cs | 11 +-- Tranga/MangaConnectors/Manganato.cs | 11 +-- Tranga/MangaConnectors/Mangasee.cs | 15 ++-- 11 files changed, 284 insertions(+), 49 deletions(-) create mode 100644 Tranga/Jobs/DownloadChapter.cs create mode 100644 Tranga/Jobs/DownloadNewChapters.cs create mode 100644 Tranga/Jobs/Job.cs create mode 100644 Tranga/Jobs/JobBoss.cs create mode 100644 Tranga/Jobs/ProgressToken.cs rename Tranga/MangaConnectors/{Connector.cs => MangaConnector.cs} (86%) diff --git a/Tranga/GlobalBase.cs b/Tranga/GlobalBase.cs index 5f66fcb..9b05fdd 100644 --- a/Tranga/GlobalBase.cs +++ b/Tranga/GlobalBase.cs @@ -2,7 +2,7 @@ namespace Tranga; -public class GlobalBase +public abstract class GlobalBase { protected Logger? logger { get; init; } protected TrangaSettings settings { get; init; } diff --git a/Tranga/Jobs/DownloadChapter.cs b/Tranga/Jobs/DownloadChapter.cs new file mode 100644 index 0000000..9110af9 --- /dev/null +++ b/Tranga/Jobs/DownloadChapter.cs @@ -0,0 +1,23 @@ +using Tranga.MangaConnectors; + +namespace Tranga.Jobs; + +public class DownloadChapter : Job +{ + public Chapter chapter { get; init; } + + public DownloadChapter(MangaConnector connector, Chapter chapter) : base(connector) + { + this.chapter = chapter; + } + + protected override IEnumerable ExecuteReturnSubTasksInternal() + { + Task downloadTask = new(delegate + { + mangaConnector.DownloadChapter(chapter, this.progressToken); + }); + downloadTask.Start(); + return Array.Empty(); + } +} \ No newline at end of file diff --git a/Tranga/Jobs/DownloadNewChapters.cs b/Tranga/Jobs/DownloadNewChapters.cs new file mode 100644 index 0000000..a31ff6a --- /dev/null +++ b/Tranga/Jobs/DownloadNewChapters.cs @@ -0,0 +1,27 @@ +using Tranga.MangaConnectors; + +namespace Tranga.Jobs; + +public class DownloadNewChapters : Job +{ + public Publication publication { get; init; } + + public DownloadNewChapters(MangaConnector connector, Publication publication, bool recurring = false) : base (connector, recurring) + { + this.publication = publication; + } + + protected override IEnumerable ExecuteReturnSubTasksInternal() + { + Chapter[] chapters = mangaConnector.GetNewChapters(publication); + this.progressToken.increments = chapters.Length; + List subJobs = new(); + foreach (Chapter chapter in chapters) + { + DownloadChapter downloadChapterJob = new(this.mangaConnector, chapter); + subJobs.Add(downloadChapterJob); + } + progressToken.Complete(); + return subJobs; + } +} \ No newline at end of file diff --git a/Tranga/Jobs/Job.cs b/Tranga/Jobs/Job.cs new file mode 100644 index 0000000..44e4774 --- /dev/null +++ b/Tranga/Jobs/Job.cs @@ -0,0 +1,73 @@ +using Tranga.MangaConnectors; + +namespace Tranga.Jobs; + +public abstract class Job +{ + public MangaConnector mangaConnector { get; init; } + public ProgressToken progressToken { get; private set; } + public bool recurring { get; init; } + public TimeSpan? recurrenceTime { get; set; } + public DateTime? lastExecution { get; private set; } + public DateTime nextExecution => NextExecution(); + + public Job(MangaConnector connector, bool recurring = false, TimeSpan? recurrenceTime = null) + { + this.mangaConnector = connector; + this.progressToken = new ProgressToken(0); + this.recurring = recurring; + if (recurring && recurrenceTime is null) + throw new ArgumentException("If recurrence is set to true, a recurrence time has to be provided."); + this.recurrenceTime = recurrenceTime; + } + + public Job(MangaConnector connector, ProgressToken progressToken, bool recurring = false, TimeSpan? recurrenceTime = null) + { + this.mangaConnector = connector; + this.progressToken = progressToken; + this.recurring = recurring; + if (recurring && recurrenceTime is null) + throw new ArgumentException("If recurrence is set to true, a recurrence time has to be provided."); + this.recurrenceTime = recurrenceTime; + } + + public Job(MangaConnector connector, int taskIncrements, bool recurring = false, TimeSpan? recurrenceTime = null) + { + this.mangaConnector = connector; + this.progressToken = new ProgressToken(taskIncrements); + this.recurring = recurring; + if (recurring && recurrenceTime is null) + throw new ArgumentException("If recurrence is set to true, a recurrence time has to be provided."); + this.recurrenceTime = recurrenceTime; + } + + private DateTime NextExecution() + { + if(recurring && recurrenceTime.HasValue && lastExecution.HasValue) + return lastExecution.Value.Add(recurrenceTime.Value); + if(recurring && recurrenceTime.HasValue && !lastExecution.HasValue) + return DateTime.Now; + return DateTime.MaxValue; + } + + public void Reset() + { + this.progressToken = new ProgressToken(this.progressToken.increments); + } + + public void Cancel() + { + this.progressToken.cancellationRequested = true; + this.progressToken.Complete(); + } + + public IEnumerable ExecuteReturnSubTasks() + { + progressToken.Start(); + IEnumerable ret = ExecuteReturnSubTasksInternal(); + lastExecution = DateTime.Now; + return ret; + } + + protected abstract IEnumerable ExecuteReturnSubTasksInternal(); +} \ No newline at end of file diff --git a/Tranga/Jobs/JobBoss.cs b/Tranga/Jobs/JobBoss.cs new file mode 100644 index 0000000..c05beae --- /dev/null +++ b/Tranga/Jobs/JobBoss.cs @@ -0,0 +1,70 @@ +using System.Runtime.CompilerServices; +using Tranga.MangaConnectors; + +namespace Tranga.Jobs; + +public class JobBoss : GlobalBase +{ + private HashSet jobs { get; init; } + private Dictionary> mangaConnectorJobQueue { get; init; } + + public JobBoss(GlobalBase clone) : base(clone) + { + this.jobs = new(); + this.mangaConnectorJobQueue = new(); + } + + public void MonitorJobs() + { + foreach (Job job in jobs.Where(job => job.nextExecution < DateTime.Now && !QueueContainsJob(job)).OrderBy(job => job.nextExecution)) + AddJobToQueue(job); + CheckJobQueue(); + } + + public void AddJob(Job job) + { + this.jobs.Add(job); + } + + public void RemoveJob(Job job) + { + job.Cancel(); + this.jobs.Remove(job); + } + + private bool QueueContainsJob(Job job) + { + mangaConnectorJobQueue.TryAdd(job.mangaConnector, new Queue()); + return mangaConnectorJobQueue[job.mangaConnector].Contains(job); + } + + private void AddJobToQueue(Job job) + { + Log($"Adding Job to Queue. {job}"); + mangaConnectorJobQueue.TryAdd(job.mangaConnector, new Queue()); + Queue connectorJobQueue = mangaConnectorJobQueue[job.mangaConnector]; + if(!connectorJobQueue.Contains(job)) + connectorJobQueue.Enqueue(job); + } + + public void AddJobsToQueue(IEnumerable jobs) + { + foreach(Job job in jobs) + AddJobToQueue(job); + } + + private void CheckJobQueue() + { + foreach (Queue jobQueue in mangaConnectorJobQueue.Values) + { + Job queueHead = jobQueue.Peek(); + if (queueHead.progressToken.state == ProgressToken.State.Complete) + { + if(queueHead.recurring) + queueHead.Reset(); + jobQueue.Dequeue(); + AddJobsToQueue(jobQueue.Peek().ExecuteReturnSubTasks()); + } + } + } + } \ No newline at end of file diff --git a/Tranga/Jobs/ProgressToken.cs b/Tranga/Jobs/ProgressToken.cs new file mode 100644 index 0000000..9b534a8 --- /dev/null +++ b/Tranga/Jobs/ProgressToken.cs @@ -0,0 +1,44 @@ +namespace Tranga.Jobs; + +public class ProgressToken +{ + public bool cancellationRequested { get; set; } + public int increments { get; set; } + public int incrementsCompleted { get; set; } + public float progress => GetProgress(); + + public enum State { Running, Complete, Standby } + public State state { get; private set; } + + public ProgressToken(int increments) + { + this.cancellationRequested = false; + this.increments = increments; + this.incrementsCompleted = 0; + this.state = State.Standby; + } + + private float GetProgress() + { + if(increments > 0 && incrementsCompleted > 0) + return (float)incrementsCompleted / (float)increments; + return 0; + } + + public void Increment() + { + this.incrementsCompleted++; + if (incrementsCompleted > increments) + state = State.Complete; + } + + public void Start() + { + state = State.Running; + } + + public void Complete() + { + state = State.Complete; + } +} \ No newline at end of file diff --git a/Tranga/MangaConnectors/Connector.cs b/Tranga/MangaConnectors/MangaConnector.cs similarity index 86% rename from Tranga/MangaConnectors/Connector.cs rename to Tranga/MangaConnectors/MangaConnector.cs index 1c839dd..84decdc 100644 --- a/Tranga/MangaConnectors/Connector.cs +++ b/Tranga/MangaConnectors/MangaConnector.cs @@ -3,6 +3,7 @@ using System.IO.Compression; using System.Net; using System.Runtime.InteropServices; using System.Text.RegularExpressions; +using Tranga.Jobs; using static System.IO.UnixFileMode; namespace Tranga.MangaConnectors; @@ -11,11 +12,11 @@ namespace Tranga.MangaConnectors; /// Base-Class for all Connectors /// Provides some methods to be used by all Connectors, as well as a DownloadClient /// -public abstract class Connector : GlobalBase +public abstract class MangaConnector : GlobalBase { internal DownloadClient downloadClient { get; init; } = null!; - protected Connector(GlobalBase clone) : base(clone) + protected MangaConnector(GlobalBase clone) : base(clone) { if (!Directory.Exists(settings.coverImageCache)) Directory.CreateDirectory(settings.coverImageCache); @@ -45,13 +46,11 @@ public abstract class Connector : GlobalBase /// /// Publication to check /// Language to receive chapters for - /// /// List of Chapters that were previously not in collection - public List GetNewChaptersList(Publication publication, string language, ref HashSet collection) + public Chapter[] GetNewChapters(Publication publication, string language = "en") { Log($"Getting new Chapters for {publication}"); Chapter[] newChapters = this.GetChapters(publication, language); - collection.Add(publication); NumberFormatInfo decimalPoint = new (){ NumberDecimalSeparator = "." }; Log($"Checking for duplicates {publication}"); List newChaptersList = newChapters.Where(nChapter => @@ -59,7 +58,7 @@ public abstract class Connector : GlobalBase !nChapter.CheckChapterIsDownloaded(settings.downloadLocation)).ToList(); Log($"{newChaptersList.Count} new chapters. {publication}"); - return newChaptersList; + return newChaptersList.ToArray(); } public Chapter[] SelectChapters(Publication publication, string searchTerm, string? language = null) @@ -135,14 +134,7 @@ public abstract class Connector : GlobalBase return Array.Empty(); } - /// - /// Retrieves the Chapter (+Images) from the website. - /// Should later call DownloadChapterImages to retrieve the individual Images of the Chapter and create .cbz archive. - /// - /// Publication that contains Chapter - /// Chapter with Images to retrieve - /// - public abstract HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, CancellationToken? cancellationToken = null); + public abstract HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null); /// /// Copies the already downloaded cover from cache to downloadLocation @@ -186,20 +178,13 @@ public abstract class Connector : GlobalBase return requestResult.statusCode; } - /// - /// Downloads all Images from URLs, Compresses to zip(cbz) and saves. - /// - /// List of URLs to download Images from - /// Full path to save archive to (without file ending .cbz) - /// Path of the generate Chapter ComicInfo.xml, if it was generated - /// RequestType for RateLimits - /// Used in http request header - /// - protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, string? comicInfoPath = null, string? referrer = null, CancellationToken? cancellationToken = null) + protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, string? comicInfoPath = null, string? referrer = null, ProgressToken? progressToken = null) { - if (cancellationToken?.IsCancellationRequested ?? false) + if (progressToken?.cancellationRequested ?? false) return HttpStatusCode.RequestTimeout; Log($"Downloading Images for {saveArchiveFilePath}"); + if(progressToken is not null) + progressToken.increments = imageUrls.Length; //Check if Publication Directory already exists string directoryPath = Path.GetDirectoryName(saveArchiveFilePath)!; if (!Directory.Exists(directoryPath)) @@ -220,9 +205,16 @@ public abstract class Connector : GlobalBase Log($"Downloading image {chapter + 1:000}/{imageUrls.Length:000}"); //TODO HttpStatusCode status = DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer); if ((int)status < 200 || (int)status >= 300) + { + progressToken?.Complete(); return status; - if (cancellationToken?.IsCancellationRequested ?? false) + } + if (progressToken?.cancellationRequested ?? false) + { + progressToken?.Complete(); return HttpStatusCode.RequestTimeout; + } + progressToken?.Increment(); } if(comicInfoPath is not null) @@ -234,6 +226,8 @@ public abstract class Connector : GlobalBase if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) File.SetUnixFileMode(saveArchiveFilePath, GroupRead | GroupWrite | OtherRead | OtherWrite | UserRead | UserWrite); Directory.Delete(tempFolder, true); //Cleanup + + progressToken?.Complete(); return HttpStatusCode.OK; } diff --git a/Tranga/MangaConnectors/MangaDex.cs b/Tranga/MangaConnectors/MangaDex.cs index 7fa1af6..025a5d8 100644 --- a/Tranga/MangaConnectors/MangaDex.cs +++ b/Tranga/MangaConnectors/MangaDex.cs @@ -2,9 +2,10 @@ using System.Net; using System.Text.Json; using System.Text.Json.Nodes; +using Tranga.Jobs; namespace Tranga.MangaConnectors; -public class MangaDex : Connector +public class MangaDex : MangaConnector { public override string name { get; } @@ -203,11 +204,11 @@ public class MangaDex : Connector return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray(); } - public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, CancellationToken? cancellationToken = null) + public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) { - if (cancellationToken?.IsCancellationRequested ?? false) + if (progressToken?.cancellationRequested ?? false) return HttpStatusCode.RequestTimeout; - Log($"Retrieving chapter-info {chapter} {publication}"); + Log($"Retrieving chapter-info {chapter} {chapter.parentPublication}"); //Request URLs for Chapter-Images DownloadClient.RequestResult requestResult = downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer); @@ -229,7 +230,7 @@ public class MangaDex : Connector File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString()); //Download Chapter-Images - return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), (byte)RequestType.AtHomeServer, comicInfoPath, cancellationToken:cancellationToken); + return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), (byte)RequestType.AtHomeServer, comicInfoPath, progressToken:progressToken); } private string? GetCoverUrl(string publicationId, string? posterId) diff --git a/Tranga/MangaConnectors/MangaKatana.cs b/Tranga/MangaConnectors/MangaKatana.cs index f0fbe40..0e61710 100644 --- a/Tranga/MangaConnectors/MangaKatana.cs +++ b/Tranga/MangaConnectors/MangaKatana.cs @@ -2,10 +2,11 @@ using System.Net; using System.Text.RegularExpressions; using HtmlAgilityPack; +using Tranga.Jobs; namespace Tranga.MangaConnectors; -public class MangaKatana : Connector +public class MangaKatana : MangaConnector { public override string name { get; } @@ -181,11 +182,11 @@ public class MangaKatana : Connector return ret; } - public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, CancellationToken? cancellationToken = null) + public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) { - if (cancellationToken?.IsCancellationRequested ?? false) + if (progressToken?.cancellationRequested ?? false) return HttpStatusCode.RequestTimeout; - Log($"Retrieving chapter-info {chapter} {publication}"); + Log($"Retrieving chapter-info {chapter} {chapter.parentPublication}"); string requestUrl = chapter.url; // Leaving this in to check if the page exists DownloadClient.RequestResult requestResult = @@ -198,7 +199,7 @@ public class MangaKatana : Connector string comicInfoPath = Path.GetTempFileName(); File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString()); - return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://mangakatana.com/", cancellationToken); + return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken); } private string[] ParseImageUrlsFromHtml(string mangaUrl) diff --git a/Tranga/MangaConnectors/Manganato.cs b/Tranga/MangaConnectors/Manganato.cs index 76d0230..5ef0e98 100644 --- a/Tranga/MangaConnectors/Manganato.cs +++ b/Tranga/MangaConnectors/Manganato.cs @@ -2,10 +2,11 @@ using System.Net; using System.Text.RegularExpressions; using HtmlAgilityPack; +using Tranga.Jobs; namespace Tranga.MangaConnectors; -public class Manganato : Connector +public class Manganato : MangaConnector { public override string name { get; } @@ -168,11 +169,11 @@ public class Manganato : Connector return ret; } - public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, CancellationToken? cancellationToken = null) + public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) { - if (cancellationToken?.IsCancellationRequested ?? false) + if (progressToken?.cancellationRequested ?? false) return HttpStatusCode.RequestTimeout; - Log($"Retrieving chapter-info {chapter} {publication}"); + Log($"Retrieving chapter-info {chapter} {chapter.parentPublication}"); string requestUrl = chapter.url; DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(requestUrl, 1); @@ -184,7 +185,7 @@ public class Manganato : Connector string comicInfoPath = Path.GetTempFileName(); File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString()); - return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://chapmanganato.com/", cancellationToken); + return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://chapmanganato.com/", progressToken:progressToken); } private string[] ParseImageUrlsFromHtml(Stream html) diff --git a/Tranga/MangaConnectors/Mangasee.cs b/Tranga/MangaConnectors/Mangasee.cs index 0e27809..a02ce20 100644 --- a/Tranga/MangaConnectors/Mangasee.cs +++ b/Tranga/MangaConnectors/Mangasee.cs @@ -5,10 +5,11 @@ using System.Xml.Linq; using HtmlAgilityPack; using Newtonsoft.Json; using PuppeteerSharp; +using Tranga.Jobs; namespace Tranga.MangaConnectors; -public class Mangasee : Connector +public class Mangasee : MangaConnector { public override string name { get; } private IBrowser? _browser; @@ -237,19 +238,19 @@ public class Mangasee : Connector return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray(); } - public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, CancellationToken? cancellationToken = null) + public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) { - if (cancellationToken?.IsCancellationRequested ?? false) + if (progressToken?.cancellationRequested ?? false) return HttpStatusCode.RequestTimeout; - while (this._browser is null && !(cancellationToken?.IsCancellationRequested??false)) + while (this._browser is null && !(progressToken?.cancellationRequested??false)) { Log("Waiting for headless browser to download..."); Thread.Sleep(1000); } - if (cancellationToken?.IsCancellationRequested??false) + if (progressToken?.cancellationRequested??false) return HttpStatusCode.RequestTimeout; - Log($"Retrieving chapter-info {chapter} {publication}"); + Log($"Retrieving chapter-info {chapter} {chapter.parentPublication}"); IPage page = _browser!.NewPageAsync().Result; IResponse response = page.GoToAsync(chapter.url).Result; if (response.Ok) @@ -266,7 +267,7 @@ public class Mangasee : Connector string comicInfoPath = Path.GetTempFileName(); File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString()); - return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, cancellationToken:cancellationToken); + return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, progressToken:progressToken); } return response.Status; }