From 9af5c1603eb085d01e023e5da1b42dafb5afbe5d Mon Sep 17 00:00:00 2001 From: glax Date: Tue, 20 Jun 2023 15:46:54 +0200 Subject: [PATCH] Using HttpStatusCode to signify Task-Success When DownloadChapterTask returns notfound, do not retry. --- Tranga/Connector.cs | 42 ++++++++++---------- Tranga/Connectors/MangaDex.cs | 20 +++++----- Tranga/Connectors/Manganato.cs | 16 ++++---- Tranga/Connectors/Mangasee.cs | 13 +++--- Tranga/TaskManager.cs | 25 +++++++----- Tranga/TrangaTask.cs | 21 ++++++---- Tranga/TrangaTasks/DownloadChapterTask.cs | 14 +++---- Tranga/TrangaTasks/MonitorPublicationTask.cs | 11 ++--- Tranga/TrangaTasks/UpdateLibrariesTask.cs | 11 ++--- 9 files changed, 93 insertions(+), 80 deletions(-) diff --git a/Tranga/Connector.cs b/Tranga/Connector.cs index a3913fd..d0f6783 100644 --- a/Tranga/Connector.cs +++ b/Tranga/Connector.cs @@ -1,4 +1,5 @@ using System.IO.Compression; +using System.Net; using System.Runtime.InteropServices; using System.Text.RegularExpressions; using System.Xml.Linq; @@ -133,7 +134,7 @@ public abstract class Connector /// Chapter with Images to retrieve /// Will be used for progress-tracking /// - public abstract bool DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null); + public abstract HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null); /// /// Copies the already downloaded cover from cache to downloadLocation @@ -212,15 +213,15 @@ public abstract class Connector /// /// RequestType for Rate-Limit /// referrer used in html request header - private bool DownloadImage(string imageUrl, string fullPath, byte requestType, string? referrer = null) + private HttpStatusCode DownloadImage(string imageUrl, string fullPath, byte requestType, string? referrer = null) { DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType, referrer); - if (!requestResult.success || requestResult.result == Stream.Null) - return false; + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.result == Stream.Null) + return requestResult.statusCode; byte[] buffer = new byte[requestResult.result.Length]; requestResult.result.ReadExactly(buffer, 0, buffer.Length); File.WriteAllBytes(fullPath, buffer); - return true; + return requestResult.statusCode; } /// @@ -233,10 +234,10 @@ public abstract class Connector /// RequestType for RateLimits /// Used in http request header /// - protected bool DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, DownloadChapterTask parentTask, string? comicInfoPath = null, string? referrer = null, CancellationToken? cancellationToken = null) + protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, DownloadChapterTask parentTask, string? comicInfoPath = null, string? referrer = null, CancellationToken? cancellationToken = null) { - if (cancellationToken?.IsCancellationRequested??false) - return false; + if (cancellationToken?.IsCancellationRequested ?? false) + return HttpStatusCode.RequestTimeout; logger?.WriteLine("Connector", $"Downloading Images for {saveArchiveFilePath}"); //Check if Publication Directory already exists string directoryPath = Path.GetDirectoryName(saveArchiveFilePath)!; @@ -244,7 +245,7 @@ public abstract class Connector Directory.CreateDirectory(directoryPath); if (File.Exists(saveArchiveFilePath)) //Don't download twice. - return false; + return HttpStatusCode.OK; //Create a temporary folder to store images string tempFolder = Directory.CreateTempSubdirectory().FullName; @@ -256,11 +257,12 @@ public abstract class Connector string[] split = imageUrl.Split('.'); string extension = split[^1]; logger?.WriteLine("Connector", $"Downloading Image {chapter + 1:000}/{imageUrls.Length:000} {parentTask.publication.sortName} {parentTask.publication.internalId} Vol.{parentTask.chapter.volumeNumber} Ch.{parentTask.chapter.chapterNumber} {parentTask.progress:P2}"); - if (!DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer)) - return false; + HttpStatusCode status = DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer); + if ((int)status < 200 || (int)status >= 300) + return status; parentTask.IncrementProgress(1.0 / imageUrls.Length); - if (cancellationToken?.IsCancellationRequested??false) - return false; + if (cancellationToken?.IsCancellationRequested ?? false) + return HttpStatusCode.RequestTimeout; } if(comicInfoPath is not null) @@ -272,7 +274,7 @@ public abstract class Connector if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) File.SetUnixFileMode(saveArchiveFilePath, GroupRead | GroupWrite | OtherRead | OtherWrite | UserRead | UserWrite); Directory.Delete(tempFolder, true); //Cleanup - return true; + return HttpStatusCode.OK; } protected string SaveCoverImageToCache(string url, byte requestType) @@ -332,7 +334,7 @@ public abstract class Connector else { logger?.WriteLine(this.GetType().ToString(), "RequestType not configured for rate-limit."); - return new RequestResult(false, Stream.Null); + return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null); } TimeSpan rateLimitTimeout = _rateLimit[requestType] @@ -362,19 +364,19 @@ public abstract class Connector if (!response.IsSuccessStatusCode) { logger?.WriteLine(this.GetType().ToString(), $"Request-Error {response.StatusCode}: {response.ReasonPhrase}"); - return new RequestResult(false, Stream.Null); + return new RequestResult(response.StatusCode, Stream.Null); } - return new RequestResult(true, response.Content.ReadAsStream()); + return new RequestResult(response.StatusCode, response.Content.ReadAsStream()); } public struct RequestResult { - public bool success { get; } + public HttpStatusCode statusCode { get; } public Stream result { get; } - public RequestResult(bool success, Stream result) + public RequestResult(HttpStatusCode statusCode, Stream result) { - this.success = success; + this.statusCode = statusCode; this.result = result; } } diff --git a/Tranga/Connectors/MangaDex.cs b/Tranga/Connectors/MangaDex.cs index e56f5da..422975d 100644 --- a/Tranga/Connectors/MangaDex.cs +++ b/Tranga/Connectors/MangaDex.cs @@ -46,7 +46,7 @@ public class MangaDex : Connector DownloadClient.RequestResult requestResult = downloadClient.MakeRequest( $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", (byte)RequestType.Manga); - if (!requestResult.success) + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) break; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); @@ -165,7 +165,7 @@ public class MangaDex : Connector DownloadClient.RequestResult requestResult = downloadClient.MakeRequest( $"https://api.mangadex.org/manga/{publication.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", (byte)RequestType.Feed); - if (!requestResult.success) + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) break; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); @@ -207,19 +207,19 @@ public class MangaDex : Connector return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray(); } - public override bool DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null) + public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null) { - if (cancellationToken?.IsCancellationRequested??false) - return false; + if (cancellationToken?.IsCancellationRequested ?? false) + return HttpStatusCode.RequestTimeout; logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}"); //Request URLs for Chapter-Images DownloadClient.RequestResult requestResult = downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer); - if (!requestResult.success) - return false; + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + return requestResult.statusCode; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); if (result is null) - return false; + return HttpStatusCode.NoContent; string baseUrl = result["baseUrl"]!.GetValue(); string hash = result["chapter"]!["hash"]!.GetValue(); @@ -248,7 +248,7 @@ public class MangaDex : Connector //Request information where to download Cover DownloadClient.RequestResult requestResult = downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", (byte)RequestType.CoverUrl); - if (!requestResult.success) + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return null; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); if (result is null) @@ -268,7 +268,7 @@ public class MangaDex : Connector { DownloadClient.RequestResult requestResult = downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", (byte)RequestType.Author); - if (!requestResult.success) + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return ret; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); if (result is null) diff --git a/Tranga/Connectors/Manganato.cs b/Tranga/Connectors/Manganato.cs index 83b6e6d..8378088 100644 --- a/Tranga/Connectors/Manganato.cs +++ b/Tranga/Connectors/Manganato.cs @@ -27,7 +27,7 @@ public class Manganato : Connector string requestUrl = $"https://manganato.com/search/story/{sanitizedTitle}"; DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(requestUrl, (byte)1); - if (!requestResult.success) + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return Array.Empty(); return ParsePublicationsFromHtml(requestResult.result); @@ -52,7 +52,7 @@ public class Manganato : Connector { DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(url, (byte)1); - if (!requestResult.success) + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return Array.Empty(); ret.Add(ParseSinglePublicationFromHtml(requestResult.result, url.Split('/')[^1])); @@ -131,7 +131,7 @@ public class Manganato : Connector string requestUrl = $"https://chapmanganato.com/{publication.publicationId}"; DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(requestUrl, (byte)1); - if (!requestResult.success) + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return Array.Empty(); //Return Chapters ordered by Chapter-Number @@ -169,16 +169,16 @@ public class Manganato : Connector return ret; } - public override bool DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null) + public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null) { - if (cancellationToken?.IsCancellationRequested??false) - return false; + if (cancellationToken?.IsCancellationRequested ?? false) + return HttpStatusCode.RequestTimeout; logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}"); string requestUrl = chapter.url; DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(requestUrl, (byte)1); - if (!requestResult.success) - return false; + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + return requestResult.statusCode; string[] imageUrls = ParseImageUrlsFromHtml(requestResult.result); diff --git a/Tranga/Connectors/Mangasee.cs b/Tranga/Connectors/Mangasee.cs index ec9f0fc..6753af6 100644 --- a/Tranga/Connectors/Mangasee.cs +++ b/Tranga/Connectors/Mangasee.cs @@ -80,7 +80,7 @@ public class Mangasee : Connector string requestUrl = $"https://mangasee123.com/_search.php"; DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(requestUrl, (byte)1); - if (!requestResult.success) + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return Array.Empty(); return ParsePublicationsFromHtml(requestResult.result, publicationTitle); @@ -110,7 +110,7 @@ public class Mangasee : Connector { DownloadClient.RequestResult requestResult = downloadClient.MakeRequest($"https://mangasee123.com/manga/{orderedItem.i}", (byte)1); - if (!requestResult.success) + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return Array.Empty(); ret.Add(ParseSinglePublicationFromHtml(requestResult.result, orderedItem.s, orderedItem.i, orderedItem.a)); } @@ -209,17 +209,17 @@ public class Mangasee : Connector return ret.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray(); } - public override bool DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null) + public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null) { if (cancellationToken?.IsCancellationRequested ?? false) - return false; + return HttpStatusCode.RequestTimeout; while (this._browser is null && !(cancellationToken?.IsCancellationRequested??false)) { logger?.WriteLine(this.GetType().ToString(), "Waiting for headless browser to download..."); Thread.Sleep(1000); } if (cancellationToken?.IsCancellationRequested??false) - return false; + return HttpStatusCode.RequestTimeout; logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}"); IPage page = _browser!.NewPageAsync().Result; @@ -240,7 +240,6 @@ public class Mangasee : Connector return DownloadChapterImages(urls.ToArray(), GetArchiveFilePath(publication, chapter), (byte)1, parentTask, comicInfoPath, cancellationToken:cancellationToken); } - - return false; + return response.Status; } } \ No newline at end of file diff --git a/Tranga/TaskManager.cs b/Tranga/TaskManager.cs index e82c089..0795f0f 100644 --- a/Tranga/TaskManager.cs +++ b/Tranga/TaskManager.cs @@ -82,18 +82,21 @@ public class TaskManager } } - foreach (TrangaTask failedTask in _allTasks.Where(taskQuery => - taskQuery.state is TrangaTask.ExecutionState.Failed)) + TrangaTask[] failedDownloadChapterTasks = _allTasks.Where(taskQuery => + taskQuery.state is TrangaTask.ExecutionState.Failed && taskQuery is DownloadChapterTask).ToArray(); + foreach (TrangaTask failedDownloadChapterTask in failedDownloadChapterTasks) { - switch (failedTask.task) - { - case TrangaTask.Task.DownloadChapter: - DeleteTask(failedTask); - TrangaTask newTask = failedTask.Clone(); - failedTask.parentTask?.AddChildTask(newTask); - AddTask(newTask); - break; - } + DeleteTask(failedDownloadChapterTask); + TrangaTask newTask = failedDownloadChapterTask.Clone(); + failedDownloadChapterTask.parentTask?.AddChildTask(newTask); + AddTask(newTask); + } + + TrangaTask[] successfulDownloadChapterTasks = _allTasks.Where(taskQuery => + taskQuery.state is TrangaTask.ExecutionState.Success && taskQuery is DownloadChapterTask).ToArray(); + foreach(TrangaTask successfulDownloadChapterTask in successfulDownloadChapterTasks) + { + DeleteTask(successfulDownloadChapterTask); } if(waitingTasksCount != _allTasks.Count(task => task.state is TrangaTask.ExecutionState.Waiting)) diff --git a/Tranga/TrangaTask.cs b/Tranga/TrangaTask.cs index d2c0263..f0c92f2 100644 --- a/Tranga/TrangaTask.cs +++ b/Tranga/TrangaTask.cs @@ -1,4 +1,5 @@ -using System.Text.Json.Serialization; +using System.Net; +using System.Text.Json.Serialization; using Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -32,7 +33,7 @@ public abstract class TrangaTask [Newtonsoft.Json.JsonIgnore]public TimeSpan executionApproximatelyRemaining => executionApproximatelyFinished.Subtract(DateTime.Now); [Newtonsoft.Json.JsonIgnore]public DateTime nextExecution => lastExecuted.Add(reoccurrence); - public enum ExecutionState { Waiting, Enqueued, Running, Failed } + public enum ExecutionState { Waiting, Enqueued, Running, Failed, Success } protected TrangaTask(Task task, TimeSpan reoccurrence, TrangaTask? parentTask = null) { @@ -53,7 +54,7 @@ public abstract class TrangaTask /// /// /// - protected abstract bool ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null); + protected abstract HttpStatusCode ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null); public abstract TrangaTask Clone(); @@ -73,18 +74,24 @@ public abstract class TrangaTask this.parentTask.state = ExecutionState.Running; this.executionStarted = DateTime.Now; this.lastChange = DateTime.Now; - bool success = ExecuteTask(taskManager, logger, cancellationToken); + HttpStatusCode statusCode = ExecuteTask(taskManager, logger, cancellationToken); while(childTasks.Any(ct => ct.state is ExecutionState.Enqueued or ExecutionState.Running)) Thread.Sleep(1000); - if (success) + if ((int)statusCode >= 200 && (int)statusCode < 300) { this.lastExecuted = DateTime.Now; - this.state = ExecutionState.Waiting; + if (this is DownloadChapterTask) + this.state = ExecutionState.Success; + else + this.state = ExecutionState.Waiting; } else { + if (this is DownloadChapterTask && statusCode == HttpStatusCode.NotFound) + this.state = ExecutionState.Success; + else + this.state = ExecutionState.Failed; this.lastExecuted = DateTime.MaxValue; - this.state = ExecutionState.Failed; } if(this.parentTask is not null) this.parentTask.state = ExecutionState.Waiting; diff --git a/Tranga/TrangaTasks/DownloadChapterTask.cs b/Tranga/TrangaTasks/DownloadChapterTask.cs index e60c51a..c870be6 100644 --- a/Tranga/TrangaTasks/DownloadChapterTask.cs +++ b/Tranga/TrangaTasks/DownloadChapterTask.cs @@ -1,4 +1,5 @@ -using Logging; +using System.Net; +using Logging; namespace Tranga.TrangaTasks; @@ -19,15 +20,14 @@ public class DownloadChapterTask : TrangaTask this.language = language; } - protected override bool ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null) + protected override HttpStatusCode ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null) { - if (cancellationToken?.IsCancellationRequested??false) - return false; + if (cancellationToken?.IsCancellationRequested ?? false) + return HttpStatusCode.RequestTimeout; Connector connector = taskManager.GetConnector(this.connectorName); connector.CopyCoverFromCacheToDownloadLocation(this.publication, taskManager.settings); - bool downloadSuccess = connector.DownloadChapter(this.publication, this.chapter, this, cancellationToken); - taskManager.DeleteTask(this); - if(downloadSuccess && parentTask is not null) + HttpStatusCode downloadSuccess = connector.DownloadChapter(this.publication, this.chapter, this, cancellationToken); + if((int)downloadSuccess >= 200 && (int)downloadSuccess < 300 && parentTask is not null) foreach(NotificationManager nm in taskManager.settings.notificationManagers) nm.SendNotification("New Chapter downloaded", $"{this.publication.sortName} {this.chapter.chapterNumber} {this.chapter.name}"); return downloadSuccess; diff --git a/Tranga/TrangaTasks/MonitorPublicationTask.cs b/Tranga/TrangaTasks/MonitorPublicationTask.cs index a8b794a..e36cf44 100644 --- a/Tranga/TrangaTasks/MonitorPublicationTask.cs +++ b/Tranga/TrangaTasks/MonitorPublicationTask.cs @@ -1,4 +1,5 @@ -using Logging; +using System.Net; +using Logging; namespace Tranga.TrangaTasks; @@ -14,10 +15,10 @@ public class MonitorPublicationTask : TrangaTask this.language = language; } - protected override bool ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null) + protected override HttpStatusCode ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null) { - if (cancellationToken?.IsCancellationRequested??false) - return false; + if (cancellationToken?.IsCancellationRequested ?? false) + return HttpStatusCode.RequestTimeout; Connector connector = taskManager.GetConnector(this.connectorName); //Check if Publication already has a Folder @@ -36,7 +37,7 @@ public class MonitorPublicationTask : TrangaTask taskManager.AddTask(newTask); } - return true; + return HttpStatusCode.OK; } public override TrangaTask Clone() diff --git a/Tranga/TrangaTasks/UpdateLibrariesTask.cs b/Tranga/TrangaTasks/UpdateLibrariesTask.cs index fa00b38..545ac9e 100644 --- a/Tranga/TrangaTasks/UpdateLibrariesTask.cs +++ b/Tranga/TrangaTasks/UpdateLibrariesTask.cs @@ -1,4 +1,5 @@ -using Logging; +using System.Net; +using Logging; namespace Tranga.TrangaTasks; @@ -8,13 +9,13 @@ public class UpdateLibrariesTask : TrangaTask { } - protected override bool ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null) + protected override HttpStatusCode ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null) { - if (cancellationToken?.IsCancellationRequested??false) - return false; + if (cancellationToken?.IsCancellationRequested ?? false) + return HttpStatusCode.RequestTimeout; foreach(LibraryManager lm in taskManager.settings.libraryManagers) lm.UpdateLibrary(); - return true; + return HttpStatusCode.OK; } public override TrangaTask Clone()