From 189569ccdfdda10263f317c73e66fc33c1db4c71 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 28 Feb 2024 20:38:22 +0100 Subject: [PATCH 01/13] dev image --- .github/workflows/docker-image-dev.yml | 45 ++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/docker-image-dev.yml diff --git a/.github/workflows/docker-image-dev.yml b/.github/workflows/docker-image-dev.yml new file mode 100644 index 0000000..32c681c --- /dev/null +++ b/.github/workflows/docker-image-dev.yml @@ -0,0 +1,45 @@ +name: Docker Image CI + +on: + push: + branches: [ "dev" ] + workflow_dispatch: + +jobs: + + build: + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + # https://github.com/docker/setup-qemu-action#usage + - name: Set up QEMU + uses: docker/setup-qemu-action@v2.2.0 + + # https://github.com/marketplace/actions/docker-setup-buildx + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3.1.0 + + # https://github.com/docker/login-action#docker-hub + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # https://github.com/docker/build-push-action#multi-platform-image + - name: Build and push API + uses: docker/build-push-action@v4.1.1 + with: + context: ./ + file: ./Dockerfile + #platforms: linux/amd64,linux/arm64,linux/riscv64,linux/ppc64le,linux/s390x,linux/386,linux/mips64le,linux/mips64,linux/arm/v7,linux/arm/v6 + platforms: linux/amd64 + pull: true + push: true + tags: | + glax/tranga-api:dev \ No newline at end of file From 2ad04c5c46585fc8516a87c06c37ed21d1ece0de Mon Sep 17 00:00:00 2001 From: Glax Date: Tue, 5 Mar 2024 02:35:37 +0100 Subject: [PATCH 02/13] Change LogFilePath to LogFolderPath #139 --- CLI/Program.cs | 4 ++-- Logging/Logger.cs | 14 +++++++------- Tranga/TrangaArgs.cs | 10 +++++----- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/CLI/Program.cs b/CLI/Program.cs index 4bb7182..ec21751 100644 --- a/CLI/Program.cs +++ b/CLI/Program.cs @@ -44,8 +44,8 @@ internal sealed class TrangaCli : Command if(settings.fileLogger is true) enabledLoggers.Add(Logger.LoggerType.FileLogger); - string? logFilePath = settings.fileLoggerPath ?? ""; - Logger logger = new(enabledLoggers.ToArray(), Console.Out, Console.OutputEncoding, logFilePath); + string? logFolderPath = settings.fileLoggerPath ?? ""; + Logger logger = new(enabledLoggers.ToArray(), Console.Out, Console.OutputEncoding, logFolderPath); TrangaSettings? trangaSettings = null; diff --git a/Logging/Logger.cs b/Logging/Logger.cs index f34bfe1..6ab86b0 100644 --- a/Logging/Logger.cs +++ b/Logging/Logger.cs @@ -20,17 +20,17 @@ public class Logger : TextWriter private readonly FormattedConsoleLogger? _formattedConsoleLogger; private readonly MemoryLogger _memoryLogger; - public Logger(LoggerType[] enabledLoggers, TextWriter? stdOut, Encoding? encoding, string? logFilePath) + public Logger(LoggerType[] enabledLoggers, TextWriter? stdOut, Encoding? encoding, string? logFolderPath) { this.Encoding = encoding ?? Encoding.UTF8; - if(enabledLoggers.Contains(LoggerType.FileLogger) && (logFilePath is null || logFilePath == "")) + DateTime now = DateTime.Now; + if(enabledLoggers.Contains(LoggerType.FileLogger) && (logFolderPath is null || logFolderPath == "")) { - DateTime now = DateTime.Now; - logFilePath = Path.Join(LogDirectoryPath, + string filePath = Path.Join(LogDirectoryPath, $"{now.ToShortDateString()}_{now.Hour}-{now.Minute}-{now.Second}.log"); - _fileLogger = new FileLogger(logFilePath, encoding); - }else if (enabledLoggers.Contains(LoggerType.FileLogger) && logFilePath is not null) - _fileLogger = new FileLogger(logFilePath, encoding); + _fileLogger = new FileLogger(filePath, encoding); + }else if (enabledLoggers.Contains(LoggerType.FileLogger) && logFolderPath is not null) + _fileLogger = new FileLogger(Path.Join(logFolderPath, $"{now.ToShortDateString()}_{now.Hour}-{now.Minute}-{now.Second}.log") , encoding); if (enabledLoggers.Contains(LoggerType.ConsoleLogger) && stdOut is not null) diff --git a/Tranga/TrangaArgs.cs b/Tranga/TrangaArgs.cs index 5739316..a4cf2ec 100644 --- a/Tranga/TrangaArgs.cs +++ b/Tranga/TrangaArgs.cs @@ -17,16 +17,16 @@ public partial class Tranga : GlobalBase string[]? consoleLogger = GetArg(args, ArgEnum.ConsoleLogger); string[]? fileLogger = GetArg(args, ArgEnum.FileLogger); - string? filePath = GetArg(args, ArgEnum.FileLoggerPath)?[0]; - if (filePath is not null && !Directory.Exists(new FileInfo(filePath).DirectoryName)) - Directory.CreateDirectory(new FileInfo(filePath).DirectoryName!); + string? directoryPath = GetArg(args, ArgEnum.FileLoggerPath)?[0]; + if (directoryPath is not null && !Directory.Exists(directoryPath)) + Directory.CreateDirectory(directoryPath); List enabledLoggers = new(); if(consoleLogger is not null) enabledLoggers.Add(Logger.LoggerType.ConsoleLogger); if (fileLogger is not null) enabledLoggers.Add(Logger.LoggerType.FileLogger); - Logger logger = new(enabledLoggers.ToArray(), Console.Out, Console.OutputEncoding, filePath); + Logger logger = new(enabledLoggers.ToArray(), Console.Out, Console.OutputEncoding, directoryPath); TrangaSettings? settings = null; string[]? downloadLocationPath = GetArg(args, ArgEnum.DownloadLocation); @@ -109,7 +109,7 @@ public partial class Tranga : GlobalBase { ArgEnum.WorkingDirectory, new(new []{"-w", "--workingDirectory"}, 1, "Directory in which application-data is saved") }, { ArgEnum.ConsoleLogger, new(new []{"-c", "--consoleLogger"}, 0, "Enables the consoleLogger") }, { ArgEnum.FileLogger, new(new []{"-f", "--fileLogger"}, 0, "Enables the fileLogger") }, - { ArgEnum.FileLoggerPath, new (new []{"-l", "--fPath"}, 1, "LogFilePath" ) }, + { ArgEnum.FileLoggerPath, new (new []{"-l", "--fPath"}, 1, "Log Folder Path" ) }, { ArgEnum.Help, new(new []{"-h", "--help"}, 0, "Print this") } //{ ArgEnum., new(new []{""}, 1, "") } }; From 590547e40701ea85a56176214e8a6244bffec350 Mon Sep 17 00:00:00 2001 From: Glax Date: Tue, 5 Mar 2024 02:55:10 +0100 Subject: [PATCH 03/13] Add Logline to print current logfilePath. --- Logging/Logger.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Logging/Logger.cs b/Logging/Logger.cs index 6ab86b0..66fd8e5 100644 --- a/Logging/Logger.cs +++ b/Logging/Logger.cs @@ -43,6 +43,7 @@ public class Logger : TextWriter throw new ArgumentException($"stdOut can not be null for LoggerType {LoggerType.ConsoleLogger}"); } _memoryLogger = new MemoryLogger(encoding); + WriteLine(GetType().ToString(), $"Logfile: {logFilePath}"); } public void WriteLine(string caller, string? value) From 17ef5eae0f565ca845c2159acc7b21d70b574a73 Mon Sep 17 00:00:00 2001 From: Glax Date: Sat, 30 Mar 2024 21:53:11 +0100 Subject: [PATCH 04/13] Fix MangaDex request for new Chapter. --- Tranga/MangaConnectors/MangaDex.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/MangaDex.cs b/Tranga/MangaConnectors/MangaDex.cs index 08c4bac..aef00df 100644 --- a/Tranga/MangaConnectors/MangaDex.cs +++ b/Tranga/MangaConnectors/MangaDex.cs @@ -252,7 +252,7 @@ public class MangaDex : MangaConnector Log($"Retrieving chapter-info {chapter} {chapterParentManga}"); //Request URLs for Chapter-Images RequestResult requestResult = - downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", RequestType.MangaDexImage); + downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false", RequestType.MangaDexImage); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) { progressToken?.Cancel(); From 94582496ef181603bf8f42d52269fb6cc6365fac Mon Sep 17 00:00:00 2001 From: Glax Date: Mon, 1 Apr 2024 20:00:02 +0200 Subject: [PATCH 05/13] Mangadex do not try downloading externally linked chapters, or chapters that have no pages. https://github.com/C9Glax/tranga/issues/153 --- Tranga/MangaConnectors/MangaDex.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Tranga/MangaConnectors/MangaDex.cs b/Tranga/MangaConnectors/MangaDex.cs index aef00df..84b8891 100644 --- a/Tranga/MangaConnectors/MangaDex.cs +++ b/Tranga/MangaConnectors/MangaDex.cs @@ -216,6 +216,7 @@ public class MangaDex : MangaConnector { JsonObject chapter = (JsonObject)jsonNode!; JsonObject attributes = chapter["attributes"]!.AsObject(); + string chapterId = chapter["id"]!.GetValue(); string? title = attributes.ContainsKey("title") && attributes["title"] is not null @@ -230,6 +231,14 @@ public class MangaDex : MangaConnector ? attributes["chapter"]!.GetValue() : "null"; + + if (attributes.ContainsKey("pages") && attributes["pages"] is not null && + attributes["pages"]!.GetValue() < 1) + { + Log($"Skipping {chapterId} Vol.{volume} Ch.{chapterNum} {title} because it has no pages or is externally linked."); + continue; + } + if(chapterNum is not "null") chapters.Add(new Chapter(manga, title, volume, chapterNum, chapterId)); } From 6a8697fc3a8490f9f6de402a27105eb3bb7024b0 Mon Sep 17 00:00:00 2001 From: Glax Date: Mon, 1 Apr 2024 20:12:25 +0200 Subject: [PATCH 06/13] Manga4Life fix bug that made it impossible for Manga to be loaded if they did not have a "Load more Chapters" button. https://github.com/C9Glax/tranga/issues/149 Created a check if the button exists before trying to click it. --- Tranga/MangaConnectors/ChromiumDownloadClient.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/ChromiumDownloadClient.cs b/Tranga/MangaConnectors/ChromiumDownloadClient.cs index 961d878..5ae80ad 100644 --- a/Tranga/MangaConnectors/ChromiumDownloadClient.cs +++ b/Tranga/MangaConnectors/ChromiumDownloadClient.cs @@ -2,6 +2,7 @@ using System.Text; using HtmlAgilityPack; using PuppeteerSharp; +using PuppeteerSharp.Input; namespace Tranga.MangaConnectors; @@ -81,7 +82,7 @@ internal class ChromiumDownloadClient : DownloadClient { if (content.Contains("text/html")) { - if(clickButton is not null) + if (clickButton is not null && page.QuerySelectorAsync(clickButton).Result is not null) page.ClickAsync(clickButton).Wait(); string htmlString = page.GetContentAsync().Result; stream = new MemoryStream(Encoding.Default.GetBytes(htmlString)); From 537ad3a5f88a57e87927fdc9280dced62393675b Mon Sep 17 00:00:00 2001 From: Glax Date: Mon, 1 Apr 2024 20:35:47 +0200 Subject: [PATCH 07/13] https://github.com/C9Glax/tranga/issues/142 Cleanup old temporary Folders and files --- Tranga/Jobs/JobBoss.cs | 2 +- Tranga/MangaConnectors/MangaConnector.cs | 6 ++++-- Tranga/Tranga.cs | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Tranga/Jobs/JobBoss.cs b/Tranga/Jobs/JobBoss.cs index f603313..99670b4 100644 --- a/Tranga/Jobs/JobBoss.cs +++ b/Tranga/Jobs/JobBoss.cs @@ -164,7 +164,7 @@ public class JobBoss : GlobalBase } HashSet coverFileNames = cachedPublications.Select(manga => manga.coverFileNameInCache!).ToHashSet(); - foreach (string fileName in Directory.GetFiles(settings.coverImageCache)) + foreach (string fileName in Directory.GetFiles(settings.coverImageCache)) //Cleanup Unused Covers { if(!coverFileNames.Any(existingManga => fileName.Contains(existingManga))) File.Delete(fileName); diff --git a/Tranga/MangaConnectors/MangaConnector.cs b/Tranga/MangaConnectors/MangaConnector.cs index bd80aed..302a064 100644 --- a/Tranga/MangaConnectors/MangaConnector.cs +++ b/Tranga/MangaConnectors/MangaConnector.cs @@ -237,7 +237,7 @@ public abstract class MangaConnector : GlobalBase return HttpStatusCode.Created; //Create a temporary folder to store images - string tempFolder = Directory.CreateTempSubdirectory().FullName; + string tempFolder = Directory.CreateTempSubdirectory("trangatemp").FullName; int chapter = 0; //Download all Images to temporary Folder @@ -260,8 +260,10 @@ public abstract class MangaConnector : GlobalBase progressToken?.Increment(); } - if(comicInfoPath is not null) + if(comicInfoPath is not null){ File.Copy(comicInfoPath, Path.Join(tempFolder, "ComicInfo.xml")); + File.Delete(comicInfoPath); //Delete tmp-file + } Log($"Creating archive {saveArchiveFilePath}"); //ZIP-it and ship-it diff --git a/Tranga/Tranga.cs b/Tranga/Tranga.cs index e146f6f..9dcc387 100644 --- a/Tranga/Tranga.cs +++ b/Tranga/Tranga.cs @@ -26,6 +26,8 @@ public partial class Tranga : GlobalBase new Bato(this), new MangaLife(this) }; + foreach(DirectoryInfo dir in new DirectoryInfo(Path.GetTempPath()).GetDirectories("trangatemp"))//Cleanup old temp folders + dir.Delete(); jobBoss = new(this, this._connectors); StartJobBoss(); this._server = new Server(this); From b27114eaad24bc3cb13f650b8953c19c78574c11 Mon Sep 17 00:00:00 2001 From: Glax Date: Mon, 15 Apr 2024 14:50:03 +0200 Subject: [PATCH 08/13] April Fools Mode https://github.com/C9Glax/tranga/issues/155 --- Tranga/Server.cs | 13 +++++++++++++ Tranga/Tranga.cs | 15 ++++++++++++++- Tranga/TrangaSettings.cs | 9 ++++++++- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/Tranga/Server.cs b/Tranga/Server.cs index cf2d15a..75b159f 100644 --- a/Tranga/Server.cs +++ b/Tranga/Server.cs @@ -206,6 +206,9 @@ public class Server : GlobalBase case "Settings/customRequestLimit": SendResponse(HttpStatusCode.OK, response, settings.requestLimits); break; + case "Settings/AprilFoolsMode": + SendResponse(HttpStatusCode.OK, response, settings.aprilFoolsMode); + break; case "NotificationConnectors": SendResponse(HttpStatusCode.OK, response, notificationConnectors); break; @@ -405,6 +408,16 @@ public class Server : GlobalBase settings.UpdateDownloadLocation(downloadLocation, moveFiles); SendResponse(HttpStatusCode.Accepted, response); break; + case "Settings/AprilFoolsMode": + if (!requestVariables.TryGetValue("enabled", out string? aprilFoolsModeEnabledStr) || + bool.TryParse(aprilFoolsModeEnabledStr, out bool aprilFoolsModeEnabled)) + { + SendResponse(HttpStatusCode.BadRequest, response); + break; + } + settings.UpdateAprilFoolsMode(aprilFoolsModeEnabled); + SendResponse(HttpStatusCode.Accepted, response); + break; /*case "Settings/UpdateWorkingDirectory": if (!requestVariables.TryGetValue("workingDirectory", out string? workingDirectory)) { diff --git a/Tranga/Tranga.cs b/Tranga/Tranga.cs index 9dcc387..f5a0f91 100644 --- a/Tranga/Tranga.cs +++ b/Tranga/Tranga.cs @@ -73,10 +73,23 @@ public partial class Tranga : GlobalBase { while (keepRunning) { - jobBoss.CheckJobs(); + if(!settings.aprilFoolsMode & !IsAprilFirst()) + jobBoss.CheckJobs(); + else + Log("April Fools Mode in Effect"); Thread.Sleep(100); } }); t.Start(); } + + private bool IsAprilFirst() + { + //UTC 01 Apr +-12hrs + DateTime start = new DateTime(DateTime.Now.Year, 03, 31, 12, 0, 0, DateTimeKind.Utc); + DateTime end = new DateTime(DateTime.Now.Year, 04, 02, 12, 0, 0, DateTimeKind.Utc); + if (DateTime.UtcNow > start && DateTime.UtcNow < end) + return true; + return false; + } } \ No newline at end of file diff --git a/Tranga/TrangaSettings.cs b/Tranga/TrangaSettings.cs index 4fae2fb..0bb8399 100644 --- a/Tranga/TrangaSettings.cs +++ b/Tranga/TrangaSettings.cs @@ -20,7 +20,8 @@ public class TrangaSettings [JsonIgnore] public string jobsFolderPath => Path.Join(workingDirectory, "jobs"); [JsonIgnore] public string coverImageCache => Path.Join(workingDirectory, "imageCache"); [JsonIgnore] internal static readonly string DefaultUserAgent = $"Tranga ({Enum.GetName(Environment.OSVersion.Platform)}; {(Environment.Is64BitOperatingSystem ? "x64" : "")}) / 1.0"; - public ushort? version { get; set; } = 1; + public ushort? version { get; } = 1; + public bool aprilFoolsMode { get; private set; } = true; [JsonIgnore]internal static readonly Dictionary DefaultRequestLimits = new () { {RequestType.MangaInfo, 250}, @@ -102,6 +103,12 @@ public class TrangaSettings })!; } + public void UpdateAprilFoolsMode(bool enabled) + { + this.aprilFoolsMode = enabled; + ExportSettings(); + } + public void UpdateDownloadLocation(string newPath, bool moveFiles = true) { if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) From 82bdb248b92d972ef28a8204ced7506bfe19404f Mon Sep 17 00:00:00 2001 From: Glax Date: Mon, 15 Apr 2024 14:50:44 +0200 Subject: [PATCH 09/13] userAgent private set in settings --- Tranga/TrangaSettings.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/TrangaSettings.cs b/Tranga/TrangaSettings.cs index 0bb8399..7d674bd 100644 --- a/Tranga/TrangaSettings.cs +++ b/Tranga/TrangaSettings.cs @@ -13,7 +13,7 @@ public class TrangaSettings public string downloadLocation { get; private set; } public string workingDirectory { get; private set; } public int apiPortNumber { get; init; } - public string userAgent { get; set; } = DefaultUserAgent; + public string userAgent { get; private set; } = DefaultUserAgent; [JsonIgnore] public string settingsFilePath => Path.Join(workingDirectory, "settings.json"); [JsonIgnore] public string libraryConnectorsFilePath => Path.Join(workingDirectory, "libraryConnectors.json"); [JsonIgnore] public string notificationConnectorsFilePath => Path.Join(workingDirectory, "notificationConnectors.json"); From c25a4f69eca0d90fe3facecec37740bcc3502641 Mon Sep 17 00:00:00 2001 From: Glax Date: Mon, 15 Apr 2024 14:51:01 +0200 Subject: [PATCH 10/13] Cleanup --- Tranga/Server.cs | 2 +- Tranga/TrangaSettings.cs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Tranga/Server.cs b/Tranga/Server.cs index 75b159f..9b99c7a 100644 --- a/Tranga/Server.cs +++ b/Tranga/Server.cs @@ -400,7 +400,7 @@ public class Server : GlobalBase case "Settings/UpdateDownloadLocation": if (!requestVariables.TryGetValue("downloadLocation", out string? downloadLocation) || !requestVariables.TryGetValue("moveFiles", out string? moveFilesStr) || - !Boolean.TryParse(moveFilesStr, out bool moveFiles)) + !bool.TryParse(moveFilesStr, out bool moveFiles)) { SendResponse(HttpStatusCode.BadRequest, response); break; diff --git a/Tranga/TrangaSettings.cs b/Tranga/TrangaSettings.cs index 7d674bd..596ab36 100644 --- a/Tranga/TrangaSettings.cs +++ b/Tranga/TrangaSettings.cs @@ -1,5 +1,4 @@ -using System.Net.Http.Headers; -using System.Runtime.InteropServices; +using System.Runtime.InteropServices; using Newtonsoft.Json; using Tranga.LibraryConnectors; using Tranga.MangaConnectors; From 431fde0d762eda186a5144cc224824c2b126d59b Mon Sep 17 00:00:00 2001 From: Glax Date: Tue, 16 Apr 2024 04:18:56 +0200 Subject: [PATCH 11/13] Wrong April Fools check. Resolves https://github.com/C9Glax/tranga/issues/159 --- Tranga/Tranga.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/Tranga.cs b/Tranga/Tranga.cs index f5a0f91..cd3b3bc 100644 --- a/Tranga/Tranga.cs +++ b/Tranga/Tranga.cs @@ -73,7 +73,7 @@ public partial class Tranga : GlobalBase { while (keepRunning) { - if(!settings.aprilFoolsMode & !IsAprilFirst()) + if(!settings.aprilFoolsMode || !IsAprilFirst()) jobBoss.CheckJobs(); else Log("April Fools Mode in Effect"); From 21a7392493e1e13f488bfa6544ac9673d2e002dc Mon Sep 17 00:00:00 2001 From: Glax Date: Thu, 18 Apr 2024 18:01:02 +0200 Subject: [PATCH 12/13] Resolves #160, Rated Manga on Mangadex. --- Tranga/MangaConnectors/MangaDex.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tranga/MangaConnectors/MangaDex.cs b/Tranga/MangaConnectors/MangaDex.cs index 84b8891..3021367 100644 --- a/Tranga/MangaConnectors/MangaDex.cs +++ b/Tranga/MangaConnectors/MangaDex.cs @@ -25,7 +25,7 @@ public class MangaDex : MangaConnector //Request next Page RequestResult requestResult = downloadClient.MakeRequest( - $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", RequestType.MangaInfo); + $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&contentRating%5B%5D=pornographic", RequestType.MangaInfo); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) break; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); @@ -200,7 +200,7 @@ public class MangaDex : MangaConnector //Request next "Page" RequestResult requestResult = downloadClient.MakeRequest( - $"https://api.mangadex.org/manga/{manga.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", RequestType.MangaDexFeed); + $"https://api.mangadex.org/manga/{manga.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&contentRating%5B%5D=pornographic", RequestType.MangaDexFeed); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) break; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); From 8f8d0198616323d1fe1d6af461837270ea32f5a0 Mon Sep 17 00:00:00 2001 From: Glax Date: Thu, 18 Apr 2024 18:56:34 +0200 Subject: [PATCH 13/13] Streamlined MangaDex information retrieval --- Tranga/MangaConnectors/MangaDex.cs | 238 ++++++++++++----------------- 1 file changed, 98 insertions(+), 140 deletions(-) diff --git a/Tranga/MangaConnectors/MangaDex.cs b/Tranga/MangaConnectors/MangaDex.cs index 3021367..a50e445 100644 --- a/Tranga/MangaConnectors/MangaDex.cs +++ b/Tranga/MangaConnectors/MangaDex.cs @@ -20,12 +20,18 @@ public class MangaDex : MangaConnector int total = int.MaxValue; //How many total results are there, is updated on first request HashSet retManga = new(); int loadedPublicationData = 0; + List results = new(); + + //Request all search-results while (offset < total) //As long as we haven't requested all "Pages" { //Request next Page - RequestResult requestResult = - downloadClient.MakeRequest( - $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&contentRating%5B%5D=pornographic", RequestType.MangaInfo); + RequestResult requestResult = downloadClient.MakeRequest( + $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}" + + $"&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica" + + $"&contentRating%5B%5D=pornographic" + + $"&includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author" + + $"&includes%5B%5D=artist&includes%5B%5D=tag", RequestType.MangaInfo); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) break; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); @@ -39,18 +45,14 @@ public class MangaDex : MangaConnector else continue; if (result.ContainsKey("data")) - { - JsonArray mangaInResult = result["data"]!.AsArray(); //Manga-data-Array - //Loop each Manga and extract information from JSON - foreach (JsonNode? mangaNode in mangaInResult) - { - if(mangaNode is null) - continue; - Log($"Getting publication data. {++loadedPublicationData}/{total}"); - if(MangaFromJsonObject((JsonObject) mangaNode) is { } manga) - retManga.Add(manga); //Add Publication (Manga) to result - } - }//else continue; + results.AddRange(result["data"]!.AsArray()!);//Manga-data-Array + } + + foreach (JsonNode mangaNode in results) + { + Log($"Getting publication data. {++loadedPublicationData}/{total}"); + if(MangaFromJsonObject(mangaNode.AsObject()) is { } manga) + retManga.Add(manga); //Add Publication (Manga) to result } Log($"Retrieved {retManga.Count} publications. Term=\"{publicationTitle}\""); return retManga.ToArray(); @@ -78,94 +80,96 @@ public class MangaDex : MangaConnector private Manga? MangaFromJsonObject(JsonObject manga) { - if (!manga.ContainsKey("attributes")) + if (!manga.TryGetPropertyValue("id", out JsonNode? idNode)) return null; - JsonObject attributes = manga["attributes"]!.AsObject(); + string publicationId = idNode!.GetValue(); + + if (!manga.TryGetPropertyValue("attributes", out JsonNode? attributesNode)) + return null; + JsonObject attributes = attributesNode!.AsObject(); + + if (!attributes.TryGetPropertyValue("title", out JsonNode? titleNode)) + return null; + string title = titleNode!.AsObject().ContainsKey("en") switch + { + true => titleNode.AsObject()["en"]!.GetValue(), + false => titleNode.AsObject().First().Value!.GetValue() + }; - if(!manga.ContainsKey("id")) - return null; - string publicationId = manga["id"]!.GetValue(); - - if(!attributes.ContainsKey("title")) - return null; - string title = attributes["title"]!.AsObject().ContainsKey("en") && attributes["title"]!["en"] is not null - ? attributes["title"]!["en"]!.GetValue() - : attributes["title"]![((IDictionary)attributes["title"]!.AsObject()).Keys.First()]!.GetValue(); - - if(!attributes.ContainsKey("description")) - return null; - string? description = attributes["description"]!.AsObject().ContainsKey("en") && attributes["description"]!["en"] is not null - ? attributes["description"]!["en"]!.GetValue() - : null; - - if(!attributes.ContainsKey("altTitles")) - return null; - JsonArray altTitlesObject = attributes["altTitles"]!.AsArray(); Dictionary altTitlesDict = new(); - foreach (JsonNode? altTitleNode in altTitlesObject) + if (attributes.TryGetPropertyValue("altTitles", out JsonNode? altTitlesNode)) { - JsonObject altTitleObject = (JsonObject)altTitleNode!; - string key = ((IDictionary)altTitleObject).Keys.ToArray()[0]; - altTitlesDict.TryAdd(key, altTitleObject[key]!.GetValue()); - } - - if(!attributes.ContainsKey("tags")) - return null; - JsonArray tagsObject = attributes["tags"]!.AsArray(); - HashSet tags = new(); - foreach (JsonNode? tagNode in tagsObject) - { - JsonObject tagObject = (JsonObject)tagNode!; - if(tagObject["attributes"]!["name"]!.AsObject().ContainsKey("en")) - tags.Add(tagObject["attributes"]!["name"]!["en"]!.GetValue()); - } - - string? posterId = null; - HashSet authorIds = new(); - if (manga.ContainsKey("relationships") && manga["relationships"] is not null) - { - JsonArray relationships = manga["relationships"]!.AsArray(); - posterId = relationships.FirstOrDefault(relationship => relationship!["type"]!.GetValue() == "cover_art")!["id"]!.GetValue(); - foreach (JsonNode? node in relationships.Where(relationship => - relationship!["type"]!.GetValue() == "author")) - authorIds.Add(node!["id"]!.GetValue()); - } - string? coverUrl = GetCoverUrl(publicationId, posterId); - string? coverCacheName = null; - if (coverUrl is not null) - coverCacheName = SaveCoverImageToCache(coverUrl, RequestType.MangaCover); - - List authors = GetAuthors(authorIds); - - Dictionary linksDict = new(); - if (attributes.ContainsKey("links") && attributes["links"] is not null) - { - JsonObject linksObject = attributes["links"]!.AsObject(); - foreach (string key in ((IDictionary)linksObject).Keys) + foreach (JsonNode? altTitleNode in altTitlesNode!.AsArray()) { - linksDict.Add(key, linksObject[key]!.GetValue()); + JsonObject altTitleNodeObject = altTitleNode!.AsObject(); + altTitlesDict.TryAdd(altTitleNodeObject.First().Key, altTitleNodeObject.First().Value!.GetValue()); } } - int? year = attributes.ContainsKey("year") && attributes["year"] is not null - ? attributes["year"]!.GetValue() - : null; + if (!attributes.TryGetPropertyValue("description", out JsonNode? descriptionNode)) + return null; + string description = descriptionNode!.AsObject().ContainsKey("en") switch + { + true => descriptionNode.AsObject()["en"]!.GetValue(), + false => descriptionNode.AsObject().First().Value!.GetValue() + }; + + Dictionary linksDict = new(); + if (attributes.TryGetPropertyValue("links", out JsonNode? linksNode)) + foreach (KeyValuePair linkKv in linksNode!.AsObject()) + linksDict.TryAdd(linkKv.Key, linkKv.Value.GetValue()); string? originalLanguage = - attributes.ContainsKey("originalLanguage") && attributes["originalLanguage"] is not null - ? attributes["originalLanguage"]!.GetValue() - : null; - - if(!attributes.ContainsKey("status")) - return null; - string status = attributes["status"]!.GetValue(); - Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased; - switch (status.ToLower()) + attributes.TryGetPropertyValue("originalLanguage", out JsonNode? originalLanguageNode) switch + { + true => originalLanguageNode!.GetValue(), + false => null + }; + + Manga.ReleaseStatusByte status = Manga.ReleaseStatusByte.Unreleased; + if (attributes.TryGetPropertyValue("status", out JsonNode? statusNode)) { - case "ongoing": releaseStatus = Manga.ReleaseStatusByte.Continuing; break; - case "completed": releaseStatus = Manga.ReleaseStatusByte.Completed; break; - case "hiatus": releaseStatus = Manga.ReleaseStatusByte.OnHiatus; break; - case "cancelled": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break; + status = statusNode!.GetValue().ToLower() switch + { + "ongoing" => Manga.ReleaseStatusByte.Continuing, + "completed" => Manga.ReleaseStatusByte.Completed, + "hiatus" => Manga.ReleaseStatusByte.OnHiatus, + "cancelled" => Manga.ReleaseStatusByte.Cancelled, + _ => Manga.ReleaseStatusByte.Unreleased + }; + } + + int? year = attributes.TryGetPropertyValue("year", out JsonNode? yearNode) switch + { + true => yearNode!.GetValue(), + false => null + }; + + HashSet tags = new(128); + if (attributes.TryGetPropertyValue("tags", out JsonNode? tagsNode)) + foreach (JsonNode? tagNode in tagsNode!.AsArray()) + tags.Add(tagNode!["attributes"]!["name"]!["en"]!.GetValue()); + + + if (!manga.TryGetPropertyValue("relationships", out JsonNode? relationshipsNode)) + return null; + + JsonNode? coverNode = relationshipsNode!.AsArray() + .FirstOrDefault(rel => rel!["type"]!.GetValue().Equals("cover_art")); + if (coverNode is null) + return null; + string fileName = coverNode["attributes"]!["fileName"]!.GetValue(); + string coverUrl = $"https://uploads.mangadex.org/covers/{publicationId}/{fileName}"; + string coverCacheName = SaveCoverImageToCache(coverUrl, RequestType.MangaCover); + + List authors = new(); + JsonNode?[] authorNodes = relationshipsNode.AsArray() + .Where(rel => rel!["type"]!.GetValue().Equals("author") || rel!["type"]!.GetValue().Equals("artist")).ToArray(); + foreach (JsonNode? authorNode in authorNodes) + { + string authorName = authorNode!["attributes"]!["name"]!.GetValue(); + if(!authors.Contains(authorName)) + authors.Add(authorName); } Manga pub = new( @@ -179,9 +183,9 @@ public class MangaDex : MangaConnector linksDict, year, originalLanguage, - status, + Enum.GetName(status) ?? "", publicationId, - releaseStatus + status ); cachedPublications.Add(pub); return pub; @@ -288,50 +292,4 @@ public class MangaDex : MangaConnector //Download Chapter-Images return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, progressToken:progressToken); } - - private string? GetCoverUrl(string publicationId, string? posterId) - { - Log($"Getting CoverUrl for Publication {publicationId}"); - if (posterId is null) - { - Log("No cover."); - return null; - } - - //Request information where to download Cover - RequestResult requestResult = - downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", RequestType.MangaCover); - if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) - return null; - JsonObject? result = JsonSerializer.Deserialize(requestResult.result); - if (result is null) - return null; - - string fileName = result["data"]!["attributes"]!["fileName"]!.GetValue(); - - string coverUrl = $"https://uploads.mangadex.org/covers/{publicationId}/{fileName}"; - Log($"Cover-Url {publicationId} -> {coverUrl}"); - return coverUrl; - } - - private List GetAuthors(IEnumerable authorIds) - { - Log("Retrieving authors."); - List ret = new(); - foreach (string authorId in authorIds) - { - RequestResult requestResult = - downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", RequestType.MangaDexAuthor); - if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) - return ret; - JsonObject? result = JsonSerializer.Deserialize(requestResult.result); - if (result is null) - return ret; - - string authorName = result["data"]!["attributes"]!["name"]!.GetValue(); - ret.Add(authorName); - Log($"Got author {authorId} -> {authorName}"); - } - return ret; - } } \ No newline at end of file