From aa692f6978bdb8ff41f3e3f4a95d1bea783392ad Mon Sep 17 00:00:00 2001 From: Glax Date: Fri, 2 Feb 2024 18:45:12 +0100 Subject: [PATCH 01/20] #108 --- Tranga/Jobs/JobJsonConverter.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Tranga/Jobs/JobJsonConverter.cs b/Tranga/Jobs/JobJsonConverter.cs index eaca679..d3ca6e0 100644 --- a/Tranga/Jobs/JobJsonConverter.cs +++ b/Tranga/Jobs/JobJsonConverter.cs @@ -38,6 +38,9 @@ public class JobJsonConverter : JsonConverter jo.GetValue("parentJobId")!.Value()); }else if ((jo.ContainsKey("jobType") && jo["jobType"]!.Value() == (byte)Job.JobType.DownloadNewChaptersJob) || jo.ContainsKey("translatedLanguage"))//TODO change to jobType { + DateTime lastExecution = jo.GetValue("lastExecution") is {} le + ? le.ToObject() + : DateTime.UnixEpoch; return new DownloadNewChapters(this._clone, jo.GetValue("mangaConnector")!.ToObject(JsonSerializer.Create(new JsonSerializerSettings() { @@ -47,7 +50,7 @@ public class JobJsonConverter : JsonConverter } }))!, jo.GetValue("manga")!.ToObject(), - jo.GetValue("lastExecution")!.ToObject(), + lastExecution, jo.GetValue("recurring")!.Value(), jo.GetValue("recurrenceTime")!.ToObject(), jo.GetValue("parentJobId")!.Value()); From 1afe36a525bee240b325b139ff1e9730825368c4 Mon Sep 17 00:00:00 2001 From: Glax Date: Fri, 2 Feb 2024 18:46:09 +0100 Subject: [PATCH 02/20] add todo --- Tranga/Jobs/JobJsonConverter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/Jobs/JobJsonConverter.cs b/Tranga/Jobs/JobJsonConverter.cs index d3ca6e0..5b12143 100644 --- a/Tranga/Jobs/JobJsonConverter.cs +++ b/Tranga/Jobs/JobJsonConverter.cs @@ -40,7 +40,7 @@ public class JobJsonConverter : JsonConverter { DateTime lastExecution = jo.GetValue("lastExecution") is {} le ? le.ToObject() - : DateTime.UnixEpoch; + : DateTime.UnixEpoch; //TODO do null checks on all variables return new DownloadNewChapters(this._clone, jo.GetValue("mangaConnector")!.ToObject(JsonSerializer.Create(new JsonSerializerSettings() { From db64b717eb620b543f6d6eb35f6a94e56cdc5e8d Mon Sep 17 00:00:00 2001 From: Glax Date: Fri, 2 Feb 2024 19:38:16 +0100 Subject: [PATCH 03/20] Fix regex for parsing publicationId --- Tranga/MangaConnectors/MangaLife.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tranga/MangaConnectors/MangaLife.cs b/Tranga/MangaConnectors/MangaLife.cs index 886f6f0..d50b6ef 100644 --- a/Tranga/MangaConnectors/MangaLife.cs +++ b/Tranga/MangaConnectors/MangaLife.cs @@ -39,8 +39,8 @@ public class MangaLife : MangaConnector public override Manga? GetMangaFromUrl(string url) { - Regex publicationIdRex = new(@"https:\/\/manga4life.com\/manga\/(.*)(\/.*)*"); - string publicationId = publicationIdRex.Match(url).Groups[1].Value; + Regex publicationIdRex = new(@"https:\/\/(www\.)?manga4life.com\/manga\/(.*)(\/.*)*"); + string publicationId = publicationIdRex.Match(url).Groups[2].Value; RequestResult requestResult = this.downloadClient.MakeRequest(url, 1); if(requestResult.htmlDocument is not null) @@ -142,7 +142,7 @@ public class MangaLife : MangaConnector HtmlNodeCollection chapterNodes = result.htmlDocument.DocumentNode.SelectNodes( "//a[contains(concat(' ',normalize-space(@class),' '),' ChapterLink ')]"); string[] urls = chapterNodes.Select(node => node.GetAttributeValue("href", "")).ToArray(); - Regex urlRex = new Regex(@"-chapter-([0-9\\.]+)(-index-([0-9\\.]+))?"); + Regex urlRex = new (@"-chapter-([0-9\\.]+)(-index-([0-9\\.]+))?"); List chapters = new(); foreach (string url in urls) From d8df6eccb14e33af6d85b7afd944f38a3c7b0a71 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 7 Feb 2024 14:53:57 +0100 Subject: [PATCH 04/20] Mangasee fix cloudflare 520 --- Tranga/MangaConnectors/Mangasee.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/Mangasee.cs b/Tranga/MangaConnectors/Mangasee.cs index a3d0955..37d5060 100644 --- a/Tranga/MangaConnectors/Mangasee.cs +++ b/Tranga/MangaConnectors/Mangasee.cs @@ -122,7 +122,7 @@ public class Mangasee : MangaConnector string publicationId = publicationIdRex.Match(url).Groups[1].Value; RequestResult requestResult = this.downloadClient.MakeRequest(url, 1); - if(requestResult.htmlDocument is not null) + if((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 && requestResult.htmlDocument is not null) return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId); return null; } From 2eaeadb92c1de49781dd06bb6e2b73de524a4bc5 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 7 Feb 2024 15:29:42 +0100 Subject: [PATCH 05/20] #113 whitespaces --- Tranga/Chapter.cs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Tranga/Chapter.cs b/Tranga/Chapter.cs index dc1c216..5af23bd 100644 --- a/Tranga/Chapter.cs +++ b/Tranga/Chapter.cs @@ -28,12 +28,14 @@ public readonly struct Chapter : IComparable this.chapterNumber = chapterNumber; this.url = url; - string chapterName = string.Concat(LegalCharacters.Matches(name ?? "")); - string volStr = volumeNumber is not null ? $"Vol.{volumeNumber} " : ""; - string chNumberStr = $"Ch.{chapterNumber} "; - string chNameStr = chapterName.Length > 0 ? $"- {chapterName}" : ""; - chNameStr = IllegalStrings.Replace(chNameStr, ""); - this.fileName = $"{volStr}{chNumberStr}{chNameStr}"; + List chapterNameStrings = new(); + + if(volumeNumber is not null && volumeNumber.Length > 0) + chapterNameStrings.Add($"Vol.{volumeNumber}"); + chapterNameStrings.Add($"Ch.{chapterNumber}"); + if(name is not null && name.Length > 0) //chapterName + chapterNameStrings.Add(IllegalStrings.Replace(string.Concat(LegalCharacters.Matches(name)), "")); + this.fileName = string.Join(' ', chapterNameStrings); } public override string ToString() From 5bc414fd590066363aab0bb608e859eca580f3e3 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 7 Feb 2024 15:34:20 +0100 Subject: [PATCH 06/20] #113 old formatting of fileNames --- Tranga/Chapter.cs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/Tranga/Chapter.cs b/Tranga/Chapter.cs index 5af23bd..99d52fd 100644 --- a/Tranga/Chapter.cs +++ b/Tranga/Chapter.cs @@ -27,15 +27,20 @@ public readonly struct Chapter : IComparable this.volumeNumber = volumeNumber ?? "0"; this.chapterNumber = chapterNumber; this.url = url; + + string chapterVolNumStr; + if (volumeNumber is not null && volumeNumber.Length > 0) + chapterVolNumStr = $"Vol.{volumeNumber} Ch.{chapterNumber}"; + else + chapterVolNumStr = $"Ch.{chapterNumber}"; - List chapterNameStrings = new(); - - if(volumeNumber is not null && volumeNumber.Length > 0) - chapterNameStrings.Add($"Vol.{volumeNumber}"); - chapterNameStrings.Add($"Ch.{chapterNumber}"); - if(name is not null && name.Length > 0) //chapterName - chapterNameStrings.Add(IllegalStrings.Replace(string.Concat(LegalCharacters.Matches(name)), "")); - this.fileName = string.Join(' ', chapterNameStrings); + if (name is not null && name.Length > 0) + { + string chapterName = IllegalStrings.Replace(string.Concat(LegalCharacters.Matches(name)), ""); + this.fileName = $"{chapterVolNumStr} - {chapterName}"; + } + else + this.fileName = chapterVolNumStr; } public override string ToString() From 2f0fbbd3cbaab1186419305b62597e5d3d407f2e Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 7 Feb 2024 15:50:26 +0100 Subject: [PATCH 07/20] #111 Fix renaming of chapters. Fixed check if Chapter exists --- Tranga/Chapter.cs | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/Tranga/Chapter.cs b/Tranga/Chapter.cs index 99d52fd..12151f7 100644 --- a/Tranga/Chapter.cs +++ b/Tranga/Chapter.cs @@ -88,23 +88,20 @@ public readonly struct Chapter : IComparable /// true if chapter is present internal bool CheckChapterIsDownloaded(string downloadLocation) { - string newFilePath = GetArchiveFilePath(downloadLocation); if (!Directory.Exists(Path.Join(downloadLocation, parentManga.folderName))) return false; FileInfo[] archives = new DirectoryInfo(Path.Join(downloadLocation, parentManga.folderName)).GetFiles(); - Regex chapterInfoRex = new(@"Ch\.[0-9.]+"); - Regex chapterRex = new(@"[0-9]+(\.[0-9]+)?"); - - if (File.Exists(newFilePath)) - return true; + Regex volChRex = new(@".*Vol(?:ume)?\.?([0-9]+).*Ch(?:apter)?\.?([0-9]+[\.0-9]*).*"); - string cn = this.chapterNumber; - if (archives.FirstOrDefault(archive => chapterRex.Match(chapterInfoRex.Match(archive.Name).Value).Value == cn) is { } path) + Chapter t = this; + return archives.Select(archive => archive.Name).Any(archiveFileName => { - File.Move(path.FullName, newFilePath); - return true; - } - return false; + Match m = volChRex.Match(archiveFileName); + string archiveVolNum = m.Groups[1].Value; + string archiveChNum = m.Groups[2].Value; + return archiveVolNum == t.volumeNumber && + archiveChNum == t.chapterNumber; + }); } /// /// Creates full file path of chapter-archive From 84272ddd1e3086c91a39b7fa88a47554899a293b Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 7 Feb 2024 18:08:57 +0100 Subject: [PATCH 08/20] https://github.com/C9Glax/tranga/issues/111#issuecomment-1932447848 --- Tranga/Chapter.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tranga/Chapter.cs b/Tranga/Chapter.cs index 12151f7..8d0622c 100644 --- a/Tranga/Chapter.cs +++ b/Tranga/Chapter.cs @@ -91,13 +91,13 @@ public readonly struct Chapter : IComparable if (!Directory.Exists(Path.Join(downloadLocation, parentManga.folderName))) return false; FileInfo[] archives = new DirectoryInfo(Path.Join(downloadLocation, parentManga.folderName)).GetFiles(); - Regex volChRex = new(@".*Vol(?:ume)?\.?([0-9]+).*Ch(?:apter)?\.?([0-9]+[\.0-9]*).*"); + Regex volChRex = new(@"(?:Vol(?:ume)?.*([0-9]+).*)?Ch(?:apter)?.*([0-9]+[\.0-9]*)"); Chapter t = this; return archives.Select(archive => archive.Name).Any(archiveFileName => { Match m = volChRex.Match(archiveFileName); - string archiveVolNum = m.Groups[1].Value; + string archiveVolNum = m.Groups[1].Success ? m.Groups[1].Value : "0"; string archiveChNum = m.Groups[2].Value; return archiveVolNum == t.volumeNumber && archiveChNum == t.chapterNumber; From 33d78ed757069691d2c08651794b87a7cc6688d6 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 7 Feb 2024 18:18:33 +0100 Subject: [PATCH 09/20] https://github.com/C9Glax/tranga/issues/111#issuecomment-1932447848 --- Tranga/Chapter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/Chapter.cs b/Tranga/Chapter.cs index 8d0622c..54679d2 100644 --- a/Tranga/Chapter.cs +++ b/Tranga/Chapter.cs @@ -91,7 +91,7 @@ public readonly struct Chapter : IComparable if (!Directory.Exists(Path.Join(downloadLocation, parentManga.folderName))) return false; FileInfo[] archives = new DirectoryInfo(Path.Join(downloadLocation, parentManga.folderName)).GetFiles(); - Regex volChRex = new(@"(?:Vol(?:ume)?.*([0-9]+).*)?Ch(?:apter)?.*([0-9]+[\.0-9]*)"); + Regex volChRex = new(@"(?:Vol(?:ume)?\D*([0-9]+).*)?Ch(?:apter)?\D*([0-9]+[\.0-9]*)"); Chapter t = this; return archives.Select(archive => archive.Name).Any(archiveFileName => From bb6c553afadd3fcb6e742ca55ee207aeae777b51 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 7 Feb 2024 19:05:11 +0100 Subject: [PATCH 10/20] One more Regex... --- Tranga/Chapter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/Chapter.cs b/Tranga/Chapter.cs index 54679d2..75fc377 100644 --- a/Tranga/Chapter.cs +++ b/Tranga/Chapter.cs @@ -91,7 +91,7 @@ public readonly struct Chapter : IComparable if (!Directory.Exists(Path.Join(downloadLocation, parentManga.folderName))) return false; FileInfo[] archives = new DirectoryInfo(Path.Join(downloadLocation, parentManga.folderName)).GetFiles(); - Regex volChRex = new(@"(?:Vol(?:ume)?\D*([0-9]+).*)?Ch(?:apter)?\D*([0-9]+[\.0-9]*)"); + Regex volChRex = new(@"(?:Vol(?:ume)?\.([0-9]+)\D*)?Ch(?:apter)?\.([0-9]+[\.0-9]*)"); Chapter t = this; return archives.Select(archive => archive.Name).Any(archiveFileName => From c03e927565aa80a687b558b81314508bfdd50d59 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 7 Feb 2024 19:23:55 +0100 Subject: [PATCH 11/20] Fix Mangaworld #103 Plurals --- Tranga/MangaConnectors/Mangaworld.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Tranga/MangaConnectors/Mangaworld.cs b/Tranga/MangaConnectors/Mangaworld.cs index 7f4dc93..f6b63bb 100644 --- a/Tranga/MangaConnectors/Mangaworld.cs +++ b/Tranga/MangaConnectors/Mangaworld.cs @@ -69,7 +69,7 @@ public class Mangaworld: MangaConnector if (requestResult.htmlDocument is null) return null; - Regex idRex = new (@"https:\/\/www\.mangaworld\.[a-z]{0,63}\/manga\/([0-9]+\/[0-9A-z\-]+)"); + Regex idRex = new (@"https:\/\/www\.mangaworld\.[a-z]{0,63}\/manga\/([0-9]+\/[0-9A-z\-]+)\/.*"); string id = idRex.Match(url).Groups[1].Value; return ParseSinglePublicationFromHtml(requestResult.htmlDocument, id); } @@ -87,19 +87,19 @@ public class Mangaworld: MangaConnector HtmlNode metadata = infoNode.Descendants().First(d => d.HasClass("meta-data")); - HtmlNode altTitlesNode = metadata.SelectSingleNode("//span[text()='Titoli alternativi: ']/..").ChildNodes[1]; + HtmlNode altTitlesNode = metadata.SelectSingleNode("//span[text()='Titoli alternativi: ' or text()='Titolo alternativo: ']/..").ChildNodes[1]; string[] alts = altTitlesNode.InnerText.Split(", "); for(int i = 0; i < alts.Length; i++) altTitles.Add(i.ToString(), alts[i]); HtmlNode genresNode = - metadata.SelectSingleNode("//span[text()='Generi: ']/.."); + metadata.SelectSingleNode("//span[text()='Generi: ' or text()='Genero: ']/.."); HashSet tags = genresNode.SelectNodes("a").Select(node => node.InnerText).ToHashSet(); HtmlNode authorsNode = - metadata.SelectSingleNode("//span[text()='Autore: ']/.."); - string[] authors = new[] { authorsNode.SelectNodes("a").First().InnerText }; + metadata.SelectSingleNode("//span[text()='Autore: ' or text()='Autori: ']/.."); + string[] authors = authorsNode.SelectNodes("a").Select(node => node.InnerText).ToArray(); string status = metadata.SelectSingleNode("//span[text()='Stato: ']/..").SelectNodes("a").First().InnerText; // ReSharper disable 5 times StringLiteralTypo From 299fa6afdaf597e6278a7c12763a7c34cc952c86 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 7 Feb 2024 19:37:35 +0100 Subject: [PATCH 12/20] I hate Regex --- Tranga/Chapter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/Chapter.cs b/Tranga/Chapter.cs index 75fc377..4650732 100644 --- a/Tranga/Chapter.cs +++ b/Tranga/Chapter.cs @@ -91,7 +91,7 @@ public readonly struct Chapter : IComparable if (!Directory.Exists(Path.Join(downloadLocation, parentManga.folderName))) return false; FileInfo[] archives = new DirectoryInfo(Path.Join(downloadLocation, parentManga.folderName)).GetFiles(); - Regex volChRex = new(@"(?:Vol(?:ume)?\.([0-9]+)\D*)?Ch(?:apter)?\.([0-9]+[\.0-9]*)"); + Regex volChRex = new(@"(?:Vol(?:ume)?\.([0-9]+)\D*)?Ch(?:apter)?\.([0-9]+(?:\.[0-9])*)"); Chapter t = this; return archives.Select(archive => archive.Name).Any(archiveFileName => From 9b990aeceaa3d1c91d08179465eb940d91aa8bc1 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 7 Feb 2024 19:40:07 +0100 Subject: [PATCH 13/20] With a passion --- Tranga/Chapter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/Chapter.cs b/Tranga/Chapter.cs index 4650732..565960c 100644 --- a/Tranga/Chapter.cs +++ b/Tranga/Chapter.cs @@ -91,7 +91,7 @@ public readonly struct Chapter : IComparable if (!Directory.Exists(Path.Join(downloadLocation, parentManga.folderName))) return false; FileInfo[] archives = new DirectoryInfo(Path.Join(downloadLocation, parentManga.folderName)).GetFiles(); - Regex volChRex = new(@"(?:Vol(?:ume)?\.([0-9]+)\D*)?Ch(?:apter)?\.([0-9]+(?:\.[0-9])*)"); + Regex volChRex = new(@"(?:Vol(?:ume)?\.([0-9]+)\D*)?Ch(?:apter)?\.([0-9]+(?:\.[0-9]+)*)"); Chapter t = this; return archives.Select(archive => archive.Name).Any(archiveFileName => From c008d55f26cc936626e762ca84266c0f2996da08 Mon Sep 17 00:00:00 2001 From: Glax Date: Thu, 8 Feb 2024 11:05:44 +0100 Subject: [PATCH 14/20] #103 Regeeeeex --- Tranga/MangaConnectors/Mangaworld.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/Mangaworld.cs b/Tranga/MangaConnectors/Mangaworld.cs index f6b63bb..8f41e8b 100644 --- a/Tranga/MangaConnectors/Mangaworld.cs +++ b/Tranga/MangaConnectors/Mangaworld.cs @@ -69,7 +69,7 @@ public class Mangaworld: MangaConnector if (requestResult.htmlDocument is null) return null; - Regex idRex = new (@"https:\/\/www\.mangaworld\.[a-z]{0,63}\/manga\/([0-9]+\/[0-9A-z\-]+)\/.*"); + Regex idRex = new (@"https:\/\/www\.mangaworld\.[a-z]{0,63}\/manga\/([0-9]+\/[0-9A-z\-]+).*"); string id = idRex.Match(url).Groups[1].Value; return ParseSinglePublicationFromHtml(requestResult.htmlDocument, id); } From 0b4461265c7fcf691f215a23f00b80dc5af3d49a Mon Sep 17 00:00:00 2001 From: Glax Date: Sun, 11 Feb 2024 00:35:33 +0100 Subject: [PATCH 15/20] #109 Rate Limits Moved Config for RateLimits to TrangaSettings Updated API: Settings/customRequestLimit requestType in RequestType.cs requestsPerMinute as int --- Tranga/Chapter.cs | 5 +-- Tranga/MangaConnectors/Bato.cs | 23 ++++++------- .../MangaConnectors/ChromiumDownloadClient.cs | 2 +- Tranga/MangaConnectors/DownloadClient.cs | 32 +++++++---------- Tranga/MangaConnectors/HttpDownloadClient.cs | 3 +- Tranga/MangaConnectors/MangaConnector.cs | 6 ++-- Tranga/MangaConnectors/MangaDex.cs | 34 +++++-------------- Tranga/MangaConnectors/MangaKatana.cs | 17 ++++------ Tranga/MangaConnectors/MangaLife.cs | 17 ++++------ Tranga/MangaConnectors/Manganato.cs | 17 ++++------ Tranga/MangaConnectors/Mangasee.cs | 15 ++++---- Tranga/MangaConnectors/Mangaworld.cs | 17 ++++------ Tranga/MangaConnectors/RequestType.cs | 12 +++++++ Tranga/Server.cs | 15 ++++---- Tranga/TrangaSettings.cs | 20 ++++++++--- 15 files changed, 110 insertions(+), 125 deletions(-) create mode 100644 Tranga/MangaConnectors/RequestType.cs diff --git a/Tranga/Chapter.cs b/Tranga/Chapter.cs index 565960c..41d1417 100644 --- a/Tranga/Chapter.cs +++ b/Tranga/Chapter.cs @@ -20,12 +20,13 @@ public readonly struct Chapter : IComparable private static readonly Regex LegalCharacters = new (@"([A-z]*[0-9]* *\.*-*,*\]*\[*'*\'*\)*\(*~*!*)*"); private static readonly Regex IllegalStrings = new(@"Vol(ume)?.?", RegexOptions.IgnoreCase); + private static readonly Regex Digits = new(@"[0-9]*"); public Chapter(Manga parentManga, string? name, string? volumeNumber, string chapterNumber, string url) { this.parentManga = parentManga; this.name = name; - this.volumeNumber = volumeNumber ?? "0"; - this.chapterNumber = chapterNumber; + this.volumeNumber = volumeNumber is not null ? string.Concat(Digits.Matches(volumeNumber).Select(x => x.Value)) : "0"; + this.chapterNumber = string.Concat(Digits.Matches(chapterNumber).Select(x => x.Value)); this.url = url; string chapterVolNumStr; diff --git a/Tranga/MangaConnectors/Bato.cs b/Tranga/MangaConnectors/Bato.cs index ff29f18..96d97c0 100644 --- a/Tranga/MangaConnectors/Bato.cs +++ b/Tranga/MangaConnectors/Bato.cs @@ -7,12 +7,10 @@ namespace Tranga.MangaConnectors; public class Bato : MangaConnector { + public Bato(GlobalBase clone) : base(clone, "Bato") { - this.downloadClient = new HttpDownloadClient(clone, new Dictionary() - { - {1, 60} - }); + this.downloadClient = new HttpDownloadClient(clone); } public override Manga[] GetManga(string publicationTitle = "") @@ -21,7 +19,7 @@ public class Bato : MangaConnector string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower(); string requestUrl = $"https://bato.to/v3x-search?word={sanitizedTitle}&lang=en"; RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, 1); + downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return Array.Empty(); @@ -43,8 +41,7 @@ public class Bato : MangaConnector public override Manga? GetMangaFromUrl(string url) { - RequestResult requestResult = - downloadClient.MakeRequest(url, 1); + RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return null; if (requestResult.htmlDocument is null) @@ -89,7 +86,7 @@ public class Bato : MangaConnector string posterUrl = document.DocumentNode.SelectNodes("//img") .First(child => child.GetAttributeValue("data-hk", "") == "0-1-0").GetAttributeValue("src", "").Replace("&", "&"); - string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1); + string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover); List genreNodes = document.DocumentNode.SelectSingleNode("//b[text()='Genres:']/..").SelectNodes("span").ToList(); string[] tags = genreNodes.Select(node => node.FirstChild.InnerText).ToArray(); @@ -129,7 +126,7 @@ public class Bato : MangaConnector string requestUrl = $"https://bato.to/title/{manga.publicationId}"; // Leaving this in for verification if the page exists RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, 1); + downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return Array.Empty(); @@ -141,7 +138,7 @@ public class Bato : MangaConnector private List ParseChaptersFromHtml(Manga manga, string mangaUrl) { - RequestResult result = downloadClient.MakeRequest(mangaUrl, 1); + RequestResult result = downloadClient.MakeRequest(mangaUrl, RequestType.Default); if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null) { Log("Failed to load site"); @@ -184,7 +181,7 @@ public class Bato : MangaConnector string requestUrl = chapter.url; // Leaving this in to check if the page exists RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, 1); + downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) { progressToken?.Cancel(); @@ -196,13 +193,13 @@ public class Bato : MangaConnector string comicInfoPath = Path.GetTempFileName(); File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString()); - return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken); + return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken); } private string[] ParseImageUrlsFromHtml(string mangaUrl) { RequestResult requestResult = - downloadClient.MakeRequest(mangaUrl, 1); + downloadClient.MakeRequest(mangaUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) { return Array.Empty(); diff --git a/Tranga/MangaConnectors/ChromiumDownloadClient.cs b/Tranga/MangaConnectors/ChromiumDownloadClient.cs index cf8f1fd..961d878 100644 --- a/Tranga/MangaConnectors/ChromiumDownloadClient.cs +++ b/Tranga/MangaConnectors/ChromiumDownloadClient.cs @@ -53,7 +53,7 @@ internal class ChromiumDownloadClient : DownloadClient }); } - public ChromiumDownloadClient(GlobalBase clone, Dictionary rateLimitRequestsPerMinute) : base(clone, rateLimitRequestsPerMinute) + public ChromiumDownloadClient(GlobalBase clone) : base(clone) { this.browser = DownloadBrowser().Result; } diff --git a/Tranga/MangaConnectors/DownloadClient.cs b/Tranga/MangaConnectors/DownloadClient.cs index d60ee5f..0e9b343 100644 --- a/Tranga/MangaConnectors/DownloadClient.cs +++ b/Tranga/MangaConnectors/DownloadClient.cs @@ -5,37 +5,29 @@ namespace Tranga.MangaConnectors; internal abstract class DownloadClient : GlobalBase { - private readonly Dictionary _lastExecutedRateLimit; - private readonly Dictionary _rateLimit; + private readonly Dictionary _lastExecutedRateLimit; - protected DownloadClient(GlobalBase clone, Dictionary rateLimitRequestsPerMinute) : base(clone) + protected DownloadClient(GlobalBase clone) : base(clone) { this._lastExecutedRateLimit = new(); - _rateLimit = new(); - foreach (KeyValuePair limit in rateLimitRequestsPerMinute) - _rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value)); - } - - internal void SetCustomRequestLimit(byte requestType, int limit) - { - if (_rateLimit.ContainsKey(requestType)) - _rateLimit[requestType] = TimeSpan.FromMinutes(1).Divide(limit); - else - _rateLimit.Add(requestType, TimeSpan.FromMinutes(1).Divide(limit)); } - public RequestResult MakeRequest(string url, byte requestType, string? referrer = null, string? clickButton = null) + public RequestResult MakeRequest(string url, RequestType requestType, string? referrer = null, string? clickButton = null) { - if (_rateLimit.TryGetValue(requestType, out TimeSpan value)) - _lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value)); - else + if (!settings.requestLimits.ContainsKey(requestType)) { Log("RequestType not configured for rate-limit."); return new RequestResult(HttpStatusCode.NotAcceptable, null, Stream.Null); } - TimeSpan rateLimitTimeout = _rateLimit[requestType] - .Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType])); + int rateLimit = settings.userAgent == TrangaSettings.DefaultUserAgent + ? TrangaSettings.DefaultRequestLimits[requestType] + : settings.requestLimits[requestType]; + + TimeSpan timeBetweenRequests = TimeSpan.FromMinutes(1).Divide(rateLimit); + _lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(timeBetweenRequests)); + + TimeSpan rateLimitTimeout = timeBetweenRequests.Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType])); if (rateLimitTimeout > TimeSpan.Zero) { diff --git a/Tranga/MangaConnectors/HttpDownloadClient.cs b/Tranga/MangaConnectors/HttpDownloadClient.cs index 342b8a1..46a0008 100644 --- a/Tranga/MangaConnectors/HttpDownloadClient.cs +++ b/Tranga/MangaConnectors/HttpDownloadClient.cs @@ -11,8 +11,7 @@ internal class HttpDownloadClient : DownloadClient Timeout = TimeSpan.FromSeconds(10) }; - - public HttpDownloadClient(GlobalBase clone, Dictionary rateLimitRequestsPerMinute) : base(clone, rateLimitRequestsPerMinute) + public HttpDownloadClient(GlobalBase clone) : base(clone) { Client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", settings.userAgent); } diff --git a/Tranga/MangaConnectors/MangaConnector.cs b/Tranga/MangaConnectors/MangaConnector.cs index 8d7df98..bd80aed 100644 --- a/Tranga/MangaConnectors/MangaConnector.cs +++ b/Tranga/MangaConnectors/MangaConnector.cs @@ -202,7 +202,7 @@ public abstract class MangaConnector : GlobalBase /// /// RequestType for Rate-Limit /// referrer used in html request header - private HttpStatusCode DownloadImage(string imageUrl, string fullPath, byte requestType, string? referrer = null) + private HttpStatusCode DownloadImage(string imageUrl, string fullPath, RequestType requestType, string? referrer = null) { RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType, referrer); @@ -217,7 +217,7 @@ public abstract class MangaConnector : GlobalBase return requestResult.statusCode; } - protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, string? comicInfoPath = null, string? referrer = null, ProgressToken? progressToken = null) + protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, RequestType requestType, string? comicInfoPath = null, string? referrer = null, ProgressToken? progressToken = null) { if (progressToken?.cancellationRequested ?? false) return HttpStatusCode.RequestTimeout; @@ -274,7 +274,7 @@ public abstract class MangaConnector : GlobalBase return HttpStatusCode.OK; } - protected string SaveCoverImageToCache(string url, byte requestType) + protected string SaveCoverImageToCache(string url, RequestType requestType) { string filetype = url.Split('/')[^1].Split('?')[0].Split('.')[^1]; string filename = $"{DateTime.Now.Ticks.ToString()}.{filetype}"; diff --git a/Tranga/MangaConnectors/MangaDex.cs b/Tranga/MangaConnectors/MangaDex.cs index eb12f8b..08c4bac 100644 --- a/Tranga/MangaConnectors/MangaDex.cs +++ b/Tranga/MangaConnectors/MangaDex.cs @@ -7,25 +7,9 @@ using JsonSerializer = System.Text.Json.JsonSerializer; namespace Tranga.MangaConnectors; public class MangaDex : MangaConnector { - private enum RequestType : byte - { - Manga, - Feed, - AtHomeServer, - CoverUrl, - Author, - } - public MangaDex(GlobalBase clone) : base(clone, "MangaDex") { - this.downloadClient = new HttpDownloadClient(clone, new Dictionary() - { - {(byte)RequestType.Manga, 250}, - {(byte)RequestType.Feed, 250}, - {(byte)RequestType.AtHomeServer, 40}, - {(byte)RequestType.CoverUrl, 250}, - {(byte)RequestType.Author, 250} - }); + this.downloadClient = new HttpDownloadClient(clone); } public override Manga[] GetManga(string publicationTitle = "") @@ -41,7 +25,7 @@ public class MangaDex : MangaConnector //Request next Page RequestResult requestResult = downloadClient.MakeRequest( - $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", (byte)RequestType.Manga); + $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", RequestType.MangaInfo); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) break; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); @@ -75,7 +59,7 @@ public class MangaDex : MangaConnector public override Manga? GetMangaFromId(string publicationId) { RequestResult requestResult = - downloadClient.MakeRequest($"https://api.mangadex.org/manga/{publicationId}", (byte)RequestType.Manga); + downloadClient.MakeRequest($"https://api.mangadex.org/manga/{publicationId}", RequestType.MangaInfo); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return null; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); @@ -149,7 +133,7 @@ public class MangaDex : MangaConnector string? coverUrl = GetCoverUrl(publicationId, posterId); string? coverCacheName = null; if (coverUrl is not null) - coverCacheName = SaveCoverImageToCache(coverUrl, (byte)RequestType.AtHomeServer); + coverCacheName = SaveCoverImageToCache(coverUrl, RequestType.MangaCover); List authors = GetAuthors(authorIds); @@ -216,7 +200,7 @@ public class MangaDex : MangaConnector //Request next "Page" RequestResult requestResult = downloadClient.MakeRequest( - $"https://api.mangadex.org/manga/{manga.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", (byte)RequestType.Feed); + $"https://api.mangadex.org/manga/{manga.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", RequestType.MangaDexFeed); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) break; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); @@ -268,7 +252,7 @@ public class MangaDex : MangaConnector Log($"Retrieving chapter-info {chapter} {chapterParentManga}"); //Request URLs for Chapter-Images RequestResult requestResult = - downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer); + downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", RequestType.MangaDexImage); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) { progressToken?.Cancel(); @@ -293,7 +277,7 @@ public class MangaDex : MangaConnector File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString()); //Download Chapter-Images - return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), (byte)RequestType.AtHomeServer, comicInfoPath, progressToken:progressToken); + return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, progressToken:progressToken); } private string? GetCoverUrl(string publicationId, string? posterId) @@ -307,7 +291,7 @@ public class MangaDex : MangaConnector //Request information where to download Cover RequestResult requestResult = - downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", (byte)RequestType.CoverUrl); + downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", RequestType.MangaCover); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return null; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); @@ -328,7 +312,7 @@ public class MangaDex : MangaConnector foreach (string authorId in authorIds) { RequestResult requestResult = - downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", (byte)RequestType.Author); + downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", RequestType.MangaDexAuthor); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return ret; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); diff --git a/Tranga/MangaConnectors/MangaKatana.cs b/Tranga/MangaConnectors/MangaKatana.cs index 9703934..a62315b 100644 --- a/Tranga/MangaConnectors/MangaKatana.cs +++ b/Tranga/MangaConnectors/MangaKatana.cs @@ -9,10 +9,7 @@ public class MangaKatana : MangaConnector { public MangaKatana(GlobalBase clone) : base(clone, "MangaKatana") { - this.downloadClient = new HttpDownloadClient(clone, new Dictionary() - { - {1, 60} - }); + this.downloadClient = new HttpDownloadClient(clone); } public override Manga[] GetManga(string publicationTitle = "") @@ -21,7 +18,7 @@ public class MangaKatana : MangaConnector string sanitizedTitle = string.Join('_', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower(); string requestUrl = $"https://mangakatana.com/?search={sanitizedTitle}&search_by=book_name"; RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, 1); + downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return Array.Empty(); @@ -47,7 +44,7 @@ public class MangaKatana : MangaConnector public override Manga? GetMangaFromUrl(string url) { RequestResult requestResult = - downloadClient.MakeRequest(url, 1); + downloadClient.MakeRequest(url, RequestType.MangaInfo); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return null; return ParseSinglePublicationFromHtml(requestResult.result, url.Split('/')[^1]); @@ -131,7 +128,7 @@ public class MangaKatana : MangaConnector string posterUrl = document.DocumentNode.SelectSingleNode("//*[@id='single_book']/div[1]/div").Descendants("img").First() .GetAttributes().First(a => a.Name == "src").Value; - string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1); + string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover); string description = document.DocumentNode.SelectSingleNode("//*[@id='single_book']/div[3]/p").InnerText; while (description.StartsWith('\n')) @@ -158,7 +155,7 @@ public class MangaKatana : MangaConnector string requestUrl = $"https://mangakatana.com/manga/{manga.publicationId}"; // Leaving this in for verification if the page exists RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, 1); + downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return Array.Empty(); @@ -210,7 +207,7 @@ public class MangaKatana : MangaConnector string requestUrl = chapter.url; // Leaving this in to check if the page exists RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, 1); + downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) { progressToken?.Cancel(); @@ -222,7 +219,7 @@ public class MangaKatana : MangaConnector string comicInfoPath = Path.GetTempFileName(); File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString()); - return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken); + return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken); } private string[] ParseImageUrlsFromHtml(string mangaUrl) diff --git a/Tranga/MangaConnectors/MangaLife.cs b/Tranga/MangaConnectors/MangaLife.cs index d50b6ef..fd92377 100644 --- a/Tranga/MangaConnectors/MangaLife.cs +++ b/Tranga/MangaConnectors/MangaLife.cs @@ -9,10 +9,7 @@ public class MangaLife : MangaConnector { public MangaLife(GlobalBase clone) : base(clone, "Manga4Life") { - this.downloadClient = new ChromiumDownloadClient(clone, new Dictionary() - { - { 1, 60 } - }); + this.downloadClient = new ChromiumDownloadClient(clone); } public override Manga[] GetManga(string publicationTitle = "") @@ -21,7 +18,7 @@ public class MangaLife : MangaConnector string sanitizedTitle = WebUtility.UrlEncode(publicationTitle); string requestUrl = $"https://manga4life.com/search/?name={sanitizedTitle}"; RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, 1); + downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return Array.Empty(); @@ -42,7 +39,7 @@ public class MangaLife : MangaConnector Regex publicationIdRex = new(@"https:\/\/(www\.)?manga4life.com\/manga\/(.*)(\/.*)*"); string publicationId = publicationIdRex.Match(url).Groups[2].Value; - RequestResult requestResult = this.downloadClient.MakeRequest(url, 1); + RequestResult requestResult = this.downloadClient.MakeRequest(url, RequestType.MangaInfo); if(requestResult.htmlDocument is not null) return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId); return null; @@ -81,7 +78,7 @@ public class MangaLife : MangaConnector HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//img"); string posterUrl = posterNode.GetAttributeValue("src", ""); - string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1); + string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover); HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//h1"); string sortName = titleNode.InnerText; @@ -133,7 +130,7 @@ public class MangaLife : MangaConnector public override Chapter[] GetChapters(Manga manga, string language="en") { Log($"Getting chapters {manga}"); - RequestResult result = downloadClient.MakeRequest($"https://manga4life.com/manga/{manga.publicationId}", 1, clickButton:"[class*='ShowAllChapters']"); + RequestResult result = downloadClient.MakeRequest($"https://manga4life.com/manga/{manga.publicationId}", RequestType.Default, clickButton:"[class*='ShowAllChapters']"); if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null) { return Array.Empty(); @@ -179,7 +176,7 @@ public class MangaLife : MangaConnector Log($"Retrieving chapter-info {chapter} {chapterParentManga}"); - RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, 1); + RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, RequestType.Default); if (requestResult.htmlDocument is null) { progressToken?.Cancel(); @@ -197,6 +194,6 @@ public class MangaLife : MangaConnector string comicInfoPath = Path.GetTempFileName(); File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString()); - return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, progressToken:progressToken); + return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, progressToken:progressToken); } } \ No newline at end of file diff --git a/Tranga/MangaConnectors/Manganato.cs b/Tranga/MangaConnectors/Manganato.cs index ff301b9..d7bbd79 100644 --- a/Tranga/MangaConnectors/Manganato.cs +++ b/Tranga/MangaConnectors/Manganato.cs @@ -9,10 +9,7 @@ public class Manganato : MangaConnector { public Manganato(GlobalBase clone) : base(clone, "Manganato") { - this.downloadClient = new HttpDownloadClient(clone, new Dictionary() - { - {1, 60} - }); + this.downloadClient = new HttpDownloadClient(clone); } public override Manga[] GetManga(string publicationTitle = "") @@ -21,7 +18,7 @@ public class Manganato : MangaConnector string sanitizedTitle = string.Join('_', Regex.Matches(publicationTitle, "[A-z]*").Where(str => str.Length > 0)).ToLower(); string requestUrl = $"https://manganato.com/search/story/{sanitizedTitle}"; RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, 1); + downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return Array.Empty(); @@ -62,7 +59,7 @@ public class Manganato : MangaConnector public override Manga? GetMangaFromUrl(string url) { RequestResult requestResult = - downloadClient.MakeRequest(url, 1); + downloadClient.MakeRequest(url, RequestType.MangaInfo); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return null; @@ -121,7 +118,7 @@ public class Manganato : MangaConnector string posterUrl = document.DocumentNode.Descendants("span").First(s => s.HasClass("info-image")).Descendants("img").First() .GetAttributes().First(a => a.Name == "src").Value; - string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1); + string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover); string description = document.DocumentNode.Descendants("div").First(d => d.HasClass("panel-story-info-description")) .InnerText.Replace("Description :", ""); @@ -143,7 +140,7 @@ public class Manganato : MangaConnector Log($"Getting chapters {manga}"); string requestUrl = $"https://chapmanganato.com/{manga.publicationId}"; RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, 1); + downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return Array.Empty(); @@ -192,7 +189,7 @@ public class Manganato : MangaConnector Log($"Retrieving chapter-info {chapter} {chapterParentManga}"); string requestUrl = chapter.url; RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, 1); + downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) { progressToken?.Cancel(); @@ -210,7 +207,7 @@ public class Manganato : MangaConnector string comicInfoPath = Path.GetTempFileName(); File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString()); - return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://chapmanganato.com/", progressToken:progressToken); + return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, "https://chapmanganato.com/", progressToken:progressToken); } private string[] ParseImageUrlsFromHtml(HtmlDocument document) diff --git a/Tranga/MangaConnectors/Mangasee.cs b/Tranga/MangaConnectors/Mangasee.cs index 37d5060..c778793 100644 --- a/Tranga/MangaConnectors/Mangasee.cs +++ b/Tranga/MangaConnectors/Mangasee.cs @@ -12,10 +12,7 @@ public class Mangasee : MangaConnector { public Mangasee(GlobalBase clone) : base(clone, "Mangasee") { - this.downloadClient = new ChromiumDownloadClient(clone, new Dictionary() - { - { 1, 60 } - }); + this.downloadClient = new ChromiumDownloadClient(clone); } private struct SearchResult @@ -30,7 +27,7 @@ public class Mangasee : MangaConnector Log($"Searching Publications. Term=\"{publicationTitle}\""); string requestUrl = "https://mangasee123.com/_search.php"; RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, 1); + downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) { Log($"Failed to retrieve search: {requestResult.statusCode}"); @@ -121,7 +118,7 @@ public class Mangasee : MangaConnector Regex publicationIdRex = new(@"https:\/\/mangasee123.com\/manga\/(.*)(\/.*)*"); string publicationId = publicationIdRex.Match(url).Groups[1].Value; - RequestResult requestResult = this.downloadClient.MakeRequest(url, 1); + RequestResult requestResult = this.downloadClient.MakeRequest(url, RequestType.MangaInfo); if((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 && requestResult.htmlDocument is not null) return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId); return null; @@ -136,7 +133,7 @@ public class Mangasee : MangaConnector HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//img"); string posterUrl = posterNode.GetAttributeValue("src", ""); - string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1); + string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover); HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//h1"); string sortName = titleNode.InnerText; @@ -232,7 +229,7 @@ public class Mangasee : MangaConnector Log($"Retrieving chapter-info {chapter} {chapterParentManga}"); - RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, 1); + RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, RequestType.Default); if (requestResult.htmlDocument is null) { progressToken?.Cancel(); @@ -250,6 +247,6 @@ public class Mangasee : MangaConnector string comicInfoPath = Path.GetTempFileName(); File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString()); - return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, progressToken:progressToken); + return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, progressToken:progressToken); } } \ No newline at end of file diff --git a/Tranga/MangaConnectors/Mangaworld.cs b/Tranga/MangaConnectors/Mangaworld.cs index 8f41e8b..a457999 100644 --- a/Tranga/MangaConnectors/Mangaworld.cs +++ b/Tranga/MangaConnectors/Mangaworld.cs @@ -9,10 +9,7 @@ public class Mangaworld: MangaConnector { public Mangaworld(GlobalBase clone) : base(clone, "Mangaworld") { - this.downloadClient = new HttpDownloadClient(clone, new Dictionary() - { - {1, 60} - }); + this.downloadClient = new HttpDownloadClient(clone); } public override Manga[] GetManga(string publicationTitle = "") @@ -21,7 +18,7 @@ public class Mangaworld: MangaConnector string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(str => str.Length > 0)).ToLower(); string requestUrl = $"https://www.mangaworld.ac/archive?keyword={sanitizedTitle}"; RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, 1); + downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return Array.Empty(); @@ -62,7 +59,7 @@ public class Mangaworld: MangaConnector public override Manga? GetMangaFromUrl(string url) { RequestResult requestResult = - downloadClient.MakeRequest(url, 1); + downloadClient.MakeRequest(url, RequestType.MangaInfo); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return null; @@ -114,7 +111,7 @@ public class Mangaworld: MangaConnector string posterUrl = document.DocumentNode.SelectSingleNode("//img[@class='rounded']").GetAttributeValue("src", ""); - string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1); + string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover); string description = document.DocumentNode.SelectSingleNode("//div[@id='noidungm']").InnerText; @@ -132,7 +129,7 @@ public class Mangaworld: MangaConnector Log($"Getting chapters {manga}"); string requestUrl = $"https://www.mangaworld.ac/manga/{manga.publicationId}"; RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, 1); + downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return Array.Empty(); @@ -191,7 +188,7 @@ public class Mangaworld: MangaConnector Log($"Retrieving chapter-info {chapter} {chapterParentManga}"); string requestUrl = $"{chapter.url}?style=list"; RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, 1); + downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) { progressToken?.Cancel(); @@ -209,7 +206,7 @@ public class Mangaworld: MangaConnector string comicInfoPath = Path.GetTempFileName(); File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString()); - return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://www.mangaworld.bz/", progressToken:progressToken); + return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, "https://www.mangaworld.bz/", progressToken:progressToken); } private string[] ParseImageUrlsFromHtml(HtmlDocument document) diff --git a/Tranga/MangaConnectors/RequestType.cs b/Tranga/MangaConnectors/RequestType.cs new file mode 100644 index 0000000..eb89b42 --- /dev/null +++ b/Tranga/MangaConnectors/RequestType.cs @@ -0,0 +1,12 @@ +namespace Tranga.MangaConnectors; + +public enum RequestType : byte +{ + Default = 0, + MangaDexFeed = 1, + MangaImage = 2, + MangaCover = 3, + MangaDexAuthor = 4, + MangaDexImage = 5, + MangaInfo = 6 +} \ No newline at end of file diff --git a/Tranga/Server.cs b/Tranga/Server.cs index 6aa518f..c12f1fa 100644 --- a/Tranga/Server.cs +++ b/Tranga/Server.cs @@ -399,16 +399,19 @@ public class Server : GlobalBase case "Settings/customRequestLimit": if (!requestVariables.TryGetValue("requestType", out string? requestTypeStr) || !requestVariables.TryGetValue("requestsPerMinute", out string? requestsPerMinuteStr) || - !requestVariables.TryGetValue("connector", out connectorName) || - !byte.TryParse(requestTypeStr, out byte requestType) || - !int.TryParse(requestsPerMinuteStr, out int requestsPerMinute) || - !_parent.TryGetConnector(connectorName, out connector)) + !Enum.TryParse(requestTypeStr, out RequestType requestType) || + !int.TryParse(requestsPerMinuteStr, out int requestsPerMinute)) { SendResponse(HttpStatusCode.BadRequest, response); break; } - connector!.downloadClient.SetCustomRequestLimit(requestType, requestsPerMinute); - SendResponse(HttpStatusCode.Accepted, response); + + if (settings.requestLimits.ContainsKey(requestType)) + { + settings.requestLimits[requestType] = requestsPerMinute; + SendResponse(HttpStatusCode.Accepted, response); + }else + SendResponse(HttpStatusCode.BadRequest, response); break; case "NotificationConnectors/Update": if (!requestVariables.TryGetValue("notificationConnector", out string? notificationConnectorStr) || diff --git a/Tranga/TrangaSettings.cs b/Tranga/TrangaSettings.cs index a75c8dd..4058188 100644 --- a/Tranga/TrangaSettings.cs +++ b/Tranga/TrangaSettings.cs @@ -2,6 +2,7 @@ using System.Runtime.InteropServices; using Newtonsoft.Json; using Tranga.LibraryConnectors; +using Tranga.MangaConnectors; using Tranga.NotificationConnectors; using static System.IO.UnixFileMode; @@ -13,15 +14,26 @@ public class TrangaSettings public string workingDirectory { get; private set; } public int apiPortNumber { get; init; } public string styleSheet { get; private set; } - - public string userAgent { get; set; } = - $"Tranga ({Enum.GetName(Environment.OSVersion.Platform)}; {(Environment.Is64BitOperatingSystem ? "x64" : "")}) / 1.0"; + public string userAgent { get; set; } = DefaultUserAgent; [JsonIgnore] public string settingsFilePath => Path.Join(workingDirectory, "settings.json"); [JsonIgnore] public string libraryConnectorsFilePath => Path.Join(workingDirectory, "libraryConnectors.json"); [JsonIgnore] public string notificationConnectorsFilePath => Path.Join(workingDirectory, "notificationConnectors.json"); [JsonIgnore] public string jobsFolderPath => Path.Join(workingDirectory, "jobs"); [JsonIgnore] public string coverImageCache => Path.Join(workingDirectory, "imageCache"); - public ushort? version { get; set; } + [JsonIgnore] internal static readonly string DefaultUserAgent = $"Tranga ({Enum.GetName(Environment.OSVersion.Platform)}; {(Environment.Is64BitOperatingSystem ? "x64" : "")}) / 1.0"; + public ushort? version { get; set; } = 1; + [JsonIgnore]internal static readonly Dictionary DefaultRequestLimits = new () + { + {RequestType.MangaInfo, 250}, + {RequestType.MangaDexFeed, 250}, + {RequestType.MangaDexImage, 40}, + {RequestType.MangaImage, 60}, + {RequestType.MangaCover, 250}, + {RequestType.MangaDexAuthor, 250}, + {RequestType.Default, 60} + }; + + public Dictionary requestLimits { get; set; } = DefaultRequestLimits; public TrangaSettings(string? downloadLocation = null, string? workingDirectory = null, int? apiPortNumber = null) { From 0aa92a791364610c3836ed8c2b07e0b5394a5c8e Mon Sep 17 00:00:00 2001 From: Glax Date: Mon, 12 Feb 2024 11:22:19 +0100 Subject: [PATCH 16/20] Correct VolumeNumbers for Mangasee --- Tranga/MangaConnectors/Mangasee.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Tranga/MangaConnectors/Mangasee.cs b/Tranga/MangaConnectors/Mangasee.cs index c778793..37ff6c4 100644 --- a/Tranga/MangaConnectors/Mangasee.cs +++ b/Tranga/MangaConnectors/Mangasee.cs @@ -191,13 +191,15 @@ public class Mangasee : MangaConnector XDocument doc = XDocument.Load($"https://mangasee123.com/rss/{manga.publicationId}.xml"); XElement[] chapterItems = doc.Descendants("item").ToArray(); List chapters = new(); + Regex chVolRex = new(@".*chapter-([0-9\.]+)(?:-index-([0-9\.]+))?.*"); foreach (XElement chapter in chapterItems) { - string volumeNumber = "1"; string url = chapter.Descendants("link").First().Value; - string chapterNumber = Regex.Match(url, @"-chapter-([0-9\.]+)").Groups[1].ToString(); + Match m = chVolRex.Match(url); + string volumeNumber = m.Groups[2].Success ? m.Groups[2].Value : "0"; + string chapterNumber = m.Groups[2].Value; - url = url.Replace(Regex.Match(url,"(-page-[0-9])").Value,""); + url = string.Concat(Regex.Match(url, @"(.*)-page-[0-9]+(\.html)").Groups.Values.Select(v => v.Value)); chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, url)); } From 4888e18fd262b54d99a125810159241db529c66c Mon Sep 17 00:00:00 2001 From: Glax Date: Mon, 12 Feb 2024 20:49:33 +0100 Subject: [PATCH 17/20] Correct ChapterNumbers for Mangasee --- Tranga/MangaConnectors/Mangasee.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/Mangasee.cs b/Tranga/MangaConnectors/Mangasee.cs index 37ff6c4..c05d8be 100644 --- a/Tranga/MangaConnectors/Mangasee.cs +++ b/Tranga/MangaConnectors/Mangasee.cs @@ -197,7 +197,7 @@ public class Mangasee : MangaConnector string url = chapter.Descendants("link").First().Value; Match m = chVolRex.Match(url); string volumeNumber = m.Groups[2].Success ? m.Groups[2].Value : "0"; - string chapterNumber = m.Groups[2].Value; + string chapterNumber = m.Groups[1].Value; url = string.Concat(Regex.Match(url, @"(.*)-page-[0-9]+(\.html)").Groups.Values.Select(v => v.Value)); chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, url)); From 9691eb0d08a96c8ebfec8ec23877a7d08788f9a3 Mon Sep 17 00:00:00 2001 From: Glax Date: Mon, 12 Feb 2024 21:02:01 +0100 Subject: [PATCH 18/20] Correct ChapterNumbers for Mangasee --- Tranga/MangaConnectors/Mangasee.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/Mangasee.cs b/Tranga/MangaConnectors/Mangasee.cs index c05d8be..5d45ac6 100644 --- a/Tranga/MangaConnectors/Mangasee.cs +++ b/Tranga/MangaConnectors/Mangasee.cs @@ -196,7 +196,7 @@ public class Mangasee : MangaConnector { string url = chapter.Descendants("link").First().Value; Match m = chVolRex.Match(url); - string volumeNumber = m.Groups[2].Success ? m.Groups[2].Value : "0"; + string? volumeNumber = m.Groups[2].Success ? m.Groups[2].Value : null; string chapterNumber = m.Groups[1].Value; url = string.Concat(Regex.Match(url, @"(.*)-page-[0-9]+(\.html)").Groups.Values.Select(v => v.Value)); From 2a824bbb8d002122f735e7f0b9749e92277cbc01 Mon Sep 17 00:00:00 2001 From: Glax Date: Mon, 12 Feb 2024 21:04:14 +0100 Subject: [PATCH 19/20] Correct "1" ChapterNumbers for Mangasee --- Tranga/MangaConnectors/Mangasee.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/Mangasee.cs b/Tranga/MangaConnectors/Mangasee.cs index 5d45ac6..3a0efde 100644 --- a/Tranga/MangaConnectors/Mangasee.cs +++ b/Tranga/MangaConnectors/Mangasee.cs @@ -196,7 +196,7 @@ public class Mangasee : MangaConnector { string url = chapter.Descendants("link").First().Value; Match m = chVolRex.Match(url); - string? volumeNumber = m.Groups[2].Success ? m.Groups[2].Value : null; + string? volumeNumber = m.Groups[2].Success ? m.Groups[2].Value : "1"; string chapterNumber = m.Groups[1].Value; url = string.Concat(Regex.Match(url, @"(.*)-page-[0-9]+(\.html)").Groups.Values.Select(v => v.Value)); From 597abde115b5e2a42c7b1c4a18dbb7052fad0a43 Mon Sep 17 00:00:00 2001 From: Glax Date: Tue, 27 Feb 2024 22:04:14 +0100 Subject: [PATCH 20/20] Fix wrong chapter (and volume) numbers for chapters --- Tranga/Chapter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/Chapter.cs b/Tranga/Chapter.cs index 41d1417..3178458 100644 --- a/Tranga/Chapter.cs +++ b/Tranga/Chapter.cs @@ -20,7 +20,7 @@ public readonly struct Chapter : IComparable private static readonly Regex LegalCharacters = new (@"([A-z]*[0-9]* *\.*-*,*\]*\[*'*\'*\)*\(*~*!*)*"); private static readonly Regex IllegalStrings = new(@"Vol(ume)?.?", RegexOptions.IgnoreCase); - private static readonly Regex Digits = new(@"[0-9]*"); + private static readonly Regex Digits = new(@"[0-9\.]*"); public Chapter(Manga parentManga, string? name, string? volumeNumber, string chapterNumber, string url) { this.parentManga = parentManga;