From 95da90021306265f2dc34421ed544f432da34841 Mon Sep 17 00:00:00 2001 From: Glax Date: Sat, 20 Jan 2024 16:33:47 +0100 Subject: [PATCH 01/11] Add url to Request-Error Output --- Tranga/MangaConnectors/HttpDownloadClient.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tranga/MangaConnectors/HttpDownloadClient.cs b/Tranga/MangaConnectors/HttpDownloadClient.cs index d8a3d40..54c9a22 100644 --- a/Tranga/MangaConnectors/HttpDownloadClient.cs +++ b/Tranga/MangaConnectors/HttpDownloadClient.cs @@ -44,10 +44,10 @@ internal class HttpDownloadClient : DownloadClient switch (e) { case TaskCanceledException: - Log($"Request timed out.\n\r{e}"); + Log($"Request timed out {url}.\n\r{e}"); return new RequestResult(HttpStatusCode.RequestTimeout, null, Stream.Null); case HttpRequestException: - Log($"Request failed\n\r{e}"); + Log($"Request failed {url}\n\r{e}"); return new RequestResult(HttpStatusCode.BadRequest, null, Stream.Null); } } @@ -55,7 +55,7 @@ internal class HttpDownloadClient : DownloadClient if (!response.IsSuccessStatusCode) { - Log($"Request-Error {response.StatusCode}: {response.ReasonPhrase}"); + Log($"Request-Error {response.StatusCode}: {url}"); return new RequestResult(response.StatusCode, null, Stream.Null); } From cd5ca0e3026b03344917a2b3febb3abae13e9986 Mon Sep 17 00:00:00 2001 From: Glax Date: Sat, 20 Jan 2024 16:44:22 +0100 Subject: [PATCH 02/11] Fix #90 --- Tranga/Jobs/UpdateMetadata.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tranga/Jobs/UpdateMetadata.cs b/Tranga/Jobs/UpdateMetadata.cs index c400c8f..f342ef8 100644 --- a/Tranga/Jobs/UpdateMetadata.cs +++ b/Tranga/Jobs/UpdateMetadata.cs @@ -33,10 +33,11 @@ public class UpdateMetadata : Job this.manga.UpdateMetadata(updatedManga); this.manga.SaveSeriesInfoJson(settings.downloadLocation, true); + /* //TODO remove if this fixes #90 if (parentJobId is not null && jobBoss.GetJobById(this.parentJobId) is DownloadNewChapters dncJob) { dncJob.manga = updatedManga; - } + }*/ this.progressToken.Complete(); } else From 54142e61fe9a3db64c249ce8dae04d8e83d67132 Mon Sep 17 00:00:00 2001 From: Glax Date: Sat, 20 Jan 2024 17:20:56 +0100 Subject: [PATCH 03/11] Fix #103 --- Tranga/MangaConnectors/Mangaworld.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/Mangaworld.cs b/Tranga/MangaConnectors/Mangaworld.cs index a669072..4a32777 100644 --- a/Tranga/MangaConnectors/Mangaworld.cs +++ b/Tranga/MangaConnectors/Mangaworld.cs @@ -69,7 +69,7 @@ public class Mangaworld: MangaConnector if (requestResult.htmlDocument is null) return null; - Regex idRex = new (@"https:\/\/www\.mangaworld\.bz\/manga\/([0-9]+\/[0-9A-z\-]+)"); + Regex idRex = new (@"https:\/\/www\.mangaworld\.[a-z]{0,63}\/manga\/([0-9]+\/[0-9A-z\-]+)"); string id = idRex.Match(url).Groups[1].Value; return ParseSinglePublicationFromHtml(requestResult.htmlDocument, id); } From e49db9a4cb57f5012cb7e6bc3b7d13a6fe36681b Mon Sep 17 00:00:00 2001 From: glax Date: Thu, 25 Jan 2024 16:40:04 +0100 Subject: [PATCH 04/11] Change toplevel domain #103 --- Tranga/MangaConnectors/Mangaworld.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tranga/MangaConnectors/Mangaworld.cs b/Tranga/MangaConnectors/Mangaworld.cs index 4a32777..7f4dc93 100644 --- a/Tranga/MangaConnectors/Mangaworld.cs +++ b/Tranga/MangaConnectors/Mangaworld.cs @@ -19,7 +19,7 @@ public class Mangaworld: MangaConnector { Log($"Searching Publications. Term=\"{publicationTitle}\""); string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(str => str.Length > 0)).ToLower(); - string requestUrl = $"https://www.mangaworld.bz/archive?keyword={sanitizedTitle}"; + string requestUrl = $"https://www.mangaworld.ac/archive?keyword={sanitizedTitle}"; RequestResult requestResult = downloadClient.MakeRequest(requestUrl, 1); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) @@ -56,7 +56,7 @@ public class Mangaworld: MangaConnector public override Manga? GetMangaFromId(string publicationId) { - return GetMangaFromUrl($"https://www.mangaworld.bz/manga/{publicationId}"); + return GetMangaFromUrl($"https://www.mangaworld.ac/manga/{publicationId}"); } public override Manga? GetMangaFromUrl(string url) @@ -130,7 +130,7 @@ public class Mangaworld: MangaConnector public override Chapter[] GetChapters(Manga manga, string language="en") { Log($"Getting chapters {manga}"); - string requestUrl = $"https://www.mangaworld.bz/manga/{manga.publicationId}"; + string requestUrl = $"https://www.mangaworld.ac/manga/{manga.publicationId}"; RequestResult requestResult = downloadClient.MakeRequest(requestUrl, 1); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) From 31a4e693e03bfd5b5f898991e6105195908ac27e Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 31 Jan 2024 18:30:44 +0100 Subject: [PATCH 05/11] Custom Request Limits #109 --- Tranga/MangaConnectors/DownloadClient.cs | 8 +++++++ Tranga/MangaConnectors/HttpDownloadClient.cs | 14 +++++------- Tranga/Server.cs | 23 ++++++++++++++++++++ Tranga/TrangaSettings.cs | 1 + 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/Tranga/MangaConnectors/DownloadClient.cs b/Tranga/MangaConnectors/DownloadClient.cs index 2e19f98..d60ee5f 100644 --- a/Tranga/MangaConnectors/DownloadClient.cs +++ b/Tranga/MangaConnectors/DownloadClient.cs @@ -15,6 +15,14 @@ internal abstract class DownloadClient : GlobalBase foreach (KeyValuePair limit in rateLimitRequestsPerMinute) _rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value)); } + + internal void SetCustomRequestLimit(byte requestType, int limit) + { + if (_rateLimit.ContainsKey(requestType)) + _rateLimit[requestType] = TimeSpan.FromMinutes(1).Divide(limit); + else + _rateLimit.Add(requestType, TimeSpan.FromMinutes(1).Divide(limit)); + } public RequestResult MakeRequest(string url, byte requestType, string? referrer = null, string? clickButton = null) { diff --git a/Tranga/MangaConnectors/HttpDownloadClient.cs b/Tranga/MangaConnectors/HttpDownloadClient.cs index 54c9a22..d5843f3 100644 --- a/Tranga/MangaConnectors/HttpDownloadClient.cs +++ b/Tranga/MangaConnectors/HttpDownloadClient.cs @@ -8,20 +8,16 @@ internal class HttpDownloadClient : DownloadClient { private static readonly HttpClient Client = new() { - Timeout = TimeSpan.FromSeconds(60), - DefaultRequestHeaders = - { - UserAgent = - { - new ProductInfoHeaderValue("Tranga", "0.1") - } - } + Timeout = TimeSpan.FromSeconds(10) }; public HttpDownloadClient(GlobalBase clone, Dictionary rateLimitRequestsPerMinute) : base(clone, rateLimitRequestsPerMinute) { - + if (settings.customUserAgent is null || settings.customUserAgent.Length < 1) + Client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("Tranga", "1.0")); + else + Client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", settings.customUserAgent); } protected override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null) diff --git a/Tranga/Server.cs b/Tranga/Server.cs index 7f3e3ce..bacb1da 100644 --- a/Tranga/Server.cs +++ b/Tranga/Server.cs @@ -384,6 +384,29 @@ public class Server : GlobalBase settings.UpdateWorkingDirectory(workingDirectory); SendResponse(HttpStatusCode.Accepted, response); break;*/ + case "Settings/customUserAgent": + if(!requestVariables.TryGetValue("userAgent", out string? customUserAgent)) + { + SendResponse(HttpStatusCode.BadRequest, response); + break; + } + settings.customUserAgent = customUserAgent; + SendResponse(HttpStatusCode.Accepted, response); + break; + case "Settings/customRequestLimit": + if (!requestVariables.TryGetValue("requestType", out string? requestTypeStr) || + !requestVariables.TryGetValue("requestsPerMinute", out string? requestsPerMinuteStr) || + !requestVariables.TryGetValue("connector", out connectorName) || + !byte.TryParse(requestTypeStr, out byte requestType) || + !int.TryParse(requestsPerMinuteStr, out int requestsPerMinute) || + !_parent.TryGetConnector(connectorName, out connector)) + { + SendResponse(HttpStatusCode.BadRequest, response); + break; + } + connector!.downloadClient.SetCustomRequestLimit(requestType, requestsPerMinute); + SendResponse(HttpStatusCode.Accepted, response); + break; case "NotificationConnectors/Update": if (!requestVariables.TryGetValue("notificationConnector", out string? notificationConnectorStr) || !Enum.TryParse(notificationConnectorStr, out NotificationConnector.NotificationConnectorType notificationConnectorType)) diff --git a/Tranga/TrangaSettings.cs b/Tranga/TrangaSettings.cs index 335f0f6..ad03f9b 100644 --- a/Tranga/TrangaSettings.cs +++ b/Tranga/TrangaSettings.cs @@ -12,6 +12,7 @@ public class TrangaSettings public string workingDirectory { get; private set; } public int apiPortNumber { get; init; } public string styleSheet { get; private set; } + public string? customUserAgent { get; set; } = null; [JsonIgnore] public string settingsFilePath => Path.Join(workingDirectory, "settings.json"); [JsonIgnore] public string libraryConnectorsFilePath => Path.Join(workingDirectory, "libraryConnectors.json"); [JsonIgnore] public string notificationConnectorsFilePath => Path.Join(workingDirectory, "notificationConnectors.json"); From 5d12be2983106caeaf51db75acc0958c961a9cb7 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 31 Jan 2024 18:30:57 +0100 Subject: [PATCH 06/11] Fix crash when Request times out on ChromiumDownloadClient --- Tranga/MangaConnectors/ChromiumDownloadClient.cs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Tranga/MangaConnectors/ChromiumDownloadClient.cs b/Tranga/MangaConnectors/ChromiumDownloadClient.cs index 78a1534..cf8f1fd 100644 --- a/Tranga/MangaConnectors/ChromiumDownloadClient.cs +++ b/Tranga/MangaConnectors/ChromiumDownloadClient.cs @@ -62,8 +62,17 @@ internal class ChromiumDownloadClient : DownloadClient { IPage page = this.browser.NewPageAsync().Result; page.DefaultTimeout = 10000; - IResponse response = page.GoToAsync(url, WaitUntilNavigation.Networkidle0).Result; - Log("Page loaded."); + IResponse response; + try + { + response = page.GoToAsync(url, WaitUntilNavigation.Networkidle0).Result; + Log("Page loaded."); + } + catch (Exception e) + { + Log($"Could not load Page:\n{e.Message}"); + return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null); + } Stream stream = Stream.Null; HtmlDocument? document = null; From 3b818ff1aff9f9b7b2a409dfa6905a3496e715a3 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 31 Jan 2024 18:35:50 +0100 Subject: [PATCH 07/11] typo --- Tranga/MangaConnectors/MangaConnector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/MangaConnector.cs b/Tranga/MangaConnectors/MangaConnector.cs index 52f80f6..8d7df98 100644 --- a/Tranga/MangaConnectors/MangaConnector.cs +++ b/Tranga/MangaConnectors/MangaConnector.cs @@ -178,7 +178,7 @@ public abstract class MangaConnector : GlobalBase string fileInCache = Path.Join(settings.coverImageCache, manga.coverFileNameInCache); if (!File.Exists(fileInCache)) { - Log($"Cloníng cover failed: File missing {fileInCache}."); + Log($"Cloning cover failed: File missing {fileInCache}."); if (retries > 0 && manga.coverUrl is not null) { Log($"Trying {retries} more times"); From bae157cdb4094648ea1d8ba0e21cda423ecd4eb8 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 31 Jan 2024 18:36:14 +0100 Subject: [PATCH 08/11] Cleanup #90 --- Tranga/Jobs/UpdateMetadata.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Tranga/Jobs/UpdateMetadata.cs b/Tranga/Jobs/UpdateMetadata.cs index f342ef8..5f80b76 100644 --- a/Tranga/Jobs/UpdateMetadata.cs +++ b/Tranga/Jobs/UpdateMetadata.cs @@ -32,12 +32,6 @@ public class UpdateMetadata : Job this.manga.UpdateMetadata(updatedManga); this.manga.SaveSeriesInfoJson(settings.downloadLocation, true); - - /* //TODO remove if this fixes #90 - if (parentJobId is not null && jobBoss.GetJobById(this.parentJobId) is DownloadNewChapters dncJob) - { - dncJob.manga = updatedManga; - }*/ this.progressToken.Complete(); } else From 3c4867a276545519380477824c5fdc63792ee08b Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 31 Jan 2024 18:39:28 +0100 Subject: [PATCH 09/11] #105 --- Tranga/Jobs/UpdateMetadata.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Tranga/Jobs/UpdateMetadata.cs b/Tranga/Jobs/UpdateMetadata.cs index 5f80b76..e9107ce 100644 --- a/Tranga/Jobs/UpdateMetadata.cs +++ b/Tranga/Jobs/UpdateMetadata.cs @@ -27,8 +27,11 @@ public class UpdateMetadata : Job Manga? possibleUpdatedManga = mangaConnector.GetMangaFromId(manga.publicationId); if (possibleUpdatedManga is { } updatedManga) { - if(updatedManga.Equals(this.manga))//Check if anything changed + if (updatedManga.Equals(this.manga)) //Check if anything changed + { + this.progressToken.Complete(); return Array.Empty(); + } this.manga.UpdateMetadata(updatedManga); this.manga.SaveSeriesInfoJson(settings.downloadLocation, true); From 2dd82aad13c5396da37180411403fef5c8b58719 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 31 Jan 2024 18:46:37 +0100 Subject: [PATCH 10/11] https://datatracker.ietf.org/doc/html/rfc2616 --- Tranga/MangaConnectors/HttpDownloadClient.cs | 5 +---- Tranga/Server.cs | 7 +++++-- Tranga/TrangaSettings.cs | 10 ++++++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Tranga/MangaConnectors/HttpDownloadClient.cs b/Tranga/MangaConnectors/HttpDownloadClient.cs index d5843f3..e2d9e3d 100644 --- a/Tranga/MangaConnectors/HttpDownloadClient.cs +++ b/Tranga/MangaConnectors/HttpDownloadClient.cs @@ -14,10 +14,7 @@ internal class HttpDownloadClient : DownloadClient public HttpDownloadClient(GlobalBase clone, Dictionary rateLimitRequestsPerMinute) : base(clone, rateLimitRequestsPerMinute) { - if (settings.customUserAgent is null || settings.customUserAgent.Length < 1) - Client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("Tranga", "1.0")); - else - Client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", settings.customUserAgent); + Client.DefaultRequestHeaders.UserAgent.Add(settings.userAgent); } protected override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null) diff --git a/Tranga/Server.cs b/Tranga/Server.cs index bacb1da..6aa518f 100644 --- a/Tranga/Server.cs +++ b/Tranga/Server.cs @@ -200,6 +200,9 @@ public class Server : GlobalBase case "Settings": SendResponse(HttpStatusCode.OK, response, settings); break; + case "Settings/userAgent": + SendResponse(HttpStatusCode.OK, response, settings.userAgent); + break; case "NotificationConnectors": SendResponse(HttpStatusCode.OK, response, notificationConnectors); break; @@ -384,13 +387,13 @@ public class Server : GlobalBase settings.UpdateWorkingDirectory(workingDirectory); SendResponse(HttpStatusCode.Accepted, response); break;*/ - case "Settings/customUserAgent": + case "Settings/userAgent": if(!requestVariables.TryGetValue("userAgent", out string? customUserAgent)) { SendResponse(HttpStatusCode.BadRequest, response); break; } - settings.customUserAgent = customUserAgent; + settings.UpdateUserAgent(customUserAgent); SendResponse(HttpStatusCode.Accepted, response); break; case "Settings/customRequestLimit": diff --git a/Tranga/TrangaSettings.cs b/Tranga/TrangaSettings.cs index ad03f9b..d9bffd3 100644 --- a/Tranga/TrangaSettings.cs +++ b/Tranga/TrangaSettings.cs @@ -1,4 +1,5 @@ -using System.Runtime.InteropServices; +using System.Net.Http.Headers; +using System.Runtime.InteropServices; using Newtonsoft.Json; using Tranga.LibraryConnectors; using Tranga.NotificationConnectors; @@ -12,7 +13,7 @@ public class TrangaSettings public string workingDirectory { get; private set; } public int apiPortNumber { get; init; } public string styleSheet { get; private set; } - public string? customUserAgent { get; set; } = null; + public ProductInfoHeaderValue userAgent { get; set; } = new ("Tranga", "1.0"); [JsonIgnore] public string settingsFilePath => Path.Join(workingDirectory, "settings.json"); [JsonIgnore] public string libraryConnectorsFilePath => Path.Join(workingDirectory, "libraryConnectors.json"); [JsonIgnore] public string notificationConnectorsFilePath => Path.Join(workingDirectory, "notificationConnectors.json"); @@ -120,6 +121,11 @@ public class TrangaSettings ExportSettings(); } + public void UpdateUserAgent(string customUserAgent) + { + this.userAgent = new ProductInfoHeaderValue(ProductHeaderValue.Parse(customUserAgent)); + } + public void ExportSettings() { if (File.Exists(settingsFilePath)) From 3ca6245fc2bdadddb51efcdc1d13c02c6535ff45 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 31 Jan 2024 19:00:38 +0100 Subject: [PATCH 11/11] safe Useragent as string and export settings after changing --- Tranga/MangaConnectors/HttpDownloadClient.cs | 2 +- Tranga/TrangaSettings.cs | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Tranga/MangaConnectors/HttpDownloadClient.cs b/Tranga/MangaConnectors/HttpDownloadClient.cs index e2d9e3d..342b8a1 100644 --- a/Tranga/MangaConnectors/HttpDownloadClient.cs +++ b/Tranga/MangaConnectors/HttpDownloadClient.cs @@ -14,7 +14,7 @@ internal class HttpDownloadClient : DownloadClient public HttpDownloadClient(GlobalBase clone, Dictionary rateLimitRequestsPerMinute) : base(clone, rateLimitRequestsPerMinute) { - Client.DefaultRequestHeaders.UserAgent.Add(settings.userAgent); + Client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", settings.userAgent); } protected override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null) diff --git a/Tranga/TrangaSettings.cs b/Tranga/TrangaSettings.cs index d9bffd3..a75c8dd 100644 --- a/Tranga/TrangaSettings.cs +++ b/Tranga/TrangaSettings.cs @@ -13,7 +13,9 @@ public class TrangaSettings public string workingDirectory { get; private set; } public int apiPortNumber { get; init; } public string styleSheet { get; private set; } - public ProductInfoHeaderValue userAgent { get; set; } = new ("Tranga", "1.0"); + + public string userAgent { get; set; } = + $"Tranga ({Enum.GetName(Environment.OSVersion.Platform)}; {(Environment.Is64BitOperatingSystem ? "x64" : "")}) / 1.0"; [JsonIgnore] public string settingsFilePath => Path.Join(workingDirectory, "settings.json"); [JsonIgnore] public string libraryConnectorsFilePath => Path.Join(workingDirectory, "libraryConnectors.json"); [JsonIgnore] public string notificationConnectorsFilePath => Path.Join(workingDirectory, "notificationConnectors.json"); @@ -123,7 +125,8 @@ public class TrangaSettings public void UpdateUserAgent(string customUserAgent) { - this.userAgent = new ProductInfoHeaderValue(ProductHeaderValue.Parse(customUserAgent)); + this.userAgent = customUserAgent; + ExportSettings(); } public void ExportSettings()