From 01cb74c08805a47db59faf7bc46e6bb45d70066a Mon Sep 17 00:00:00 2001 From: glax Date: Mon, 22 May 2023 18:15:24 +0200 Subject: [PATCH] First attempt at #18 Rate Limits --- Tranga/Connector.cs | 48 ++++++++++++++++++++++------------- Tranga/Connectors/MangaDex.cs | 34 +++++++++++++++++-------- 2 files changed, 53 insertions(+), 29 deletions(-) diff --git a/Tranga/Connector.cs b/Tranga/Connector.cs index d3e49b4..83c21ff 100644 --- a/Tranga/Connector.cs +++ b/Tranga/Connector.cs @@ -12,14 +12,13 @@ namespace Tranga; public abstract class Connector { internal string downloadLocation { get; } //Location of local files - protected DownloadClient downloadClient { get; } + protected DownloadClient downloadClient { get; init; } protected Logger? logger; - protected Connector(string downloadLocation, uint downloadDelay, Logger? logger) + protected Connector(string downloadLocation, Logger? logger) { this.downloadLocation = downloadLocation; - this.downloadClient = new DownloadClient(downloadDelay); this.logger = logger; } @@ -109,21 +108,22 @@ public abstract class Connector { return Path.Join(downloadLocation, publication.folderName, $"{chapter.fileName}.cbz"); } - + /// /// Downloads Image from URL and saves it to the given path(incl. fileName) /// /// /// /// DownloadClient of the connector - protected static void DownloadImage(string imageUrl, string fullPath, DownloadClient downloadClient) + /// Requesttype for ratelimit + protected static void DownloadImage(string imageUrl, string fullPath, DownloadClient downloadClient, byte requestType) { - DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl); + DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType); byte[] buffer = new byte[requestResult.result.Length]; requestResult.result.ReadExactly(buffer, 0, buffer.Length); File.WriteAllBytes(fullPath, buffer); } - + /// /// Downloads all Images from URLs, Compresses to zip(cbz) and saves. /// @@ -131,7 +131,8 @@ public abstract class Connector /// Full path to save archive to (without file ending .cbz) /// DownloadClient of the connector /// Path of the generate Chapter ComicInfo.xml, if it was generated - protected static void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, DownloadClient downloadClient, Logger? logger, string? comicInfoPath = null) + /// RequestType for RateLimits + protected static void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, DownloadClient downloadClient, byte requestType, Logger? logger, string? comicInfoPath = null) { logger?.WriteLine("Connector", "Downloading Images"); //Check if Publication Directory already exists @@ -151,7 +152,7 @@ public abstract class Connector { string[] split = imageUrl.Split('.'); string extension = split[^1]; - DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), downloadClient); + DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), downloadClient, requestType); } if(comicInfoPath is not null) @@ -165,30 +166,41 @@ public abstract class Connector protected class DownloadClient { - private readonly TimeSpan _requestSpeed; - private DateTime _lastRequest; private static readonly HttpClient Client = new(); + private readonly Dictionary _lastExecutedRateLimit; + private readonly Dictionary _RateLimit; + /// /// Creates a httpClient /// /// minimum delay between requests (to avoid spam) - public DownloadClient(uint delay) + /// Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType + public DownloadClient(Dictionary rateLimitRequestsPerMinute) { - _requestSpeed = TimeSpan.FromMilliseconds(delay); - _lastRequest = DateTime.Now.Subtract(_requestSpeed); + _lastExecutedRateLimit = new(); + _RateLimit = new(); + foreach(KeyValuePair limit in rateLimitRequestsPerMinute) + _RateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value)); } - + /// /// Request Webpage /// /// + /// For RateLimits: Same Endpoints use same type /// RequestResult with StatusCode and Stream of received data - public RequestResult MakeRequest(string url) + public RequestResult MakeRequest(string url, byte requestType) { - while((DateTime.Now - _lastRequest) < _requestSpeed) + if (_RateLimit.TryGetValue(requestType, out TimeSpan value)) + _lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value)); + else + return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null); + + + while(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]) < _RateLimit[requestType]) Thread.Sleep(10); - _lastRequest = DateTime.Now; + _lastExecutedRateLimit[requestType] = DateTime.Now; HttpRequestMessage requestMessage = new(HttpMethod.Get, url); HttpResponseMessage response = Client.Send(requestMessage); diff --git a/Tranga/Connectors/MangaDex.cs b/Tranga/Connectors/MangaDex.cs index c221a67..0c916aa 100644 --- a/Tranga/Connectors/MangaDex.cs +++ b/Tranga/Connectors/MangaDex.cs @@ -9,14 +9,26 @@ public class MangaDex : Connector { public override string name { get; } - public MangaDex(string downloadLocation, uint downloadDelay, Logger? logger) : base(downloadLocation, downloadDelay, logger) + private enum RequestType : byte { - name = "MangaDex"; + Manga, + Feed, + AtHomeServer, + Cover, + Author } - - public MangaDex(string downloadLocation, Logger? logger) : base(downloadLocation, 750, logger) + + public MangaDex(string downloadLocation, Logger? logger) : base(downloadLocation, logger) { name = "MangaDex"; + this.downloadClient = new DownloadClient(new Dictionary() + { + {(byte)RequestType.Manga, 250}, + {(byte)RequestType.Feed, 250}, + {(byte)RequestType.AtHomeServer, 60}, + {(byte)RequestType.Cover, 250}, + {(byte)RequestType.Author, 250} + }); } public override Publication[] GetPublications(string publicationTitle = "") @@ -31,7 +43,7 @@ public class MangaDex : Connector //Request next Page DownloadClient.RequestResult requestResult = downloadClient.MakeRequest( - $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}"); + $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", (byte)RequestType.Manga); if (requestResult.statusCode != HttpStatusCode.OK) break; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); @@ -141,7 +153,7 @@ public class MangaDex : Connector //Request next "Page" DownloadClient.RequestResult requestResult = downloadClient.MakeRequest( - $"https://api.mangadex.org/manga/{publication.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}"); + $"https://api.mangadex.org/manga/{publication.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", (byte)RequestType.Feed); if (requestResult.statusCode != HttpStatusCode.OK) break; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); @@ -188,7 +200,7 @@ public class MangaDex : Connector logger?.WriteLine(this.GetType().ToString(), $"Download Chapter {publication.sortName} {chapter.volumeNumber}-{chapter.chapterNumber}"); //Request URLs for Chapter-Images DownloadClient.RequestResult requestResult = - downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'"); + downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer); if (requestResult.statusCode != HttpStatusCode.OK) return; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); @@ -207,7 +219,7 @@ public class MangaDex : Connector File.WriteAllText(comicInfoPath, CreateComicInfo(publication, chapter, logger)); //Download Chapter-Images - DownloadChapterImages(imageUrls.ToArray(), CreateFullFilepath(publication, chapter), downloadClient, logger, comicInfoPath); + DownloadChapterImages(imageUrls.ToArray(), CreateFullFilepath(publication, chapter), downloadClient, (byte)RequestType.AtHomeServer, logger, comicInfoPath); } private string? GetCoverUrl(string publicationId, string? posterId) @@ -220,7 +232,7 @@ public class MangaDex : Connector //Request information where to download Cover DownloadClient.RequestResult requestResult = - downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}"); + downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", (byte)RequestType.Cover); if (requestResult.statusCode != HttpStatusCode.OK) return null; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); @@ -239,7 +251,7 @@ public class MangaDex : Connector return null; DownloadClient.RequestResult requestResult = - downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}"); + downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", (byte)RequestType.Author); if (requestResult.statusCode != HttpStatusCode.OK) return null; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); @@ -281,6 +293,6 @@ public class MangaDex : Connector Directory.CreateDirectory(outFolderPath); //Download cover-Image - DownloadImage(coverUrl, Path.Join(downloadLocation, publication.folderName, $"cover.{extension}"), this.downloadClient); + DownloadImage(coverUrl, Path.Join(downloadLocation, publication.folderName, $"cover.{extension}"), this.downloadClient, (byte)RequestType.AtHomeServer); } } \ No newline at end of file