diff --git a/Tranga/Connector.cs b/Tranga/Connector.cs
index d3e49b4..83c21ff 100644
--- a/Tranga/Connector.cs
+++ b/Tranga/Connector.cs
@@ -12,14 +12,13 @@ namespace Tranga;
public abstract class Connector
{
internal string downloadLocation { get; } //Location of local files
- protected DownloadClient downloadClient { get; }
+ protected DownloadClient downloadClient { get; init; }
protected Logger? logger;
- protected Connector(string downloadLocation, uint downloadDelay, Logger? logger)
+ protected Connector(string downloadLocation, Logger? logger)
{
this.downloadLocation = downloadLocation;
- this.downloadClient = new DownloadClient(downloadDelay);
this.logger = logger;
}
@@ -109,21 +108,22 @@ public abstract class Connector
{
return Path.Join(downloadLocation, publication.folderName, $"{chapter.fileName}.cbz");
}
-
+
///
/// Downloads Image from URL and saves it to the given path(incl. fileName)
///
///
///
/// DownloadClient of the connector
- protected static void DownloadImage(string imageUrl, string fullPath, DownloadClient downloadClient)
+ /// Requesttype for ratelimit
+ protected static void DownloadImage(string imageUrl, string fullPath, DownloadClient downloadClient, byte requestType)
{
- DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl);
+ DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType);
byte[] buffer = new byte[requestResult.result.Length];
requestResult.result.ReadExactly(buffer, 0, buffer.Length);
File.WriteAllBytes(fullPath, buffer);
}
-
+
///
/// Downloads all Images from URLs, Compresses to zip(cbz) and saves.
///
@@ -131,7 +131,8 @@ public abstract class Connector
/// Full path to save archive to (without file ending .cbz)
/// DownloadClient of the connector
/// Path of the generate Chapter ComicInfo.xml, if it was generated
- protected static void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, DownloadClient downloadClient, Logger? logger, string? comicInfoPath = null)
+ /// RequestType for RateLimits
+ protected static void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, DownloadClient downloadClient, byte requestType, Logger? logger, string? comicInfoPath = null)
{
logger?.WriteLine("Connector", "Downloading Images");
//Check if Publication Directory already exists
@@ -151,7 +152,7 @@ public abstract class Connector
{
string[] split = imageUrl.Split('.');
string extension = split[^1];
- DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), downloadClient);
+ DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), downloadClient, requestType);
}
if(comicInfoPath is not null)
@@ -165,30 +166,41 @@ public abstract class Connector
protected class DownloadClient
{
- private readonly TimeSpan _requestSpeed;
- private DateTime _lastRequest;
private static readonly HttpClient Client = new();
+ private readonly Dictionary _lastExecutedRateLimit;
+ private readonly Dictionary _RateLimit;
+
///
/// Creates a httpClient
///
/// minimum delay between requests (to avoid spam)
- public DownloadClient(uint delay)
+ /// Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType
+ public DownloadClient(Dictionary rateLimitRequestsPerMinute)
{
- _requestSpeed = TimeSpan.FromMilliseconds(delay);
- _lastRequest = DateTime.Now.Subtract(_requestSpeed);
+ _lastExecutedRateLimit = new();
+ _RateLimit = new();
+ foreach(KeyValuePair limit in rateLimitRequestsPerMinute)
+ _RateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value));
}
-
+
///
/// Request Webpage
///
///
+ /// For RateLimits: Same Endpoints use same type
/// RequestResult with StatusCode and Stream of received data
- public RequestResult MakeRequest(string url)
+ public RequestResult MakeRequest(string url, byte requestType)
{
- while((DateTime.Now - _lastRequest) < _requestSpeed)
+ if (_RateLimit.TryGetValue(requestType, out TimeSpan value))
+ _lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
+ else
+ return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null);
+
+
+ while(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]) < _RateLimit[requestType])
Thread.Sleep(10);
- _lastRequest = DateTime.Now;
+ _lastExecutedRateLimit[requestType] = DateTime.Now;
HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
HttpResponseMessage response = Client.Send(requestMessage);
diff --git a/Tranga/Connectors/MangaDex.cs b/Tranga/Connectors/MangaDex.cs
index c221a67..0c916aa 100644
--- a/Tranga/Connectors/MangaDex.cs
+++ b/Tranga/Connectors/MangaDex.cs
@@ -9,14 +9,26 @@ public class MangaDex : Connector
{
public override string name { get; }
- public MangaDex(string downloadLocation, uint downloadDelay, Logger? logger) : base(downloadLocation, downloadDelay, logger)
+ private enum RequestType : byte
{
- name = "MangaDex";
+ Manga,
+ Feed,
+ AtHomeServer,
+ Cover,
+ Author
}
-
- public MangaDex(string downloadLocation, Logger? logger) : base(downloadLocation, 750, logger)
+
+ public MangaDex(string downloadLocation, Logger? logger) : base(downloadLocation, logger)
{
name = "MangaDex";
+ this.downloadClient = new DownloadClient(new Dictionary()
+ {
+ {(byte)RequestType.Manga, 250},
+ {(byte)RequestType.Feed, 250},
+ {(byte)RequestType.AtHomeServer, 60},
+ {(byte)RequestType.Cover, 250},
+ {(byte)RequestType.Author, 250}
+ });
}
public override Publication[] GetPublications(string publicationTitle = "")
@@ -31,7 +43,7 @@ public class MangaDex : Connector
//Request next Page
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(
- $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}");
+ $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", (byte)RequestType.Manga);
if (requestResult.statusCode != HttpStatusCode.OK)
break;
JsonObject? result = JsonSerializer.Deserialize(requestResult.result);
@@ -141,7 +153,7 @@ public class MangaDex : Connector
//Request next "Page"
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(
- $"https://api.mangadex.org/manga/{publication.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}");
+ $"https://api.mangadex.org/manga/{publication.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", (byte)RequestType.Feed);
if (requestResult.statusCode != HttpStatusCode.OK)
break;
JsonObject? result = JsonSerializer.Deserialize(requestResult.result);
@@ -188,7 +200,7 @@ public class MangaDex : Connector
logger?.WriteLine(this.GetType().ToString(), $"Download Chapter {publication.sortName} {chapter.volumeNumber}-{chapter.chapterNumber}");
//Request URLs for Chapter-Images
DownloadClient.RequestResult requestResult =
- downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'");
+ downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer);
if (requestResult.statusCode != HttpStatusCode.OK)
return;
JsonObject? result = JsonSerializer.Deserialize(requestResult.result);
@@ -207,7 +219,7 @@ public class MangaDex : Connector
File.WriteAllText(comicInfoPath, CreateComicInfo(publication, chapter, logger));
//Download Chapter-Images
- DownloadChapterImages(imageUrls.ToArray(), CreateFullFilepath(publication, chapter), downloadClient, logger, comicInfoPath);
+ DownloadChapterImages(imageUrls.ToArray(), CreateFullFilepath(publication, chapter), downloadClient, (byte)RequestType.AtHomeServer, logger, comicInfoPath);
}
private string? GetCoverUrl(string publicationId, string? posterId)
@@ -220,7 +232,7 @@ public class MangaDex : Connector
//Request information where to download Cover
DownloadClient.RequestResult requestResult =
- downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}");
+ downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", (byte)RequestType.Cover);
if (requestResult.statusCode != HttpStatusCode.OK)
return null;
JsonObject? result = JsonSerializer.Deserialize(requestResult.result);
@@ -239,7 +251,7 @@ public class MangaDex : Connector
return null;
DownloadClient.RequestResult requestResult =
- downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}");
+ downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", (byte)RequestType.Author);
if (requestResult.statusCode != HttpStatusCode.OK)
return null;
JsonObject? result = JsonSerializer.Deserialize(requestResult.result);
@@ -281,6 +293,6 @@ public class MangaDex : Connector
Directory.CreateDirectory(outFolderPath);
//Download cover-Image
- DownloadImage(coverUrl, Path.Join(downloadLocation, publication.folderName, $"cover.{extension}"), this.downloadClient);
+ DownloadImage(coverUrl, Path.Join(downloadLocation, publication.folderName, $"cover.{extension}"), this.downloadClient, (byte)RequestType.AtHomeServer);
}
}
\ No newline at end of file