glax/Tranga
2
0
Fork 0
Code

First attempt at #18 Rate Limits

Browse Source
This commit is contained in:
glax 2023-05-22 18:15:24 +02:00
parent ec480dffad
commit 01cb74c088
2 changed files with 53 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
Tranga/Connector.cs
View File

@ -12,14 +12,13 @@ namespace Tranga;
public abstract class Connector public abstract class Connector
{ {
internal string downloadLocation { get; } //Location of local files internal string downloadLocation { get; } //Location of local files
protected DownloadClient downloadClient { get; } protected DownloadClient downloadClient { get; init; }
protected Logger? logger; protected Logger? logger;
protected Connector(string downloadLocation, uint downloadDelay, Logger? logger) protected Connector(string downloadLocation, Logger? logger)
{ {
this.downloadLocation = downloadLocation; this.downloadLocation = downloadLocation;
this.downloadClient = new DownloadClient(downloadDelay);
this.logger = logger; this.logger = logger;
} }
@ -116,9 +115,10 @@ public abstract class Connector
/// <param name="imageUrl"></param> /// <param name="imageUrl"></param>
/// <param name="fullPath"></param> /// <param name="fullPath"></param>
/// <param name="downloadClient">DownloadClient of the connector</param> /// <param name="downloadClient">DownloadClient of the connector</param>
protected static void DownloadImage(string imageUrl, string fullPath, DownloadClient downloadClient) /// <param name="requestType">Requesttype for ratelimit</param>
protected static void DownloadImage(string imageUrl, string fullPath, DownloadClient downloadClient, byte requestType)
{ {
DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl); DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType);
byte[] buffer = new byte[requestResult.result.Length]; byte[] buffer = new byte[requestResult.result.Length];
requestResult.result.ReadExactly(buffer, 0, buffer.Length); requestResult.result.ReadExactly(buffer, 0, buffer.Length);
File.WriteAllBytes(fullPath, buffer); File.WriteAllBytes(fullPath, buffer);
@ -131,7 +131,8 @@ public abstract class Connector
/// <param name="saveArchiveFilePath">Full path to save archive to (without file ending .cbz)</param> /// <param name="saveArchiveFilePath">Full path to save archive to (without file ending .cbz)</param>
/// <param name="downloadClient">DownloadClient of the connector</param> /// <param name="downloadClient">DownloadClient of the connector</param>
/// <param name="comicInfoPath">Path of the generate Chapter ComicInfo.xml, if it was generated</param> /// <param name="comicInfoPath">Path of the generate Chapter ComicInfo.xml, if it was generated</param>
protected static void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, DownloadClient downloadClient, Logger? logger, string? comicInfoPath = null) /// <param name="requestType">RequestType for RateLimits</param>
protected static void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, DownloadClient downloadClient, byte requestType, Logger? logger, string? comicInfoPath = null)
{ {
logger?.WriteLine("Connector", "Downloading Images"); logger?.WriteLine("Connector", "Downloading Images");
//Check if Publication Directory already exists //Check if Publication Directory already exists
@ -151,7 +152,7 @@ public abstract class Connector
{ {
string[] split = imageUrl.Split('.'); string[] split = imageUrl.Split('.');
string extension = split[^1]; string extension = split[^1];
DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), downloadClient); DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), downloadClient, requestType);
} }
if(comicInfoPath is not null) if(comicInfoPath is not null)
@ -165,30 +166,41 @@ public abstract class Connector
protected class DownloadClient protected class DownloadClient
{ {
private readonly TimeSpan _requestSpeed;
private DateTime _lastRequest;
private static readonly HttpClient Client = new(); private static readonly HttpClient Client = new();
private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit;
private readonly Dictionary<byte, TimeSpan> _RateLimit;
/// <summary> /// <summary>
/// Creates a httpClient /// Creates a httpClient
/// </summary> /// </summary>
/// <param name="delay">minimum delay between requests (to avoid spam)</param> /// <param name="delay">minimum delay between requests (to avoid spam)</param>
public DownloadClient(uint delay) /// <param name="rateLimitRequestsPerMinute">Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType</param>
public DownloadClient(Dictionary<byte, int> rateLimitRequestsPerMinute)
{ {
_requestSpeed = TimeSpan.FromMilliseconds(delay); _lastExecutedRateLimit = new();
_lastRequest = DateTime.Now.Subtract(_requestSpeed); _RateLimit = new();
foreach(KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute)
_RateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value));
} }
/// <summary> /// <summary>
/// Request Webpage /// Request Webpage
/// </summary> /// </summary>
/// <param name="url"></param> /// <param name="url"></param>
/// <param name="requestType">For RateLimits: Same Endpoints use same type</param>
/// <returns>RequestResult with StatusCode and Stream of received data</returns> /// <returns>RequestResult with StatusCode and Stream of received data</returns>
public RequestResult MakeRequest(string url) public RequestResult MakeRequest(string url, byte requestType)
{ {
while((DateTime.Now - _lastRequest) < _requestSpeed) if (_RateLimit.TryGetValue(requestType, out TimeSpan value))
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
else
return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null);
while(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]) < _RateLimit[requestType])
Thread.Sleep(10); Thread.Sleep(10);
_lastRequest = DateTime.Now; _lastExecutedRateLimit[requestType] = DateTime.Now;
HttpRequestMessage requestMessage = new(HttpMethod.Get, url); HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
HttpResponseMessage response = Client.Send(requestMessage); HttpResponseMessage response = Client.Send(requestMessage);

32
Tranga/Connectors/MangaDex.cs
View File

@ -9,14 +9,26 @@ public class MangaDex : Connector
{ {
public override string name { get; } public override string name { get; }
public MangaDex(string downloadLocation, uint downloadDelay, Logger? logger) : base(downloadLocation, downloadDelay, logger) private enum RequestType : byte
{ {
name = "MangaDex"; Manga,
Feed,
AtHomeServer,
Cover,
Author
} }
public MangaDex(string downloadLocation, Logger? logger) : base(downloadLocation, 750, logger) public MangaDex(string downloadLocation, Logger? logger) : base(downloadLocation, logger)
{ {
name = "MangaDex"; name = "MangaDex";
this.downloadClient = new DownloadClient(new Dictionary<byte, int>()
{
{(byte)RequestType.Manga, 250},
{(byte)RequestType.Feed, 250},
{(byte)RequestType.AtHomeServer, 60},
{(byte)RequestType.Cover, 250},
{(byte)RequestType.Author, 250}
});
} }
public override Publication[] GetPublications(string publicationTitle = "") public override Publication[] GetPublications(string publicationTitle = "")
@ -31,7 +43,7 @@ public class MangaDex : Connector
//Request next Page //Request next Page
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest( downloadClient.MakeRequest(
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}"); $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", (byte)RequestType.Manga);
if (requestResult.statusCode != HttpStatusCode.OK) if (requestResult.statusCode != HttpStatusCode.OK)
break; break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result); JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -141,7 +153,7 @@ public class MangaDex : Connector
//Request next "Page" //Request next "Page"
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest( downloadClient.MakeRequest(
$"https://api.mangadex.org/manga/{publication.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}"); $"https://api.mangadex.org/manga/{publication.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", (byte)RequestType.Feed);
if (requestResult.statusCode != HttpStatusCode.OK) if (requestResult.statusCode != HttpStatusCode.OK)
break; break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result); JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -188,7 +200,7 @@ public class MangaDex : Connector
logger?.WriteLine(this.GetType().ToString(), $"Download Chapter {publication.sortName} {chapter.volumeNumber}-{chapter.chapterNumber}"); logger?.WriteLine(this.GetType().ToString(), $"Download Chapter {publication.sortName} {chapter.volumeNumber}-{chapter.chapterNumber}");
//Request URLs for Chapter-Images //Request URLs for Chapter-Images
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'"); downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer);
if (requestResult.statusCode != HttpStatusCode.OK) if (requestResult.statusCode != HttpStatusCode.OK)
return; return;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result); JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -207,7 +219,7 @@ public class MangaDex : Connector
File.WriteAllText(comicInfoPath, CreateComicInfo(publication, chapter, logger)); File.WriteAllText(comicInfoPath, CreateComicInfo(publication, chapter, logger));
//Download Chapter-Images //Download Chapter-Images
DownloadChapterImages(imageUrls.ToArray(), CreateFullFilepath(publication, chapter), downloadClient, logger, comicInfoPath); DownloadChapterImages(imageUrls.ToArray(), CreateFullFilepath(publication, chapter), downloadClient, (byte)RequestType.AtHomeServer, logger, comicInfoPath);
} }
private string? GetCoverUrl(string publicationId, string? posterId) private string? GetCoverUrl(string publicationId, string? posterId)
@ -220,7 +232,7 @@ public class MangaDex : Connector
//Request information where to download Cover //Request information where to download Cover
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}"); downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", (byte)RequestType.Cover);
if (requestResult.statusCode != HttpStatusCode.OK) if (requestResult.statusCode != HttpStatusCode.OK)
return null; return null;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result); JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -239,7 +251,7 @@ public class MangaDex : Connector
return null; return null;
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}"); downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", (byte)RequestType.Author);
if (requestResult.statusCode != HttpStatusCode.OK) if (requestResult.statusCode != HttpStatusCode.OK)
return null; return null;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result); JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -281,6 +293,6 @@ public class MangaDex : Connector
Directory.CreateDirectory(outFolderPath); Directory.CreateDirectory(outFolderPath);
//Download cover-Image //Download cover-Image
DownloadImage(coverUrl, Path.Join(downloadLocation, publication.folderName, $"cover.{extension}"), this.downloadClient); DownloadImage(coverUrl, Path.Join(downloadLocation, publication.folderName, $"cover.{extension}"), this.downloadClient, (byte)RequestType.AtHomeServer);
} }
} }