glax/Tranga-Website
1
0
Fork 0
Code Issues Pull Requests Releases Activity

First attempt at #18 Rate Limits

Browse Source
This commit is contained in:
glax 2023-05-22 18:15:24 +02:00
parent ec480dffad
commit 01cb74c088
2 changed files with 53 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
Tranga/Connector.cs
View File

@ -12,14 +12,13 @@ namespace Tranga;
public abstract class Connector
{
internal string downloadLocation { get; } //Location of local files
protected DownloadClient downloadClient { get; }
protected DownloadClient downloadClient { get; init; }
protected Logger? logger;
protected Connector(string downloadLocation, uint downloadDelay, Logger? logger)
protected Connector(string downloadLocation, Logger? logger)
{
this.downloadLocation = downloadLocation;
this.downloadClient = new DownloadClient(downloadDelay);
this.logger = logger;
}
@ -109,21 +108,22 @@ public abstract class Connector
{
return Path.Join(downloadLocation, publication.folderName, $"{chapter.fileName}.cbz");
}
/// <summary>
/// Downloads Image from URL and saves it to the given path(incl. fileName)
/// </summary>
/// <param name="imageUrl"></param>
/// <param name="fullPath"></param>
/// <param name="downloadClient">DownloadClient of the connector</param>
protected static void DownloadImage(string imageUrl, string fullPath, DownloadClient downloadClient)
/// <param name="requestType">Requesttype for ratelimit</param>
protected static void DownloadImage(string imageUrl, string fullPath, DownloadClient downloadClient, byte requestType)
{
DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl);
DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType);
byte[] buffer = new byte[requestResult.result.Length];
requestResult.result.ReadExactly(buffer, 0, buffer.Length);
File.WriteAllBytes(fullPath, buffer);
}
/// <summary>
/// Downloads all Images from URLs, Compresses to zip(cbz) and saves.
/// </summary>
@ -131,7 +131,8 @@ public abstract class Connector
/// <param name="saveArchiveFilePath">Full path to save archive to (without file ending .cbz)</param>
/// <param name="downloadClient">DownloadClient of the connector</param>
/// <param name="comicInfoPath">Path of the generate Chapter ComicInfo.xml, if it was generated</param>
protected static void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, DownloadClient downloadClient, Logger? logger, string? comicInfoPath = null)
/// <param name="requestType">RequestType for RateLimits</param>
protected static void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, DownloadClient downloadClient, byte requestType, Logger? logger, string? comicInfoPath = null)
{
logger?.WriteLine("Connector", "Downloading Images");
//Check if Publication Directory already exists
@ -151,7 +152,7 @@ public abstract class Connector
{
string[] split = imageUrl.Split('.');
string extension = split[^1];
DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), downloadClient);
DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), downloadClient, requestType);
}
if(comicInfoPath is not null)
@ -165,30 +166,41 @@ public abstract class Connector
protected class DownloadClient
{
private readonly TimeSpan _requestSpeed;
private DateTime _lastRequest;
private static readonly HttpClient Client = new();
private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit;
private readonly Dictionary<byte, TimeSpan> _RateLimit;
/// <summary>
/// Creates a httpClient
/// </summary>
/// <param name="delay">minimum delay between requests (to avoid spam)</param>
public DownloadClient(uint delay)
/// <param name="rateLimitRequestsPerMinute">Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType</param>
public DownloadClient(Dictionary<byte, int> rateLimitRequestsPerMinute)
{
_requestSpeed = TimeSpan.FromMilliseconds(delay);
_lastRequest = DateTime.Now.Subtract(_requestSpeed);
_lastExecutedRateLimit = new();
_RateLimit = new();
foreach(KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute)
_RateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value));
}
/// <summary>
/// Request Webpage
/// </summary>
/// <param name="url"></param>
/// <param name="requestType">For RateLimits: Same Endpoints use same type</param>
/// <returns>RequestResult with StatusCode and Stream of received data</returns>
public RequestResult MakeRequest(string url)
public RequestResult MakeRequest(string url, byte requestType)
{
while((DateTime.Now - _lastRequest) < _requestSpeed)
if (_RateLimit.TryGetValue(requestType, out TimeSpan value))
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
else
return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null);
while(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]) < _RateLimit[requestType])
Thread.Sleep(10);
_lastRequest = DateTime.Now;
_lastExecutedRateLimit[requestType] = DateTime.Now;
HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
HttpResponseMessage response = Client.Send(requestMessage);

34
Tranga/Connectors/MangaDex.cs
View File

@ -9,14 +9,26 @@ public class MangaDex : Connector
{
public override string name { get; }
public MangaDex(string downloadLocation, uint downloadDelay, Logger? logger) : base(downloadLocation, downloadDelay, logger)
private enum RequestType : byte
{
name = "MangaDex";
Manga,
Feed,
AtHomeServer,
Cover,
Author
}
public MangaDex(string downloadLocation, Logger? logger) : base(downloadLocation, 750, logger)
public MangaDex(string downloadLocation, Logger? logger) : base(downloadLocation, logger)
{
name = "MangaDex";
this.downloadClient = new DownloadClient(new Dictionary<byte, int>()
{
{(byte)RequestType.Manga, 250},
{(byte)RequestType.Feed, 250},
{(byte)RequestType.AtHomeServer, 60},
{(byte)RequestType.Cover, 250},
{(byte)RequestType.Author, 250}
});
}
public override Publication[] GetPublications(string publicationTitle = "")
@ -31,7 +43,7 @@ public class MangaDex : Connector
//Request next Page
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}");
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", (byte)RequestType.Manga);
if (requestResult.statusCode != HttpStatusCode.OK)
break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -141,7 +153,7 @@ public class MangaDex : Connector
//Request next "Page"
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(
$"https://api.mangadex.org/manga/{publication.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}");
$"https://api.mangadex.org/manga/{publication.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", (byte)RequestType.Feed);
if (requestResult.statusCode != HttpStatusCode.OK)
break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -188,7 +200,7 @@ public class MangaDex : Connector
logger?.WriteLine(this.GetType().ToString(), $"Download Chapter {publication.sortName} {chapter.volumeNumber}-{chapter.chapterNumber}");
//Request URLs for Chapter-Images
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'");
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer);
if (requestResult.statusCode != HttpStatusCode.OK)
return;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -207,7 +219,7 @@ public class MangaDex : Connector
File.WriteAllText(comicInfoPath, CreateComicInfo(publication, chapter, logger));
//Download Chapter-Images
DownloadChapterImages(imageUrls.ToArray(), CreateFullFilepath(publication, chapter), downloadClient, logger, comicInfoPath);
DownloadChapterImages(imageUrls.ToArray(), CreateFullFilepath(publication, chapter), downloadClient, (byte)RequestType.AtHomeServer, logger, comicInfoPath);
}
private string? GetCoverUrl(string publicationId, string? posterId)
@ -220,7 +232,7 @@ public class MangaDex : Connector
//Request information where to download Cover
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}");
downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", (byte)RequestType.Cover);
if (requestResult.statusCode != HttpStatusCode.OK)
return null;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -239,7 +251,7 @@ public class MangaDex : Connector
return null;
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}");
downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", (byte)RequestType.Author);
if (requestResult.statusCode != HttpStatusCode.OK)
return null;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -281,6 +293,6 @@ public class MangaDex : Connector
Directory.CreateDirectory(outFolderPath);
//Download cover-Image
DownloadImage(coverUrl, Path.Join(downloadLocation, publication.folderName, $"cover.{extension}"), this.downloadClient);
DownloadImage(coverUrl, Path.Join(downloadLocation, publication.folderName, $"cover.{extension}"), this.downloadClient, (byte)RequestType.AtHomeServer);
}
}