glax/Tranga-Website
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Better Rate-Limits

Browse Source 
Added Logger to DownloadClient
This commit is contained in:
glax 2023-05-22 21:38:23 +02:00
parent 7b6253de0f
commit 30b6c4680b
2 changed files with 32 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
Tranga/Connector.cs
View File

@ -169,19 +169,21 @@ public abstract class Connector
private static readonly HttpClient Client = new(); private static readonly HttpClient Client = new();
private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit; private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit;
private readonly Dictionary<byte, TimeSpan> _RateLimit; private readonly Dictionary<byte, TimeSpan> _rateLimit;
private Logger? logger;
/// <summary> /// <summary>
/// Creates a httpClient /// Creates a httpClient
/// </summary> /// </summary>
/// <param name="delay">minimum delay between requests (to avoid spam)</param> /// <param name="delay">minimum delay between requests (to avoid spam)</param>
/// <param name="rateLimitRequestsPerMinute">Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType</param> /// <param name="rateLimitRequestsPerMinute">Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType</param>
public DownloadClient(Dictionary<byte, int> rateLimitRequestsPerMinute) public DownloadClient(Dictionary<byte, int> rateLimitRequestsPerMinute, Logger? logger)
{ {
this.logger = logger;
_lastExecutedRateLimit = new(); _lastExecutedRateLimit = new();
_RateLimit = new(); _rateLimit = new();
foreach(KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute) foreach(KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute)
_RateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value)); _rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value));
} }
/// <summary> /// <summary>
@ -192,19 +194,36 @@ public abstract class Connector
/// <returns>RequestResult with StatusCode and Stream of received data</returns> /// <returns>RequestResult with StatusCode and Stream of received data</returns>
public RequestResult MakeRequest(string url, byte requestType) public RequestResult MakeRequest(string url, byte requestType)
{ {
if (_RateLimit.TryGetValue(requestType, out TimeSpan value)) if (_rateLimit.TryGetValue(requestType, out TimeSpan value))
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value)); _lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
else else
{
logger?.WriteLine(this.GetType().ToString(), "RequestType not configured for rate-limit.");
return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null); return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null);
}
while(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]) < _RateLimit[requestType])
Thread.Sleep(10);
_lastExecutedRateLimit[requestType] = DateTime.Now;
HttpRequestMessage requestMessage = new(HttpMethod.Get, url); TimeSpan rateLimitTimeout = _rateLimit[requestType]
HttpResponseMessage response = Client.Send(requestMessage); .Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]));
Thread.Sleep(rateLimitTimeout);
HttpResponseMessage? response = null;
while (response is null)
{
try
{
HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
_lastExecutedRateLimit[requestType] = DateTime.Now;
response = Client.Send(requestMessage);
}
catch (HttpRequestException e)
{
Thread.Sleep(_rateLimit[requestType] * 2);
}
}
Stream resultString = response.IsSuccessStatusCode ? response.Content.ReadAsStream() : Stream.Null; Stream resultString = response.IsSuccessStatusCode ? response.Content.ReadAsStream() : Stream.Null;
if (!response.IsSuccessStatusCode)
logger?.WriteLine(this.GetType().ToString(), $"Request-Error {response.StatusCode}: {response.ReasonPhrase}");
return new RequestResult(response.StatusCode, resultString); return new RequestResult(response.StatusCode, resultString);
} }

2
Tranga/Connectors/MangaDex.cs
View File

@ -28,7 +28,7 @@ public class MangaDex : Connector
{(byte)RequestType.AtHomeServer, 40}, {(byte)RequestType.AtHomeServer, 40},
{(byte)RequestType.Cover, 250}, {(byte)RequestType.Cover, 250},
{(byte)RequestType.Author, 250} {(byte)RequestType.Author, 250}
}); }, logger);
} }
public override Publication[] GetPublications(string publicationTitle = "") public override Publication[] GetPublications(string publicationTitle = "")