diff --git a/Tranga/Connectors/Connector.cs b/Tranga/Connectors/Connector.cs index e5f85d8..6f1d1a7 100644 --- a/Tranga/Connectors/Connector.cs +++ b/Tranga/Connectors/Connector.cs @@ -16,8 +16,8 @@ namespace Tranga.Connectors; public abstract class Connector { protected TrangaSettings settings { get; } - protected DownloadClient downloadClient { get; init; } = null!; - + internal DownloadClient downloadClient { get; init; } = null!; + protected readonly Logger? logger; @@ -272,108 +272,4 @@ public abstract class Connector logger?.WriteLine(this.GetType().ToString(), $"Saving image to {saveImagePath}"); return filename; } - - protected class DownloadClient - { - private static readonly HttpClient Client = new() - { - Timeout = TimeSpan.FromSeconds(60) - }; - - private readonly Dictionary _lastExecutedRateLimit; - private readonly Dictionary _rateLimit; - // ReSharper disable once InconsistentNaming - private readonly Logger? logger; - - /// - /// Creates a httpClient - /// - /// Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType - /// - public DownloadClient(Dictionary rateLimitRequestsPerMinute, Logger? logger) - { - this.logger = logger; - _lastExecutedRateLimit = new(); - _rateLimit = new(); - foreach(KeyValuePair limit in rateLimitRequestsPerMinute) - _rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value)); - } - - /// - /// Request Webpage - /// - /// - /// For RateLimits: Same Endpoints use same type - /// Used in http request header - /// RequestResult with StatusCode and Stream of received data - public RequestResult MakeRequest(string url, byte requestType, string? referrer = null) - { - if (_rateLimit.TryGetValue(requestType, out TimeSpan value)) - _lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value)); - else - { - logger?.WriteLine(this.GetType().ToString(), "RequestType not configured for rate-limit."); - return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null); - } - - TimeSpan rateLimitTimeout = _rateLimit[requestType] - .Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType])); - - if(rateLimitTimeout > TimeSpan.Zero) - Thread.Sleep(rateLimitTimeout); - - HttpResponseMessage? response = null; - while (response is null) - { - try - { - HttpRequestMessage requestMessage = new(HttpMethod.Get, url); - if(referrer is not null) - requestMessage.Headers.Referrer = new Uri(referrer); - _lastExecutedRateLimit[requestType] = DateTime.Now; - response = Client.Send(requestMessage); - } - catch (HttpRequestException e) - { - logger?.WriteLine(this.GetType().ToString(), e.Message); - logger?.WriteLine(this.GetType().ToString(), $"Waiting {_rateLimit[requestType] * 2}... Retrying."); - Thread.Sleep(_rateLimit[requestType] * 2); - } - } - if (!response.IsSuccessStatusCode) - { - logger?.WriteLine(this.GetType().ToString(), $"Request-Error {response.StatusCode}: {response.ReasonPhrase}"); - return new RequestResult(response.StatusCode, Stream.Null); - } - - // Request has been redirected to another page. For example, it redirects directly to the results when there is only 1 result - if(response.RequestMessage is not null && response.RequestMessage.RequestUri is not null) - { - return new RequestResult(response.StatusCode, response.Content.ReadAsStream(), true, response.RequestMessage.RequestUri.AbsoluteUri); - } - - return new RequestResult(response.StatusCode, response.Content.ReadAsStream()); - } - - public struct RequestResult - { - public HttpStatusCode statusCode { get; } - public Stream result { get; } - public bool HasBeenRedirected { get; } - public string? RedirectedToUrl { get; } - - public RequestResult(HttpStatusCode statusCode, Stream result) - { - this.statusCode = statusCode; - this.result = result; - } - - public RequestResult(HttpStatusCode statusCode, Stream result, bool hasBeenRedirected, string redirectedTo) - : this(statusCode, result) - { - this.HasBeenRedirected = hasBeenRedirected; - RedirectedToUrl = redirectedTo; - } - } - } } \ No newline at end of file diff --git a/Tranga/DownloadClient.cs b/Tranga/DownloadClient.cs new file mode 100644 index 0000000..585b15a --- /dev/null +++ b/Tranga/DownloadClient.cs @@ -0,0 +1,108 @@ +using System.Net; +using Logging; + +namespace Tranga; + +internal class DownloadClient + { + private static readonly HttpClient Client = new() + { + Timeout = TimeSpan.FromSeconds(60) + }; + + private readonly Dictionary _lastExecutedRateLimit; + private readonly Dictionary _rateLimit; + // ReSharper disable once InconsistentNaming + private readonly Logger? logger; + + /// + /// Creates a httpClient + /// + /// Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType + /// + public DownloadClient(Dictionary rateLimitRequestsPerMinute, Logger? logger) + { + this.logger = logger; + _lastExecutedRateLimit = new(); + _rateLimit = new(); + foreach(KeyValuePair limit in rateLimitRequestsPerMinute) + _rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value)); + } + + /// + /// Request Webpage + /// + /// + /// For RateLimits: Same Endpoints use same type + /// Used in http request header + /// RequestResult with StatusCode and Stream of received data + public RequestResult MakeRequest(string url, byte requestType, string? referrer = null) + { + if (_rateLimit.TryGetValue(requestType, out TimeSpan value)) + _lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value)); + else + { + logger?.WriteLine(this.GetType().ToString(), "RequestType not configured for rate-limit."); + return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null); + } + + TimeSpan rateLimitTimeout = _rateLimit[requestType] + .Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType])); + + if(rateLimitTimeout > TimeSpan.Zero) + Thread.Sleep(rateLimitTimeout); + + HttpResponseMessage? response = null; + while (response is null) + { + try + { + HttpRequestMessage requestMessage = new(HttpMethod.Get, url); + if(referrer is not null) + requestMessage.Headers.Referrer = new Uri(referrer); + _lastExecutedRateLimit[requestType] = DateTime.Now; + response = Client.Send(requestMessage); + } + catch (HttpRequestException e) + { + logger?.WriteLine(this.GetType().ToString(), e.Message); + logger?.WriteLine(this.GetType().ToString(), $"Waiting {_rateLimit[requestType] * 2}... Retrying."); + Thread.Sleep(_rateLimit[requestType] * 2); + } + } + if (!response.IsSuccessStatusCode) + { + logger?.WriteLine(this.GetType().ToString(), $"Request-Error {response.StatusCode}: {response.ReasonPhrase}"); + return new RequestResult(response.StatusCode, Stream.Null); + } + + // Request has been redirected to another page. For example, it redirects directly to the results when there is only 1 result + if(response.RequestMessage is not null && response.RequestMessage.RequestUri is not null) + { + return new RequestResult(response.StatusCode, response.Content.ReadAsStream(), true, response.RequestMessage.RequestUri.AbsoluteUri); + } + + return new RequestResult(response.StatusCode, response.Content.ReadAsStream()); + } + + public struct RequestResult + { + public HttpStatusCode statusCode { get; } + public Stream result { get; } + public bool HasBeenRedirected { get; } + public string? RedirectedToUrl { get; } + + public RequestResult(HttpStatusCode statusCode, Stream result) + { + this.statusCode = statusCode; + this.result = result; + } + + public RequestResult(HttpStatusCode statusCode, Stream result, bool hasBeenRedirected, string redirectedTo) + : this(statusCode, result) + { + this.HasBeenRedirected = hasBeenRedirected; + RedirectedToUrl = redirectedTo; + } + } + } \ No newline at end of file