glax/Tranga
2
0
Fork 0
Code

Moved downloadclient to separate file

Browse Source
This commit is contained in:
glax 2023-07-30 17:04:43 +02:00
parent 579e400a5d
commit b4bef25a22
2 changed files with 110 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
Tranga/Connectors/Connector.cs
View File

@ -16,8 +16,8 @@ namespace Tranga.Connectors;
public abstract class Connector public abstract class Connector
{ {
protected TrangaSettings settings { get; } protected TrangaSettings settings { get; }
protected DownloadClient downloadClient { get; init; } = null!; internal DownloadClient downloadClient { get; init; } = null!;
protected readonly Logger? logger; protected readonly Logger? logger;
@ -272,108 +272,4 @@ public abstract class Connector
logger?.WriteLine(this.GetType().ToString(), $"Saving image to {saveImagePath}"); logger?.WriteLine(this.GetType().ToString(), $"Saving image to {saveImagePath}");
return filename; return filename;
} }
protected class DownloadClient
{
private static readonly HttpClient Client = new()
{
Timeout = TimeSpan.FromSeconds(60)
};
private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit;
private readonly Dictionary<byte, TimeSpan> _rateLimit;
// ReSharper disable once InconsistentNaming
private readonly Logger? logger;
/// <summary>
/// Creates a httpClient
/// </summary>
/// <param name="rateLimitRequestsPerMinute">Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType</param>
/// <param name="logger"></param>
public DownloadClient(Dictionary<byte, int> rateLimitRequestsPerMinute, Logger? logger)
{
this.logger = logger;
_lastExecutedRateLimit = new();
_rateLimit = new();
foreach(KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute)
_rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value));
}
/// <summary>
/// Request Webpage
/// </summary>
/// <param name="url"></param>
/// <param name="requestType">For RateLimits: Same Endpoints use same type</param>
/// <param name="referrer">Used in http request header</param>
/// <returns>RequestResult with StatusCode and Stream of received data</returns>
public RequestResult MakeRequest(string url, byte requestType, string? referrer = null)
{
if (_rateLimit.TryGetValue(requestType, out TimeSpan value))
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
else
{
logger?.WriteLine(this.GetType().ToString(), "RequestType not configured for rate-limit.");
return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null);
}
TimeSpan rateLimitTimeout = _rateLimit[requestType]
.Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]));
if(rateLimitTimeout > TimeSpan.Zero)
Thread.Sleep(rateLimitTimeout);
HttpResponseMessage? response = null;
while (response is null)
{
try
{
HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
if(referrer is not null)
requestMessage.Headers.Referrer = new Uri(referrer);
_lastExecutedRateLimit[requestType] = DateTime.Now;
response = Client.Send(requestMessage);
}
catch (HttpRequestException e)
{
logger?.WriteLine(this.GetType().ToString(), e.Message);
logger?.WriteLine(this.GetType().ToString(), $"Waiting {_rateLimit[requestType] * 2}... Retrying.");
Thread.Sleep(_rateLimit[requestType] * 2);
}
}
if (!response.IsSuccessStatusCode)
{
logger?.WriteLine(this.GetType().ToString(), $"Request-Error {response.StatusCode}: {response.ReasonPhrase}");
return new RequestResult(response.StatusCode, Stream.Null);
}
// Request has been redirected to another page. For example, it redirects directly to the results when there is only 1 result
if(response.RequestMessage is not null && response.RequestMessage.RequestUri is not null)
{
return new RequestResult(response.StatusCode, response.Content.ReadAsStream(), true, response.RequestMessage.RequestUri.AbsoluteUri);
}
return new RequestResult(response.StatusCode, response.Content.ReadAsStream());
}
public struct RequestResult
{
public HttpStatusCode statusCode { get; }
public Stream result { get; }
public bool HasBeenRedirected { get; }
public string? RedirectedToUrl { get; }
public RequestResult(HttpStatusCode statusCode, Stream result)
{
this.statusCode = statusCode;
this.result = result;
}
public RequestResult(HttpStatusCode statusCode, Stream result, bool hasBeenRedirected, string redirectedTo)
: this(statusCode, result)
{
this.HasBeenRedirected = hasBeenRedirected;
RedirectedToUrl = redirectedTo;
}
}
}
} }

108
Tranga/DownloadClient.cs Normal file
View File

@ -0,0 +1,108 @@
using System.Net;
using Logging;
namespace Tranga;
internal class DownloadClient
{
private static readonly HttpClient Client = new()
{
Timeout = TimeSpan.FromSeconds(60)
};
private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit;
private readonly Dictionary<byte, TimeSpan> _rateLimit;
// ReSharper disable once InconsistentNaming
private readonly Logger? logger;
/// <summary>
/// Creates a httpClient
/// </summary>
/// <param name="rateLimitRequestsPerMinute">Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType</param>
/// <param name="logger"></param>
public DownloadClient(Dictionary<byte, int> rateLimitRequestsPerMinute, Logger? logger)
{
this.logger = logger;
_lastExecutedRateLimit = new();
_rateLimit = new();
foreach(KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute)
_rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value));
}
/// <summary>
/// Request Webpage
/// </summary>
/// <param name="url"></param>
/// <param name="requestType">For RateLimits: Same Endpoints use same type</param>
/// <param name="referrer">Used in http request header</param>
/// <returns>RequestResult with StatusCode and Stream of received data</returns>
public RequestResult MakeRequest(string url, byte requestType, string? referrer = null)
{
if (_rateLimit.TryGetValue(requestType, out TimeSpan value))
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
else
{
logger?.WriteLine(this.GetType().ToString(), "RequestType not configured for rate-limit.");
return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null);
}
TimeSpan rateLimitTimeout = _rateLimit[requestType]
.Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]));
if(rateLimitTimeout > TimeSpan.Zero)
Thread.Sleep(rateLimitTimeout);
HttpResponseMessage? response = null;
while (response is null)
{
try
{
HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
if(referrer is not null)
requestMessage.Headers.Referrer = new Uri(referrer);
_lastExecutedRateLimit[requestType] = DateTime.Now;
response = Client.Send(requestMessage);
}
catch (HttpRequestException e)
{
logger?.WriteLine(this.GetType().ToString(), e.Message);
logger?.WriteLine(this.GetType().ToString(), $"Waiting {_rateLimit[requestType] * 2}... Retrying.");
Thread.Sleep(_rateLimit[requestType] * 2);
}
}
if (!response.IsSuccessStatusCode)
{
logger?.WriteLine(this.GetType().ToString(), $"Request-Error {response.StatusCode}: {response.ReasonPhrase}");
return new RequestResult(response.StatusCode, Stream.Null);
}
// Request has been redirected to another page. For example, it redirects directly to the results when there is only 1 result
if(response.RequestMessage is not null && response.RequestMessage.RequestUri is not null)
{
return new RequestResult(response.StatusCode, response.Content.ReadAsStream(), true, response.RequestMessage.RequestUri.AbsoluteUri);
}
return new RequestResult(response.StatusCode, response.Content.ReadAsStream());
}
public struct RequestResult
{
public HttpStatusCode statusCode { get; }
public Stream result { get; }
public bool HasBeenRedirected { get; }
public string? RedirectedToUrl { get; }
public RequestResult(HttpStatusCode statusCode, Stream result)
{
this.statusCode = statusCode;
this.result = result;
}
public RequestResult(HttpStatusCode statusCode, Stream result, bool hasBeenRedirected, string redirectedTo)
: this(statusCode, result)
{
this.HasBeenRedirected = hasBeenRedirected;
RedirectedToUrl = redirectedTo;
}
}
}