glax/Tranga
1
0
Fork 0
mirror of https://github.com/C9Glax/tranga.git synced 2025-07-02 00:44:16 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

DownloadClient and MangaConnector improvements

Browse Source 
DownloadClient is now abstract for HttpDownloadClient and ChromiumDownloadClient
The chromium client will exit the headless browser (on clean exit of the program).
The field "name" of MangaConnector is no longer abstract, instead set through constructor.
This commit is contained in:
glax
2023-09-08 23:27:09 +02:00
parent 017701867d
commit 569622099d
9 changed files with 279 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
Tranga/MangaConnectors/DownloadClient.cs
View File

@ -1,107 +1,66 @@
using System.Net;
using System.Net.Http.Headers;
using HtmlAgilityPack;
namespace Tranga.MangaConnectors;
internal class DownloadClient : GlobalBase
internal abstract class DownloadClient : GlobalBase
{
private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit;
private readonly Dictionary<byte, TimeSpan> _rateLimit;
protected DownloadClient(GlobalBase clone, Dictionary<byte, int> rateLimitRequestsPerMinute) : base(clone)
{
private static readonly HttpClient Client = new()
this._lastExecutedRateLimit = new();
_rateLimit = new();
foreach (KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute)
_rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value));
}
public RequestResult MakeRequest(string url, byte requestType, string? referrer = null)
{
if (_rateLimit.TryGetValue(requestType, out TimeSpan value))
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
else
{
Timeout = TimeSpan.FromSeconds(60),
DefaultRequestHeaders =
{
UserAgent =
{
new ProductInfoHeaderValue("Tranga", "0.1")
}
}
};
private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit;
private readonly Dictionary<byte, TimeSpan> _rateLimit;
public DownloadClient(GlobalBase clone, Dictionary<byte, int> rateLimitRequestsPerMinute) : base(clone)
{
_lastExecutedRateLimit = new();
_rateLimit = new();
foreach(KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute)
_rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value));
Log("RequestType not configured for rate-limit.");
return new RequestResult(HttpStatusCode.NotAcceptable, null, Stream.Null);
}
/// <summary>
/// Request Webpage
/// </summary>
/// <param name="url"></param>
/// <param name="requestType">For RateLimits: Same Endpoints use same type</param>
/// <param name="referrer">Used in http request header</param>
/// <returns>RequestResult with StatusCode and Stream of received data</returns>
public RequestResult MakeRequest(string url, byte requestType, string? referrer = null)
TimeSpan rateLimitTimeout = _rateLimit[requestType]
.Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]));
if (rateLimitTimeout > TimeSpan.Zero)
Thread.Sleep(rateLimitTimeout);
RequestResult result = MakeRequestInternal(url, referrer);
_lastExecutedRateLimit[requestType] = DateTime.Now;
return result;
}
protected abstract RequestResult MakeRequestInternal(string url, string? referrer = null);
public abstract void Close();
public struct RequestResult
{
public HttpStatusCode statusCode { get; }
public Stream result { get; }
public bool hasBeenRedirected { get; }
public string? redirectedToUrl { get; }
public HtmlDocument? htmlDocument { get; }
public RequestResult(HttpStatusCode statusCode, HtmlDocument? htmlDocument, Stream result)
{
if (_rateLimit.TryGetValue(requestType, out TimeSpan value))
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
else
{
Log("RequestType not configured for rate-limit.");
return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null);
}
TimeSpan rateLimitTimeout = _rateLimit[requestType]
.Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]));
if(rateLimitTimeout > TimeSpan.Zero)
Thread.Sleep(rateLimitTimeout);
HttpResponseMessage? response = null;
while (response is null)
{
try
{
HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
if(referrer is not null)
requestMessage.Headers.Referrer = new Uri(referrer);
_lastExecutedRateLimit[requestType] = DateTime.Now;
//Log($"Requesting {requestType} {url}");
response = Client.Send(requestMessage);
}
catch (HttpRequestException e)
{
Log("Exception:\n\t{0}\n\tWaiting {1} before retrying.", e.Message, _rateLimit[requestType] * 2);
Thread.Sleep(_rateLimit[requestType] * 2);
}
}
if (!response.IsSuccessStatusCode)
{
Log($"Request-Error {response.StatusCode}: {response.ReasonPhrase}");
return new RequestResult(response.StatusCode, Stream.Null);
}
// Request has been redirected to another page. For example, it redirects directly to the results when there is only 1 result
if(response.RequestMessage is not null && response.RequestMessage.RequestUri is not null)
{
return new RequestResult(response.StatusCode, response.Content.ReadAsStream(), true, response.RequestMessage.RequestUri.AbsoluteUri);
}
return new RequestResult(response.StatusCode, response.Content.ReadAsStream());
this.statusCode = statusCode;
this.htmlDocument = htmlDocument;
this.result = result;
}
public struct RequestResult
public RequestResult(HttpStatusCode statusCode, HtmlDocument? htmlDocument, Stream result, bool hasBeenRedirected, string redirectedTo)
: this(statusCode, htmlDocument, result)
{
public HttpStatusCode statusCode { get; }
public Stream result { get; }
public bool hasBeenRedirected { get; }
public string? redirectedToUrl { get; }
public RequestResult(HttpStatusCode statusCode, Stream result)
{
this.statusCode = statusCode;
this.result = result;
}
public RequestResult(HttpStatusCode statusCode, Stream result, bool hasBeenRedirected, string redirectedTo)
: this(statusCode, result)
{
this.hasBeenRedirected = hasBeenRedirected;
redirectedToUrl = redirectedTo;
}
this.hasBeenRedirected = hasBeenRedirected;
redirectedToUrl = redirectedTo;
}
}
}
}