glax/Tranga
1
0
Fork 0
mirror of https://github.com/C9Glax/tranga.git synced 2025-10-11 13:19:48 +02:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Use DelegatingHandler for RateLimits
Some checks failed
Docker Image CI / build (push) Has been cancelled
Details

Browse Source
This commit is contained in:
glax
2025-10-06 22:48:46 +02:00
parent 02cec53963
commit ed07bba841
15 changed files with 142 additions and 538 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
API/MangaDownloadClients/HttpDownloadClient.cs
View File

@@ -1,89 +1,59 @@
using System.Net;
using HtmlAgilityPack;
using log4net;
namespace API.MangaDownloadClients;
internal class HttpDownloadClient : DownloadClient
internal class HttpDownloadClient : IDownloadClient
{
private static readonly FlareSolverrDownloadClient FlareSolverrDownloadClient = new();
internal override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null)
private static readonly HttpClient Client = new(handler: Tranga.RateLimitHandler)
{
Timeout = TimeSpan.FromSeconds(10),
DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrHigher,
DefaultRequestHeaders = { { "User-Agent", Tranga.Settings.UserAgent } }
};
private static readonly FlareSolverrDownloadClient FlareSolverrDownloadClient = new(Client);
private ILog Log { get; } = LogManager.GetLogger(typeof(HttpDownloadClient));
public async Task<HttpResponseMessage> MakeRequest(string url, RequestType requestType, string? referrer = null)
{
Log.Debug($"Using {typeof(HttpDownloadClient).FullName} for {url}");
if (clickButton is not null)
Log.Warn("Client can not click button");
HttpClient client = new();
client.Timeout = TimeSpan.FromSeconds(10);
client.DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrHigher;
client.DefaultRequestHeaders.Add("User-Agent", Tranga.Settings.UserAgent);
HttpResponseMessage? response;
Uri uri = new(url);
HttpRequestMessage requestMessage = new(HttpMethod.Get, uri);
HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
if (referrer is not null)
requestMessage.Headers.Referrer = new (referrer);
Log.Debug($"Requesting {url}");
try
{
response = client.Send(requestMessage);
}
catch (HttpRequestException e)
{
Log.Error(e);
return new (HttpStatusCode.Unused, null, Stream.Null);
}
HttpResponseMessage response = await Client.SendAsync(requestMessage);
Log.Debug($"Request {url} returned {(int)response.StatusCode} {response.StatusCode}");
if(response.IsSuccessStatusCode)
return response;
if (!response.IsSuccessStatusCode)
{
Log.Debug($"Request returned status code {(int)response.StatusCode} {response.StatusCode}");
if (response.Headers.Server.Any(s =>
(s.Product?.Name ?? "").Contains("cloudflare", StringComparison.InvariantCultureIgnoreCase)))
{
Log.Debug("Retrying with FlareSolverr!");
return FlareSolverrDownloadClient.MakeRequestInternal(url, referrer, clickButton);
}
else
{
Log.Debug($"Request returned status code {(int)response.StatusCode} {response.StatusCode}:\n" +
$"=====\n" +
$"Request:\n" +
$"{requestMessage.Method} {requestMessage.RequestUri}\n" +
$"{requestMessage.Version} {requestMessage.VersionPolicy}\n" +
$"Headers:\n\t{string.Join("\n\t", requestMessage.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
$"{requestMessage.Content?.ReadAsStringAsync().Result}" +
$"=====\n" +
$"Response:\n" +
$"{response.Version}\n" +
$"Headers:\n\t{string.Join("\n\t", response.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
$"{response.Content.ReadAsStringAsync().Result}");
return await FlareSolverrDownloadClient.MakeRequest(url, requestType, referrer);
}
Log.Debug($"Request returned status code {(int)response.StatusCode} {response.StatusCode}:\n" +
$"=====\n" +
$"Request:\n" +
$"{requestMessage.Method} {requestMessage.RequestUri}\n" +
$"{requestMessage.Version} {requestMessage.VersionPolicy}\n" +
$"Headers:\n\t{string.Join("\n\t", requestMessage.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
$"{requestMessage.Content?.ReadAsStringAsync().Result}" +
$"=====\n" +
$"Response:\n" +
$"{response.Version}\n" +
$"Headers:\n\t{string.Join("\n\t", response.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
$"{response.Content.ReadAsStringAsync().Result}");
return new(HttpStatusCode.InternalServerError);
}
Stream stream;
try
{
stream = response.Content.ReadAsStream();
}
catch (Exception e)
catch (HttpRequestException e)
{
Log.Error(e);
return new (HttpStatusCode.Unused, null, Stream.Null);
return new(HttpStatusCode.InternalServerError);
}
HtmlDocument? document = null;
if (response.Content.Headers.ContentType?.MediaType == "text/html")
{
StreamReader reader = new (stream);
document = new ();
document.LoadHtml(reader.ReadToEnd());
stream.Position = 0;
}
// Request has been redirected to another page. For example, it redirects directly to the results when there is only 1 result
if (response.RequestMessage is not null && response.RequestMessage.RequestUri is not null && response.RequestMessage.RequestUri != uri)
{
return new (response.StatusCode, document, stream, true, response.RequestMessage.RequestUri.AbsoluteUri);
}
return new (response.StatusCode, document, stream);
}
}