diff --git a/API/Controllers/SettingsController.cs b/API/Controllers/SettingsController.cs index fce6d47..4233ff8 100644 --- a/API/Controllers/SettingsController.cs +++ b/API/Controllers/SettingsController.cs @@ -60,17 +60,6 @@ public class SettingsController() : Controller return TypedResults.Ok(); } - /// - /// Get all Request-Limits - /// - /// - [HttpGet("RequestLimits")] - [ProducesResponseType>(Status200OK, "application/json")] - public Ok> GetRequestLimits() - { - return TypedResults.Ok(Tranga.Settings.RequestLimits); - } - /// /// Update all Request-Limits to new values /// @@ -81,48 +70,6 @@ public class SettingsController() : Controller { return TypedResults.StatusCode(Status501NotImplemented); } - - /// - /// Updates a Request-Limit value - /// - /// Type of Request - /// New limit in Requests/Minute - /// - /// Limit needs to be greater than 0 - [HttpPatch("RequestLimits/{RequestType}")] - [ProducesResponseType(Status200OK)] - [ProducesResponseType(Status400BadRequest)] - public Results SetRequestLimit(RequestType RequestType, [FromBody]int requestLimit) - { - if (requestLimit <= 0) - return TypedResults.BadRequest(); - Tranga.Settings.SetRequestLimit(RequestType, requestLimit); - return TypedResults.Ok(); - } - - /// - /// Reset Request-Limit - /// - /// - [HttpDelete("RequestLimits/{RequestType}")] - [ProducesResponseType(Status200OK)] - public Ok ResetRequestLimits(RequestType RequestType) - { - Tranga.Settings.SetRequestLimit(RequestType, TrangaSettings.DefaultRequestLimits[RequestType]); - return TypedResults.Ok(); - } - - /// - /// Reset Request-Limit - /// - /// - [HttpDelete("RequestLimits")] - [ProducesResponseType(Status200OK)] - public Ok ResetRequestLimits() - { - Tranga.Settings.ResetRequestLimits(); - return TypedResults.Ok(); - } /// /// Returns Level of Image-Compression for Images @@ -260,12 +207,12 @@ public class SettingsController() : Controller [HttpPost("FlareSolverr/Test")] [ProducesResponseType(Status200OK)] [ProducesResponseType(Status500InternalServerError)] - public Results TestFlareSolverrReachable() + public async Task> TestFlareSolverrReachable() { const string knownProtectedUrl = "https://prowlarr.servarr.com/v1/ping"; - FlareSolverrDownloadClient client = new(); - RequestResult result = client.MakeRequestInternal(knownProtectedUrl); - return (int)result.statusCode >= 200 && (int)result.statusCode < 300 ? TypedResults.Ok() : TypedResults.InternalServerError(); + FlareSolverrDownloadClient client = new(new ()); + HttpResponseMessage result = await client.MakeRequest(knownProtectedUrl, RequestType.Default); + return (int)result.StatusCode >= 200 && (int)result.StatusCode < 300 ? TypedResults.Ok() : TypedResults.InternalServerError(); } /// diff --git a/API/MangaConnectors/MangaConnector.cs b/API/MangaConnectors/MangaConnector.cs index c462856..5514c0f 100644 --- a/API/MangaConnectors/MangaConnector.cs +++ b/API/MangaConnectors/MangaConnector.cs @@ -5,9 +5,7 @@ using API.MangaDownloadClients; using API.Schema.MangaContext; using log4net; using Microsoft.EntityFrameworkCore; -using Newtonsoft.Json; using SixLabors.ImageSharp; -using SixLabors.ImageSharp.Formats.Jpeg; using SixLabors.ImageSharp.Processing; namespace API.MangaConnectors; @@ -15,7 +13,7 @@ namespace API.MangaConnectors; [PrimaryKey("Name")] public abstract class MangaConnector(string name, string[] supportedLanguages, string[] baseUris, string iconUrl) { - [NotMapped] internal DownloadClient downloadClient { get; init; } = null!; + [NotMapped] internal IDownloadClient downloadClient { get; init; } = null!; [NotMapped] protected ILog Log { get; init; } = LogManager.GetLogger(name); [StringLength(32)] public string Name { get; init; } = name; [StringLength(8)] public string[] SupportedLanguages { get; init; } = supportedLanguages; @@ -50,14 +48,14 @@ public abstract class MangaConnector(string name, string[] supportedLanguages, s if (File.Exists(saveImagePath)) return filename; - RequestResult coverResult = downloadClient.MakeRequest(mangaId.Obj.CoverUrl, RequestType.MangaCover, $"https://{match.Groups[1].Value}"); - if ((int)coverResult.statusCode < 200 || (int)coverResult.statusCode >= 300) + HttpResponseMessage coverResult = downloadClient.MakeRequest(mangaId.Obj.CoverUrl, RequestType.MangaCover, $"https://{match.Groups[1].Value}").Result; + if ((int)coverResult.StatusCode < 200 || (int)coverResult.StatusCode >= 300) return SaveCoverImageToCache(mangaId, --retries); try { using MemoryStream ms = new(); - coverResult.result.CopyTo(ms); + coverResult.Content.ReadAsStream().CopyTo(ms); byte[] imageBytes = ms.ToArray(); Directory.CreateDirectory(TrangaSettings.CoverImageCacheOriginal); File.WriteAllBytes(saveImagePath, imageBytes); diff --git a/API/MangaConnectors/MangaDex.cs b/API/MangaConnectors/MangaDex.cs index e6f4188..b88905b 100644 --- a/API/MangaConnectors/MangaDex.cs +++ b/API/MangaConnectors/MangaDex.cs @@ -35,14 +35,14 @@ public class MangaDex : MangaConnector $"&includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author&includes%5B%5D=artist&includes%5B%5D=tag'"; offset += Limit; - RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed); - if ((int)result.statusCode < 200 || (int)result.statusCode >= 300) + HttpResponseMessage result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed).Result; + if ((int)result.StatusCode < 200 || (int)result.StatusCode >= 300) { Log.Error("Request failed"); return []; } - using StreamReader sr = new (result.result); + using StreamReader sr = new (result.Content.ReadAsStream()); JObject jObject = JObject.Parse(sr.ReadToEnd()); if (jObject.Value("result") != "ok") @@ -96,14 +96,14 @@ public class MangaDex : MangaConnector $"https://api.mangadex.org/manga/{mangaIdOnSite}" + $"?includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author&includes%5B%5D=artist&includes%5B%5D=tag'"; - RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed); - if ((int)result.statusCode < 200 || (int)result.statusCode >= 300) + HttpResponseMessage result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed).Result; + if ((int)result.StatusCode < 200 || (int)result.StatusCode >= 300) { Log.Error("Request failed"); return null; } - using StreamReader sr = new (result.result); + using StreamReader sr = new (result.Content.ReadAsStream()); JObject jObject = JObject.Parse(sr.ReadToEnd()); if (jObject.Value("result") != "ok") @@ -138,14 +138,14 @@ public class MangaDex : MangaConnector $"contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&includeFutureUpdates=0&includes%5B%5D="; offset += Limit; - RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed); - if ((int)result.statusCode < 200 || (int)result.statusCode >= 300) + HttpResponseMessage result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed).Result; + if ((int)result.StatusCode < 200 || (int)result.StatusCode >= 300) { Log.Error("Request failed"); return []; } - using StreamReader sr = new (result.result); + using StreamReader sr = new (result.Content.ReadAsStream()); JObject jObject = JObject.Parse(sr.ReadToEnd()); if (jObject.Value("result") != "ok") @@ -191,14 +191,14 @@ public class MangaDex : MangaConnector string id = match.Groups[1].Value; string requestUrl = $"https://api.mangadex.org/at-home/server/{id}"; - RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.Default); - if ((int)result.statusCode < 200 || (int)result.statusCode >= 300) + HttpResponseMessage result = downloadClient.MakeRequest(requestUrl, RequestType.Default).Result; + if ((int)result.StatusCode < 200 || (int)result.StatusCode >= 300) { Log.Error("Request failed"); return []; } - using StreamReader sr = new (result.result); + using StreamReader sr = new (result.Content.ReadAsStream()); JObject jObject = JObject.Parse(sr.ReadToEnd()); if (jObject.Value("result") != "ok") diff --git a/API/MangaConnectors/MangaPark.cs b/API/MangaConnectors/MangaPark.cs index 51bc614..091d01d 100644 --- a/API/MangaConnectors/MangaPark.cs +++ b/API/MangaConnectors/MangaPark.cs @@ -36,7 +36,7 @@ public class MangaPark : MangaConnector for (int page = 1;; page++) // break; in loop { Uri searchUri = new(baseUri, $"search?word={HttpUtility.UrlEncode(mangaSearchName)}&lang={Tranga.Settings.DownloadLanguage}&page={page}"); - if (downloadClient.MakeRequest(searchUri.ToString(), RequestType.Default) is { statusCode: >= HttpStatusCode.OK and < HttpStatusCode.Ambiguous } result) + if (downloadClient.MakeRequest(searchUri.ToString(), RequestType.Default).Result is { StatusCode: >= HttpStatusCode.OK and < HttpStatusCode.Ambiguous } result) { HtmlDocument document = result.CreateDocument(); // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract HAP sucks with nullable types @@ -73,8 +73,8 @@ public class MangaPark : MangaConnector public override (Manga, MangaConnectorId)? GetMangaFromUrl(string url) { - if (downloadClient.MakeRequest(url, RequestType.Default) is - { statusCode: >= HttpStatusCode.OK and < HttpStatusCode.Ambiguous } result) + if (downloadClient.MakeRequest(url, RequestType.Default).Result is + { StatusCode: >= HttpStatusCode.OK and < HttpStatusCode.Ambiguous } result) { HtmlDocument document= result.CreateDocument(); @@ -145,8 +145,8 @@ public class MangaPark : MangaConnector List<(Chapter, MangaConnectorId)> ret = []; - if (downloadClient.MakeRequest(requestUri.ToString(), RequestType.Default) is - { statusCode: >= HttpStatusCode.OK and < HttpStatusCode.Ambiguous } result) + if (downloadClient.MakeRequest(requestUri.ToString(), RequestType.Default).Result is + { StatusCode: >= HttpStatusCode.OK and < HttpStatusCode.Ambiguous } result) { HtmlDocument document= result.CreateDocument(); @@ -220,8 +220,8 @@ public class MangaPark : MangaConnector Log.Debug($"Using domain {domain}"); Uri baseUri = new ($"https://{domain}/"); Uri requestUri = new (baseUri, $"title/{chapterId.IdOnConnectorSite}"); - if (downloadClient.MakeRequest(requestUri.ToString(), RequestType.Default) is - { statusCode: >= HttpStatusCode.OK and < HttpStatusCode.Ambiguous } result) + if (downloadClient.MakeRequest(requestUri.ToString(), RequestType.Default).Result is + { StatusCode: >= HttpStatusCode.OK and < HttpStatusCode.Ambiguous } result) { HtmlDocument document = result.CreateDocument(); @@ -240,10 +240,10 @@ public class MangaPark : MangaConnector internal static class MangaParkHelper { - internal static HtmlDocument CreateDocument(this RequestResult result) + internal static HtmlDocument CreateDocument(this HttpResponseMessage result) { HtmlDocument document = new(); - StreamReader sr = new (result.result); + StreamReader sr = new (result.Content.ReadAsStream()); string htmlStr = sr.ReadToEnd().Replace("q:key", "qkey"); document.LoadHtml(htmlStr); diff --git a/API/MangaConnectors/Mangaworld.cs b/API/MangaConnectors/Mangaworld.cs index e8b84cf..9def2c1 100644 --- a/API/MangaConnectors/Mangaworld.cs +++ b/API/MangaConnectors/Mangaworld.cs @@ -30,11 +30,11 @@ public sealed class Mangaworld : MangaConnector Uri baseUri = new ("https://www.mangaworld.cx/"); Uri searchUrl = new (baseUri, "archive?keyword=" + HttpUtility.UrlEncode(mangaSearchName)); - RequestResult res = downloadClient.MakeRequest(searchUrl.ToString(), RequestType.Default); - if ((int)res.statusCode < 200 || (int)res.statusCode >= 300) + HttpResponseMessage res = downloadClient.MakeRequest(searchUrl.ToString(), RequestType.Default).Result; + if ((int)res.StatusCode < 200 || (int)res.StatusCode >= 300) return []; - using StreamReader sr = new (res.result); + using StreamReader sr = new (res.Content.ReadAsStream()); string html = sr.ReadToEnd(); HtmlDocument doc = new (); @@ -85,11 +85,11 @@ public sealed class Mangaworld : MangaConnector string slug = parts[1]; string url = $"https://www.mangaworld.cx/manga/{id}/{slug}/"; - RequestResult res = downloadClient.MakeRequest(url, RequestType.MangaInfo); - if ((int)res.statusCode < 200 || (int)res.statusCode >= 300) + HttpResponseMessage res = downloadClient.MakeRequest(url, RequestType.MangaInfo).Result; + if ((int)res.StatusCode < 200 || (int)res.StatusCode >= 300) return null; - using StreamReader sr = new (res.result); + using StreamReader sr = new (res.Content.ReadAsStream()); string html = sr.ReadToEnd(); HtmlDocument doc = new (); @@ -175,11 +175,11 @@ public sealed class Mangaworld : MangaConnector { string url = EnsureListStyle(chapterId.WebsiteUrl ?? $"https://www.mangaworld.cx/manga/{chapterId.IdOnConnectorSite}"); - RequestResult res = downloadClient.MakeRequest(url, RequestType.MangaInfo); - if ((int)res.statusCode < 200 || (int)res.statusCode >= 300) + HttpResponseMessage res = downloadClient.MakeRequest(url, RequestType.MangaInfo).Result; + if ((int)res.StatusCode < 200 || (int)res.StatusCode >= 300) return []; - using StreamReader sr = new (res.result); + using StreamReader sr = new (res.Content.ReadAsStream()); string html = sr.ReadToEnd(); Uri baseUri = new (url); @@ -354,20 +354,20 @@ public sealed class Mangaworld : MangaConnector baseUri = new (seriesUrl); // 1) tenta client "Default" - RequestResult res = downloadClient.MakeRequest(seriesUrl, RequestType.Default); - if ((int)res.statusCode >= 200 && (int)res.statusCode < 300) + HttpResponseMessage res = downloadClient.MakeRequest(seriesUrl, RequestType.Default).Result; + if ((int)res.StatusCode >= 200 && (int)res.StatusCode < 300) { - using StreamReader sr = new (res.result); + using StreamReader sr = new (res.Content.ReadAsStream()); string html = sr.ReadToEnd(); if (!LooksLikeChallenge(html)) return html; } // 2) fallback: client “MangaInfo” (proxy/Flare se configurato) - RequestResult res2 = downloadClient.MakeRequest(seriesUrl, RequestType.MangaInfo); - if ((int)res2.statusCode >= 200 && (int)res2.statusCode < 300) + HttpResponseMessage res2 = downloadClient.MakeRequest(seriesUrl, RequestType.MangaInfo).Result; + if ((int)res2.StatusCode >= 200 && (int)res2.StatusCode < 300) { - using StreamReader sr2 = new StreamReader(res2.result); + using StreamReader sr2 = new StreamReader(res2.Content.ReadAsStream()); return sr2.ReadToEnd(); } diff --git a/API/MangaDownloadClients/DownloadClient.cs b/API/MangaDownloadClients/DownloadClient.cs index 0df3f51..6cce650 100644 --- a/API/MangaDownloadClients/DownloadClient.cs +++ b/API/MangaDownloadClients/DownloadClient.cs @@ -1,56 +1,6 @@ -using System.Collections.Concurrent; -using System.Net; -using log4net; +namespace API.MangaDownloadClients; -namespace API.MangaDownloadClients; - -public abstract class DownloadClient +public interface IDownloadClient { - private static readonly ConcurrentDictionary LastExecutedRateLimit = new(); - protected ILog Log { get; init; } - - protected DownloadClient() - { - this.Log = LogManager.GetLogger(GetType()); - } - - // TODO Requests still go too fast across threads! - public RequestResult MakeRequest(string url, RequestType requestType, string? referrer = null, string? clickButton = null) - { - Log.Debug($"Requesting {requestType} {url}"); - - // If we don't have a RequestLimit set for a Type, use the default one - if (!Tranga.Settings.RequestLimits.ContainsKey(requestType)) - requestType = RequestType.Default; - - int rateLimit = Tranga.Settings.RequestLimits[requestType]; - // TODO this probably needs a better check whether the useragent matches... - // If the UserAgent is the default one, do not exceed the default request-limits. - if (Tranga.Settings.UserAgent == TrangaSettings.DefaultUserAgent && rateLimit > TrangaSettings.DefaultRequestLimits[requestType]) - rateLimit = TrangaSettings.DefaultRequestLimits[requestType]; - - // Apply the delay - TimeSpan timeBetweenRequests = TimeSpan.FromMinutes(1).Divide(rateLimit); - DateTime now = DateTime.UtcNow; - LastExecutedRateLimit.TryAdd(requestType, now.Subtract(timeBetweenRequests)); - - TimeSpan rateLimitTimeout = timeBetweenRequests.Subtract(now.Subtract(LastExecutedRateLimit[requestType])); - Log.Debug($"Request limit {requestType} {rateLimit}/Minute timeBetweenRequests: {timeBetweenRequests:ss'.'fffff}"); - - if (rateLimitTimeout > TimeSpan.Zero) - { - Log.Info($"Waiting {rateLimitTimeout} for {url}"); - Thread.Sleep(rateLimitTimeout); - } - - // Make the request - RequestResult result = MakeRequestInternal(url, referrer, clickButton); - - // Update the time the last request was made - LastExecutedRateLimit[requestType] = DateTime.UtcNow; - Log.Debug($"Result {url}: {result}"); - return result; - } - - internal abstract RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null); + internal Task MakeRequest(string url, RequestType requestType, string? referrer = null); } \ No newline at end of file diff --git a/API/MangaDownloadClients/FlareSolverrDownloadClient.cs b/API/MangaDownloadClients/FlareSolverrDownloadClient.cs index f4861c8..86dd1c5 100644 --- a/API/MangaDownloadClients/FlareSolverrDownloadClient.cs +++ b/API/MangaDownloadClients/FlareSolverrDownloadClient.cs @@ -1,25 +1,26 @@ using System.Diagnostics.CodeAnalysis; using System.Net; -using System.Text; using System.Text.Json; using HtmlAgilityPack; +using log4net; using Newtonsoft.Json; using Newtonsoft.Json.Linq; namespace API.MangaDownloadClients; -public class FlareSolverrDownloadClient : DownloadClient +public class FlareSolverrDownloadClient(HttpClient client) : IDownloadClient { - internal override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null) + private ILog Log { get; } = LogManager.GetLogger(typeof(FlareSolverrDownloadClient)); + + public async Task MakeRequest(string url, RequestType requestType, string? referrer = null) { - if (clickButton is not null) - Log.Warn("Client can not click button"); + Log.Debug($"Using {typeof(FlareSolverrDownloadClient).FullName} for {url}"); if(referrer is not null) Log.Warn("Client can not set referrer"); if (Tranga.Settings.FlareSolverrUrl == string.Empty) { Log.Error("FlareSolverr URL is empty"); - return new(HttpStatusCode.InternalServerError, null, Stream.Null); + return new(HttpStatusCode.InternalServerError); } Uri flareSolverrUri = new (Tranga.Settings.FlareSolverrUrl); @@ -29,13 +30,6 @@ public class FlareSolverrDownloadClient : DownloadClient Path = "v1" }.Uri; - HttpClient client = new() - { - Timeout = TimeSpan.FromSeconds(10), - DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrHigher, - DefaultRequestHeaders = { { "User-Agent", Tranga.Settings.UserAgent } } - }; - JObject requestObj = new() { ["cmd"] = "request.get", @@ -52,12 +46,12 @@ public class FlareSolverrDownloadClient : DownloadClient HttpResponseMessage? response; try { - response = client.Send(requestMessage); + response = await client.SendAsync(requestMessage); } catch (HttpRequestException e) { Log.Error(e); - return new (HttpStatusCode.Unused, null, Stream.Null); + return new (HttpStatusCode.InternalServerError); } if (!response.IsSuccessStatusCode) @@ -74,7 +68,7 @@ public class FlareSolverrDownloadClient : DownloadClient $"{response.Version}\n" + $"Headers:\n\t{string.Join("\n\t", response.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" + $"{response.Content.ReadAsStringAsync().Result}"); - return new (response.StatusCode, null, Stream.Null); + return response; } string responseString = response.Content.ReadAsStringAsync().Result; @@ -82,51 +76,45 @@ public class FlareSolverrDownloadClient : DownloadClient if (!IsInCorrectFormat(responseObj, out string? reason)) { Log.Error($"Wrong format: {reason}"); - return new(HttpStatusCode.Unused, null, Stream.Null); + return new(HttpStatusCode.InternalServerError); } string statusResponse = responseObj["status"]!.Value()!; if (statusResponse != "ok") { Log.Debug($"Status is not ok: {statusResponse}"); - return new(HttpStatusCode.Unused, null, Stream.Null); + return new(HttpStatusCode.InternalServerError); } JObject solution = (responseObj["solution"] as JObject)!; if (!Enum.TryParse(solution["status"]!.Value().ToString(), out HttpStatusCode statusCode)) { Log.Error($"Wrong format: Cant parse status code: {solution["status"]!.Value()}"); - return new(HttpStatusCode.Unused, null, Stream.Null); + return new(HttpStatusCode.InternalServerError); } if (statusCode < HttpStatusCode.OK || statusCode >= HttpStatusCode.MultipleChoices) { Log.Debug($"Status is: {statusCode}"); - return new(statusCode, null, Stream.Null); + return new (statusCode); } if (solution["response"]!.Value() is not { } htmlString) { Log.Error("Wrong format: Cant find response in solution"); - return new(HttpStatusCode.Unused, null, Stream.Null); + return new(HttpStatusCode.InternalServerError); } - if (IsJson(htmlString, out HtmlDocument document, out string? json)) + if (IsJson(htmlString, out string? json)) { - MemoryStream ms = new(); - ms.Write(Encoding.UTF8.GetBytes(json)); - ms.Position = 0; - return new(statusCode, document, ms); + return new(statusCode) { Content = new StringContent(json) }; } else { - MemoryStream ms = new(); - ms.Write(Encoding.UTF8.GetBytes(htmlString)); - ms.Position = 0; - return new(statusCode, document, ms); + return new(statusCode) { Content = new StringContent(htmlString) }; } } - private bool IsInCorrectFormat(JObject responseObj, [NotNullWhen(false)]out string? reason) + private static bool IsInCorrectFormat(JObject responseObj, [NotNullWhen(false)]out string? reason) { reason = null; if (!responseObj.ContainsKey("status")) @@ -157,10 +145,10 @@ public class FlareSolverrDownloadClient : DownloadClient return true; } - private bool IsJson(string htmlString, out HtmlDocument document, [NotNullWhen(true)]out string? jsonString) + private static bool IsJson(string htmlString, [NotNullWhen(true)]out string? jsonString) { jsonString = null; - document = new(); + HtmlDocument document = new(); document.LoadHtml(htmlString); HtmlNode pre = document.DocumentNode.SelectSingleNode("//pre"); diff --git a/API/MangaDownloadClients/HttpDownloadClient.cs b/API/MangaDownloadClients/HttpDownloadClient.cs index d0babb0..e4b5eea 100644 --- a/API/MangaDownloadClients/HttpDownloadClient.cs +++ b/API/MangaDownloadClients/HttpDownloadClient.cs @@ -1,89 +1,59 @@ using System.Net; -using HtmlAgilityPack; +using log4net; namespace API.MangaDownloadClients; -internal class HttpDownloadClient : DownloadClient +internal class HttpDownloadClient : IDownloadClient { - private static readonly FlareSolverrDownloadClient FlareSolverrDownloadClient = new(); - internal override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null) + private static readonly HttpClient Client = new(handler: Tranga.RateLimitHandler) + { + Timeout = TimeSpan.FromSeconds(10), + DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrHigher, + DefaultRequestHeaders = { { "User-Agent", Tranga.Settings.UserAgent } } + }; + private static readonly FlareSolverrDownloadClient FlareSolverrDownloadClient = new(Client); + private ILog Log { get; } = LogManager.GetLogger(typeof(HttpDownloadClient)); + + public async Task MakeRequest(string url, RequestType requestType, string? referrer = null) { Log.Debug($"Using {typeof(HttpDownloadClient).FullName} for {url}"); - if (clickButton is not null) - Log.Warn("Client can not click button"); - HttpClient client = new(); - client.Timeout = TimeSpan.FromSeconds(10); - client.DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrHigher; - client.DefaultRequestHeaders.Add("User-Agent", Tranga.Settings.UserAgent); - HttpResponseMessage? response; - Uri uri = new(url); - HttpRequestMessage requestMessage = new(HttpMethod.Get, uri); + HttpRequestMessage requestMessage = new(HttpMethod.Get, url); if (referrer is not null) requestMessage.Headers.Referrer = new (referrer); Log.Debug($"Requesting {url}"); + try { - response = client.Send(requestMessage); - } - catch (HttpRequestException e) - { - Log.Error(e); - return new (HttpStatusCode.Unused, null, Stream.Null); - } + HttpResponseMessage response = await Client.SendAsync(requestMessage); + Log.Debug($"Request {url} returned {(int)response.StatusCode} {response.StatusCode}"); + if(response.IsSuccessStatusCode) + return response; - if (!response.IsSuccessStatusCode) - { - Log.Debug($"Request returned status code {(int)response.StatusCode} {response.StatusCode}"); if (response.Headers.Server.Any(s => (s.Product?.Name ?? "").Contains("cloudflare", StringComparison.InvariantCultureIgnoreCase))) { Log.Debug("Retrying with FlareSolverr!"); - return FlareSolverrDownloadClient.MakeRequestInternal(url, referrer, clickButton); - } - else - { - Log.Debug($"Request returned status code {(int)response.StatusCode} {response.StatusCode}:\n" + - $"=====\n" + - $"Request:\n" + - $"{requestMessage.Method} {requestMessage.RequestUri}\n" + - $"{requestMessage.Version} {requestMessage.VersionPolicy}\n" + - $"Headers:\n\t{string.Join("\n\t", requestMessage.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" + - $"{requestMessage.Content?.ReadAsStringAsync().Result}" + - $"=====\n" + - $"Response:\n" + - $"{response.Version}\n" + - $"Headers:\n\t{string.Join("\n\t", response.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" + - $"{response.Content.ReadAsStringAsync().Result}"); + return await FlareSolverrDownloadClient.MakeRequest(url, requestType, referrer); } + + Log.Debug($"Request returned status code {(int)response.StatusCode} {response.StatusCode}:\n" + + $"=====\n" + + $"Request:\n" + + $"{requestMessage.Method} {requestMessage.RequestUri}\n" + + $"{requestMessage.Version} {requestMessage.VersionPolicy}\n" + + $"Headers:\n\t{string.Join("\n\t", requestMessage.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" + + $"{requestMessage.Content?.ReadAsStringAsync().Result}" + + $"=====\n" + + $"Response:\n" + + $"{response.Version}\n" + + $"Headers:\n\t{string.Join("\n\t", response.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" + + $"{response.Content.ReadAsStringAsync().Result}"); + return new(HttpStatusCode.InternalServerError); } - - Stream stream; - try - { - stream = response.Content.ReadAsStream(); - } - catch (Exception e) + catch (HttpRequestException e) { Log.Error(e); - return new (HttpStatusCode.Unused, null, Stream.Null); + return new(HttpStatusCode.InternalServerError); } - - HtmlDocument? document = null; - - if (response.Content.Headers.ContentType?.MediaType == "text/html") - { - StreamReader reader = new (stream); - document = new (); - document.LoadHtml(reader.ReadToEnd()); - stream.Position = 0; - } - - // Request has been redirected to another page. For example, it redirects directly to the results when there is only 1 result - if (response.RequestMessage is not null && response.RequestMessage.RequestUri is not null && response.RequestMessage.RequestUri != uri) - { - return new (response.StatusCode, document, stream, true, response.RequestMessage.RequestUri.AbsoluteUri); - } - - return new (response.StatusCode, document, stream); } } \ No newline at end of file diff --git a/API/MangaDownloadClients/RateLimitHandler.cs b/API/MangaDownloadClients/RateLimitHandler.cs new file mode 100644 index 0000000..3213e74 --- /dev/null +++ b/API/MangaDownloadClients/RateLimitHandler.cs @@ -0,0 +1,31 @@ +using System.Net; +using System.Threading.RateLimiting; +using log4net; + +namespace API.MangaDownloadClients; + +public class RateLimitHandler() : DelegatingHandler(new HttpClientHandler()) +{ + private ILog Log { get; init; } = LogManager.GetLogger(typeof(RateLimitHandler)); + + private readonly RateLimiter _limiter = new SlidingWindowRateLimiter(new () + { + AutoReplenishment = true, + PermitLimit = 240, + QueueLimit = 120, + QueueProcessingOrder = QueueProcessingOrder.OldestFirst, + SegmentsPerWindow = 60, + Window = TimeSpan.FromSeconds(60) + }); + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Log.Debug($"Requesting lease {request.RequestUri}"); + using RateLimitLease lease = await _limiter.AcquireAsync(permitCount: 1, cancellationToken); + Log.Debug($"lease {lease.IsAcquired} {request.RequestUri}"); + + return lease.IsAcquired + ? await base.SendAsync(request, cancellationToken) + : new (HttpStatusCode.TooManyRequests); + } +} \ No newline at end of file diff --git a/API/MangaDownloadClients/RequestResult.cs b/API/MangaDownloadClients/RequestResult.cs deleted file mode 100644 index ae9cec1..0000000 --- a/API/MangaDownloadClients/RequestResult.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System.Net; -using HtmlAgilityPack; - -namespace API.MangaDownloadClients; - -public struct RequestResult -{ - public HttpStatusCode statusCode { get; } - public Stream result { get; } - public bool hasBeenRedirected { get; } - public string? redirectedToUrl { get; } - public HtmlDocument? htmlDocument { get; } - - public RequestResult(HttpStatusCode statusCode, HtmlDocument? htmlDocument, Stream result) - { - this.statusCode = statusCode; - this.htmlDocument = htmlDocument; - this.result = result; - } - - public RequestResult(HttpStatusCode statusCode, HtmlDocument? htmlDocument, Stream result, bool hasBeenRedirected, string redirectedTo) - : this(statusCode, htmlDocument, result) - { - this.hasBeenRedirected = hasBeenRedirected; - redirectedToUrl = redirectedTo; - } - - public override string ToString() - { - return - $"{(int)statusCode} {statusCode.ToString()} {(hasBeenRedirected ? "Redirected: " : "")} {redirectedToUrl}"; - } -} \ No newline at end of file diff --git a/API/Tranga.cs b/API/Tranga.cs index 9eb0aac..a05eb02 100644 --- a/API/Tranga.cs +++ b/API/Tranga.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using System.Diagnostics.CodeAnalysis; using API.MangaConnectors; +using API.MangaDownloadClients; using API.Schema.LibraryContext; using API.Schema.MangaContext; using API.Schema.MangaContext.MetadataFetchers; @@ -35,6 +36,8 @@ public static class Tranga internal static readonly CleanupMangaconnectorIdsWithoutConnector CleanupMangaconnectorIdsWithoutConnector = new(); // ReSharper restore MemberCanBePrivate.Global + internal static readonly RateLimitHandler RateLimitHandler = new(); + internal static void StartupTasks() { AddWorker(SendNotificationsWorker); diff --git a/API/TrangaSettings.cs b/API/TrangaSettings.cs index 7b205e4..0cbe2ff 100644 --- a/API/TrangaSettings.cs +++ b/API/TrangaSettings.cs @@ -1,5 +1,4 @@ using System.Runtime.InteropServices; -using API.MangaDownloadClients; using API.Workers; using Newtonsoft.Json; using Newtonsoft.Json.Converters; @@ -38,17 +37,6 @@ public struct TrangaSettings() /// public string ChapterNamingScheme { get; set; } = "%M - ?V(Vol.%V )Ch.%C?T( - %T)"; public int WorkCycleTimeoutMs { get; set; } = 20000; - [JsonIgnore] - internal static readonly Dictionary DefaultRequestLimits = new () - { - {RequestType.MangaInfo, 360}, - {RequestType.MangaDexFeed, 360}, - {RequestType.MangaDexImage, 60}, - {RequestType.MangaImage, 240}, - {RequestType.MangaCover, 60}, - {RequestType.Default, 360} - }; - public Dictionary RequestLimits { get; set; } = DefaultRequestLimits; public string DownloadLanguage { get; set; } = "en"; @@ -78,18 +66,6 @@ public struct TrangaSettings() Save(); } - public void SetRequestLimit(RequestType type, int value) - { - this.RequestLimits[type] = value; - Save(); - } - - public void ResetRequestLimits() - { - this.RequestLimits = DefaultRequestLimits; - Save(); - } - public void UpdateImageCompression(int value) { this.ImageCompression = value; diff --git a/API/Workers/MangaDownloadWorkers/DownloadChapterFromMangaconnectorWorker.cs b/API/Workers/MangaDownloadWorkers/DownloadChapterFromMangaconnectorWorker.cs index 99c2beb..55dc59f 100644 --- a/API/Workers/MangaDownloadWorkers/DownloadChapterFromMangaconnectorWorker.cs +++ b/API/Workers/MangaDownloadWorkers/DownloadChapterFromMangaconnectorWorker.cs @@ -286,14 +286,14 @@ public class DownloadChapterFromMangaconnectorWorker(MangaConnectorId c private Stream? DownloadImage(string imageUrl) { HttpDownloadClient downloadClient = new(); - RequestResult requestResult = downloadClient.MakeRequest(imageUrl, RequestType.MangaImage); + HttpResponseMessage requestResult = downloadClient.MakeRequest(imageUrl, RequestType.MangaImage).Result; - if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + if ((int)requestResult.StatusCode < 200 || (int)requestResult.StatusCode >= 300) return null; - if (requestResult.result == Stream.Null) + if (requestResult.Content.ReadAsStream() == Stream.Null) return null; - return ProcessImage(requestResult.result, out Stream processedImage) ? processedImage : null; + return ProcessImage(requestResult.Content.ReadAsStream(), out Stream processedImage) ? processedImage : null; } public override string ToString() => $"{base.ToString()} {_mangaConnectorIdId}"; diff --git a/API/openapi/API_v2.json b/API/openapi/API_v2.json index 506142d..d725b20 100644 --- a/API/openapi/API_v2.json +++ b/API/openapi/API_v2.json @@ -2764,51 +2764,6 @@ } }, "/v2/Settings/RequestLimits": { - "get": { - "tags": [ - "Settings" - ], - "summary": "Get all Request-Limits", - "responses": { - "200": { - "description": "", - "content": { - "application/json; x-version=2.0": { - "schema": { - "type": "object", - "properties": { - "Default": { - "type": "integer", - "format": "int32" - }, - "MangaDexFeed": { - "type": "integer", - "format": "int32" - }, - "MangaImage": { - "type": "integer", - "format": "int32" - }, - "MangaCover": { - "type": "integer", - "format": "int32" - }, - "MangaDexImage": { - "type": "integer", - "format": "int32" - }, - "MangaInfo": { - "type": "integer", - "format": "int32" - } - }, - "additionalProperties": false - } - } - } - } - } - }, "patch": { "tags": [ "Settings" @@ -2820,145 +2775,6 @@ "description": "Not Implemented" } } - }, - "delete": { - "tags": [ - "Settings" - ], - "summary": "Reset Request-Limit", - "responses": { - "200": { - "description": "", - "content": { - "text/plain; x-version=2.0": { - "schema": { - "type": "string" - } - }, - "application/json; x-version=2.0": { - "schema": { - "type": "string" - } - }, - "text/json; x-version=2.0": { - "schema": { - "type": "string" - } - } - } - } - } - } - }, - "/v2/Settings/RequestLimits/{RequestType}": { - "patch": { - "tags": [ - "Settings" - ], - "summary": "Updates a Request-Limit value", - "parameters": [ - { - "name": "RequestType", - "in": "path", - "description": "Type of Request", - "required": true, - "schema": { - "$ref": "#/components/schemas/RequestType" - } - } - ], - "requestBody": { - "description": "New limit in Requests/Minute", - "content": { - "application/json-patch+json; x-version=2.0": { - "schema": { - "type": "integer", - "format": "int32" - } - }, - "application/json; x-version=2.0": { - "schema": { - "type": "integer", - "format": "int32" - } - }, - "text/json; x-version=2.0": { - "schema": { - "type": "integer", - "format": "int32" - } - }, - "application/*+json; x-version=2.0": { - "schema": { - "type": "integer", - "format": "int32" - } - } - } - }, - "responses": { - "200": { - "description": "" - }, - "400": { - "description": "Limit needs to be greater than 0", - "content": { - "text/plain; x-version=2.0": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - }, - "application/json; x-version=2.0": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - }, - "text/json; x-version=2.0": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - } - } - } - }, - "delete": { - "tags": [ - "Settings" - ], - "summary": "Reset Request-Limit", - "parameters": [ - { - "name": "RequestType", - "in": "path", - "required": true, - "schema": { - "$ref": "#/components/schemas/RequestType" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "text/plain; x-version=2.0": { - "schema": { - "type": "string" - } - }, - "application/json; x-version=2.0": { - "schema": { - "type": "string" - } - }, - "text/json; x-version=2.0": { - "schema": { - "type": "string" - } - } - } - } - } } }, "/v2/Settings/ImageCompressionLevel": { @@ -4313,17 +4129,6 @@ }, "additionalProperties": { } }, - "RequestType": { - "enum": [ - "Default", - "MangaDexFeed", - "MangaImage", - "MangaCover", - "MangaDexImage", - "MangaInfo" - ], - "type": "string" - }, "TrangaSettings": { "type": "object", "properties": { @@ -4356,37 +4161,6 @@ "type": "integer", "format": "int32" }, - "requestLimits": { - "type": "object", - "properties": { - "Default": { - "type": "integer", - "format": "int32" - }, - "MangaDexFeed": { - "type": "integer", - "format": "int32" - }, - "MangaImage": { - "type": "integer", - "format": "int32" - }, - "MangaCover": { - "type": "integer", - "format": "int32" - }, - "MangaDexImage": { - "type": "integer", - "format": "int32" - }, - "MangaInfo": { - "type": "integer", - "format": "int32" - } - }, - "additionalProperties": false, - "nullable": true - }, "downloadLanguage": { "type": "string", "nullable": true diff --git a/docker-compose.local.yaml b/docker-compose.local.yaml index 2fc3a8c..80c61dd 100644 --- a/docker-compose.local.yaml +++ b/docker-compose.local.yaml @@ -22,7 +22,7 @@ services: max-size: "10m" max-file: "5" tranga-pg: - image: postgres:latest + image: postgres:17 container_name: tranga-pg ports: - "5432:5432"