Use DelegatingHandler for RateLimits

API/Controllers/SettingsController.cs

@@ -60,17 +60,6 @@ public class SettingsController() : Controller
         return TypedResults.Ok();
     }
     
-    /// <summary>
-    /// Get all Request-Limits
-    /// </summary>
-    /// <response code="200"></response>
-    [HttpGet("RequestLimits")]
-    [ProducesResponseType<Dictionary<RequestType,int>>(Status200OK, "application/json")]
-    public Ok<Dictionary<RequestType,int>> GetRequestLimits()
-    {
-        return TypedResults.Ok(Tranga.Settings.RequestLimits);
-    }
-    
     /// <summary>
     /// Update all Request-Limits to new values
     /// </summary>
@@ -82,48 +71,6 @@ public class SettingsController() : Controller
         return TypedResults.StatusCode(Status501NotImplemented);
     }
     
-    /// <summary>
-    /// Updates a Request-Limit value
-    /// </summary>
-    /// <param name="RequestType">Type of Request</param>
-    /// <param name="requestLimit">New limit in Requests/Minute</param>
-    /// <response code="200"></response>
-    /// <response code="400">Limit needs to be greater than 0</response>
-    [HttpPatch("RequestLimits/{RequestType}")]
-    [ProducesResponseType(Status200OK)]
-    [ProducesResponseType(Status400BadRequest)]
-    public Results<Ok, BadRequest> SetRequestLimit(RequestType RequestType, [FromBody]int requestLimit)
-    {
-        if (requestLimit <= 0)
-            return TypedResults.BadRequest();
-        Tranga.Settings.SetRequestLimit(RequestType, requestLimit);
-        return TypedResults.Ok();
-    }
-    
-    /// <summary>
-    /// Reset Request-Limit
-    /// </summary>
-    /// <response code="200"></response>
-    [HttpDelete("RequestLimits/{RequestType}")]
-    [ProducesResponseType<string>(Status200OK)]
-    public Ok ResetRequestLimits(RequestType RequestType)
-    {
-        Tranga.Settings.SetRequestLimit(RequestType, TrangaSettings.DefaultRequestLimits[RequestType]);
-        return TypedResults.Ok();
-    }
-    
-    /// <summary>
-    /// Reset Request-Limit
-    /// </summary>
-    /// <response code="200"></response>
-    [HttpDelete("RequestLimits")]
-    [ProducesResponseType<string>(Status200OK)]
-    public Ok ResetRequestLimits()
-    {
-        Tranga.Settings.ResetRequestLimits();
-        return TypedResults.Ok();
-    }
-    
     /// <summary>
     /// Returns Level of Image-Compression for Images
     /// </summary>
@@ -260,12 +207,12 @@ public class SettingsController() : Controller
     [HttpPost("FlareSolverr/Test")]
     [ProducesResponseType(Status200OK)]
     [ProducesResponseType(Status500InternalServerError)]
-    public Results<Ok, InternalServerError> TestFlareSolverrReachable()
+    public async Task<Results<Ok, InternalServerError>> TestFlareSolverrReachable()
     {
         const string knownProtectedUrl = "https://prowlarr.servarr.com/v1/ping";
-        FlareSolverrDownloadClient client = new();
-        RequestResult result = client.MakeRequestInternal(knownProtectedUrl);
-        return (int)result.statusCode >= 200 && (int)result.statusCode < 300 ? TypedResults.Ok() : TypedResults.InternalServerError(); 
+        FlareSolverrDownloadClient client = new(new ());
+        HttpResponseMessage result = await client.MakeRequest(knownProtectedUrl, RequestType.Default);
+        return (int)result.StatusCode >= 200 && (int)result.StatusCode < 300 ? TypedResults.Ok() : TypedResults.InternalServerError(); 
     }
 
     /// <summary>

API/MangaConnectors/MangaConnector.cs

@@ -5,9 +5,7 @@ using API.MangaDownloadClients;
 using API.Schema.MangaContext;
 using log4net;
 using Microsoft.EntityFrameworkCore;
-using Newtonsoft.Json;
 using SixLabors.ImageSharp;
-using SixLabors.ImageSharp.Formats.Jpeg;
 using SixLabors.ImageSharp.Processing;
 
 namespace API.MangaConnectors;
@@ -15,7 +13,7 @@ namespace API.MangaConnectors;
 [PrimaryKey("Name")]
 public abstract class MangaConnector(string name, string[] supportedLanguages, string[] baseUris, string iconUrl)
 {
-    [NotMapped] internal DownloadClient downloadClient { get; init; } = null!;
+    [NotMapped] internal IDownloadClient downloadClient { get; init; } = null!;
     [NotMapped] protected ILog Log { get; init; } = LogManager.GetLogger(name);
     [StringLength(32)] public string Name { get; init; } = name;
     [StringLength(8)] public string[] SupportedLanguages { get; init; } = supportedLanguages;
@@ -50,14 +48,14 @@ public abstract class MangaConnector(string name, string[] supportedLanguages, s
         if (File.Exists(saveImagePath))
             return filename;
         
-        RequestResult coverResult = downloadClient.MakeRequest(mangaId.Obj.CoverUrl, RequestType.MangaCover, $"https://{match.Groups[1].Value}");
-        if ((int)coverResult.statusCode < 200 || (int)coverResult.statusCode >= 300)
+        HttpResponseMessage coverResult = downloadClient.MakeRequest(mangaId.Obj.CoverUrl, RequestType.MangaCover, $"https://{match.Groups[1].Value}").Result;
+        if ((int)coverResult.StatusCode < 200 || (int)coverResult.StatusCode >= 300)
             return SaveCoverImageToCache(mangaId, --retries);
             
         try
         {
             using MemoryStream ms = new();
-            coverResult.result.CopyTo(ms);
+            coverResult.Content.ReadAsStream().CopyTo(ms);
             byte[] imageBytes = ms.ToArray();
             Directory.CreateDirectory(TrangaSettings.CoverImageCacheOriginal);
             File.WriteAllBytes(saveImagePath, imageBytes);

API/MangaConnectors/MangaDex.cs

@@ -35,14 +35,14 @@ public class MangaDex : MangaConnector
                 $"&includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author&includes%5B%5D=artist&includes%5B%5D=tag'";
             offset += Limit;
 
-            RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed);
-            if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
+            HttpResponseMessage result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed).Result;
+            if ((int)result.StatusCode < 200 || (int)result.StatusCode >= 300)
             {
                 Log.Error("Request failed");
                 return [];
             }
 
-            using StreamReader sr = new (result.result);
+            using StreamReader sr = new (result.Content.ReadAsStream());
             JObject jObject = JObject.Parse(sr.ReadToEnd());
 
             if (jObject.Value<string>("result") != "ok")
@@ -96,14 +96,14 @@ public class MangaDex : MangaConnector
             $"https://api.mangadex.org/manga/{mangaIdOnSite}" +
             $"?includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author&includes%5B%5D=artist&includes%5B%5D=tag'";
         
-        RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed);
-        if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
+        HttpResponseMessage result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed).Result;
+        if ((int)result.StatusCode < 200 || (int)result.StatusCode >= 300)
         {
             Log.Error("Request failed");
             return null;
         }
 
-        using StreamReader sr = new (result.result);
+        using StreamReader sr = new (result.Content.ReadAsStream());
         JObject jObject = JObject.Parse(sr.ReadToEnd());
 
         if (jObject.Value<string>("result") != "ok")
@@ -138,14 +138,14 @@ public class MangaDex : MangaConnector
                 $"contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&includeFutureUpdates=0&includes%5B%5D=";
             offset += Limit;
 
-            RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed);
-            if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
+            HttpResponseMessage result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed).Result;
+            if ((int)result.StatusCode < 200 || (int)result.StatusCode >= 300)
             {
                 Log.Error("Request failed");
                 return [];
             }
 
-            using StreamReader sr = new (result.result);
+            using StreamReader sr = new (result.Content.ReadAsStream());
             JObject jObject = JObject.Parse(sr.ReadToEnd());
 
             if (jObject.Value<string>("result") != "ok")
@@ -191,14 +191,14 @@ public class MangaDex : MangaConnector
         string id = match.Groups[1].Value;
         string requestUrl = $"https://api.mangadex.org/at-home/server/{id}";
         
-        RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.Default);
-        if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
+        HttpResponseMessage result = downloadClient.MakeRequest(requestUrl, RequestType.Default).Result;
+        if ((int)result.StatusCode < 200 || (int)result.StatusCode >= 300)
         {
             Log.Error("Request failed");
             return [];
         }
 
-        using StreamReader sr = new (result.result);
+        using StreamReader sr = new (result.Content.ReadAsStream());
         JObject jObject = JObject.Parse(sr.ReadToEnd());
         
         if (jObject.Value<string>("result") != "ok")

API/MangaConnectors/MangaPark.cs

@@ -36,7 +36,7 @@ public class MangaPark : MangaConnector
         for (int page = 1;; page++) // break; in loop
         {
             Uri searchUri = new(baseUri, $"search?word={HttpUtility.UrlEncode(mangaSearchName)}&lang={Tranga.Settings.DownloadLanguage}&page={page}");
-            if (downloadClient.MakeRequest(searchUri.ToString(), RequestType.Default) is { statusCode: >= HttpStatusCode.OK and < HttpStatusCode.Ambiguous } result)
+            if (downloadClient.MakeRequest(searchUri.ToString(), RequestType.Default).Result is { StatusCode: >= HttpStatusCode.OK and < HttpStatusCode.Ambiguous } result)
             {
                 HtmlDocument document = result.CreateDocument();
                 // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract HAP sucks with nullable types
@@ -73,8 +73,8 @@ public class MangaPark : MangaConnector
 
     public override (Manga, MangaConnectorId<Manga>)? GetMangaFromUrl(string url)
     {
-        if (downloadClient.MakeRequest(url, RequestType.Default) is
-            { statusCode: >= HttpStatusCode.OK and < HttpStatusCode.Ambiguous } result)
+        if (downloadClient.MakeRequest(url, RequestType.Default).Result is
+            { StatusCode: >= HttpStatusCode.OK and < HttpStatusCode.Ambiguous } result)
         {
             HtmlDocument document= result.CreateDocument();
 
@@ -145,8 +145,8 @@ public class MangaPark : MangaConnector
 
         List<(Chapter, MangaConnectorId<Chapter>)> ret = [];
         
-        if (downloadClient.MakeRequest(requestUri.ToString(), RequestType.Default) is
-            { statusCode: >= HttpStatusCode.OK and < HttpStatusCode.Ambiguous } result)
+        if (downloadClient.MakeRequest(requestUri.ToString(), RequestType.Default).Result is
+            { StatusCode: >= HttpStatusCode.OK and < HttpStatusCode.Ambiguous } result)
         {
             HtmlDocument document= result.CreateDocument();
 
@@ -220,8 +220,8 @@ public class MangaPark : MangaConnector
         Log.Debug($"Using domain {domain}");
         Uri baseUri = new ($"https://{domain}/");
         Uri requestUri = new (baseUri, $"title/{chapterId.IdOnConnectorSite}");
-        if (downloadClient.MakeRequest(requestUri.ToString(), RequestType.Default) is
-            { statusCode: >= HttpStatusCode.OK and < HttpStatusCode.Ambiguous } result)
+        if (downloadClient.MakeRequest(requestUri.ToString(), RequestType.Default).Result is
+            { StatusCode: >= HttpStatusCode.OK and < HttpStatusCode.Ambiguous } result)
         {
             HtmlDocument document = result.CreateDocument();
 
@@ -240,10 +240,10 @@ public class MangaPark : MangaConnector
 
 internal static class MangaParkHelper
 {
-    internal static HtmlDocument CreateDocument(this RequestResult result)
+    internal static HtmlDocument CreateDocument(this HttpResponseMessage result)
     {
         HtmlDocument document = new();
-        StreamReader sr = new (result.result);
+        StreamReader sr = new (result.Content.ReadAsStream());
         string htmlStr = sr.ReadToEnd().Replace("q:key", "qkey");
         document.LoadHtml(htmlStr);
         

API/MangaConnectors/Mangaworld.cs

@@ -30,11 +30,11 @@ public sealed class Mangaworld : MangaConnector
         Uri baseUri = new ("https://www.mangaworld.cx/");
         Uri searchUrl = new (baseUri, "archive?keyword=" + HttpUtility.UrlEncode(mangaSearchName));
 
-        RequestResult res = downloadClient.MakeRequest(searchUrl.ToString(), RequestType.Default);
-        if ((int)res.statusCode < 200 || (int)res.statusCode >= 300)
+        HttpResponseMessage res = downloadClient.MakeRequest(searchUrl.ToString(), RequestType.Default).Result;
+        if ((int)res.StatusCode < 200 || (int)res.StatusCode >= 300)
             return [];
 
-        using StreamReader sr = new (res.result);
+        using StreamReader sr = new (res.Content.ReadAsStream());
         string html = sr.ReadToEnd();
 
         HtmlDocument doc = new ();
@@ -85,11 +85,11 @@ public sealed class Mangaworld : MangaConnector
         string slug = parts[1];
 
         string url = $"https://www.mangaworld.cx/manga/{id}/{slug}/";
-        RequestResult res = downloadClient.MakeRequest(url, RequestType.MangaInfo);
-        if ((int)res.statusCode < 200 || (int)res.statusCode >= 300)
+        HttpResponseMessage res = downloadClient.MakeRequest(url, RequestType.MangaInfo).Result;
+        if ((int)res.StatusCode < 200 || (int)res.StatusCode >= 300)
             return null;
 
-        using StreamReader sr = new (res.result);
+        using StreamReader sr = new (res.Content.ReadAsStream());
         string html = sr.ReadToEnd();
 
         HtmlDocument doc = new ();
@@ -175,11 +175,11 @@ public sealed class Mangaworld : MangaConnector
     {
         string url = EnsureListStyle(chapterId.WebsiteUrl ?? $"https://www.mangaworld.cx/manga/{chapterId.IdOnConnectorSite}");
 
-        RequestResult res = downloadClient.MakeRequest(url, RequestType.MangaInfo);
-        if ((int)res.statusCode < 200 || (int)res.statusCode >= 300)
+        HttpResponseMessage res = downloadClient.MakeRequest(url, RequestType.MangaInfo).Result;
+        if ((int)res.StatusCode < 200 || (int)res.StatusCode >= 300)
             return [];
 
-        using StreamReader sr = new (res.result);
+        using StreamReader sr = new (res.Content.ReadAsStream());
         string html = sr.ReadToEnd();
 
         Uri baseUri = new (url);
@@ -354,20 +354,20 @@ public sealed class Mangaworld : MangaConnector
         baseUri = new (seriesUrl);
 
         // 1) tenta client "Default"
-        RequestResult res = downloadClient.MakeRequest(seriesUrl, RequestType.Default);
-        if ((int)res.statusCode >= 200 && (int)res.statusCode < 300)
+        HttpResponseMessage res = downloadClient.MakeRequest(seriesUrl, RequestType.Default).Result;
+        if ((int)res.StatusCode >= 200 && (int)res.StatusCode < 300)
         {
-            using StreamReader sr = new (res.result);
+            using StreamReader sr = new (res.Content.ReadAsStream());
             string html = sr.ReadToEnd();
             if (!LooksLikeChallenge(html))
                 return html;
         }
 
         // 2) fallback: client “MangaInfo” (proxy/Flare se configurato)
-        RequestResult res2 = downloadClient.MakeRequest(seriesUrl, RequestType.MangaInfo);
-        if ((int)res2.statusCode >= 200 && (int)res2.statusCode < 300)
+        HttpResponseMessage res2 = downloadClient.MakeRequest(seriesUrl, RequestType.MangaInfo).Result;
+        if ((int)res2.StatusCode >= 200 && (int)res2.StatusCode < 300)
         {
-            using StreamReader sr2 = new StreamReader(res2.result);
+            using StreamReader sr2 = new StreamReader(res2.Content.ReadAsStream());
             return sr2.ReadToEnd();
         }
 

API/MangaDownloadClients/DownloadClient.cs

@@ -1,56 +1,6 @@
-using System.Collections.Concurrent;
-using System.Net;
-using log4net;
+namespace API.MangaDownloadClients;
 
-namespace API.MangaDownloadClients;
-
-public abstract class DownloadClient
+public interface IDownloadClient
 {
-    private static readonly ConcurrentDictionary<RequestType, DateTime> LastExecutedRateLimit = new();
-    protected ILog Log { get; init; }
-
-    protected DownloadClient()
-    {
-        this.Log = LogManager.GetLogger(GetType());
-    }
-    
-    // TODO Requests still go too fast across threads!
-    public RequestResult MakeRequest(string url, RequestType requestType, string? referrer = null, string? clickButton = null)
-    {
-        Log.Debug($"Requesting {requestType} {url}");
-        
-        // If we don't have a RequestLimit set for a Type, use the default one
-        if (!Tranga.Settings.RequestLimits.ContainsKey(requestType))
-            requestType = RequestType.Default;
-
-        int rateLimit = Tranga.Settings.RequestLimits[requestType];
-        // TODO this probably needs a better check whether the useragent matches...
-        // If the UserAgent is the default one, do not exceed the default request-limits.
-        if (Tranga.Settings.UserAgent == TrangaSettings.DefaultUserAgent && rateLimit > TrangaSettings.DefaultRequestLimits[requestType])
-            rateLimit = TrangaSettings.DefaultRequestLimits[requestType];
-        
-        // Apply the delay
-        TimeSpan timeBetweenRequests = TimeSpan.FromMinutes(1).Divide(rateLimit);
-        DateTime now = DateTime.UtcNow;
-        LastExecutedRateLimit.TryAdd(requestType, now.Subtract(timeBetweenRequests));
-
-        TimeSpan rateLimitTimeout = timeBetweenRequests.Subtract(now.Subtract(LastExecutedRateLimit[requestType]));
-        Log.Debug($"Request limit {requestType} {rateLimit}/Minute timeBetweenRequests: {timeBetweenRequests:ss'.'fffff}");
-
-        if (rateLimitTimeout > TimeSpan.Zero)
-        {
-            Log.Info($"Waiting {rateLimitTimeout} for {url}");
-            Thread.Sleep(rateLimitTimeout);
-        }
-
-        // Make the request
-        RequestResult result = MakeRequestInternal(url, referrer, clickButton);
-        
-        // Update the time the last request was made
-        LastExecutedRateLimit[requestType] = DateTime.UtcNow;
-        Log.Debug($"Result {url}: {result}");
-        return result;
-    }
-
-    internal abstract RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null);
+    internal Task<HttpResponseMessage> MakeRequest(string url, RequestType requestType, string? referrer = null);
 }

API/MangaDownloadClients/FlareSolverrDownloadClient.cs

@@ -1,25 +1,26 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Net;
-using System.Text;
 using System.Text.Json;
 using HtmlAgilityPack;
+using log4net;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 
 namespace API.MangaDownloadClients;
 
-public class FlareSolverrDownloadClient : DownloadClient
+public class FlareSolverrDownloadClient(HttpClient client) : IDownloadClient
 {
-    internal override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null)
+    private ILog Log { get; } = LogManager.GetLogger(typeof(FlareSolverrDownloadClient));
+
+    public async Task<HttpResponseMessage> MakeRequest(string url, RequestType requestType, string? referrer = null)
     {
-        if (clickButton is not null)
-            Log.Warn("Client can not click button");
+        Log.Debug($"Using {typeof(FlareSolverrDownloadClient).FullName} for {url}");
         if(referrer is not null)
             Log.Warn("Client can not set referrer");
         if (Tranga.Settings.FlareSolverrUrl == string.Empty)
         {
             Log.Error("FlareSolverr URL is empty");
-            return new(HttpStatusCode.InternalServerError, null, Stream.Null);
+            return new(HttpStatusCode.InternalServerError);
         }
         
         Uri flareSolverrUri = new (Tranga.Settings.FlareSolverrUrl);
@@ -29,13 +30,6 @@ public class FlareSolverrDownloadClient : DownloadClient
                 Path = "v1"
             }.Uri;
         
-        HttpClient client = new()
-        {
-            Timeout = TimeSpan.FromSeconds(10),
-            DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrHigher,
-            DefaultRequestHeaders = { { "User-Agent", Tranga.Settings.UserAgent } }
-        };
-
         JObject requestObj = new()
         {
             ["cmd"] = "request.get",
@@ -52,12 +46,12 @@ public class FlareSolverrDownloadClient : DownloadClient
         HttpResponseMessage? response;
         try
         {
-            response = client.Send(requestMessage);
+            response = await client.SendAsync(requestMessage);
         }
         catch (HttpRequestException e)
         {
             Log.Error(e);
-            return new (HttpStatusCode.Unused, null, Stream.Null);
+            return new (HttpStatusCode.InternalServerError);
         }
 
         if (!response.IsSuccessStatusCode)
@@ -74,7 +68,7 @@ public class FlareSolverrDownloadClient : DownloadClient
                       $"{response.Version}\n" +
                       $"Headers:\n\t{string.Join("\n\t", response.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
                       $"{response.Content.ReadAsStringAsync().Result}");
-            return new (response.StatusCode,  null, Stream.Null);
+            return response;
         }
 
         string responseString = response.Content.ReadAsStringAsync().Result;
@@ -82,51 +76,45 @@ public class FlareSolverrDownloadClient : DownloadClient
         if (!IsInCorrectFormat(responseObj, out string? reason))
         {
             Log.Error($"Wrong format: {reason}");
-            return new(HttpStatusCode.Unused, null, Stream.Null);
+            return new(HttpStatusCode.InternalServerError);
         }
 
         string statusResponse = responseObj["status"]!.Value<string>()!;
         if (statusResponse != "ok")
         {
             Log.Debug($"Status is not ok: {statusResponse}");
-            return new(HttpStatusCode.Unused, null, Stream.Null);
+            return new(HttpStatusCode.InternalServerError);
         }
         JObject solution = (responseObj["solution"] as JObject)!;
 
         if (!Enum.TryParse(solution["status"]!.Value<int>().ToString(), out HttpStatusCode statusCode))
         {
             Log.Error($"Wrong format: Cant parse status code: {solution["status"]!.Value<int>()}");
-            return new(HttpStatusCode.Unused, null, Stream.Null);
+            return new(HttpStatusCode.InternalServerError);
         }
         if (statusCode < HttpStatusCode.OK || statusCode >= HttpStatusCode.MultipleChoices)
         {
             Log.Debug($"Status is: {statusCode}");
-            return new(statusCode, null, Stream.Null);
+            return new (statusCode);
         }
 
         if (solution["response"]!.Value<string>() is not { } htmlString)
         {
             Log.Error("Wrong format: Cant find response in solution");
-            return new(HttpStatusCode.Unused, null, Stream.Null);
+            return new(HttpStatusCode.InternalServerError);
         }
 
-        if (IsJson(htmlString, out HtmlDocument document, out string? json))
+        if (IsJson(htmlString, out string? json))
         {
-            MemoryStream ms = new();
-            ms.Write(Encoding.UTF8.GetBytes(json));
-            ms.Position = 0;
-            return new(statusCode, document, ms);
+            return new(statusCode) { Content = new StringContent(json) };
         }
         else
         {
-            MemoryStream ms = new();
-            ms.Write(Encoding.UTF8.GetBytes(htmlString));
-            ms.Position = 0;
-            return new(statusCode, document, ms);
+            return new(statusCode) { Content = new StringContent(htmlString) };
         }
     }
 
-    private bool IsInCorrectFormat(JObject responseObj, [NotNullWhen(false)]out string? reason)
+    private static bool IsInCorrectFormat(JObject responseObj, [NotNullWhen(false)]out string? reason)
     {
         reason = null;
         if (!responseObj.ContainsKey("status"))
@@ -157,10 +145,10 @@ public class FlareSolverrDownloadClient : DownloadClient
         return true;
     }
 
-    private bool IsJson(string htmlString, out HtmlDocument document, [NotNullWhen(true)]out string? jsonString)
+    private static bool IsJson(string htmlString, [NotNullWhen(true)]out string? jsonString)
     {
         jsonString = null;
-        document = new();
+        HtmlDocument document = new();
         document.LoadHtml(htmlString);
 
         HtmlNode pre = document.DocumentNode.SelectSingleNode("//pre");

API/MangaDownloadClients/HttpDownloadClient.cs

@@ -1,47 +1,41 @@
 using System.Net;
-using HtmlAgilityPack;
+using log4net;
 
 namespace API.MangaDownloadClients;
 
-internal class HttpDownloadClient : DownloadClient
+internal class HttpDownloadClient : IDownloadClient
 {
-    private static readonly FlareSolverrDownloadClient FlareSolverrDownloadClient = new();
-    internal override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null)
+    private static readonly HttpClient Client = new(handler: Tranga.RateLimitHandler)
+    {
+        Timeout = TimeSpan.FromSeconds(10),
+        DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrHigher,
+        DefaultRequestHeaders = { { "User-Agent", Tranga.Settings.UserAgent } }
+    };
+    private static readonly FlareSolverrDownloadClient FlareSolverrDownloadClient = new(Client);
+    private ILog Log { get; } = LogManager.GetLogger(typeof(HttpDownloadClient));
+    
+    public async Task<HttpResponseMessage> MakeRequest(string url, RequestType requestType, string? referrer = null)
     {
         Log.Debug($"Using {typeof(HttpDownloadClient).FullName} for {url}");
-        if (clickButton is not null)
-            Log.Warn("Client can not click button");
-        HttpClient client = new();
-        client.Timeout = TimeSpan.FromSeconds(10);
-        client.DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrHigher;
-        client.DefaultRequestHeaders.Add("User-Agent", Tranga.Settings.UserAgent);
-        HttpResponseMessage? response;
-        Uri uri = new(url);
-        HttpRequestMessage requestMessage = new(HttpMethod.Get, uri);
+        HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
         if (referrer is not null)
             requestMessage.Headers.Referrer = new (referrer);
         Log.Debug($"Requesting {url}");
+        
         try
         {
-            response = client.Send(requestMessage);
-        }
-        catch (HttpRequestException e)
-        {
-            Log.Error(e);
-            return new (HttpStatusCode.Unused, null, Stream.Null);
-        }
+            HttpResponseMessage response = await Client.SendAsync(requestMessage);
+            Log.Debug($"Request {url} returned {(int)response.StatusCode} {response.StatusCode}");
+            if(response.IsSuccessStatusCode)
+                return response;
 
-        if (!response.IsSuccessStatusCode)
-        {
-            Log.Debug($"Request returned status code {(int)response.StatusCode} {response.StatusCode}");
             if (response.Headers.Server.Any(s =>
                     (s.Product?.Name ?? "").Contains("cloudflare", StringComparison.InvariantCultureIgnoreCase)))
             {
                 Log.Debug("Retrying with FlareSolverr!");
-                return FlareSolverrDownloadClient.MakeRequestInternal(url, referrer, clickButton);
+                return await FlareSolverrDownloadClient.MakeRequest(url, requestType, referrer);
             }
-            else
-            {
+            
             Log.Debug($"Request returned status code {(int)response.StatusCode} {response.StatusCode}:\n" +
                       $"=====\n" +
                       $"Request:\n" +
@@ -54,36 +48,12 @@ internal class HttpDownloadClient : DownloadClient
                       $"{response.Version}\n" +
                       $"Headers:\n\t{string.Join("\n\t", response.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
                       $"{response.Content.ReadAsStringAsync().Result}");
+            return new(HttpStatusCode.InternalServerError);
         }
-        }
-
-        Stream stream;
-        try
-        {
-            stream = response.Content.ReadAsStream();
-        }
-        catch (Exception e)
+        catch (HttpRequestException e)
         {
             Log.Error(e);
-            return new (HttpStatusCode.Unused, null, Stream.Null);
-        }
-
-        HtmlDocument? document = null;
-
-        if (response.Content.Headers.ContentType?.MediaType == "text/html")
-        {
-            StreamReader reader = new (stream);
-            document = new ();
-            document.LoadHtml(reader.ReadToEnd());
-            stream.Position = 0;
-        }
-
-        // Request has been redirected to another page. For example, it redirects directly to the results when there is only 1 result
-        if (response.RequestMessage is not null && response.RequestMessage.RequestUri is not null && response.RequestMessage.RequestUri != uri)
-        {
-            return new (response.StatusCode, document, stream, true, response.RequestMessage.RequestUri.AbsoluteUri);
-        }
-
-        return new (response.StatusCode, document, stream);
+            return new(HttpStatusCode.InternalServerError);
+        }
     }
 }

API/MangaDownloadClients/RateLimitHandler.cs

@@ -0,0 +1,31 @@
+using System.Net;
+using System.Threading.RateLimiting;
+using log4net;
+
+namespace API.MangaDownloadClients;
+
+public class RateLimitHandler() : DelegatingHandler(new HttpClientHandler())
+{
+    private ILog Log { get; init; } = LogManager.GetLogger(typeof(RateLimitHandler));
+
+    private readonly RateLimiter _limiter = new SlidingWindowRateLimiter(new ()
+    {
+        AutoReplenishment = true,
+        PermitLimit = 240,
+        QueueLimit = 120,
+        QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
+        SegmentsPerWindow = 60,
+        Window = TimeSpan.FromSeconds(60)
+    });
+    
+    protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+    {
+        Log.Debug($"Requesting lease {request.RequestUri}");
+        using RateLimitLease lease = await _limiter.AcquireAsync(permitCount: 1, cancellationToken);
+        Log.Debug($"lease {lease.IsAcquired} {request.RequestUri}");
+
+        return lease.IsAcquired
+            ? await base.SendAsync(request, cancellationToken)
+            : new (HttpStatusCode.TooManyRequests);
+    }
+}

API/MangaDownloadClients/RequestResult.cs

@@ -1,33 +0,0 @@
-using System.Net;
-using HtmlAgilityPack;
-
-namespace API.MangaDownloadClients;
-
-public struct RequestResult
-{
-    public HttpStatusCode statusCode { get; }
-    public Stream result { get; }
-    public bool hasBeenRedirected { get; }
-    public string? redirectedToUrl { get; }
-    public HtmlDocument? htmlDocument { get; }
-
-    public RequestResult(HttpStatusCode statusCode, HtmlDocument? htmlDocument, Stream result)
-    {
-        this.statusCode = statusCode;
-        this.htmlDocument = htmlDocument;
-        this.result = result;
-    }
-
-    public RequestResult(HttpStatusCode statusCode, HtmlDocument? htmlDocument, Stream result, bool hasBeenRedirected, string redirectedTo)
-        : this(statusCode, htmlDocument, result)
-    {
-        this.hasBeenRedirected = hasBeenRedirected;
-        redirectedToUrl = redirectedTo;
-    }
-
-    public override string ToString()
-    {
-        return
-            $"{(int)statusCode} {statusCode.ToString()} {(hasBeenRedirected ? "Redirected: " : "")} {redirectedToUrl}";
-    }
-}

API/Tranga.cs

@@ -1,6 +1,7 @@
 using System.Collections.Concurrent;
 using System.Diagnostics.CodeAnalysis;
 using API.MangaConnectors;
+using API.MangaDownloadClients;
 using API.Schema.LibraryContext;
 using API.Schema.MangaContext;
 using API.Schema.MangaContext.MetadataFetchers;
@@ -35,6 +36,8 @@ public static class Tranga
     internal static readonly CleanupMangaconnectorIdsWithoutConnector CleanupMangaconnectorIdsWithoutConnector = new();
     // ReSharper restore MemberCanBePrivate.Global
 
+    internal static readonly RateLimitHandler RateLimitHandler = new();
+
     internal static void StartupTasks()
     {
         AddWorker(SendNotificationsWorker);

API/TrangaSettings.cs

@@ -1,5 +1,4 @@
 using System.Runtime.InteropServices;
-using API.MangaDownloadClients;
 using API.Workers;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
@@ -38,17 +37,6 @@ public struct TrangaSettings()
     /// </summary>
     public string ChapterNamingScheme { get; set; } = "%M - ?V(Vol.%V )Ch.%C?T( - %T)";
     public int WorkCycleTimeoutMs { get; set; } = 20000;
-    [JsonIgnore]
-    internal static readonly Dictionary<RequestType, int> DefaultRequestLimits = new ()
-    {
-        {RequestType.MangaInfo, 360},
-        {RequestType.MangaDexFeed, 360},
-        {RequestType.MangaDexImage, 60},
-        {RequestType.MangaImage, 240},
-        {RequestType.MangaCover, 60},
-        {RequestType.Default, 360}
-    };
-    public Dictionary<RequestType, int> RequestLimits { get; set; } = DefaultRequestLimits;
 
     public string DownloadLanguage { get; set; } = "en";
     
@@ -78,18 +66,6 @@ public struct TrangaSettings()
         Save();
     }
 
-    public void SetRequestLimit(RequestType type, int value)
-    {
-        this.RequestLimits[type] = value;
-        Save();
-    }
-
-    public void ResetRequestLimits()
-    {
-        this.RequestLimits = DefaultRequestLimits;
-        Save();
-    }
-
     public void UpdateImageCompression(int value)
     {
         this.ImageCompression = value;

API/Workers/MangaDownloadWorkers/DownloadChapterFromMangaconnectorWorker.cs

@@ -286,14 +286,14 @@ public class DownloadChapterFromMangaconnectorWorker(MangaConnectorId<Chapter> c
     private Stream? DownloadImage(string imageUrl)
     {
         HttpDownloadClient downloadClient = new();
-        RequestResult requestResult = downloadClient.MakeRequest(imageUrl, RequestType.MangaImage);
+        HttpResponseMessage requestResult = downloadClient.MakeRequest(imageUrl, RequestType.MangaImage).Result;
 
-        if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
+        if ((int)requestResult.StatusCode < 200 || (int)requestResult.StatusCode >= 300)
             return null;
-        if (requestResult.result == Stream.Null)
+        if (requestResult.Content.ReadAsStream() == Stream.Null)
             return null;
         
-        return ProcessImage(requestResult.result, out Stream processedImage) ? processedImage : null;
+        return ProcessImage(requestResult.Content.ReadAsStream(), out Stream processedImage) ? processedImage : null;
     }
 
     public override string ToString() => $"{base.ToString()} {_mangaConnectorIdId}";

API/openapi/API_v2.json

@@ -2764,51 +2764,6 @@
       }
     },
     "/v2/Settings/RequestLimits": {
-      "get": {
-        "tags": [
-          "Settings"
-        ],
-        "summary": "Get all Request-Limits",
-        "responses": {
-          "200": {
-            "description": "",
-            "content": {
-              "application/json; x-version=2.0": {
-                "schema": {
-                  "type": "object",
-                  "properties": {
-                    "Default": {
-                      "type": "integer",
-                      "format": "int32"
-                    },
-                    "MangaDexFeed": {
-                      "type": "integer",
-                      "format": "int32"
-                    },
-                    "MangaImage": {
-                      "type": "integer",
-                      "format": "int32"
-                    },
-                    "MangaCover": {
-                      "type": "integer",
-                      "format": "int32"
-                    },
-                    "MangaDexImage": {
-                      "type": "integer",
-                      "format": "int32"
-                    },
-                    "MangaInfo": {
-                      "type": "integer",
-                      "format": "int32"
-                    }
-                  },
-                  "additionalProperties": false
-                }
-              }
-            }
-          }
-        }
-      },
       "patch": {
         "tags": [
           "Settings"
@@ -2820,145 +2775,6 @@
             "description": "Not Implemented"
           }
         }
-      },
-      "delete": {
-        "tags": [
-          "Settings"
-        ],
-        "summary": "Reset Request-Limit",
-        "responses": {
-          "200": {
-            "description": "",
-            "content": {
-              "text/plain; x-version=2.0": {
-                "schema": {
-                  "type": "string"
-                }
-              },
-              "application/json; x-version=2.0": {
-                "schema": {
-                  "type": "string"
-                }
-              },
-              "text/json; x-version=2.0": {
-                "schema": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    },
-    "/v2/Settings/RequestLimits/{RequestType}": {
-      "patch": {
-        "tags": [
-          "Settings"
-        ],
-        "summary": "Updates a Request-Limit value",
-        "parameters": [
-          {
-            "name": "RequestType",
-            "in": "path",
-            "description": "Type of Request",
-            "required": true,
-            "schema": {
-              "$ref": "#/components/schemas/RequestType"
-            }
-          }
-        ],
-        "requestBody": {
-          "description": "New limit in Requests/Minute",
-          "content": {
-            "application/json-patch+json; x-version=2.0": {
-              "schema": {
-                "type": "integer",
-                "format": "int32"
-              }
-            },
-            "application/json; x-version=2.0": {
-              "schema": {
-                "type": "integer",
-                "format": "int32"
-              }
-            },
-            "text/json; x-version=2.0": {
-              "schema": {
-                "type": "integer",
-                "format": "int32"
-              }
-            },
-            "application/*+json; x-version=2.0": {
-              "schema": {
-                "type": "integer",
-                "format": "int32"
-              }
-            }
-          }
-        },
-        "responses": {
-          "200": {
-            "description": ""
-          },
-          "400": {
-            "description": "Limit needs to be greater than 0",
-            "content": {
-              "text/plain; x-version=2.0": {
-                "schema": {
-                  "$ref": "#/components/schemas/ProblemDetails"
-                }
-              },
-              "application/json; x-version=2.0": {
-                "schema": {
-                  "$ref": "#/components/schemas/ProblemDetails"
-                }
-              },
-              "text/json; x-version=2.0": {
-                "schema": {
-                  "$ref": "#/components/schemas/ProblemDetails"
-                }
-              }
-            }
-          }
-        }
-      },
-      "delete": {
-        "tags": [
-          "Settings"
-        ],
-        "summary": "Reset Request-Limit",
-        "parameters": [
-          {
-            "name": "RequestType",
-            "in": "path",
-            "required": true,
-            "schema": {
-              "$ref": "#/components/schemas/RequestType"
-            }
-          }
-        ],
-        "responses": {
-          "200": {
-            "description": "",
-            "content": {
-              "text/plain; x-version=2.0": {
-                "schema": {
-                  "type": "string"
-                }
-              },
-              "application/json; x-version=2.0": {
-                "schema": {
-                  "type": "string"
-                }
-              },
-              "text/json; x-version=2.0": {
-                "schema": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
       }
     },
     "/v2/Settings/ImageCompressionLevel": {
@@ -4313,17 +4129,6 @@
         },
         "additionalProperties": { }
       },
-      "RequestType": {
-        "enum": [
-          "Default",
-          "MangaDexFeed",
-          "MangaImage",
-          "MangaCover",
-          "MangaDexImage",
-          "MangaInfo"
-        ],
-        "type": "string"
-      },
       "TrangaSettings": {
         "type": "object",
         "properties": {
@@ -4356,37 +4161,6 @@
             "type": "integer",
             "format": "int32"
           },
-          "requestLimits": {
-            "type": "object",
-            "properties": {
-              "Default": {
-                "type": "integer",
-                "format": "int32"
-              },
-              "MangaDexFeed": {
-                "type": "integer",
-                "format": "int32"
-              },
-              "MangaImage": {
-                "type": "integer",
-                "format": "int32"
-              },
-              "MangaCover": {
-                "type": "integer",
-                "format": "int32"
-              },
-              "MangaDexImage": {
-                "type": "integer",
-                "format": "int32"
-              },
-              "MangaInfo": {
-                "type": "integer",
-                "format": "int32"
-              }
-            },
-            "additionalProperties": false,
-            "nullable": true
-          },
           "downloadLanguage": {
             "type": "string",
             "nullable": true

docker-compose.local.yaml

@@ -22,7 +22,7 @@ services:
         max-size: "10m"
         max-file: "5"
   tranga-pg:
-    image: postgres:latest
+    image: postgres:17
     container_name: tranga-pg
     ports:
       - "5432:5432"