Use DelegatingHandler for RateLimits

API/MangaDownloadClients/DownloadClient.cs

@@ -1,56 +1,6 @@
-using System.Collections.Concurrent;
-using System.Net;
-using log4net;
+namespace API.MangaDownloadClients;
 
-namespace API.MangaDownloadClients;
-
-public abstract class DownloadClient
+public interface IDownloadClient
 {
-    private static readonly ConcurrentDictionary<RequestType, DateTime> LastExecutedRateLimit = new();
-    protected ILog Log { get; init; }
-
-    protected DownloadClient()
-    {
-        this.Log = LogManager.GetLogger(GetType());
-    }
-    
-    // TODO Requests still go too fast across threads!
-    public RequestResult MakeRequest(string url, RequestType requestType, string? referrer = null, string? clickButton = null)
-    {
-        Log.Debug($"Requesting {requestType} {url}");
-        
-        // If we don't have a RequestLimit set for a Type, use the default one
-        if (!Tranga.Settings.RequestLimits.ContainsKey(requestType))
-            requestType = RequestType.Default;
-
-        int rateLimit = Tranga.Settings.RequestLimits[requestType];
-        // TODO this probably needs a better check whether the useragent matches...
-        // If the UserAgent is the default one, do not exceed the default request-limits.
-        if (Tranga.Settings.UserAgent == TrangaSettings.DefaultUserAgent && rateLimit > TrangaSettings.DefaultRequestLimits[requestType])
-            rateLimit = TrangaSettings.DefaultRequestLimits[requestType];
-        
-        // Apply the delay
-        TimeSpan timeBetweenRequests = TimeSpan.FromMinutes(1).Divide(rateLimit);
-        DateTime now = DateTime.UtcNow;
-        LastExecutedRateLimit.TryAdd(requestType, now.Subtract(timeBetweenRequests));
-
-        TimeSpan rateLimitTimeout = timeBetweenRequests.Subtract(now.Subtract(LastExecutedRateLimit[requestType]));
-        Log.Debug($"Request limit {requestType} {rateLimit}/Minute timeBetweenRequests: {timeBetweenRequests:ss'.'fffff}");
-
-        if (rateLimitTimeout > TimeSpan.Zero)
-        {
-            Log.Info($"Waiting {rateLimitTimeout} for {url}");
-            Thread.Sleep(rateLimitTimeout);
-        }
-
-        // Make the request
-        RequestResult result = MakeRequestInternal(url, referrer, clickButton);
-        
-        // Update the time the last request was made
-        LastExecutedRateLimit[requestType] = DateTime.UtcNow;
-        Log.Debug($"Result {url}: {result}");
-        return result;
-    }
-
-    internal abstract RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null);
+    internal Task<HttpResponseMessage> MakeRequest(string url, RequestType requestType, string? referrer = null);
 }

API/MangaDownloadClients/FlareSolverrDownloadClient.cs

@@ -1,25 +1,26 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Net;
-using System.Text;
 using System.Text.Json;
 using HtmlAgilityPack;
+using log4net;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 
 namespace API.MangaDownloadClients;
 
-public class FlareSolverrDownloadClient : DownloadClient
+public class FlareSolverrDownloadClient(HttpClient client) : IDownloadClient
 {
-    internal override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null)
+    private ILog Log { get; } = LogManager.GetLogger(typeof(FlareSolverrDownloadClient));
+
+    public async Task<HttpResponseMessage> MakeRequest(string url, RequestType requestType, string? referrer = null)
     {
-        if (clickButton is not null)
-            Log.Warn("Client can not click button");
+        Log.Debug($"Using {typeof(FlareSolverrDownloadClient).FullName} for {url}");
         if(referrer is not null)
             Log.Warn("Client can not set referrer");
         if (Tranga.Settings.FlareSolverrUrl == string.Empty)
         {
             Log.Error("FlareSolverr URL is empty");
-            return new(HttpStatusCode.InternalServerError, null, Stream.Null);
+            return new(HttpStatusCode.InternalServerError);
         }
         
         Uri flareSolverrUri = new (Tranga.Settings.FlareSolverrUrl);
@@ -29,13 +30,6 @@ public class FlareSolverrDownloadClient : DownloadClient
                 Path = "v1"
             }.Uri;
         
-        HttpClient client = new()
-        {
-            Timeout = TimeSpan.FromSeconds(10),
-            DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrHigher,
-            DefaultRequestHeaders = { { "User-Agent", Tranga.Settings.UserAgent } }
-        };
-
         JObject requestObj = new()
         {
             ["cmd"] = "request.get",
@@ -52,12 +46,12 @@ public class FlareSolverrDownloadClient : DownloadClient
         HttpResponseMessage? response;
         try
         {
-            response = client.Send(requestMessage);
+            response = await client.SendAsync(requestMessage);
         }
         catch (HttpRequestException e)
         {
             Log.Error(e);
-            return new (HttpStatusCode.Unused, null, Stream.Null);
+            return new (HttpStatusCode.InternalServerError);
         }
 
         if (!response.IsSuccessStatusCode)
@@ -74,7 +68,7 @@ public class FlareSolverrDownloadClient : DownloadClient
                       $"{response.Version}\n" +
                       $"Headers:\n\t{string.Join("\n\t", response.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
                       $"{response.Content.ReadAsStringAsync().Result}");
-            return new (response.StatusCode,  null, Stream.Null);
+            return response;
         }
 
         string responseString = response.Content.ReadAsStringAsync().Result;
@@ -82,51 +76,45 @@ public class FlareSolverrDownloadClient : DownloadClient
         if (!IsInCorrectFormat(responseObj, out string? reason))
         {
             Log.Error($"Wrong format: {reason}");
-            return new(HttpStatusCode.Unused, null, Stream.Null);
+            return new(HttpStatusCode.InternalServerError);
         }
 
         string statusResponse = responseObj["status"]!.Value<string>()!;
         if (statusResponse != "ok")
         {
             Log.Debug($"Status is not ok: {statusResponse}");
-            return new(HttpStatusCode.Unused, null, Stream.Null);
+            return new(HttpStatusCode.InternalServerError);
         }
         JObject solution = (responseObj["solution"] as JObject)!;
 
         if (!Enum.TryParse(solution["status"]!.Value<int>().ToString(), out HttpStatusCode statusCode))
         {
             Log.Error($"Wrong format: Cant parse status code: {solution["status"]!.Value<int>()}");
-            return new(HttpStatusCode.Unused, null, Stream.Null);
+            return new(HttpStatusCode.InternalServerError);
         }
         if (statusCode < HttpStatusCode.OK || statusCode >= HttpStatusCode.MultipleChoices)
         {
             Log.Debug($"Status is: {statusCode}");
-            return new(statusCode, null, Stream.Null);
+            return new (statusCode);
         }
 
         if (solution["response"]!.Value<string>() is not { } htmlString)
         {
             Log.Error("Wrong format: Cant find response in solution");
-            return new(HttpStatusCode.Unused, null, Stream.Null);
+            return new(HttpStatusCode.InternalServerError);
         }
 
-        if (IsJson(htmlString, out HtmlDocument document, out string? json))
+        if (IsJson(htmlString, out string? json))
         {
-            MemoryStream ms = new();
-            ms.Write(Encoding.UTF8.GetBytes(json));
-            ms.Position = 0;
-            return new(statusCode, document, ms);
+            return new(statusCode) { Content = new StringContent(json) };
         }
         else
         {
-            MemoryStream ms = new();
-            ms.Write(Encoding.UTF8.GetBytes(htmlString));
-            ms.Position = 0;
-            return new(statusCode, document, ms);
+            return new(statusCode) { Content = new StringContent(htmlString) };
         }
     }
 
-    private bool IsInCorrectFormat(JObject responseObj, [NotNullWhen(false)]out string? reason)
+    private static bool IsInCorrectFormat(JObject responseObj, [NotNullWhen(false)]out string? reason)
     {
         reason = null;
         if (!responseObj.ContainsKey("status"))
@@ -157,10 +145,10 @@ public class FlareSolverrDownloadClient : DownloadClient
         return true;
     }
 
-    private bool IsJson(string htmlString, out HtmlDocument document, [NotNullWhen(true)]out string? jsonString)
+    private static bool IsJson(string htmlString, [NotNullWhen(true)]out string? jsonString)
     {
         jsonString = null;
-        document = new();
+        HtmlDocument document = new();
         document.LoadHtml(htmlString);
 
         HtmlNode pre = document.DocumentNode.SelectSingleNode("//pre");

API/MangaDownloadClients/HttpDownloadClient.cs

@@ -1,89 +1,59 @@
 using System.Net;
-using HtmlAgilityPack;
+using log4net;
 
 namespace API.MangaDownloadClients;
 
-internal class HttpDownloadClient : DownloadClient
+internal class HttpDownloadClient : IDownloadClient
 {
-    private static readonly FlareSolverrDownloadClient FlareSolverrDownloadClient = new();
-    internal override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null)
+    private static readonly HttpClient Client = new(handler: Tranga.RateLimitHandler)
+    {
+        Timeout = TimeSpan.FromSeconds(10),
+        DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrHigher,
+        DefaultRequestHeaders = { { "User-Agent", Tranga.Settings.UserAgent } }
+    };
+    private static readonly FlareSolverrDownloadClient FlareSolverrDownloadClient = new(Client);
+    private ILog Log { get; } = LogManager.GetLogger(typeof(HttpDownloadClient));
+    
+    public async Task<HttpResponseMessage> MakeRequest(string url, RequestType requestType, string? referrer = null)
     {
         Log.Debug($"Using {typeof(HttpDownloadClient).FullName} for {url}");
-        if (clickButton is not null)
-            Log.Warn("Client can not click button");
-        HttpClient client = new();
-        client.Timeout = TimeSpan.FromSeconds(10);
-        client.DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrHigher;
-        client.DefaultRequestHeaders.Add("User-Agent", Tranga.Settings.UserAgent);
-        HttpResponseMessage? response;
-        Uri uri = new(url);
-        HttpRequestMessage requestMessage = new(HttpMethod.Get, uri);
+        HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
         if (referrer is not null)
             requestMessage.Headers.Referrer = new (referrer);
         Log.Debug($"Requesting {url}");
+        
         try
         {
-            response = client.Send(requestMessage);
-        }
-        catch (HttpRequestException e)
-        {
-            Log.Error(e);
-            return new (HttpStatusCode.Unused, null, Stream.Null);
-        }
+            HttpResponseMessage response = await Client.SendAsync(requestMessage);
+            Log.Debug($"Request {url} returned {(int)response.StatusCode} {response.StatusCode}");
+            if(response.IsSuccessStatusCode)
+                return response;
 
-        if (!response.IsSuccessStatusCode)
-        {
-            Log.Debug($"Request returned status code {(int)response.StatusCode} {response.StatusCode}");
             if (response.Headers.Server.Any(s =>
                     (s.Product?.Name ?? "").Contains("cloudflare", StringComparison.InvariantCultureIgnoreCase)))
             {
                 Log.Debug("Retrying with FlareSolverr!");
-                return FlareSolverrDownloadClient.MakeRequestInternal(url, referrer, clickButton);
-            }
-            else
-            {
-                Log.Debug($"Request returned status code {(int)response.StatusCode} {response.StatusCode}:\n" +
-                          $"=====\n" +
-                          $"Request:\n" +
-                          $"{requestMessage.Method} {requestMessage.RequestUri}\n" +
-                          $"{requestMessage.Version} {requestMessage.VersionPolicy}\n" +
-                          $"Headers:\n\t{string.Join("\n\t", requestMessage.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
-                          $"{requestMessage.Content?.ReadAsStringAsync().Result}" +
-                          $"=====\n" +
-                          $"Response:\n" +
-                          $"{response.Version}\n" +
-                          $"Headers:\n\t{string.Join("\n\t", response.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
-                          $"{response.Content.ReadAsStringAsync().Result}");
+                return await FlareSolverrDownloadClient.MakeRequest(url, requestType, referrer);
             }
+            
+            Log.Debug($"Request returned status code {(int)response.StatusCode} {response.StatusCode}:\n" +
+                      $"=====\n" +
+                      $"Request:\n" +
+                      $"{requestMessage.Method} {requestMessage.RequestUri}\n" +
+                      $"{requestMessage.Version} {requestMessage.VersionPolicy}\n" +
+                      $"Headers:\n\t{string.Join("\n\t", requestMessage.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
+                      $"{requestMessage.Content?.ReadAsStringAsync().Result}" +
+                      $"=====\n" +
+                      $"Response:\n" +
+                      $"{response.Version}\n" +
+                      $"Headers:\n\t{string.Join("\n\t", response.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
+                      $"{response.Content.ReadAsStringAsync().Result}");
+            return new(HttpStatusCode.InternalServerError);
         }
-
-        Stream stream;
-        try
-        {
-            stream = response.Content.ReadAsStream();
-        }
-        catch (Exception e)
+        catch (HttpRequestException e)
         {
             Log.Error(e);
-            return new (HttpStatusCode.Unused, null, Stream.Null);
+            return new(HttpStatusCode.InternalServerError);
         }
-
-        HtmlDocument? document = null;
-
-        if (response.Content.Headers.ContentType?.MediaType == "text/html")
-        {
-            StreamReader reader = new (stream);
-            document = new ();
-            document.LoadHtml(reader.ReadToEnd());
-            stream.Position = 0;
-        }
-
-        // Request has been redirected to another page. For example, it redirects directly to the results when there is only 1 result
-        if (response.RequestMessage is not null && response.RequestMessage.RequestUri is not null && response.RequestMessage.RequestUri != uri)
-        {
-            return new (response.StatusCode, document, stream, true, response.RequestMessage.RequestUri.AbsoluteUri);
-        }
-
-        return new (response.StatusCode, document, stream);
     }
 }

API/MangaDownloadClients/RateLimitHandler.cs

@@ -0,0 +1,31 @@
+using System.Net;
+using System.Threading.RateLimiting;
+using log4net;
+
+namespace API.MangaDownloadClients;
+
+public class RateLimitHandler() : DelegatingHandler(new HttpClientHandler())
+{
+    private ILog Log { get; init; } = LogManager.GetLogger(typeof(RateLimitHandler));
+
+    private readonly RateLimiter _limiter = new SlidingWindowRateLimiter(new ()
+    {
+        AutoReplenishment = true,
+        PermitLimit = 240,
+        QueueLimit = 120,
+        QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
+        SegmentsPerWindow = 60,
+        Window = TimeSpan.FromSeconds(60)
+    });
+    
+    protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+    {
+        Log.Debug($"Requesting lease {request.RequestUri}");
+        using RateLimitLease lease = await _limiter.AcquireAsync(permitCount: 1, cancellationToken);
+        Log.Debug($"lease {lease.IsAcquired} {request.RequestUri}");
+
+        return lease.IsAcquired
+            ? await base.SendAsync(request, cancellationToken)
+            : new (HttpStatusCode.TooManyRequests);
+    }
+}

API/MangaDownloadClients/RequestResult.cs

@@ -1,33 +0,0 @@
-using System.Net;
-using HtmlAgilityPack;
-
-namespace API.MangaDownloadClients;
-
-public struct RequestResult
-{
-    public HttpStatusCode statusCode { get; }
-    public Stream result { get; }
-    public bool hasBeenRedirected { get; }
-    public string? redirectedToUrl { get; }
-    public HtmlDocument? htmlDocument { get; }
-
-    public RequestResult(HttpStatusCode statusCode, HtmlDocument? htmlDocument, Stream result)
-    {
-        this.statusCode = statusCode;
-        this.htmlDocument = htmlDocument;
-        this.result = result;
-    }
-
-    public RequestResult(HttpStatusCode statusCode, HtmlDocument? htmlDocument, Stream result, bool hasBeenRedirected, string redirectedTo)
-        : this(statusCode, htmlDocument, result)
-    {
-        this.hasBeenRedirected = hasBeenRedirected;
-        redirectedToUrl = redirectedTo;
-    }
-
-    public override string ToString()
-    {
-        return
-            $"{(int)statusCode} {statusCode.ToString()} {(hasBeenRedirected ? "Redirected: " : "")} {redirectedToUrl}";
-    }
-}