Add more Logging

API/MangaDownloadClients/ChromiumDownloadClient.cs

@@ -2,6 +2,7 @@
 using System.Text;
 using System.Text.RegularExpressions;
 using HtmlAgilityPack;
+using log4net;
 using PuppeteerSharp;
 
 namespace API.MangaDownloadClients;
@@ -13,7 +14,7 @@ internal class ChromiumDownloadClient : DownloadClient
     private readonly Thread _closeStalePagesThread;
     private readonly List<KeyValuePair<IPage, DateTime>> _openPages = new ();
     
-    private static async Task<IBrowser> StartBrowser()
+    private static async Task<IBrowser> StartBrowser(ILog log)
     {
         return await Puppeteer.LaunchAsync(new LaunchOptions
         {
@@ -24,14 +25,14 @@ internal class ChromiumDownloadClient : DownloadClient
                 "--disable-setuid-sandbox",
                 "--no-sandbox"},
             Timeout = 30000
-        });
+        }, new LoggerFactory([new Provider(log)]));
     }
 
     public ChromiumDownloadClient()
     {
         _httpDownloadClient = new();
         if(_browser is null)
-            _browser = StartBrowser().Result;
+            _browser = StartBrowser(Log).Result;
         _closeStalePagesThread = new Thread(CheckStalePages);
         _closeStalePagesThread.Start();
     }
@@ -41,8 +42,10 @@ internal class ChromiumDownloadClient : DownloadClient
         while (true)
         {
             Thread.Sleep(TimeSpan.FromHours(1));
+            Log.Debug("Removing stale pages");
             foreach ((IPage? key, DateTime value) in _openPages.Where(kv => kv.Value.Subtract(DateTime.Now) > TimeSpan.FromHours(1)))
             {
+                Log.Debug($"Closing {key.Url}");
                 key.CloseAsync().Wait();
             }
         }
@@ -51,6 +54,7 @@ internal class ChromiumDownloadClient : DownloadClient
     private readonly Regex _imageUrlRex = new(@"https?:\/\/.*\.(?:p?jpe?g|gif|a?png|bmp|avif|webp)(\?.*)?");
     internal override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null)
     {
+        Log.Debug($"Requesting {url}");
         return _imageUrlRex.IsMatch(url)
             ? _httpDownloadClient.MakeRequestInternal(url, referrer)
             : MakeRequestBrowser(url, referrer, clickButton);
@@ -68,11 +72,11 @@ internal class ChromiumDownloadClient : DownloadClient
         try
         {
             response = page.GoToAsync(url, WaitUntilNavigation.Networkidle0).Result;
-            //Log($"Page loaded. {url}");
+            Log.Debug($"Page loaded. {url}");
         }
         catch (Exception e)
         {
-            //Log($"Could not load Page {url}\n{e.Message}");
+            Log.Info($"Could not load Page {url}\n{e.Message}");
             page.CloseAsync();
             _openPages.Remove(_openPages.Find(i => i.Key == page));
             return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null);
@@ -107,4 +111,41 @@ internal class ChromiumDownloadClient : DownloadClient
         _openPages.Remove(_openPages.Find(i => i.Key == page));
         return new RequestResult(response.Status, document, stream, false, "");
     }
+
+    private class Provider(ILog log) : ILoggerProvider
+    {
+        public void Dispose()
+        {
+            
+        }
+
+        public ILogger CreateLogger(string categoryName)
+        {
+            return new ChromiumLogger(log);
+        }
+    }
+
+    private class ChromiumLogger(ILog log) : ILogger
+    {
+        public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
+        {
+            string message = formatter.Invoke(state, exception);
+            switch(logLevel)
+            {
+                case LogLevel.Critical: log.Fatal(message); break;
+                case LogLevel.Error: log.Error(message); break;
+                case LogLevel.Warning: log.Warn(message); break;
+                case LogLevel.Information: log.Info(message); break;
+                case LogLevel.Debug: log.Debug(message); break;
+                default: log.Info(message); break;
+            }
+        }
+
+        public bool IsEnabled(LogLevel logLevel) => true;
+
+        public IDisposable? BeginScope<TState>(TState state) where TState : notnull
+        {
+            return null;
+        }
+    }
 }

API/MangaDownloadClients/DownloadClient.cs

@@ -16,6 +16,7 @@ internal abstract class DownloadClient
     
     public RequestResult MakeRequest(string url, RequestType requestType, string? referrer = null, string? clickButton = null)
     {
+        Log.Debug($"Requesting {url}");
         if (!TrangaSettings.requestLimits.ContainsKey(requestType))
         {
             return new RequestResult(HttpStatusCode.NotAcceptable, null, Stream.Null);
@@ -24,14 +25,17 @@ internal abstract class DownloadClient
         int rateLimit = TrangaSettings.userAgent == TrangaSettings.DefaultUserAgent
             ? TrangaSettings.DefaultRequestLimits[requestType]
             : TrangaSettings.requestLimits[requestType];
+        Log.Debug($"Request limit {rateLimit}");
         
         TimeSpan timeBetweenRequests = TimeSpan.FromMinutes(1).Divide(rateLimit);
         _lastExecutedRateLimit.TryAdd(requestType, DateTime.UtcNow.Subtract(timeBetweenRequests));
 
         TimeSpan rateLimitTimeout = timeBetweenRequests.Subtract(DateTime.UtcNow.Subtract(_lastExecutedRateLimit[requestType]));
 
+        
         if (rateLimitTimeout > TimeSpan.Zero)
         {
+            Log.Debug($"Timeout: {rateLimitTimeout}");
             Thread.Sleep(rateLimitTimeout);
         }
 

API/MangaDownloadClients/HttpDownloadClient.cs

@@ -1,5 +1,4 @@
 using System.Net;
-using API.Schema;
 using HtmlAgilityPack;
 
 namespace API.MangaDownloadClients;
@@ -18,16 +17,15 @@ internal class HttpDownloadClient : DownloadClient
     
     internal override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null)
     {
-        //TODO
+        if (clickButton is not null)
-        //if (clickButton is not null)
+            Log.Warn("Can not click button on static site.");
-            //Log("Can not click button on static site.");
         HttpResponseMessage? response = null;
         while (response is null)
         {
             HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
             if (referrer is not null)
                 requestMessage.Headers.Referrer = new Uri(referrer);
-            //Log($"Requesting {requestType} {url}");
+            Log.Debug($"Requesting {url}");
             try
             {
                 response = Client.Send(requestMessage);

API/Schema/MangaConnectors/MangaDex.cs

@@ -22,26 +22,32 @@ public class MangaDex : MangaConnector
         int offset = 0; //"Page"
         int total = int.MaxValue; //How many total results are there, is updated on first request
         HashSet<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)> retManga = new();
-        int loadedPublicationData = 0;
         List<JsonNode> results = new();
         
         //Request all search-results
         while (offset < total) //As long as we haven't requested all "Pages"
         {
             //Request next Page
-            RequestResult requestResult = downloadClient.MakeRequest(
+            string requestUrl =
-                    $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}" +
+                $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}" +
-                    $"&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica" +
+                $"&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica" +
-                    $"&contentRating%5B%5D=pornographic" +
+                $"&contentRating%5B%5D=pornographic" +
-                    $"&includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author" +
+                $"&includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author" +
-                    $"&includes%5B%5D=artist&includes%5B%5D=tag", RequestType.MangaInfo);
+                $"&includes%5B%5D=artist&includes%5B%5D=tag";
+            RequestResult requestResult = downloadClient.MakeRequest(requestUrl, RequestType.MangaInfo);
             if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
+            {
+                Log.Info($"{requestResult.statusCode}: {requestUrl}");
                 break;
+            }
             JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
             
             offset += limit;
             if (result is null)
+            {
+                Log.Info($"result was null: {requestUrl}");
                 break;
+            }
             
             if(result.ContainsKey("total"))
                 total = result["total"]!.GetValue<int>(); //Update the total number of Publications
@@ -61,13 +67,18 @@ public class MangaDex : MangaConnector
 
     public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(string publicationId)
     {
-        RequestResult requestResult =
+        string url = $"https://api.mangadex.org/manga/{publicationId}" +
-            downloadClient.MakeRequest($"https://api.mangadex.org/manga/{publicationId}?includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author&includes%5B%5D=artist&includes%5B%5D=tag", RequestType.MangaInfo);
+                     $"?includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author&includes%5B%5D=artist&includes%5B%5D=tag";
+        RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
         if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
+        {
+            Log.Info($"{requestResult.statusCode}: {url}");
             return null;
+        }
         JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
         if(result is not null)
             return MangaFromJsonObject(result["data"]!.AsObject());
+        Log.Info($"result was null: {url}");
         return null;
     }
 
@@ -81,15 +92,24 @@ public class MangaDex : MangaConnector
     private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? MangaFromJsonObject(JsonObject manga)
     {
         if (!manga.TryGetPropertyValue("id", out JsonNode? idNode))
+        {
+            Log.Info("id was null");
             return null;
+        }
         string publicationId = idNode!.GetValue<string>();
             
         if (!manga.TryGetPropertyValue("attributes", out JsonNode? attributesNode))
+        {
+            Log.Info("attributes was null");
             return null;
+        }
         JsonObject attributes = attributesNode!.AsObject();
 
         if (!attributes.TryGetPropertyValue("title", out JsonNode? titleNode))
+        {
+            Log.Info("title was null");
             return null;
+        }
         string sortName = titleNode!.AsObject().ContainsKey("en") switch
         {
             true => titleNode.AsObject()["en"]!.GetValue<string>(),
@@ -108,7 +128,10 @@ public class MangaDex : MangaConnector
         List<MangaAltTitle> altTitles = altTitlesDict.Select(t => new MangaAltTitle(t.Key, t.Value)).ToList();
 
         if (!attributes.TryGetPropertyValue("description", out JsonNode? descriptionNode))
+        {
+            Log.Info("description was null");
             return null;
+        }
         string description = descriptionNode!.AsObject().ContainsKey("en") switch
         {
             true => descriptionNode.AsObject()["en"]!.GetValue<string>(),
@@ -154,12 +177,18 @@ public class MangaDex : MangaConnector
         List<MangaTag> mangaTags = tags.Select(t => new MangaTag(t)).ToList();
         
         if (!manga.TryGetPropertyValue("relationships", out JsonNode? relationshipsNode))
+        {
+            Log.Info("relationships was null");
             return null;
+        }
         
         JsonNode? coverNode = relationshipsNode!.AsArray()
             .FirstOrDefault(rel => rel!["type"]!.GetValue<string>().Equals("cover_art"));
         if (coverNode is null)
+        {
+            Log.Info("coverNode was null");
             return null;
+        }
         string fileName = coverNode["attributes"]!["fileName"]!.GetValue<string>();
         string coverUrl = $"https://uploads.mangadex.org/covers/{publicationId}/{fileName}";
         
@@ -195,16 +224,22 @@ public class MangaDex : MangaConnector
         while (offset < total)
         {
             //Request next "Page"
-            RequestResult requestResult =
+            string requestUrl = $"https://api.mangadex.org/manga/{manga.IdOnConnectorSite}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}" +
-                downloadClient.MakeRequest(
+                                $"&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&contentRating%5B%5D=pornographic";
-                    $"https://api.mangadex.org/manga/{manga.IdOnConnectorSite}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&contentRating%5B%5D=pornographic", RequestType.MangaDexFeed);
+            RequestResult requestResult = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed);
             if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
+            {
+                Log.Info($"{requestResult.statusCode}: {requestUrl}");
                 break;
+            }
             JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
             
             offset += limit;
             if (result is null)
+            {
+                Log.Info($"result was null: {requestUrl}");
                 break;
+            }
             
             total = result["total"]!.GetValue<int>();
             JsonArray chaptersInResult = result["data"]!.AsArray();
@@ -235,6 +270,7 @@ public class MangaDex : MangaConnector
                 if (attributes.ContainsKey("pages") && attributes["pages"] is not null &&
                     attributes["pages"]!.GetValue<int>() < 1)
                 {
+                    Log.Info($"No pages: {chapterId}");
                     continue;
                 }
                 
@@ -246,6 +282,7 @@ public class MangaDex : MangaConnector
                 }
                 catch (Exception e)
                 {
+                    Log.Debug(e);
                 }
             }
         }
@@ -258,16 +295,23 @@ public class MangaDex : MangaConnector
     {//Request URLs for Chapter-Images
         Match m = Regex.Match(chapter.Url, @"https?:\/\/mangadex.org\/chapter\/([0-9\-a-z]+)");
         if (!m.Success)
+        {
+            Log.Error($"Could not parse Chapter ID: {chapter.Url}");
             return [];
+        }
+
+        string url = $"https://api.mangadex.org/at-home/server/{m.Groups[1].Value}?forcePort443=false";
         RequestResult requestResult =
-            downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{m.Groups[1].Value}?forcePort443=false", RequestType.MangaDexImage);
+            downloadClient.MakeRequest(url, RequestType.MangaDexImage);
         if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
         {
+            Log.Info($"{requestResult.statusCode}: {url}");
             return [];
         }
         JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
         if (result is null)
         {
+            Log.Info($"Result was null: {url}");
             return [];
         }
         string baseUrl = result["baseUrl"]!.GetValue<string>();