diff --git a/API/MangaDownloadClients/ChromiumDownloadClient.cs b/API/MangaDownloadClients/ChromiumDownloadClient.cs index eb0ef0e..228624e 100644 --- a/API/MangaDownloadClients/ChromiumDownloadClient.cs +++ b/API/MangaDownloadClients/ChromiumDownloadClient.cs @@ -2,6 +2,7 @@ using System.Text; using System.Text.RegularExpressions; using HtmlAgilityPack; +using log4net; using PuppeteerSharp; namespace API.MangaDownloadClients; @@ -13,7 +14,7 @@ internal class ChromiumDownloadClient : DownloadClient private readonly Thread _closeStalePagesThread; private readonly List> _openPages = new (); - private static async Task StartBrowser() + private static async Task StartBrowser(ILog log) { return await Puppeteer.LaunchAsync(new LaunchOptions { @@ -24,14 +25,14 @@ internal class ChromiumDownloadClient : DownloadClient "--disable-setuid-sandbox", "--no-sandbox"}, Timeout = 30000 - }); + }, new LoggerFactory([new Provider(log)])); } public ChromiumDownloadClient() { _httpDownloadClient = new(); if(_browser is null) - _browser = StartBrowser().Result; + _browser = StartBrowser(Log).Result; _closeStalePagesThread = new Thread(CheckStalePages); _closeStalePagesThread.Start(); } @@ -41,8 +42,10 @@ internal class ChromiumDownloadClient : DownloadClient while (true) { Thread.Sleep(TimeSpan.FromHours(1)); + Log.Debug("Removing stale pages"); foreach ((IPage? key, DateTime value) in _openPages.Where(kv => kv.Value.Subtract(DateTime.Now) > TimeSpan.FromHours(1))) { + Log.Debug($"Closing {key.Url}"); key.CloseAsync().Wait(); } } @@ -51,6 +54,7 @@ internal class ChromiumDownloadClient : DownloadClient private readonly Regex _imageUrlRex = new(@"https?:\/\/.*\.(?:p?jpe?g|gif|a?png|bmp|avif|webp)(\?.*)?"); internal override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null) { + Log.Debug($"Requesting {url}"); return _imageUrlRex.IsMatch(url) ? _httpDownloadClient.MakeRequestInternal(url, referrer) : MakeRequestBrowser(url, referrer, clickButton); @@ -68,11 +72,11 @@ internal class ChromiumDownloadClient : DownloadClient try { response = page.GoToAsync(url, WaitUntilNavigation.Networkidle0).Result; - //Log($"Page loaded. {url}"); + Log.Debug($"Page loaded. {url}"); } catch (Exception e) { - //Log($"Could not load Page {url}\n{e.Message}"); + Log.Info($"Could not load Page {url}\n{e.Message}"); page.CloseAsync(); _openPages.Remove(_openPages.Find(i => i.Key == page)); return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null); @@ -107,4 +111,41 @@ internal class ChromiumDownloadClient : DownloadClient _openPages.Remove(_openPages.Find(i => i.Key == page)); return new RequestResult(response.Status, document, stream, false, ""); } + + private class Provider(ILog log) : ILoggerProvider + { + public void Dispose() + { + + } + + public ILogger CreateLogger(string categoryName) + { + return new ChromiumLogger(log); + } + } + + private class ChromiumLogger(ILog log) : ILogger + { + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + string message = formatter.Invoke(state, exception); + switch(logLevel) + { + case LogLevel.Critical: log.Fatal(message); break; + case LogLevel.Error: log.Error(message); break; + case LogLevel.Warning: log.Warn(message); break; + case LogLevel.Information: log.Info(message); break; + case LogLevel.Debug: log.Debug(message); break; + default: log.Info(message); break; + } + } + + public bool IsEnabled(LogLevel logLevel) => true; + + public IDisposable? BeginScope(TState state) where TState : notnull + { + return null; + } + } } \ No newline at end of file diff --git a/API/MangaDownloadClients/DownloadClient.cs b/API/MangaDownloadClients/DownloadClient.cs index c611a33..60d5760 100644 --- a/API/MangaDownloadClients/DownloadClient.cs +++ b/API/MangaDownloadClients/DownloadClient.cs @@ -16,6 +16,7 @@ internal abstract class DownloadClient public RequestResult MakeRequest(string url, RequestType requestType, string? referrer = null, string? clickButton = null) { + Log.Debug($"Requesting {url}"); if (!TrangaSettings.requestLimits.ContainsKey(requestType)) { return new RequestResult(HttpStatusCode.NotAcceptable, null, Stream.Null); @@ -24,14 +25,17 @@ internal abstract class DownloadClient int rateLimit = TrangaSettings.userAgent == TrangaSettings.DefaultUserAgent ? TrangaSettings.DefaultRequestLimits[requestType] : TrangaSettings.requestLimits[requestType]; + Log.Debug($"Request limit {rateLimit}"); TimeSpan timeBetweenRequests = TimeSpan.FromMinutes(1).Divide(rateLimit); _lastExecutedRateLimit.TryAdd(requestType, DateTime.UtcNow.Subtract(timeBetweenRequests)); TimeSpan rateLimitTimeout = timeBetweenRequests.Subtract(DateTime.UtcNow.Subtract(_lastExecutedRateLimit[requestType])); + if (rateLimitTimeout > TimeSpan.Zero) { + Log.Debug($"Timeout: {rateLimitTimeout}"); Thread.Sleep(rateLimitTimeout); } diff --git a/API/MangaDownloadClients/HttpDownloadClient.cs b/API/MangaDownloadClients/HttpDownloadClient.cs index b72b29b..db67b86 100644 --- a/API/MangaDownloadClients/HttpDownloadClient.cs +++ b/API/MangaDownloadClients/HttpDownloadClient.cs @@ -1,5 +1,4 @@ using System.Net; -using API.Schema; using HtmlAgilityPack; namespace API.MangaDownloadClients; @@ -18,16 +17,15 @@ internal class HttpDownloadClient : DownloadClient internal override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null) { - //TODO - //if (clickButton is not null) - //Log("Can not click button on static site."); + if (clickButton is not null) + Log.Warn("Can not click button on static site."); HttpResponseMessage? response = null; while (response is null) { HttpRequestMessage requestMessage = new(HttpMethod.Get, url); if (referrer is not null) requestMessage.Headers.Referrer = new Uri(referrer); - //Log($"Requesting {requestType} {url}"); + Log.Debug($"Requesting {url}"); try { response = Client.Send(requestMessage); diff --git a/API/Schema/MangaConnectors/MangaDex.cs b/API/Schema/MangaConnectors/MangaDex.cs index 13ba7f2..0cc9a00 100644 --- a/API/Schema/MangaConnectors/MangaDex.cs +++ b/API/Schema/MangaConnectors/MangaDex.cs @@ -22,26 +22,32 @@ public class MangaDex : MangaConnector int offset = 0; //"Page" int total = int.MaxValue; //How many total results are there, is updated on first request HashSet<(Manga, List?, List?, List?, List?)> retManga = new(); - int loadedPublicationData = 0; List results = new(); //Request all search-results while (offset < total) //As long as we haven't requested all "Pages" { //Request next Page - RequestResult requestResult = downloadClient.MakeRequest( - $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}" + - $"&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica" + - $"&contentRating%5B%5D=pornographic" + - $"&includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author" + - $"&includes%5B%5D=artist&includes%5B%5D=tag", RequestType.MangaInfo); + string requestUrl = + $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}" + + $"&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica" + + $"&contentRating%5B%5D=pornographic" + + $"&includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author" + + $"&includes%5B%5D=artist&includes%5B%5D=tag"; + RequestResult requestResult = downloadClient.MakeRequest(requestUrl, RequestType.MangaInfo); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + { + Log.Info($"{requestResult.statusCode}: {requestUrl}"); break; + } JsonObject? result = JsonSerializer.Deserialize(requestResult.result); offset += limit; if (result is null) + { + Log.Info($"result was null: {requestUrl}"); break; + } if(result.ContainsKey("total")) total = result["total"]!.GetValue(); //Update the total number of Publications @@ -61,13 +67,18 @@ public class MangaDex : MangaConnector public override (Manga, List?, List?, List?, List?)? GetMangaFromId(string publicationId) { - RequestResult requestResult = - downloadClient.MakeRequest($"https://api.mangadex.org/manga/{publicationId}?includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author&includes%5B%5D=artist&includes%5B%5D=tag", RequestType.MangaInfo); + string url = $"https://api.mangadex.org/manga/{publicationId}" + + $"?includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author&includes%5B%5D=artist&includes%5B%5D=tag"; + RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + { + Log.Info($"{requestResult.statusCode}: {url}"); return null; + } JsonObject? result = JsonSerializer.Deserialize(requestResult.result); if(result is not null) return MangaFromJsonObject(result["data"]!.AsObject()); + Log.Info($"result was null: {url}"); return null; } @@ -81,15 +92,24 @@ public class MangaDex : MangaConnector private (Manga, List?, List?, List?, List?)? MangaFromJsonObject(JsonObject manga) { if (!manga.TryGetPropertyValue("id", out JsonNode? idNode)) + { + Log.Info("id was null"); return null; + } string publicationId = idNode!.GetValue(); if (!manga.TryGetPropertyValue("attributes", out JsonNode? attributesNode)) + { + Log.Info("attributes was null"); return null; + } JsonObject attributes = attributesNode!.AsObject(); if (!attributes.TryGetPropertyValue("title", out JsonNode? titleNode)) + { + Log.Info("title was null"); return null; + } string sortName = titleNode!.AsObject().ContainsKey("en") switch { true => titleNode.AsObject()["en"]!.GetValue(), @@ -108,7 +128,10 @@ public class MangaDex : MangaConnector List altTitles = altTitlesDict.Select(t => new MangaAltTitle(t.Key, t.Value)).ToList(); if (!attributes.TryGetPropertyValue("description", out JsonNode? descriptionNode)) + { + Log.Info("description was null"); return null; + } string description = descriptionNode!.AsObject().ContainsKey("en") switch { true => descriptionNode.AsObject()["en"]!.GetValue(), @@ -154,12 +177,18 @@ public class MangaDex : MangaConnector List mangaTags = tags.Select(t => new MangaTag(t)).ToList(); if (!manga.TryGetPropertyValue("relationships", out JsonNode? relationshipsNode)) + { + Log.Info("relationships was null"); return null; + } JsonNode? coverNode = relationshipsNode!.AsArray() .FirstOrDefault(rel => rel!["type"]!.GetValue().Equals("cover_art")); if (coverNode is null) + { + Log.Info("coverNode was null"); return null; + } string fileName = coverNode["attributes"]!["fileName"]!.GetValue(); string coverUrl = $"https://uploads.mangadex.org/covers/{publicationId}/{fileName}"; @@ -195,16 +224,22 @@ public class MangaDex : MangaConnector while (offset < total) { //Request next "Page" - RequestResult requestResult = - downloadClient.MakeRequest( - $"https://api.mangadex.org/manga/{manga.IdOnConnectorSite}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&contentRating%5B%5D=pornographic", RequestType.MangaDexFeed); + string requestUrl = $"https://api.mangadex.org/manga/{manga.IdOnConnectorSite}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}" + + $"&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&contentRating%5B%5D=pornographic"; + RequestResult requestResult = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + { + Log.Info($"{requestResult.statusCode}: {requestUrl}"); break; + } JsonObject? result = JsonSerializer.Deserialize(requestResult.result); offset += limit; if (result is null) + { + Log.Info($"result was null: {requestUrl}"); break; + } total = result["total"]!.GetValue(); JsonArray chaptersInResult = result["data"]!.AsArray(); @@ -235,6 +270,7 @@ public class MangaDex : MangaConnector if (attributes.ContainsKey("pages") && attributes["pages"] is not null && attributes["pages"]!.GetValue() < 1) { + Log.Info($"No pages: {chapterId}"); continue; } @@ -246,6 +282,7 @@ public class MangaDex : MangaConnector } catch (Exception e) { + Log.Debug(e); } } } @@ -258,16 +295,23 @@ public class MangaDex : MangaConnector {//Request URLs for Chapter-Images Match m = Regex.Match(chapter.Url, @"https?:\/\/mangadex.org\/chapter\/([0-9\-a-z]+)"); if (!m.Success) + { + Log.Error($"Could not parse Chapter ID: {chapter.Url}"); return []; + } + + string url = $"https://api.mangadex.org/at-home/server/{m.Groups[1].Value}?forcePort443=false"; RequestResult requestResult = - downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{m.Groups[1].Value}?forcePort443=false", RequestType.MangaDexImage); + downloadClient.MakeRequest(url, RequestType.MangaDexImage); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) { + Log.Info($"{requestResult.statusCode}: {url}"); return []; } JsonObject? result = JsonSerializer.Deserialize(requestResult.result); if (result is null) { + Log.Info($"Result was null: {url}"); return []; } string baseUrl = result["baseUrl"]!.GetValue();