mirror of
https://github.com/C9Glax/tranga.git
synced 2025-05-14 18:33:02 +02:00
Add more Logging
This commit is contained in:
parent
a490e233d7
commit
ec5d048df5
@ -2,6 +2,7 @@
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using HtmlAgilityPack;
|
||||
using log4net;
|
||||
using PuppeteerSharp;
|
||||
|
||||
namespace API.MangaDownloadClients;
|
||||
@ -13,7 +14,7 @@ internal class ChromiumDownloadClient : DownloadClient
|
||||
private readonly Thread _closeStalePagesThread;
|
||||
private readonly List<KeyValuePair<IPage, DateTime>> _openPages = new ();
|
||||
|
||||
private static async Task<IBrowser> StartBrowser()
|
||||
private static async Task<IBrowser> StartBrowser(ILog log)
|
||||
{
|
||||
return await Puppeteer.LaunchAsync(new LaunchOptions
|
||||
{
|
||||
@ -24,14 +25,14 @@ internal class ChromiumDownloadClient : DownloadClient
|
||||
"--disable-setuid-sandbox",
|
||||
"--no-sandbox"},
|
||||
Timeout = 30000
|
||||
});
|
||||
}, new LoggerFactory([new Provider(log)]));
|
||||
}
|
||||
|
||||
public ChromiumDownloadClient()
|
||||
{
|
||||
_httpDownloadClient = new();
|
||||
if(_browser is null)
|
||||
_browser = StartBrowser().Result;
|
||||
_browser = StartBrowser(Log).Result;
|
||||
_closeStalePagesThread = new Thread(CheckStalePages);
|
||||
_closeStalePagesThread.Start();
|
||||
}
|
||||
@ -41,8 +42,10 @@ internal class ChromiumDownloadClient : DownloadClient
|
||||
while (true)
|
||||
{
|
||||
Thread.Sleep(TimeSpan.FromHours(1));
|
||||
Log.Debug("Removing stale pages");
|
||||
foreach ((IPage? key, DateTime value) in _openPages.Where(kv => kv.Value.Subtract(DateTime.Now) > TimeSpan.FromHours(1)))
|
||||
{
|
||||
Log.Debug($"Closing {key.Url}");
|
||||
key.CloseAsync().Wait();
|
||||
}
|
||||
}
|
||||
@ -51,6 +54,7 @@ internal class ChromiumDownloadClient : DownloadClient
|
||||
private readonly Regex _imageUrlRex = new(@"https?:\/\/.*\.(?:p?jpe?g|gif|a?png|bmp|avif|webp)(\?.*)?");
|
||||
internal override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null)
|
||||
{
|
||||
Log.Debug($"Requesting {url}");
|
||||
return _imageUrlRex.IsMatch(url)
|
||||
? _httpDownloadClient.MakeRequestInternal(url, referrer)
|
||||
: MakeRequestBrowser(url, referrer, clickButton);
|
||||
@ -68,11 +72,11 @@ internal class ChromiumDownloadClient : DownloadClient
|
||||
try
|
||||
{
|
||||
response = page.GoToAsync(url, WaitUntilNavigation.Networkidle0).Result;
|
||||
//Log($"Page loaded. {url}");
|
||||
Log.Debug($"Page loaded. {url}");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
//Log($"Could not load Page {url}\n{e.Message}");
|
||||
Log.Info($"Could not load Page {url}\n{e.Message}");
|
||||
page.CloseAsync();
|
||||
_openPages.Remove(_openPages.Find(i => i.Key == page));
|
||||
return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null);
|
||||
@ -107,4 +111,41 @@ internal class ChromiumDownloadClient : DownloadClient
|
||||
_openPages.Remove(_openPages.Find(i => i.Key == page));
|
||||
return new RequestResult(response.Status, document, stream, false, "");
|
||||
}
|
||||
|
||||
private class Provider(ILog log) : ILoggerProvider
|
||||
{
|
||||
public void Dispose()
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public ILogger CreateLogger(string categoryName)
|
||||
{
|
||||
return new ChromiumLogger(log);
|
||||
}
|
||||
}
|
||||
|
||||
private class ChromiumLogger(ILog log) : ILogger
|
||||
{
|
||||
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
|
||||
{
|
||||
string message = formatter.Invoke(state, exception);
|
||||
switch(logLevel)
|
||||
{
|
||||
case LogLevel.Critical: log.Fatal(message); break;
|
||||
case LogLevel.Error: log.Error(message); break;
|
||||
case LogLevel.Warning: log.Warn(message); break;
|
||||
case LogLevel.Information: log.Info(message); break;
|
||||
case LogLevel.Debug: log.Debug(message); break;
|
||||
default: log.Info(message); break;
|
||||
}
|
||||
}
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -16,6 +16,7 @@ internal abstract class DownloadClient
|
||||
|
||||
public RequestResult MakeRequest(string url, RequestType requestType, string? referrer = null, string? clickButton = null)
|
||||
{
|
||||
Log.Debug($"Requesting {url}");
|
||||
if (!TrangaSettings.requestLimits.ContainsKey(requestType))
|
||||
{
|
||||
return new RequestResult(HttpStatusCode.NotAcceptable, null, Stream.Null);
|
||||
@ -24,14 +25,17 @@ internal abstract class DownloadClient
|
||||
int rateLimit = TrangaSettings.userAgent == TrangaSettings.DefaultUserAgent
|
||||
? TrangaSettings.DefaultRequestLimits[requestType]
|
||||
: TrangaSettings.requestLimits[requestType];
|
||||
Log.Debug($"Request limit {rateLimit}");
|
||||
|
||||
TimeSpan timeBetweenRequests = TimeSpan.FromMinutes(1).Divide(rateLimit);
|
||||
_lastExecutedRateLimit.TryAdd(requestType, DateTime.UtcNow.Subtract(timeBetweenRequests));
|
||||
|
||||
TimeSpan rateLimitTimeout = timeBetweenRequests.Subtract(DateTime.UtcNow.Subtract(_lastExecutedRateLimit[requestType]));
|
||||
|
||||
|
||||
if (rateLimitTimeout > TimeSpan.Zero)
|
||||
{
|
||||
Log.Debug($"Timeout: {rateLimitTimeout}");
|
||||
Thread.Sleep(rateLimitTimeout);
|
||||
}
|
||||
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
using System.Net;
|
||||
using API.Schema;
|
||||
using HtmlAgilityPack;
|
||||
|
||||
namespace API.MangaDownloadClients;
|
||||
@ -18,16 +17,15 @@ internal class HttpDownloadClient : DownloadClient
|
||||
|
||||
internal override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null)
|
||||
{
|
||||
//TODO
|
||||
//if (clickButton is not null)
|
||||
//Log("Can not click button on static site.");
|
||||
if (clickButton is not null)
|
||||
Log.Warn("Can not click button on static site.");
|
||||
HttpResponseMessage? response = null;
|
||||
while (response is null)
|
||||
{
|
||||
HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
|
||||
if (referrer is not null)
|
||||
requestMessage.Headers.Referrer = new Uri(referrer);
|
||||
//Log($"Requesting {requestType} {url}");
|
||||
Log.Debug($"Requesting {url}");
|
||||
try
|
||||
{
|
||||
response = Client.Send(requestMessage);
|
||||
|
||||
@ -22,26 +22,32 @@ public class MangaDex : MangaConnector
|
||||
int offset = 0; //"Page"
|
||||
int total = int.MaxValue; //How many total results are there, is updated on first request
|
||||
HashSet<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)> retManga = new();
|
||||
int loadedPublicationData = 0;
|
||||
List<JsonNode> results = new();
|
||||
|
||||
//Request all search-results
|
||||
while (offset < total) //As long as we haven't requested all "Pages"
|
||||
{
|
||||
//Request next Page
|
||||
RequestResult requestResult = downloadClient.MakeRequest(
|
||||
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}" +
|
||||
$"&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica" +
|
||||
$"&contentRating%5B%5D=pornographic" +
|
||||
$"&includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author" +
|
||||
$"&includes%5B%5D=artist&includes%5B%5D=tag", RequestType.MangaInfo);
|
||||
string requestUrl =
|
||||
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}" +
|
||||
$"&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica" +
|
||||
$"&contentRating%5B%5D=pornographic" +
|
||||
$"&includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author" +
|
||||
$"&includes%5B%5D=artist&includes%5B%5D=tag";
|
||||
RequestResult requestResult = downloadClient.MakeRequest(requestUrl, RequestType.MangaInfo);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
{
|
||||
Log.Info($"{requestResult.statusCode}: {requestUrl}");
|
||||
break;
|
||||
}
|
||||
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
|
||||
|
||||
offset += limit;
|
||||
if (result is null)
|
||||
{
|
||||
Log.Info($"result was null: {requestUrl}");
|
||||
break;
|
||||
}
|
||||
|
||||
if(result.ContainsKey("total"))
|
||||
total = result["total"]!.GetValue<int>(); //Update the total number of Publications
|
||||
@ -61,13 +67,18 @@ public class MangaDex : MangaConnector
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(string publicationId)
|
||||
{
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest($"https://api.mangadex.org/manga/{publicationId}?includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author&includes%5B%5D=artist&includes%5B%5D=tag", RequestType.MangaInfo);
|
||||
string url = $"https://api.mangadex.org/manga/{publicationId}" +
|
||||
$"?includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author&includes%5B%5D=artist&includes%5B%5D=tag";
|
||||
RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
{
|
||||
Log.Info($"{requestResult.statusCode}: {url}");
|
||||
return null;
|
||||
}
|
||||
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
|
||||
if(result is not null)
|
||||
return MangaFromJsonObject(result["data"]!.AsObject());
|
||||
Log.Info($"result was null: {url}");
|
||||
return null;
|
||||
}
|
||||
|
||||
@ -81,15 +92,24 @@ public class MangaDex : MangaConnector
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? MangaFromJsonObject(JsonObject manga)
|
||||
{
|
||||
if (!manga.TryGetPropertyValue("id", out JsonNode? idNode))
|
||||
{
|
||||
Log.Info("id was null");
|
||||
return null;
|
||||
}
|
||||
string publicationId = idNode!.GetValue<string>();
|
||||
|
||||
if (!manga.TryGetPropertyValue("attributes", out JsonNode? attributesNode))
|
||||
{
|
||||
Log.Info("attributes was null");
|
||||
return null;
|
||||
}
|
||||
JsonObject attributes = attributesNode!.AsObject();
|
||||
|
||||
if (!attributes.TryGetPropertyValue("title", out JsonNode? titleNode))
|
||||
{
|
||||
Log.Info("title was null");
|
||||
return null;
|
||||
}
|
||||
string sortName = titleNode!.AsObject().ContainsKey("en") switch
|
||||
{
|
||||
true => titleNode.AsObject()["en"]!.GetValue<string>(),
|
||||
@ -108,7 +128,10 @@ public class MangaDex : MangaConnector
|
||||
List<MangaAltTitle> altTitles = altTitlesDict.Select(t => new MangaAltTitle(t.Key, t.Value)).ToList();
|
||||
|
||||
if (!attributes.TryGetPropertyValue("description", out JsonNode? descriptionNode))
|
||||
{
|
||||
Log.Info("description was null");
|
||||
return null;
|
||||
}
|
||||
string description = descriptionNode!.AsObject().ContainsKey("en") switch
|
||||
{
|
||||
true => descriptionNode.AsObject()["en"]!.GetValue<string>(),
|
||||
@ -154,12 +177,18 @@ public class MangaDex : MangaConnector
|
||||
List<MangaTag> mangaTags = tags.Select(t => new MangaTag(t)).ToList();
|
||||
|
||||
if (!manga.TryGetPropertyValue("relationships", out JsonNode? relationshipsNode))
|
||||
{
|
||||
Log.Info("relationships was null");
|
||||
return null;
|
||||
}
|
||||
|
||||
JsonNode? coverNode = relationshipsNode!.AsArray()
|
||||
.FirstOrDefault(rel => rel!["type"]!.GetValue<string>().Equals("cover_art"));
|
||||
if (coverNode is null)
|
||||
{
|
||||
Log.Info("coverNode was null");
|
||||
return null;
|
||||
}
|
||||
string fileName = coverNode["attributes"]!["fileName"]!.GetValue<string>();
|
||||
string coverUrl = $"https://uploads.mangadex.org/covers/{publicationId}/{fileName}";
|
||||
|
||||
@ -195,16 +224,22 @@ public class MangaDex : MangaConnector
|
||||
while (offset < total)
|
||||
{
|
||||
//Request next "Page"
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(
|
||||
$"https://api.mangadex.org/manga/{manga.IdOnConnectorSite}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&contentRating%5B%5D=pornographic", RequestType.MangaDexFeed);
|
||||
string requestUrl = $"https://api.mangadex.org/manga/{manga.IdOnConnectorSite}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}" +
|
||||
$"&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&contentRating%5B%5D=pornographic";
|
||||
RequestResult requestResult = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
{
|
||||
Log.Info($"{requestResult.statusCode}: {requestUrl}");
|
||||
break;
|
||||
}
|
||||
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
|
||||
|
||||
offset += limit;
|
||||
if (result is null)
|
||||
{
|
||||
Log.Info($"result was null: {requestUrl}");
|
||||
break;
|
||||
}
|
||||
|
||||
total = result["total"]!.GetValue<int>();
|
||||
JsonArray chaptersInResult = result["data"]!.AsArray();
|
||||
@ -235,6 +270,7 @@ public class MangaDex : MangaConnector
|
||||
if (attributes.ContainsKey("pages") && attributes["pages"] is not null &&
|
||||
attributes["pages"]!.GetValue<int>() < 1)
|
||||
{
|
||||
Log.Info($"No pages: {chapterId}");
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -246,6 +282,7 @@ public class MangaDex : MangaConnector
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log.Debug(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -258,16 +295,23 @@ public class MangaDex : MangaConnector
|
||||
{//Request URLs for Chapter-Images
|
||||
Match m = Regex.Match(chapter.Url, @"https?:\/\/mangadex.org\/chapter\/([0-9\-a-z]+)");
|
||||
if (!m.Success)
|
||||
{
|
||||
Log.Error($"Could not parse Chapter ID: {chapter.Url}");
|
||||
return [];
|
||||
}
|
||||
|
||||
string url = $"https://api.mangadex.org/at-home/server/{m.Groups[1].Value}?forcePort443=false";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{m.Groups[1].Value}?forcePort443=false", RequestType.MangaDexImage);
|
||||
downloadClient.MakeRequest(url, RequestType.MangaDexImage);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
{
|
||||
Log.Info($"{requestResult.statusCode}: {url}");
|
||||
return [];
|
||||
}
|
||||
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
|
||||
if (result is null)
|
||||
{
|
||||
Log.Info($"Result was null: {url}");
|
||||
return [];
|
||||
}
|
||||
string baseUrl = result["baseUrl"]!.GetValue<string>();
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user