glax/Tranga
1
0
Fork 0
mirror of https://github.com/C9Glax/tranga.git synced 2025-09-10 03:48:19 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update packages, remove puppeteer

Browse Source
This commit is contained in:
glax
2025-09-01 20:20:04 +02:00
parent 5883ed6426
commit c81a3633de
3 changed files with 10 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
API/API.csproj
View File

@@ -11,26 +11,24 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="Asp.Versioning.Mvc.ApiExplorer" Version="8.1.0" /> <PackageReference Include="Asp.Versioning.Mvc.ApiExplorer" Version="8.1.0" />
<PackageReference Include="HtmlAgilityPack" Version="1.12.0" /> <PackageReference Include="HtmlAgilityPack" Version="1.12.2" />
<PackageReference Include="JikanDotNet" Version="2.9.1" /> <PackageReference Include="JikanDotNet" Version="2.9.1" />
<PackageReference Include="log4net" Version="3.0.4" /> <PackageReference Include="log4net" Version="3.2.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="9.0.3" /> <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="9.0.8" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="9.0.3" /> <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="9.0.8" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.3"> <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.8">
<PrivateAssets>all</PrivateAssets> <PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference> </PackageReference>
<PackageReference Include="Microsoft.EntityFrameworkCore.Proxies" Version="9.0.5" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" /> <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="Npgsql" Version="9.0.3" /> <PackageReference Include="Npgsql" Version="9.0.3" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="9.0.4" /> <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="9.0.4" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL.Design" Version="1.1.0" /> <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL.Design" Version="1.1.0" />
<PackageReference Include="PuppeteerSharp" Version="20.1.3" /> <PackageReference Include="SixLabors.ImageSharp" Version="3.1.11" />
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.7" /> <PackageReference Include="Soenneker.Utils.String.NeedlemanWunsch" Version="3.0.978" />
<PackageReference Include="Soenneker.Utils.String.NeedlemanWunsch" Version="3.0.929" /> <PackageReference Include="Swashbuckle.AspNetCore" Version="9.0.4" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="8.1.0" /> <PackageReference Include="Swashbuckle.AspNetCore.Newtonsoft" Version="9.0.4" />
<PackageReference Include="Swashbuckle.AspNetCore.Newtonsoft" Version="8.1.0" /> <PackageReference Include="System.Drawing.Common" Version="9.0.8" />
<PackageReference Include="System.Drawing.Common" Version="9.0.3" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>

151
API/MangaDownloadClients/ChromiumDownloadClient.cs
View File

@@ -1,151 +0,0 @@
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using log4net;
using PuppeteerSharp;
namespace API.MangaDownloadClients;
internal class ChromiumDownloadClient : DownloadClient
{
private static IBrowser? _browser;
private readonly HttpDownloadClient _httpDownloadClient;
private readonly Thread _closeStalePagesThread;
private readonly List<KeyValuePair<IPage, DateTime>> _openPages = new ();
private static async Task<IBrowser> StartBrowser(ILog log)
{
return await Puppeteer.LaunchAsync(new LaunchOptions
{
Headless = true,
Args = new [] {
"--disable-gpu",
"--disable-dev-shm-usage",
"--disable-setuid-sandbox",
"--no-sandbox"},
Timeout = 30000
}, new LoggerFactory([new Provider(log)]));
}
public ChromiumDownloadClient()
{
_httpDownloadClient = new();
if(_browser is null)
_browser = StartBrowser(Log).Result;
_closeStalePagesThread = new Thread(CheckStalePages);
_closeStalePagesThread.Start();
}
private void CheckStalePages()
{
while (true)
{
Thread.Sleep(TimeSpan.FromHours(1));
Log.Debug("Removing stale pages");
foreach ((IPage key, DateTime _) in _openPages.Where(kv => kv.Value.Subtract(DateTime.Now) > TimeSpan.FromHours(1)))
{
Log.Debug($"Closing {key.Url}");
key.CloseAsync().Wait();
}
}
}
private readonly Regex _imageUrlRex = new(@"https?:\/\/.*\.(?:p?jpe?g|gif|a?png|bmp|avif|webp)(\?.*)?");
internal override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null)
{
Log.Debug($"Requesting {url}");
return _imageUrlRex.IsMatch(url)
? _httpDownloadClient.MakeRequestInternal(url, referrer)
: MakeRequestBrowser(url, referrer, clickButton);
}
private RequestResult MakeRequestBrowser(string url, string? referrer = null, string? clickButton = null)
{
if (_browser is null)
return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null);
IPage page = _browser.NewPageAsync().Result;
_openPages.Add(new(page, DateTime.Now));
page.SetExtraHttpHeadersAsync(new() { { "Referer", referrer } });
page.DefaultTimeout = 30000;
IResponse response;
try
{
response = page.GoToAsync(url, WaitUntilNavigation.Networkidle0).Result;
Log.Debug($"Page loaded. {url}");
}
catch (Exception e)
{
Log.Info($"Could not load Page {url}\n{e.Message}");
page.CloseAsync();
_openPages.Remove(_openPages.Find(i => i.Key == page));
return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null);
}
Stream stream = Stream.Null;
HtmlDocument? document = null;
if (response.Headers.TryGetValue("Content-Type", out string? content))
{
if (content.Contains("text/html"))
{
if (clickButton is not null && page.QuerySelectorAsync(clickButton).Result is not null)
page.ClickAsync(clickButton).Wait();
string htmlString = page.GetContentAsync().Result;
stream = new MemoryStream(Encoding.Default.GetBytes(htmlString));
document = new ();
document.LoadHtml(htmlString);
}else if (content.Contains("image"))
{
stream = new MemoryStream(response.BufferAsync().Result);
}
}
else
{
page.CloseAsync().Wait();
_openPages.Remove(_openPages.Find(i => i.Key == page));
return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null);
}
page.CloseAsync().Wait();
_openPages.Remove(_openPages.Find(i => i.Key == page));
return new RequestResult(response.Status, document, stream, false, "");
}
private class Provider(ILog log) : ILoggerProvider
{
public void Dispose()
{
}
public ILogger CreateLogger(string categoryName)
{
return new ChromiumLogger(log);
}
}
private class ChromiumLogger(ILog log) : ILogger
{
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
{
string message = formatter.Invoke(state, exception);
switch(logLevel)
{
case LogLevel.Critical: log.Fatal(message); break;
case LogLevel.Error: log.Error(message); break;
case LogLevel.Warning: log.Warn(message); break;
case LogLevel.Information: log.Info(message); break;
case LogLevel.Debug: log.Debug(message); break;
default: log.Info(message); break;
}
}
public bool IsEnabled(LogLevel logLevel) => true;
public IDisposable? BeginScope<TState>(TState state) where TState : notnull
{
return null;
}
}
}

9
Dockerfile
View File

@@ -3,15 +3,6 @@ ARG DOTNET=9.0
FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/aspnet:$DOTNET AS base FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/aspnet:$DOTNET AS base
WORKDIR /publish WORKDIR /publish
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
ENV XDG_CONFIG_HOME=/tmp/.chromium
ENV XDG_CACHE_HOME=/tmp/.chromium
RUN apt-get update \
&& apt-get upgrade -y \
&& apt-get install -y chromium \
&& apt-get autopurge -y \
&& apt-get autoclean -y
FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:$DOTNET AS build-env FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:$DOTNET AS build-env
WORKDIR /src WORKDIR /src