Compare commits

..

No commits in common. "c94c55300cca0b4e69296721063c062bd796deab" and "9521f66bac8b09a7a6191e12519585c3ce1d1e9b" have entirely different histories.

6 changed files with 33 additions and 94 deletions

View File

@ -50,7 +50,7 @@ public class MangaConnectorController(PgsqlContext context) : Controller
/// <summary> /// <summary>
/// Enabled or disables a Connector /// Enabled or disables a Connector
/// </summary> /// </summary>
/// <param name="MangaConnectorName">ID of the connector</param> /// <param name="id">ID of the connector</param>
/// <param name="enabled">Set true to enable</param> /// <param name="enabled">Set true to enable</param>
/// <response code="200"></response> /// <response code="200"></response>
/// <response code="404">Connector with ID not found.</response> /// <response code="404">Connector with ID not found.</response>
@ -59,11 +59,11 @@ public class MangaConnectorController(PgsqlContext context) : Controller
[ProducesResponseType(Status200OK)] [ProducesResponseType(Status200OK)]
[ProducesResponseType(Status404NotFound)] [ProducesResponseType(Status404NotFound)]
[ProducesResponseType<string>(Status500InternalServerError, "text/plain")] [ProducesResponseType<string>(Status500InternalServerError, "text/plain")]
public IActionResult SetEnabled(string MangaConnectorName, bool enabled) public IActionResult SetEnabled(string id, bool enabled)
{ {
try try
{ {
MangaConnector? connector = context.MangaConnectors.Find(MangaConnectorName); MangaConnector? connector = context.MangaConnectors.Find(id);
if (connector is null) if (connector is null)
return NotFound(); return NotFound();

View File

@ -10,8 +10,6 @@ internal class ChromiumDownloadClient : DownloadClient
{ {
private static IBrowser? _browser; private static IBrowser? _browser;
private readonly HttpDownloadClient _httpDownloadClient; private readonly HttpDownloadClient _httpDownloadClient;
private readonly Thread _closeStalePagesThread;
private readonly List<KeyValuePair<IPage, DateTime>> _openPages = new ();
private static async Task<IBrowser> StartBrowser() private static async Task<IBrowser> StartBrowser()
{ {
@ -24,7 +22,31 @@ internal class ChromiumDownloadClient : DownloadClient
"--disable-setuid-sandbox", "--disable-setuid-sandbox",
"--no-sandbox"}, "--no-sandbox"},
Timeout = 30000 Timeout = 30000
}); }, new LoggerFactory([new LogProvider()])); //TODO
}
private class LogProvider : ILoggerProvider
{
//TODO
public void Dispose() { }
public ILogger CreateLogger(string categoryName) => new Logger();
}
private class Logger : ILogger
{
public Logger() : base() { }
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
{
if (logLevel <= LogLevel.Information)
return;
//TODO
}
public bool IsEnabled(LogLevel logLevel) => true;
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
} }
public ChromiumDownloadClient() public ChromiumDownloadClient()
@ -32,20 +54,6 @@ internal class ChromiumDownloadClient : DownloadClient
_httpDownloadClient = new(); _httpDownloadClient = new();
if(_browser is null) if(_browser is null)
_browser = StartBrowser().Result; _browser = StartBrowser().Result;
_closeStalePagesThread = new Thread(CheckStalePages);
_closeStalePagesThread.Start();
}
private void CheckStalePages()
{
while (true)
{
Thread.Sleep(TimeSpan.FromHours(1));
foreach ((IPage? key, DateTime value) in _openPages.Where(kv => kv.Value.Subtract(DateTime.Now) > TimeSpan.FromHours(1)))
{
key.CloseAsync().Wait();
}
}
} }
private readonly Regex _imageUrlRex = new(@"https?:\/\/.*\.(?:p?jpe?g|gif|a?png|bmp|avif|webp)(\?.*)?"); private readonly Regex _imageUrlRex = new(@"https?:\/\/.*\.(?:p?jpe?g|gif|a?png|bmp|avif|webp)(\?.*)?");
@ -61,9 +69,8 @@ internal class ChromiumDownloadClient : DownloadClient
if (_browser is null) if (_browser is null)
return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null); return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null);
IPage page = _browser.NewPageAsync().Result; IPage page = _browser.NewPageAsync().Result;
_openPages.Add(new(page, DateTime.Now));
page.SetExtraHttpHeadersAsync(new() { { "Referer", referrer } }); page.SetExtraHttpHeadersAsync(new() { { "Referer", referrer } });
page.DefaultTimeout = 30000; page.DefaultTimeout = 10000;
IResponse response; IResponse response;
try try
{ {
@ -74,7 +81,6 @@ internal class ChromiumDownloadClient : DownloadClient
{ {
//Log($"Could not load Page {url}\n{e.Message}"); //Log($"Could not load Page {url}\n{e.Message}");
page.CloseAsync(); page.CloseAsync();
_openPages.Remove(_openPages.Find(i => i.Key == page));
return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null); return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null);
} }
@ -98,13 +104,11 @@ internal class ChromiumDownloadClient : DownloadClient
} }
else else
{ {
page.CloseAsync().Wait(); page.CloseAsync();
_openPages.Remove(_openPages.Find(i => i.Key == page));
return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null); return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null);
} }
page.CloseAsync().Wait(); page.CloseAsync();
_openPages.Remove(_openPages.Find(i => i.Key == page));
return new RequestResult(response.Status, document, stream, false, ""); return new RequestResult(response.Status, document, stream, false, "");
} }
} }

View File

@ -117,8 +117,7 @@ using (var scope = app.Services.CreateScope())
new Mangaworld(), new Mangaworld(),
new ManhuaPlus(), new ManhuaPlus(),
new Weebcentral(), new Weebcentral(),
new Manganato(), new Manganato()
new Global(scope.ServiceProvider.GetService<PgsqlContext>()!)
]; ];
MangaConnector[] newConnectors = connectors.Where(c => !context.MangaConnectors.Contains(c)).ToArray(); MangaConnector[] newConnectors = connectors.Where(c => !context.MangaConnectors.Contains(c)).ToArray();
context.MangaConnectors.AddRange(newConnectors); context.MangaConnectors.AddRange(newConnectors);

View File

@ -1,55 +0,0 @@
namespace API.Schema.MangaConnectors;
public class Global : MangaConnector
{
private PgsqlContext context { get; init; }
public Global(PgsqlContext context) : base("Global", ["all"], [""], "")
{
this.context = context;
}
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(string publicationTitle = "")
{
//Get all enabled Connectors
MangaConnector[] enabledConnectors = context.MangaConnectors.Where(c => c.Enabled && c.Name != "Global").ToArray();
//Create Task for each MangaConnector to search simulatneously
Task<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[]>[] tasks =
enabledConnectors.Select(c =>
new Task<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[]>(() => c.GetManga(publicationTitle))).ToArray();
foreach (var task in tasks)
task.Start();
//Wait for all tasks to finish
do
{
Thread.Sleep(50);
}while(tasks.Any(t => t.Status < TaskStatus.RanToCompletion));
//Concatenate all results into one
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] ret =
tasks.Select(t => t.IsCompletedSuccessfully ? t.Result : []).ToArray().SelectMany(i => i).ToArray();
return ret;
}
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromUrl(string url)
{
MangaConnector? mc = context.MangaConnectors.ToArray().FirstOrDefault(c => c.ValidateUrl(url));
return mc?.GetMangaFromUrl(url) ?? null;
}
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(string publicationId)
{
return null;
}
public override Chapter[] GetChapters(Manga manga, string language = "en")
{
return manga.MangaConnector?.GetChapters(manga) ?? [];
}
internal override string[] GetChapterImageUrls(Chapter chapter)
{
return chapter.ParentManga?.MangaConnector?.GetChapterImageUrls(chapter) ?? [];
}
}

View File

@ -25,7 +25,6 @@ public class PgsqlContext(DbContextOptions<PgsqlContext> options) : DbContext(op
{ {
modelBuilder.Entity<MangaConnector>() modelBuilder.Entity<MangaConnector>()
.HasDiscriminator(c => c.Name) .HasDiscriminator(c => c.Name)
.HasValue<Global>("Global")
.HasValue<AsuraToon>("AsuraToon") .HasValue<AsuraToon>("AsuraToon")
.HasValue<Bato>("Bato") .HasValue<Bato>("Bato")
.HasValue<MangaHere>("MangaHere") .HasValue<MangaHere>("MangaHere")

View File

@ -4,7 +4,6 @@ using API.Schema.MangaConnectors;
using API.Schema.NotificationConnectors; using API.Schema.NotificationConnectors;
using log4net; using log4net;
using log4net.Config; using log4net.Config;
using Microsoft.EntityFrameworkCore;
namespace API; namespace API;
@ -129,14 +128,7 @@ public static class Tranga
context.Jobs.Update(thread.job); context.Jobs.Update(thread.job);
} }
try context.SaveChanges();
{
context.SaveChanges();
}
catch (DbUpdateException e)
{
}
Thread.Sleep(TrangaSettings.startNewJobTimeoutMs); Thread.Sleep(TrangaSettings.startNewJobTimeoutMs);
} }
} }