mirror of
https://github.com/C9Glax/tranga.git
synced 2025-05-09 00:22:08 +02:00
Compare commits
7 Commits
d6018b60ae
...
6315940cd6
Author | SHA1 | Date | |
---|---|---|---|
![]() |
6315940cd6 | ||
![]() |
ef7ebf022d | ||
![]() |
725813c2f3 | ||
![]() |
a69e12179b | ||
![]() |
45ca2695eb | ||
![]() |
bd9e79d026 | ||
![]() |
6bbd09072b |
@ -1,4 +1,4 @@
|
|||||||
using API.Schema;
|
using API.Schema;
|
||||||
using API.Schema.MangaConnectors;
|
using API.Schema.MangaConnectors;
|
||||||
using Asp.Versioning;
|
using Asp.Versioning;
|
||||||
using Microsoft.AspNetCore.Mvc;
|
using Microsoft.AspNetCore.Mvc;
|
||||||
@ -82,8 +82,9 @@ public class SearchController(PgsqlContext context) : Controller
|
|||||||
{
|
{
|
||||||
if (manga is null)
|
if (manga is null)
|
||||||
return null;
|
return null;
|
||||||
|
|
||||||
Manga? existing = context.Manga.FirstOrDefault(m =>
|
Manga? existing = context.Manga.FirstOrDefault(m =>
|
||||||
m.MangaConnector == manga.MangaConnector && m.ConnectorId == manga.ConnectorId);
|
m.MangaId == manga.MangaId);
|
||||||
|
|
||||||
if (tags is not null)
|
if (tags is not null)
|
||||||
{
|
{
|
||||||
|
@ -24,9 +24,9 @@ internal abstract class DownloadClient
|
|||||||
: TrangaSettings.requestLimits[requestType];
|
: TrangaSettings.requestLimits[requestType];
|
||||||
|
|
||||||
TimeSpan timeBetweenRequests = TimeSpan.FromMinutes(1).Divide(rateLimit);
|
TimeSpan timeBetweenRequests = TimeSpan.FromMinutes(1).Divide(rateLimit);
|
||||||
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(timeBetweenRequests));
|
_lastExecutedRateLimit.TryAdd(requestType, DateTime.UtcNow.Subtract(timeBetweenRequests));
|
||||||
|
|
||||||
TimeSpan rateLimitTimeout = timeBetweenRequests.Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]));
|
TimeSpan rateLimitTimeout = timeBetweenRequests.Subtract(DateTime.UtcNow.Subtract(_lastExecutedRateLimit[requestType]));
|
||||||
|
|
||||||
if (rateLimitTimeout > TimeSpan.Zero)
|
if (rateLimitTimeout > TimeSpan.Zero)
|
||||||
{
|
{
|
||||||
@ -34,7 +34,7 @@ internal abstract class DownloadClient
|
|||||||
}
|
}
|
||||||
|
|
||||||
RequestResult result = MakeRequestInternal(url, referrer, clickButton);
|
RequestResult result = MakeRequestInternal(url, referrer, clickButton);
|
||||||
_lastExecutedRateLimit[requestType] = DateTime.Now;
|
_lastExecutedRateLimit[requestType] = DateTime.UtcNow;
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -124,7 +124,7 @@ using (var scope = app.Services.CreateScope())
|
|||||||
|
|
||||||
TrangaSettings.Load();
|
TrangaSettings.Load();
|
||||||
Tranga.StartLogger();
|
Tranga.StartLogger();
|
||||||
Tranga.JobStarterThread.Start(app.Services.CreateScope().ServiceProvider.GetService<PgsqlContext>());
|
Tranga.JobStarterThread.Start(app.Services);
|
||||||
Tranga.NotificationSenderThread.Start(app.Services.CreateScope().ServiceProvider.GetService<PgsqlContext>());
|
Tranga.NotificationSenderThread.Start(app.Services.CreateScope().ServiceProvider.GetService<PgsqlContext>());
|
||||||
|
|
||||||
app.UseCors("AllowAll");
|
app.UseCors("AllowAll");
|
||||||
|
@ -80,8 +80,7 @@ public class Chapter : IComparable<Chapter>
|
|||||||
{
|
{
|
||||||
string oldPath = GetArchiveFilePath();
|
string oldPath = GetArchiveFilePath();
|
||||||
ArchiveFileName = BuildArchiveFileName();
|
ArchiveFileName = BuildArchiveFileName();
|
||||||
if (Downloaded) return new MoveFileOrFolderJob(oldPath, GetArchiveFilePath());
|
return Downloaded ? new MoveFileOrFolderJob(oldPath, GetArchiveFilePath()) : null;
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@ -101,9 +100,12 @@ public class Chapter : IComparable<Chapter>
|
|||||||
|
|
||||||
private static int CompareChapterNumbers(string ch1, string ch2)
|
private static int CompareChapterNumbers(string ch1, string ch2)
|
||||||
{
|
{
|
||||||
int[] ch1Arr = ch1.Split('.').Select(c => int.Parse(c)).ToArray();
|
int[] ch1Arr = ch1.Split('.').Select(c => int.TryParse(c, out int result) ? result : -1).ToArray();
|
||||||
int[] ch2Arr = ch2.Split('.').Select(c => int.Parse(c)).ToArray();
|
int[] ch2Arr = ch2.Split('.').Select(c => int.TryParse(c, out int result) ? result : -1).ToArray();
|
||||||
|
|
||||||
|
if (ch1Arr.Contains(-1) || ch2Arr.Contains(-1))
|
||||||
|
throw new ArgumentException("Chapter number is not in correct format");
|
||||||
|
|
||||||
int i = 0, j = 0;
|
int i = 0, j = 0;
|
||||||
|
|
||||||
while (i < ch1Arr.Length && j < ch2Arr.Length)
|
while (i < ch1Arr.Length && j < ch2Arr.Length)
|
||||||
|
@ -79,7 +79,6 @@ public class DownloadMangaCoverJob(string chapterId, string? parentJobId = null,
|
|||||||
{
|
{
|
||||||
if (!TrangaSettings.bwImages && TrangaSettings.compression == 100)
|
if (!TrangaSettings.bwImages && TrangaSettings.compression == 100)
|
||||||
return;
|
return;
|
||||||
DateTime start = DateTime.Now;
|
|
||||||
using Image image = Image.Load(imagePath);
|
using Image image = Image.Load(imagePath);
|
||||||
File.Delete(imagePath);
|
File.Delete(imagePath);
|
||||||
if(TrangaSettings.bwImages)
|
if(TrangaSettings.bwImages)
|
||||||
|
@ -12,11 +12,22 @@ public class DownloadNewChaptersJob(ulong recurrenceMs, string mangaId, string?
|
|||||||
|
|
||||||
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
|
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
|
||||||
{
|
{
|
||||||
Manga m = Manga ?? context.Manga.Find(MangaId)!;
|
/*
|
||||||
MangaConnector connector = m.MangaConnector ?? context.MangaConnectors.Find(m.MangaConnectorId)!;
|
* For some reason, directly using Manga from above instead of finding it again causes DBContext to consider
|
||||||
Chapter[] newChapters = connector.GetNewChapters(m);
|
* Manga as a new entity and Postgres throws a Duplicate PK exception.
|
||||||
|
* m.MangaConnector does not have this issue (IDK why).
|
||||||
|
*/
|
||||||
|
Manga m = context.Manga.Find(MangaId)!;
|
||||||
|
MangaConnector connector = context.MangaConnectors.Find(m.MangaConnectorId)!;
|
||||||
|
// This gets all chapters that are not downloaded
|
||||||
|
Chapter[] allNewChapters = connector.GetNewChapters(m);
|
||||||
|
|
||||||
|
// This filters out chapters that are not downloaded but already exist in the DB
|
||||||
|
string[] chapterIds = context.Chapters.Where(chapter => chapter.ParentMangaId == m.MangaId).Select(chapter => chapter.ChapterId).ToArray();
|
||||||
|
Chapter[] newChapters = allNewChapters.Where(chapter => !chapterIds.Contains(chapter.ChapterId)).ToArray();
|
||||||
context.Chapters.AddRangeAsync(newChapters).Wait();
|
context.Chapters.AddRangeAsync(newChapters).Wait();
|
||||||
context.SaveChangesAsync().Wait();
|
context.SaveChangesAsync().Wait();
|
||||||
return newChapters.Select(chapter => new DownloadSingleChapterJob(chapter.ChapterId, this.JobId));
|
|
||||||
|
return allNewChapters.Select(chapter => new DownloadSingleChapterJob(chapter.ChapterId, this.JobId));
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -81,7 +81,7 @@ public class DownloadSingleChapterJob(string chapterId, string? parentJobId = nu
|
|||||||
{
|
{
|
||||||
if (!TrangaSettings.bwImages && TrangaSettings.compression == 100)
|
if (!TrangaSettings.bwImages && TrangaSettings.compression == 100)
|
||||||
return;
|
return;
|
||||||
DateTime start = DateTime.Now;
|
DateTime start = DateTime.UtcNow;
|
||||||
using Image image = Image.Load(imagePath);
|
using Image image = Image.Load(imagePath);
|
||||||
File.Delete(imagePath);
|
File.Delete(imagePath);
|
||||||
if(TrangaSettings.bwImages)
|
if(TrangaSettings.bwImages)
|
||||||
|
@ -43,8 +43,11 @@ public abstract class Job
|
|||||||
RecurrenceMs = recurrenceMs;
|
RecurrenceMs = recurrenceMs;
|
||||||
}
|
}
|
||||||
|
|
||||||
public IEnumerable<Job> Run(PgsqlContext context)
|
public IEnumerable<Job> Run(IServiceProvider serviceProvider)
|
||||||
{
|
{
|
||||||
|
using IServiceScope scope = serviceProvider.CreateScope();
|
||||||
|
PgsqlContext context = scope.ServiceProvider.GetRequiredService<PgsqlContext>();
|
||||||
|
|
||||||
this.state = JobState.Running;
|
this.state = JobState.Running;
|
||||||
IEnumerable<Job> newJobs = RunInternal(context);
|
IEnumerable<Job> newJobs = RunInternal(context);
|
||||||
this.state = JobState.Completed;
|
this.state = JobState.Completed;
|
||||||
|
@ -30,6 +30,7 @@ public class Manga
|
|||||||
public float IgnoreChapterBefore { get; internal set; }
|
public float IgnoreChapterBefore { get; internal set; }
|
||||||
|
|
||||||
public string MangaConnectorId { get; private set; }
|
public string MangaConnectorId { get; private set; }
|
||||||
|
|
||||||
public MangaConnector? MangaConnector { get; private set; }
|
public MangaConnector? MangaConnector { get; private set; }
|
||||||
|
|
||||||
public ICollection<Author>? Authors { get; internal set; }
|
public ICollection<Author>? Authors { get; internal set; }
|
||||||
@ -57,7 +58,7 @@ public class Manga
|
|||||||
string? coverFileNameInCache, uint year, string? originalLanguage, MangaReleaseStatus releaseStatus,
|
string? coverFileNameInCache, uint year, string? originalLanguage, MangaReleaseStatus releaseStatus,
|
||||||
float ignoreChapterBefore, string mangaConnectorId)
|
float ignoreChapterBefore, string mangaConnectorId)
|
||||||
{
|
{
|
||||||
MangaId = TokenGen.CreateToken(typeof(Manga), MangaConnectorId, ConnectorId);
|
MangaId = TokenGen.CreateToken(typeof(Manga), mangaConnectorId, connectorId);
|
||||||
ConnectorId = connectorId;
|
ConnectorId = connectorId;
|
||||||
Name = name;
|
Name = name;
|
||||||
Description = description;
|
Description = description;
|
||||||
|
@ -97,7 +97,7 @@ public class Bato : MangaConnector
|
|||||||
if (!uint.TryParse(
|
if (!uint.TryParse(
|
||||||
document.DocumentNode.SelectSingleNode("//span[text()='Original Publication:']/..").LastChild.InnerText.Split('-')[0],
|
document.DocumentNode.SelectSingleNode("//span[text()='Original Publication:']/..").LastChild.InnerText.Split('-')[0],
|
||||||
out uint year))
|
out uint year))
|
||||||
year = (uint)DateTime.Now.Year;
|
year = (uint)DateTime.UtcNow.Year;
|
||||||
|
|
||||||
string status = document.DocumentNode.SelectSingleNode("//span[text()='Original Publication:']/..")
|
string status = document.DocumentNode.SelectSingleNode("//span[text()='Original Publication:']/..")
|
||||||
.ChildNodes[2].InnerText;
|
.ChildNodes[2].InnerText;
|
||||||
|
@ -127,7 +127,7 @@ public class MangaKatana : MangaConnector
|
|||||||
while (description.StartsWith('\n'))
|
while (description.StartsWith('\n'))
|
||||||
description = description.Substring(1);
|
description = description.Substring(1);
|
||||||
|
|
||||||
uint year = (uint)DateTime.Now.Year;
|
uint year = (uint)DateTime.UtcNow.Year;
|
||||||
string yearString = infoTable.Descendants("div").First(d => d.HasClass("updateAt"))
|
string yearString = infoTable.Descendants("div").First(d => d.HasClass("updateAt"))
|
||||||
.InnerText.Split('-')[^1];
|
.InnerText.Split('-')[^1];
|
||||||
|
|
||||||
|
@ -17,37 +17,38 @@ public class Weebcentral : MangaConnector
|
|||||||
downloadClient = new ChromiumDownloadClient();
|
downloadClient = new ChromiumDownloadClient();
|
||||||
}
|
}
|
||||||
|
|
||||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(string publicationTitle = "")
|
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(
|
||||||
|
string publicationTitle = "")
|
||||||
{
|
{
|
||||||
const int limit = 32; //How many values we want returned at once
|
const int limit = 32; //How many values we want returned at once
|
||||||
var offset = 0; //"Page"
|
int offset = 0; //"Page"
|
||||||
var requestUrl =
|
string requestUrl =
|
||||||
$"{_baseUrl}/search/data?limit={limit}&offset={offset}&text={publicationTitle}&sort=Best+Match&order=Ascending&official=Any&display_mode=Minimal%20Display";
|
$"{_baseUrl}/search/data?limit={limit}&offset={offset}&text={publicationTitle}&sort=Best+Match&order=Ascending&official=Any&display_mode=Minimal%20Display";
|
||||||
var requestResult =
|
RequestResult requestResult =
|
||||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 ||
|
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 ||
|
||||||
requestResult.htmlDocument == null)
|
requestResult.htmlDocument == null)
|
||||||
{
|
|
||||||
return [];
|
return [];
|
||||||
}
|
|
||||||
|
|
||||||
var publications = ParsePublicationsFromHtml(requestResult.htmlDocument);
|
(Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>)[] publications =
|
||||||
|
ParsePublicationsFromHtml(requestResult.htmlDocument);
|
||||||
|
|
||||||
return publications;
|
return publications;
|
||||||
}
|
}
|
||||||
|
|
||||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] ParsePublicationsFromHtml(HtmlDocument document)
|
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] ParsePublicationsFromHtml(
|
||||||
|
HtmlDocument document)
|
||||||
{
|
{
|
||||||
if (document.DocumentNode.SelectNodes("//article") == null)
|
if (document.DocumentNode.SelectNodes("//article") == null)
|
||||||
return [];
|
return [];
|
||||||
|
|
||||||
var urls = document.DocumentNode.SelectNodes("/html/body/article/a[@class='link link-hover']")
|
List<string> urls = document.DocumentNode.SelectNodes("/html/body/article/a[@class='link link-hover']")
|
||||||
.Select(elem => elem.GetAttributeValue("href", "")).ToList();
|
.Select(elem => elem.GetAttributeValue("href", "")).ToList();
|
||||||
|
|
||||||
List<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)> ret = new();
|
List<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)> ret = new();
|
||||||
foreach (var url in urls)
|
foreach (string url in urls)
|
||||||
{
|
{
|
||||||
var manga = GetMangaFromUrl(url);
|
(Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>)? manga = GetMangaFromUrl(url);
|
||||||
if (manga is { } x)
|
if (manga is { } x)
|
||||||
ret.Add(x);
|
ret.Add(x);
|
||||||
}
|
}
|
||||||
@ -55,30 +56,32 @@ public class Weebcentral : MangaConnector
|
|||||||
return ret.ToArray();
|
return ret.ToArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromUrl(string url)
|
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)?
|
||||||
|
GetMangaFromUrl(string url)
|
||||||
{
|
{
|
||||||
Regex publicationIdRex = new(@"https:\/\/weebcentral\.com\/series\/(\w*)\/(.*)");
|
Regex publicationIdRex = new(@"https:\/\/weebcentral\.com\/series\/(\w*)\/(.*)");
|
||||||
var publicationId = publicationIdRex.Match(url).Groups[1].Value;
|
string publicationId = publicationIdRex.Match(url).Groups[1].Value;
|
||||||
|
|
||||||
var requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||||
if ((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 &&
|
if ((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 &&
|
||||||
requestResult.htmlDocument is not null)
|
requestResult.htmlDocument is not null)
|
||||||
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url);
|
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?) ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl)
|
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?) ParseSinglePublicationFromHtml(
|
||||||
|
HtmlDocument document, string publicationId, string websiteUrl)
|
||||||
{
|
{
|
||||||
var posterNode =
|
HtmlNode? posterNode =
|
||||||
document.DocumentNode.SelectSingleNode("//section[@class='flex items-center justify-center']/picture/img");
|
document.DocumentNode.SelectSingleNode("//section[@class='flex items-center justify-center']/picture/img");
|
||||||
var coverUrl = posterNode?.GetAttributeValue("src", "") ?? "";
|
string coverUrl = posterNode?.GetAttributeValue("src", "") ?? "";
|
||||||
|
|
||||||
var titleNode = document.DocumentNode.SelectSingleNode("//section/h1");
|
HtmlNode? titleNode = document.DocumentNode.SelectSingleNode("//section/h1");
|
||||||
var sortName = titleNode?.InnerText ?? "Undefined";
|
string sortName = titleNode?.InnerText ?? "Undefined";
|
||||||
|
|
||||||
HtmlNode[] authorsNodes =
|
HtmlNode[] authorsNodes =
|
||||||
document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Author(s): ']/span")?.ToArray() ?? [];
|
document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Author(s): ']/span")?.ToArray() ?? [];
|
||||||
var authorNames = authorsNodes.Select(n => n.InnerText).ToList();
|
List<string> authorNames = authorsNodes.Select(n => n.InnerText).ToList();
|
||||||
List<Author> authors = authorNames.Select(n => new Author(n)).ToList();
|
List<Author> authors = authorNames.Select(n => new Author(n)).ToList();
|
||||||
|
|
||||||
HtmlNode[] genreNodes =
|
HtmlNode[] genreNodes =
|
||||||
@ -86,9 +89,9 @@ public class Weebcentral : MangaConnector
|
|||||||
HashSet<string> tags = genreNodes.Select(n => n.InnerText).ToHashSet();
|
HashSet<string> tags = genreNodes.Select(n => n.InnerText).ToHashSet();
|
||||||
List<MangaTag> mangaTags = tags.Select(t => new MangaTag(t)).ToList();
|
List<MangaTag> mangaTags = tags.Select(t => new MangaTag(t)).ToList();
|
||||||
|
|
||||||
var statusNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Status: ']/a");
|
HtmlNode? statusNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Status: ']/a");
|
||||||
var status = statusNode?.InnerText ?? "";
|
string status = statusNode?.InnerText ?? "";
|
||||||
var releaseStatus = MangaReleaseStatus.Unreleased;
|
MangaReleaseStatus releaseStatus = MangaReleaseStatus.Unreleased;
|
||||||
switch (status.ToLower())
|
switch (status.ToLower())
|
||||||
{
|
{
|
||||||
case "cancelled": releaseStatus = MangaReleaseStatus.Cancelled; break;
|
case "cancelled": releaseStatus = MangaReleaseStatus.Cancelled; break;
|
||||||
@ -97,33 +100,34 @@ public class Weebcentral : MangaConnector
|
|||||||
case "ongoing": releaseStatus = MangaReleaseStatus.Continuing; break;
|
case "ongoing": releaseStatus = MangaReleaseStatus.Continuing; break;
|
||||||
}
|
}
|
||||||
|
|
||||||
var yearNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Released: ']/span");
|
HtmlNode? yearNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Released: ']/span");
|
||||||
var year = uint.Parse(yearNode?.InnerText ?? "0");
|
uint year = uint.Parse(yearNode?.InnerText ?? "0");
|
||||||
|
|
||||||
var descriptionNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Description']/p");
|
HtmlNode? descriptionNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Description']/p");
|
||||||
var description = descriptionNode?.InnerText ?? "Undefined";
|
string description = descriptionNode?.InnerText ?? "Undefined";
|
||||||
|
|
||||||
HtmlNode[] altTitleNodes = document.DocumentNode
|
HtmlNode[] altTitleNodes = document.DocumentNode
|
||||||
.SelectNodes("//ul/li[strong/text() = 'Associated Name(s)']/ul/li")?.ToArray() ?? [];
|
.SelectNodes("//ul/li[strong/text() = 'Associated Name(s)']/ul/li")?.ToArray() ?? [];
|
||||||
Dictionary<string, string> altTitlesDict = new(), links = new();
|
Dictionary<string, string> altTitlesDict = new(), links = new();
|
||||||
for (var i = 0; i < altTitleNodes.Length; i++)
|
for (int i = 0; i < altTitleNodes.Length; i++)
|
||||||
altTitlesDict.Add(i.ToString(), altTitleNodes[i].InnerText);
|
altTitlesDict.Add(i.ToString(), altTitleNodes[i].InnerText);
|
||||||
List<MangaAltTitle> altTitles = altTitlesDict.Select(a => new MangaAltTitle(a.Key, a.Value)).ToList();
|
List<MangaAltTitle> altTitles = altTitlesDict.Select(a => new MangaAltTitle(a.Key, a.Value)).ToList();
|
||||||
|
|
||||||
var originalLanguage = "";
|
string originalLanguage = "";
|
||||||
|
|
||||||
Manga manga = new (publicationId, sortName, description, websiteUrl, coverUrl, null, year,
|
Manga manga = new(publicationId, sortName, description, websiteUrl, coverUrl, null, year,
|
||||||
originalLanguage, releaseStatus, -1,
|
originalLanguage, releaseStatus, -1,
|
||||||
this,
|
this,
|
||||||
authors,
|
authors,
|
||||||
mangaTags,
|
mangaTags,
|
||||||
[],
|
[],
|
||||||
altTitles);
|
altTitles);
|
||||||
|
|
||||||
return (manga, authors, mangaTags, [], altTitles);
|
return (manga, authors, mangaTags, [], altTitles);
|
||||||
}
|
}
|
||||||
|
|
||||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(string publicationId)
|
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(
|
||||||
|
string publicationId)
|
||||||
{
|
{
|
||||||
return GetMangaFromUrl($"https://weebcentral.com/series/{publicationId}");
|
return GetMangaFromUrl($"https://weebcentral.com/series/{publicationId}");
|
||||||
}
|
}
|
||||||
@ -136,68 +140,67 @@ public class Weebcentral : MangaConnector
|
|||||||
private SearchResult[] FilteredResults(string publicationTitle, SearchResult[] unfilteredSearchResults)
|
private SearchResult[] FilteredResults(string publicationTitle, SearchResult[] unfilteredSearchResults)
|
||||||
{
|
{
|
||||||
Dictionary<SearchResult, int> similarity = new();
|
Dictionary<SearchResult, int> similarity = new();
|
||||||
foreach (var sr in unfilteredSearchResults)
|
foreach (SearchResult sr in unfilteredSearchResults)
|
||||||
{
|
{
|
||||||
List<int> scores = new();
|
List<int> scores = new();
|
||||||
var filteredPublicationString = ToFilteredString(publicationTitle);
|
string filteredPublicationString = ToFilteredString(publicationTitle);
|
||||||
var filteredSString = ToFilteredString(sr.s);
|
string filteredSString = ToFilteredString(sr.s);
|
||||||
scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredSString, filteredPublicationString));
|
scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredSString, filteredPublicationString));
|
||||||
foreach (var srA in sr.a)
|
foreach (string srA in sr.a)
|
||||||
{
|
{
|
||||||
var filteredAString = ToFilteredString(srA);
|
string filteredAString = ToFilteredString(srA);
|
||||||
scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredAString, filteredPublicationString));
|
scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredAString, filteredPublicationString));
|
||||||
}
|
}
|
||||||
|
|
||||||
similarity.Add(sr, scores.Sum() / scores.Count);
|
similarity.Add(sr, scores.Sum() / scores.Count);
|
||||||
}
|
}
|
||||||
|
|
||||||
var ret = similarity.OrderBy(s => s.Value).Take(10).Select(s => s.Key).ToList();
|
List<SearchResult> ret = similarity.OrderBy(s => s.Value).Take(10).Select(s => s.Key).ToList();
|
||||||
return ret.ToArray();
|
return ret.ToArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
public override Chapter[] GetChapters(Manga manga, string language = "en")
|
public override Chapter[] GetChapters(Manga manga, string language = "en")
|
||||||
{
|
{
|
||||||
var requestUrl = $"{_baseUrl}/series/{manga.ConnectorId}/full-chapter-list";
|
string requestUrl = $"{_baseUrl}/series/{manga.ConnectorId}/full-chapter-list";
|
||||||
var requestResult =
|
RequestResult requestResult =
|
||||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||||
return Array.Empty<Chapter>();
|
return [];
|
||||||
|
|
||||||
//Return Chapters ordered by Chapter-Number
|
//Return Chapters ordered by Chapter-Number
|
||||||
if (requestResult.htmlDocument is null)
|
if (requestResult.htmlDocument is null)
|
||||||
return Array.Empty<Chapter>();
|
return [];
|
||||||
var chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument);
|
List<Chapter> chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument);
|
||||||
return chapters.Order().ToArray();
|
return chapters.Order().ToArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<Chapter> ParseChaptersFromHtml(Manga manga, HtmlDocument document)
|
private List<Chapter> ParseChaptersFromHtml(Manga manga, HtmlDocument document)
|
||||||
{
|
{
|
||||||
var chaptersWrapper = document.DocumentNode.SelectSingleNode("/html/body");
|
HtmlNode? chaptersWrapper = document.DocumentNode.SelectSingleNode("/html/body");
|
||||||
|
|
||||||
Regex chapterRex = new(@".* (\d+)");
|
Regex chapterRex = new(@"(\d+(?:\.\d+)*)");
|
||||||
Regex idRex = new(@"https:\/\/weebcentral\.com\/chapters\/(\w*)");
|
Regex idRex = new(@"https:\/\/weebcentral\.com\/chapters\/(\w*)");
|
||||||
|
|
||||||
var ret = chaptersWrapper.Descendants("a").Select(elem =>
|
List<Chapter> ret = chaptersWrapper.Descendants("a").Select(elem =>
|
||||||
{
|
{
|
||||||
var url = elem.GetAttributeValue("href", "") ?? "Undefined";
|
string url = elem.GetAttributeValue("href", "") ?? "Undefined";
|
||||||
|
|
||||||
if (!url.StartsWith("https://") && !url.StartsWith("http://"))
|
if (!url.StartsWith("https://") && !url.StartsWith("http://"))
|
||||||
return new Chapter(manga, "undefined", "-1", null, null);
|
return new Chapter(manga, "undefined", "-1");
|
||||||
|
|
||||||
var idMatch = idRex.Match(url);
|
Match idMatch = idRex.Match(url);
|
||||||
var id = idMatch.Success ? idMatch.Groups[1].Value : null;
|
string? id = idMatch.Success ? idMatch.Groups[1].Value : null;
|
||||||
|
|
||||||
var chapterNode = elem.SelectSingleNode("span[@class='grow flex items-center gap-2']/span")?.InnerText ??
|
string chapterNode = elem.SelectSingleNode("span[@class='grow flex items-center gap-2']/span")?.InnerText ??
|
||||||
"Undefined";
|
"Undefined";
|
||||||
|
|
||||||
var chapterNumberMatch = chapterRex.Match(chapterNode);
|
Match chapterNumberMatch = chapterRex.Match(chapterNode);
|
||||||
|
|
||||||
if(!chapterNumberMatch.Success)
|
if (!chapterNumberMatch.Success)
|
||||||
return new Chapter(manga, "undefined", "-1", null, null);
|
return new Chapter(manga, "undefined", "-1");
|
||||||
|
|
||||||
string chapterNumber = new(chapterNumberMatch.Groups[1].Value);
|
string chapterNumber = chapterNumberMatch.Groups[1].Value;
|
||||||
var chapter = new Chapter(manga, url, chapterNumber, null, null);
|
return new Chapter(manga, url, chapterNumber);
|
||||||
return chapter;
|
|
||||||
}).Where(elem => elem.ChapterNumber.CompareTo("-1") != 0 && elem.Url != "undefined").ToList();
|
}).Where(elem => elem.ChapterNumber.CompareTo("-1") != 0 && elem.Url != "undefined").ToList();
|
||||||
|
|
||||||
ret.Reverse();
|
ret.Reverse();
|
||||||
@ -206,17 +209,15 @@ public class Weebcentral : MangaConnector
|
|||||||
|
|
||||||
internal override string[] GetChapterImageUrls(Chapter chapter)
|
internal override string[] GetChapterImageUrls(Chapter chapter)
|
||||||
{
|
{
|
||||||
var requestResult = downloadClient.MakeRequest(chapter.Url, RequestType.Default);
|
RequestResult requestResult = downloadClient.MakeRequest(chapter.Url, RequestType.Default);
|
||||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 ||requestResult.htmlDocument is null)
|
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 ||
|
||||||
{
|
requestResult.htmlDocument is null) return [];
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
var document = requestResult.htmlDocument;
|
HtmlDocument? document = requestResult.htmlDocument;
|
||||||
|
|
||||||
var imageNodes =
|
HtmlNode[] imageNodes =
|
||||||
document.DocumentNode.SelectNodes($"//section[@hx-get='{chapter.Url}/images']/img")?.ToArray() ?? [];
|
document.DocumentNode.SelectNodes($"//section[@hx-get='{chapter.Url}/images']/img")?.ToArray() ?? [];
|
||||||
var urls = imageNodes.Select(imgNode => imgNode.GetAttributeValue("src", "")).ToArray();
|
string[] urls = imageNodes.Select(imgNode => imgNode.GetAttributeValue("src", "")).ToArray();
|
||||||
return urls;
|
return urls;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -43,7 +43,7 @@ public static class Tranga
|
|||||||
if (notifications.Any())
|
if (notifications.Any())
|
||||||
{
|
{
|
||||||
DateTime max = notifications.MaxBy(n => n.Date)!.Date;
|
DateTime max = notifications.MaxBy(n => n.Date)!.Date;
|
||||||
if (DateTime.Now.Subtract(max) > TrangaSettings.NotificationUrgencyDelay(urgency))
|
if (DateTime.UtcNow.Subtract(max) > TrangaSettings.NotificationUrgencyDelay(urgency))
|
||||||
{
|
{
|
||||||
foreach (NotificationConnector notificationConnector in context.NotificationConnectors)
|
foreach (NotificationConnector notificationConnector in context.NotificationConnectors)
|
||||||
{
|
{
|
||||||
@ -56,18 +56,22 @@ public static class Tranga
|
|||||||
context.SaveChanges();
|
context.SaveChanges();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void JobStarter(object? pgsqlContext)
|
private static void JobStarter(object? serviceProviderObj)
|
||||||
{
|
{
|
||||||
if(pgsqlContext is null) return;
|
if(serviceProviderObj is null) return;
|
||||||
PgsqlContext context = (PgsqlContext)pgsqlContext;
|
IServiceProvider serviceProvider = (IServiceProvider)serviceProviderObj;
|
||||||
|
using IServiceScope scope = serviceProvider.CreateScope();
|
||||||
string TRANGA = "\n\n _______ \n|_ _|.----..---.-..-----..-----..---.-.\n | | | _|| _ || || _ || _ |\n |___| |__| |___._||__|__||___ ||___._|\n |_____| \n\n";
|
PgsqlContext? context = scope.ServiceProvider.GetService<PgsqlContext>();
|
||||||
|
if (context is null) return;
|
||||||
|
|
||||||
|
string TRANGA =
|
||||||
|
"\n\n _______ \n|_ _|.----..---.-..-----..-----..---.-.\n | | | _|| _ || || _ || _ |\n |___| |__| |___._||__|__||___ ||___._|\n |_____| \n\n";
|
||||||
Log.Info(TRANGA);
|
Log.Info(TRANGA);
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
List<Job> completedJobs = context.Jobs.Where(j => j.state == JobState.Completed).ToList();
|
List<Job> completedJobs = context.Jobs.Where(j => j.state == JobState.Completed).ToList();
|
||||||
foreach (Job job in completedJobs)
|
foreach (Job job in completedJobs)
|
||||||
if(job.RecurrenceMs <= 0)
|
if (job.RecurrenceMs <= 0)
|
||||||
context.Jobs.Remove(job);
|
context.Jobs.Remove(job);
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -75,16 +79,17 @@ public static class Tranga
|
|||||||
job.state = JobState.Waiting;
|
job.state = JobState.Waiting;
|
||||||
context.Jobs.Update(job);
|
context.Jobs.Update(job);
|
||||||
}
|
}
|
||||||
|
|
||||||
List<Job> runJobs = context.Jobs.Where(j => j.state <= JobState.Running).ToList().Where(j => j.NextExecution < DateTime.UtcNow).ToList();
|
List<Job> runJobs = context.Jobs.Where(j => j.state <= JobState.Running).ToList()
|
||||||
|
.Where(j => j.NextExecution < DateTime.UtcNow).ToList();
|
||||||
foreach (Job job in runJobs)
|
foreach (Job job in runJobs)
|
||||||
{
|
{
|
||||||
// If the job is already running, skip it
|
// If the job is already running, skip it
|
||||||
if (RunningJobs.Values.Any(j => j.JobId == job.JobId)) continue;
|
if (RunningJobs.Values.Any(j => j.JobId == job.JobId)) continue;
|
||||||
|
|
||||||
Thread t = new (() =>
|
Thread t = new(() =>
|
||||||
{
|
{
|
||||||
IEnumerable<Job> newJobs = job.Run(context);
|
IEnumerable<Job> newJobs = job.Run(serviceProvider);
|
||||||
context.Jobs.AddRange(newJobs);
|
context.Jobs.AddRange(newJobs);
|
||||||
});
|
});
|
||||||
RunningJobs.Add(t, job);
|
RunningJobs.Add(t, job);
|
||||||
@ -99,7 +104,7 @@ public static class Tranga
|
|||||||
RunningJobs.Remove(thread.thread);
|
RunningJobs.Remove(thread.thread);
|
||||||
context.Jobs.Update(thread.job);
|
context.Jobs.Update(thread.job);
|
||||||
}
|
}
|
||||||
|
|
||||||
context.SaveChanges();
|
context.SaveChanges();
|
||||||
Thread.Sleep(2000);
|
Thread.Sleep(2000);
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user