glax/Tranga
1
0
Fork 0
mirror of https://github.com/C9Glax/tranga.git synced 2025-02-23 07:40:13 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

[postgres-Server-V2] Update Weebcentral with changes from V1

Browse Source
This commit is contained in:
Alessandro Benetton 2025-02-02 00:18:38 +01:00
parent 725813c2f3
commit ef7ebf022d
No known key found for this signature in database
GPG Key ID: ED63A304CE2C0303
1 changed files with 71 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
API/Schema/MangaConnectors/WeebCentral.cs
View File

@ -17,37 +17,38 @@ public class Weebcentral : MangaConnector
downloadClient = new ChromiumDownloadClient(); downloadClient = new ChromiumDownloadClient();
} }
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(string publicationTitle = "") public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(
string publicationTitle = "")
{ {
const int limit = 32; //How many values we want returned at once const int limit = 32; //How many values we want returned at once
var offset = 0; //"Page" int offset = 0; //"Page"
var requestUrl = string requestUrl =
$"{_baseUrl}/search/data?limit={limit}&offset={offset}&text={publicationTitle}&sort=Best+Match&order=Ascending&official=Any&display_mode=Minimal%20Display"; $"{_baseUrl}/search/data?limit={limit}&offset={offset}&text={publicationTitle}&sort=Best+Match&order=Ascending&official=Any&display_mode=Minimal%20Display";
var requestResult = RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, RequestType.Default); downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 ||
requestResult.htmlDocument == null) requestResult.htmlDocument == null)
{
return []; return [];
}
var publications = ParsePublicationsFromHtml(requestResult.htmlDocument); (Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>)[] publications =
ParsePublicationsFromHtml(requestResult.htmlDocument);
return publications; return publications;
} }
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] ParsePublicationsFromHtml(HtmlDocument document) private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] ParsePublicationsFromHtml(
HtmlDocument document)
{ {
if (document.DocumentNode.SelectNodes("//article") == null) if (document.DocumentNode.SelectNodes("//article") == null)
return []; return [];
var urls = document.DocumentNode.SelectNodes("/html/body/article/a[@class='link link-hover']") List<string> urls = document.DocumentNode.SelectNodes("/html/body/article/a[@class='link link-hover']")
.Select(elem => elem.GetAttributeValue("href", "")).ToList(); .Select(elem => elem.GetAttributeValue("href", "")).ToList();
List<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)> ret = new(); List<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)> ret = new();
foreach (var url in urls) foreach (string url in urls)
{ {
var manga = GetMangaFromUrl(url); (Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>)? manga = GetMangaFromUrl(url);
if (manga is { } x) if (manga is { } x)
ret.Add(x); ret.Add(x);
} }
@ -55,30 +56,32 @@ public class Weebcentral : MangaConnector
return ret.ToArray(); return ret.ToArray();
} }
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromUrl(string url) public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)?
GetMangaFromUrl(string url)
{ {
Regex publicationIdRex = new(@"https:\/\/weebcentral\.com\/series\/(\w*)\/(.*)"); Regex publicationIdRex = new(@"https:\/\/weebcentral\.com\/series\/(\w*)\/(.*)");
var publicationId = publicationIdRex.Match(url).Groups[1].Value; string publicationId = publicationIdRex.Match(url).Groups[1].Value;
var requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo); RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
if ((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 && if ((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 &&
requestResult.htmlDocument is not null) requestResult.htmlDocument is not null)
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url); return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url);
return null; return null;
} }
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?) ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl) private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?) ParseSinglePublicationFromHtml(
HtmlDocument document, string publicationId, string websiteUrl)
{ {
var posterNode = HtmlNode? posterNode =
document.DocumentNode.SelectSingleNode("//section[@class='flex items-center justify-center']/picture/img"); document.DocumentNode.SelectSingleNode("//section[@class='flex items-center justify-center']/picture/img");
var coverUrl = posterNode?.GetAttributeValue("src", "") ?? ""; string coverUrl = posterNode?.GetAttributeValue("src", "") ?? "";
var titleNode = document.DocumentNode.SelectSingleNode("//section/h1"); HtmlNode? titleNode = document.DocumentNode.SelectSingleNode("//section/h1");
var sortName = titleNode?.InnerText ?? "Undefined"; string sortName = titleNode?.InnerText ?? "Undefined";
HtmlNode[] authorsNodes = HtmlNode[] authorsNodes =
document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Author(s): ']/span")?.ToArray() ?? []; document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Author(s): ']/span")?.ToArray() ?? [];
var authorNames = authorsNodes.Select(n => n.InnerText).ToList(); List<string> authorNames = authorsNodes.Select(n => n.InnerText).ToList();
List<Author> authors = authorNames.Select(n => new Author(n)).ToList(); List<Author> authors = authorNames.Select(n => new Author(n)).ToList();
HtmlNode[] genreNodes = HtmlNode[] genreNodes =
@ -86,9 +89,9 @@ public class Weebcentral : MangaConnector
HashSet<string> tags = genreNodes.Select(n => n.InnerText).ToHashSet(); HashSet<string> tags = genreNodes.Select(n => n.InnerText).ToHashSet();
List<MangaTag> mangaTags = tags.Select(t => new MangaTag(t)).ToList(); List<MangaTag> mangaTags = tags.Select(t => new MangaTag(t)).ToList();
var statusNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Status: ']/a"); HtmlNode? statusNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Status: ']/a");
var status = statusNode?.InnerText ?? ""; string status = statusNode?.InnerText ?? "";
var releaseStatus = MangaReleaseStatus.Unreleased; MangaReleaseStatus releaseStatus = MangaReleaseStatus.Unreleased;
switch (status.ToLower()) switch (status.ToLower())
{ {
case "cancelled": releaseStatus = MangaReleaseStatus.Cancelled; break; case "cancelled": releaseStatus = MangaReleaseStatus.Cancelled; break;
@ -97,20 +100,20 @@ public class Weebcentral : MangaConnector
case "ongoing": releaseStatus = MangaReleaseStatus.Continuing; break; case "ongoing": releaseStatus = MangaReleaseStatus.Continuing; break;
} }
var yearNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Released: ']/span"); HtmlNode? yearNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Released: ']/span");
var year = uint.Parse(yearNode?.InnerText ?? "0"); uint year = uint.Parse(yearNode?.InnerText ?? "0");
var descriptionNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Description']/p"); HtmlNode? descriptionNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Description']/p");
var description = descriptionNode?.InnerText ?? "Undefined"; string description = descriptionNode?.InnerText ?? "Undefined";
HtmlNode[] altTitleNodes = document.DocumentNode HtmlNode[] altTitleNodes = document.DocumentNode
.SelectNodes("//ul/li[strong/text() = 'Associated Name(s)']/ul/li")?.ToArray() ?? []; .SelectNodes("//ul/li[strong/text() = 'Associated Name(s)']/ul/li")?.ToArray() ?? [];
Dictionary<string, string> altTitlesDict = new(), links = new(); Dictionary<string, string> altTitlesDict = new(), links = new();
for (var i = 0; i < altTitleNodes.Length; i++) for (int i = 0; i < altTitleNodes.Length; i++)
altTitlesDict.Add(i.ToString(), altTitleNodes[i].InnerText); altTitlesDict.Add(i.ToString(), altTitleNodes[i].InnerText);
List<MangaAltTitle> altTitles = altTitlesDict.Select(a => new MangaAltTitle(a.Key, a.Value)).ToList(); List<MangaAltTitle> altTitles = altTitlesDict.Select(a => new MangaAltTitle(a.Key, a.Value)).ToList();
var originalLanguage = ""; string originalLanguage = "";
Manga manga = new(publicationId, sortName, description, websiteUrl, coverUrl, null, year, Manga manga = new(publicationId, sortName, description, websiteUrl, coverUrl, null, year,
originalLanguage, releaseStatus, -1, originalLanguage, releaseStatus, -1,
@ -123,7 +126,8 @@ public class Weebcentral : MangaConnector
return (manga, authors, mangaTags, [], altTitles); return (manga, authors, mangaTags, [], altTitles);
} }
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(string publicationId) public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(
string publicationId)
{ {
return GetMangaFromUrl($"https://weebcentral.com/series/{publicationId}"); return GetMangaFromUrl($"https://weebcentral.com/series/{publicationId}");
} }
@ -136,68 +140,67 @@ public class Weebcentral : MangaConnector
private SearchResult[] FilteredResults(string publicationTitle, SearchResult[] unfilteredSearchResults) private SearchResult[] FilteredResults(string publicationTitle, SearchResult[] unfilteredSearchResults)
{ {
Dictionary<SearchResult, int> similarity = new(); Dictionary<SearchResult, int> similarity = new();
foreach (var sr in unfilteredSearchResults) foreach (SearchResult sr in unfilteredSearchResults)
{ {
List<int> scores = new(); List<int> scores = new();
var filteredPublicationString = ToFilteredString(publicationTitle); string filteredPublicationString = ToFilteredString(publicationTitle);
var filteredSString = ToFilteredString(sr.s); string filteredSString = ToFilteredString(sr.s);
scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredSString, filteredPublicationString)); scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredSString, filteredPublicationString));
foreach (var srA in sr.a) foreach (string srA in sr.a)
{ {
var filteredAString = ToFilteredString(srA); string filteredAString = ToFilteredString(srA);
scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredAString, filteredPublicationString)); scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredAString, filteredPublicationString));
} }
similarity.Add(sr, scores.Sum() / scores.Count); similarity.Add(sr, scores.Sum() / scores.Count);
} }
var ret = similarity.OrderBy(s => s.Value).Take(10).Select(s => s.Key).ToList(); List<SearchResult> ret = similarity.OrderBy(s => s.Value).Take(10).Select(s => s.Key).ToList();
return ret.ToArray(); return ret.ToArray();
} }
public override Chapter[] GetChapters(Manga manga, string language = "en") public override Chapter[] GetChapters(Manga manga, string language = "en")
{ {
var requestUrl = $"{_baseUrl}/series/{manga.ConnectorId}/full-chapter-list"; string requestUrl = $"{_baseUrl}/series/{manga.ConnectorId}/full-chapter-list";
var requestResult = RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, RequestType.Default); downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Chapter>(); return [];
//Return Chapters ordered by Chapter-Number //Return Chapters ordered by Chapter-Number
if (requestResult.htmlDocument is null) if (requestResult.htmlDocument is null)
return Array.Empty<Chapter>(); return [];
var chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument); List<Chapter> chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument);
return chapters.Order().ToArray(); return chapters.Order().ToArray();
} }
private List<Chapter> ParseChaptersFromHtml(Manga manga, HtmlDocument document) private List<Chapter> ParseChaptersFromHtml(Manga manga, HtmlDocument document)
{ {
var chaptersWrapper = document.DocumentNode.SelectSingleNode("/html/body"); HtmlNode? chaptersWrapper = document.DocumentNode.SelectSingleNode("/html/body");
Regex chapterRex = new(@".* (\d+)"); Regex chapterRex = new(@"(\d+(?:\.\d+)*)");
Regex idRex = new(@"https:\/\/weebcentral\.com\/chapters\/(\w*)"); Regex idRex = new(@"https:\/\/weebcentral\.com\/chapters\/(\w*)");
var ret = chaptersWrapper.Descendants("a").Select(elem => List<Chapter> ret = chaptersWrapper.Descendants("a").Select(elem =>
{ {
var url = elem.GetAttributeValue("href", "") ?? "Undefined"; string url = elem.GetAttributeValue("href", "") ?? "Undefined";
if (!url.StartsWith("https://") && !url.StartsWith("http://")) if (!url.StartsWith("https://") && !url.StartsWith("http://"))
return new Chapter(manga, "undefined", "-1", null, null); return new Chapter(manga, "undefined", "-1");
var idMatch = idRex.Match(url); Match idMatch = idRex.Match(url);
var id = idMatch.Success ? idMatch.Groups[1].Value : null; string? id = idMatch.Success ? idMatch.Groups[1].Value : null;
var chapterNode = elem.SelectSingleNode("span[@class='grow flex items-center gap-2']/span")?.InnerText ?? string chapterNode = elem.SelectSingleNode("span[@class='grow flex items-center gap-2']/span")?.InnerText ??
"Undefined"; "Undefined";
var chapterNumberMatch = chapterRex.Match(chapterNode); Match chapterNumberMatch = chapterRex.Match(chapterNode);
if (!chapterNumberMatch.Success) if (!chapterNumberMatch.Success)
return new Chapter(manga, "undefined", "-1", null, null); return new Chapter(manga, "undefined", "-1");
string chapterNumber = new(chapterNumberMatch.Groups[1].Value); string chapterNumber = chapterNumberMatch.Groups[1].Value;
var chapter = new Chapter(manga, url, chapterNumber, null, null); return new Chapter(manga, url, chapterNumber);
return chapter;
}).Where(elem => elem.ChapterNumber.CompareTo("-1") != 0 && elem.Url != "undefined").ToList(); }).Where(elem => elem.ChapterNumber.CompareTo("-1") != 0 && elem.Url != "undefined").ToList();
ret.Reverse(); ret.Reverse();
@ -206,17 +209,15 @@ public class Weebcentral : MangaConnector
internal override string[] GetChapterImageUrls(Chapter chapter) internal override string[] GetChapterImageUrls(Chapter chapter)
{ {
var requestResult = downloadClient.MakeRequest(chapter.Url, RequestType.Default); RequestResult requestResult = downloadClient.MakeRequest(chapter.Url, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 ||requestResult.htmlDocument is null) if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 ||
{ requestResult.htmlDocument is null) return [];
return [];
}
var document = requestResult.htmlDocument; HtmlDocument? document = requestResult.htmlDocument;
var imageNodes = HtmlNode[] imageNodes =
document.DocumentNode.SelectNodes($"//section[@hx-get='{chapter.Url}/images']/img")?.ToArray() ?? []; document.DocumentNode.SelectNodes($"//section[@hx-get='{chapter.Url}/images']/img")?.ToArray() ?? [];
var urls = imageNodes.Select(imgNode => imgNode.GetAttributeValue("src", "")).ToArray(); string[] urls = imageNodes.Select(imgNode => imgNode.GetAttributeValue("src", "")).ToArray();
return urls; return urls;
} }