From 68cc23e158c9187425fe028c2ffde13593c4efc4 Mon Sep 17 00:00:00 2001 From: Glax Date: Mon, 3 Mar 2025 15:02:40 +0100 Subject: [PATCH] Mangaconnector-Update: Remove MangaNato Update Weebcentral --- API/Program.cs | 1 - API/Schema/MangaConnectors/Manganato.cs | 224 ---------------------- API/Schema/MangaConnectors/WeebCentral.cs | 217 ++++++++------------- API/Schema/PgsqlContext.cs | 1 - Tranga/MangaConnectors/WeebCentral.cs | 209 -------------------- 5 files changed, 81 insertions(+), 571 deletions(-) delete mode 100644 API/Schema/MangaConnectors/Manganato.cs delete mode 100644 Tranga/MangaConnectors/WeebCentral.cs diff --git a/API/Program.cs b/API/Program.cs index 869d655..fe444af 100644 --- a/API/Program.cs +++ b/API/Program.cs @@ -104,7 +104,6 @@ using (var scope = app.Services.CreateScope()) new MangaDex(), new MangaHere(), new MangaKatana(), - new Manganato(), new Mangaworld(), new ManhuaPlus(), new Weebcentral() diff --git a/API/Schema/MangaConnectors/Manganato.cs b/API/Schema/MangaConnectors/Manganato.cs deleted file mode 100644 index 0ff9b8c..0000000 --- a/API/Schema/MangaConnectors/Manganato.cs +++ /dev/null @@ -1,224 +0,0 @@ -using System.Globalization; -using System.Net; -using System.Text.RegularExpressions; -using API.MangaDownloadClients; -using HtmlAgilityPack; - -namespace API.Schema.MangaConnectors; - -public class Manganato : MangaConnector -{ - public Manganato() : base("Manganato", ["en"], ["manganato.com"]) - { - this.downloadClient = new HttpDownloadClient(); - } - - public override (Manga, List?, List?, List?, List?)[] GetManga(string publicationTitle = "") - { - string sanitizedTitle = string.Join('_', Regex.Matches(publicationTitle, "[A-z]*").Where(str => str.Length > 0)).ToLower(); - string requestUrl = $"https://manganato.com/search/story/{sanitizedTitle}"; - RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, RequestType.Default); - if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 ||requestResult.htmlDocument is null) - return []; - (Manga, List?, List?, List?, List?)[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument); - return publications; - } - - private (Manga, List?, List?, List?, List?)[] ParsePublicationsFromHtml(HtmlDocument document) - { - List searchResults = document.DocumentNode.Descendants("div").Where(n => n.HasClass("search-story-item")).ToList(); - List urls = new(); - foreach (HtmlNode mangaResult in searchResults) - { - urls.Add(mangaResult.Descendants("a").First(n => n.HasClass("item-title")).GetAttributes() - .First(a => a.Name == "href").Value); - } - - List<(Manga, List?, List?, List?, List?)> ret = new(); - foreach (string url in urls) - { - (Manga, List?, List?, List?, List?)? manga = GetMangaFromUrl(url); - if (manga is { } x) - ret.Add(x); - } - - return ret.ToArray(); - } - - public override (Manga, List?, List?, List?, List?)? GetMangaFromId(string publicationId) - { - return GetMangaFromUrl($"https://chapmanganato.com/{publicationId}"); - } - - public override (Manga, List?, List?, List?, List?)? GetMangaFromUrl(string url) - { - RequestResult requestResult = - downloadClient.MakeRequest(url, RequestType.MangaInfo); - if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) - return null; - - if (requestResult.htmlDocument is null) - return null; - return ParseSinglePublicationFromHtml(requestResult.htmlDocument, url.Split('/')[^1], url); - } - - private (Manga, List?, List?, List?, List?) ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl) - { - Dictionary altTitlesDict = new(); - Dictionary? links = null; - HashSet tags = new(); - string[] authorNames = []; - string originalLanguage = ""; - MangaReleaseStatus releaseStatus = MangaReleaseStatus.Unreleased; - - HtmlNode infoNode = document.DocumentNode.Descendants("div").First(d => d.HasClass("story-info-right")); - - string sortName = infoNode.Descendants("h1").First().InnerText; - - HtmlNode infoTable = infoNode.Descendants().First(d => d.Name == "table"); - - foreach (HtmlNode row in infoTable.Descendants("tr")) - { - string key = row.SelectNodes("td").First().InnerText.ToLower(); - string value = row.SelectNodes("td").Last().InnerText; - string keySanitized = string.Concat(Regex.Matches(key, "[a-z]")); - - switch (keySanitized) - { - case "alternative": - string[] alts = value.Split(" ; "); - for(int i = 0; i < alts.Length; i++) - altTitlesDict.Add(i.ToString(), alts[i]); - break; - case "authors": - authorNames = value.Split('-'); - for (int i = 0; i < authorNames.Length; i++) - authorNames[i] = authorNames[i].Replace("\r\n", ""); - break; - case "status": - switch (value.ToLower()) - { - case "ongoing": releaseStatus = MangaReleaseStatus.Continuing; break; - case "completed": releaseStatus = MangaReleaseStatus.Completed; break; - } - break; - case "genres": - string[] genres = value.Split(" - "); - for (int i = 0; i < genres.Length; i++) - genres[i] = genres[i].Replace("\r\n", ""); - tags = genres.ToHashSet(); - break; - } - } - List authors = authorNames.Select(n => new Author(n)).ToList(); - List mangaTags = tags.Select(n => new MangaTag(n)).ToList(); - List mangaAltTitles = altTitlesDict.Select(a => new MangaAltTitle(a.Key, a.Value)).ToList(); - - string coverUrl = document.DocumentNode.Descendants("span").First(s => s.HasClass("info-image")).Descendants("img").First() - .GetAttributes().First(a => a.Name == "src").Value; - - string description = document.DocumentNode.Descendants("div").First(d => d.HasClass("panel-story-info-description")) - .InnerText.Replace("Description :", ""); - while (description.StartsWith('\n')) - description = description.Substring(1); - - string pattern = "MMM dd,yyyy HH:mm"; - - HtmlNode? oldestChapter = document.DocumentNode - .SelectNodes("//span[contains(concat(' ',normalize-space(@class),' '),' chapter-time ')]").MaxBy( - node => DateTime.ParseExact(node.GetAttributeValue("title", "Dec 31 2400, 23:59"), pattern, - CultureInfo.InvariantCulture).Millisecond); - - - uint year = (uint)DateTime.ParseExact(oldestChapter?.GetAttributeValue("title", "Dec 31 2400, 23:59")??"Dec 31 2400, 23:59", pattern, - CultureInfo.InvariantCulture).Year; - - Manga manga = new (publicationId, sortName, description, websiteUrl, coverUrl, null, year, - originalLanguage, releaseStatus, -1, - this, - authors, - mangaTags, - [], - mangaAltTitles); - - return (manga, authors, mangaTags, [], mangaAltTitles); - } - - public override Chapter[] GetChapters(Manga manga, string language="en") - { - string requestUrl = $"https://chapmanganato.com/{manga.MangaId}"; - RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, RequestType.Default); - if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) - return []; - - //Return Chapters ordered by Chapter-Number - if (requestResult.htmlDocument is null) - return []; - List chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument); - return chapters.Order().ToArray(); - } - - - private List ParseChaptersFromHtml(Manga manga, HtmlDocument document) - { - List ret = new(); - - HtmlNode chapterList = document.DocumentNode.Descendants("ul").First(l => l.HasClass("row-content-chapter")); - - Regex volRex = new(@"Vol\.([0-9]+).*"); - Regex chapterRex = new(@"https:\/\/chapmanganato.[A-z]+\/manga-[A-z0-9]+\/chapter-([0-9\.]+)"); - Regex nameRex = new(@"Chapter ([0-9]+(\.[0-9]+)*){1}:? (.*)"); - - foreach (HtmlNode chapterInfo in chapterList.Descendants("li")) - { - string fullString = chapterInfo.Descendants("a").First(d => d.HasClass("chapter-name")).InnerText; - - string url = chapterInfo.Descendants("a").First(d => d.HasClass("chapter-name")) - .GetAttributeValue("href", ""); - - int? volumeNumber = volRex.IsMatch(fullString) - ? int.Parse(volRex.Match(fullString).Groups[1].Value) - : null; - string chapterNumber = new(chapterRex.Match(url).Groups[1].Value); - string chapterName = nameRex.Match(fullString).Groups[3].Value; - try - { - ret.Add(new Chapter(manga, url, chapterNumber, volumeNumber, chapterName)); - } - catch (Exception e) - { - } - } - ret.Reverse(); - return ret; - } - - internal override string[] GetChapterImageUrls(Chapter chapter) - { - string requestUrl = chapter.Url; - RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, RequestType.Default); - if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || - requestResult.htmlDocument is null) - { - return []; - } - - string[] imageUrls = ParseImageUrlsFromHtml(requestResult.htmlDocument); - return imageUrls; - } - - private string[] ParseImageUrlsFromHtml(HtmlDocument document) - { - List ret = new(); - - HtmlNode imageContainer = - document.DocumentNode.Descendants("div").First(i => i.HasClass("container-chapter-reader")); - foreach(HtmlNode imageNode in imageContainer.Descendants("img")) - ret.Add(imageNode.GetAttributeValue("src", "")); - - return ret.ToArray(); - } -} \ No newline at end of file diff --git a/API/Schema/MangaConnectors/WeebCentral.cs b/API/Schema/MangaConnectors/WeebCentral.cs index aa1e6a9..dbf0a85 100644 --- a/API/Schema/MangaConnectors/WeebCentral.cs +++ b/API/Schema/MangaConnectors/WeebCentral.cs @@ -1,14 +1,12 @@ -using System.Text.RegularExpressions; +using System.Net; +using System.Text.RegularExpressions; using API.MangaDownloadClients; using HtmlAgilityPack; -using Soenneker.Utils.String.NeedlemanWunsch; namespace API.Schema.MangaConnectors; public class Weebcentral : MangaConnector { - private readonly string _baseUrl = "https://weebcentral.com"; - private readonly string[] _filterWords = { "a", "the", "of", "as", "to", "no", "for", "on", "with", "be", "and", "in", "wa", "at", "be", "ni" }; @@ -17,152 +15,108 @@ public class Weebcentral : MangaConnector downloadClient = new ChromiumDownloadClient(); } - public override (Manga, List?, List?, List?, List?)[] GetManga( - string publicationTitle = "") + public override (Manga, List?, List?, List?, List?)[] GetManga(string publicationTitle = "") { const int limit = 32; //How many values we want returned at once - int offset = 0; //"Page" - string requestUrl = - $"{_baseUrl}/search/data?limit={limit}&offset={offset}&text={publicationTitle}&sort=Best+Match&order=Ascending&official=Any&display_mode=Minimal%20Display"; - RequestResult requestResult = + var offset = 0; //"Page" + var requestUrl = + $"{BaseUris[0]}/search/data?limit={limit}&offset={offset}&text={publicationTitle}&sort=Best+Match&order=Ascending&official=Any&display_mode=Minimal%20Display"; + var requestResult = downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.htmlDocument == null) + { return []; + } - (Manga, List, List, List, List)[] publications = - ParsePublicationsFromHtml(requestResult.htmlDocument); - + var publications = ParsePublicationsFromHtml(requestResult.htmlDocument); + return publications; } - private (Manga, List?, List?, List?, List?)[] ParsePublicationsFromHtml( - HtmlDocument document) + private (Manga, List?, List?, List?, List?)[] ParsePublicationsFromHtml(HtmlDocument document) { if (document.DocumentNode.SelectNodes("//article") == null) return []; - List urls = document.DocumentNode.SelectNodes("/html/body/article/a[@class='link link-hover']") + var urls = document.DocumentNode.SelectNodes("/html/body/article/a[@class='link link-hover']") .Select(elem => elem.GetAttributeValue("href", "")).ToList(); List<(Manga, List?, List?, List?, List?)> ret = new(); - foreach (string url in urls) + foreach (var url in urls) { - (Manga, List, List, List, List)? manga = GetMangaFromUrl(url); - if (manga is { } x) - ret.Add(x); + (Manga, List?, List?, List?, List?)? manga = GetMangaFromUrl(url); + if (manga is { }) + ret.Add(((Manga, List?, List?, List?, List?))manga); } return ret.ToArray(); } - public override (Manga, List?, List?, List?, List?)? - GetMangaFromUrl(string url) + public override (Manga, List?, List?, List?, List?)? GetMangaFromUrl(string url) { Regex publicationIdRex = new(@"https:\/\/weebcentral\.com\/series\/(\w*)\/(.*)"); - string publicationId = publicationIdRex.Match(url).Groups[1].Value; + var publicationId = publicationIdRex.Match(url).Groups[1].Value; - RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo); + var requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo); if ((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 && requestResult.htmlDocument is not null) return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url); return null; } - private (Manga, List?, List?, List?, List?) ParseSinglePublicationFromHtml( - HtmlDocument document, string publicationId, string websiteUrl) + private (Manga, List?, List?, List?, List?) ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl) { - HtmlNode? posterNode = + HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//section[@class='flex items-center justify-center']/picture/img"); - string coverUrl = posterNode?.GetAttributeValue("src", "") ?? ""; + string posterUrl = posterNode?.GetAttributeValue("src", "") ?? ""; - HtmlNode? titleNode = document.DocumentNode.SelectSingleNode("//section/h1"); + HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//section/h1"); string sortName = titleNode?.InnerText ?? "Undefined"; HtmlNode[] authorsNodes = document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Author(s): ']/span")?.ToArray() ?? []; - List authorNames = authorsNodes.Select(n => n.InnerText).ToList(); - List authors = authorNames.Select(n => new Author(n)).ToList(); + List authors = authorsNodes.Select(n => new Author(n.InnerText)).ToList(); HtmlNode[] genreNodes = document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Tags(s): ']/span")?.ToArray() ?? []; - HashSet tags = genreNodes.Select(n => n.InnerText).ToHashSet(); - List mangaTags = tags.Select(t => new MangaTag(t)).ToList(); + List tags = genreNodes.Select(n => new MangaTag(n.InnerText)).ToList(); - HtmlNode? statusNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Status: ']/a"); - string status = statusNode?.InnerText ?? ""; - MangaReleaseStatus releaseStatus = MangaReleaseStatus.Unreleased; - switch (status.ToLower()) + HtmlNode statusNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Status: ']/a"); + string statusText = statusNode?.InnerText ?? ""; + MangaReleaseStatus releaseStatus = statusText.ToLower() switch { - case "cancelled": releaseStatus = MangaReleaseStatus.Cancelled; break; - case "hiatus": releaseStatus = MangaReleaseStatus.OnHiatus; break; - case "complete": releaseStatus = MangaReleaseStatus.Completed; break; - case "ongoing": releaseStatus = MangaReleaseStatus.Continuing; break; - } + "cancelled" => MangaReleaseStatus.Cancelled, + "hiatus" => MangaReleaseStatus.OnHiatus, + "complete" => MangaReleaseStatus.Completed, + "ongoing" => MangaReleaseStatus.Continuing, + _ => MangaReleaseStatus.Unreleased + }; - HtmlNode? yearNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Released: ']/span"); - uint year = uint.Parse(yearNode?.InnerText ?? "0"); + HtmlNode yearNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Released: ']/span"); + uint year = Convert.ToUInt32(yearNode?.InnerText ?? "0"); - HtmlNode? descriptionNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Description']/p"); + HtmlNode descriptionNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Description']/p"); string description = descriptionNode?.InnerText ?? "Undefined"; HtmlNode[] altTitleNodes = document.DocumentNode .SelectNodes("//ul/li[strong/text() = 'Associated Name(s)']/ul/li")?.ToArray() ?? []; - Dictionary altTitlesDict = new(), links = new(); - for (int i = 0; i < altTitleNodes.Length; i++) - altTitlesDict.Add(i.ToString(), altTitleNodes[i].InnerText); - List altTitles = altTitlesDict.Select(a => new MangaAltTitle(a.Key, a.Value)).ToList(); + List altTitles = altTitleNodes.Select(n => new MangaAltTitle("", n.InnerText)).ToList(); - string originalLanguage = ""; - - Manga manga = new(publicationId, sortName, description, websiteUrl, coverUrl, null, year, - originalLanguage, releaseStatus, -1, - this, - authors, - mangaTags, - [], - altTitles); - - return (manga, authors, mangaTags, [], altTitles); + Manga m = new(publicationId, sortName, description, websiteUrl, posterUrl, null, year, null, releaseStatus, -1, + this, authors, tags, [], altTitles); + return (m, authors, tags, [], altTitles); } - public override (Manga, List?, List?, List?, List?)? GetMangaFromId( - string publicationId) + public override (Manga, List?, List?, List?, List?)? GetMangaFromId(string publicationId) { return GetMangaFromUrl($"https://weebcentral.com/series/{publicationId}"); } - private string ToFilteredString(string input) - { - return string.Join(' ', input.ToLower().Split(' ').Where(word => _filterWords.Contains(word) == false)); - } - - private SearchResult[] FilteredResults(string publicationTitle, SearchResult[] unfilteredSearchResults) - { - Dictionary similarity = new(); - foreach (SearchResult sr in unfilteredSearchResults) - { - List scores = new(); - string filteredPublicationString = ToFilteredString(publicationTitle); - string filteredSString = ToFilteredString(sr.s); - scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredSString, filteredPublicationString)); - foreach (string srA in sr.a) - { - string filteredAString = ToFilteredString(srA); - scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredAString, filteredPublicationString)); - } - - similarity.Add(sr, scores.Sum() / scores.Count); - } - - List ret = similarity.OrderBy(s => s.Value).Take(10).Select(s => s.Key).ToList(); - return ret.ToArray(); - } - public override Chapter[] GetChapters(Manga manga, string language = "en") { - string requestUrl = $"{_baseUrl}/series/{manga.ConnectorId}/full-chapter-list"; - RequestResult requestResult = + var requestUrl = $"{BaseUris[0]}/series/{manga.MangaConnectorId}/full-chapter-list"; + var requestResult = downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return []; @@ -170,61 +124,52 @@ public class Weebcentral : MangaConnector //Return Chapters ordered by Chapter-Number if (requestResult.htmlDocument is null) return []; - List chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument); - return chapters.Order().ToArray(); - } - - private List ParseChaptersFromHtml(Manga manga, HtmlDocument document) - { - HtmlNode? chaptersWrapper = document.DocumentNode.SelectSingleNode("/html/body"); - - Regex chapterRex = new(@"(\d+(?:\.\d+)*)"); - Regex idRex = new(@"https:\/\/weebcentral\.com\/chapters\/(\w*)"); - - List ret = chaptersWrapper.Descendants("a").Select(elem => - { - string url = elem.GetAttributeValue("href", "") ?? "Undefined"; - - if (!url.StartsWith("https://") && !url.StartsWith("http://")) - return new Chapter(manga, "undefined", "-1"); - - Match idMatch = idRex.Match(url); - string? id = idMatch.Success ? idMatch.Groups[1].Value : null; - - string chapterNode = elem.SelectSingleNode("span[@class='grow flex items-center gap-2']/span")?.InnerText ?? - "Undefined"; - - Match chapterNumberMatch = chapterRex.Match(chapterNode); - - if (!chapterNumberMatch.Success) - return new Chapter(manga, "undefined", "-1"); - - string chapterNumber = chapterNumberMatch.Groups[1].Value; - return new Chapter(manga, url, chapterNumber); - }).Where(elem => elem.ChapterNumber.CompareTo("-1") != 0 && elem.Url != "undefined").ToList(); - - ret.Reverse(); - return ret; + var chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument); + return chapters.Order().ToArray(); } internal override string[] GetChapterImageUrls(Chapter chapter) { - RequestResult requestResult = downloadClient.MakeRequest(chapter.Url, RequestType.Default); - if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || - requestResult.htmlDocument is null) return []; + var requestResult = downloadClient.MakeRequest(chapter.Url, RequestType.Default); + if (requestResult.htmlDocument is null) + return []; - HtmlDocument? document = requestResult.htmlDocument; + var document = requestResult.htmlDocument; - HtmlNode[] imageNodes = + var imageNodes = document.DocumentNode.SelectNodes($"//section[@hx-get='{chapter.Url}/images']/img")?.ToArray() ?? []; - string[] urls = imageNodes.Select(imgNode => imgNode.GetAttributeValue("src", "")).ToArray(); + var urls = imageNodes.Select(imgNode => imgNode.GetAttributeValue("src", "")).ToArray(); + return urls; } - private struct SearchResult + private List ParseChaptersFromHtml(Manga manga, HtmlDocument document) { - public string i { get; set; } - public string s { get; set; } - public string[] a { get; set; } + var chaptersWrapper = document.DocumentNode.SelectSingleNode("/html/body"); + + Regex chapterRex = new(@"(\d+(?:\.\d+)*)"); + Regex idRex = new(@"https:\/\/weebcentral\.com\/chapters\/(\w*)"); + + var ret = chaptersWrapper.Descendants("a").Select(elem => + { + var url = elem.GetAttributeValue("href", "") ?? "Undefined"; + + if (!url.StartsWith("https://") && !url.StartsWith("http://")) + return new Chapter(manga, "", ""); + + var idMatch = idRex.Match(url); + var id = idMatch.Success ? idMatch.Groups[1].Value : null; + + var chapterNode = elem.SelectSingleNode("span[@class='grow flex items-center gap-2']/span")?.InnerText ?? + "Undefined"; + + var chapterNumberMatch = chapterRex.Match(chapterNode); + var chapterNumber = chapterNumberMatch.Success ? chapterNumberMatch.Groups[1].Value : "-1"; + + return new Chapter(manga, url, chapterNumber); + }).Where(elem => elem.ChapterNumber != String.Empty && elem.Url != string.Empty).ToList(); + + ret.Reverse(); + return ret; } } \ No newline at end of file diff --git a/API/Schema/PgsqlContext.cs b/API/Schema/PgsqlContext.cs index 915fed6..0915b55 100644 --- a/API/Schema/PgsqlContext.cs +++ b/API/Schema/PgsqlContext.cs @@ -28,7 +28,6 @@ public class PgsqlContext(DbContextOptions options) : DbContext(op .HasValue("Bato") .HasValue("MangaHere") .HasValue("MangaKatana") - .HasValue("Manganato") .HasValue("Mangaworld") .HasValue("ManhuaPlus") .HasValue("Weebcentral") diff --git a/Tranga/MangaConnectors/WeebCentral.cs b/Tranga/MangaConnectors/WeebCentral.cs deleted file mode 100644 index c5d355f..0000000 --- a/Tranga/MangaConnectors/WeebCentral.cs +++ /dev/null @@ -1,209 +0,0 @@ -using System.Net; -using System.Text.RegularExpressions; -using HtmlAgilityPack; -using Soenneker.Utils.String.NeedlemanWunsch; -using Tranga.Jobs; - -namespace Tranga.MangaConnectors; - -public class Weebcentral : MangaConnector -{ - private readonly string _baseUrl = "https://weebcentral.com"; - - private readonly string[] _filterWords = - { "a", "the", "of", "as", "to", "no", "for", "on", "with", "be", "and", "in", "wa", "at", "be", "ni" }; - - public Weebcentral(GlobalBase clone) : base(clone, "Weebcentral", ["en"]) - { - downloadClient = new ChromiumDownloadClient(clone); - } - - public override Manga[] GetManga(string publicationTitle = "") - { - Log($"Searching Publications. Term=\"{publicationTitle}\""); - const int limit = 32; //How many values we want returned at once - var offset = 0; //"Page" - var requestUrl = - $"{_baseUrl}/search/data?limit={limit}&offset={offset}&text={publicationTitle}&sort=Best+Match&order=Ascending&official=Any&display_mode=Minimal%20Display"; - var requestResult = - downloadClient.MakeRequest(requestUrl, RequestType.Default); - if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || - requestResult.htmlDocument == null) - { - Log($"Failed to retrieve search: {requestResult.statusCode}"); - return []; - } - - var publications = ParsePublicationsFromHtml(requestResult.htmlDocument); - Log($"Retrieved {publications.Length} publications. Term=\"{publicationTitle}\""); - - return publications; - } - - private Manga[] ParsePublicationsFromHtml(HtmlDocument document) - { - if (document.DocumentNode.SelectNodes("//article") == null) - return []; - - var urls = document.DocumentNode.SelectNodes("/html/body/article/a[@class='link link-hover']") - .Select(elem => elem.GetAttributeValue("href", "")).ToList(); - - HashSet ret = new(); - foreach (var url in urls) - { - var manga = GetMangaFromUrl(url); - if (manga is not null) - ret.Add((Manga)manga); - } - - return ret.ToArray(); - } - - public override Manga? GetMangaFromUrl(string url) - { - Regex publicationIdRex = new(@"https:\/\/weebcentral\.com\/series\/(\w*)\/(.*)"); - var publicationId = publicationIdRex.Match(url).Groups[1].Value; - - var requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo); - if ((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 && - requestResult.htmlDocument is not null) - return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url); - return null; - } - - private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl) - { - var posterNode = - document.DocumentNode.SelectSingleNode("//section[@class='flex items-center justify-center']/picture/img"); - var posterUrl = posterNode?.GetAttributeValue("src", "") ?? ""; - var coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover); - - var titleNode = document.DocumentNode.SelectSingleNode("//section/h1"); - var sortName = titleNode?.InnerText ?? "Undefined"; - - HtmlNode[] authorsNodes = - document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Author(s): ']/span")?.ToArray() ?? []; - var authors = authorsNodes.Select(n => n.InnerText).ToList(); - - HtmlNode[] genreNodes = - document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Tags(s): ']/span")?.ToArray() ?? []; - HashSet tags = genreNodes.Select(n => n.InnerText).ToHashSet(); - - var statusNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Status: ']/a"); - var status = statusNode?.InnerText ?? ""; - Log("unable to parse status"); - var releaseStatus = Manga.ReleaseStatusByte.Unreleased; - switch (status.ToLower()) - { - case "cancelled": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break; - case "hiatus": releaseStatus = Manga.ReleaseStatusByte.OnHiatus; break; - case "complete": releaseStatus = Manga.ReleaseStatusByte.Completed; break; - case "ongoing": releaseStatus = Manga.ReleaseStatusByte.Continuing; break; - } - - var yearNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Released: ']/span"); - var year = Convert.ToInt32(yearNode?.InnerText ?? "0"); - - var descriptionNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Description']/p"); - var description = descriptionNode?.InnerText ?? "Undefined"; - - HtmlNode[] altTitleNodes = document.DocumentNode - .SelectNodes("//ul/li[strong/text() = 'Associated Name(s)']/ul/li")?.ToArray() ?? []; - Dictionary altTitles = new(), links = new(); - for (var i = 0; i < altTitleNodes.Length; i++) - altTitles.Add(i.ToString(), altTitleNodes[i].InnerText); - - var originalLanguage = ""; - - Manga manga = new(sortName, authors.ToList(), description, altTitles, tags.ToArray(), posterUrl, - coverFileNameInCache, links, - year, originalLanguage, publicationId, releaseStatus, websiteUrl); - AddMangaToCache(manga); - return manga; - } - - public override Manga? GetMangaFromId(string publicationId) - { - return GetMangaFromUrl($"https://weebcentral.com/series/{publicationId}"); - } - - public override Chapter[] GetChapters(Manga manga, string language = "en") - { - Log($"Getting chapters {manga}"); - var requestUrl = $"{_baseUrl}/series/{manga.publicationId}/full-chapter-list"; - var requestResult = - downloadClient.MakeRequest(requestUrl, RequestType.Default); - if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) - return []; - - //Return Chapters ordered by Chapter-Number - if (requestResult.htmlDocument is null) - return []; - var chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument); - Log($"Got {chapters.Count} chapters. {manga}"); - return chapters.Order().ToArray(); - } - - private List ParseChaptersFromHtml(Manga manga, HtmlDocument document) - { - var chaptersWrapper = document.DocumentNode.SelectSingleNode("/html/body"); - - Regex chapterRex = new(@"(\d+(?:\.\d+)*)"); - Regex idRex = new(@"https:\/\/weebcentral\.com\/chapters\/(\w*)"); - - var ret = chaptersWrapper.Descendants("a").Select(elem => - { - var url = elem.GetAttributeValue("href", "") ?? "Undefined"; - - if (!url.StartsWith("https://") && !url.StartsWith("http://")) - return new Chapter(manga, null, null, "-1", "undefined"); - - var idMatch = idRex.Match(url); - var id = idMatch.Success ? idMatch.Groups[1].Value : null; - - var chapterNode = elem.SelectSingleNode("span[@class='grow flex items-center gap-2']/span")?.InnerText ?? - "Undefined"; - - var chapterNumberMatch = chapterRex.Match(chapterNode); - var chapterNumber = chapterNumberMatch.Success ? chapterNumberMatch.Groups[1].Value : "-1"; - - return new Chapter(manga, null, null, chapterNumber, url, id); - }).Where(elem => elem.chapterNumber != -1 && elem.url != "undefined").ToList(); - - ret.Reverse(); - return ret; - } - - public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) - { - if (progressToken?.cancellationRequested ?? false) - { - progressToken.Cancel(); - return HttpStatusCode.RequestTimeout; - } - - var chapterParentManga = chapter.parentManga; - if (progressToken?.cancellationRequested ?? false) - { - progressToken.Cancel(); - return HttpStatusCode.RequestTimeout; - } - - Log($"Retrieving chapter-info {chapter} {chapterParentManga}"); - - var requestResult = downloadClient.MakeRequest(chapter.url, RequestType.Default); - if (requestResult.htmlDocument is null) - { - progressToken?.Cancel(); - return HttpStatusCode.RequestTimeout; - } - - var document = requestResult.htmlDocument; - - var imageNodes = - document.DocumentNode.SelectNodes($"//section[@hx-get='{chapter.url}/images']/img")?.ToArray() ?? []; - var urls = imageNodes.Select(imgNode => imgNode.GetAttributeValue("src", "")).ToArray(); - - return DownloadChapterImages(urls, chapter, RequestType.MangaImage, progressToken: progressToken); - } -} \ No newline at end of file