From 71f663ca2f236f7a01ad4d02bae95569b19c5e41 Mon Sep 17 00:00:00 2001 From: Alessandro Benetton Date: Sat, 1 Mar 2025 11:39:12 +0100 Subject: [PATCH 01/15] [cuttingedge] fix(Weebcentral): File name also depends on original chapter name --- Tranga/Chapter.cs | 2 +- Tranga/MangaConnectors/WeebCentral.cs | 90 ++++++++++++++------------- 2 files changed, 47 insertions(+), 45 deletions(-) diff --git a/Tranga/Chapter.cs b/Tranga/Chapter.cs index 7622bd7..a4fa2c8 100644 --- a/Tranga/Chapter.cs +++ b/Tranga/Chapter.cs @@ -44,7 +44,7 @@ public readonly struct Chapter : IComparable if (name is not null && name.Length > 0) { string chapterName = IllegalStrings.Replace(string.Concat(LegalCharacters.Matches(name)), ""); - this.fileName = $"{chapterVolNumStr} - {chapterName}"; + this.fileName = chapterName.Length > 0 ? $"{chapterVolNumStr} - {chapterName}" : chapterVolNumStr; } else this.fileName = chapterVolNumStr; diff --git a/Tranga/MangaConnectors/WeebCentral.cs b/Tranga/MangaConnectors/WeebCentral.cs index c5d355f..d4636d1 100644 --- a/Tranga/MangaConnectors/WeebCentral.cs +++ b/Tranga/MangaConnectors/WeebCentral.cs @@ -1,7 +1,6 @@ using System.Net; using System.Text.RegularExpressions; using HtmlAgilityPack; -using Soenneker.Utils.String.NeedlemanWunsch; using Tranga.Jobs; namespace Tranga.MangaConnectors; @@ -22,10 +21,10 @@ public class Weebcentral : MangaConnector { Log($"Searching Publications. Term=\"{publicationTitle}\""); const int limit = 32; //How many values we want returned at once - var offset = 0; //"Page" - var requestUrl = + int offset = 0; //"Page" + string requestUrl = $"{_baseUrl}/search/data?limit={limit}&offset={offset}&text={publicationTitle}&sort=Best+Match&order=Ascending&official=Any&display_mode=Minimal%20Display"; - var requestResult = + RequestResult requestResult = downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.htmlDocument == null) @@ -34,7 +33,7 @@ public class Weebcentral : MangaConnector return []; } - var publications = ParsePublicationsFromHtml(requestResult.htmlDocument); + Manga[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument); Log($"Retrieved {publications.Length} publications. Term=\"{publicationTitle}\""); return publications; @@ -45,13 +44,13 @@ public class Weebcentral : MangaConnector if (document.DocumentNode.SelectNodes("//article") == null) return []; - var urls = document.DocumentNode.SelectNodes("/html/body/article/a[@class='link link-hover']") + List urls = document.DocumentNode.SelectNodes("/html/body/article/a[@class='link link-hover']") .Select(elem => elem.GetAttributeValue("href", "")).ToList(); HashSet ret = new(); - foreach (var url in urls) + foreach (string url in urls) { - var manga = GetMangaFromUrl(url); + Manga? manga = GetMangaFromUrl(url); if (manga is not null) ret.Add((Manga)manga); } @@ -62,9 +61,9 @@ public class Weebcentral : MangaConnector public override Manga? GetMangaFromUrl(string url) { Regex publicationIdRex = new(@"https:\/\/weebcentral\.com\/series\/(\w*)\/(.*)"); - var publicationId = publicationIdRex.Match(url).Groups[1].Value; + string publicationId = publicationIdRex.Match(url).Groups[1].Value; - var requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo); + RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo); if ((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 && requestResult.htmlDocument is not null) return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url); @@ -73,26 +72,26 @@ public class Weebcentral : MangaConnector private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl) { - var posterNode = + HtmlNode? posterNode = document.DocumentNode.SelectSingleNode("//section[@class='flex items-center justify-center']/picture/img"); - var posterUrl = posterNode?.GetAttributeValue("src", "") ?? ""; - var coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover); + string posterUrl = posterNode?.GetAttributeValue("src", "") ?? ""; + string coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover); - var titleNode = document.DocumentNode.SelectSingleNode("//section/h1"); - var sortName = titleNode?.InnerText ?? "Undefined"; + HtmlNode? titleNode = document.DocumentNode.SelectSingleNode("//section/h1"); + string sortName = titleNode?.InnerText ?? "Undefined"; HtmlNode[] authorsNodes = document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Author(s): ']/span")?.ToArray() ?? []; - var authors = authorsNodes.Select(n => n.InnerText).ToList(); + List authors = authorsNodes.Select(n => n.InnerText).ToList(); HtmlNode[] genreNodes = document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Tags(s): ']/span")?.ToArray() ?? []; HashSet tags = genreNodes.Select(n => n.InnerText).ToHashSet(); - var statusNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Status: ']/a"); - var status = statusNode?.InnerText ?? ""; + HtmlNode? statusNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Status: ']/a"); + string status = statusNode?.InnerText ?? ""; Log("unable to parse status"); - var releaseStatus = Manga.ReleaseStatusByte.Unreleased; + Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased; switch (status.ToLower()) { case "cancelled": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break; @@ -101,19 +100,19 @@ public class Weebcentral : MangaConnector case "ongoing": releaseStatus = Manga.ReleaseStatusByte.Continuing; break; } - var yearNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Released: ']/span"); - var year = Convert.ToInt32(yearNode?.InnerText ?? "0"); + HtmlNode? yearNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Released: ']/span"); + int year = Convert.ToInt32(yearNode?.InnerText ?? "0"); - var descriptionNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Description']/p"); - var description = descriptionNode?.InnerText ?? "Undefined"; + HtmlNode? descriptionNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Description']/p"); + string description = descriptionNode?.InnerText ?? "Undefined"; HtmlNode[] altTitleNodes = document.DocumentNode .SelectNodes("//ul/li[strong/text() = 'Associated Name(s)']/ul/li")?.ToArray() ?? []; Dictionary altTitles = new(), links = new(); - for (var i = 0; i < altTitleNodes.Length; i++) + for (int i = 0; i < altTitleNodes.Length; i++) altTitles.Add(i.ToString(), altTitleNodes[i].InnerText); - var originalLanguage = ""; + string originalLanguage = ""; Manga manga = new(sortName, authors.ToList(), description, altTitles, tags.ToArray(), posterUrl, coverFileNameInCache, links, @@ -130,8 +129,8 @@ public class Weebcentral : MangaConnector public override Chapter[] GetChapters(Manga manga, string language = "en") { Log($"Getting chapters {manga}"); - var requestUrl = $"{_baseUrl}/series/{manga.publicationId}/full-chapter-list"; - var requestResult = + string requestUrl = $"{_baseUrl}/series/{manga.publicationId}/full-chapter-list"; + RequestResult requestResult = downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return []; @@ -139,35 +138,38 @@ public class Weebcentral : MangaConnector //Return Chapters ordered by Chapter-Number if (requestResult.htmlDocument is null) return []; - var chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument); + List chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument); Log($"Got {chapters.Count} chapters. {manga}"); - return chapters.Order().ToArray(); + return chapters.OrderBy(c => c.name).ThenBy(c => c.volumeNumber).ThenBy(c => c.chapterNumber).ToArray(); } private List ParseChaptersFromHtml(Manga manga, HtmlDocument document) { - var chaptersWrapper = document.DocumentNode.SelectSingleNode("/html/body"); + HtmlNode? chaptersWrapper = document.DocumentNode.SelectSingleNode("/html/body"); Regex chapterRex = new(@"(\d+(?:\.\d+)*)"); + Regex chapterNameRex = new(@"(\w* )+"); Regex idRex = new(@"https:\/\/weebcentral\.com\/chapters\/(\w*)"); - var ret = chaptersWrapper.Descendants("a").Select(elem => + List ret = chaptersWrapper.Descendants("a").Select(elem => { - var url = elem.GetAttributeValue("href", "") ?? "Undefined"; + string url = elem.GetAttributeValue("href", "") ?? "Undefined"; if (!url.StartsWith("https://") && !url.StartsWith("http://")) return new Chapter(manga, null, null, "-1", "undefined"); - var idMatch = idRex.Match(url); - var id = idMatch.Success ? idMatch.Groups[1].Value : null; + Match idMatch = idRex.Match(url); + string? id = idMatch.Success ? idMatch.Groups[1].Value : null; - var chapterNode = elem.SelectSingleNode("span[@class='grow flex items-center gap-2']/span")?.InnerText ?? - "Undefined"; + string chapterNode = elem.SelectSingleNode("span[@class='grow flex items-center gap-2']/span")?.InnerText ?? + "Undefined"; - var chapterNumberMatch = chapterRex.Match(chapterNode); - var chapterNumber = chapterNumberMatch.Success ? chapterNumberMatch.Groups[1].Value : "-1"; + Match chapterNumberMatch = chapterRex.Match(chapterNode); + string chapterNumber = chapterNumberMatch.Success ? chapterNumberMatch.Groups[1].Value : "-1"; + Match chapterNameMatch = chapterNameRex.Match(chapterNode); + string chapterName = chapterNameMatch.Success ? chapterNameMatch.Groups[1].Value.Trim() : ""; - return new Chapter(manga, null, null, chapterNumber, url, id); + return new Chapter(manga, chapterName != "" ? chapterName : null, null, chapterNumber, url, id); }).Where(elem => elem.chapterNumber != -1 && elem.url != "undefined").ToList(); ret.Reverse(); @@ -182,7 +184,7 @@ public class Weebcentral : MangaConnector return HttpStatusCode.RequestTimeout; } - var chapterParentManga = chapter.parentManga; + Manga chapterParentManga = chapter.parentManga; if (progressToken?.cancellationRequested ?? false) { progressToken.Cancel(); @@ -191,18 +193,18 @@ public class Weebcentral : MangaConnector Log($"Retrieving chapter-info {chapter} {chapterParentManga}"); - var requestResult = downloadClient.MakeRequest(chapter.url, RequestType.Default); + RequestResult requestResult = downloadClient.MakeRequest(chapter.url, RequestType.Default); if (requestResult.htmlDocument is null) { progressToken?.Cancel(); return HttpStatusCode.RequestTimeout; } - var document = requestResult.htmlDocument; + HtmlDocument? document = requestResult.htmlDocument; - var imageNodes = + HtmlNode[] imageNodes = document.DocumentNode.SelectNodes($"//section[@hx-get='{chapter.url}/images']/img")?.ToArray() ?? []; - var urls = imageNodes.Select(imgNode => imgNode.GetAttributeValue("src", "")).ToArray(); + string[] urls = imageNodes.Select(imgNode => imgNode.GetAttributeValue("src", "")).ToArray(); return DownloadChapterImages(urls, chapter, RequestType.MangaImage, progressToken: progressToken); } From 5a4bc1c6de4c4e925329d462bb15d97595bc4407 Mon Sep 17 00:00:00 2001 From: Alessandro Benetton Date: Sat, 1 Mar 2025 12:06:51 +0100 Subject: [PATCH 02/15] [cuttingedge] fix(Weebcentral): Handle case of chapter name with multiple number parts --- Tranga/MangaConnectors/WeebCentral.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Tranga/MangaConnectors/WeebCentral.cs b/Tranga/MangaConnectors/WeebCentral.cs index d4636d1..9b5fe98 100644 --- a/Tranga/MangaConnectors/WeebCentral.cs +++ b/Tranga/MangaConnectors/WeebCentral.cs @@ -164,10 +164,14 @@ public class Weebcentral : MangaConnector string chapterNode = elem.SelectSingleNode("span[@class='grow flex items-center gap-2']/span")?.InnerText ?? "Undefined"; - Match chapterNumberMatch = chapterRex.Match(chapterNode); - string chapterNumber = chapterNumberMatch.Success ? chapterNumberMatch.Groups[1].Value : "-1"; - Match chapterNameMatch = chapterNameRex.Match(chapterNode); - string chapterName = chapterNameMatch.Success ? chapterNameMatch.Groups[1].Value.Trim() : ""; + MatchCollection chapterNumberMatch = chapterRex.Matches(chapterNode); + string chapterNumber = chapterNumberMatch.Count > 0 ? chapterNumberMatch[^1].Groups[1].Value : "-1"; + MatchCollection chapterNameMatch = chapterNameRex.Matches(chapterNode); + string chapterName = chapterNameMatch.Count > 0 + ? string.Join(" - ", + chapterNameMatch.Select(m => m.Groups[1].Value.Trim()) + .Where(name => name.Length > 0 && !name.Equals("Chapter", StringComparison.OrdinalIgnoreCase)).ToArray()).Trim() + : ""; return new Chapter(manga, chapterName != "" ? chapterName : null, null, chapterNumber, url, id); }).Where(elem => elem.chapterNumber != -1 && elem.url != "undefined").ToList(); From 4bafffded4a1b1a6e9cb041bcd16f87c0d965afe Mon Sep 17 00:00:00 2001 From: Alessandro Benetton Date: Sun, 2 Mar 2025 15:56:12 +0100 Subject: [PATCH 03/15] [cuttingedge] feat(Weebcentra): When ordering chapters, order name desc to put special chapters first --- Tranga/MangaConnectors/WeebCentral.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/WeebCentral.cs b/Tranga/MangaConnectors/WeebCentral.cs index 9b5fe98..1ed1438 100644 --- a/Tranga/MangaConnectors/WeebCentral.cs +++ b/Tranga/MangaConnectors/WeebCentral.cs @@ -140,7 +140,7 @@ public class Weebcentral : MangaConnector return []; List chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument); Log($"Got {chapters.Count} chapters. {manga}"); - return chapters.OrderBy(c => c.name).ThenBy(c => c.volumeNumber).ThenBy(c => c.chapterNumber).ToArray(); + return chapters.OrderByDescending(c => c.name).ThenBy(c => c.volumeNumber).ThenBy(c => c.chapterNumber).ToArray(); } private List ParseChaptersFromHtml(Manga manga, HtmlDocument document) From 9a066e7ac7dade3662a55ddf736bbd61037d3fe5 Mon Sep 17 00:00:00 2001 From: Alessandro Benetton Date: Sun, 2 Mar 2025 15:57:09 +0100 Subject: [PATCH 04/15] [cuttingedge] fix(Weebcentral): Updated CheckChapterIsDownloaded logic to also consider chapter name if present --- Tranga/Chapter.cs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Tranga/Chapter.cs b/Tranga/Chapter.cs index a4fa2c8..20ff851 100644 --- a/Tranga/Chapter.cs +++ b/Tranga/Chapter.cs @@ -96,17 +96,15 @@ public readonly struct Chapter : IComparable if(mangaArchive is null) { FileInfo[] archives = new DirectoryInfo(mangaDirectory).GetFiles("*.cbz"); - Regex volChRex = new(@"(?:Vol(?:ume)?\.([0-9]+)\D*)?Ch(?:apter)?\.([0-9]+(?:\.[0-9]+)*)"); + Regex volChRex = new(@"(?:Vol(?:ume)?\.([0-9]+)\D*)?Ch(?:apter)?\.([0-9]+(?:\.[0-9]+)*)(?: - (.*))?.cbz"); Chapter t = this; mangaArchive = archives.FirstOrDefault(archive => { Match m = volChRex.Match(archive.Name); - if (m.Groups[1].Success) - return m.Groups[1].Value == t.volumeNumber.ToString(GlobalBase.numberFormatDecimalPoint) && - m.Groups[2].Value == t.chapterNumber.ToString(GlobalBase.numberFormatDecimalPoint); - else - return m.Groups[2].Value == t.chapterNumber.ToString(GlobalBase.numberFormatDecimalPoint); + return (!m.Groups[1].Success || m.Groups[1].Value == t.volumeNumber.ToString(GlobalBase.numberFormatDecimalPoint)) && + m.Groups[2].Value == t.chapterNumber.ToString(GlobalBase.numberFormatDecimalPoint) && + (t.name == null || m.Groups[3].Value == t.name); }); } From c6bb1c918037fced9337eef436243ad890fb0f04 Mon Sep 17 00:00:00 2001 From: Alessandro Benetton Date: Sun, 2 Mar 2025 16:06:21 +0100 Subject: [PATCH 05/15] [cuttingedge] fix(Chapter): Minor logic change to account for all chapterName cases --- Tranga/Chapter.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Tranga/Chapter.cs b/Tranga/Chapter.cs index 20ff851..2992efd 100644 --- a/Tranga/Chapter.cs +++ b/Tranga/Chapter.cs @@ -102,9 +102,14 @@ public readonly struct Chapter : IComparable mangaArchive = archives.FirstOrDefault(archive => { Match m = volChRex.Match(archive.Name); + /* + * 1. If the volumeNumber is not present in the filename, it is not checked. + * 2. Check the chapterNumber in the chapter against the one in the filename. + * 3. The chpaterName has to either be absent both in the chapter and the filename or match. + */ return (!m.Groups[1].Success || m.Groups[1].Value == t.volumeNumber.ToString(GlobalBase.numberFormatDecimalPoint)) && m.Groups[2].Value == t.chapterNumber.ToString(GlobalBase.numberFormatDecimalPoint) && - (t.name == null || m.Groups[3].Value == t.name); + ((!m.Groups[3].Success && string.IsNullOrEmpty(t.name)) || m.Groups[3].Value == t.name); }); } From 1d1d01b6e54faa2ab962c09ea8c081c217df06f3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 05:09:43 +0000 Subject: [PATCH 06/15] Bump docker/setup-qemu-action from 3.5.0 to 3.6.0 Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 3.5.0 to 3.6.0. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/v3.5.0...v3.6.0) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/docker-image-cuttingedge.yml | 2 +- .github/workflows/docker-image-dev.yml | 2 +- .github/workflows/docker-image-master.yml | 2 +- .github/workflows/docker-image-serverv2.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-image-cuttingedge.yml b/.github/workflows/docker-image-cuttingedge.yml index 6521384..8a3d7e6 100644 --- a/.github/workflows/docker-image-cuttingedge.yml +++ b/.github/workflows/docker-image-cuttingedge.yml @@ -17,7 +17,7 @@ jobs: # https://github.com/docker/setup-qemu-action#usage - name: Set up QEMU - uses: docker/setup-qemu-action@v3.5.0 + uses: docker/setup-qemu-action@v3.6.0 # https://github.com/marketplace/actions/docker-setup-buildx - name: Set up Docker Buildx diff --git a/.github/workflows/docker-image-dev.yml b/.github/workflows/docker-image-dev.yml index 827a70b..76528ab 100644 --- a/.github/workflows/docker-image-dev.yml +++ b/.github/workflows/docker-image-dev.yml @@ -17,7 +17,7 @@ jobs: # https://github.com/docker/setup-qemu-action#usage - name: Set up QEMU - uses: docker/setup-qemu-action@v3.5.0 + uses: docker/setup-qemu-action@v3.6.0 # https://github.com/marketplace/actions/docker-setup-buildx - name: Set up Docker Buildx diff --git a/.github/workflows/docker-image-master.yml b/.github/workflows/docker-image-master.yml index e1e277c..fb13c57 100644 --- a/.github/workflows/docker-image-master.yml +++ b/.github/workflows/docker-image-master.yml @@ -17,7 +17,7 @@ jobs: # https://github.com/docker/setup-qemu-action#usage - name: Set up QEMU - uses: docker/setup-qemu-action@v3.5.0 + uses: docker/setup-qemu-action@v3.6.0 # https://github.com/marketplace/actions/docker-setup-buildx - name: Set up Docker Buildx diff --git a/.github/workflows/docker-image-serverv2.yml b/.github/workflows/docker-image-serverv2.yml index c3875f1..3480170 100644 --- a/.github/workflows/docker-image-serverv2.yml +++ b/.github/workflows/docker-image-serverv2.yml @@ -17,7 +17,7 @@ jobs: # https://github.com/docker/setup-qemu-action#usage - name: Set up QEMU - uses: docker/setup-qemu-action@v3.5.0 + uses: docker/setup-qemu-action@v3.6.0 # https://github.com/marketplace/actions/docker-setup-buildx - name: Set up Docker Buildx From 95d1e37b476f4eab310df4e889d1a5aecc000e9f Mon Sep 17 00:00:00 2001 From: merlinmarijn Date: Mon, 3 Mar 2025 22:27:37 +0100 Subject: [PATCH 07/15] Update Manganato.cs --- Tranga/MangaConnectors/Manganato.cs | 87 ++++++++++++----------------- 1 file changed, 37 insertions(+), 50 deletions(-) diff --git a/Tranga/MangaConnectors/Manganato.cs b/Tranga/MangaConnectors/Manganato.cs index 7d79414..58bee53 100644 --- a/Tranga/MangaConnectors/Manganato.cs +++ b/Tranga/MangaConnectors/Manganato.cs @@ -17,7 +17,7 @@ public class Manganato : MangaConnector { Log($"Searching Publications. Term=\"{publicationTitle}\""); string sanitizedTitle = string.Join('_', Regex.Matches(publicationTitle, "[A-z]*").Where(str => str.Length > 0)).ToLower(); - string requestUrl = $"https://manganato.com/search/story/{sanitizedTitle}"; + string requestUrl = $"https://manganato.gg/search/story/{sanitizedTitle}"; RequestResult requestResult = downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) @@ -32,13 +32,13 @@ public class Manganato : MangaConnector private Manga[] ParsePublicationsFromHtml(HtmlDocument document) { - List searchResults = document.DocumentNode.Descendants("div").Where(n => n.HasClass("search-story-item")).ToList(); + List searchResults = document.DocumentNode.Descendants("div").Where(n => n.HasClass("panel_story_list")).ToList(); Log($"{searchResults.Count} items."); List urls = new(); foreach (HtmlNode mangaResult in searchResults) { - urls.Add(mangaResult.Descendants("a").First(n => n.HasClass("item-title")).GetAttributes() - .First(a => a.Name == "href").Value); + urls.Add(mangaResult.Descendants("h3").First(n => n.HasClass("story_name")) + .Descendants("a").First().GetAttributeValue("href", "")); } HashSet ret = new(); @@ -78,61 +78,49 @@ public class Manganato : MangaConnector string originalLanguage = ""; Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased; - HtmlNode infoNode = document.DocumentNode.Descendants("div").First(d => d.HasClass("story-info-right")); + HtmlNode infoNode = document.DocumentNode.Descendants("ul").First(d => d.HasClass("manga-info-text")); string sortName = infoNode.Descendants("h1").First().InnerText; - HtmlNode infoTable = infoNode.Descendants().First(d => d.Name == "table"); + //HtmlNode infoTable = infoNode.Descendants().First(d => d.Name == "table"); - foreach (HtmlNode row in infoTable.Descendants("tr")) + foreach (HtmlNode li in infoNode.Descendants("li")) { - string key = row.SelectNodes("td").First().InnerText.ToLower(); - string value = row.SelectNodes("td").Last().InnerText; - string keySanitized = string.Concat(Regex.Matches(key, "[a-z]")); - - switch (keySanitized) + string text = li.InnerText.Trim().ToLower(); + + if (text.StartsWith("author(s) :")) { - case "alternative": - string[] alts = value.Split(" ; "); - for(int i = 0; i < alts.Length; i++) - altTitles.Add(i.ToString(), alts[i]); - break; - case "authors": - authors = value.Split('-'); - for (int i = 0; i < authors.Length; i++) - authors[i] = authors[i].Replace("\r\n", ""); - break; - case "status": - switch (value.ToLower()) - { - case "ongoing": releaseStatus = Manga.ReleaseStatusByte.Continuing; break; - case "completed": releaseStatus = Manga.ReleaseStatusByte.Completed; break; - } - break; - case "genres": - string[] genres = value.Split(" - "); - for (int i = 0; i < genres.Length; i++) - genres[i] = genres[i].Replace("\r\n", ""); - tags = genres.ToHashSet(); - break; + authors = li.Descendants("a").Select(a => a.InnerText.Trim()).ToArray(); + } + else if (text.StartsWith("status :")) + { + string status = text.Replace("status :", "").Trim().ToLower(); + if (status == "ongoing") + releaseStatus = Manga.ReleaseStatusByte.Continuing; + else + releaseStatus = Enum.Parse(status, true); + } + else if (li.HasClass("genres")) + { + tags = li.Descendants("a").Select(a => a.InnerText.Trim()).ToHashSet(); } } - string posterUrl = document.DocumentNode.Descendants("span").First(s => s.HasClass("info-image")).Descendants("img").First() + string posterUrl = document.DocumentNode.Descendants("div").First(s => s.HasClass("manga-info-pic")).Descendants("img").First() .GetAttributes().First(a => a.Name == "src").Value; string coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover); - string description = document.DocumentNode.Descendants("div").First(d => d.HasClass("panel-story-info-description")) + string description = document.DocumentNode.SelectSingleNode("//div[@id='contentBox']") .InnerText.Replace("Description :", ""); while (description.StartsWith('\n')) description = description.Substring(1); - string pattern = "MMM dd,yyyy HH:mm"; + string pattern = "MMM-dd-yyyy HH:mm"; HtmlNode? oldestChapter = document.DocumentNode - .SelectNodes("//span[contains(concat(' ',normalize-space(@class),' '),' chapter-time ')]").MaxBy( - node => DateTime.ParseExact(node.GetAttributeValue("title", "Dec 31 2400, 23:59"), pattern, + .SelectNodes("//div[contains(concat(' ',normalize-space(@class),' '),' row ')]/span[@title]").MaxBy( + node => DateTime.ParseExact(node.GetAttributeValue("title", "Dec-31-2400 23:59"), pattern, CultureInfo.InvariantCulture).Millisecond); @@ -148,7 +136,7 @@ public class Manganato : MangaConnector public override Chapter[] GetChapters(Manga manga, string language="en") { Log($"Getting chapters {manga}"); - string requestUrl = $"https://chapmanganato.com/{manga.publicationId}"; + string requestUrl = manga.websiteUrl; RequestResult requestResult = downloadClient.MakeRequest(requestUrl, RequestType.Default); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) @@ -166,21 +154,20 @@ public class Manganato : MangaConnector { List ret = new(); - HtmlNode chapterList = document.DocumentNode.Descendants("ul").First(l => l.HasClass("row-content-chapter")); + HtmlNode chapterList = document.DocumentNode.Descendants("div").First(l => l.HasClass("chapter-list")); Regex volRex = new(@"Vol\.([0-9]+).*"); Regex chapterRex = new(@"https:\/\/chapmanganato.[A-z]+\/manga-[A-z0-9]+\/chapter-([0-9\.]+)"); Regex nameRex = new(@"Chapter ([0-9]+(\.[0-9]+)*){1}:? (.*)"); - foreach (HtmlNode chapterInfo in chapterList.Descendants("li")) + foreach (HtmlNode chapterInfo in chapterList.Descendants("div").Where(x => x.HasClass("row"))) { - string fullString = chapterInfo.Descendants("a").First(d => d.HasClass("chapter-name")).InnerText; - - string url = chapterInfo.Descendants("a").First(d => d.HasClass("chapter-name")) - .GetAttributeValue("href", ""); - string? volumeNumber = volRex.IsMatch(fullString) ? volRex.Match(fullString).Groups[1].Value : null; - string chapterNumber = chapterRex.Match(url).Groups[1].Value; - string chapterName = nameRex.Match(fullString).Groups[3].Value; + string url = chapterInfo.Descendants("a").First().GetAttributeValue("href", ""); + string chapterName = chapterInfo.Descendants("a").First().GetAttributeValue("title", ""); + string chapterNumber = Regex.Match(chapterName, @"Chapter ([0-9]+(\.[0-9]+)*)").Groups[1].Value; + string? volumeNumber = Regex.Match(chapterName, @"Vol\.([0-9]+)").Groups[1].Value; + if (string.IsNullOrWhiteSpace(volumeNumber)) + volumeNumber = "0"; try { ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url)); From 8df05d7e8ad9b9244f79ed1bc5a9863d351b0c83 Mon Sep 17 00:00:00 2001 From: merlinmarijn Date: Mon, 3 Mar 2025 22:59:25 +0100 Subject: [PATCH 08/15] fixed image referrer --- Tranga/MangaConnectors/Manganato.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/Manganato.cs b/Tranga/MangaConnectors/Manganato.cs index 58bee53..9e54be1 100644 --- a/Tranga/MangaConnectors/Manganato.cs +++ b/Tranga/MangaConnectors/Manganato.cs @@ -109,7 +109,7 @@ public class Manganato : MangaConnector string posterUrl = document.DocumentNode.Descendants("div").First(s => s.HasClass("manga-info-pic")).Descendants("img").First() .GetAttributes().First(a => a.Name == "src").Value; - string coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover); + string coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover, "https://www.manganato.gg/"); string description = document.DocumentNode.SelectSingleNode("//div[@id='contentBox']") .InnerText.Replace("Description :", ""); From ed3ca5dba868603faa3cd85211e8d88c9eb50b33 Mon Sep 17 00:00:00 2001 From: merlinmarijn Date: Mon, 3 Mar 2025 23:04:43 +0100 Subject: [PATCH 09/15] removed leftover comment --- Tranga/MangaConnectors/Manganato.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/Tranga/MangaConnectors/Manganato.cs b/Tranga/MangaConnectors/Manganato.cs index 9e54be1..0924756 100644 --- a/Tranga/MangaConnectors/Manganato.cs +++ b/Tranga/MangaConnectors/Manganato.cs @@ -82,8 +82,6 @@ public class Manganato : MangaConnector string sortName = infoNode.Descendants("h1").First().InnerText; - //HtmlNode infoTable = infoNode.Descendants().First(d => d.Name == "table"); - foreach (HtmlNode li in infoNode.Descendants("li")) { string text = li.InnerText.Trim().ToLower(); From 59511056d0283166897828dc9966d45905a5da7c Mon Sep 17 00:00:00 2001 From: merlinmarijn Date: Mon, 3 Mar 2025 23:43:35 +0100 Subject: [PATCH 10/15] added try around getting urls --- Tranga/MangaConnectors/Manganato.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Tranga/MangaConnectors/Manganato.cs b/Tranga/MangaConnectors/Manganato.cs index 0924756..5f3c63b 100644 --- a/Tranga/MangaConnectors/Manganato.cs +++ b/Tranga/MangaConnectors/Manganato.cs @@ -37,8 +37,14 @@ public class Manganato : MangaConnector List urls = new(); foreach (HtmlNode mangaResult in searchResults) { + try + { urls.Add(mangaResult.Descendants("h3").First(n => n.HasClass("story_name")) .Descendants("a").First().GetAttributeValue("href", "")); + } catch + { + //failed to get a url, send it to the void + } } HashSet ret = new(); From c9bc79fbd5f4c3190f2816d9544da452d0e4c64d Mon Sep 17 00:00:00 2001 From: Glax Date: Fri, 7 Mar 2025 10:19:08 +0100 Subject: [PATCH 11/15] Update new_connector.yml --- .github/ISSUE_TEMPLATE/new_connector.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/new_connector.yml b/.github/ISSUE_TEMPLATE/new_connector.yml index 41ef0fd..2df60f7 100644 --- a/.github/ISSUE_TEMPLATE/new_connector.yml +++ b/.github/ISSUE_TEMPLATE/new_connector.yml @@ -12,7 +12,7 @@ body: - type: checkboxes attributes: label: Is the Website free to access? - description: We can't support pay-to-use sites. + description: We can't support pay-to-use sites, or captcha-proxied sites as Cloudflare. options: - label: The Website is freely accessible. required: true @@ -20,4 +20,4 @@ body: attributes: label: Anything else? validations: - required: false \ No newline at end of file + required: false From 49dab9a6704a075779fbf9dbabda4fd42a1f975b Mon Sep 17 00:00:00 2001 From: merlinmarijn Date: Fri, 7 Mar 2025 19:57:27 +0100 Subject: [PATCH 12/15] Referrer policy changed - Updated: image hosting platform seem to have changed a policy requiring now to send the referrer from the actual site instead of just allowing any connecting regardless of the referrer address --- Tranga/MangaConnectors/Manganato.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/Manganato.cs b/Tranga/MangaConnectors/Manganato.cs index 5f3c63b..d647050 100644 --- a/Tranga/MangaConnectors/Manganato.cs +++ b/Tranga/MangaConnectors/Manganato.cs @@ -212,7 +212,7 @@ public class Manganato : MangaConnector string[] imageUrls = ParseImageUrlsFromHtml(requestResult.htmlDocument); - return DownloadChapterImages(imageUrls, chapter, RequestType.MangaImage, "https://chapmanganato.com/", progressToken:progressToken); + return DownloadChapterImages(imageUrls, chapter, RequestType.MangaImage, "https://www.manganato.gg", progressToken:progressToken); } private string[] ParseImageUrlsFromHtml(HtmlDocument document) From 70391c83c1c99dd6cd2171009d4ed91b67873c1c Mon Sep 17 00:00:00 2001 From: merlinmarijn Date: Fri, 7 Mar 2025 21:39:17 +0100 Subject: [PATCH 13/15] Update Manganato.cs i found out, i am stupid --- Tranga/MangaConnectors/Manganato.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tranga/MangaConnectors/Manganato.cs b/Tranga/MangaConnectors/Manganato.cs index d647050..d9b0090 100644 --- a/Tranga/MangaConnectors/Manganato.cs +++ b/Tranga/MangaConnectors/Manganato.cs @@ -167,8 +167,9 @@ public class Manganato : MangaConnector foreach (HtmlNode chapterInfo in chapterList.Descendants("div").Where(x => x.HasClass("row"))) { string url = chapterInfo.Descendants("a").First().GetAttributeValue("href", ""); - string chapterName = chapterInfo.Descendants("a").First().GetAttributeValue("title", ""); - string chapterNumber = Regex.Match(chapterName, @"Chapter ([0-9]+(\.[0-9]+)*)").Groups[1].Value; + var name = chapterInfo.Descendants("a").First().InnerText.Trim(); + string chapterName = nameRex.Match(name).Groups[3].Value; + string chapterNumber = Regex.Match(name, @"Chapter ([0-9]+(\.[0-9]+)*)").Groups[1].Value; string? volumeNumber = Regex.Match(chapterName, @"Vol\.([0-9]+)").Groups[1].Value; if (string.IsNullOrWhiteSpace(volumeNumber)) volumeNumber = "0"; From 14524407f912bc40ab882b8cb5126903de4c8823 Mon Sep 17 00:00:00 2001 From: merlinmarijn Date: Fri, 7 Mar 2025 22:29:40 +0100 Subject: [PATCH 14/15] Update Manganato.cs --- Tranga/MangaConnectors/Manganato.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Tranga/MangaConnectors/Manganato.cs b/Tranga/MangaConnectors/Manganato.cs index d9b0090..afd2944 100644 --- a/Tranga/MangaConnectors/Manganato.cs +++ b/Tranga/MangaConnectors/Manganato.cs @@ -32,7 +32,7 @@ public class Manganato : MangaConnector private Manga[] ParsePublicationsFromHtml(HtmlDocument document) { - List searchResults = document.DocumentNode.Descendants("div").Where(n => n.HasClass("panel_story_list")).ToList(); + List searchResults = document.DocumentNode.Descendants("div").Where(n => n.HasClass("story_item")).ToList(); Log($"{searchResults.Count} items."); List urls = new(); foreach (HtmlNode mangaResult in searchResults) @@ -99,7 +99,9 @@ public class Manganato : MangaConnector else if (text.StartsWith("status :")) { string status = text.Replace("status :", "").Trim().ToLower(); - if (status == "ongoing") + if (status == "") + releaseStatus = Manga.ReleaseStatusByte.Continuing; + else if (status == "ongoing") releaseStatus = Manga.ReleaseStatusByte.Continuing; else releaseStatus = Enum.Parse(status, true); From ebfa34e386cd20ec434361640d8b4d3841fb65ab Mon Sep 17 00:00:00 2001 From: merlinmarijn Date: Fri, 7 Mar 2025 22:33:24 +0100 Subject: [PATCH 15/15] Update Manganato.cs --- Tranga/MangaConnectors/Manganato.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/Manganato.cs b/Tranga/MangaConnectors/Manganato.cs index afd2944..4a1d8c8 100644 --- a/Tranga/MangaConnectors/Manganato.cs +++ b/Tranga/MangaConnectors/Manganato.cs @@ -99,7 +99,7 @@ public class Manganato : MangaConnector else if (text.StartsWith("status :")) { string status = text.Replace("status :", "").Trim().ToLower(); - if (status == "") + if (string.IsNullOrWhiteSpace(status)) releaseStatus = Manga.ReleaseStatusByte.Continuing; else if (status == "ongoing") releaseStatus = Manga.ReleaseStatusByte.Continuing;