From 1785aa28ea7f3ed247c69c654d642a32bcd5472b Mon Sep 17 00:00:00 2001 From: glax Date: Tue, 10 Oct 2023 22:34:47 +0200 Subject: [PATCH 1/8] Change coverCacheFilenames, to avoid conflicts and malformatted filenames --- Tranga/MangaConnectors/MangaConnector.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/MangaConnector.cs b/Tranga/MangaConnectors/MangaConnector.cs index a6bcef0..5610eaf 100644 --- a/Tranga/MangaConnectors/MangaConnector.cs +++ b/Tranga/MangaConnectors/MangaConnector.cs @@ -247,7 +247,8 @@ public abstract class MangaConnector : GlobalBase protected string SaveCoverImageToCache(string url, byte requestType) { - string filename = url.Split('/')[^1].Split('?')[0]; + string filetype = url.Split('/')[^1].Split('?')[0].Split('.')[^1]; + string filename = $"{DateTime.Now.Ticks.ToString()}.{filetype}"; string saveImagePath = Path.Join(settings.coverImageCache, filename); if (File.Exists(saveImagePath)) From fafcdac00aa26c1f14d786e5d6d230ebfd8f9b13 Mon Sep 17 00:00:00 2001 From: glax Date: Tue, 10 Oct 2023 22:40:07 +0200 Subject: [PATCH 2/8] Fix file-extension on image download --- Tranga/MangaConnectors/MangaConnector.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Tranga/MangaConnectors/MangaConnector.cs b/Tranga/MangaConnectors/MangaConnector.cs index 5610eaf..cd42d3e 100644 --- a/Tranga/MangaConnectors/MangaConnector.cs +++ b/Tranga/MangaConnectors/MangaConnector.cs @@ -213,8 +213,7 @@ public abstract class MangaConnector : GlobalBase //Download all Images to temporary Folder foreach (string imageUrl in imageUrls) { - string[] split = imageUrl.Split('.'); - string extension = split[^1]; + string extension = imageUrl.Split('.')[^1].Split('?')[0]; Log($"Downloading image {chapter + 1:000}/{imageUrls.Length:000}"); //TODO HttpStatusCode status = DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer); Log($"{saveArchiveFilePath} {chapter + 1:000}/{imageUrls.Length:000} {status}"); From e642d50c4707a720e76bdc5c8d4e6ccc728eb2d8 Mon Sep 17 00:00:00 2001 From: glax Date: Tue, 10 Oct 2023 22:40:44 +0200 Subject: [PATCH 3/8] #64 Bato Comment: This website suuuucks to scrape. There is gonna be so many issues --- Tranga/MangaConnectors/Bato.cs | 207 ++++++++++++++++++ .../MangaConnectorJsonConverter.cs | 2 + Tranga/Tranga.cs | 3 +- 3 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 Tranga/MangaConnectors/Bato.cs diff --git a/Tranga/MangaConnectors/Bato.cs b/Tranga/MangaConnectors/Bato.cs new file mode 100644 index 0000000..d62670e --- /dev/null +++ b/Tranga/MangaConnectors/Bato.cs @@ -0,0 +1,207 @@ +using System.Net; +using System.Text.RegularExpressions; +using HtmlAgilityPack; +using Tranga.Jobs; + +namespace Tranga.MangaConnectors; + +public class Bato : MangaConnector +{ + public Bato(GlobalBase clone) : base(clone, "Bato") + { + this.downloadClient = new HttpDownloadClient(clone, new Dictionary() + { + {1, 60} + }); + } + + public override Manga[] GetManga(string publicationTitle = "") + { + Log($"Searching Publications. Term=\"{publicationTitle}\""); + string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower(); + string requestUrl = $"https://bato.to/v3x-search?word={sanitizedTitle}&lang=en"; + DownloadClient.RequestResult requestResult = + downloadClient.MakeRequest(requestUrl, 1); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + return Array.Empty(); + + if (requestResult.htmlDocument is null) + { + Log($"Failed to retrieve site"); + return Array.Empty(); + } + + Manga[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument); + Log($"Retrieved {publications.Length} publications. Term=\"{publicationTitle}\""); + return publications; + } + + public override Manga? GetMangaFromUrl(string url) + { + DownloadClient.RequestResult requestResult = + downloadClient.MakeRequest(url, 1); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + return null; + if (requestResult.htmlDocument is null) + { + Log($"Failed to retrieve site"); + return null; + } + return ParseSinglePublicationFromHtml(requestResult.htmlDocument, url.Split('/')[^1]); + } + + private Manga[] ParsePublicationsFromHtml(HtmlDocument document) + { + HtmlNode mangaList = document.DocumentNode.SelectSingleNode("//div[@data-hk='0-0-2']"); + + List urls = mangaList.ChildNodes + .Select(node => $"https://bato.to{node.Descendants("div").First().FirstChild.GetAttributeValue("href", "")}").ToList(); + + HashSet ret = new(); + foreach (string url in urls) + { + Manga? manga = GetMangaFromUrl(url); + if (manga is not null) + ret.Add((Manga)manga); + } + + return ret.ToArray(); + } + + private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId) + { + HtmlNode infoNode = document.DocumentNode.SelectSingleNode("/html/body/div/main/div[1]/div[2]"); + + string sortName = infoNode.Descendants("h3").First().InnerText; + string description = document.DocumentNode + .SelectSingleNode("//div[contains(concat(' ',normalize-space(@class),' '),'prose')]").InnerText; + + string[] altTitlesList = infoNode.ChildNodes[1].ChildNodes[2].InnerText.Split('/'); + int i = 0; + Dictionary altTitles = altTitlesList.ToDictionary(s => i++.ToString(), s => s); + + string posterUrl = document.DocumentNode.SelectNodes("//img") + .First(child => child.GetAttributeValue("data-hk", "") == "0-1-0").GetAttributeValue("src", "").Replace("&", "&"); + string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1); + + List genreNodes = document.DocumentNode.SelectSingleNode("//b[text()='Genres:']/..").SelectNodes("span").ToList(); + string[] tags = genreNodes.Select(node => node.FirstChild.InnerText).ToArray(); + + List authorsNodes = infoNode.ChildNodes[1].ChildNodes[3].Descendants("a").ToList(); + List authors = authorsNodes.Select(node => node.InnerText).ToList(); + + HtmlNode? originalLanguageNode = document.DocumentNode.SelectSingleNode("//span[text()='Tr From']/.."); + string originalLanguage = originalLanguageNode is not null ? originalLanguageNode.LastChild.InnerText : ""; + + if (!int.TryParse( + document.DocumentNode.SelectSingleNode("//span[text()='Original Publication:']/..").LastChild.InnerText.Split('-')[0], + out int year)) + year = DateTime.Now.Year; + + string status = document.DocumentNode.SelectSingleNode("//span[text()='Original Publication:']/..") + .ChildNodes[2].InnerText; + + Manga manga = new (sortName, authors, description, altTitles, tags, posterUrl, coverFileNameInCache, new Dictionary(), + year, originalLanguage, status, publicationId); + cachedPublications.Add(manga); + return manga; + } + + public override Chapter[] GetChapters(Manga manga, string language="en") + { + Log($"Getting chapters {manga}"); + string requestUrl = $"https://bato.to/title/{manga.publicationId}"; + // Leaving this in for verification if the page exists + DownloadClient.RequestResult requestResult = + downloadClient.MakeRequest(requestUrl, 1); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + return Array.Empty(); + + //Return Chapters ordered by Chapter-Number + List chapters = ParseChaptersFromHtml(manga, requestUrl); + Log($"Got {chapters.Count} chapters. {manga}"); + return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, numberFormatDecimalPoint)).ToArray(); + } + + private List ParseChaptersFromHtml(Manga manga, string mangaUrl) + { + // Using HtmlWeb will include the chapters since they are loaded with js + HtmlWeb web = new(); + HtmlDocument document = web.Load(mangaUrl); + + List ret = new(); + + HtmlNode chapterList = + document.DocumentNode.SelectSingleNode("/html/body/div/main/div[3]/astro-island/div/div[2]/div/div/astro-slot"); + + Regex chapterNumberRex = new(@"Chapter ([0-9\.]+)"); + + foreach (HtmlNode chapterInfo in chapterList.SelectNodes("div")) + { + HtmlNode infoNode = chapterInfo.FirstChild.FirstChild; + string fullString = infoNode.InnerText; + + string? volumeNumber = null; + string chapterNumber = chapterNumberRex.Match(fullString).Groups[1].Value; + string chapterName = chapterNumber; + string url = $"https://bato.to{infoNode.GetAttributeValue("href", "")}?load=2"; + ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url)); + } + + return ret; + } + + public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) + { + if (progressToken?.cancellationRequested ?? false) + { + progressToken.Cancel(); + return HttpStatusCode.RequestTimeout; + } + + Manga chapterParentManga = chapter.parentManga; + Log($"Retrieving chapter-info {chapter} {chapterParentManga}"); + string requestUrl = chapter.url; + // Leaving this in to check if the page exists + DownloadClient.RequestResult requestResult = + downloadClient.MakeRequest(requestUrl, 1); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + { + progressToken?.Cancel(); + return requestResult.statusCode; + } + + string[] imageUrls = ParseImageUrlsFromHtml(requestUrl); + + string comicInfoPath = Path.GetTempFileName(); + File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString()); + + return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken); + } + + private string[] ParseImageUrlsFromHtml(string mangaUrl) + { + DownloadClient.RequestResult requestResult = + downloadClient.MakeRequest(mangaUrl, 1); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + { + return Array.Empty(); + } + if (requestResult.htmlDocument is null) + { + Log($"Failed to retrieve site"); + return Array.Empty(); + } + + HtmlDocument document = requestResult.htmlDocument; + + HtmlNode images = document.DocumentNode.SelectNodes("//astro-island").First(node => + node.GetAttributeValue("component-url", "").Contains("/_astro/ImageList.")); + + string weirdString = images.OuterHtml; + string weirdString2 = Regex.Match(weirdString, @"props=\""(.*)}\""").Groups[1].Value; + string[] urls = Regex.Matches(weirdString2, @"https:\/\/[A-z\-0-9\.\?\&\;\=\/]*").Select(m => m.Value.Replace("\\"]", "").Replace("amp;", "")).ToArray(); + + return urls; + } +} \ No newline at end of file diff --git a/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs b/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs index 62f389b..c28359f 100644 --- a/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs +++ b/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs @@ -34,6 +34,8 @@ public class MangaConnectorJsonConverter : JsonConverter return this._connectors.First(c => c is Mangasee); case "Mangaworld": return this._connectors.First(c => c is Mangaworld); + case "Bato": + return this._connectors.First(c => c is Bato); } throw new Exception(); diff --git a/Tranga/Tranga.cs b/Tranga/Tranga.cs index 23846b8..34b5de6 100644 --- a/Tranga/Tranga.cs +++ b/Tranga/Tranga.cs @@ -22,7 +22,8 @@ public partial class Tranga : GlobalBase new Mangasee(this), new MangaDex(this), new MangaKatana(this), - new Mangaworld(this) + new Mangaworld(this), + new Bato(this) }; jobBoss = new(this, this._connectors); StartJobBoss(); From fec970d7d681bc4e3f3db6a84644fd94b9f07159 Mon Sep 17 00:00:00 2001 From: glax Date: Tue, 10 Oct 2023 22:43:34 +0200 Subject: [PATCH 4/8] #64 fix empty search --- Tranga/MangaConnectors/Bato.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tranga/MangaConnectors/Bato.cs b/Tranga/MangaConnectors/Bato.cs index d62670e..ae525fe 100644 --- a/Tranga/MangaConnectors/Bato.cs +++ b/Tranga/MangaConnectors/Bato.cs @@ -53,6 +53,8 @@ public class Bato : MangaConnector private Manga[] ParsePublicationsFromHtml(HtmlDocument document) { HtmlNode mangaList = document.DocumentNode.SelectSingleNode("//div[@data-hk='0-0-2']"); + if (!mangaList.ChildNodes.Any(node => node.Name == "div")) + return Array.Empty(); List urls = mangaList.ChildNodes .Select(node => $"https://bato.to{node.Descendants("div").First().FirstChild.GetAttributeValue("href", "")}").ToList(); From 238a2775f49f8e810dd9c78f085329fc1912a54c Mon Sep 17 00:00:00 2001 From: glax Date: Tue, 10 Oct 2023 22:45:11 +0200 Subject: [PATCH 5/8] Author formatting bato --- Tranga/MangaConnectors/Bato.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tranga/MangaConnectors/Bato.cs b/Tranga/MangaConnectors/Bato.cs index ae525fe..eb40673 100644 --- a/Tranga/MangaConnectors/Bato.cs +++ b/Tranga/MangaConnectors/Bato.cs @@ -90,7 +90,7 @@ public class Bato : MangaConnector string[] tags = genreNodes.Select(node => node.FirstChild.InnerText).ToArray(); List authorsNodes = infoNode.ChildNodes[1].ChildNodes[3].Descendants("a").ToList(); - List authors = authorsNodes.Select(node => node.InnerText).ToList(); + List authors = authorsNodes.Select(node => node.InnerText.Replace("amp;", "")).ToList(); HtmlNode? originalLanguageNode = document.DocumentNode.SelectSingleNode("//span[text()='Tr From']/.."); string originalLanguage = originalLanguageNode is not null ? originalLanguageNode.LastChild.InnerText : ""; From 51a6f216af029590ab16dc69c71bfc7e42c3b5a2 Mon Sep 17 00:00:00 2001 From: glax Date: Tue, 10 Oct 2023 22:51:24 +0200 Subject: [PATCH 6/8] Remove extraneous covers from imageCache. --- Tranga/Jobs/JobBoss.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Tranga/Jobs/JobBoss.cs b/Tranga/Jobs/JobBoss.cs index cd320ed..18ac437 100644 --- a/Tranga/Jobs/JobBoss.cs +++ b/Tranga/Jobs/JobBoss.cs @@ -162,14 +162,21 @@ public class JobBoss : GlobalBase new JobJsonConverter(this, new MangaConnectorJsonConverter(this, connectors)))!; this.jobs.Add(job); } - + //Connect jobs to parent-jobs and add Publications to cache foreach (Job job in this.jobs) { this.jobs.FirstOrDefault(jjob => jjob.id == job.parentJobId)?.AddSubJob(job); - if(job is DownloadNewChapters dncJob) + if (job is DownloadNewChapters dncJob) cachedPublications.Add(dncJob.manga); } + + HashSet coverFileNames = cachedPublications.Select(manga => manga.coverFileNameInCache!).ToHashSet(); + foreach (string fileName in Directory.GetFiles(settings.coverImageCache)) + { + if(!coverFileNames.Any(existingManga => fileName.Contains(existingManga))) + File.Delete(fileName); + } } private void UpdateJobFile(Job job) From 334795b26307e64edf0eaa56ac3542aa45988391 Mon Sep 17 00:00:00 2001 From: glax Date: Tue, 10 Oct 2023 22:58:05 +0200 Subject: [PATCH 7/8] Update readme to reflect new connectors --- README.md | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 9cd6099..3f87f68 100644 --- a/README.md +++ b/README.md @@ -21,11 +21,6 @@

-# Important for existing users: -Tranga just had a complete rewrite. Old settings, tasks, etc. will not work. -~~For the time being the docker-tag `latest` will be the old, discontinued branch.~~ `cuttingedge` is the active branch and -will soon be moved to the `latest` branch (This is now the case). There is no migration-tool. Make a backup of old files. -
Table of Contents @@ -57,10 +52,12 @@ will soon be moved to the `latest` branch (This is now the case). There is no mi Tranga can download Chapters and Metadata from "Scanlation" sites such as -- [MangaDex.org](https://mangadex.org/) -- [Manganato.com](https://manganato.com/) -- [Mangasee](https://mangasee123.com/) -- [MangaKatana](https://mangakatana.com) +- [MangaDex.org](https://mangadex.org/) (Multilingual) +- [Manganato.com](https://manganato.com/) (en) +- [Mangasee.com](https://mangasee123.com/) (en) +- [MangaKatana.com](https://mangakatana.com) (en) +- [Mangaworld.bz](https://www.mangaworld.bz/) (it) +- [Bato.to](https://bato.to/v3x) (en) - ❓ Open an [issue](https://github.com/C9Glax/tranga/issues) and trigger an scan with [Komga](https://komga.org/) and [Kavita](https://www.kavitareader.com/). From 9bf650f5fca100378979b3fd87237dfbe85de016 Mon Sep 17 00:00:00 2001 From: glax Date: Thu, 12 Oct 2023 20:45:56 +0200 Subject: [PATCH 8/8] New Issue Template: New Connector --- .github/ISSUE_TEMPLATE/new_connector.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/new_connector.yml diff --git a/.github/ISSUE_TEMPLATE/new_connector.yml b/.github/ISSUE_TEMPLATE/new_connector.yml new file mode 100644 index 0000000..41ef0fd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/new_connector.yml @@ -0,0 +1,23 @@ +name: New Connector Request +description: Request a new site to be added +title: "[New Connector]: " +labels: ["New Connector"] +body: + - type: input + attributes: + label: Website-Link + placeholder: https:// + validations: + required: true + - type: checkboxes + attributes: + label: Is the Website free to access? + description: We can't support pay-to-use sites. + options: + - label: The Website is freely accessible. + required: true + - type: textarea + attributes: + label: Anything else? + validations: + required: false \ No newline at end of file