diff --git a/.github/ISSUE_TEMPLATE/new_connector.yml b/.github/ISSUE_TEMPLATE/new_connector.yml new file mode 100644 index 0000000..41ef0fd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/new_connector.yml @@ -0,0 +1,23 @@ +name: New Connector Request +description: Request a new site to be added +title: "[New Connector]: " +labels: ["New Connector"] +body: + - type: input + attributes: + label: Website-Link + placeholder: https:// + validations: + required: true + - type: checkboxes + attributes: + label: Is the Website free to access? + description: We can't support pay-to-use sites. + options: + - label: The Website is freely accessible. + required: true + - type: textarea + attributes: + label: Anything else? + validations: + required: false \ No newline at end of file diff --git a/README.md b/README.md index 9cd6099..3f87f68 100644 --- a/README.md +++ b/README.md @@ -21,11 +21,6 @@

-# Important for existing users: -Tranga just had a complete rewrite. Old settings, tasks, etc. will not work. -~~For the time being the docker-tag `latest` will be the old, discontinued branch.~~ `cuttingedge` is the active branch and -will soon be moved to the `latest` branch (This is now the case). There is no migration-tool. Make a backup of old files. -
Table of Contents @@ -57,10 +52,12 @@ will soon be moved to the `latest` branch (This is now the case). There is no mi Tranga can download Chapters and Metadata from "Scanlation" sites such as -- [MangaDex.org](https://mangadex.org/) -- [Manganato.com](https://manganato.com/) -- [Mangasee](https://mangasee123.com/) -- [MangaKatana](https://mangakatana.com) +- [MangaDex.org](https://mangadex.org/) (Multilingual) +- [Manganato.com](https://manganato.com/) (en) +- [Mangasee.com](https://mangasee123.com/) (en) +- [MangaKatana.com](https://mangakatana.com) (en) +- [Mangaworld.bz](https://www.mangaworld.bz/) (it) +- [Bato.to](https://bato.to/v3x) (en) - ❓ Open an [issue](https://github.com/C9Glax/tranga/issues) and trigger an scan with [Komga](https://komga.org/) and [Kavita](https://www.kavitareader.com/). diff --git a/Tranga/Jobs/JobBoss.cs b/Tranga/Jobs/JobBoss.cs index cd320ed..18ac437 100644 --- a/Tranga/Jobs/JobBoss.cs +++ b/Tranga/Jobs/JobBoss.cs @@ -162,14 +162,21 @@ public class JobBoss : GlobalBase new JobJsonConverter(this, new MangaConnectorJsonConverter(this, connectors)))!; this.jobs.Add(job); } - + //Connect jobs to parent-jobs and add Publications to cache foreach (Job job in this.jobs) { this.jobs.FirstOrDefault(jjob => jjob.id == job.parentJobId)?.AddSubJob(job); - if(job is DownloadNewChapters dncJob) + if (job is DownloadNewChapters dncJob) cachedPublications.Add(dncJob.manga); } + + HashSet coverFileNames = cachedPublications.Select(manga => manga.coverFileNameInCache!).ToHashSet(); + foreach (string fileName in Directory.GetFiles(settings.coverImageCache)) + { + if(!coverFileNames.Any(existingManga => fileName.Contains(existingManga))) + File.Delete(fileName); + } } private void UpdateJobFile(Job job) diff --git a/Tranga/MangaConnectors/Bato.cs b/Tranga/MangaConnectors/Bato.cs new file mode 100644 index 0000000..eb40673 --- /dev/null +++ b/Tranga/MangaConnectors/Bato.cs @@ -0,0 +1,209 @@ +using System.Net; +using System.Text.RegularExpressions; +using HtmlAgilityPack; +using Tranga.Jobs; + +namespace Tranga.MangaConnectors; + +public class Bato : MangaConnector +{ + public Bato(GlobalBase clone) : base(clone, "Bato") + { + this.downloadClient = new HttpDownloadClient(clone, new Dictionary() + { + {1, 60} + }); + } + + public override Manga[] GetManga(string publicationTitle = "") + { + Log($"Searching Publications. Term=\"{publicationTitle}\""); + string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower(); + string requestUrl = $"https://bato.to/v3x-search?word={sanitizedTitle}&lang=en"; + DownloadClient.RequestResult requestResult = + downloadClient.MakeRequest(requestUrl, 1); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + return Array.Empty(); + + if (requestResult.htmlDocument is null) + { + Log($"Failed to retrieve site"); + return Array.Empty(); + } + + Manga[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument); + Log($"Retrieved {publications.Length} publications. Term=\"{publicationTitle}\""); + return publications; + } + + public override Manga? GetMangaFromUrl(string url) + { + DownloadClient.RequestResult requestResult = + downloadClient.MakeRequest(url, 1); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + return null; + if (requestResult.htmlDocument is null) + { + Log($"Failed to retrieve site"); + return null; + } + return ParseSinglePublicationFromHtml(requestResult.htmlDocument, url.Split('/')[^1]); + } + + private Manga[] ParsePublicationsFromHtml(HtmlDocument document) + { + HtmlNode mangaList = document.DocumentNode.SelectSingleNode("//div[@data-hk='0-0-2']"); + if (!mangaList.ChildNodes.Any(node => node.Name == "div")) + return Array.Empty(); + + List urls = mangaList.ChildNodes + .Select(node => $"https://bato.to{node.Descendants("div").First().FirstChild.GetAttributeValue("href", "")}").ToList(); + + HashSet ret = new(); + foreach (string url in urls) + { + Manga? manga = GetMangaFromUrl(url); + if (manga is not null) + ret.Add((Manga)manga); + } + + return ret.ToArray(); + } + + private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId) + { + HtmlNode infoNode = document.DocumentNode.SelectSingleNode("/html/body/div/main/div[1]/div[2]"); + + string sortName = infoNode.Descendants("h3").First().InnerText; + string description = document.DocumentNode + .SelectSingleNode("//div[contains(concat(' ',normalize-space(@class),' '),'prose')]").InnerText; + + string[] altTitlesList = infoNode.ChildNodes[1].ChildNodes[2].InnerText.Split('/'); + int i = 0; + Dictionary altTitles = altTitlesList.ToDictionary(s => i++.ToString(), s => s); + + string posterUrl = document.DocumentNode.SelectNodes("//img") + .First(child => child.GetAttributeValue("data-hk", "") == "0-1-0").GetAttributeValue("src", "").Replace("&", "&"); + string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1); + + List genreNodes = document.DocumentNode.SelectSingleNode("//b[text()='Genres:']/..").SelectNodes("span").ToList(); + string[] tags = genreNodes.Select(node => node.FirstChild.InnerText).ToArray(); + + List authorsNodes = infoNode.ChildNodes[1].ChildNodes[3].Descendants("a").ToList(); + List authors = authorsNodes.Select(node => node.InnerText.Replace("amp;", "")).ToList(); + + HtmlNode? originalLanguageNode = document.DocumentNode.SelectSingleNode("//span[text()='Tr From']/.."); + string originalLanguage = originalLanguageNode is not null ? originalLanguageNode.LastChild.InnerText : ""; + + if (!int.TryParse( + document.DocumentNode.SelectSingleNode("//span[text()='Original Publication:']/..").LastChild.InnerText.Split('-')[0], + out int year)) + year = DateTime.Now.Year; + + string status = document.DocumentNode.SelectSingleNode("//span[text()='Original Publication:']/..") + .ChildNodes[2].InnerText; + + Manga manga = new (sortName, authors, description, altTitles, tags, posterUrl, coverFileNameInCache, new Dictionary(), + year, originalLanguage, status, publicationId); + cachedPublications.Add(manga); + return manga; + } + + public override Chapter[] GetChapters(Manga manga, string language="en") + { + Log($"Getting chapters {manga}"); + string requestUrl = $"https://bato.to/title/{manga.publicationId}"; + // Leaving this in for verification if the page exists + DownloadClient.RequestResult requestResult = + downloadClient.MakeRequest(requestUrl, 1); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + return Array.Empty(); + + //Return Chapters ordered by Chapter-Number + List chapters = ParseChaptersFromHtml(manga, requestUrl); + Log($"Got {chapters.Count} chapters. {manga}"); + return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, numberFormatDecimalPoint)).ToArray(); + } + + private List ParseChaptersFromHtml(Manga manga, string mangaUrl) + { + // Using HtmlWeb will include the chapters since they are loaded with js + HtmlWeb web = new(); + HtmlDocument document = web.Load(mangaUrl); + + List ret = new(); + + HtmlNode chapterList = + document.DocumentNode.SelectSingleNode("/html/body/div/main/div[3]/astro-island/div/div[2]/div/div/astro-slot"); + + Regex chapterNumberRex = new(@"Chapter ([0-9\.]+)"); + + foreach (HtmlNode chapterInfo in chapterList.SelectNodes("div")) + { + HtmlNode infoNode = chapterInfo.FirstChild.FirstChild; + string fullString = infoNode.InnerText; + + string? volumeNumber = null; + string chapterNumber = chapterNumberRex.Match(fullString).Groups[1].Value; + string chapterName = chapterNumber; + string url = $"https://bato.to{infoNode.GetAttributeValue("href", "")}?load=2"; + ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url)); + } + + return ret; + } + + public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) + { + if (progressToken?.cancellationRequested ?? false) + { + progressToken.Cancel(); + return HttpStatusCode.RequestTimeout; + } + + Manga chapterParentManga = chapter.parentManga; + Log($"Retrieving chapter-info {chapter} {chapterParentManga}"); + string requestUrl = chapter.url; + // Leaving this in to check if the page exists + DownloadClient.RequestResult requestResult = + downloadClient.MakeRequest(requestUrl, 1); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + { + progressToken?.Cancel(); + return requestResult.statusCode; + } + + string[] imageUrls = ParseImageUrlsFromHtml(requestUrl); + + string comicInfoPath = Path.GetTempFileName(); + File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString()); + + return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken); + } + + private string[] ParseImageUrlsFromHtml(string mangaUrl) + { + DownloadClient.RequestResult requestResult = + downloadClient.MakeRequest(mangaUrl, 1); + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) + { + return Array.Empty(); + } + if (requestResult.htmlDocument is null) + { + Log($"Failed to retrieve site"); + return Array.Empty(); + } + + HtmlDocument document = requestResult.htmlDocument; + + HtmlNode images = document.DocumentNode.SelectNodes("//astro-island").First(node => + node.GetAttributeValue("component-url", "").Contains("/_astro/ImageList.")); + + string weirdString = images.OuterHtml; + string weirdString2 = Regex.Match(weirdString, @"props=\""(.*)}\""").Groups[1].Value; + string[] urls = Regex.Matches(weirdString2, @"https:\/\/[A-z\-0-9\.\?\&\;\=\/]*").Select(m => m.Value.Replace("\\"]", "").Replace("amp;", "")).ToArray(); + + return urls; + } +} \ No newline at end of file diff --git a/Tranga/MangaConnectors/MangaConnector.cs b/Tranga/MangaConnectors/MangaConnector.cs index a6bcef0..cd42d3e 100644 --- a/Tranga/MangaConnectors/MangaConnector.cs +++ b/Tranga/MangaConnectors/MangaConnector.cs @@ -213,8 +213,7 @@ public abstract class MangaConnector : GlobalBase //Download all Images to temporary Folder foreach (string imageUrl in imageUrls) { - string[] split = imageUrl.Split('.'); - string extension = split[^1]; + string extension = imageUrl.Split('.')[^1].Split('?')[0]; Log($"Downloading image {chapter + 1:000}/{imageUrls.Length:000}"); //TODO HttpStatusCode status = DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer); Log($"{saveArchiveFilePath} {chapter + 1:000}/{imageUrls.Length:000} {status}"); @@ -247,7 +246,8 @@ public abstract class MangaConnector : GlobalBase protected string SaveCoverImageToCache(string url, byte requestType) { - string filename = url.Split('/')[^1].Split('?')[0]; + string filetype = url.Split('/')[^1].Split('?')[0].Split('.')[^1]; + string filename = $"{DateTime.Now.Ticks.ToString()}.{filetype}"; string saveImagePath = Path.Join(settings.coverImageCache, filename); if (File.Exists(saveImagePath)) diff --git a/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs b/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs index 62f389b..c28359f 100644 --- a/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs +++ b/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs @@ -34,6 +34,8 @@ public class MangaConnectorJsonConverter : JsonConverter return this._connectors.First(c => c is Mangasee); case "Mangaworld": return this._connectors.First(c => c is Mangaworld); + case "Bato": + return this._connectors.First(c => c is Bato); } throw new Exception(); diff --git a/Tranga/Tranga.cs b/Tranga/Tranga.cs index 23846b8..34b5de6 100644 --- a/Tranga/Tranga.cs +++ b/Tranga/Tranga.cs @@ -22,7 +22,8 @@ public partial class Tranga : GlobalBase new Mangasee(this), new MangaDex(this), new MangaKatana(this), - new Mangaworld(this) + new Mangaworld(this), + new Bato(this) }; jobBoss = new(this, this._connectors); StartJobBoss();