From f532e2ff76f56518e099c1b3af69d02d06671d01 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 15 Jan 2025 22:13:50 +0100 Subject: [PATCH 1/3] JobBoss LoadJobsList change: Fix Directory.Exists jobsFolderPath to create new Directory Fix Loading Job fails leading to crash. --- Tranga/Jobs/JobBoss.cs | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/Tranga/Jobs/JobBoss.cs b/Tranga/Jobs/JobBoss.cs index b1b38dd..2fe58a3 100644 --- a/Tranga/Jobs/JobBoss.cs +++ b/Tranga/Jobs/JobBoss.cs @@ -1,4 +1,5 @@ -using System.Runtime.InteropServices; +using System.Diagnostics; +using System.Runtime.InteropServices; using System.Text.RegularExpressions; using Newtonsoft.Json; using Tranga.MangaConnectors; @@ -145,26 +146,24 @@ public class JobBoss : GlobalBase private void LoadJobsList(HashSet connectors) { - Directory.CreateDirectory(TrangaSettings.jobsFolderPath); - if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - File.SetUnixFileMode(TrangaSettings.jobsFolderPath, UserRead | UserWrite | UserExecute | GroupRead | OtherRead); if (!Directory.Exists(TrangaSettings.jobsFolderPath)) //No jobs to load + { + Directory.CreateDirectory(TrangaSettings.jobsFolderPath); + if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + File.SetUnixFileMode(TrangaSettings.jobsFolderPath, UserRead | UserWrite | UserExecute | GroupRead | OtherRead); return; + } //Load json-job-files foreach (FileInfo file in Directory.GetFiles(TrangaSettings.jobsFolderPath, "*.json").Select(f => new FileInfo(f))) { Log($"Adding {file.Name}"); - Job? job = JsonConvert.DeserializeObject(File.ReadAllText(file.FullName), - new JobJsonConverter(this, new MangaConnectorJsonConverter(this, connectors))); - if (job is null) - { - string newName = file.FullName + ".failed"; - Log($"Failed loading file {file.Name}.\nMoving to {newName}"); - File.Move(file.FullName, newName); - } - else + try { + Job? job = JsonConvert.DeserializeObject(File.ReadAllText(file.FullName), + new JobJsonConverter(this, new MangaConnectorJsonConverter(this, connectors))); + if (job is null) throw new NullReferenceException(); + Log($"Adding Job {job}"); if (!AddJob(job, file.FullName)) //If we detect a duplicate, delete the file. { @@ -173,6 +172,16 @@ public class JobBoss : GlobalBase Log($"Duplicate detected or otherwise not able to add job to list.\nMoved job {job} to {path}"); } } + catch (Exception e) + { + if (e is not UnreachableException or NullReferenceException) + throw; + Log(e.Message); + string newName = file.FullName + ".failed"; + Log($"Failed loading file {file.Name}.\nMoving to {newName}"); + File.Move(file.FullName, newName); + continue; + } } //Connect jobs to parent-jobs and add Publications to cache From 2350c5a04b654508bb36c0352315fcf138987d23 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 15 Jan 2025 22:13:58 +0100 Subject: [PATCH 2/3] Remove Mangasee --- .../MangaConnectorJsonConverter.cs | 1 - Tranga/MangaConnectors/Mangasee.cs | 234 ------------------ Tranga/Tranga.cs | 1 - 3 files changed, 236 deletions(-) delete mode 100644 Tranga/MangaConnectors/Mangasee.cs diff --git a/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs b/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs index b0cc3dc..f54dbe3 100644 --- a/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs +++ b/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs @@ -32,7 +32,6 @@ public class MangaConnectorJsonConverter : JsonConverter "MangaDex" => this._connectors.First(c => c is MangaDex), "Manganato" => this._connectors.First(c => c is Manganato), "MangaKatana" => this._connectors.First(c => c is MangaKatana), - "Mangasee" => this._connectors.First(c => c is Mangasee), "Mangaworld" => this._connectors.First(c => c is Mangaworld), "Bato" => this._connectors.First(c => c is Bato), "Manga4Life" => this._connectors.First(c => c is MangaLife), diff --git a/Tranga/MangaConnectors/Mangasee.cs b/Tranga/MangaConnectors/Mangasee.cs deleted file mode 100644 index f912f6d..0000000 --- a/Tranga/MangaConnectors/Mangasee.cs +++ /dev/null @@ -1,234 +0,0 @@ -using System.Data; -using System.Net; -using System.Text.RegularExpressions; -using System.Xml.Linq; -using HtmlAgilityPack; -using Newtonsoft.Json; -using Soenneker.Utils.String.NeedlemanWunsch; -using Tranga.Jobs; - -namespace Tranga.MangaConnectors; - -public class Mangasee : MangaConnector -{ - public Mangasee(GlobalBase clone) : base(clone, "Mangasee", ["en"]) - { - this.downloadClient = new ChromiumDownloadClient(clone); - } - - private struct SearchResult - { - public string i { get; set; } - public string s { get; set; } - public string[] a { get; set; } - } - - public override Manga[] GetManga(string publicationTitle = "") - { - Log($"Searching Publications. Term=\"{publicationTitle}\""); - string requestUrl = "https://mangasee123.com/_search.php"; - RequestResult requestResult = - downloadClient.MakeRequest(requestUrl, RequestType.Default); - if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) - { - Log($"Failed to retrieve search: {requestResult.statusCode}"); - return Array.Empty(); - } - - try - { - SearchResult[] searchResults = JsonConvert.DeserializeObject(requestResult.htmlDocument!.DocumentNode.InnerText) ?? - throw new NoNullAllowedException(); - SearchResult[] filteredResults = FilteredResults(publicationTitle, searchResults); - Log($"Total available manga: {searchResults.Length} Filtered down to: {filteredResults.Length}"); - - - string[] urls = filteredResults.Select(result => $"https://mangasee123.com/manga/{result.i}").ToArray(); - List searchResultManga = new(); - foreach (string url in urls) - { - Manga? newManga = GetMangaFromUrl(url); - if(newManga is { } manga) - searchResultManga.Add(manga); - } - Log($"Retrieved {searchResultManga.Count} publications. Term=\"{publicationTitle}\""); - return searchResultManga.ToArray(); - } - catch (NoNullAllowedException) - { - Log("Failed to retrieve search"); - return Array.Empty(); - } - } - - private readonly string[] _filterWords = {"a", "the", "of", "as", "to", "no", "for", "on", "with", "be", "and", "in", "wa", "at", "be", "ni"}; - private string ToFilteredString(string input) => string.Join(' ', input.ToLower().Split(' ').Where(word => _filterWords.Contains(word) == false)); - private SearchResult[] FilteredResults(string publicationTitle, SearchResult[] unfilteredSearchResults) - { - Dictionary similarity = new(); - foreach (SearchResult sr in unfilteredSearchResults) - { - List scores = new(); - string filteredPublicationString = ToFilteredString(publicationTitle); - string filteredSString = ToFilteredString(sr.s); - scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredSString, filteredPublicationString)); - foreach (string srA in sr.a) - { - string filteredAString = ToFilteredString(srA); - scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredAString, filteredPublicationString)); - } - similarity.Add(sr, scores.Sum() / scores.Count); - } - - List ret = similarity.OrderBy(s => s.Value).Take(10).Select(s => s.Key).ToList(); - return ret.ToArray(); - } - - public override Manga? GetMangaFromId(string publicationId) - { - return GetMangaFromUrl($"https://mangasee123.com/manga/{publicationId}"); - } - - public override Manga? GetMangaFromUrl(string url) - { - Regex publicationIdRex = new(@"https:\/\/mangasee123.com\/manga\/(.*)(\/.*)*"); - string publicationId = publicationIdRex.Match(url).Groups[1].Value; - - RequestResult requestResult = this.downloadClient.MakeRequest(url, RequestType.MangaInfo); - if((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 && requestResult.htmlDocument is not null) - return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url); - return null; - } - - private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl) - { - string originalLanguage = "", status = ""; - Dictionary altTitles = new(), links = new(); - HashSet tags = new(); - Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased; - - HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//img"); - string posterUrl = posterNode.GetAttributeValue("src", ""); - string coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover); - - HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//h1"); - string sortName = titleNode.InnerText; - - HtmlNode[] authorsNodes = document.DocumentNode - .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Author(s):']/..").Descendants("a") - .ToArray(); - List authors = new(); - foreach (HtmlNode authorNode in authorsNodes) - authors.Add(authorNode.InnerText); - - HtmlNode[] genreNodes = document.DocumentNode - .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Genre(s):']/..").Descendants("a") - .ToArray(); - foreach (HtmlNode genreNode in genreNodes) - tags.Add(genreNode.InnerText); - - HtmlNode yearNode = document.DocumentNode - .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Released:']/..").Descendants("a") - .First(); - int year = Convert.ToInt32(yearNode.InnerText); - - HtmlNode[] statusNodes = document.DocumentNode - .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Status:']/..").Descendants("a") - .ToArray(); - foreach (HtmlNode statusNode in statusNodes) - if (statusNode.InnerText.Contains("publish", StringComparison.CurrentCultureIgnoreCase)) - status = statusNode.InnerText.Split(' ')[0]; - switch (status.ToLower()) - { - case "cancelled": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break; - case "hiatus": releaseStatus = Manga.ReleaseStatusByte.OnHiatus; break; - case "discontinued": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break; - case "complete": releaseStatus = Manga.ReleaseStatusByte.Completed; break; - case "ongoing": releaseStatus = Manga.ReleaseStatusByte.Continuing; break; - } - - HtmlNode descriptionNode = document.DocumentNode - .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Description:']/..") - .Descendants("div").First(); - string description = descriptionNode.InnerText; - - Manga manga = new(sortName, authors.ToList(), description, altTitles, tags.ToArray(), posterUrl, - coverFileNameInCache, links, - year, originalLanguage, publicationId, releaseStatus, websiteUrl: websiteUrl); - AddMangaToCache(manga); - return manga; - } - - public override Chapter[] GetChapters(Manga manga, string language="en") - { - Log($"Getting chapters {manga}"); - try - { - XDocument doc = XDocument.Load($"https://mangasee123.com/rss/{manga.publicationId}.xml"); - XElement[] chapterItems = doc.Descendants("item").ToArray(); - List chapters = new(); - Regex chVolRex = new(@".*chapter-([0-9\.]+)(?:-index-([0-9\.]+))?.*"); - foreach (XElement chapter in chapterItems) - { - string url = chapter.Descendants("link").First().Value; - Match m = chVolRex.Match(url); - string? volumeNumber = m.Groups[2].Success ? m.Groups[2].Value : "1"; - string chapterNumber = m.Groups[1].Value; - - string chapterUrl = Regex.Replace(url, @"-page-[0-9]+(\.html)", ".html"); - try - { - chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, chapterUrl)); - } - catch (Exception e) - { - Log($"Failed to load chapter {chapterNumber}: {e.Message}"); - } - } - - //Return Chapters ordered by Chapter-Number - Log($"Got {chapters.Count} chapters. {manga}"); - return chapters.Order().ToArray(); - } - catch (HttpRequestException e) - { - Log($"Failed to load https://mangasee123.com/rss/{manga.publicationId}.xml \n\r{e}"); - return Array.Empty(); - } - } - - public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) - { - if (progressToken?.cancellationRequested ?? false) - { - progressToken.Cancel(); - return HttpStatusCode.RequestTimeout; - } - - Manga chapterParentManga = chapter.parentManga; - if (progressToken?.cancellationRequested ?? false) - { - progressToken.Cancel(); - return HttpStatusCode.RequestTimeout; - } - - Log($"Retrieving chapter-info {chapter} {chapterParentManga}"); - - RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, RequestType.Default); - if (requestResult.htmlDocument is null) - { - progressToken?.Cancel(); - return HttpStatusCode.RequestTimeout; - } - - HtmlDocument document = requestResult.htmlDocument; - - HtmlNode gallery = document.DocumentNode.Descendants("div").First(div => div.HasClass("ImageGallery")); - HtmlNode[] images = gallery.Descendants("img").Where(img => img.HasClass("img-fluid")).ToArray(); - List urls = new(); - foreach(HtmlNode galleryImage in images) - urls.Add(galleryImage.GetAttributeValue("src", "")); - - return DownloadChapterImages(urls.ToArray(), chapter, RequestType.MangaImage, progressToken:progressToken); - } -} \ No newline at end of file diff --git a/Tranga/Tranga.cs b/Tranga/Tranga.cs index 65ea58e..3292e6d 100644 --- a/Tranga/Tranga.cs +++ b/Tranga/Tranga.cs @@ -18,7 +18,6 @@ public partial class Tranga : GlobalBase _connectors = new HashSet() { new Manganato(this), - new Mangasee(this), new MangaDex(this), new MangaKatana(this), new Mangaworld(this), From 123a8b06b29fcfa5fdf781ac778c33bdd8fdb3a6 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 15 Jan 2025 22:15:33 +0100 Subject: [PATCH 3/3] jobloading errormessage --- Tranga/Jobs/JobBoss.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tranga/Jobs/JobBoss.cs b/Tranga/Jobs/JobBoss.cs index 2fe58a3..4631a66 100644 --- a/Tranga/Jobs/JobBoss.cs +++ b/Tranga/Jobs/JobBoss.cs @@ -178,7 +178,8 @@ public class JobBoss : GlobalBase throw; Log(e.Message); string newName = file.FullName + ".failed"; - Log($"Failed loading file {file.Name}.\nMoving to {newName}"); + Log($"Failed loading file {file.Name}.\nMoving to {newName}.\n" + + $"If you think this is a bug, upload contents of the file to the Bugreport!"); File.Move(file.FullName, newName); continue; }