diff --git a/Logging/MemoryLogger.cs b/Logging/MemoryLogger.cs index a612587..019600f 100644 --- a/Logging/MemoryLogger.cs +++ b/Logging/MemoryLogger.cs @@ -62,7 +62,7 @@ public class MemoryLogger : LoggerBase ret.Add(_logMessages.GetValueAtIndex(_lastLogMessageIndex + retIndex).ToString()); } } - catch (NullReferenceException e)//Called when LogMessage has not finished writing + catch (NullReferenceException)//Called when LogMessage has not finished writing { break; } diff --git a/Tranga.sln.DotSettings b/Tranga.sln.DotSettings index 1162c8f..672ec3b 100644 --- a/Tranga.sln.DotSettings +++ b/Tranga.sln.DotSettings @@ -8,5 +8,6 @@ True True True + True True True \ No newline at end of file diff --git a/Tranga/Jobs/JobBoss.cs b/Tranga/Jobs/JobBoss.cs index 7be8a0c..cd320ed 100644 --- a/Tranga/Jobs/JobBoss.cs +++ b/Tranga/Jobs/JobBoss.cs @@ -27,7 +27,7 @@ public class JobBoss : GlobalBase { Log($"Added {job}"); this.jobs.Add(job); - ExportJob(job); + UpdateJobFile(job); } } @@ -37,6 +37,10 @@ public class JobBoss : GlobalBase AddJob(job); } + /// + /// Compares contents of the provided job and all current jobs + /// Does not check if objects are the same + /// public bool ContainsJobLike(Job job) { if (job is DownloadChapter dcJob) @@ -57,13 +61,14 @@ public class JobBoss : GlobalBase this.jobs.Remove(job); if(job.subJobs is not null && job.subJobs.Any()) RemoveJobs(job.subJobs); - ExportJob(job); + UpdateJobFile(job); } public void RemoveJobs(IEnumerable jobsToRemove) { - Log($"Removing {jobsToRemove.Count()} jobs."); - foreach (Job? job in jobsToRemove) + List toRemove = jobsToRemove.ToList(); //Prevent multiple enumeration + Log($"Removing {toRemove.Count()} jobs."); + foreach (Job? job in toRemove) if(job is not null) RemoveJob(job); } @@ -96,7 +101,7 @@ public class JobBoss : GlobalBase Chapter? chapter = null) { if (chapter is not null) - return GetJobsLike(mangaConnector?.name, chapter.Value.parentManga.internalId, chapter?.chapterNumber); + return GetJobsLike(mangaConnector?.name, chapter.Value.parentManga.internalId, chapter.Value.chapterNumber); else return GetJobsLike(mangaConnector?.name, publication?.internalId); } @@ -122,47 +127,52 @@ public class JobBoss : GlobalBase private bool QueueContainsJob(Job job) { - mangaConnectorJobQueue.TryAdd(job.mangaConnector, new Queue()); + if (mangaConnectorJobQueue.TryAdd(job.mangaConnector, new Queue()))//If we can add the queue, there is certainly no job in it + return true; return mangaConnectorJobQueue[job.mangaConnector].Contains(job); } public void AddJobToQueue(Job job) { Log($"Adding Job to Queue. {job}"); - mangaConnectorJobQueue.TryAdd(job.mangaConnector, new Queue()); - Queue connectorJobQueue = mangaConnectorJobQueue[job.mangaConnector]; - if(!connectorJobQueue.Contains(job)) - connectorJobQueue.Enqueue(job); + if(!QueueContainsJob(job)) + mangaConnectorJobQueue[job.mangaConnector].Enqueue(job); job.ExecutionEnqueue(); } - public void AddJobsToQueue(IEnumerable jobs) + private void AddJobsToQueue(IEnumerable newJobs) { - foreach(Job job in jobs) + foreach(Job job in newJobs) AddJobToQueue(job); } - public void LoadJobsList(HashSet connectors) + private void LoadJobsList(HashSet connectors) { - Directory.CreateDirectory(settings.jobsFolderPath); + if (!Directory.Exists(settings.jobsFolderPath)) //No jobs to load + { + Directory.CreateDirectory(settings.jobsFolderPath); + return; + } Regex idRex = new (@"(.*)\.json"); - foreach (FileInfo file in new DirectoryInfo(settings.jobsFolderPath).EnumerateFiles()) - if (idRex.IsMatch(file.Name)) - { - Job job = JsonConvert.DeserializeObject(File.ReadAllText(file.FullName), - new JobJsonConverter(this, new MangaConnectorJsonConverter(this, connectors)))!; - this.jobs.Add(job); - } - - foreach (Job job in this.jobs) - this.jobs.FirstOrDefault(jjob => jjob.id == job.parentJobId)?.AddSubJob(job); + //Load json-job-files + foreach (FileInfo file in new DirectoryInfo(settings.jobsFolderPath).EnumerateFiles().Where(fileInfo => idRex.IsMatch(fileInfo.Name))) + { + Job job = JsonConvert.DeserializeObject(File.ReadAllText(file.FullName), + new JobJsonConverter(this, new MangaConnectorJsonConverter(this, connectors)))!; + this.jobs.Add(job); + } - foreach (DownloadNewChapters ncJob in this.jobs.Where(job => job is DownloadNewChapters)) - cachedPublications.Add(ncJob.manga); + //Connect jobs to parent-jobs and add Publications to cache + foreach (Job job in this.jobs) + { + this.jobs.FirstOrDefault(jjob => jjob.id == job.parentJobId)?.AddSubJob(job); + if(job is DownloadNewChapters dncJob) + cachedPublications.Add(dncJob.manga); + } } - public void ExportJob(Job job) + private void UpdateJobFile(Job job) { string jobFilePath = Path.Join(settings.jobsFolderPath, $"{job.id}.json"); @@ -190,11 +200,11 @@ public class JobBoss : GlobalBase } } - public void ExportJobsList() + private void UpdateAllJobFiles() { Log("Exporting Jobs"); foreach (Job job in this.jobs) - ExportJob(job); + UpdateJobFile(job); //Remove files with jobs not in this.jobs-list Regex idRex = new (@"(.*)\.json"); @@ -228,9 +238,10 @@ public class JobBoss : GlobalBase Job queueHead = jobQueue.Peek(); if (queueHead.progressToken.state is ProgressToken.State.Complete or ProgressToken.State.Cancelled) { - queueHead.ResetProgress(); if(!queueHead.recurring) RemoveJob(queueHead); + else + queueHead.ResetProgress(); jobQueue.Dequeue(); Log($"Next job in {jobs.MinBy(job => job.nextExecution)?.nextExecution.Subtract(DateTime.Now)} {jobs.MinBy(job => job.nextExecution)?.id}"); }else if (queueHead.progressToken.state is ProgressToken.State.Standby) @@ -238,6 +249,10 @@ public class JobBoss : GlobalBase Job[] subJobs = jobQueue.Peek().ExecuteReturnSubTasks().ToArray(); AddJobs(subJobs); AddJobsToQueue(subJobs); + }else if (queueHead.progressToken.state is ProgressToken.State.Running && DateTime.Now.Subtract(queueHead.progressToken.lastUpdate) > TimeSpan.FromMinutes(5)) + { + Log($"{queueHead} inactive for more than 5 minutes. Cancelling."); + queueHead.Cancel(); } } } diff --git a/Tranga/Jobs/ProgressToken.cs b/Tranga/Jobs/ProgressToken.cs index f23819d..e718c7d 100644 --- a/Tranga/Jobs/ProgressToken.cs +++ b/Tranga/Jobs/ProgressToken.cs @@ -6,7 +6,7 @@ public class ProgressToken public int increments { get; set; } public int incrementsCompleted { get; set; } public float progress => GetProgress(); - + public DateTime lastUpdate { get; private set; } public DateTime executionStarted { get; private set; } public TimeSpan timeRemaining => GetTimeRemaining(); @@ -20,12 +20,13 @@ public class ProgressToken this.incrementsCompleted = 0; this.state = State.Waiting; this.executionStarted = DateTime.UnixEpoch; + this.lastUpdate = DateTime.UnixEpoch; } private float GetProgress() { if(increments > 0 && incrementsCompleted > 0) - return (float)incrementsCompleted / (float)increments; + return incrementsCompleted / (float)increments; return 0; } @@ -38,6 +39,7 @@ public class ProgressToken public void Increment() { + this.lastUpdate = DateTime.Now; this.incrementsCompleted++; if (incrementsCompleted > increments) state = State.Complete; @@ -45,27 +47,32 @@ public class ProgressToken public void Standby() { + this.lastUpdate = DateTime.Now; state = State.Standby; } public void Start() { + this.lastUpdate = DateTime.Now; state = State.Running; this.executionStarted = DateTime.Now; } public void Complete() { + this.lastUpdate = DateTime.Now; state = State.Complete; } public void Cancel() { + this.lastUpdate = DateTime.Now; state = State.Cancelled; } public void Waiting() { + this.lastUpdate = DateTime.Now; state = State.Waiting; } } \ No newline at end of file diff --git a/Tranga/LibraryConnectors/LibraryManagerJsonConverter.cs b/Tranga/LibraryConnectors/LibraryManagerJsonConverter.cs index be8983f..23fbf3a 100644 --- a/Tranga/LibraryConnectors/LibraryManagerJsonConverter.cs +++ b/Tranga/LibraryConnectors/LibraryManagerJsonConverter.cs @@ -5,7 +5,7 @@ namespace Tranga.LibraryConnectors; public class LibraryManagerJsonConverter : JsonConverter { - private GlobalBase _clone; + private readonly GlobalBase _clone; internal LibraryManagerJsonConverter(GlobalBase clone) { diff --git a/Tranga/MangaConnectors/ChromiumDownloadClient.cs b/Tranga/MangaConnectors/ChromiumDownloadClient.cs index bbb5949..9877a54 100644 --- a/Tranga/MangaConnectors/ChromiumDownloadClient.cs +++ b/Tranga/MangaConnectors/ChromiumDownloadClient.cs @@ -48,7 +48,8 @@ internal class ChromiumDownloadClient : DownloadClient "--disable-gpu", "--disable-dev-shm-usage", "--disable-setuid-sandbox", - "--no-sandbox"} + "--no-sandbox"}, + Timeout = 10000 }); } @@ -59,8 +60,10 @@ internal class ChromiumDownloadClient : DownloadClient protected override RequestResult MakeRequestInternal(string url, string? referrer = null) { - IPage page = this.browser!.NewPageAsync().Result; + IPage page = this.browser.NewPageAsync().Result; + page.DefaultTimeout = 10000; IResponse response = page.GoToAsync(url, WaitUntilNavigation.DOMContentLoaded).Result; + Log("Page loaded."); Stream stream = Stream.Null; HtmlDocument? document = null; @@ -83,7 +86,7 @@ internal class ChromiumDownloadClient : DownloadClient page.CloseAsync(); return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null); } - + page.CloseAsync(); return new RequestResult(response.Status, document, stream, false, ""); } diff --git a/Tranga/MangaConnectors/DownloadClient.cs b/Tranga/MangaConnectors/DownloadClient.cs index 1b87413..80b050f 100644 --- a/Tranga/MangaConnectors/DownloadClient.cs +++ b/Tranga/MangaConnectors/DownloadClient.cs @@ -30,7 +30,10 @@ internal abstract class DownloadClient : GlobalBase .Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType])); if (rateLimitTimeout > TimeSpan.Zero) + { + Log($"Waiting {rateLimitTimeout.TotalSeconds} seconds"); Thread.Sleep(rateLimitTimeout); + } RequestResult result = MakeRequestInternal(url, referrer); _lastExecutedRateLimit[requestType] = DateTime.Now; diff --git a/Tranga/MangaConnectors/MangaConnector.cs b/Tranga/MangaConnectors/MangaConnector.cs index d8ddacb..a6bcef0 100644 --- a/Tranga/MangaConnectors/MangaConnector.cs +++ b/Tranga/MangaConnectors/MangaConnector.cs @@ -58,9 +58,9 @@ public abstract class MangaConnector : GlobalBase Log($"Getting new Chapters for {manga}"); Chapter[] newChapters = this.GetChapters(manga, language); Log($"Checking for duplicates {manga}"); - List newChaptersList = newChapters.Where(nChapter => - float.Parse(nChapter.chapterNumber, numberFormatDecimalPoint) > manga.ignoreChaptersBelow && - !nChapter.CheckChapterIsDownloaded(settings.downloadLocation)).ToList(); + List newChaptersList = newChapters.Where(nChapter => float.TryParse(nChapter.chapterNumber, numberFormatDecimalPoint, out float chapterNumber) + && chapterNumber > manga.ignoreChaptersBelow + && !nChapter.CheckChapterIsDownloaded(settings.downloadLocation)).ToList(); Log($"{newChaptersList.Count} new chapters. {manga}"); return newChaptersList.ToArray(); @@ -175,11 +175,15 @@ public abstract class MangaConnector : GlobalBase private HttpStatusCode DownloadImage(string imageUrl, string fullPath, byte requestType, string? referrer = null) { DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType, referrer); - if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.result == Stream.Null) + + if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) return requestResult.statusCode; - byte[] buffer = new byte[requestResult.result.Length]; - requestResult.result.ReadExactly(buffer, 0, buffer.Length); - File.WriteAllBytes(fullPath, buffer); + if (requestResult.result == Stream.Null) + return HttpStatusCode.NotFound; + + FileStream fs = new (fullPath, FileMode.Create); + requestResult.result.CopyTo(fs); + fs.Close(); return requestResult.statusCode; } @@ -213,6 +217,7 @@ public abstract class MangaConnector : GlobalBase string extension = split[^1]; Log($"Downloading image {chapter + 1:000}/{imageUrls.Length:000}"); //TODO HttpStatusCode status = DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer); + Log($"{saveArchiveFilePath} {chapter + 1:000}/{imageUrls.Length:000} {status}"); if ((int)status < 200 || (int)status >= 300) { progressToken?.Complete(); @@ -220,7 +225,7 @@ public abstract class MangaConnector : GlobalBase } if (progressToken?.cancellationRequested ?? false) { - progressToken?.Complete(); + progressToken.Complete(); return HttpStatusCode.RequestTimeout; } progressToken?.Increment(); diff --git a/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs b/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs index b9f3af0..62f389b 100644 --- a/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs +++ b/Tranga/MangaConnectors/MangaConnectorJsonConverter.cs @@ -6,12 +6,12 @@ namespace Tranga.MangaConnectors; public class MangaConnectorJsonConverter : JsonConverter { private GlobalBase _clone; - private HashSet connectors; + private readonly HashSet _connectors; internal MangaConnectorJsonConverter(GlobalBase clone, HashSet connectors) { this._clone = clone; - this.connectors = connectors; + this._connectors = connectors; } public override bool CanConvert(Type objectType) @@ -25,15 +25,15 @@ public class MangaConnectorJsonConverter : JsonConverter switch (jo.GetValue("name")!.Value()!) { case "MangaDex": - return this.connectors.First(c => c is MangaDex); + return this._connectors.First(c => c is MangaDex); case "Manganato": - return this.connectors.First(c => c is Manganato); + return this._connectors.First(c => c is Manganato); case "MangaKatana": - return this.connectors.First(c => c is MangaKatana); + return this._connectors.First(c => c is MangaKatana); case "Mangasee": - return this.connectors.First(c => c is Mangasee); + return this._connectors.First(c => c is Mangasee); case "Mangaworld": - return this.connectors.First(c => c is Mangaworld); + return this._connectors.First(c => c is Mangaworld); } throw new Exception(); diff --git a/Tranga/MangaConnectors/MangaDex.cs b/Tranga/MangaConnectors/MangaDex.cs index cd0226e..f7f05e5 100644 --- a/Tranga/MangaConnectors/MangaDex.cs +++ b/Tranga/MangaConnectors/MangaDex.cs @@ -1,5 +1,4 @@ -using System.Globalization; -using System.Net; +using System.Net; using System.Text.Json.Nodes; using System.Text.RegularExpressions; using Tranga.Jobs; @@ -51,16 +50,23 @@ public class MangaDex : MangaConnector if (result is null) break; - total = result["total"]!.GetValue(); //Update the total number of Publications - - JsonArray mangaInResult = result["data"]!.AsArray(); //Manga-data-Array - //Loop each Manga and extract information from JSON - foreach (JsonNode? mangaNode in mangaInResult) + if(result.ContainsKey("total")) + total = result["total"]!.GetValue(); //Update the total number of Publications + else continue; + + if (result.ContainsKey("data")) { - Log($"Getting publication data. {++loadedPublicationData}/{total}"); - Manga manga = MangaFromJsonObject((JsonObject)mangaNode); - retManga.Add(manga); //Add Publication (Manga) to result - } + JsonArray mangaInResult = result["data"]!.AsArray(); //Manga-data-Array + //Loop each Manga and extract information from JSON + foreach (JsonNode? mangaNode in mangaInResult) + { + if(mangaNode is null) + continue; + Log($"Getting publication data. {++loadedPublicationData}/{total}"); + if(MangaFromJsonObject((JsonObject) mangaNode) is { } manga) + retManga.Add(manga); //Add Publication (Manga) to result + } + }//else continue; } Log($"Retrieved {retManga.Count} publications. Term=\"{publicationTitle}\""); return retManga.ToArray(); @@ -81,20 +87,30 @@ public class MangaDex : MangaConnector return null; } - private Manga MangaFromJsonObject(JsonObject manga) + private Manga? MangaFromJsonObject(JsonObject manga) { + if (!manga.ContainsKey("attributes")) + return null; JsonObject attributes = manga["attributes"]!.AsObject(); - + + if(!manga.ContainsKey("id")) + return null; string publicationId = manga["id"]!.GetValue(); + if(!attributes.ContainsKey("title")) + return null; string title = attributes["title"]!.AsObject().ContainsKey("en") && attributes["title"]!["en"] is not null ? attributes["title"]!["en"]!.GetValue() : attributes["title"]![((IDictionary)attributes["title"]!.AsObject()).Keys.First()]!.GetValue(); + if(!attributes.ContainsKey("description")) + return null; string? description = attributes["description"]!.AsObject().ContainsKey("en") && attributes["description"]!["en"] is not null ? attributes["description"]!["en"]!.GetValue() : null; + if(!attributes.ContainsKey("altTitles")) + return null; JsonArray altTitlesObject = attributes["altTitles"]!.AsArray(); Dictionary altTitlesDict = new(); foreach (JsonNode? altTitleNode in altTitlesObject) @@ -104,6 +120,8 @@ public class MangaDex : MangaConnector altTitlesDict.TryAdd(key, altTitleObject[key]!.GetValue()); } + if(!attributes.ContainsKey("tags")) + return null; JsonArray tagsObject = attributes["tags"]!.AsArray(); HashSet tags = new(); foreach (JsonNode? tagNode in tagsObject) @@ -149,6 +167,8 @@ public class MangaDex : MangaConnector ? attributes["originalLanguage"]!.GetValue() : null; + if(!attributes.ContainsKey("status")) + return null; string status = attributes["status"]!.GetValue(); Manga pub = new( @@ -226,7 +246,7 @@ public class MangaDex : MangaConnector { if (progressToken?.cancellationRequested ?? false) { - progressToken?.Cancel(); + progressToken.Cancel(); return HttpStatusCode.RequestTimeout; } diff --git a/Tranga/MangaConnectors/MangaKatana.cs b/Tranga/MangaConnectors/MangaKatana.cs index 70c60b2..10f301c 100644 --- a/Tranga/MangaConnectors/MangaKatana.cs +++ b/Tranga/MangaConnectors/MangaKatana.cs @@ -1,5 +1,4 @@ -using System.Globalization; -using System.Net; +using System.Net; using System.Text.RegularExpressions; using HtmlAgilityPack; using Tranga.Jobs; @@ -187,7 +186,7 @@ public class MangaKatana : MangaConnector { if (progressToken?.cancellationRequested ?? false) { - progressToken?.Cancel(); + progressToken.Cancel(); return HttpStatusCode.RequestTimeout; } diff --git a/Tranga/MangaConnectors/Manganato.cs b/Tranga/MangaConnectors/Manganato.cs index d56a298..7635fd5 100644 --- a/Tranga/MangaConnectors/Manganato.cs +++ b/Tranga/MangaConnectors/Manganato.cs @@ -1,5 +1,4 @@ -using System.Globalization; -using System.Net; +using System.Net; using System.Text.RegularExpressions; using HtmlAgilityPack; using Tranga.Jobs; @@ -35,7 +34,8 @@ public class Manganato : MangaConnector private Manga[] ParsePublicationsFromHtml(HtmlDocument document) { - IEnumerable searchResults = document.DocumentNode.Descendants("div").Where(n => n.HasClass("search-story-item")); + List searchResults = document.DocumentNode.Descendants("div").Where(n => n.HasClass("search-story-item")).ToList(); + Log($"{searchResults.Count} items."); List urls = new(); foreach (HtmlNode mangaResult in searchResults) { @@ -141,7 +141,12 @@ public class Manganato : MangaConnector return Array.Empty(); List chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument); Log($"Got {chapters.Count} chapters. {manga}"); - return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, numberFormatDecimalPoint)).ToArray(); + return chapters.OrderBy(chapter => + { + if (float.TryParse(chapter.chapterNumber, numberFormatDecimalPoint, out float chapterNumber)) + return chapterNumber; + else return 0; + }).ToArray(); } private List ParseChaptersFromHtml(Manga manga, HtmlDocument document) @@ -159,7 +164,7 @@ public class Manganato : MangaConnector string fullString = chapterInfo.Descendants("a").First(d => d.HasClass("chapter-name")).InnerText; string? volumeNumber = volRex.IsMatch(fullString) ? volRex.Match(fullString).Groups[1].Value : null; - string chapterNumber = chapterRex.Match(fullString).Groups[1].Value; + string chapterNumber = chapterRex.IsMatch(fullString) ? chapterRex.Match(fullString).Groups[1].Value : fullString; string chapterName = nameRex.Match(fullString).Groups[3].Value; string url = chapterInfo.Descendants("a").First(d => d.HasClass("chapter-name")) .GetAttributeValue("href", ""); @@ -173,7 +178,7 @@ public class Manganato : MangaConnector { if (progressToken?.cancellationRequested ?? false) { - progressToken?.Cancel(); + progressToken.Cancel(); return HttpStatusCode.RequestTimeout; } diff --git a/Tranga/MangaConnectors/Mangasee.cs b/Tranga/MangaConnectors/Mangasee.cs index 382b1c2..22ae2ed 100644 --- a/Tranga/MangaConnectors/Mangasee.cs +++ b/Tranga/MangaConnectors/Mangasee.cs @@ -2,7 +2,6 @@ using System.Text.RegularExpressions; using System.Xml.Linq; using HtmlAgilityPack; -using Newtonsoft.Json; using Tranga.Jobs; namespace Tranga.MangaConnectors; @@ -20,7 +19,8 @@ public class Mangasee : MangaConnector public override Manga[] GetManga(string publicationTitle = "") { Log($"Searching Publications. Term=\"{publicationTitle}\""); - string requestUrl = $"https://mangasee123.com/_search.php"; + string sanitizedTitle = string.Join('+', Regex.Matches(publicationTitle, "[A-z]*").Where(str => str.Length > 0)).ToLower(); + string requestUrl = $"https://mangasee123.com/search/?name={sanitizedTitle}"; DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(requestUrl, 1); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) @@ -28,7 +28,7 @@ public class Mangasee : MangaConnector if (requestResult.htmlDocument is null) return Array.Empty(); - Manga[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument, publicationTitle); + Manga[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument); Log($"Retrieved {publications.Length} publications. Term=\"{publicationTitle}\""); return publications; } @@ -44,37 +44,25 @@ public class Mangasee : MangaConnector return null; } - private Manga[] ParsePublicationsFromHtml(HtmlDocument document, string publicationTitle) + private Manga[] ParsePublicationsFromHtml(HtmlDocument document) { - string jsonString = document.DocumentNode.SelectSingleNode("//body").InnerText; - List result = JsonConvert.DeserializeObject>(jsonString)!; - Dictionary queryFiltered = new(); - foreach (SearchResultItem resultItem in result) - { - int matches = resultItem.GetMatches(publicationTitle); - if (matches > 0) - queryFiltered.TryAdd(resultItem, matches); - } - - queryFiltered = queryFiltered.Where(item => item.Value >= publicationTitle.Split(' ').Length - 1) - .ToDictionary(item => item.Key, item => item.Value); - - Log($"Retrieved {queryFiltered.Count} publications."); + HtmlNode resultsNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']/div[last()]/div[1]/div"); + Log($"{resultsNode.SelectNodes("div").Count} items."); HashSet ret = new(); - List orderedFiltered = - queryFiltered.OrderBy(item => item.Value).ToDictionary(item => item.Key, item => item.Value).Keys.ToList(); - foreach (SearchResultItem orderedItem in orderedFiltered) + foreach (HtmlNode resultNode in resultsNode.SelectNodes("div")) { - Manga? manga = GetMangaFromUrl($"https://mangasee123.com/manga/{orderedItem.i}"); + string url = resultNode.Descendants().First(d => d.HasClass("SeriesName")).GetAttributeValue("href", ""); + Manga? manga = GetMangaFromUrl($"https://mangasee123.com{url}"); if (manga is not null) ret.Add((Manga)manga); } + return ret.ToArray(); } - + private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId) { string originalLanguage = "", status = ""; @@ -88,71 +76,42 @@ public class Mangasee : MangaConnector HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//h1"); string sortName = titleNode.InnerText; - HtmlNode[] authorsNodes = document.DocumentNode.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Author(s):']/..").Descendants("a").ToArray(); + HtmlNode[] authorsNodes = document.DocumentNode + .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Author(s):']/..").Descendants("a") + .ToArray(); List authors = new(); - foreach(HtmlNode authorNode in authorsNodes) + foreach (HtmlNode authorNode in authorsNodes) authors.Add(authorNode.InnerText); - - HtmlNode[] genreNodes = document.DocumentNode.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Genre(s):']/..").Descendants("a").ToArray(); + + HtmlNode[] genreNodes = document.DocumentNode + .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Genre(s):']/..").Descendants("a") + .ToArray(); foreach (HtmlNode genreNode in genreNodes) tags.Add(genreNode.InnerText); - - HtmlNode yearNode = document.DocumentNode.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Released:']/..").Descendants("a").First(); + + HtmlNode yearNode = document.DocumentNode + .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Released:']/..").Descendants("a") + .First(); int year = Convert.ToInt32(yearNode.InnerText); - - HtmlNode[] statusNodes = document.DocumentNode.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Status:']/..").Descendants("a").ToArray(); - foreach(HtmlNode statusNode in statusNodes) + + HtmlNode[] statusNodes = document.DocumentNode + .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Status:']/..").Descendants("a") + .ToArray(); + foreach (HtmlNode statusNode in statusNodes) if (statusNode.InnerText.Contains("publish", StringComparison.CurrentCultureIgnoreCase)) status = statusNode.InnerText.Split(' ')[0]; - - HtmlNode descriptionNode = document.DocumentNode.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Description:']/..").Descendants("div").First(); + + HtmlNode descriptionNode = document.DocumentNode + .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Description:']/..") + .Descendants("div").First(); string description = descriptionNode.InnerText; - - Manga manga = new (sortName, authors.ToList(), description, altTitles, tags.ToArray(), posterUrl, coverFileNameInCache, links, + + Manga manga = new(sortName, authors.ToList(), description, altTitles, tags.ToArray(), posterUrl, + coverFileNameInCache, links, year, originalLanguage, status, publicationId); cachedPublications.Add(manga); return manga; } - - // ReSharper disable once ClassNeverInstantiated.Local Will be instantiated during deserialization - private class SearchResultItem - { - public string i { get; init; } - public string s { get; init; } - public string[] a { get; init; } - - [JsonConstructor] - public SearchResultItem(string i, string s, string[] a) - { - this.i = i; - this.s = s; - this.a = a; - } - - public int GetMatches(string title) - { - int ret = 0; - Regex cleanRex = new("[A-z0-9]*"); - string[] badWords = { "a", "an", "no", "ni", "so", "as", "and", "the", "of", "that", "in", "is", "for" }; - - string[] titleTerms = title.Split(new[] { ' ', '-' }).Where(str => !badWords.Contains(str)).ToArray(); - - foreach (Match matchTerm in cleanRex.Matches(this.i)) - ret += titleTerms.Count(titleTerm => - titleTerm.Equals(matchTerm.Value, StringComparison.OrdinalIgnoreCase)); - - foreach (Match matchTerm in cleanRex.Matches(this.s)) - ret += titleTerms.Count(titleTerm => - titleTerm.Equals(matchTerm.Value, StringComparison.OrdinalIgnoreCase)); - - foreach(string alt in this.a) - foreach (Match matchTerm in cleanRex.Matches(alt)) - ret += titleTerms.Count(titleTerm => - titleTerm.Equals(matchTerm.Value, StringComparison.OrdinalIgnoreCase)); - - return ret; - } - } public override Chapter[] GetChapters(Manga manga, string language="en") { @@ -163,11 +122,10 @@ public class Mangasee : MangaConnector foreach (XElement chapter in chapterItems) { string volumeNumber = "1"; - string chapterName = chapter.Descendants("title").First().Value; - string chapterNumber = Regex.Matches(chapterName, "[0-9]+")[^1].ToString(); - string url = chapter.Descendants("link").First().Value; - url = url.Replace(Regex.Matches(url,"(-page-[0-9])")[0].ToString(),""); + string chapterNumber = Regex.Match(url, @"-chapter-([0-9\.]+)").Groups[1].ToString(); + + url = url.Replace(Regex.Match(url,"(-page-[0-9])").Value,""); chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, url)); } @@ -180,14 +138,14 @@ public class Mangasee : MangaConnector { if (progressToken?.cancellationRequested ?? false) { - progressToken?.Cancel(); + progressToken.Cancel(); return HttpStatusCode.RequestTimeout; } Manga chapterParentManga = chapter.parentManga; if (progressToken?.cancellationRequested ?? false) { - progressToken?.Cancel(); + progressToken.Cancel(); return HttpStatusCode.RequestTimeout; } diff --git a/Tranga/MangaConnectors/Mangaworld.cs b/Tranga/MangaConnectors/Mangaworld.cs index 8b47866..1939423 100644 --- a/Tranga/MangaConnectors/Mangaworld.cs +++ b/Tranga/MangaConnectors/Mangaworld.cs @@ -69,11 +69,8 @@ public class Mangaworld: MangaConnector private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId) { - string status = ""; Dictionary altTitles = new(); Dictionary? links = null; - HashSet tags = new(); - string[] authors = Array.Empty(); string originalLanguage = ""; HtmlNode infoNode = document.DocumentNode.Descendants("div").First(d => d.HasClass("info")); @@ -90,13 +87,13 @@ public class Mangaworld: MangaConnector HtmlNode genresNode = metadata.SelectSingleNode("//span[text()='Generi: ']/.."); - tags = genresNode.SelectNodes("a").Select(node => node.InnerText).ToHashSet(); + HashSet tags = genresNode.SelectNodes("a").Select(node => node.InnerText).ToHashSet(); HtmlNode authorsNode = metadata.SelectSingleNode("//span[text()='Autore: ']/.."); - authors = new[] { authorsNode.SelectNodes("a").First().InnerText }; + string[] authors = new[] { authorsNode.SelectNodes("a").First().InnerText }; - status = metadata.SelectSingleNode("//span[text()='Stato: ']/..").SelectNodes("a").First().InnerText; + string status = metadata.SelectSingleNode("//span[text()='Stato: ']/..").SelectNodes("a").First().InnerText; string posterUrl = document.DocumentNode.SelectSingleNode("//img[@class='rounded']").GetAttributeValue("src", ""); @@ -169,7 +166,7 @@ public class Mangaworld: MangaConnector { if (progressToken?.cancellationRequested ?? false) { - progressToken?.Cancel(); + progressToken.Cancel(); return HttpStatusCode.RequestTimeout; } diff --git a/Tranga/Server.cs b/Tranga/Server.cs index bd9625e..7ba6122 100644 --- a/Tranga/Server.cs +++ b/Tranga/Server.cs @@ -52,7 +52,7 @@ public class Server : GlobalBase }); t.Start(); } - catch (HttpListenerException e) + catch (HttpListenerException) { } diff --git a/Tranga/TrangaArgs.cs b/Tranga/TrangaArgs.cs index 23be84e..86d9028 100644 --- a/Tranga/TrangaArgs.cs +++ b/Tranga/TrangaArgs.cs @@ -59,7 +59,7 @@ public partial class Tranga : GlobalBase private static void PrintHelp() { Console.WriteLine("Tranga-Help:"); - foreach (Argument argument in arguments.Values) + foreach (Argument argument in Arguments.Values) { foreach(string name in argument.names) Console.Write("{0} ", name); @@ -82,14 +82,14 @@ public partial class Tranga : GlobalBase { List argsList = args.ToList(); List ret = new(); - foreach (string name in arguments[arg].names) + foreach (string name in Arguments[arg].names) { int argIndex = argsList.IndexOf(name); if (argIndex != -1) { - if (arguments[arg].parameterCount == 0) + if (Arguments[arg].parameterCount == 0) return ret.ToArray(); - for (int parameterIndex = 1; parameterIndex <= arguments[arg].parameterCount; parameterIndex++) + for (int parameterIndex = 1; parameterIndex <= Arguments[arg].parameterCount; parameterIndex++) { if(argIndex + parameterIndex >= argsList.Count || args[argIndex + parameterIndex].Contains('-'))//End of arguments, or no parameter provided, when one is required Console.WriteLine($"No parameter provided for argument {name}. -h for help."); @@ -100,7 +100,7 @@ public partial class Tranga : GlobalBase return ret.Any() ? ret.ToArray() : null; } - private static Dictionary arguments = new() + private static readonly Dictionary Arguments = new() { { ArgEnum.DownloadLocation, new(new []{"-d", "--downloadLocation"}, 1, "Directory to which downloaded Manga are saved") }, { ArgEnum.WorkingDirectory, new(new []{"-w", "--workingDirectory"}, 1, "Directory in which application-data is saved") }, diff --git a/Tranga/TrangaSettings.cs b/Tranga/TrangaSettings.cs index 1da2d2e..ed613cc 100644 --- a/Tranga/TrangaSettings.cs +++ b/Tranga/TrangaSettings.cs @@ -46,7 +46,7 @@ public class TrangaSettings this.downloadLocation = downloadLocation!; this.workingDirectory = workingDirectory!; } - UpdateDownloadLocation(this.downloadLocation!, false); + UpdateDownloadLocation(this.downloadLocation, false); } public HashSet LoadLibraryConnectors(GlobalBase clone)