Merge branch 'cuttingedge-merge-ServerV2' into cuttingedge

Fix Directory.Exists jobsFolderPath to create new Directory
Fix Loading Job fails leading to crash.

Tranga/Jobs/JobBoss.cs

@@ -1,4 +1,5 @@
-using System.Runtime.InteropServices;
+using System.Diagnostics;
+using System.Runtime.InteropServices;
 using System.Text.RegularExpressions;
 using Newtonsoft.Json;
 using Tranga.MangaConnectors;
@@ -145,26 +146,24 @@ public class JobBoss : GlobalBase
 
     private void LoadJobsList(HashSet<MangaConnector> connectors)
     {
-        Directory.CreateDirectory(TrangaSettings.jobsFolderPath);
-        if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
-            File.SetUnixFileMode(TrangaSettings.jobsFolderPath, UserRead | UserWrite | UserExecute | GroupRead | OtherRead);
         if (!Directory.Exists(TrangaSettings.jobsFolderPath)) //No jobs to load
+        {
+            Directory.CreateDirectory(TrangaSettings.jobsFolderPath);
+            if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+                File.SetUnixFileMode(TrangaSettings.jobsFolderPath, UserRead | UserWrite | UserExecute | GroupRead | OtherRead);
             return;
+        }
 
         //Load json-job-files
         foreach (FileInfo file in Directory.GetFiles(TrangaSettings.jobsFolderPath, "*.json").Select(f => new FileInfo(f)))
         {
             Log($"Adding {file.Name}");
-            Job? job = JsonConvert.DeserializeObject<Job>(File.ReadAllText(file.FullName),
-                new JobJsonConverter(this, new MangaConnectorJsonConverter(this, connectors)));
-            if (job is null)
-            {
-                string newName = file.FullName + ".failed";
-                Log($"Failed loading file {file.Name}.\nMoving to {newName}");
-                File.Move(file.FullName, newName);
-            }
-            else
+            try
             {
+                Job? job = JsonConvert.DeserializeObject<Job>(File.ReadAllText(file.FullName),
+                    new JobJsonConverter(this, new MangaConnectorJsonConverter(this, connectors)));
+                if (job is null) throw new NullReferenceException();
+                
                 Log($"Adding Job {job}");
                 if (!AddJob(job, file.FullName)) //If we detect a duplicate, delete the file.
                 {
@@ -173,6 +172,17 @@ public class JobBoss : GlobalBase
                     Log($"Duplicate detected or otherwise not able to add job to list.\nMoved job {job} to {path}");
                 }
             }
+            catch (Exception e)
+            {
+                if (e is not UnreachableException or NullReferenceException)
+                    throw;
+                Log(e.Message);
+                string newName = file.FullName + ".failed";
+                Log($"Failed loading file {file.Name}.\nMoving to {newName}.\n" +
+                    $"If you think this is a bug, upload contents of the file to the Bugreport!");
+                File.Move(file.FullName, newName);
+                continue;
+            }
         }
 
         //Connect jobs to parent-jobs and add Publications to cache

Tranga/MangaConnectors/ChromiumDownloadClient.cs

@@ -10,7 +10,6 @@ namespace Tranga.MangaConnectors;
 internal class ChromiumDownloadClient : DownloadClient
 {
     private static IBrowser? _browser;
-    private const int StartTimeoutMs = 10000;
     private readonly HttpDownloadClient _httpDownloadClient;
     
     private static async Task<IBrowser> StartBrowser(Logging.Logger? logger = null)
@@ -24,7 +23,7 @@ internal class ChromiumDownloadClient : DownloadClient
                 "--disable-dev-shm-usage",
                 "--disable-setuid-sandbox",
                 "--no-sandbox"},
-            Timeout = StartTimeoutMs
+            Timeout = TrangaSettings.ChromiumStartupTimeoutMs
         }, new LoggerFactory([new LogProvider(logger)]));
     }
 
@@ -70,17 +69,19 @@ internal class ChromiumDownloadClient : DownloadClient
 
     private RequestResult MakeRequestBrowser(string url, string? referrer = null, string? clickButton = null)
     {
+        if (_browser is null)
+            return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null);
         IPage page = _browser.NewPageAsync().Result;
-        page.DefaultTimeout = 10000;
+        page.DefaultTimeout = TrangaSettings.ChromiumPageTimeoutMs;
         IResponse response;
         try
         {
             response = page.GoToAsync(url, WaitUntilNavigation.Networkidle0).Result;
-            Log("Page loaded.");
+            Log($"Page loaded. {url}");
         }
         catch (Exception e)
         {
-            Log($"Could not load Page:\n{e.Message}");
+            Log($"Could not load Page {url}\n{e.Message}");
             page.CloseAsync();
             return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null);
         }

Tranga/MangaConnectors/MangaConnectorJsonConverter.cs

@@ -32,7 +32,6 @@ public class MangaConnectorJsonConverter : JsonConverter
             "MangaDex" => this._connectors.First(c => c is MangaDex),
             "Manganato" => this._connectors.First(c => c is Manganato),
             "MangaKatana" => this._connectors.First(c => c is MangaKatana),
-            "Mangasee" => this._connectors.First(c => c is Mangasee),
             "Mangaworld" => this._connectors.First(c => c is Mangaworld),
             "Bato" => this._connectors.First(c => c is Bato),
             "Manga4Life" => this._connectors.First(c => c is MangaLife),

Tranga/MangaConnectors/Mangasee.cs

@@ -1,234 +0,0 @@
-using System.Data;
-using System.Net;
-using System.Text.RegularExpressions;
-using System.Xml.Linq;
-using HtmlAgilityPack;
-using Newtonsoft.Json;
-using Soenneker.Utils.String.NeedlemanWunsch;
-using Tranga.Jobs;
-
-namespace Tranga.MangaConnectors;
-
-public class Mangasee : MangaConnector
-{
-    public Mangasee(GlobalBase clone) : base(clone, "Mangasee", ["en"])
-    {
-        this.downloadClient = new ChromiumDownloadClient(clone);
-    }
-
-    private struct SearchResult
-    {
-        public string i { get; set; }
-        public string s { get; set; }
-        public string[] a { get; set; }
-    }
-
-    public override Manga[] GetManga(string publicationTitle = "")
-    {
-        Log($"Searching Publications. Term=\"{publicationTitle}\"");
-        string requestUrl = "https://mangasee123.com/_search.php";
-        RequestResult requestResult =
-            downloadClient.MakeRequest(requestUrl, RequestType.Default);
-        if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
-        {
-            Log($"Failed to retrieve search: {requestResult.statusCode}");
-            return Array.Empty<Manga>();
-        }
-        
-        try
-        {
-            SearchResult[] searchResults = JsonConvert.DeserializeObject<SearchResult[]>(requestResult.htmlDocument!.DocumentNode.InnerText) ??
-                                           throw new NoNullAllowedException();
-            SearchResult[] filteredResults = FilteredResults(publicationTitle, searchResults);
-            Log($"Total available manga: {searchResults.Length} Filtered down to: {filteredResults.Length}");
-            
-
-            string[] urls = filteredResults.Select(result => $"https://mangasee123.com/manga/{result.i}").ToArray();
-            List<Manga> searchResultManga = new();
-            foreach (string url in urls)
-            {
-                Manga? newManga = GetMangaFromUrl(url);
-                if(newManga is { } manga)
-                    searchResultManga.Add(manga);
-            }
-            Log($"Retrieved {searchResultManga.Count} publications. Term=\"{publicationTitle}\"");
-            return searchResultManga.ToArray();
-        }
-        catch (NoNullAllowedException)
-        {
-            Log("Failed to retrieve search");
-            return Array.Empty<Manga>();
-        }
-    }
-
-    private readonly string[] _filterWords = {"a", "the", "of", "as", "to", "no", "for", "on", "with", "be", "and", "in", "wa", "at", "be", "ni"};
-    private string ToFilteredString(string input) => string.Join(' ', input.ToLower().Split(' ').Where(word => _filterWords.Contains(word) == false));
-    private SearchResult[] FilteredResults(string publicationTitle, SearchResult[] unfilteredSearchResults)
-    {
-        Dictionary<SearchResult, int> similarity = new();
-        foreach (SearchResult sr in unfilteredSearchResults)
-        {
-            List<int> scores = new();
-            string filteredPublicationString = ToFilteredString(publicationTitle);
-            string filteredSString = ToFilteredString(sr.s);
-            scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredSString, filteredPublicationString));
-            foreach (string srA in sr.a)
-            {
-                string filteredAString = ToFilteredString(srA);
-                scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredAString, filteredPublicationString));
-            }
-            similarity.Add(sr, scores.Sum() / scores.Count);
-        }
-
-        List<SearchResult> ret = similarity.OrderBy(s => s.Value).Take(10).Select(s => s.Key).ToList();
-        return ret.ToArray();
-    }
-
-    public override Manga? GetMangaFromId(string publicationId)
-    {
-        return GetMangaFromUrl($"https://mangasee123.com/manga/{publicationId}");
-    }
-
-    public override Manga? GetMangaFromUrl(string url)
-    {
-        Regex publicationIdRex = new(@"https:\/\/mangasee123.com\/manga\/(.*)(\/.*)*");
-        string publicationId = publicationIdRex.Match(url).Groups[1].Value;
-
-        RequestResult requestResult = this.downloadClient.MakeRequest(url, RequestType.MangaInfo);
-        if((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 && requestResult.htmlDocument is not null)
-            return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url);
-        return null;
-    }
-
-    private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl)
-    {
-        string originalLanguage = "", status = "";
-        Dictionary<string, string> altTitles = new(), links = new();
-        HashSet<string> tags = new();
-        Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased;
-
-        HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//img");
-        string posterUrl = posterNode.GetAttributeValue("src", "");
-        string coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover);
-
-        HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//h1");
-        string sortName = titleNode.InnerText;
-
-        HtmlNode[] authorsNodes = document.DocumentNode
-            .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Author(s):']/..").Descendants("a")
-            .ToArray();
-        List<string> authors = new();
-        foreach (HtmlNode authorNode in authorsNodes)
-            authors.Add(authorNode.InnerText);
-
-        HtmlNode[] genreNodes = document.DocumentNode
-            .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Genre(s):']/..").Descendants("a")
-            .ToArray();
-        foreach (HtmlNode genreNode in genreNodes)
-            tags.Add(genreNode.InnerText);
-
-        HtmlNode yearNode = document.DocumentNode
-            .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Released:']/..").Descendants("a")
-            .First();
-        int year = Convert.ToInt32(yearNode.InnerText);
-
-        HtmlNode[] statusNodes = document.DocumentNode
-            .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Status:']/..").Descendants("a")
-            .ToArray();
-        foreach (HtmlNode statusNode in statusNodes)
-            if (statusNode.InnerText.Contains("publish", StringComparison.CurrentCultureIgnoreCase))
-                status = statusNode.InnerText.Split(' ')[0];
-        switch (status.ToLower())
-        {
-            case "cancelled": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break;
-            case "hiatus": releaseStatus = Manga.ReleaseStatusByte.OnHiatus; break;
-            case "discontinued": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break;
-            case "complete": releaseStatus = Manga.ReleaseStatusByte.Completed; break;
-            case "ongoing": releaseStatus = Manga.ReleaseStatusByte.Continuing; break;
-        }
-
-        HtmlNode descriptionNode = document.DocumentNode
-            .SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Description:']/..")
-            .Descendants("div").First();
-        string description = descriptionNode.InnerText;
-
-        Manga manga = new(sortName, authors.ToList(), description, altTitles, tags.ToArray(), posterUrl,
-            coverFileNameInCache, links,
-            year, originalLanguage, publicationId, releaseStatus, websiteUrl: websiteUrl);
-        AddMangaToCache(manga);
-        return manga;
-    }
-
-    public override Chapter[] GetChapters(Manga manga, string language="en")
-    {
-        Log($"Getting chapters {manga}");
-        try
-        {
-            XDocument doc = XDocument.Load($"https://mangasee123.com/rss/{manga.publicationId}.xml");
-            XElement[] chapterItems = doc.Descendants("item").ToArray();
-            List<Chapter> chapters = new();
-            Regex chVolRex = new(@".*chapter-([0-9\.]+)(?:-index-([0-9\.]+))?.*");
-            foreach (XElement chapter in chapterItems)
-            {
-                string url = chapter.Descendants("link").First().Value;
-                Match m = chVolRex.Match(url);
-                string? volumeNumber = m.Groups[2].Success ? m.Groups[2].Value : "1";
-                string chapterNumber = m.Groups[1].Value;
-
-                string chapterUrl = Regex.Replace(url, @"-page-[0-9]+(\.html)", ".html");
-                try
-                {
-                    chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, chapterUrl));
-                }
-                catch (Exception e)
-                {
-                    Log($"Failed to load chapter {chapterNumber}: {e.Message}");
-                }
-            }
-
-            //Return Chapters ordered by Chapter-Number
-            Log($"Got {chapters.Count} chapters. {manga}");
-            return chapters.Order().ToArray();
-        }
-        catch (HttpRequestException e)
-        {
-            Log($"Failed to load https://mangasee123.com/rss/{manga.publicationId}.xml \n\r{e}");
-            return Array.Empty<Chapter>();
-        }
-    }
-
-    public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
-    {
-        if (progressToken?.cancellationRequested ?? false)
-        {
-            progressToken.Cancel();
-            return HttpStatusCode.RequestTimeout;
-        }
-
-        Manga chapterParentManga = chapter.parentManga;
-        if (progressToken?.cancellationRequested ?? false)
-        {
-            progressToken.Cancel();
-            return HttpStatusCode.RequestTimeout;
-        }
-
-        Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
-
-        RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, RequestType.Default);
-        if (requestResult.htmlDocument is null)
-        {
-            progressToken?.Cancel();
-            return HttpStatusCode.RequestTimeout;
-        }
-
-        HtmlDocument document = requestResult.htmlDocument;
-        
-        HtmlNode gallery = document.DocumentNode.Descendants("div").First(div => div.HasClass("ImageGallery"));
-        HtmlNode[] images = gallery.Descendants("img").Where(img => img.HasClass("img-fluid")).ToArray();
-        List<string> urls = new();
-        foreach(HtmlNode galleryImage in images)
-            urls.Add(galleryImage.GetAttributeValue("src", ""));
-        
-        return DownloadChapterImages(urls.ToArray(), chapter, RequestType.MangaImage, progressToken:progressToken);
-    }
-}

Tranga/Tranga.cs

@@ -18,7 +18,6 @@ public partial class Tranga : GlobalBase
         _connectors = new HashSet<MangaConnector>()
         {  
             new Manganato(this),
-            new Mangasee(this),
             new MangaDex(this),
             new MangaKatana(this),
             new Mangaworld(this),

Tranga/TrangaSettings.cs

@@ -35,6 +35,8 @@ public static class TrangaSettings
     };
 
     public static Dictionary<RequestType, int> requestLimits { get; set; } = DefaultRequestLimits;
+    public static int ChromiumStartupTimeoutMs { get; set; } = 30000;
+    public static int ChromiumPageTimeoutMs { get; set; } = 30000;
 
     public static void LoadFromWorkingDirectory(string directory)
     {
@@ -167,6 +169,8 @@ public static class TrangaSettings
         jobj.Add("requestLimits", JToken.FromObject(requestLimits));
         jobj.Add("bufferLibraryUpdates", JToken.FromObject(bufferLibraryUpdates));
         jobj.Add("bufferNotifications", JToken.FromObject(bufferNotifications));
+        jobj.Add("chromiumStartTimeout", JToken.FromObject(ChromiumStartupTimeoutMs));
+        jobj.Add("chromiumPageTimeout", JToken.FromObject(ChromiumPageTimeoutMs));
         return jobj;
     }
 
@@ -191,5 +195,9 @@ public static class TrangaSettings
             bufferLibraryUpdates = blu.Value<bool>()!;
         if (jobj.TryGetValue("bufferNotifications", out JToken? bn))
             bufferNotifications = bn.Value<bool>()!;
+        if (jobj.TryGetValue("chromiumStartTimeout", out JToken? cst))
+            ChromiumStartupTimeoutMs = cst.Value<int>();
+        if (jobj.TryGetValue("chromiumPageTimeout", out JToken? cpt))
+            ChromiumPageTimeoutMs = cpt.Value<int>();
     }
 }