Update readme HtmlAgilityPack

executionStarted,
executionApproximatelyFinished,
executionApproximatelyRemaining
to track progress

README.md

@@ -55,8 +55,9 @@ Tranga can download Chapters and Metadata from Scanlation sites such as
 
 - [MangaDex.org](https://mangadex.org/)
 - [Manganato.com](https://manganato.com/)
+- [Mangasee](https://mangasee123.com/)
 
-and automatically start updates in [Komga](https://komga.org/) to import them.
+and automatically start updates in [Komga](https://komga.org/) and [Kavita](https://www.kavitareader.com/) to import them.
 
 ### Inspiration:
 
@@ -72,6 +73,8 @@ That is why I wanted to create my own project, in a language I understand, and t
 
 - .NET-Core
 - Newtonsoft.JSON
+- [PuppeteerSharp](https://www.puppeteersharp.com/)
+- [Html Agility Pack (HAP)](https://html-agility-pack.net/)
 - Love <3 Blåhaj 🦈
 
 <p align="right">(<a href="#readme-top">back to top</a>)</p>
@@ -108,14 +111,12 @@ Wherever you are mounting `/usr/share/Tranga-API` you also need to mount that sa
 
 ### Prerequisites
 
-[.NET-Core 7.0](https://dotnet.microsoft.com/en-us/download/dotnet/7.0)
+[.NET-Core 7.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/7.0)
 
 <!-- ROADMAP -->
 ## Roadmap
 
-- [x] Web-UI #1
-- [ ] More Connectors
-  - [x] Manganato #2
+- [ ] Docker ARM support
 - [ ] ? 
 
 See the [open issues](https://git.bernloehr.eu/glax/Tranga/issues) for a full list of proposed features (and known issues).

Tranga-CLI/Tranga_Cli.cs

@@ -135,8 +135,12 @@ public static class Tranga_Cli
                 switch (selection)
                 {
                     case ConsoleKey.L:
-                        PrintTasks(taskManager.GetAllTasks(), logger);
-                        Console.WriteLine("Press any key.");
+                        while (!Console.KeyAvailable)
+                        {
+                            PrintTasks(taskManager.GetAllTasks(), logger);
+                            Console.WriteLine("Press any key.");
+                            Thread.Sleep(500);
+                        }
                         Console.ReadKey();
                         break;
                     case ConsoleKey.C:
@@ -160,17 +164,26 @@ public static class Tranga_Cli
                         Console.ReadKey();
                         break;
                     case ConsoleKey.R:
-                        PrintTasks(
-                            taskManager.GetAllTasks().Where(eTask => eTask.state == TrangaTask.ExecutionState.Running)
-                                .ToArray(), logger);
-                        Console.WriteLine("Press any key.");
+                        while (!Console.KeyAvailable)
+                        {
+                            PrintTasks(
+                                taskManager.GetAllTasks().Where(eTask => eTask.state == TrangaTask.ExecutionState.Running)
+                                    .ToArray(), logger);
+                            Console.WriteLine("Press any key.");
+                            Thread.Sleep(500);
+                        }
                         Console.ReadKey();
                         break;
                     case ConsoleKey.K:
-                        PrintTasks(
-                            taskManager.GetAllTasks().Where(qTask => qTask.state is TrangaTask.ExecutionState.Enqueued)
-                                .ToArray(), logger);
-                        Console.WriteLine("Press any key.");
+                        while (!Console.KeyAvailable)
+                        {
+                            PrintTasks(
+                                taskManager.GetAllTasks()
+                                    .Where(qTask => qTask.state is TrangaTask.ExecutionState.Enqueued)
+                                    .ToArray(), logger);
+                            Console.WriteLine("Press any key.");
+                            Thread.Sleep(500);
+                        }
                         Console.ReadKey();
                         break;
                     case ConsoleKey.F:
@@ -240,13 +253,13 @@ public static class Tranga_Cli
         int tIndex = 0;
         Console.WriteLine($"Tasks (Running/Queue/Total): {taskRunningCount}/{taskEnqueuedCount}/{taskCount}");
         string header =
-            $"{"",-5}{"Task",-20} | {"Last Executed",-20} | {"Reoccurrence",-12} | {"State",-10} | {"Progress",-9} | {"Connector",-15} | Publication/Manga ";
+            $"{"",-5}{"Task",-20} | {"Last Executed",-20} | {"Reoccurrence",-12} | {"State",-10} | {"Progress",-9} | {"Finished",-20} | {"Remaining",-12} | {"Connector",-15} | Publication/Manga ";
         Console.WriteLine(header);
         Console.WriteLine(new string('-', header.Length));
         foreach (TrangaTask trangaTask in tasks)
         {
             string[] taskSplit = trangaTask.ToString().Split(", ");
-            Console.WriteLine($"{tIndex++:000}: {taskSplit[0],-20} | {taskSplit[1],-20} | {taskSplit[2],-12} | {taskSplit[3],-10} | {(taskSplit.Length > 4 ? taskSplit[4] : ""),-9} | {(taskSplit.Length > 5 ? taskSplit[5] : ""),-15} | {(taskSplit.Length > 6 ? taskSplit[6] : "")} {(taskSplit.Length > 7 ? taskSplit[7] : "")} {(taskSplit.Length > 8 ? taskSplit[8] : "")}");
+            Console.WriteLine($"{tIndex++:000}: {taskSplit[0],-20} | {taskSplit[1],-20} | {taskSplit[2],-12} | {taskSplit[3],-10} | {taskSplit[4],-9} | {taskSplit[5],-20} | {taskSplit[6][..12],-12} | {(taskSplit.Length > 7 ? taskSplit[7] : ""),-15} | {(taskSplit.Length > 8 ? taskSplit[8] : "")} {(taskSplit.Length > 9 ? taskSplit[9] : "")} {(taskSplit.Length > 10 ? taskSplit[10] : "")}");
         }
             
     }

Tranga/Connector.cs

@@ -183,7 +183,7 @@ public abstract class Connector
             string extension = split[^1];
             logger?.WriteLine("Connector", $"Downloading Image {chapter + 1:000}/{imageUrls.Length:000} {parentTask.publication.sortName} {parentTask.publication.internalId} Vol.{parentTask.chapter.volumeNumber} Ch.{parentTask.chapter.chapterNumber} {parentTask.progress:P2}");
             DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer);
-            parentTask.progress += 1f / imageUrls.Length;
+            parentTask.IncrementProgress(1f / imageUrls.Length);
         }
         
         if(comicInfoPath is not null)

Tranga/Connectors/Mangasee.cs

@@ -31,34 +31,35 @@ public class Mangasee : Connector
     private async void DownloadBrowser()
     {
         BrowserFetcher browserFetcher = new BrowserFetcher();
-        if (browserFetcher.LocalRevisions().Contains(ChromiumVersion))
-            return;
-        else
-            foreach(string rev in browserFetcher.LocalRevisions())
-                browserFetcher.Remove(rev);
-        
-        logger?.WriteLine(this.GetType().ToString(), "Downloading headless browser");
-        DateTime last = DateTime.Now.Subtract(TimeSpan.FromSeconds(5));
-        browserFetcher.DownloadProgressChanged += async (sender, args) =>
+        foreach(string rev in browserFetcher.LocalRevisions().Where(rev => rev != ChromiumVersion))
+            browserFetcher.Remove(rev);
+        if (!browserFetcher.LocalRevisions().Contains(ChromiumVersion))
         {
-            double currentBytes = Convert.ToDouble(args.BytesReceived) / Convert.ToDouble(args.TotalBytesToReceive);
-            if (args.TotalBytesToReceive == args.BytesReceived)
+            logger?.WriteLine(this.GetType().ToString(), "Downloading headless browser");
+            DateTime last = DateTime.Now.Subtract(TimeSpan.FromSeconds(5));
+            browserFetcher.DownloadProgressChanged += (sender, args) =>
             {
-                logger?.WriteLine(this.GetType().ToString(), "Browser downloaded. Launching...");
-            }
-            else if (DateTime.Now > last.AddSeconds(5))
-            {
-                logger?.WriteLine(this.GetType().ToString(), $"Browser progress: {currentBytes:P2}");
-                last = DateTime.Now;
-            }
+                double currentBytes = Convert.ToDouble(args.BytesReceived) / Convert.ToDouble(args.TotalBytesToReceive);
+                if (args.TotalBytesToReceive == args.BytesReceived)
+                {
+                    logger?.WriteLine(this.GetType().ToString(), "Browser downloaded.");
+                }
+                else if (DateTime.Now > last.AddSeconds(5))
+                {
+                    logger?.WriteLine(this.GetType().ToString(), $"Browser download progress: {currentBytes:P2}");
+                    last = DateTime.Now;
+                }
 
-        };
-        if (!browserFetcher.CanDownloadAsync(ChromiumVersion).Result)
-        {
-            logger?.WriteLine(this.GetType().ToString(), $"Can't download browser version {ChromiumVersion}");
-            return;
+            };
+            if (!browserFetcher.CanDownloadAsync(ChromiumVersion).Result)
+            {
+                logger?.WriteLine(this.GetType().ToString(), $"Can't download browser version {ChromiumVersion}");
+                return;
+            }
+            await browserFetcher.DownloadAsync(ChromiumVersion);
         }
-        await browserFetcher.DownloadAsync(ChromiumVersion);
+        
+        logger?.WriteLine(this.GetType().ToString(), "Starting browser.");
         this._browser = await Puppeteer.LaunchAsync(new LaunchOptions
         {
             Headless = true,
@@ -173,11 +174,14 @@ public class Mangasee : Connector
             year, originalLanguage, status, publicationId);
     }
     
+    // ReSharper disable once ClassNeverInstantiated.Local Will be instantiated during deserialization
     private class SearchResultItem
     {
+#pragma warning disable CS8618 //Will always be set
         public string i { get; set; }
         public string s { get; set; }
         public string[] a { get; set; }
+#pragma warning restore CS8618
     }
 
     public override Chapter[] GetChapters(Publication publication, string language = "")
@@ -207,7 +211,8 @@ public class Mangasee : Connector
             logger?.WriteLine(this.GetType().ToString(), "Waiting for headless browser to download...");
             Thread.Sleep(1000);
         }
-
+        
+        logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
         IPage page = _browser.NewPageAsync().Result;
         IResponse response = page.GoToAsync(chapter.url).Result;
         if (response.Ok)

Tranga/TaskManager.cs

@@ -207,7 +207,7 @@ public class TaskManager
     /// </summary>
     /// <param name="task">TrangaTask.Task type</param>
     /// <param name="connectorName">Name of Connector that was used</param>
-    /// <param name="publication">Publication that was used</param>
+    /// <param name="publicationId">Publication that was used</param>
     public void DeleteTask(TrangaTask.Task task, string? connectorName, string? publicationId)
     {
         logger?.WriteLine(this.GetType().ToString(), $"Removing Task {task} {publicationId}");
@@ -220,13 +220,19 @@ public class TaskManager
                 _allTasks.RemoveWhere(trangaTask => trangaTask.task is TrangaTask.Task.UpdateLibraries);
                 break;
             case TrangaTask.Task.DownloadNewChapters:
-                if(connectorName is null || publicationId is null)
+                if (connectorName is null || publicationId is null)
                     logger?.WriteLine(this.GetType().ToString(), "connectorName and publication can not be null");
                 else
+                {
                     _allTasks.RemoveWhere(mTask =>
-                            mTask.GetType() == typeof(DownloadNewChaptersTask) &&
-                            ((DownloadNewChaptersTask)mTask).publication.internalId != publicationId &&
-                            ((DownloadNewChaptersTask)mTask).connectorName != connectorName!);
+                        mTask.GetType() == typeof(DownloadNewChaptersTask) &&
+                        ((DownloadNewChaptersTask)mTask).publication.internalId == publicationId &&
+                        ((DownloadNewChaptersTask)mTask).connectorName == connectorName!);
+                    _allTasks.RemoveWhere(mTask =>
+                        mTask.GetType() == typeof(DownloadChapterTask) &&
+                        ((DownloadChapterTask)mTask).publication.internalId == publicationId &&
+                        ((DownloadChapterTask)mTask).connectorName == connectorName!);
+                }
                 break;
         }
         ExportDataAndSettings();

Tranga/TrangaTask.cs

@@ -21,8 +21,17 @@ public abstract class TrangaTask
     public DateTime lastExecuted { get; set; }
     public Task task { get; }
     [Newtonsoft.Json.JsonIgnore]public ExecutionState state { get; set; }
-    [Newtonsoft.Json.JsonIgnore]public float progress { get; set; }
+    [Newtonsoft.Json.JsonIgnore]public float progress { get; protected set; }
     [Newtonsoft.Json.JsonIgnore]public DateTime nextExecution => lastExecuted.Add(reoccurrence);
+    [Newtonsoft.Json.JsonIgnore]public DateTime executionStarted { get; protected set; }
+
+    [Newtonsoft.Json.JsonIgnore]
+    public DateTime executionApproximatelyFinished => this.progress != 0
+        ? this.executionStarted.Add(DateTime.Now.Subtract(this.executionStarted) / this.progress)
+        : DateTime.MaxValue;
+    
+    [Newtonsoft.Json.JsonIgnore]
+    public TimeSpan executionApproximatelyRemaining => this.executionApproximatelyFinished.Subtract(DateTime.Now);
 
     public enum ExecutionState
     {
@@ -37,6 +46,13 @@ public abstract class TrangaTask
         this.lastExecuted = DateTime.Now.Subtract(reoccurrence);
         this.task = task;
         this.progress = 0f;
+        this.executionStarted = DateTime.Now;
+    }
+
+    public float IncrementProgress(float amount)
+    {
+        this.progress += amount;
+        return this.progress;
     }
     
     /// <summary>
@@ -55,6 +71,7 @@ public abstract class TrangaTask
     {
         logger?.WriteLine(this.GetType().ToString(), $"Executing Task {this}");
         this.state = ExecutionState.Running;
+        this.executionStarted = DateTime.Now;
         ExecuteTask(taskManager, logger);
         this.lastExecuted = DateTime.Now;
         this.state = ExecutionState.Waiting;
@@ -76,7 +93,7 @@ public abstract class TrangaTask
 
     public override string ToString()
     {
-        return $"{task}, {lastExecuted}, {reoccurrence}, {state}, {progress:P2}";
+        return $"{task}, {lastExecuted}, {reoccurrence}, {state}, {progress:P2}, {executionApproximatelyFinished}, {executionApproximatelyRemaining}";
     }
     
     public class TrangaTaskJsonConverter : JsonConverter

Tranga/TrangaTasks/DownloadChapterTask.cs

@@ -1,4 +1,5 @@
 using Logging;
+using Newtonsoft.Json;
 
 namespace Tranga.TrangaTasks;
 
@@ -8,12 +9,15 @@ public class DownloadChapterTask : TrangaTask
     public Publication publication { get; }
     public string language { get; }
     public Chapter chapter { get; }
-    public DownloadChapterTask(Task task, string connectorName, Publication publication, Chapter chapter, string language = "en") : base(task, TimeSpan.Zero)
+    [JsonIgnore]private DownloadNewChaptersTask? parentTask { get; init; }
+    
+    public DownloadChapterTask(Task task, string connectorName, Publication publication, Chapter chapter, string language = "en", DownloadNewChaptersTask? parentTask = null) : base(task, TimeSpan.Zero)
     {
         this.chapter = chapter;
         this.connectorName = connectorName;
         this.publication = publication;
         this.language = language;
+        this.parentTask = parentTask;
     }
 
     protected override void ExecuteTask(TaskManager taskManager, Logger? logger)
@@ -24,6 +28,13 @@ public class DownloadChapterTask : TrangaTask
         taskManager.DeleteTask(this);
     }
     
+    public new float IncrementProgress(float amount)
+    {
+        this.progress += amount;
+        parentTask?.IncrementProgress(amount);
+        return this.progress;
+    }
+
     public override string ToString()
     {
         return $"{base.ToString()}, {connectorName}, {publication.sortName} {publication.internalId}, Vol.{chapter.volumeNumber} Ch.{chapter.chapterNumber}";

Tranga/TrangaTasks/DownloadNewChaptersTask.cs

@@ -1,4 +1,5 @@
 using Logging;
+using Newtonsoft.Json;
 
 namespace Tranga.TrangaTasks;
 
@@ -7,34 +8,38 @@ public class DownloadNewChaptersTask : TrangaTask
     public string connectorName { get; }
     public Publication publication { get; }
     public string language { get; }
+    [JsonIgnore]private int childTaskAmount { get; set; }
+    
     public DownloadNewChaptersTask(Task task, string connectorName, Publication publication, TimeSpan reoccurrence, string language = "en") : base(task, reoccurrence)
     {
         this.connectorName = connectorName;
         this.publication = publication;
         this.language = language;
+        childTaskAmount = 0;
+    }
+    
+    public new float IncrementProgress(float amount)
+    {
+        this.progress += amount / this.childTaskAmount;
+        return this.progress;
     }
 
     protected override void ExecuteTask(TaskManager taskManager, Logger? logger)
     {
         Publication pub = publication!;
         Connector connector = taskManager.GetConnector(this.connectorName);
-        this.progress = 0.1f;
         
         //Check if Publication already has a Folder
         pub.CreatePublicationFolder(taskManager.settings.downloadLocation);
-        this.progress = 0.2f;
         List<Chapter> newChapters = GetNewChaptersList(connector, pub, language!, ref taskManager.chapterCollection);
-        this.progress = 0.6f;
+        this.childTaskAmount = newChapters.Count;
 
         connector.CopyCoverFromCacheToDownloadLocation(pub, taskManager.settings);
-        this.progress = 0.7f;
         
         pub.SaveSeriesInfoJson(connector.downloadLocation);
-        this.progress = 0.8f;
 
         foreach (Chapter newChapter in newChapters)
-            taskManager.AddTask(new DownloadChapterTask(Task.DownloadChapter, this.connectorName!, pub, newChapter, this.language));
-        this.progress = 1f;
+            taskManager.AddTask(new DownloadChapterTask(Task.DownloadChapter, this.connectorName!, pub, newChapter, this.language, this));
     }
     
     /// <summary>