Sanitization Manganto

Fixed CLI output

README.md

@@ -115,7 +115,7 @@ Wherever you are mounting `/usr/share/Tranga-API` you also need to mount that sa
 
 - [x] Web-UI #1
 - [ ] More Connectors
-  - [ ] Manganato #2
+  - [x] Manganato #2
 - [ ] ? 
 
 See the [open issues](https://git.bernloehr.eu/glax/Tranga/issues) for a full list of proposed features (and known issues).

Tranga-CLI/Tranga_Cli.cs

@@ -202,13 +202,13 @@ public static class Tranga_Cli
         int tIndex = 0;
         Console.WriteLine($"Tasks (Running/Queue/Total): {taskRunningCount}/{taskEnqueuedCount}/{taskCount}");
         string header =
-            $"{"",-5}{"Task",-20} | {"Last Executed",-20} | {"Reoccurrence",-12} | {"State",-10} | {"Connector",-15} | Publication/Manga";
+            $"{"",-5}{"Task",-20} | {"Last Executed",-20} | {"Reoccurrence",-12} | {"State",-10} | {"Connector",-15} | {"Progress",-9} | Publication/Manga";
         Console.WriteLine(header);
         Console.WriteLine(new string('-', header.Length));
         foreach (TrangaTask trangaTask in tasks)
         {
             string[] taskSplit = trangaTask.ToString().Split(", ");
-            Console.WriteLine($"{tIndex++:000}: {taskSplit[0],-20} | {taskSplit[1],-20} | {taskSplit[2],-12} | {taskSplit[3],-10} | {(taskSplit.Length > 4 ? taskSplit[4] : ""),-15} | {(taskSplit.Length > 5 ? taskSplit[5] : "")}");
+            Console.WriteLine($"{tIndex++:000}: {taskSplit[0],-20} | {taskSplit[1],-20} | {taskSplit[2],-12} | {taskSplit[3],-10} | {(taskSplit.Length > 4 ? taskSplit[4] : ""),-15} | {(taskSplit.Length > 5 ? taskSplit[5] : ""),-9} |{(taskSplit.Length > 6 ? taskSplit[6] : "")}");
         }
             
     }

Tranga/Connector.cs

@@ -1,7 +1,9 @@
 using System.IO.Compression;
 using System.Net;
+using System.Runtime.InteropServices;
 using System.Xml.Linq;
 using Logging;
+using static System.IO.UnixFileMode;
 
 namespace Tranga;
 
@@ -56,7 +58,8 @@ public abstract class Connector
     /// </summary>
     /// <param name="publication">Publication that contains Chapter</param>
     /// <param name="chapter">Chapter with Images to retrieve</param>
-    public abstract void DownloadChapter(Publication publication, Chapter chapter);
+    /// <param name="parentTask">Will be used for progress-tracking</param>
+    public abstract void DownloadChapter(Publication publication, Chapter chapter, TrangaTask parentTask);
 
     /// <summary>
     /// Copies the already downloaded cover from cache to downloadLocation
@@ -67,9 +70,7 @@ public abstract class Connector
     {
         logger?.WriteLine(this.GetType().ToString(), $"Cloning cover {publication.sortName} {publication.internalId}");
         //Check if Publication already has a Folder and cover
-        string publicationFolder = Path.Join(downloadLocation, publication.folderName);
-        if(!Directory.Exists(publicationFolder))
-            Directory.CreateDirectory(publicationFolder);
+        string publicationFolder = publication.CreatePublicationFolder(downloadLocation);
         DirectoryInfo dirInfo = new (publicationFolder);
         if (dirInfo.EnumerateFiles().Any(info => info.Name.Contains("cover.")))
         {
@@ -81,6 +82,8 @@ public abstract class Connector
         string newFilePath = Path.Join(publicationFolder, $"cover.{Path.GetFileName(fileInCache).Split('.')[^1]}" );
         logger?.WriteLine(this.GetType().ToString(), $"Cloning cover {fileInCache} -> {newFilePath}");
         File.Copy(fileInCache, newFilePath, true);
+        if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+            File.SetUnixFileMode(newFilePath, GroupRead | GroupWrite | OtherRead | OtherWrite | UserRead | UserWrite);
     }
 
     /// <summary>
@@ -97,7 +100,7 @@ public abstract class Connector
             new XElement("Title", chapter.name),
             new XElement("Writer", publication.author),
             new XElement("Volume", chapter.volumeNumber),
-            new XElement("Number", chapter.chapterNumber) //TODO check if this is correct at some point
+            new XElement("Number", chapter.chapterNumber)
         );
         return comicInfo.ToString();
     }
@@ -126,12 +129,17 @@ public abstract class Connector
     /// <param name="imageUrl"></param>
     /// <param name="fullPath"></param>
     /// <param name="requestType">RequestType for Rate-Limit</param>
+    /// <param name="referrer">referrer used in html request header</param>
     private void DownloadImage(string imageUrl, string fullPath, byte requestType, string? referrer = null)
     {
         DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType, referrer);
+        if (requestResult.result != Stream.Null)
+        {
             byte[] buffer = new byte[requestResult.result.Length];
             requestResult.result.ReadExactly(buffer, 0, buffer.Length);
             File.WriteAllBytes(fullPath, buffer);
+        }else
+            logger?.WriteLine(this.GetType().ToString(), "No Stream-Content in result.");
     }
 
     /// <summary>
@@ -139,9 +147,11 @@ public abstract class Connector
     /// </summary>
     /// <param name="imageUrls">List of URLs to download Images from</param>
     /// <param name="saveArchiveFilePath">Full path to save archive to (without file ending .cbz)</param>
+    /// <param name="parentTask">Used for progress tracking</param>
     /// <param name="comicInfoPath">Path of the generate Chapter ComicInfo.xml, if it was generated</param>
     /// <param name="requestType">RequestType for RateLimits</param>
-    protected void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, string? comicInfoPath = null, string? referrer = null)
+    /// <param name="referrer">Used in http request header</param>
+    protected void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, TrangaTask parentTask, string? comicInfoPath = null, string? referrer = null)
     {
         logger?.WriteLine("Connector", $"Downloading Images for {saveArchiveFilePath}");
         //Check if Publication Directory already exists
@@ -161,8 +171,9 @@ public abstract class Connector
         {
             string[] split = imageUrl.Split('.');
             string extension = split[^1];
-            logger?.WriteLine("Connector", $"Downloading Image {chapter + 1}/{imageUrls.Length}");
+            logger?.WriteLine("Connector", $"Downloading Image {chapter + 1:000}/{imageUrls.Length:000} {(parentTask.publication?.sortName)![..(int)(parentTask.publication?.sortName.Length > 25 ? 25 : parentTask.publication?.sortName.Length)!],-25} {(parentTask.publication?.internalId)![..(int)(parentTask.publication?.internalId.Length > 25 ? 25 : parentTask.publication?.internalId.Length)!],-25} Total Task Progress: {parentTask.progress:00.0}%");
             DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer);
+            parentTask.tasksFinished++;
         }
         
         if(comicInfoPath is not null)
@@ -171,6 +182,8 @@ public abstract class Connector
         logger?.WriteLine("Connector", $"Creating archive {saveArchiveFilePath}");
         //ZIP-it and ship-it
         ZipFile.CreateFromDirectory(tempFolder, saveArchiveFilePath);
+        if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+            File.SetUnixFileMode(saveArchiveFilePath, GroupRead | GroupWrite | OtherRead | OtherWrite | UserRead | UserWrite);
         Directory.Delete(tempFolder, true); //Cleanup
     }
     
@@ -218,6 +231,7 @@ public abstract class Connector
         /// </summary>
         /// <param name="url"></param>
         /// <param name="requestType">For RateLimits: Same Endpoints use same type</param>
+        /// <param name="referrer">Used in http request header</param>
         /// <returns>RequestResult with StatusCode and Stream of received data</returns>
         public RequestResult MakeRequest(string url, byte requestType, string? referrer = null)
         {
@@ -249,7 +263,7 @@ public abstract class Connector
                 catch (HttpRequestException e)
                 {
                     logger?.WriteLine(this.GetType().ToString(), e.Message);
-                    logger?.WriteLine(this.GetType().ToString(), $"Waiting {_rateLimit[requestType] * 2}");
+                    logger?.WriteLine(this.GetType().ToString(), $"Waiting {_rateLimit[requestType] * 2}... Retrying.");
                     Thread.Sleep(_rateLimit[requestType] * 2);
                 }
             }

Tranga/Connectors/MangaDex.cs

@@ -204,7 +204,7 @@ public class MangaDex : Connector
         return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray();
     }
 
-    public override void DownloadChapter(Publication publication, Chapter chapter)
+    public override void DownloadChapter(Publication publication, Chapter chapter, TrangaTask parentTask)
     {
         logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
         //Request URLs for Chapter-Images
@@ -228,7 +228,7 @@ public class MangaDex : Connector
         File.WriteAllText(comicInfoPath, GetComicInfoXmlString(publication, chapter, logger));
         
         //Download Chapter-Images
-        DownloadChapterImages(imageUrls.ToArray(), GetArchiveFilePath(publication, chapter), (byte)RequestType.AtHomeServer, comicInfoPath);
+        DownloadChapterImages(imageUrls.ToArray(), GetArchiveFilePath(publication, chapter), (byte)RequestType.AtHomeServer, parentTask, comicInfoPath);
     }
 
     private string? GetCoverUrl(string publicationId, string? posterId)

Tranga/Connectors/Manganato.cs

@@ -1,5 +1,4 @@
-using System.Collections;
-using System.Net;
+using System.Net;
 using System.Text.RegularExpressions;
 using HtmlAgilityPack;
 using Logging;
@@ -15,14 +14,14 @@ public class Manganato : Connector
         this.name = "Manganato";
         this.downloadClient = new DownloadClient(new Dictionary<byte, int>()
         {
-            {(byte)1, 100}
+            {(byte)1, 60}
         }, logger);
     }
 
     public override Publication[] GetPublications(string publicationTitle = "")
     {
         logger?.WriteLine(this.GetType().ToString(), $"Getting Publications (title={publicationTitle})");
-        string sanitizedTitle = publicationTitle.ToLower().Replace(' ', '_');
+        string sanitizedTitle = string.Concat(Regex.Matches(publicationTitle, "[A-z]* *")).ToLower().Replace(' ', '_');
         logger?.WriteLine(this.GetType().ToString(), $"Getting Publications (title={sanitizedTitle})");
         string requestUrl = $"https://manganato.com/search/story/{sanitizedTitle}";
         DownloadClient.RequestResult requestResult =
@@ -152,7 +151,7 @@ public class Manganato : Connector
             string fullString = chapterInfo.Descendants("a").First(d => d.HasClass("chapter-name")).InnerText;
 
             string? volumeNumber = fullString.Contains("Vol.") ? fullString.Replace("Vol.", "").Split(' ')[0] : null;
-            string? chapterNumber = fullString.Split(':')[0].Split(' ')[^1];
+            string? chapterNumber = fullString.Split(':')[0].Split("Chapter ")[1].Replace('-','.');
             string chapterName = string.Concat(fullString.Split(':')[1..]);
             string url = chapterInfo.Descendants("a").First(d => d.HasClass("chapter-name"))
                 .GetAttributeValue("href", "");
@@ -162,7 +161,7 @@ public class Manganato : Connector
         return ret.ToArray();
     }
 
-    public override void DownloadChapter(Publication publication, Chapter chapter)
+    public override void DownloadChapter(Publication publication, Chapter chapter, TrangaTask parentTask)
     {
         logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
         string requestUrl = chapter.url;
@@ -176,7 +175,7 @@ public class Manganato : Connector
         string comicInfoPath = Path.GetTempFileName();
         File.WriteAllText(comicInfoPath, GetComicInfoXmlString(publication, chapter, logger));
         
-        DownloadChapterImages(imageUrls, GetArchiveFilePath(publication, chapter), (byte)1, comicInfoPath, "https://chapmanganato.com/");
+        DownloadChapterImages(imageUrls, GetArchiveFilePath(publication, chapter), (byte)1, parentTask, comicInfoPath, "https://chapmanganato.com/");
     }
 
     private string[] ParseImageUrlsFromHtml(Stream html)

Tranga/Publication.cs

@@ -1,6 +1,8 @@
-using System.Text;
+using System.Runtime.InteropServices;
+using System.Text;
 using System.Text.RegularExpressions;
 using Newtonsoft.Json;
+using static System.IO.UnixFileMode;
 
 namespace Tranga;
 
@@ -48,14 +50,24 @@ public readonly struct Publication
         this.internalId = Convert.ToBase64String(Encoding.ASCII.GetBytes($"{onlyLowerLetters}{this.year}"));
     }
 
-    public void SaveSeriesInfoJson(string downloadDirectory)
+    public string CreatePublicationFolder(string downloadDirectory)
     {
         string publicationFolder = Path.Join(downloadDirectory, this.folderName);
         if(!Directory.Exists(publicationFolder))
             Directory.CreateDirectory(publicationFolder);
+        if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+            File.SetUnixFileMode(publicationFolder, GroupRead | GroupWrite | GroupExecute | OtherRead | OtherWrite | OtherExecute | UserRead | UserWrite | UserExecute);
+        return publicationFolder;
+    }
+
+    public void SaveSeriesInfoJson(string downloadDirectory)
+    {
+        string publicationFolder = CreatePublicationFolder(downloadDirectory);
         string seriesInfoPath = Path.Join(publicationFolder, "series.json");
         if(!File.Exists(seriesInfoPath))
             File.WriteAllText(seriesInfoPath,this.GetSeriesInfoJson());
+        if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+            File.SetUnixFileMode(seriesInfoPath, GroupRead | GroupWrite | OtherRead | OtherWrite | UserRead | UserWrite);
     }
     
     /// <returns>Serialized JSON String for series.json</returns>

Tranga/TrangaTask.cs

@@ -19,6 +19,9 @@ public abstract class TrangaTask
     public Publication? publication { get; }
     public string? language { get; }
     [JsonIgnore]public ExecutionState state { get; set; }
+    [JsonIgnore] public float progress => (tasksFinished != 0f ? tasksFinished / tasksCount : 0f);
+    [JsonIgnore]public float tasksCount { get; set; }
+    [JsonIgnore]public float tasksFinished { get; set; }
 
     public enum ExecutionState
     {
@@ -35,6 +38,8 @@ public abstract class TrangaTask
         this.connectorName = connectorName;
         this.task = task;
         this.language = language;
+        this.tasksCount = 1;
+        this.tasksFinished = 0;
     }
     
     /// <summary>
@@ -74,7 +79,7 @@ public abstract class TrangaTask
 
     public override string ToString()
     {
-        return $"{task}, {lastExecuted}, {reoccurrence}, {state} {(connectorName is not null ? $", {connectorName}" : "" )} {(publication is not null ? $", {publication?.sortName}": "")}";
+        return $"{task}, {lastExecuted}, {reoccurrence}, {state} {(connectorName is not null ? $", {connectorName}" : "" )} {(publication is not null ? $", {progress:00.00}%" : "")} {(publication is not null ? $", {publication?.sortName}" : "")}";
     }
     
     public class TrangaTaskJsonConverter : JsonConverter

Tranga/TrangaTasks/DownloadNewChaptersTask.cs

@@ -14,17 +14,16 @@ public class DownloadNewChaptersTask : TrangaTask
         Connector connector = taskManager.GetConnector(this.connectorName);
 
         //Check if Publication already has a Folder
-        string publicationFolder = Path.Join(connector.downloadLocation, pub.folderName);
-        if(!Directory.Exists(publicationFolder))
-            Directory.CreateDirectory(publicationFolder);
+        pub.CreatePublicationFolder(taskManager.settings.downloadLocation);
         List<Chapter> newChapters = UpdateChapters(connector, pub, language!, ref taskManager.chapterCollection);
+        this.tasksCount = newChapters.Count;
 
         connector.CopyCoverFromCacheToDownloadLocation(pub, taskManager.settings);
         
         pub.SaveSeriesInfoJson(connector.downloadLocation);
 
         foreach(Chapter newChapter in newChapters)
-            connector.DownloadChapter(pub, newChapter);
+            connector.DownloadChapter(pub, newChapter, this);
     }
     
     /// <summary>