Fixed some nullable types

Added Logger to DownloadClient

Logging/LoggerBase.cs

@@ -52,7 +52,7 @@ public abstract class LoggerBase : TextWriter
         public override string ToString()
         {
             string dateTimeString = $"{logTime.ToShortDateString()} {logTime.ToLongTimeString()}";
-            return $"[{dateTimeString}] {caller,30} | {value}";
+            return $"[{dateTimeString}] {caller.Split(new char[]{'.','+'}).Last(),15} | {value}";
         }
     }
 }

Tranga-CLI/Tranga_Cli.cs

@@ -209,8 +209,12 @@ public static class Tranga_Cli
             $"{"",-5}{"Task",-20} | {"Last Executed",-20} | {"Reoccurrence",-12} | {"State",-10} | {"Connector",-15} | Publication/Manga";
         Console.WriteLine(header);
         Console.WriteLine(new string('-', header.Length));
-        foreach(TrangaTask trangaTask in tasks)
+        foreach (TrangaTask trangaTask in tasks)
-            Console.WriteLine($"{tIndex++:000}: {trangaTask}");
+        {
+            string[] taskSplit = trangaTask.ToString().Split(", ");
+            Console.WriteLine($"{tIndex++:000}: {taskSplit[0],-20} | {taskSplit[1],-20} | {taskSplit[2],-12} | {taskSplit[3],-10} | {(taskSplit.Length > 4 ? taskSplit[4] : ""),-15} | {(taskSplit.Length > 5 ? taskSplit[5] : "")}");
+        }
+            
     }
 
     private static TrangaTask? SelectTask(TrangaTask[] tasks, Logger logger)

Tranga/Connector.cs

@@ -12,15 +12,18 @@ namespace Tranga;
 public abstract class Connector
 {
     internal string downloadLocation { get; }  //Location of local files
-    protected DownloadClient downloadClient { get; }
+    protected DownloadClient downloadClient { get; init; }
 
     protected Logger? logger;
 
-    protected Connector(string downloadLocation, uint downloadDelay, Logger? logger)
+    protected Connector(string downloadLocation, Logger? logger)
     {
         this.downloadLocation = downloadLocation;
-        this.downloadClient = new DownloadClient(downloadDelay);
         this.logger = logger;
+        this.downloadClient = new DownloadClient(new Dictionary<byte, int>()
+        {
+            //RequestTypes for RateLimits
+        }, logger);
     }
     
     public abstract string name { get; } //Name of the Connector (e.g. Website)
@@ -85,6 +88,7 @@ public abstract class Connector
             new XElement("Tags", string.Join(',',publication.tags)),
             new XElement("LanguageISO", publication.originalLanguage),
             new XElement("Title", chapter.name),
+            new XElement("Writer", publication.author),
             new XElement("Volume", chapter.volumeNumber),
             new XElement("Number", chapter.chapterNumber) //TODO check if this is correct at some point
         );
@@ -108,21 +112,22 @@ public abstract class Connector
     {
         return Path.Join(downloadLocation, publication.folderName, $"{chapter.fileName}.cbz");
     }
-    
+
     /// <summary>
     /// Downloads Image from URL and saves it to the given path(incl. fileName)
     /// </summary>
     /// <param name="imageUrl"></param>
     /// <param name="fullPath"></param>
     /// <param name="downloadClient">DownloadClient of the connector</param>
-    protected static void DownloadImage(string imageUrl, string fullPath, DownloadClient downloadClient)
+    /// <param name="requestType">Requesttype for ratelimit</param>
+    protected static void DownloadImage(string imageUrl, string fullPath, DownloadClient downloadClient, byte requestType)
     {
-        DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl);
+        DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType);
         byte[] buffer = new byte[requestResult.result.Length];
         requestResult.result.ReadExactly(buffer, 0, buffer.Length);
         File.WriteAllBytes(fullPath, buffer);
     }
-    
+
     /// <summary>
     /// Downloads all Images from URLs, Compresses to zip(cbz) and saves.
     /// </summary>
@@ -130,7 +135,8 @@ public abstract class Connector
     /// <param name="saveArchiveFilePath">Full path to save archive to (without file ending .cbz)</param>
     /// <param name="downloadClient">DownloadClient of the connector</param>
     /// <param name="comicInfoPath">Path of the generate Chapter ComicInfo.xml, if it was generated</param>
-    protected static void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, DownloadClient downloadClient, Logger? logger, string? comicInfoPath = null)
+    /// <param name="requestType">RequestType for RateLimits</param>
+    protected static void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, DownloadClient downloadClient, byte requestType, Logger? logger, string? comicInfoPath = null)
     {
         logger?.WriteLine("Connector", "Downloading Images");
         //Check if Publication Directory already exists
@@ -150,7 +156,7 @@ public abstract class Connector
         {
             string[] split = imageUrl.Split('.');
             string extension = split[^1];
-            DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), downloadClient);
+            DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), downloadClient, requestType);
         }
         
         if(comicInfoPath is not null)
@@ -164,34 +170,66 @@ public abstract class Connector
     
     protected class DownloadClient
     {
-        private readonly TimeSpan _requestSpeed;
-        private DateTime _lastRequest;
         private static readonly HttpClient Client = new();
 
+        private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit;
+        private readonly Dictionary<byte, TimeSpan> _rateLimit;
+        private Logger? logger;
+
         /// <summary>
         /// Creates a httpClient
         /// </summary>
         /// <param name="delay">minimum delay between requests (to avoid spam)</param>
-        public DownloadClient(uint delay)
+        /// <param name="rateLimitRequestsPerMinute">Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType</param>
+        public DownloadClient(Dictionary<byte, int> rateLimitRequestsPerMinute, Logger? logger)
         {
-            _requestSpeed = TimeSpan.FromMilliseconds(delay);
+            this.logger = logger;
-            _lastRequest = DateTime.Now.Subtract(_requestSpeed);
+            _lastExecutedRateLimit = new();
+            _rateLimit = new();
+            foreach(KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute)
+                _rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value));
         }
-        
+
         /// <summary>
         /// Request Webpage
         /// </summary>
         /// <param name="url"></param>
+        /// <param name="requestType">For RateLimits: Same Endpoints use same type</param>
         /// <returns>RequestResult with StatusCode and Stream of received data</returns>
-        public RequestResult MakeRequest(string url)
+        public RequestResult MakeRequest(string url, byte requestType)
         {
-            while((DateTime.Now - _lastRequest) < _requestSpeed)
+            if (_rateLimit.TryGetValue(requestType, out TimeSpan value))
-                Thread.Sleep(10);
+                _lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
-            _lastRequest = DateTime.Now;
+            else
+            {
+                logger?.WriteLine(this.GetType().ToString(), "RequestType not configured for rate-limit.");
+                return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null);
+            }
 
-            HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
+            TimeSpan rateLimitTimeout = _rateLimit[requestType]
-            HttpResponseMessage response = Client.Send(requestMessage);
+                .Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]));
+            
+            if(rateLimitTimeout > TimeSpan.Zero)
+                Thread.Sleep(rateLimitTimeout);
+
+            HttpResponseMessage? response = null;
+            while (response is null)
+            {
+                try
+                {
+                    HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
+                    _lastExecutedRateLimit[requestType] = DateTime.Now;
+                    response = Client.Send(requestMessage);
+                }
+                catch (HttpRequestException e)
+                {
+                    logger?.WriteLine(this.GetType().ToString(), e.Message);
+                    Thread.Sleep(_rateLimit[requestType] * 2);
+                }
+            }
             Stream resultString = response.IsSuccessStatusCode ? response.Content.ReadAsStream() : Stream.Null;
+            if (!response.IsSuccessStatusCode)
+                logger?.WriteLine(this.GetType().ToString(), $"Request-Error {response.StatusCode}: {response.ReasonPhrase}");
             return new RequestResult(response.StatusCode, resultString);
         }
 

Tranga/Connectors/MangaDex.cs

@@ -9,14 +9,26 @@ public class MangaDex : Connector
 {
     public override string name { get; }
 
-    public MangaDex(string downloadLocation, uint downloadDelay, Logger? logger) : base(downloadLocation, downloadDelay, logger)
+    private enum RequestType : byte
     {
-        name = "MangaDex";
+        Manga,
+        Feed,
+        AtHomeServer,
+        Cover,
+        Author
     }
-    
+
-    public MangaDex(string downloadLocation, Logger? logger) : base(downloadLocation, 750, logger)
+    public MangaDex(string downloadLocation, Logger? logger) : base(downloadLocation, logger)
     {
         name = "MangaDex";
+        this.downloadClient = new DownloadClient(new Dictionary<byte, int>()
+        {
+            {(byte)RequestType.Manga, 250},
+            {(byte)RequestType.Feed, 250},
+            {(byte)RequestType.AtHomeServer, 40},
+            {(byte)RequestType.Cover, 250},
+            {(byte)RequestType.Author, 250}
+        }, logger);
     }
 
     public override Publication[] GetPublications(string publicationTitle = "")
@@ -31,7 +43,7 @@ public class MangaDex : Connector
             //Request next Page
             DownloadClient.RequestResult requestResult =
                 downloadClient.MakeRequest(
-                    $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}");
+                    $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", (byte)RequestType.Manga);
             if (requestResult.statusCode != HttpStatusCode.OK)
                 break;
             JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@@ -49,6 +61,8 @@ public class MangaDex : Connector
                 JsonObject manga = (JsonObject)mangeNode!;
                 JsonObject attributes = manga["attributes"]!.AsObject();
                 
+                string publicationId = manga["id"]!.GetValue<string>();
+                
                 string title = attributes["title"]!.AsObject().ContainsKey("en") && attributes["title"]!["en"] is not null
                     ? attributes["title"]!["en"]!.GetValue<string>()
                     : attributes["title"]![((IDictionary<string, JsonNode?>)attributes["title"]!.AsObject()).Keys.First()]!.GetValue<string>();
@@ -76,11 +90,15 @@ public class MangaDex : Connector
                 }
 
                 string? posterId = null;
+                string? authorId = null;
                 if (manga.ContainsKey("relationships") && manga["relationships"] is not null)
                 {
                     JsonArray relationships = manga["relationships"]!.AsArray();
                     posterId = relationships.FirstOrDefault(relationship => relationship!["type"]!.GetValue<string>() == "cover_art")!["id"]!.GetValue<string>();
+                    authorId = relationships.FirstOrDefault(relationship => relationship!["type"]!.GetValue<string>() == "author")!["id"]!.GetValue<string>();
                 }
+                string? coverUrl = GetCoverUrl(publicationId, posterId);
+                string? author = GetAuthor(authorId);
 
                 Dictionary<string, string> linksDict = new();
                 if (attributes.ContainsKey("links") && attributes["links"] is not null)
@@ -102,12 +120,9 @@ public class MangaDex : Connector
                 
                 string status = attributes["status"]!.GetValue<string>();
 
-                string publicationId = manga["id"]!.GetValue<string>();
-
-                string? coverUrl = GetCoverUrl(publicationId, posterId);
-
                 Publication pub = new (
                     title,
+                    author,
                     description,
                     altTitlesDict,
                     tags.ToArray(),
@@ -138,7 +153,7 @@ public class MangaDex : Connector
             //Request next "Page"
             DownloadClient.RequestResult requestResult =
                 downloadClient.MakeRequest(
-                    $"https://api.mangadex.org/manga/{publication.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}");
+                    $"https://api.mangadex.org/manga/{publication.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", (byte)RequestType.Feed);
             if (requestResult.statusCode != HttpStatusCode.OK)
                 break;
             JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@@ -185,7 +200,7 @@ public class MangaDex : Connector
         logger?.WriteLine(this.GetType().ToString(), $"Download Chapter {publication.sortName} {chapter.volumeNumber}-{chapter.chapterNumber}");
         //Request URLs for Chapter-Images
         DownloadClient.RequestResult requestResult =
-            downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'");
+            downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer);
         if (requestResult.statusCode != HttpStatusCode.OK)
             return;
         JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@@ -204,7 +219,7 @@ public class MangaDex : Connector
         File.WriteAllText(comicInfoPath, CreateComicInfo(publication, chapter, logger));
         
         //Download Chapter-Images
-        DownloadChapterImages(imageUrls.ToArray(), CreateFullFilepath(publication, chapter), downloadClient, logger, comicInfoPath);
+        DownloadChapterImages(imageUrls.ToArray(), CreateFullFilepath(publication, chapter), downloadClient, (byte)RequestType.AtHomeServer, logger, comicInfoPath);
     }
 
     private string? GetCoverUrl(string publicationId, string? posterId)
@@ -217,7 +232,7 @@ public class MangaDex : Connector
         
         //Request information where to download Cover
         DownloadClient.RequestResult requestResult =
-            downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}");
+            downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", (byte)RequestType.Cover);
         if (requestResult.statusCode != HttpStatusCode.OK)
             return null;
         JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@@ -230,6 +245,23 @@ public class MangaDex : Connector
         return coverUrl;
     }
 
+    private string? GetAuthor(string? authorId)
+    {
+        if (authorId is null)
+            return null;
+        
+        DownloadClient.RequestResult requestResult =
+            downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", (byte)RequestType.Author);
+        if (requestResult.statusCode != HttpStatusCode.OK)
+            return null;
+        JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
+        if (result is null)
+            return null;
+
+        string author = result["data"]!["attributes"]!["name"]!.GetValue<string>();
+        return author;
+    }
+
     public override void DownloadCover(Publication publication)
     {
         logger?.WriteLine(this.GetType().ToString(), $"Download cover {publication.sortName}");
@@ -238,29 +270,26 @@ public class MangaDex : Connector
         if(!Directory.Exists(publicationFolder))
             Directory.CreateDirectory(publicationFolder);
         DirectoryInfo dirInfo = new (publicationFolder);
-        foreach(FileInfo fileInfo in dirInfo.EnumerateFiles())
+        if (dirInfo.EnumerateFiles().Any(info => info.Name.Contains("cover.")))
-            if (fileInfo.Name.Contains("cover."))
-            {
-                logger?.WriteLine(this.GetType().ToString(), $"Cover exists {publication.sortName}");
-                return;
-            }
-
-        if (publication.posterUrl is null)
         {
-            logger?.WriteLine(this.GetType().ToString(), $"No posterurl in publication");
+            logger?.WriteLine(this.GetType().ToString(), $"Cover exists {publication.sortName}");
             return;
         }
 
-        string coverUrl = publication.posterUrl;
+        if (publication.posterUrl is null || publication.posterUrl!.Contains("http"))
+        {
+            logger?.WriteLine(this.GetType().ToString(), $"No Poster-URL in publication");
+            return;
+        }
 
         //Get file-extension (jpg, png)
-        string[] split = coverUrl.Split('.');
+        string[] split = publication.posterUrl.Split('.');
         string extension = split[^1];
 
         string outFolderPath = Path.Join(downloadLocation, publication.folderName);
         Directory.CreateDirectory(outFolderPath);
         
         //Download cover-Image
-        DownloadImage(coverUrl, Path.Join(downloadLocation, publication.folderName, $"cover.{extension}"), this.downloadClient);
+        DownloadImage(publication.posterUrl, Path.Join(downloadLocation, publication.folderName, $"cover.{extension}"), this.downloadClient, (byte)RequestType.AtHomeServer);
     }
 }

Tranga/Publication.cs

@@ -9,6 +9,7 @@ namespace Tranga;
 public readonly struct Publication
 {
     public string sortName { get; }
+    public string? author { get; }
     public Dictionary<string,string> altTitles { get; }
     // ReSharper disable trice MemberCanBePrivate.Global, trust
     public string? description { get; }
@@ -22,9 +23,10 @@ public readonly struct Publication
     public string publicationId { get; }
     public string internalId { get; }
 
-    public Publication(string sortName, string? description, Dictionary<string,string> altTitles, string[] tags, string? posterUrl, Dictionary<string,string>? links, int? year, string? originalLanguage, string status, string publicationId)
+    public Publication(string sortName, string? author, string? description, Dictionary<string,string> altTitles, string[] tags, string? posterUrl, Dictionary<string,string>? links, int? year, string? originalLanguage, string status, string publicationId)
     {
         this.sortName = sortName;
+        this.author = author;
         this.description = description;
         this.altTitles = altTitles;
         this.tags = tags;
@@ -35,8 +37,8 @@ public readonly struct Publication
         this.status = status;
         this.publicationId = publicationId;
         this.folderName = string.Concat(sortName.Split(Path.GetInvalidPathChars().Concat(Path.GetInvalidFileNameChars()).ToArray()));
-        string onlyLowerAscii = this.sortName.ToLower().Where(Char.IsAscii).ToString()!;
+        string onlyLowerLetters = string.Concat(this.sortName.ToLower().Where(Char.IsLetter));
-        this.internalId = Convert.ToBase64String(Encoding.ASCII.GetBytes($"{onlyLowerAscii}{this.year}"));
+        this.internalId = Convert.ToBase64String(Encoding.ASCII.GetBytes($"{onlyLowerLetters}{this.year}"));
     }
     
     /// <returns>Serialized JSON String for series.json</returns>

Tranga/TaskExecutor.cs

@@ -92,13 +92,13 @@ public static class TaskExecutor
     /// <param name="chapterCollection"></param>
     private static void DownloadNewChapters(Connector connector, Publication publication, string language, ref Dictionary<Publication, List<Chapter>> chapterCollection)
     {
-        List<Chapter> newChapters = UpdateChapters(connector, publication, language, ref chapterCollection);
+        //Check if Publication already has a Folder
-        connector.DownloadCover(publication);
-        
-        //Check if Publication already has a Folder and a series.json
         string publicationFolder = Path.Join(connector.downloadLocation, publication.folderName);
         if(!Directory.Exists(publicationFolder))
             Directory.CreateDirectory(publicationFolder);
+        List<Chapter> newChapters = UpdateChapters(connector, publication, language, ref chapterCollection);
+
+        connector.DownloadCover(publication);
         
         string seriesInfoPath = Path.Join(publicationFolder, "series.json");
         if(!File.Exists(seriesInfoPath))

Tranga/TaskManager.cs

@@ -25,7 +25,7 @@ public class TaskManager
     /// <param name="komgaUsername">The Komga username</param>
     /// <param name="komgaPassword">The Komga password</param>
     /// <param name="logger"></param>
-    public TaskManager(string downloadFolderPath, string? workingDirectory = null, string? komgaBaseUrl = null, string? komgaUsername = null, string? komgaPassword = null, Logger? logger = null)
+    public TaskManager(string downloadFolderPath, string workingDirectory, string? komgaBaseUrl = null, string? komgaUsername = null, string? komgaPassword = null, Logger? logger = null)
     {
         this.logger = logger;
         _allTasks = new HashSet<TrangaTask>();
@@ -169,6 +169,8 @@ public class TaskManager
                     _chapterCollection.TryAdd((Publication)publication!, new List<Chapter>());
                 _allTasks.Add(newTask);
             }
+            else
+                logger?.WriteLine(this.GetType().ToString(), $"Publication already exists {publication?.internalId}");
         }
         logger?.WriteLine(this.GetType().ToString(), $"Added new Task {newTask.ToString()}");
         ExportData();

Tranga/TrangaTask.cs

@@ -56,6 +56,6 @@ public class TrangaTask
 
     public override string ToString()
     {
-        return $"{task,-20} | {lastExecuted,-20} | {reoccurrence,-12} | {state,-10} | {connectorName,-15} | {publication?.sortName}";
+        return $"{task}, {lastExecuted}, {reoccurrence}, {state} {(connectorName is not null ? $", {connectorName}" : "" )} {(publication is not null ? $", {publication?.sortName}": "")}";
     }
 }