Merge remote-tracking branch 'upstream/cuttingedge' into cuttingedge

.github/workflows/docker-base.yml

@@ -20,7 +20,7 @@ jobs:
       # https://github.com/marketplace/actions/docker-setup-buildx
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3.1.0
+        uses: docker/setup-buildx-action@v3.3.0
       
       # https://github.com/docker/login-action#docker-hub
       - name: Login to Docker Hub
@@ -31,7 +31,7 @@ jobs:
       
       # https://github.com/docker/build-push-action#multi-platform-image
       - name: Build and push base
-        uses: docker/build-push-action@v4.1.1
+        uses: docker/build-push-action@v5.3.0
         with:
           context: ./
           file: ./Dockerfile-base

.github/workflows/docker-image-cuttingedge.yml

@@ -22,7 +22,7 @@ jobs:
       # https://github.com/marketplace/actions/docker-setup-buildx
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3.1.0
+        uses: docker/setup-buildx-action@v3.3.0
       
       # https://github.com/docker/login-action#docker-hub
       - name: Login to Docker Hub
@@ -33,7 +33,7 @@ jobs:
       
       # https://github.com/docker/build-push-action#multi-platform-image
       - name: Build and push API
-        uses: docker/build-push-action@v4.1.1
+        uses: docker/build-push-action@v5.3.0
         with:
           context: ./
           file: ./Dockerfile

.github/workflows/docker-image-dev.yml

@@ -22,7 +22,7 @@ jobs:
       # https://github.com/marketplace/actions/docker-setup-buildx
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3.1.0
+        uses: docker/setup-buildx-action@v3.3.0
       
       # https://github.com/docker/login-action#docker-hub
       - name: Login to Docker Hub
@@ -33,7 +33,7 @@ jobs:
       
       # https://github.com/docker/build-push-action#multi-platform-image
       - name: Build and push API
-        uses: docker/build-push-action@v4.1.1
+        uses: docker/build-push-action@v5.3.0
         with:
           context: ./
           file: ./Dockerfile

.github/workflows/docker-image-master.yml

@@ -22,7 +22,7 @@ jobs:
       # https://github.com/marketplace/actions/docker-setup-buildx
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3.1.0
+        uses: docker/setup-buildx-action@v3.3.0
       
       # https://github.com/docker/login-action#docker-hub
       - name: Login to Docker Hub
@@ -33,7 +33,7 @@ jobs:
       
       # https://github.com/docker/build-push-action#multi-platform-image
       - name: Build and push API
-        uses: docker/build-push-action@v4.1.1
+        uses: docker/build-push-action@v5.3.0
         with:
           context: ./
           file: ./Dockerfile

README.md

@@ -1,12 +1,3 @@
-<!-- PROJECT SHIELDS -->
-<!--
-*** I'm using markdown "reference style" links for readability.
-*** Reference links are enclosed in brackets [ ] instead of parentheses ( ).
-*** See the bottom of this document for the declaration of the reference variables
-*** for contributors-url, forks-url, etc. This is an optional, concise syntax you may use.
-*** https://www.markdownguide.org/basic-syntax/#reference-style-links
--->
-
 <!-- PROJECT LOGO -->
 <br />
 <div align="center">
@@ -61,8 +52,8 @@ Tranga can download Chapters and Metadata from "Scanlation" sites such as
 - [Manga4Life](https://manga4life.com) (en)
 - ❓ Open an [issue](https://github.com/C9Glax/tranga/issues)
 
-and trigger an scan with [Komga](https://komga.org/) and [Kavita](https://www.kavitareader.com/).  
-Notifications will can sent to your devices using [Gotify](https://gotify.net/) and [LunaSea](https://www.lunasea.app/).
+and trigger a library-scan with [Komga](https://komga.org/) and [Kavita](https://www.kavitareader.com/).  
+Notifications can be sent to your devices using [Gotify](https://gotify.net/) and [LunaSea](https://www.lunasea.app/).
 
 ### What this does and doesn't do
 
@@ -75,15 +66,15 @@ This project downloads the images for a Manga from the specified Scanlation-Webs
 It does this on an interval, and checks for any Chapters (.cbz-Archive) not already existing in your specified Download-Location. (If you rename or move files, it will download those again)  
 Tranga can (if configured) trigger a scan in Komga or Kavita, however the directory in which the Manga reside has to be available to both Tranga and Komga/Kavita.
 
-The project doesn't manage metadata, doesn't curate, change or enhance any information that isn't available on the selected Scanlation-Site.  
+The project doesn't manage metadata, and doesn't curate, change or enhance any information that isn't available on the selected Scanlation-Site.  
 It will blindly use whatever is scrapes (yes this is a glorified Web-scraper).
 
 
 ### Inspiration:
 
 Because [Kaizoku](https://github.com/oae/kaizoku) was relying on [mangal](https://github.com/metafates/mangal) and mangal
-hasn't received bugfixes for it's issues with Titles not showing up, or throwing errors because of illegal characters,
-there were no alternatives for automatic downloads. However [Kaizoku](https://github.com/oae/kaizoku) certainly had a great Web-UI.
+hasn't received bugfixes for its issues with Titles not showing up, or throwing errors because of illegal characters,
+there were no alternatives for automatic downloads. However, [Kaizoku](https://github.com/oae/kaizoku) certainly had a great Web-UI.
 
 That is why I wanted to create my own project, in a language I understand, and that I am able to maintain myself.
 
@@ -102,25 +93,15 @@ That is why I wanted to create my own project, in a language I understand, and t
 <!-- GETTING STARTED -->
 ## Getting Started
 
-There is two release types:
-
-- CLI
-- Docker
-
-### CLI
-
-Head over to [releases](https://git.bernloehr.eu/glax/Tranga/releases) and download.
-
-
-~~The CLI will guide you through setup.~~ Not in the current version.  
-Right now it is barebones with options to view logs and make HTTP-Requests
-
 ### Docker
 
 Download [docker-compose.yaml](https://git.bernloehr.eu/glax/Tranga/src/branch/master/docker-compose.yaml) and configure to your needs.  
-Mount `/Manga` to wherever you want your chapters (`.cbz`-Archives) downloaded (for exampled where Komga/Kavita can access them).  
+Mount `/Manga` to wherever you want your chapters (`.cbz`-Archives) downloaded (where Komga/Kavita can access them).  
 The `docker-compose` also includes [tranga-website](https://github.com/C9Glax/tranga-website) as frontend. For its configuration refer to the repo README.
 
+For compatibility do not execute the compose as root (which you should not do anyways...) but as user that can
+access the folder.
+
 ### Prerequisites
 
 #### To Build
@@ -131,7 +112,6 @@ The `docker-compose` also includes [tranga-website](https://github.com/C9Glax/tr
 <!-- ROADMAP -->
 ## Roadmap
 
-- [ ] Docker ARM support
 - [ ] ❓
 
 See the [open issues](https://github.com/C9Glax/tranga/issues) for a full list of proposed features (and known issues).

Tranga/MangaConnectors/ChromiumDownloadClient.cs

@@ -10,6 +10,7 @@ internal class ChromiumDownloadClient : DownloadClient
 {
     private IBrowser browser { get; set; }
     private const string ChromiumVersion = "1154303";
+    private const int StartTimeoutMs = 30000;
     
     private async Task<IBrowser> DownloadBrowser()
     {
@@ -40,7 +41,7 @@ internal class ChromiumDownloadClient : DownloadClient
             await browserFetcher.DownloadAsync(ChromiumVersion);
         }
         
-        Log("Starting Browser.");
+        Log($"Starting Browser. ({StartTimeoutMs}ms timeout)");
         return await Puppeteer.LaunchAsync(new LaunchOptions
         {
             Headless = true,
@@ -50,7 +51,7 @@ internal class ChromiumDownloadClient : DownloadClient
                 "--disable-dev-shm-usage",
                 "--disable-setuid-sandbox",
                 "--no-sandbox"},
-            Timeout = 10000
+            Timeout = StartTimeoutMs
         });
     }
 

Tranga/MangaConnectors/MangaDex.cs

@@ -20,12 +20,18 @@ public class MangaDex : MangaConnector
         int total = int.MaxValue; //How many total results are there, is updated on first request
         HashSet<Manga> retManga = new();
         int loadedPublicationData = 0;
+        List<JsonNode> results = new();
+        
+        //Request all search-results
         while (offset < total) //As long as we haven't requested all "Pages"
         {
             //Request next Page
-            RequestResult requestResult =
-                downloadClient.MakeRequest(
-                    $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", RequestType.MangaInfo);
+            RequestResult requestResult = downloadClient.MakeRequest(
+                    $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}" +
+                    $"&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica" +
+                    $"&contentRating%5B%5D=pornographic" +
+                    $"&includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author" +
+                    $"&includes%5B%5D=artist&includes%5B%5D=tag", RequestType.MangaInfo);
             if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
                 break;
             JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@@ -39,18 +45,14 @@ public class MangaDex : MangaConnector
             else continue;
 
             if (result.ContainsKey("data"))
-            {
-                JsonArray mangaInResult = result["data"]!.AsArray(); //Manga-data-Array
-                //Loop each Manga and extract information from JSON
-                foreach (JsonNode? mangaNode in mangaInResult)
-                {
-                    if(mangaNode is null)
-                        continue;
-                    Log($"Getting publication data. {++loadedPublicationData}/{total}");
-                    if(MangaFromJsonObject((JsonObject) mangaNode) is { } manga)
-                        retManga.Add(manga); //Add Publication (Manga) to result
+                results.AddRange(result["data"]!.AsArray()!);//Manga-data-Array
         }
-            }//else continue;
+        
+        foreach (JsonNode mangaNode in results)
+        {
+            Log($"Getting publication data. {++loadedPublicationData}/{total}");
+            if(MangaFromJsonObject(mangaNode.AsObject()) is { } manga)
+                retManga.Add(manga); //Add Publication (Manga) to result
         }
         Log($"Retrieved {retManga.Count} publications. Term=\"{publicationTitle}\"");
         return retManga.ToArray();
@@ -78,94 +80,96 @@ public class MangaDex : MangaConnector
 
     private Manga? MangaFromJsonObject(JsonObject manga)
     {
-        if (!manga.ContainsKey("attributes"))
+        if (!manga.TryGetPropertyValue("id", out JsonNode? idNode))
             return null;
-        JsonObject attributes = manga["attributes"]!.AsObject();
+        string publicationId = idNode!.GetValue<string>();
             
-        if(!manga.ContainsKey("id"))
+        if (!manga.TryGetPropertyValue("attributes", out JsonNode? attributesNode))
             return null;
-        string publicationId = manga["id"]!.GetValue<string>();
+        JsonObject attributes = attributesNode!.AsObject();
 
-        if(!attributes.ContainsKey("title"))
+        if (!attributes.TryGetPropertyValue("title", out JsonNode? titleNode))
             return null;
-        string title = attributes["title"]!.AsObject().ContainsKey("en") && attributes["title"]!["en"] is not null
-                    ? attributes["title"]!["en"]!.GetValue<string>()
-                    : attributes["title"]![((IDictionary<string, JsonNode?>)attributes["title"]!.AsObject()).Keys.First()]!.GetValue<string>();
+        string title = titleNode!.AsObject().ContainsKey("en") switch
+        {
+            true => titleNode.AsObject()["en"]!.GetValue<string>(),
+            false => titleNode.AsObject().First().Value!.GetValue<string>()
+        };
         
-        if(!attributes.ContainsKey("description"))
-            return null;
-        string? description = attributes["description"]!.AsObject().ContainsKey("en") && attributes["description"]!["en"] is not null
-                    ? attributes["description"]!["en"]!.GetValue<string?>()
-                    : null;
-
-        if(!attributes.ContainsKey("altTitles"))
-            return null;
-        JsonArray altTitlesObject = attributes["altTitles"]!.AsArray();
         Dictionary<string, string> altTitlesDict = new();
-        foreach (JsonNode? altTitleNode in altTitlesObject)
+        if (attributes.TryGetPropertyValue("altTitles", out JsonNode? altTitlesNode))
         {
-            JsonObject altTitleObject = (JsonObject)altTitleNode!;
-            string key = ((IDictionary<string, JsonNode?>)altTitleObject).Keys.ToArray()[0];
-            altTitlesDict.TryAdd(key, altTitleObject[key]!.GetValue<string>());
+            foreach (JsonNode? altTitleNode in altTitlesNode!.AsArray())
+            {
+                JsonObject altTitleNodeObject = altTitleNode!.AsObject();
+                altTitlesDict.TryAdd(altTitleNodeObject.First().Key, altTitleNodeObject.First().Value!.GetValue<string>());
+            }
         }
 
-        if(!attributes.ContainsKey("tags"))
+        if (!attributes.TryGetPropertyValue("description", out JsonNode? descriptionNode))
             return null;
-        JsonArray tagsObject = attributes["tags"]!.AsArray();
-        HashSet<string> tags = new();
-        foreach (JsonNode? tagNode in tagsObject)
+        string description = descriptionNode!.AsObject().ContainsKey("en") switch
         {
-            JsonObject tagObject = (JsonObject)tagNode!;
-            if(tagObject["attributes"]!["name"]!.AsObject().ContainsKey("en"))
-                tags.Add(tagObject["attributes"]!["name"]!["en"]!.GetValue<string>());
-        }
-
-        string? posterId = null;
-        HashSet<string> authorIds = new();
-        if (manga.ContainsKey("relationships") && manga["relationships"] is not null)
-        {
-            JsonArray relationships = manga["relationships"]!.AsArray();
-            posterId = relationships.FirstOrDefault(relationship => relationship!["type"]!.GetValue<string>() == "cover_art")!["id"]!.GetValue<string>();
-            foreach (JsonNode? node in relationships.Where(relationship =>
-                relationship!["type"]!.GetValue<string>() == "author"))
-                authorIds.Add(node!["id"]!.GetValue<string>());
-        }
-        string? coverUrl = GetCoverUrl(publicationId, posterId);
-        string? coverCacheName = null;
-        if (coverUrl is not null)
-            coverCacheName = SaveCoverImageToCache(coverUrl, RequestType.MangaCover);
-
-        List<string> authors = GetAuthors(authorIds);
+            true => descriptionNode.AsObject()["en"]!.GetValue<string>(),
+            false => descriptionNode.AsObject().First().Value!.GetValue<string>()
+        };
 
         Dictionary<string, string> linksDict = new();
-        if (attributes.ContainsKey("links") && attributes["links"] is not null)
-        {
-            JsonObject linksObject = attributes["links"]!.AsObject();
-            foreach (string key in ((IDictionary<string, JsonNode?>)linksObject).Keys)
-            {
-                linksDict.Add(key, linksObject[key]!.GetValue<string>());
-            }
-        }
-
-        int? year = attributes.ContainsKey("year") && attributes["year"] is not null
-            ? attributes["year"]!.GetValue<int?>()
-            : null;
+        if (attributes.TryGetPropertyValue("links", out JsonNode? linksNode))
+            foreach (KeyValuePair<string, JsonNode> linkKv in linksNode!.AsObject())
+                linksDict.TryAdd(linkKv.Key, linkKv.Value.GetValue<string>());
 
         string? originalLanguage =
-            attributes.ContainsKey("originalLanguage") && attributes["originalLanguage"] is not null
-                ? attributes["originalLanguage"]!.GetValue<string?>()
-                : null;
-
-        if(!attributes.ContainsKey("status"))
-            return null;
-        string status = attributes["status"]!.GetValue<string>();
-        Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased;
-        switch (status.ToLower())
+            attributes.TryGetPropertyValue("originalLanguage", out JsonNode? originalLanguageNode) switch
             {
-            case "ongoing": releaseStatus = Manga.ReleaseStatusByte.Continuing; break;
-            case "completed": releaseStatus = Manga.ReleaseStatusByte.Completed; break;
-            case "hiatus": releaseStatus = Manga.ReleaseStatusByte.OnHiatus; break;
-            case "cancelled": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break;
+                true => originalLanguageNode!.GetValue<string>(),
+                false => null
+            };
+        
+        Manga.ReleaseStatusByte status = Manga.ReleaseStatusByte.Unreleased;
+        if (attributes.TryGetPropertyValue("status", out JsonNode? statusNode))
+        {
+            status = statusNode!.GetValue<string>().ToLower() switch
+            {
+                "ongoing" => Manga.ReleaseStatusByte.Continuing,
+                "completed" => Manga.ReleaseStatusByte.Completed,
+                "hiatus" => Manga.ReleaseStatusByte.OnHiatus,
+                "cancelled" => Manga.ReleaseStatusByte.Cancelled,
+                _ => Manga.ReleaseStatusByte.Unreleased
+            };
+        }
+
+        int? year = attributes.TryGetPropertyValue("year", out JsonNode? yearNode) switch
+        {
+            true => yearNode!.GetValue<int>(),
+            false => null
+        };
+        
+        HashSet<string> tags = new(128);
+        if (attributes.TryGetPropertyValue("tags", out JsonNode? tagsNode))
+            foreach (JsonNode? tagNode in tagsNode!.AsArray())
+                tags.Add(tagNode!["attributes"]!["name"]!["en"]!.GetValue<string>());
+
+        
+        if (!manga.TryGetPropertyValue("relationships", out JsonNode? relationshipsNode))
+            return null;
+        
+        JsonNode? coverNode = relationshipsNode!.AsArray()
+            .FirstOrDefault(rel => rel!["type"]!.GetValue<string>().Equals("cover_art"));
+        if (coverNode is null)
+            return null;
+        string fileName = coverNode["attributes"]!["fileName"]!.GetValue<string>();
+        string coverUrl = $"https://uploads.mangadex.org/covers/{publicationId}/{fileName}";
+        string coverCacheName = SaveCoverImageToCache(coverUrl, RequestType.MangaCover);
+        
+        List<string> authors = new();
+        JsonNode?[] authorNodes = relationshipsNode.AsArray()
+            .Where(rel => rel!["type"]!.GetValue<string>().Equals("author") || rel!["type"]!.GetValue<string>().Equals("artist")).ToArray();
+        foreach (JsonNode? authorNode in authorNodes)
+        {
+            string authorName = authorNode!["attributes"]!["name"]!.GetValue<string>();
+            if(!authors.Contains(authorName))
+               authors.Add(authorName);
         }
 
         Manga pub = new(
@@ -179,9 +183,9 @@ public class MangaDex : MangaConnector
             linksDict,
             year,
             originalLanguage,
-            status,
+            Enum.GetName(status) ?? "",
             publicationId,
-            releaseStatus
+            status
         );
         cachedPublications.Add(pub);
         return pub;
@@ -200,7 +204,7 @@ public class MangaDex : MangaConnector
             //Request next "Page"
             RequestResult requestResult =
                 downloadClient.MakeRequest(
-                    $"https://api.mangadex.org/manga/{manga.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", RequestType.MangaDexFeed);
+                    $"https://api.mangadex.org/manga/{manga.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&contentRating%5B%5D=pornographic", RequestType.MangaDexFeed);
             if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
                 break;
             JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@@ -288,50 +292,4 @@ public class MangaDex : MangaConnector
         //Download Chapter-Images
         return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, progressToken:progressToken);
     }
-
-    private string? GetCoverUrl(string publicationId, string? posterId)
-    {
-        Log($"Getting CoverUrl for Publication {publicationId}");
-        if (posterId is null)
-        {
-            Log("No cover.");
-            return null;
-        }
-        
-        //Request information where to download Cover
-        RequestResult requestResult =
-            downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", RequestType.MangaCover);
-        if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
-            return null;
-        JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
-        if (result is null)
-            return null;
-
-        string fileName = result["data"]!["attributes"]!["fileName"]!.GetValue<string>();
-
-        string coverUrl = $"https://uploads.mangadex.org/covers/{publicationId}/{fileName}";
-        Log($"Cover-Url {publicationId} -> {coverUrl}");
-        return coverUrl;
-    }
-
-    private List<string> GetAuthors(IEnumerable<string> authorIds)
-    {
-        Log("Retrieving authors.");
-        List<string> ret = new();
-        foreach (string authorId in authorIds)
-        {
-            RequestResult requestResult =
-                downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", RequestType.MangaDexAuthor);
-            if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
-                return ret;
-            JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
-            if (result is null)
-                return ret;
-
-            string authorName = result["data"]!["attributes"]!["name"]!.GetValue<string>();
-            ret.Add(authorName);
-            Log($"Got author {authorId} -> {authorName}");
-        }
-        return ret;
-    }
 }