From 21a7392493e1e13f488bfa6544ac9673d2e002dc Mon Sep 17 00:00:00 2001 From: Glax Date: Thu, 18 Apr 2024 18:01:02 +0200 Subject: [PATCH 1/5] Resolves #160, Rated Manga on Mangadex. --- Tranga/MangaConnectors/MangaDex.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tranga/MangaConnectors/MangaDex.cs b/Tranga/MangaConnectors/MangaDex.cs index 84b8891..3021367 100644 --- a/Tranga/MangaConnectors/MangaDex.cs +++ b/Tranga/MangaConnectors/MangaDex.cs @@ -25,7 +25,7 @@ public class MangaDex : MangaConnector //Request next Page RequestResult requestResult = downloadClient.MakeRequest( - $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", RequestType.MangaInfo); + $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&contentRating%5B%5D=pornographic", RequestType.MangaInfo); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) break; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); @@ -200,7 +200,7 @@ public class MangaDex : MangaConnector //Request next "Page" RequestResult requestResult = downloadClient.MakeRequest( - $"https://api.mangadex.org/manga/{manga.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", RequestType.MangaDexFeed); + $"https://api.mangadex.org/manga/{manga.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&contentRating%5B%5D=pornographic", RequestType.MangaDexFeed); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) break; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); From 8f8d0198616323d1fe1d6af461837270ea32f5a0 Mon Sep 17 00:00:00 2001 From: Glax Date: Thu, 18 Apr 2024 18:56:34 +0200 Subject: [PATCH 2/5] Streamlined MangaDex information retrieval --- Tranga/MangaConnectors/MangaDex.cs | 238 ++++++++++++----------------- 1 file changed, 98 insertions(+), 140 deletions(-) diff --git a/Tranga/MangaConnectors/MangaDex.cs b/Tranga/MangaConnectors/MangaDex.cs index 3021367..a50e445 100644 --- a/Tranga/MangaConnectors/MangaDex.cs +++ b/Tranga/MangaConnectors/MangaDex.cs @@ -20,12 +20,18 @@ public class MangaDex : MangaConnector int total = int.MaxValue; //How many total results are there, is updated on first request HashSet retManga = new(); int loadedPublicationData = 0; + List results = new(); + + //Request all search-results while (offset < total) //As long as we haven't requested all "Pages" { //Request next Page - RequestResult requestResult = - downloadClient.MakeRequest( - $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&contentRating%5B%5D=pornographic", RequestType.MangaInfo); + RequestResult requestResult = downloadClient.MakeRequest( + $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}" + + $"&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica" + + $"&contentRating%5B%5D=pornographic" + + $"&includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author" + + $"&includes%5B%5D=artist&includes%5B%5D=tag", RequestType.MangaInfo); if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) break; JsonObject? result = JsonSerializer.Deserialize(requestResult.result); @@ -39,18 +45,14 @@ public class MangaDex : MangaConnector else continue; if (result.ContainsKey("data")) - { - JsonArray mangaInResult = result["data"]!.AsArray(); //Manga-data-Array - //Loop each Manga and extract information from JSON - foreach (JsonNode? mangaNode in mangaInResult) - { - if(mangaNode is null) - continue; - Log($"Getting publication data. {++loadedPublicationData}/{total}"); - if(MangaFromJsonObject((JsonObject) mangaNode) is { } manga) - retManga.Add(manga); //Add Publication (Manga) to result - } - }//else continue; + results.AddRange(result["data"]!.AsArray()!);//Manga-data-Array + } + + foreach (JsonNode mangaNode in results) + { + Log($"Getting publication data. {++loadedPublicationData}/{total}"); + if(MangaFromJsonObject(mangaNode.AsObject()) is { } manga) + retManga.Add(manga); //Add Publication (Manga) to result } Log($"Retrieved {retManga.Count} publications. Term=\"{publicationTitle}\""); return retManga.ToArray(); @@ -78,94 +80,96 @@ public class MangaDex : MangaConnector private Manga? MangaFromJsonObject(JsonObject manga) { - if (!manga.ContainsKey("attributes")) + if (!manga.TryGetPropertyValue("id", out JsonNode? idNode)) return null; - JsonObject attributes = manga["attributes"]!.AsObject(); + string publicationId = idNode!.GetValue(); + + if (!manga.TryGetPropertyValue("attributes", out JsonNode? attributesNode)) + return null; + JsonObject attributes = attributesNode!.AsObject(); + + if (!attributes.TryGetPropertyValue("title", out JsonNode? titleNode)) + return null; + string title = titleNode!.AsObject().ContainsKey("en") switch + { + true => titleNode.AsObject()["en"]!.GetValue(), + false => titleNode.AsObject().First().Value!.GetValue() + }; - if(!manga.ContainsKey("id")) - return null; - string publicationId = manga["id"]!.GetValue(); - - if(!attributes.ContainsKey("title")) - return null; - string title = attributes["title"]!.AsObject().ContainsKey("en") && attributes["title"]!["en"] is not null - ? attributes["title"]!["en"]!.GetValue() - : attributes["title"]![((IDictionary)attributes["title"]!.AsObject()).Keys.First()]!.GetValue(); - - if(!attributes.ContainsKey("description")) - return null; - string? description = attributes["description"]!.AsObject().ContainsKey("en") && attributes["description"]!["en"] is not null - ? attributes["description"]!["en"]!.GetValue() - : null; - - if(!attributes.ContainsKey("altTitles")) - return null; - JsonArray altTitlesObject = attributes["altTitles"]!.AsArray(); Dictionary altTitlesDict = new(); - foreach (JsonNode? altTitleNode in altTitlesObject) + if (attributes.TryGetPropertyValue("altTitles", out JsonNode? altTitlesNode)) { - JsonObject altTitleObject = (JsonObject)altTitleNode!; - string key = ((IDictionary)altTitleObject).Keys.ToArray()[0]; - altTitlesDict.TryAdd(key, altTitleObject[key]!.GetValue()); - } - - if(!attributes.ContainsKey("tags")) - return null; - JsonArray tagsObject = attributes["tags"]!.AsArray(); - HashSet tags = new(); - foreach (JsonNode? tagNode in tagsObject) - { - JsonObject tagObject = (JsonObject)tagNode!; - if(tagObject["attributes"]!["name"]!.AsObject().ContainsKey("en")) - tags.Add(tagObject["attributes"]!["name"]!["en"]!.GetValue()); - } - - string? posterId = null; - HashSet authorIds = new(); - if (manga.ContainsKey("relationships") && manga["relationships"] is not null) - { - JsonArray relationships = manga["relationships"]!.AsArray(); - posterId = relationships.FirstOrDefault(relationship => relationship!["type"]!.GetValue() == "cover_art")!["id"]!.GetValue(); - foreach (JsonNode? node in relationships.Where(relationship => - relationship!["type"]!.GetValue() == "author")) - authorIds.Add(node!["id"]!.GetValue()); - } - string? coverUrl = GetCoverUrl(publicationId, posterId); - string? coverCacheName = null; - if (coverUrl is not null) - coverCacheName = SaveCoverImageToCache(coverUrl, RequestType.MangaCover); - - List authors = GetAuthors(authorIds); - - Dictionary linksDict = new(); - if (attributes.ContainsKey("links") && attributes["links"] is not null) - { - JsonObject linksObject = attributes["links"]!.AsObject(); - foreach (string key in ((IDictionary)linksObject).Keys) + foreach (JsonNode? altTitleNode in altTitlesNode!.AsArray()) { - linksDict.Add(key, linksObject[key]!.GetValue()); + JsonObject altTitleNodeObject = altTitleNode!.AsObject(); + altTitlesDict.TryAdd(altTitleNodeObject.First().Key, altTitleNodeObject.First().Value!.GetValue()); } } - int? year = attributes.ContainsKey("year") && attributes["year"] is not null - ? attributes["year"]!.GetValue() - : null; + if (!attributes.TryGetPropertyValue("description", out JsonNode? descriptionNode)) + return null; + string description = descriptionNode!.AsObject().ContainsKey("en") switch + { + true => descriptionNode.AsObject()["en"]!.GetValue(), + false => descriptionNode.AsObject().First().Value!.GetValue() + }; + + Dictionary linksDict = new(); + if (attributes.TryGetPropertyValue("links", out JsonNode? linksNode)) + foreach (KeyValuePair linkKv in linksNode!.AsObject()) + linksDict.TryAdd(linkKv.Key, linkKv.Value.GetValue()); string? originalLanguage = - attributes.ContainsKey("originalLanguage") && attributes["originalLanguage"] is not null - ? attributes["originalLanguage"]!.GetValue() - : null; - - if(!attributes.ContainsKey("status")) - return null; - string status = attributes["status"]!.GetValue(); - Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased; - switch (status.ToLower()) + attributes.TryGetPropertyValue("originalLanguage", out JsonNode? originalLanguageNode) switch + { + true => originalLanguageNode!.GetValue(), + false => null + }; + + Manga.ReleaseStatusByte status = Manga.ReleaseStatusByte.Unreleased; + if (attributes.TryGetPropertyValue("status", out JsonNode? statusNode)) { - case "ongoing": releaseStatus = Manga.ReleaseStatusByte.Continuing; break; - case "completed": releaseStatus = Manga.ReleaseStatusByte.Completed; break; - case "hiatus": releaseStatus = Manga.ReleaseStatusByte.OnHiatus; break; - case "cancelled": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break; + status = statusNode!.GetValue().ToLower() switch + { + "ongoing" => Manga.ReleaseStatusByte.Continuing, + "completed" => Manga.ReleaseStatusByte.Completed, + "hiatus" => Manga.ReleaseStatusByte.OnHiatus, + "cancelled" => Manga.ReleaseStatusByte.Cancelled, + _ => Manga.ReleaseStatusByte.Unreleased + }; + } + + int? year = attributes.TryGetPropertyValue("year", out JsonNode? yearNode) switch + { + true => yearNode!.GetValue(), + false => null + }; + + HashSet tags = new(128); + if (attributes.TryGetPropertyValue("tags", out JsonNode? tagsNode)) + foreach (JsonNode? tagNode in tagsNode!.AsArray()) + tags.Add(tagNode!["attributes"]!["name"]!["en"]!.GetValue()); + + + if (!manga.TryGetPropertyValue("relationships", out JsonNode? relationshipsNode)) + return null; + + JsonNode? coverNode = relationshipsNode!.AsArray() + .FirstOrDefault(rel => rel!["type"]!.GetValue().Equals("cover_art")); + if (coverNode is null) + return null; + string fileName = coverNode["attributes"]!["fileName"]!.GetValue(); + string coverUrl = $"https://uploads.mangadex.org/covers/{publicationId}/{fileName}"; + string coverCacheName = SaveCoverImageToCache(coverUrl, RequestType.MangaCover); + + List authors = new(); + JsonNode?[] authorNodes = relationshipsNode.AsArray() + .Where(rel => rel!["type"]!.GetValue().Equals("author") || rel!["type"]!.GetValue().Equals("artist")).ToArray(); + foreach (JsonNode? authorNode in authorNodes) + { + string authorName = authorNode!["attributes"]!["name"]!.GetValue(); + if(!authors.Contains(authorName)) + authors.Add(authorName); } Manga pub = new( @@ -179,9 +183,9 @@ public class MangaDex : MangaConnector linksDict, year, originalLanguage, - status, + Enum.GetName(status) ?? "", publicationId, - releaseStatus + status ); cachedPublications.Add(pub); return pub; @@ -288,50 +292,4 @@ public class MangaDex : MangaConnector //Download Chapter-Images return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, progressToken:progressToken); } - - private string? GetCoverUrl(string publicationId, string? posterId) - { - Log($"Getting CoverUrl for Publication {publicationId}"); - if (posterId is null) - { - Log("No cover."); - return null; - } - - //Request information where to download Cover - RequestResult requestResult = - downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", RequestType.MangaCover); - if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) - return null; - JsonObject? result = JsonSerializer.Deserialize(requestResult.result); - if (result is null) - return null; - - string fileName = result["data"]!["attributes"]!["fileName"]!.GetValue(); - - string coverUrl = $"https://uploads.mangadex.org/covers/{publicationId}/{fileName}"; - Log($"Cover-Url {publicationId} -> {coverUrl}"); - return coverUrl; - } - - private List GetAuthors(IEnumerable authorIds) - { - Log("Retrieving authors."); - List ret = new(); - foreach (string authorId in authorIds) - { - RequestResult requestResult = - downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", RequestType.MangaDexAuthor); - if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) - return ret; - JsonObject? result = JsonSerializer.Deserialize(requestResult.result); - if (result is null) - return ret; - - string authorName = result["data"]!["attributes"]!["name"]!.GetValue(); - ret.Add(authorName); - Log($"Got author {authorId} -> {authorName}"); - } - return ret; - } } \ No newline at end of file From d1a6c0ad3d4238618083774d2d92e5ca9fcd00a6 Mon Sep 17 00:00:00 2001 From: Glax Date: Thu, 18 Apr 2024 22:12:49 +0200 Subject: [PATCH 3/5] Set Chromium Start Timeout to 30 seconds. Resolves #135 ? --- Tranga/MangaConnectors/ChromiumDownloadClient.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tranga/MangaConnectors/ChromiumDownloadClient.cs b/Tranga/MangaConnectors/ChromiumDownloadClient.cs index 5ae80ad..8ddf850 100644 --- a/Tranga/MangaConnectors/ChromiumDownloadClient.cs +++ b/Tranga/MangaConnectors/ChromiumDownloadClient.cs @@ -10,6 +10,7 @@ internal class ChromiumDownloadClient : DownloadClient { private IBrowser browser { get; set; } private const string ChromiumVersion = "1154303"; + private const int StartTimeoutMs = 30000; private async Task DownloadBrowser() { @@ -40,7 +41,7 @@ internal class ChromiumDownloadClient : DownloadClient await browserFetcher.DownloadAsync(ChromiumVersion); } - Log("Starting Browser."); + Log($"Starting Browser. ({StartTimeoutMs}ms timeout)"); return await Puppeteer.LaunchAsync(new LaunchOptions { Headless = true, @@ -50,7 +51,7 @@ internal class ChromiumDownloadClient : DownloadClient "--disable-dev-shm-usage", "--disable-setuid-sandbox", "--no-sandbox"}, - Timeout = 10000 + Timeout = StartTimeoutMs }); } From ff0875461007928796b5f2a3d52562b68085d99e Mon Sep 17 00:00:00 2001 From: Glax Date: Thu, 18 Apr 2024 22:52:38 +0200 Subject: [PATCH 4/5] Bump docker/setup-buildx-action@v3.3.0 Bump docker/build-push-action@v5.3.0 --- .github/workflows/docker-base.yml | 4 ++-- .github/workflows/docker-image-cuttingedge.yml | 4 ++-- .github/workflows/docker-image-dev.yml | 4 ++-- .github/workflows/docker-image-master.yml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-base.yml b/.github/workflows/docker-base.yml index c331137..64191e6 100644 --- a/.github/workflows/docker-base.yml +++ b/.github/workflows/docker-base.yml @@ -20,7 +20,7 @@ jobs: # https://github.com/marketplace/actions/docker-setup-buildx - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3.1.0 + uses: docker/setup-buildx-action@v3.3.0 # https://github.com/docker/login-action#docker-hub - name: Login to Docker Hub @@ -31,7 +31,7 @@ jobs: # https://github.com/docker/build-push-action#multi-platform-image - name: Build and push base - uses: docker/build-push-action@v4.1.1 + uses: docker/build-push-action@v5.3.0 with: context: ./ file: ./Dockerfile-base diff --git a/.github/workflows/docker-image-cuttingedge.yml b/.github/workflows/docker-image-cuttingedge.yml index d6810eb..0f12162 100644 --- a/.github/workflows/docker-image-cuttingedge.yml +++ b/.github/workflows/docker-image-cuttingedge.yml @@ -22,7 +22,7 @@ jobs: # https://github.com/marketplace/actions/docker-setup-buildx - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3.1.0 + uses: docker/setup-buildx-action@v3.3.0 # https://github.com/docker/login-action#docker-hub - name: Login to Docker Hub @@ -33,7 +33,7 @@ jobs: # https://github.com/docker/build-push-action#multi-platform-image - name: Build and push API - uses: docker/build-push-action@v4.1.1 + uses: docker/build-push-action@v5.3.0 with: context: ./ file: ./Dockerfile diff --git a/.github/workflows/docker-image-dev.yml b/.github/workflows/docker-image-dev.yml index 32c681c..71770fc 100644 --- a/.github/workflows/docker-image-dev.yml +++ b/.github/workflows/docker-image-dev.yml @@ -22,7 +22,7 @@ jobs: # https://github.com/marketplace/actions/docker-setup-buildx - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3.1.0 + uses: docker/setup-buildx-action@v3.3.0 # https://github.com/docker/login-action#docker-hub - name: Login to Docker Hub @@ -33,7 +33,7 @@ jobs: # https://github.com/docker/build-push-action#multi-platform-image - name: Build and push API - uses: docker/build-push-action@v4.1.1 + uses: docker/build-push-action@v5.3.0 with: context: ./ file: ./Dockerfile diff --git a/.github/workflows/docker-image-master.yml b/.github/workflows/docker-image-master.yml index ffee8a6..92aff40 100644 --- a/.github/workflows/docker-image-master.yml +++ b/.github/workflows/docker-image-master.yml @@ -22,7 +22,7 @@ jobs: # https://github.com/marketplace/actions/docker-setup-buildx - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3.1.0 + uses: docker/setup-buildx-action@v3.3.0 # https://github.com/docker/login-action#docker-hub - name: Login to Docker Hub @@ -33,7 +33,7 @@ jobs: # https://github.com/docker/build-push-action#multi-platform-image - name: Build and push API - uses: docker/build-push-action@v4.1.1 + uses: docker/build-push-action@v5.3.0 with: context: ./ file: ./Dockerfile From 5f03b0d89cb15f410d49005fee7619d65524f265 Mon Sep 17 00:00:00 2001 From: Glax Date: Thu, 18 Apr 2024 23:05:04 +0200 Subject: [PATCH 5/5] Closes #154 --- README.md | 38 +++++++++----------------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 409cf06..655dcfe 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,3 @@ - - -
@@ -61,8 +52,8 @@ Tranga can download Chapters and Metadata from "Scanlation" sites such as - [Manga4Life](https://manga4life.com) (en) - ❓ Open an [issue](https://github.com/C9Glax/tranga/issues) -and trigger an scan with [Komga](https://komga.org/) and [Kavita](https://www.kavitareader.com/). -Notifications will can sent to your devices using [Gotify](https://gotify.net/) and [LunaSea](https://www.lunasea.app/). +and trigger a library-scan with [Komga](https://komga.org/) and [Kavita](https://www.kavitareader.com/). +Notifications can be sent to your devices using [Gotify](https://gotify.net/) and [LunaSea](https://www.lunasea.app/). ### What this does and doesn't do @@ -75,15 +66,15 @@ This project downloads the images for a Manga from the specified Scanlation-Webs It does this on an interval, and checks for any Chapters (.cbz-Archive) not already existing in your specified Download-Location. (If you rename or move files, it will download those again) Tranga can (if configured) trigger a scan in Komga or Kavita, however the directory in which the Manga reside has to be available to both Tranga and Komga/Kavita. -The project doesn't manage metadata, doesn't curate, change or enhance any information that isn't available on the selected Scanlation-Site. +The project doesn't manage metadata, and doesn't curate, change or enhance any information that isn't available on the selected Scanlation-Site. It will blindly use whatever is scrapes (yes this is a glorified Web-scraper). ### Inspiration: Because [Kaizoku](https://github.com/oae/kaizoku) was relying on [mangal](https://github.com/metafates/mangal) and mangal -hasn't received bugfixes for it's issues with Titles not showing up, or throwing errors because of illegal characters, -there were no alternatives for automatic downloads. However [Kaizoku](https://github.com/oae/kaizoku) certainly had a great Web-UI. +hasn't received bugfixes for its issues with Titles not showing up, or throwing errors because of illegal characters, +there were no alternatives for automatic downloads. However, [Kaizoku](https://github.com/oae/kaizoku) certainly had a great Web-UI. That is why I wanted to create my own project, in a language I understand, and that I am able to maintain myself. @@ -102,25 +93,15 @@ That is why I wanted to create my own project, in a language I understand, and t ## Getting Started -There is two release types: - -- CLI -- Docker - -### CLI - -Head over to [releases](https://git.bernloehr.eu/glax/Tranga/releases) and download. - - -~~The CLI will guide you through setup.~~ Not in the current version. -Right now it is barebones with options to view logs and make HTTP-Requests - ### Docker Download [docker-compose.yaml](https://git.bernloehr.eu/glax/Tranga/src/branch/master/docker-compose.yaml) and configure to your needs. -Mount `/Manga` to wherever you want your chapters (`.cbz`-Archives) downloaded (for exampled where Komga/Kavita can access them). +Mount `/Manga` to wherever you want your chapters (`.cbz`-Archives) downloaded (where Komga/Kavita can access them). The `docker-compose` also includes [tranga-website](https://github.com/C9Glax/tranga-website) as frontend. For its configuration refer to the repo README. +For compatibility do not execute the compose as root (which you should not do anyways...) but as user that can +access the folder. + ### Prerequisites #### To Build @@ -131,7 +112,6 @@ The `docker-compose` also includes [tranga-website](https://github.com/C9Glax/tr ## Roadmap -- [ ] Docker ARM support - [ ] ❓ See the [open issues](https://github.com/C9Glax/tranga/issues) for a full list of proposed features (and known issues).