glax/Tranga
2
0
Fork 0
Code Issues Pull Requests Releases 25 Activity

Merge remote-tracking branch 'upstream/cuttingedge' into cuttingedge

Browse Source
This commit is contained in:
db-2001 2024-04-18 17:48:43 -04:00
commit 006b71b496
7 changed files with 119 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/docker-base.yml vendored
View File

@ -20,7 +20,7 @@ jobs:
# https://github.com/marketplace/actions/docker-setup-buildx
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3.1.0
uses: docker/setup-buildx-action@v3.3.0
# https://github.com/docker/login-action#docker-hub
- name: Login to Docker Hub
@ -31,7 +31,7 @@ jobs:
# https://github.com/docker/build-push-action#multi-platform-image
- name: Build and push base
uses: docker/build-push-action@v4.1.1
uses: docker/build-push-action@v5.3.0
with:
context: ./
file: ./Dockerfile-base

4
.github/workflows/docker-image-cuttingedge.yml vendored
View File

@ -22,7 +22,7 @@ jobs:
# https://github.com/marketplace/actions/docker-setup-buildx
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3.1.0
uses: docker/setup-buildx-action@v3.3.0
# https://github.com/docker/login-action#docker-hub
- name: Login to Docker Hub
@ -33,7 +33,7 @@ jobs:
# https://github.com/docker/build-push-action#multi-platform-image
- name: Build and push API
uses: docker/build-push-action@v4.1.1
uses: docker/build-push-action@v5.3.0
with:
context: ./
file: ./Dockerfile

4
.github/workflows/docker-image-dev.yml vendored
View File

@ -22,7 +22,7 @@ jobs:
# https://github.com/marketplace/actions/docker-setup-buildx
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3.1.0
uses: docker/setup-buildx-action@v3.3.0
# https://github.com/docker/login-action#docker-hub
- name: Login to Docker Hub
@ -33,7 +33,7 @@ jobs:
# https://github.com/docker/build-push-action#multi-platform-image
- name: Build and push API
uses: docker/build-push-action@v4.1.1
uses: docker/build-push-action@v5.3.0
with:
context: ./
file: ./Dockerfile

4
.github/workflows/docker-image-master.yml vendored
View File

@ -22,7 +22,7 @@ jobs:
# https://github.com/marketplace/actions/docker-setup-buildx
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3.1.0
uses: docker/setup-buildx-action@v3.3.0
# https://github.com/docker/login-action#docker-hub
- name: Login to Docker Hub
@ -33,7 +33,7 @@ jobs:
# https://github.com/docker/build-push-action#multi-platform-image
- name: Build and push API
uses: docker/build-push-action@v4.1.1
uses: docker/build-push-action@v5.3.0
with:
context: ./
file: ./Dockerfile

38
README.md
View File

@ -1,12 +1,3 @@
<!-- PROJECT SHIELDS -->
<!--
*** I'm using markdown "reference style" links for readability.
*** Reference links are enclosed in brackets [ ] instead of parentheses ( ).
*** See the bottom of this document for the declaration of the reference variables
*** for contributors-url, forks-url, etc. This is an optional, concise syntax you may use.
*** https://www.markdownguide.org/basic-syntax/#reference-style-links
-->
<!-- PROJECT LOGO -->
<br />
<div align="center">
@ -61,8 +52,8 @@ Tranga can download Chapters and Metadata from "Scanlation" sites such as
- [Manga4Life](https://manga4life.com) (en)
- ❓ Open an [issue](https://github.com/C9Glax/tranga/issues)
and trigger an scan with [Komga](https://komga.org/) and [Kavita](https://www.kavitareader.com/).
Notifications will can sent to your devices using [Gotify](https://gotify.net/) and [LunaSea](https://www.lunasea.app/).
and trigger a library-scan with [Komga](https://komga.org/) and [Kavita](https://www.kavitareader.com/).
Notifications can be sent to your devices using [Gotify](https://gotify.net/) and [LunaSea](https://www.lunasea.app/).
### What this does and doesn't do
@ -75,15 +66,15 @@ This project downloads the images for a Manga from the specified Scanlation-Webs
It does this on an interval, and checks for any Chapters (.cbz-Archive) not already existing in your specified Download-Location. (If you rename or move files, it will download those again)
Tranga can (if configured) trigger a scan in Komga or Kavita, however the directory in which the Manga reside has to be available to both Tranga and Komga/Kavita.
The project doesn't manage metadata, doesn't curate, change or enhance any information that isn't available on the selected Scanlation-Site.
The project doesn't manage metadata, and doesn't curate, change or enhance any information that isn't available on the selected Scanlation-Site.
It will blindly use whatever is scrapes (yes this is a glorified Web-scraper).
### Inspiration:
Because [Kaizoku](https://github.com/oae/kaizoku) was relying on [mangal](https://github.com/metafates/mangal) and mangal
hasn't received bugfixes for it's issues with Titles not showing up, or throwing errors because of illegal characters,
there were no alternatives for automatic downloads. However [Kaizoku](https://github.com/oae/kaizoku) certainly had a great Web-UI.
hasn't received bugfixes for its issues with Titles not showing up, or throwing errors because of illegal characters,
there were no alternatives for automatic downloads. However, [Kaizoku](https://github.com/oae/kaizoku) certainly had a great Web-UI.
That is why I wanted to create my own project, in a language I understand, and that I am able to maintain myself.
@ -102,25 +93,15 @@ That is why I wanted to create my own project, in a language I understand, and t
<!-- GETTING STARTED -->
## Getting Started
There is two release types:
- CLI
- Docker
### CLI
Head over to [releases](https://git.bernloehr.eu/glax/Tranga/releases) and download.
~~The CLI will guide you through setup.~~ Not in the current version.
Right now it is barebones with options to view logs and make HTTP-Requests
### Docker
Download [docker-compose.yaml](https://git.bernloehr.eu/glax/Tranga/src/branch/master/docker-compose.yaml) and configure to your needs.
Mount `/Manga` to wherever you want your chapters (`.cbz`-Archives) downloaded (for exampled where Komga/Kavita can access them).
Mount `/Manga` to wherever you want your chapters (`.cbz`-Archives) downloaded (where Komga/Kavita can access them).
The `docker-compose` also includes [tranga-website](https://github.com/C9Glax/tranga-website) as frontend. For its configuration refer to the repo README.
For compatibility do not execute the compose as root (which you should not do anyways...) but as user that can
access the folder.
### Prerequisites
#### To Build
@ -131,7 +112,6 @@ The `docker-compose` also includes [tranga-website](https://github.com/C9Glax/tr
<!-- ROADMAP -->
## Roadmap
- [ ] Docker ARM support
- [ ] ❓
See the [open issues](https://github.com/C9Glax/tranga/issues) for a full list of proposed features (and known issues).

5
Tranga/MangaConnectors/ChromiumDownloadClient.cs
View File

@ -10,6 +10,7 @@ internal class ChromiumDownloadClient : DownloadClient
{
private IBrowser browser { get; set; }
private const string ChromiumVersion = "1154303";
private const int StartTimeoutMs = 30000;
private async Task<IBrowser> DownloadBrowser()
{
@ -40,7 +41,7 @@ internal class ChromiumDownloadClient : DownloadClient
await browserFetcher.DownloadAsync(ChromiumVersion);
}
Log("Starting Browser.");
Log($"Starting Browser. ({StartTimeoutMs}ms timeout)");
return await Puppeteer.LaunchAsync(new LaunchOptions
{
Headless = true,
@ -50,7 +51,7 @@ internal class ChromiumDownloadClient : DownloadClient
"--disable-dev-shm-usage",
"--disable-setuid-sandbox",
"--no-sandbox"},
Timeout = 10000
Timeout = StartTimeoutMs
});
}

240
Tranga/MangaConnectors/MangaDex.cs
View File

@ -20,12 +20,18 @@ public class MangaDex : MangaConnector
int total = int.MaxValue; //How many total results are there, is updated on first request
HashSet<Manga> retManga = new();
int loadedPublicationData = 0;
List<JsonNode> results = new();
//Request all search-results
while (offset < total) //As long as we haven't requested all "Pages"
{
//Request next Page
RequestResult requestResult =
downloadClient.MakeRequest(
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", RequestType.MangaInfo);
RequestResult requestResult = downloadClient.MakeRequest(
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}" +
$"&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica" +
$"&contentRating%5B%5D=pornographic" +
$"&includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author" +
$"&includes%5B%5D=artist&includes%5B%5D=tag", RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -39,18 +45,14 @@ public class MangaDex : MangaConnector
else continue;
if (result.ContainsKey("data"))
{
JsonArray mangaInResult = result["data"]!.AsArray(); //Manga-data-Array
//Loop each Manga and extract information from JSON
foreach (JsonNode? mangaNode in mangaInResult)
{
if(mangaNode is null)
continue;
Log($"Getting publication data. {++loadedPublicationData}/{total}");
if(MangaFromJsonObject((JsonObject) mangaNode) is { } manga)
retManga.Add(manga); //Add Publication (Manga) to result
}
}//else continue;
results.AddRange(result["data"]!.AsArray()!);//Manga-data-Array
}
foreach (JsonNode mangaNode in results)
{
Log($"Getting publication data. {++loadedPublicationData}/{total}");
if(MangaFromJsonObject(mangaNode.AsObject()) is { } manga)
retManga.Add(manga); //Add Publication (Manga) to result
}
Log($"Retrieved {retManga.Count} publications. Term=\"{publicationTitle}\"");
return retManga.ToArray();
@ -78,94 +80,96 @@ public class MangaDex : MangaConnector
private Manga? MangaFromJsonObject(JsonObject manga)
{
if (!manga.ContainsKey("attributes"))
if (!manga.TryGetPropertyValue("id", out JsonNode? idNode))
return null;
JsonObject attributes = manga["attributes"]!.AsObject();
string publicationId = idNode!.GetValue<string>();
if (!manga.TryGetPropertyValue("attributes", out JsonNode? attributesNode))
return null;
JsonObject attributes = attributesNode!.AsObject();
if (!attributes.TryGetPropertyValue("title", out JsonNode? titleNode))
return null;
string title = titleNode!.AsObject().ContainsKey("en") switch
{
true => titleNode.AsObject()["en"]!.GetValue<string>(),
false => titleNode.AsObject().First().Value!.GetValue<string>()
};
if(!manga.ContainsKey("id"))
return null;
string publicationId = manga["id"]!.GetValue<string>();
if(!attributes.ContainsKey("title"))
return null;
string title = attributes["title"]!.AsObject().ContainsKey("en") && attributes["title"]!["en"] is not null
? attributes["title"]!["en"]!.GetValue<string>()
: attributes["title"]![((IDictionary<string, JsonNode?>)attributes["title"]!.AsObject()).Keys.First()]!.GetValue<string>();
if(!attributes.ContainsKey("description"))
return null;
string? description = attributes["description"]!.AsObject().ContainsKey("en") && attributes["description"]!["en"] is not null
? attributes["description"]!["en"]!.GetValue<string?>()
: null;
if(!attributes.ContainsKey("altTitles"))
return null;
JsonArray altTitlesObject = attributes["altTitles"]!.AsArray();
Dictionary<string, string> altTitlesDict = new();
foreach (JsonNode? altTitleNode in altTitlesObject)
if (attributes.TryGetPropertyValue("altTitles", out JsonNode? altTitlesNode))
{
JsonObject altTitleObject = (JsonObject)altTitleNode!;
string key = ((IDictionary<string, JsonNode?>)altTitleObject).Keys.ToArray()[0];
altTitlesDict.TryAdd(key, altTitleObject[key]!.GetValue<string>());
}
if(!attributes.ContainsKey("tags"))
return null;
JsonArray tagsObject = attributes["tags"]!.AsArray();
HashSet<string> tags = new();
foreach (JsonNode? tagNode in tagsObject)
{
JsonObject tagObject = (JsonObject)tagNode!;
if(tagObject["attributes"]!["name"]!.AsObject().ContainsKey("en"))
tags.Add(tagObject["attributes"]!["name"]!["en"]!.GetValue<string>());
}
string? posterId = null;
HashSet<string> authorIds = new();
if (manga.ContainsKey("relationships") && manga["relationships"] is not null)
{
JsonArray relationships = manga["relationships"]!.AsArray();
posterId = relationships.FirstOrDefault(relationship => relationship!["type"]!.GetValue<string>() == "cover_art")!["id"]!.GetValue<string>();
foreach (JsonNode? node in relationships.Where(relationship =>
relationship!["type"]!.GetValue<string>() == "author"))
authorIds.Add(node!["id"]!.GetValue<string>());
}
string? coverUrl = GetCoverUrl(publicationId, posterId);
string? coverCacheName = null;
if (coverUrl is not null)
coverCacheName = SaveCoverImageToCache(coverUrl, RequestType.MangaCover);
List<string> authors = GetAuthors(authorIds);
Dictionary<string, string> linksDict = new();
if (attributes.ContainsKey("links") && attributes["links"] is not null)
{
JsonObject linksObject = attributes["links"]!.AsObject();
foreach (string key in ((IDictionary<string, JsonNode?>)linksObject).Keys)
foreach (JsonNode? altTitleNode in altTitlesNode!.AsArray())
{
linksDict.Add(key, linksObject[key]!.GetValue<string>());
JsonObject altTitleNodeObject = altTitleNode!.AsObject();
altTitlesDict.TryAdd(altTitleNodeObject.First().Key, altTitleNodeObject.First().Value!.GetValue<string>());
}
}
int? year = attributes.ContainsKey("year") && attributes["year"] is not null
? attributes["year"]!.GetValue<int?>()
: null;
if (!attributes.TryGetPropertyValue("description", out JsonNode? descriptionNode))
return null;
string description = descriptionNode!.AsObject().ContainsKey("en") switch
{
true => descriptionNode.AsObject()["en"]!.GetValue<string>(),
false => descriptionNode.AsObject().First().Value!.GetValue<string>()
};
Dictionary<string, string> linksDict = new();
if (attributes.TryGetPropertyValue("links", out JsonNode? linksNode))
foreach (KeyValuePair<string, JsonNode> linkKv in linksNode!.AsObject())
linksDict.TryAdd(linkKv.Key, linkKv.Value.GetValue<string>());
string? originalLanguage =
attributes.ContainsKey("originalLanguage") && attributes["originalLanguage"] is not null
? attributes["originalLanguage"]!.GetValue<string?>()
: null;
if(!attributes.ContainsKey("status"))
return null;
string status = attributes["status"]!.GetValue<string>();
Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased;
switch (status.ToLower())
attributes.TryGetPropertyValue("originalLanguage", out JsonNode? originalLanguageNode) switch
{
true => originalLanguageNode!.GetValue<string>(),
false => null
};
Manga.ReleaseStatusByte status = Manga.ReleaseStatusByte.Unreleased;
if (attributes.TryGetPropertyValue("status", out JsonNode? statusNode))
{
case "ongoing": releaseStatus = Manga.ReleaseStatusByte.Continuing; break;
case "completed": releaseStatus = Manga.ReleaseStatusByte.Completed; break;
case "hiatus": releaseStatus = Manga.ReleaseStatusByte.OnHiatus; break;
case "cancelled": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break;
status = statusNode!.GetValue<string>().ToLower() switch
{
"ongoing" => Manga.ReleaseStatusByte.Continuing,
"completed" => Manga.ReleaseStatusByte.Completed,
"hiatus" => Manga.ReleaseStatusByte.OnHiatus,
"cancelled" => Manga.ReleaseStatusByte.Cancelled,
_ => Manga.ReleaseStatusByte.Unreleased
};
}
int? year = attributes.TryGetPropertyValue("year", out JsonNode? yearNode) switch
{
true => yearNode!.GetValue<int>(),
false => null
};
HashSet<string> tags = new(128);
if (attributes.TryGetPropertyValue("tags", out JsonNode? tagsNode))
foreach (JsonNode? tagNode in tagsNode!.AsArray())
tags.Add(tagNode!["attributes"]!["name"]!["en"]!.GetValue<string>());
if (!manga.TryGetPropertyValue("relationships", out JsonNode? relationshipsNode))
return null;
JsonNode? coverNode = relationshipsNode!.AsArray()
.FirstOrDefault(rel => rel!["type"]!.GetValue<string>().Equals("cover_art"));
if (coverNode is null)
return null;
string fileName = coverNode["attributes"]!["fileName"]!.GetValue<string>();
string coverUrl = $"https://uploads.mangadex.org/covers/{publicationId}/{fileName}";
string coverCacheName = SaveCoverImageToCache(coverUrl, RequestType.MangaCover);
List<string> authors = new();
JsonNode?[] authorNodes = relationshipsNode.AsArray()
.Where(rel => rel!["type"]!.GetValue<string>().Equals("author") || rel!["type"]!.GetValue<string>().Equals("artist")).ToArray();
foreach (JsonNode? authorNode in authorNodes)
{
string authorName = authorNode!["attributes"]!["name"]!.GetValue<string>();
if(!authors.Contains(authorName))
authors.Add(authorName);
}
Manga pub = new(
@ -179,9 +183,9 @@ public class MangaDex : MangaConnector
linksDict,
year,
originalLanguage,
status,
Enum.GetName(status) ?? "",
publicationId,
releaseStatus
status
);
cachedPublications.Add(pub);
return pub;
@ -200,7 +204,7 @@ public class MangaDex : MangaConnector
//Request next "Page"
RequestResult requestResult =
downloadClient.MakeRequest(
$"https://api.mangadex.org/manga/{manga.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", RequestType.MangaDexFeed);
$"https://api.mangadex.org/manga/{manga.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&contentRating%5B%5D=pornographic", RequestType.MangaDexFeed);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -288,50 +292,4 @@ public class MangaDex : MangaConnector
//Download Chapter-Images
return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, progressToken:progressToken);
}
private string? GetCoverUrl(string publicationId, string? posterId)
{
Log($"Getting CoverUrl for Publication {publicationId}");
if (posterId is null)
{
Log("No cover.");
return null;
}
//Request information where to download Cover
RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", RequestType.MangaCover);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
if (result is null)
return null;
string fileName = result["data"]!["attributes"]!["fileName"]!.GetValue<string>();
string coverUrl = $"https://uploads.mangadex.org/covers/{publicationId}/{fileName}";
Log($"Cover-Url {publicationId} -> {coverUrl}");
return coverUrl;
}
private List<string> GetAuthors(IEnumerable<string> authorIds)
{
Log("Retrieving authors.");
List<string> ret = new();
foreach (string authorId in authorIds)
{
RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", RequestType.MangaDexAuthor);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return ret;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
if (result is null)
return ret;
string authorName = result["data"]!["attributes"]!["name"]!.GetValue<string>();
ret.Add(authorName);
Log($"Got author {authorId} -> {authorName}");
}
return ret;
}
}