glax/Tranga
2
0
Fork 0
Code

Fixed naming errors containing Manga

Browse Source 
Added GetMangaFromUrl(url) to Mangaconnector
This commit is contained in:
glax 2023-09-02 21:52:48 +02:00
parent 14ba71005f
commit ad1d4dfe23
6 changed files with 183 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Tranga/MangaConnectors/MangaConnector.cs
View File

@ -29,7 +29,9 @@ public abstract class MangaConnector : GlobalBase
/// </summary>
/// <param name="publicationTitle">Search-Query</param>
/// <returns>Publications matching the query</returns>
public abstract Manga[] GetPublications(string publicationTitle = "");
public abstract Manga[] GetManga(string publicationTitle = "");
public abstract Manga? GetMangaFromUrl(string url);
/// <summary>
/// Returns all Chapters of the publication in the provided language.

202
Tranga/MangaConnectors/MangaDex.cs
View File

@ -1,8 +1,9 @@
using System.Globalization;
using System.Net;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.RegularExpressions;
using Tranga.Jobs;
using JsonSerializer = System.Text.Json.JsonSerializer;
namespace Tranga.MangaConnectors;
public class MangaDex : MangaConnector
@ -31,13 +32,13 @@ public class MangaDex : MangaConnector
});
}
public override Manga[] GetPublications(string publicationTitle = "")
public override Manga[] GetManga(string publicationTitle = "")
{
Log($"Searching Publications. Term=\"{publicationTitle}\"");
const int limit = 100; //How many values we want returned at once
int offset = 0; //"Page"
int total = int.MaxValue; //How many total results are there, is updated on first request
HashSet<Manga> publications = new();
HashSet<Manga> retManga = new();
int loadedPublicationData = 0;
while (offset < total) //As long as we haven't requested all "Pages"
{
@ -57,98 +58,119 @@ public class MangaDex : MangaConnector
JsonArray mangaInResult = result["data"]!.AsArray(); //Manga-data-Array
//Loop each Manga and extract information from JSON
foreach (JsonNode? mangeNode in mangaInResult)
foreach (JsonNode? mangaNode in mangaInResult)
{
Log($"Getting publication data. {++loadedPublicationData}/{total}");
JsonObject manga = (JsonObject)mangeNode!;
JsonObject attributes = manga["attributes"]!.AsObject();
string publicationId = manga["id"]!.GetValue<string>();
string title = attributes["title"]!.AsObject().ContainsKey("en") && attributes["title"]!["en"] is not null
? attributes["title"]!["en"]!.GetValue<string>()
: attributes["title"]![((IDictionary<string, JsonNode?>)attributes["title"]!.AsObject()).Keys.First()]!.GetValue<string>();
string? description = attributes["description"]!.AsObject().ContainsKey("en") && attributes["description"]!["en"] is not null
? attributes["description"]!["en"]!.GetValue<string?>()
: null;
JsonArray altTitlesObject = attributes["altTitles"]!.AsArray();
Dictionary<string, string> altTitlesDict = new();
foreach (JsonNode? altTitleNode in altTitlesObject)
{
JsonObject altTitleObject = (JsonObject)altTitleNode!;
string key = ((IDictionary<string, JsonNode?>)altTitleObject).Keys.ToArray()[0];
altTitlesDict.TryAdd(key, altTitleObject[key]!.GetValue<string>());
}
JsonArray tagsObject = attributes["tags"]!.AsArray();
HashSet<string> tags = new();
foreach (JsonNode? tagNode in tagsObject)
{
JsonObject tagObject = (JsonObject)tagNode!;
if(tagObject["attributes"]!["name"]!.AsObject().ContainsKey("en"))
tags.Add(tagObject["attributes"]!["name"]!["en"]!.GetValue<string>());
}
string? posterId = null;
HashSet<string> authorIds = new();
if (manga.ContainsKey("relationships") && manga["relationships"] is not null)
{
JsonArray relationships = manga["relationships"]!.AsArray();
posterId = relationships.FirstOrDefault(relationship => relationship!["type"]!.GetValue<string>() == "cover_art")!["id"]!.GetValue<string>();
foreach (JsonNode? node in relationships.Where(relationship =>
relationship!["type"]!.GetValue<string>() == "author"))
authorIds.Add(node!["id"]!.GetValue<string>());
}
string? coverUrl = GetCoverUrl(publicationId, posterId);
string? coverCacheName = null;
if (coverUrl is not null)
coverCacheName = SaveCoverImageToCache(coverUrl, (byte)RequestType.AtHomeServer);
List<string> authors = GetAuthors(authorIds);
Dictionary<string, string> linksDict = new();
if (attributes.ContainsKey("links") && attributes["links"] is not null)
{
JsonObject linksObject = attributes["links"]!.AsObject();
foreach (string key in ((IDictionary<string, JsonNode?>)linksObject).Keys)
{
linksDict.Add(key, linksObject[key]!.GetValue<string>());
}
}
int? year = attributes.ContainsKey("year") && attributes["year"] is not null
? attributes["year"]!.GetValue<int?>()
: null;
string? originalLanguage = attributes.ContainsKey("originalLanguage") && attributes["originalLanguage"] is not null
? attributes["originalLanguage"]!.GetValue<string?>()
: null;
string status = attributes["status"]!.GetValue<string>();
Manga pub = new (
title,
authors,
description,
altTitlesDict,
tags.ToArray(),
coverUrl,
coverCacheName,
linksDict,
year,
originalLanguage,
status,
publicationId
);
publications.Add(pub); //Add Publication (Manga) to result
Manga manga = MangaFromJsonObject((JsonObject)mangaNode);
retManga.Add(manga); //Add Publication (Manga) to result
}
}
cachedPublications.AddRange(publications);
Log($"Retrieved {publications.Count} publications. Term=\"{publicationTitle}\"");
return publications.ToArray();
cachedPublications.AddRange(retManga);
Log($"Retrieved {retManga.Count} publications. Term=\"{publicationTitle}\"");
return retManga.ToArray();
}
public override Manga? GetMangaFromUrl(string url)
{
Regex idRex = new (@"https:\/\/mangadex.org\/title\/([A-z0-9-]*)\/.*");
string id = idRex.Match(url).Value;
Log($"Got id {id} from {url}");
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/manga/{id}", (byte)RequestType.Manga);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
if(result is not null)
return MangaFromJsonObject(result);
return null;
}
private Manga MangaFromJsonObject(JsonObject manga)
{
JsonObject attributes = manga["attributes"]!.AsObject();
string publicationId = manga["id"]!.GetValue<string>();
string title = attributes["title"]!.AsObject().ContainsKey("en") && attributes["title"]!["en"] is not null
? attributes["title"]!["en"]!.GetValue<string>()
: attributes["title"]![((IDictionary<string, JsonNode?>)attributes["title"]!.AsObject()).Keys.First()]!.GetValue<string>();
string? description = attributes["description"]!.AsObject().ContainsKey("en") && attributes["description"]!["en"] is not null
? attributes["description"]!["en"]!.GetValue<string?>()
: null;
JsonArray altTitlesObject = attributes["altTitles"]!.AsArray();
Dictionary<string, string> altTitlesDict = new();
foreach (JsonNode? altTitleNode in altTitlesObject)
{
JsonObject altTitleObject = (JsonObject)altTitleNode!;
string key = ((IDictionary<string, JsonNode?>)altTitleObject).Keys.ToArray()[0];
altTitlesDict.TryAdd(key, altTitleObject[key]!.GetValue<string>());
}
JsonArray tagsObject = attributes["tags"]!.AsArray();
HashSet<string> tags = new();
foreach (JsonNode? tagNode in tagsObject)
{
JsonObject tagObject = (JsonObject)tagNode!;
if(tagObject["attributes"]!["name"]!.AsObject().ContainsKey("en"))
tags.Add(tagObject["attributes"]!["name"]!["en"]!.GetValue<string>());
}
string? posterId = null;
HashSet<string> authorIds = new();
if (manga.ContainsKey("relationships") && manga["relationships"] is not null)
{
JsonArray relationships = manga["relationships"]!.AsArray();
posterId = relationships.FirstOrDefault(relationship => relationship!["type"]!.GetValue<string>() == "cover_art")!["id"]!.GetValue<string>();
foreach (JsonNode? node in relationships.Where(relationship =>
relationship!["type"]!.GetValue<string>() == "author"))
authorIds.Add(node!["id"]!.GetValue<string>());
}
string? coverUrl = GetCoverUrl(publicationId, posterId);
string? coverCacheName = null;
if (coverUrl is not null)
coverCacheName = SaveCoverImageToCache(coverUrl, (byte)RequestType.AtHomeServer);
List<string> authors = GetAuthors(authorIds);
Dictionary<string, string> linksDict = new();
if (attributes.ContainsKey("links") && attributes["links"] is not null)
{
JsonObject linksObject = attributes["links"]!.AsObject();
foreach (string key in ((IDictionary<string, JsonNode?>)linksObject).Keys)
{
linksDict.Add(key, linksObject[key]!.GetValue<string>());
}
}
int? year = attributes.ContainsKey("year") && attributes["year"] is not null
? attributes["year"]!.GetValue<int?>()
: null;
string? originalLanguage =
attributes.ContainsKey("originalLanguage") && attributes["originalLanguage"] is not null
? attributes["originalLanguage"]!.GetValue<string?>()
: null;
string status = attributes["status"]!.GetValue<string>();
Manga pub = new(
title,
authors,
description,
altTitlesDict,
tags.ToArray(),
coverUrl,
coverCacheName,
linksDict,
year,
originalLanguage,
status,
publicationId
);
return pub;
}
public override Chapter[] GetChapters(Manga manga, string language="en")

20
Tranga/MangaConnectors/MangaKatana.cs
View File

@ -19,7 +19,7 @@ public class MangaKatana : MangaConnector
});
}
public override Manga[] GetPublications(string publicationTitle = "")
public override Manga[] GetManga(string publicationTitle = "")
{
Log($"Searching Publications. Term=\"{publicationTitle}\"");
string sanitizedTitle = string.Join('_', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower();
@ -44,6 +44,15 @@ public class MangaKatana : MangaConnector
return publications;
}
public override Manga? GetMangaFromUrl(string url)
{
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(url, 1);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
return ParseSinglePublicationFromHtml(requestResult.result, url.Split('/')[^1]);
}
private Manga[] ParsePublicationsFromHtml(Stream html)
{
StreamReader reader = new(html);
@ -63,12 +72,9 @@ public class MangaKatana : MangaConnector
HashSet<Manga> ret = new();
foreach (string url in urls)
{
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(url, 1);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Manga>();
ret.Add(ParseSinglePublicationFromHtml(requestResult.result, url.Split('/')[^1]));
Manga? manga = GetMangaFromUrl(url);
if (manga is not null)
ret.Add((Manga)manga);
}
return ret.ToArray();

21
Tranga/MangaConnectors/Manganato.cs
View File

@ -19,7 +19,7 @@ public class Manganato : MangaConnector
});
}
public override Manga[] GetPublications(string publicationTitle = "")
public override Manga[] GetManga(string publicationTitle = "")
{
Log($"Searching Publications. Term=\"{publicationTitle}\"");
string sanitizedTitle = string.Join('_', Regex.Matches(publicationTitle, "[A-z]*")).ToLower();
@ -52,17 +52,24 @@ public class Manganato : MangaConnector
HashSet<Manga> ret = new();
foreach (string url in urls)
{
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(url, 1);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Manga>();
ret.Add(ParseSinglePublicationFromHtml(requestResult.result, url.Split('/')[^1]));
Manga? manga = GetMangaFromUrl(url);
if (manga is not null)
ret.Add((Manga)manga);
}
return ret.ToArray();
}
public override Manga? GetMangaFromUrl(string url)
{
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(url, 1);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
return ParseSinglePublicationFromHtml(requestResult.result, url.Split('/')[^1]);
}
private Manga ParseSinglePublicationFromHtml(Stream html, string publicationId)
{
StreamReader reader = new (html);

80
Tranga/MangaConnectors/Mangasee.cs
View File

@ -69,7 +69,7 @@ public class Mangasee : MangaConnector
});
}
public override Manga[] GetPublications(string publicationTitle = "")
public override Manga[] GetManga(string publicationTitle = "")
{
Log($"Searching Publications. Term=\"{publicationTitle}\"");
string requestUrl = $"https://mangasee123.com/_search.php";
@ -84,6 +84,28 @@ public class Mangasee : MangaConnector
return publications;
}
public override Manga? GetMangaFromUrl(string url)
{
while (this._browser is null)
{
Log("Waiting for headless browser to download...");
Thread.Sleep(1000);
}
IPage page = _browser!.NewPageAsync().Result;
IResponse response = page.GoToAsync(url, WaitUntilNavigation.DOMContentLoaded).Result;
if (response.Ok)
{
HtmlDocument document = new();
document.LoadHtml(page.GetContentAsync().Result);
page.CloseAsync();
return ParseSinglePublicationFromHtml(document);
}
return null;
}
private Manga[] ParsePublicationsFromHtml(Stream html, string publicationTitle)
{
string jsonString = new StreamReader(html).ReadToEnd();
@ -105,73 +127,51 @@ public class Mangasee : MangaConnector
List<SearchResultItem> orderedFiltered =
queryFiltered.OrderBy(item => item.Value).ToDictionary(item => item.Key, item => item.Value).Keys.ToList();
uint index = 1;
foreach (SearchResultItem orderedItem in orderedFiltered)
{
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://mangasee123.com/manga/{orderedItem.i}", 1);
if ((int)requestResult.statusCode >= 200 || (int)requestResult.statusCode < 300)
{
Log($"Retrieving Publication info: {orderedItem.s} {index++}/{orderedFiltered.Count}");
ret.Add(ParseSinglePublicationFromHtml(requestResult.result, orderedItem.s, orderedItem.i, orderedItem.a));
}
Manga? manga = GetMangaFromUrl($"https://mangasee123.com/manga/{orderedItem.i}");
if (manga is not null)
ret.Add((Manga)manga);
}
return ret.ToArray();
}
private Manga ParseSinglePublicationFromHtml(Stream html, string sortName, string publicationId, string[] a)
private Manga ParseSinglePublicationFromHtml(HtmlDocument document)
{
StreamReader reader = new (html);
HtmlDocument document = new ();
document.LoadHtml(reader.ReadToEnd());
string originalLanguage = "", status = "";
Dictionary<string, string> altTitles = new(), links = new();
HashSet<string> tags = new();
HtmlNode posterNode =
document.DocumentNode.Descendants("img").First(img => img.HasClass("img-fluid") && img.HasClass("bottom-5"));
HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//img");
string posterUrl = posterNode.GetAttributeValue("src", "");
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1);
HtmlNode attributes = document.DocumentNode.Descendants("div")
.First(div => div.HasClass("col-md-9") && div.HasClass("col-sm-8") && div.HasClass("top-5"))
.Descendants("ul").First();
HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//h1");
string title = titleNode.InnerText;
string publicationId = title;
HtmlNode[] authorsNodes = attributes.Descendants("li")
.First(node => node.InnerText.Contains("author(s):", StringComparison.CurrentCultureIgnoreCase))
.Descendants("a").ToArray();
HtmlNode[] authorsNodes = document.DocumentNode.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Author(s):']/..").Descendants("a").ToArray();
List<string> authors = new();
foreach(HtmlNode authorNode in authorsNodes)
authors.Add(authorNode.InnerText);
HtmlNode[] genreNodes = attributes.Descendants("li")
.First(node => node.InnerText.Contains("genre(s):", StringComparison.CurrentCultureIgnoreCase))
.Descendants("a").ToArray();
HtmlNode[] genreNodes = document.DocumentNode.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Genre(s):']/..").Descendants("a").ToArray();
foreach (HtmlNode genreNode in genreNodes)
tags.Add(genreNode.InnerText);
HtmlNode yearNode = attributes.Descendants("li")
.First(node => node.InnerText.Contains("released:", StringComparison.CurrentCultureIgnoreCase))
.Descendants("a").First();
HtmlNode yearNode = document.DocumentNode.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Released:']/..").Descendants("a").First();
int year = Convert.ToInt32(yearNode.InnerText);
HtmlNode[] statusNodes = attributes.Descendants("li")
.First(node => node.InnerText.Contains("status:", StringComparison.CurrentCultureIgnoreCase))
.Descendants("a").ToArray();
HtmlNode[] statusNodes = document.DocumentNode.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Status:']/..").Descendants("a").ToArray();
foreach(HtmlNode statusNode in statusNodes)
if (statusNode.InnerText.Contains("publish", StringComparison.CurrentCultureIgnoreCase))
status = statusNode.InnerText.Split(' ')[0];
HtmlNode descriptionNode = attributes.Descendants("li").First(node => node.InnerText.Contains("description:", StringComparison.CurrentCultureIgnoreCase)).Descendants("div").First();
HtmlNode descriptionNode = document.DocumentNode.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Description:']/..").Descendants("div").First();
string description = descriptionNode.InnerText;
int i = 0;
foreach(string at in a)
altTitles.Add((i++).ToString(), at);
return new Manga(sortName, authors, description, altTitles, tags.ToArray(), posterUrl, coverFileNameInCache, links,
return new Manga(title, authors, description, altTitles, tags.ToArray(), posterUrl, coverFileNameInCache, links,
year, originalLanguage, status, publicationId);
}

2
Tranga/Server.cs
View File

@ -141,7 +141,7 @@ public class Server : GlobalBase
SendResponse(HttpStatusCode.BadRequest, response);
break;
}
SendResponse(HttpStatusCode.OK, response, connector!.GetPublications(title));
SendResponse(HttpStatusCode.OK, response, connector!.GetManga(title));
break;
case "Manga/Chapters":
if(!requestVariables.TryGetValue("connector", out connectorName) ||