mirror of
https://github.com/C9Glax/tranga.git
synced 2025-09-10 20:08:19 +02:00
MangaConnectors do not have to return an Object with 6 Parameters.
Job-Start Logic readable and optimized More robust Database design
This commit is contained in:
@@ -1,191 +0,0 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using API.MangaDownloadClients;
|
||||
using HtmlAgilityPack;
|
||||
using log4net;
|
||||
|
||||
namespace API.Schema.MangaConnectors;
|
||||
|
||||
public class AsuraToon : MangaConnector
|
||||
{
|
||||
|
||||
public AsuraToon() : base("AsuraToon", ["en"], ["asuracomic.net"], "https://asuracomic.net/images/logo.webp")
|
||||
{
|
||||
this.downloadClient = new ChromiumDownloadClient();
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(string publicationTitle = "")
|
||||
{
|
||||
string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower();
|
||||
string requestUrl = $"https://asuracomic.net/series?name={sanitizedTitle}";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return [];
|
||||
|
||||
if (requestResult.htmlDocument is null)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument);
|
||||
return publications;
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(string publicationId)
|
||||
{
|
||||
return GetMangaFromUrl($"https://asuracomic.net/series/{publicationId}");
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromUrl(string url)
|
||||
{
|
||||
RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return null;
|
||||
if (requestResult.htmlDocument is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, url.Split('/')[^1], url);
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] ParsePublicationsFromHtml(HtmlDocument document)
|
||||
{
|
||||
HtmlNodeCollection mangaList = document.DocumentNode.SelectNodes("//a[starts-with(@href,'series')]");
|
||||
if (mangaList is null || mangaList.Count < 1)
|
||||
return [];
|
||||
|
||||
IEnumerable<string> urls = mangaList.Select(a => $"https://asuracomic.net/{a.GetAttributeValue("href", "")}");
|
||||
|
||||
List<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)> ret = new();
|
||||
foreach (string url in urls)
|
||||
{
|
||||
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? manga = GetMangaFromUrl(url);
|
||||
if (manga is { } x)
|
||||
ret.Add(x);
|
||||
}
|
||||
|
||||
return ret.ToArray();
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?) ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl)
|
||||
{
|
||||
string? originalLanguage = null;
|
||||
Dictionary<string, string> altTitles = new(), links = new();
|
||||
|
||||
HtmlNodeCollection genreNodes = document.DocumentNode.SelectNodes("//h3[text()='Genres']/../div/button");
|
||||
string[] tags = genreNodes.Select(b => b.InnerText).ToArray();
|
||||
List<MangaTag> mangaTags = tags.Select(t => new MangaTag(t)).ToList();
|
||||
|
||||
HtmlNode statusNode = document.DocumentNode.SelectSingleNode("//h3[text()='Status']/../h3[2]");
|
||||
MangaReleaseStatus releaseStatus = statusNode.InnerText.ToLower() switch
|
||||
{
|
||||
"ongoing" => MangaReleaseStatus.Continuing,
|
||||
"hiatus" => MangaReleaseStatus.OnHiatus,
|
||||
"completed" => MangaReleaseStatus.Completed,
|
||||
"dropped" => MangaReleaseStatus.Cancelled,
|
||||
"season end" => MangaReleaseStatus.Continuing,
|
||||
"coming soon" => MangaReleaseStatus.Unreleased,
|
||||
_ => MangaReleaseStatus.Unreleased
|
||||
};
|
||||
|
||||
HtmlNode coverNode =
|
||||
document.DocumentNode.SelectSingleNode("//img[@alt='poster']");
|
||||
string coverUrl = coverNode.GetAttributeValue("src", "");
|
||||
|
||||
HtmlNode titleNode =
|
||||
document.DocumentNode.SelectSingleNode("//title");
|
||||
string sortName = Regex.Match(titleNode.InnerText, @"(.*) - Asura Scans").Groups[1].Value;
|
||||
|
||||
HtmlNode descriptionNode =
|
||||
document.DocumentNode.SelectSingleNode("//h3[starts-with(text(),'Synopsis')]/../span");
|
||||
string description = descriptionNode?.InnerText??"";
|
||||
|
||||
HtmlNodeCollection authorNodes = document.DocumentNode.SelectNodes("//h3[text()='Author']/../h3[not(text()='Author' or text()='_')]");
|
||||
HtmlNodeCollection artistNodes = document.DocumentNode.SelectNodes("//h3[text()='Artist']/../h3[not(text()='Artist' or text()='_')]");
|
||||
IEnumerable<string> authorNames = authorNodes is null ? [] : authorNodes.Select(a => a.InnerText);
|
||||
IEnumerable<string> artistNames = artistNodes is null ? [] : artistNodes.Select(a => a.InnerText);
|
||||
List<string> authorStrings = authorNames.Concat(artistNames).ToList();
|
||||
List<Author> authors = authorStrings.Select(author => new Author(author)).ToList();
|
||||
|
||||
HtmlNode? firstChapterNode = document.DocumentNode.SelectSingleNode("//a[contains(@href, 'chapter/1')]/../following-sibling::h3");
|
||||
uint year = uint.Parse(firstChapterNode?.InnerText.Split(' ')[^1] ?? "2000");
|
||||
|
||||
Manga manga = new (publicationId, sortName, description, websiteUrl, coverUrl, null, year,
|
||||
originalLanguage, releaseStatus, -1,
|
||||
this,
|
||||
authors,
|
||||
mangaTags,
|
||||
[],
|
||||
[]);
|
||||
|
||||
return (manga, authors, mangaTags, [], []);
|
||||
}
|
||||
|
||||
public override Chapter[] GetChapters(Manga manga, string language="en")
|
||||
{
|
||||
string requestUrl = $"https://asuracomic.net/series/{manga.MangaId}";
|
||||
// Leaving this in for verification if the page exists
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return [];
|
||||
|
||||
//Return Chapters ordered by Chapter-Number
|
||||
List<Chapter> chapters = ParseChaptersFromHtml(manga, requestUrl);
|
||||
return chapters.Order().ToArray();
|
||||
}
|
||||
|
||||
private List<Chapter> ParseChaptersFromHtml(Manga manga, string mangaUrl)
|
||||
{
|
||||
RequestResult result = downloadClient.MakeRequest(mangaUrl, RequestType.Default);
|
||||
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null)
|
||||
{
|
||||
return new List<Chapter>();
|
||||
}
|
||||
|
||||
List<Chapter> ret = new();
|
||||
|
||||
HtmlNodeCollection chapterURLNodes = result.htmlDocument.DocumentNode.SelectNodes("//a[contains(@href, '/chapter/')]");
|
||||
Regex infoRex = new(@"Chapter ([0-9]+)(.*)?");
|
||||
|
||||
foreach (HtmlNode chapterInfo in chapterURLNodes)
|
||||
{
|
||||
string chapterUrl = chapterInfo.GetAttributeValue("href", "");
|
||||
|
||||
Match match = infoRex.Match(chapterInfo.InnerText);
|
||||
string chapterNumber = new(match.Groups[1].Value);
|
||||
string? chapterName = match.Groups[2].Success && match.Groups[2].Length > 1 ? match.Groups[2].Value : null;
|
||||
string url = $"https://asuracomic.net/series/{chapterUrl}";
|
||||
try
|
||||
{
|
||||
ret.Add(new Chapter(manga, url, chapterNumber, null, chapterName));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
internal override string[] GetChapterImageUrls(Chapter chapter)
|
||||
{
|
||||
string requestUrl = chapter.Url;
|
||||
// Leaving this in to check if the page exists
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.htmlDocument is null)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
string[] imageUrls = ParseImageUrlsFromHtml(requestResult.htmlDocument);
|
||||
return imageUrls;
|
||||
}
|
||||
|
||||
private string[] ParseImageUrlsFromHtml(HtmlDocument document)
|
||||
{
|
||||
HtmlNodeCollection images = document.DocumentNode.SelectNodes("//img[contains(@alt, 'chapter page')]");
|
||||
|
||||
return images.Select(i => i.GetAttributeValue("src", "")).ToArray();
|
||||
}
|
||||
}
|
||||
@@ -1,203 +0,0 @@
|
||||
using System.Net;
|
||||
using System.Text.RegularExpressions;
|
||||
using API.MangaDownloadClients;
|
||||
using HtmlAgilityPack;
|
||||
|
||||
namespace API.Schema.MangaConnectors;
|
||||
|
||||
public class Bato : MangaConnector
|
||||
{
|
||||
|
||||
public Bato() : base("Bato", ["en"], ["bato.to"], "https://bato.to/amsta/img/batoto/favicon.ico")
|
||||
{
|
||||
this.downloadClient = new HttpDownloadClient();
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(string publicationTitle = "")
|
||||
{
|
||||
string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower();
|
||||
string requestUrl = $"https://bato.to/v3x-search?word={sanitizedTitle}&lang=en";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return [];
|
||||
|
||||
if (requestResult.htmlDocument is null)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument);
|
||||
return publications;
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(string publicationId)
|
||||
{
|
||||
return GetMangaFromUrl($"https://bato.to/title/{publicationId}");
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromUrl(string url)
|
||||
{
|
||||
RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return null;
|
||||
if (requestResult.htmlDocument is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, url.Split('/')[^1], url);
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] ParsePublicationsFromHtml(HtmlDocument document)
|
||||
{
|
||||
HtmlNode mangaList = document.DocumentNode.SelectSingleNode("//div[@data-hk='0-0-2']");
|
||||
if (!mangaList.ChildNodes.Any(node => node.Name == "div"))
|
||||
return [];
|
||||
|
||||
List<string> urls = mangaList.ChildNodes
|
||||
.Select(node => $"https://bato.to{node.Descendants("div").First().FirstChild.GetAttributeValue("href", "")}").ToList();
|
||||
|
||||
HashSet<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)> ret = new();
|
||||
foreach (string url in urls)
|
||||
{
|
||||
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? manga = GetMangaFromUrl(url);
|
||||
if (manga is { } x)
|
||||
ret.Add(x);
|
||||
}
|
||||
|
||||
return ret.ToArray();
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?) ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl)
|
||||
{
|
||||
HtmlNode infoNode = document.DocumentNode.SelectSingleNode("/html/body/div/main/div[1]/div[2]");
|
||||
|
||||
string sortName = infoNode.Descendants("h3").First().InnerText;
|
||||
string description = document.DocumentNode
|
||||
.SelectSingleNode("//div[contains(concat(' ',normalize-space(@class),' '),'prose')]").InnerText;
|
||||
|
||||
string[] altTitlesList = infoNode.ChildNodes[1].ChildNodes[2].InnerText.Split('/');
|
||||
int i = 0;
|
||||
List<MangaAltTitle> altTitles = altTitlesList.Select(a => new MangaAltTitle(i++.ToString(), a)).ToList();
|
||||
|
||||
string coverUrl = document.DocumentNode.SelectNodes("//img")
|
||||
.First(child => child.GetAttributeValue("data-hk", "") == "0-1-0").GetAttributeValue("src", "").Replace("&", "&");
|
||||
|
||||
List<HtmlNode> genreNodes = document.DocumentNode.SelectSingleNode("//b[text()='Genres:']/..").SelectNodes("span").ToList();
|
||||
string[] tags = genreNodes.Select(node => node.FirstChild.InnerText).ToArray();
|
||||
List<MangaTag> mangaTags = tags.Select(s => new MangaTag(s)).ToList();
|
||||
|
||||
List<HtmlNode> authorsNodes = infoNode.ChildNodes[1].ChildNodes[3].Descendants("a").ToList();
|
||||
List<string> authorNames = authorsNodes.Select(node => node.InnerText.Replace("amp;", "")).ToList();
|
||||
List<Author> authors = authorNames.Select(n => new Author(n)).ToList();
|
||||
|
||||
HtmlNode? originalLanguageNode = document.DocumentNode.SelectSingleNode("//span[text()='Tr From']/..");
|
||||
string originalLanguage = originalLanguageNode is not null ? originalLanguageNode.LastChild.InnerText : "";
|
||||
|
||||
if (!uint.TryParse(
|
||||
document.DocumentNode.SelectSingleNode("//span[text()='Original Publication:']/..").LastChild.InnerText.Split('-')[0],
|
||||
out uint year))
|
||||
year = (uint)DateTime.UtcNow.Year;
|
||||
|
||||
string status = document.DocumentNode.SelectSingleNode("//span[text()='Original Publication:']/..")
|
||||
.ChildNodes[2].InnerText;
|
||||
MangaReleaseStatus releaseStatus = MangaReleaseStatus.Unreleased;
|
||||
switch (status.ToLower())
|
||||
{
|
||||
case "ongoing": releaseStatus = MangaReleaseStatus.Continuing; break;
|
||||
case "completed": releaseStatus = MangaReleaseStatus.Completed; break;
|
||||
case "hiatus": releaseStatus = MangaReleaseStatus.OnHiatus; break;
|
||||
case "cancelled": releaseStatus = MangaReleaseStatus.Cancelled; break;
|
||||
case "pending": releaseStatus = MangaReleaseStatus.Unreleased; break;
|
||||
}
|
||||
|
||||
Manga manga = new (publicationId, sortName, description, websiteUrl, coverUrl, null, year,
|
||||
originalLanguage, releaseStatus, -1,
|
||||
this,
|
||||
authors,
|
||||
mangaTags,
|
||||
[],
|
||||
altTitles);
|
||||
|
||||
return (manga, authors, mangaTags, [], altTitles);
|
||||
}
|
||||
|
||||
public override Chapter[] GetChapters(Manga manga, string language="en")
|
||||
{
|
||||
string requestUrl = $"https://bato.to/title/{manga.MangaId}";
|
||||
// Leaving this in for verification if the page exists
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return [];
|
||||
|
||||
//Return Chapters ordered by Chapter-Number
|
||||
List<Chapter> chapters = ParseChaptersFromHtml(manga, requestUrl);
|
||||
return chapters.Order().ToArray();
|
||||
}
|
||||
|
||||
private List<Chapter> ParseChaptersFromHtml(Manga manga, string mangaUrl)
|
||||
{
|
||||
RequestResult result = downloadClient.MakeRequest(mangaUrl, RequestType.Default);
|
||||
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null)
|
||||
{
|
||||
return new List<Chapter>();
|
||||
}
|
||||
|
||||
List<Chapter> ret = new();
|
||||
|
||||
HtmlNode chapterList =
|
||||
result.htmlDocument.DocumentNode.SelectSingleNode("/html/body/div/main/div[3]/astro-island/div/div[2]/div/div/astro-slot");
|
||||
|
||||
Regex numberRex = new(@"\/title\/.+\/([0-9])+(?:-vol_([0-9]+))?-ch_([0-9\.]+)");
|
||||
|
||||
foreach (HtmlNode chapterInfo in chapterList.SelectNodes("div"))
|
||||
{
|
||||
HtmlNode infoNode = chapterInfo.FirstChild.FirstChild;
|
||||
string chapterUrl = infoNode.GetAttributeValue("href", "");
|
||||
|
||||
Match match = numberRex.Match(chapterUrl);
|
||||
string id = match.Groups[1].Value;
|
||||
int? volumeNumber = match.Groups[2].Success ? int.Parse(match.Groups[2].Value) : null;
|
||||
string chapterNumber = new(match.Groups[3].Value);
|
||||
string url = $"https://bato.to{chapterUrl}?load=2";
|
||||
try
|
||||
{
|
||||
ret.Add(new Chapter(manga, url, chapterNumber, volumeNumber, null));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
internal override string[] GetChapterImageUrls(Chapter chapter)
|
||||
{
|
||||
string requestUrl = chapter.Url;
|
||||
// Leaving this in to check if the page exists
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.htmlDocument is null)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
string[] imageUrls = ParseImageUrlsFromHtml(requestResult.htmlDocument);
|
||||
return imageUrls;
|
||||
}
|
||||
|
||||
private string[] ParseImageUrlsFromHtml(HtmlDocument document)
|
||||
{
|
||||
HtmlNode images = document.DocumentNode.SelectNodes("//astro-island").First(node =>
|
||||
node.GetAttributeValue("component-url", "").Contains("/_astro/ImageList."));
|
||||
|
||||
string weirdString = images.OuterHtml;
|
||||
string weirdString2 = Regex.Match(weirdString, @"props=\""(.*)}\""").Groups[1].Value;
|
||||
string[] urls = Regex.Matches(weirdString2, @"(https:\/\/[A-z\-0-9\.\?\&\;\=\/]+)\\")
|
||||
.Select(match => match.Groups[1].Value.Replace("&", "&")).ToArray();
|
||||
|
||||
return urls;
|
||||
}
|
||||
}
|
||||
@@ -8,15 +8,14 @@ public class Global : MangaConnector
|
||||
this.context = context;
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(string publicationTitle = "")
|
||||
public override Manga[] SearchManga(string mangaSearchName)
|
||||
{
|
||||
//Get all enabled Connectors
|
||||
MangaConnector[] enabledConnectors = context.MangaConnectors.Where(c => c.Enabled && c.Name != "Global").ToArray();
|
||||
|
||||
//Create Task for each MangaConnector to search simulatneously
|
||||
Task<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[]>[] tasks =
|
||||
enabledConnectors.Select(c =>
|
||||
new Task<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[]>(() => c.GetManga(publicationTitle))).ToArray();
|
||||
Task<Manga[]>[] tasks =
|
||||
enabledConnectors.Select(c => new Task<Manga[]>(() => c.SearchManga(mangaSearchName))).ToArray();
|
||||
foreach (var task in tasks)
|
||||
task.Start();
|
||||
|
||||
@@ -27,29 +26,28 @@ public class Global : MangaConnector
|
||||
}while(tasks.Any(t => t.Status < TaskStatus.RanToCompletion));
|
||||
|
||||
//Concatenate all results into one
|
||||
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] ret =
|
||||
tasks.Select(t => t.IsCompletedSuccessfully ? t.Result : []).ToArray().SelectMany(i => i).ToArray();
|
||||
Manga[] ret = tasks.Select(t => t.IsCompletedSuccessfully ? t.Result : []).ToArray().SelectMany(i => i).ToArray();
|
||||
return ret;
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromUrl(string url)
|
||||
public override Manga? GetMangaFromUrl(string url)
|
||||
{
|
||||
MangaConnector? mc = context.MangaConnectors.ToArray().FirstOrDefault(c => c.ValidateUrl(url));
|
||||
MangaConnector? mc = context.MangaConnectors.ToArray().FirstOrDefault(c => c.UrlMatchesConnector(url));
|
||||
return mc?.GetMangaFromUrl(url) ?? null;
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(string publicationId)
|
||||
public override Manga? GetMangaFromId(string mangaIdOnSite)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
public override Chapter[] GetChapters(Manga manga, string language = "en")
|
||||
public override Chapter[] GetChapters(Manga manga, string? language = null)
|
||||
{
|
||||
return manga.MangaConnector?.GetChapters(manga) ?? [];
|
||||
return manga.MangaConnector.GetChapters(manga, language);
|
||||
}
|
||||
|
||||
internal override string[] GetChapterImageUrls(Chapter chapter)
|
||||
{
|
||||
return chapter.ParentManga?.MangaConnector?.GetChapterImageUrls(chapter) ?? [];
|
||||
return chapter.ParentManga.MangaConnector.GetChapterImageUrls(chapter);
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,14 @@ namespace API.Schema.MangaConnectors;
|
||||
[PrimaryKey("Name")]
|
||||
public abstract class MangaConnector(string name, string[] supportedLanguages, string[] baseUris, string iconUrl)
|
||||
{
|
||||
[JsonIgnore]
|
||||
[NotMapped]
|
||||
internal DownloadClient downloadClient { get; init; } = null!;
|
||||
|
||||
[JsonIgnore]
|
||||
[NotMapped]
|
||||
protected ILog Log { get; init; } = LogManager.GetLogger(name);
|
||||
|
||||
[StringLength(32)]
|
||||
[Required]
|
||||
public string Name { get; init; } = name;
|
||||
@@ -26,32 +34,41 @@ public abstract class MangaConnector(string name, string[] supportedLanguages, s
|
||||
[Required]
|
||||
public bool Enabled { get; internal set; } = true;
|
||||
|
||||
public abstract (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(string publicationTitle = "");
|
||||
public abstract Manga[] SearchManga(string mangaSearchName);
|
||||
|
||||
public abstract (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromUrl(string url);
|
||||
public abstract Manga? GetMangaFromUrl(string url);
|
||||
|
||||
public abstract (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(string publicationId);
|
||||
public abstract Manga? GetMangaFromId(string mangaIdOnSite);
|
||||
|
||||
public abstract Chapter[] GetChapters(Manga manga, string language="en");
|
||||
|
||||
[JsonIgnore]
|
||||
[NotMapped]
|
||||
internal DownloadClient downloadClient { get; init; } = null!;
|
||||
|
||||
[JsonIgnore]
|
||||
[NotMapped]
|
||||
protected ILog Log { get; init; } = LogManager.GetLogger(name);
|
||||
|
||||
public Chapter[] GetNewChapters(Manga manga)
|
||||
{
|
||||
Chapter[] allChapters = GetChapters(manga);
|
||||
if (allChapters.Length < 1)
|
||||
return [];
|
||||
|
||||
return allChapters.Where(chapter => !chapter.IsDownloaded()).ToArray();
|
||||
}
|
||||
public abstract Chapter[] GetChapters(Manga manga, string? language = null);
|
||||
|
||||
internal abstract string[] GetChapterImageUrls(Chapter chapter);
|
||||
|
||||
public bool ValidateUrl(string url) => BaseUris.Any(baseUri => Regex.IsMatch(url, "https?://" + baseUri + "/.*"));
|
||||
public bool UrlMatchesConnector(string url) => BaseUris.Any(baseUri => Regex.IsMatch(url, "https?://" + baseUri + "/.*"));
|
||||
|
||||
internal string? SaveCoverImageToCache(Manga manga, int retries = 3)
|
||||
{
|
||||
if(retries < 0)
|
||||
return null;
|
||||
|
||||
Regex urlRex = new (@"https?:\/\/((?:[a-zA-Z0-9-]+\.)+[a-zA-Z0-9]+)\/(?:.+\/)*(.+\.([a-zA-Z]+))");
|
||||
//https?:\/\/[a-zA-Z0-9-]+\.([a-zA-Z0-9-]+\.[a-zA-Z0-9]+)\/(?:.+\/)*(.+\.([a-zA-Z]+)) for only second level domains
|
||||
Match match = urlRex.Match(manga.CoverUrl);
|
||||
string filename = $"{match.Groups[1].Value}-{manga.MangaId}.{match.Groups[3].Value}";
|
||||
string saveImagePath = Path.Join(TrangaSettings.coverImageCache, filename);
|
||||
|
||||
if (File.Exists(saveImagePath))
|
||||
return saveImagePath;
|
||||
|
||||
RequestResult coverResult = downloadClient.MakeRequest(manga.CoverUrl, RequestType.MangaCover, $"https://{match.Groups[1].Value}");
|
||||
if ((int)coverResult.statusCode < 200 || (int)coverResult.statusCode >= 300)
|
||||
return SaveCoverImageToCache(manga, --retries);
|
||||
|
||||
using MemoryStream ms = new();
|
||||
coverResult.result.CopyTo(ms);
|
||||
Directory.CreateDirectory(TrangaSettings.coverImageCache);
|
||||
File.WriteAllBytes(saveImagePath, ms.ToArray());
|
||||
|
||||
return saveImagePath;
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,6 @@
|
||||
using System.Net;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Text.RegularExpressions;
|
||||
using API.MangaDownloadClients;
|
||||
using JsonSerializer = System.Text.Json.JsonSerializer;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace API.Schema.MangaConnectors;
|
||||
|
||||
@@ -11,313 +9,327 @@ public class MangaDex : MangaConnector
|
||||
//https://api.mangadex.org/docs/3-enumerations/#language-codes--localization
|
||||
//https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes
|
||||
//https://gist.github.com/Josantonius/b455e315bc7f790d14b136d61d9ae469
|
||||
public MangaDex() : base("MangaDex", ["en","pt","pt-br","it","de","ru","aa","ab","ae","af","ak","am","an","ar-ae","ar-bh","ar-dz","ar-eg","ar-iq","ar-jo","ar-kw","ar-lb","ar-ly","ar-ma","ar-om","ar-qa","ar-sa","ar-sy","ar-tn","ar-ye","ar","as","av","ay","az","ba","be","bg","bh","bi","bm","bn","bo","br","bs","ca","ce","ch","co","cr","cs","cu","cv","cy","da","de-at","de-ch","de-de","de-li","de-lu","div","dv","dz","ee","el","en-au","en-bz","en-ca","en-cb","en-gb","en-ie","en-jm","en-nz","en-ph","en-tt","en-us","en-za","en-zw","eo","es-ar","es-bo","es-cl","es-co","es-cr","es-do","es-ec","es-es","es-gt","es-hn","es-la","es-mx","es-ni","es-pa","es-pe","es-pr","es-py","es-sv","es-us","es-uy","es-ve","es","et","eu","fa","ff","fi","fj","fo","fr-be","fr-ca","fr-ch","fr-fr","fr-lu","fr-mc","fr","fy","ga","gd","gl","gn","gu","gv","ha","he","hi","ho","hr-ba","hr-hr","hr","ht","hu","hy","hz","ia","id","ie","ig","ii","ik","in","io","is","it-ch","it-it","iu","iw","ja","ja-ro","ji","jv","jw","ka","kg","ki","kj","kk","kl","km","kn","ko","ko-ro","kr","ks","ku","kv","kw","ky","kz","la","lb","lg","li","ln","lo","ls","lt","lu","lv","mg","mh","mi","mk","ml","mn","mo","mr","ms-bn","ms-my","ms","mt","my","na","nb","nd","ne","ng","nl-be","nl-nl","nl","nn","no","nr","ns","nv","ny","oc","oj","om","or","os","pa","pi","pl","ps","pt-pt","qu-bo","qu-ec","qu-pe","qu","rm","rn","ro","rw","sa","sb","sc","sd","se-fi","se-no","se-se","se","sg","sh","si","sk","sl","sm","sn","so","sq","sr-ba","sr-sp","sr","ss","st","su","sv-fi","sv-se","sv","sw","sx","syr","ta","te","tg","th","ti","tk","tl","tn","to","tr","ts","tt","tw","ty","ug","uk","ur","us","uz","ve","vi","vo","wa","wo","xh","yi","yo","za","zh-cn","zh-hk","zh-mo","zh-ro","zh-sg","zh-tw","zh","zu"], ["mangadex.org"], "https://mangadex.org/favicon.ico")
|
||||
public MangaDex() : base("MangaDex",
|
||||
["en","pt","pt-br","it","de","ru","aa","ab","ae","af","ak","am","an","ar-ae","ar-bh","ar-dz","ar-eg","ar-iq","ar-jo","ar-kw","ar-lb","ar-ly","ar-ma","ar-om","ar-qa","ar-sa","ar-sy","ar-tn","ar-ye","ar","as","av","ay","az","ba","be","bg","bh","bi","bm","bn","bo","br","bs","ca","ce","ch","co","cr","cs","cu","cv","cy","da","de-at","de-ch","de-de","de-li","de-lu","div","dv","dz","ee","el","en-au","en-bz","en-ca","en-cb","en-gb","en-ie","en-jm","en-nz","en-ph","en-tt","en-us","en-za","en-zw","eo","es-ar","es-bo","es-cl","es-co","es-cr","es-do","es-ec","es-es","es-gt","es-hn","es-la","es-mx","es-ni","es-pa","es-pe","es-pr","es-py","es-sv","es-us","es-uy","es-ve","es","et","eu","fa","ff","fi","fj","fo","fr-be","fr-ca","fr-ch","fr-fr","fr-lu","fr-mc","fr","fy","ga","gd","gl","gn","gu","gv","ha","he","hi","ho","hr-ba","hr-hr","hr","ht","hu","hy","hz","ia","id","ie","ig","ii","ik","in","io","is","it-ch","it-it","iu","iw","ja","ja-ro","ji","jv","jw","ka","kg","ki","kj","kk","kl","km","kn","ko","ko-ro","kr","ks","ku","kv","kw","ky","kz","la","lb","lg","li","ln","lo","ls","lt","lu","lv","mg","mh","mi","mk","ml","mn","mo","mr","ms-bn","ms-my","ms","mt","my","na","nb","nd","ne","ng","nl-be","nl-nl","nl","nn","no","nr","ns","nv","ny","oc","oj","om","or","os","pa","pi","pl","ps","pt-pt","qu-bo","qu-ec","qu-pe","qu","rm","rn","ro","rw","sa","sb","sc","sd","se-fi","se-no","se-se","se","sg","sh","si","sk","sl","sm","sn","so","sq","sr-ba","sr-sp","sr","ss","st","su","sv-fi","sv-se","sv","sw","sx","syr","ta","te","tg","th","ti","tk","tl","tn","to","tr","ts","tt","tw","ty","ug","uk","ur","us","uz","ve","vi","vo","wa","wo","xh","yi","yo","za","zh-cn","zh-hk","zh-mo","zh-ro","zh-sg","zh-tw","zh","zu"],
|
||||
["mangadex.org"],
|
||||
"https://mangadex.org/favicon.ico")
|
||||
{
|
||||
this.downloadClient = new HttpDownloadClient();
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(string publicationTitle = "")
|
||||
private const int Limit = 100;
|
||||
public override Manga[] SearchManga(string mangaSearchName)
|
||||
{
|
||||
const int limit = 100; //How many values we want returned at once
|
||||
int offset = 0; //"Page"
|
||||
int total = int.MaxValue; //How many total results are there, is updated on first request
|
||||
HashSet<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)> retManga = new();
|
||||
List<JsonNode> results = new();
|
||||
Log.Info($"Searching Manga: {mangaSearchName}");
|
||||
List<Manga> mangas = new ();
|
||||
|
||||
//Request all search-results
|
||||
while (offset < total) //As long as we haven't requested all "Pages"
|
||||
int offset = 0;
|
||||
int total = int.MaxValue;
|
||||
while(offset < total)
|
||||
{
|
||||
//Request next Page
|
||||
string requestUrl =
|
||||
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}" +
|
||||
$"&includes[]=manga&includes[]=cover_art&includes[]=author&includes[]=artist&includes[]=tag";
|
||||
RequestResult requestResult = downloadClient.MakeRequest(requestUrl, RequestType.MangaInfo);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
{
|
||||
Log.Info($"{requestResult.statusCode}: {requestUrl}");
|
||||
break;
|
||||
}
|
||||
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
|
||||
|
||||
offset += limit;
|
||||
if (result is null)
|
||||
{
|
||||
Log.Info($"result was null: {requestUrl}");
|
||||
break;
|
||||
}
|
||||
|
||||
if(result.ContainsKey("total"))
|
||||
total = result["total"]!.GetValue<int>(); //Update the total number of Publications
|
||||
else continue;
|
||||
$"https://api.mangadex.org/manga?limit={Limit}&offset={offset}&title={mangaSearchName}" +
|
||||
$"&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica" +
|
||||
$"&includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author&includes%5B%5D=artist&includes%5B%5D=tag'";
|
||||
offset += Limit;
|
||||
|
||||
if (result.ContainsKey("data"))
|
||||
results.AddRange(result["data"]!.AsArray()!);//Manga-data-Array
|
||||
RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed);
|
||||
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
|
||||
{
|
||||
Log.Error("Request failed");
|
||||
return [];
|
||||
}
|
||||
|
||||
using StreamReader sr = new (result.result);
|
||||
JObject jObject = JObject.Parse(sr.ReadToEnd());
|
||||
|
||||
if (jObject.Value<string>("result") != "ok")
|
||||
{
|
||||
JArray? errors = jObject["errors"] as JArray;
|
||||
Log.Error($"Request failed: {string.Join(',', errors?.Select(e => e.Value<string>("title")) ?? [])}");
|
||||
return [];
|
||||
}
|
||||
|
||||
total = jObject.Value<int>("total");
|
||||
|
||||
JArray? data = jObject.Value<JArray>("data");
|
||||
if (data is null)
|
||||
{
|
||||
Log.Error("Data was null");
|
||||
return [];
|
||||
}
|
||||
|
||||
mangas.AddRange(data.Select(ParseMangaFromJToken));
|
||||
}
|
||||
|
||||
foreach (JsonNode mangaNode in results)
|
||||
{
|
||||
if(MangaFromJsonObject(mangaNode.AsObject()) is { } manga)
|
||||
retManga.Add(manga); //Add Publication (Manga) to result
|
||||
}
|
||||
return retManga.ToArray();
|
||||
Log.Info($"Search {mangaSearchName} yielded {mangas.Count} results.");
|
||||
return mangas.ToArray();
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(string publicationId)
|
||||
private static readonly Regex GetMangaIdFromUrl = new(@"https?:\/\/mangadex\.org\/title\/([a-z0-9-]+)\/?.*");
|
||||
public override Manga? GetMangaFromUrl(string url)
|
||||
{
|
||||
string url = $"https://api.mangadex.org/manga/{publicationId}" +
|
||||
$"?includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author&includes%5B%5D=artist&includes%5B%5D=tag";
|
||||
RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
Log.Info($"Getting Manga: {url}");
|
||||
if (!UrlMatchesConnector(url))
|
||||
{
|
||||
Log.Info($"{requestResult.statusCode}: {url}");
|
||||
Log.Debug($"Url is not for Connector. {url}");
|
||||
return null;
|
||||
}
|
||||
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
|
||||
if(result is not null)
|
||||
return MangaFromJsonObject(result["data"]!.AsObject());
|
||||
Log.Info($"result was null: {url}");
|
||||
return null;
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromUrl(string url)
|
||||
{
|
||||
Regex idRex = new (@"https:\/\/mangadex.org\/title\/([A-z0-9-]*)\/.*");
|
||||
string id = idRex.Match(url).Groups[1].Value;
|
||||
Match match = GetMangaIdFromUrl.Match(url);
|
||||
if (!match.Success || !match.Groups[1].Success)
|
||||
{
|
||||
Log.Debug($"Url is not for Connector (Could not retrieve id). {url}");
|
||||
return null;
|
||||
}
|
||||
string id = match.Groups[1].Value;
|
||||
|
||||
return GetMangaFromId(id);
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? MangaFromJsonObject(JsonObject manga)
|
||||
public override Manga? GetMangaFromId(string mangaIdOnSite)
|
||||
{
|
||||
if (!manga.TryGetPropertyValue("id", out JsonNode? idNode))
|
||||
Log.Info($"Getting Manga: {mangaIdOnSite}");
|
||||
string requestUrl =
|
||||
$"https://api.mangadex.org/manga/{mangaIdOnSite}" +
|
||||
$"?includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author&includes%5B%5D=artist&includes%5B%5D=tag'";
|
||||
|
||||
RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed);
|
||||
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
|
||||
{
|
||||
Log.Info("id was null");
|
||||
Log.Error("Request failed");
|
||||
return null;
|
||||
}
|
||||
string publicationId = idNode!.GetValue<string>();
|
||||
|
||||
if (!manga.TryGetPropertyValue("attributes", out JsonNode? attributesNode))
|
||||
{
|
||||
Log.Info("attributes was null");
|
||||
return null;
|
||||
}
|
||||
JsonObject attributes = attributesNode!.AsObject();
|
||||
|
||||
if (!attributes.TryGetPropertyValue("title", out JsonNode? titleNode))
|
||||
{
|
||||
Log.Info("title was null");
|
||||
return null;
|
||||
}
|
||||
string sortName = titleNode!.AsObject().ContainsKey("en") switch
|
||||
{
|
||||
true => titleNode.AsObject()["en"]!.GetValue<string>(),
|
||||
false => titleNode.AsObject().First().Value!.GetValue<string>()
|
||||
};
|
||||
|
||||
Dictionary<string, string> altTitlesDict = new();
|
||||
if (attributes.TryGetPropertyValue("altTitles", out JsonNode? altTitlesNode))
|
||||
{
|
||||
foreach (JsonNode? altTitleNode in altTitlesNode!.AsArray())
|
||||
{
|
||||
JsonObject altTitleNodeObject = altTitleNode!.AsObject();
|
||||
altTitlesDict.TryAdd(altTitleNodeObject.First().Key, altTitleNodeObject.First().Value!.GetValue<string>());
|
||||
}
|
||||
}
|
||||
List<MangaAltTitle> altTitles = altTitlesDict.Select(t => new MangaAltTitle(t.Key, t.Value)).ToList();
|
||||
using StreamReader sr = new (result.result);
|
||||
JObject jObject = JObject.Parse(sr.ReadToEnd());
|
||||
|
||||
if (!attributes.TryGetPropertyValue("description", out JsonNode? descriptionNode))
|
||||
if (jObject.Value<string>("result") != "ok")
|
||||
{
|
||||
Log.Info("description was null");
|
||||
return null;
|
||||
}
|
||||
string description = descriptionNode!.AsObject().ContainsKey("en") switch
|
||||
{
|
||||
true => descriptionNode.AsObject()["en"]!.GetValue<string>(),
|
||||
false => descriptionNode.AsObject().FirstOrDefault().Value?.GetValue<string>() ?? ""
|
||||
};
|
||||
|
||||
Dictionary<string, string> linksDict = new();
|
||||
if (attributes.TryGetPropertyValue("links", out JsonNode? linksNode) && linksNode is not null)
|
||||
foreach (KeyValuePair<string, JsonNode?> linkKv in linksNode!.AsObject())
|
||||
linksDict.TryAdd(linkKv.Key, linkKv.Value.GetValue<string>());
|
||||
List<Link> links = linksDict.Select(x => new Link(x.Key, x.Value)).ToList();
|
||||
|
||||
string? originalLanguage =
|
||||
attributes.TryGetPropertyValue("originalLanguage", out JsonNode? originalLanguageNode) switch
|
||||
{
|
||||
true => originalLanguageNode?.GetValue<string>(),
|
||||
false => null
|
||||
};
|
||||
|
||||
MangaReleaseStatus releaseStatus = MangaReleaseStatus.Unreleased;
|
||||
if (attributes.TryGetPropertyValue("status", out JsonNode? statusNode))
|
||||
{
|
||||
releaseStatus = statusNode?.GetValue<string>().ToLower() switch
|
||||
{
|
||||
"ongoing" => MangaReleaseStatus.Continuing,
|
||||
"completed" => MangaReleaseStatus.Completed,
|
||||
"hiatus" => MangaReleaseStatus.OnHiatus,
|
||||
"cancelled" => MangaReleaseStatus.Cancelled,
|
||||
_ => MangaReleaseStatus.Unreleased
|
||||
};
|
||||
}
|
||||
|
||||
uint year = attributes.TryGetPropertyValue("year", out JsonNode? yearNode) switch
|
||||
{
|
||||
true => yearNode?.GetValue<uint>()??0,
|
||||
false => 0
|
||||
};
|
||||
|
||||
HashSet<string> tags = new(128);
|
||||
if (attributes.TryGetPropertyValue("tags", out JsonNode? tagsNode))
|
||||
foreach (JsonNode? tagNode in tagsNode!.AsArray())
|
||||
tags.Add(tagNode!["attributes"]!["name"]!["en"]!.GetValue<string>());
|
||||
List<MangaTag> mangaTags = tags.Select(t => new MangaTag(t)).ToList();
|
||||
|
||||
if (!manga.TryGetPropertyValue("relationships", out JsonNode? relationshipsNode))
|
||||
{
|
||||
Log.Info("relationships was null");
|
||||
JArray? errors = jObject["errors"] as JArray;
|
||||
Log.Error($"Request failed: {string.Join(',', errors?.Select(e => e.Value<string>("title")) ?? [])}");
|
||||
return null;
|
||||
}
|
||||
|
||||
JsonNode? coverNode = relationshipsNode!.AsArray()
|
||||
.FirstOrDefault(rel => rel!["type"]!.GetValue<string>().Equals("cover_art"));
|
||||
if (coverNode is null)
|
||||
JObject? data = jObject["data"] as JObject;
|
||||
if (data is null)
|
||||
{
|
||||
Log.Info("coverNode was null");
|
||||
Log.Error("Data was null");
|
||||
return null;
|
||||
}
|
||||
string fileName = coverNode["attributes"]!["fileName"]!.GetValue<string>();
|
||||
string coverUrl = $"https://uploads.mangadex.org/covers/{publicationId}/{fileName}";
|
||||
|
||||
List<string> authorNames = new();
|
||||
JsonNode?[] authorNodes = relationshipsNode.AsArray()
|
||||
.Where(rel => rel!["type"]!.GetValue<string>().Equals("author") || rel!["type"]!.GetValue<string>().Equals("artist")).ToArray();
|
||||
foreach (JsonNode? authorNode in authorNodes)
|
||||
{
|
||||
string authorName = authorNode!["attributes"]!["name"]!.GetValue<string>();
|
||||
if(!authorNames.Contains(authorName))
|
||||
authorNames.Add(authorName);
|
||||
}
|
||||
List<Author> authors = authorNames.Select(a => new Author(a)).ToList();
|
||||
|
||||
Manga pub = new (publicationId, sortName, description, $"https://mangadex.org/title/{publicationId}", coverUrl, null, year,
|
||||
originalLanguage, releaseStatus, -1,
|
||||
this,
|
||||
authors,
|
||||
mangaTags,
|
||||
links,
|
||||
altTitles);
|
||||
|
||||
return (pub, authors, mangaTags, links, altTitles);
|
||||
Manga manga = ParseMangaFromJToken(data);
|
||||
return manga;
|
||||
}
|
||||
|
||||
public override Chapter[] GetChapters(Manga manga, string language="en")
|
||||
public override Chapter[] GetChapters(Manga manga, string? language = null)
|
||||
{
|
||||
const int limit = 100; //How many values we want returned at once
|
||||
int offset = 0; //"Page"
|
||||
int total = int.MaxValue; //How many total results are there, is updated on first request
|
||||
List<Chapter> chapters = new();
|
||||
//As long as we haven't requested all "Pages"
|
||||
while (offset < total)
|
||||
Log.Info($"Getting Chapters: {manga.IdOnConnectorSite}");
|
||||
List<Chapter> chapters = new ();
|
||||
|
||||
int offset = 0;
|
||||
int total = int.MaxValue;
|
||||
while(offset < total)
|
||||
{
|
||||
//Request next "Page"
|
||||
string requestUrl = $"https://api.mangadex.org/manga/{manga.IdOnConnectorSite}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}" +
|
||||
$"&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&contentRating%5B%5D=pornographic";
|
||||
RequestResult requestResult = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
{
|
||||
Log.Info($"{requestResult.statusCode}: {requestUrl}");
|
||||
break;
|
||||
}
|
||||
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
|
||||
|
||||
offset += limit;
|
||||
if (result is null)
|
||||
{
|
||||
Log.Info($"result was null: {requestUrl}");
|
||||
break;
|
||||
}
|
||||
|
||||
total = result["total"]!.GetValue<int>();
|
||||
JsonArray chaptersInResult = result["data"]!.AsArray();
|
||||
//Loop through all Chapters in result and extract information from JSON
|
||||
foreach (JsonNode? jsonNode in chaptersInResult)
|
||||
{
|
||||
JsonObject chapter = (JsonObject)jsonNode!;
|
||||
JsonObject attributes = chapter["attributes"]!.AsObject();
|
||||
|
||||
string chapterId = chapter["id"]!.GetValue<string>();
|
||||
string url = $"https://mangadex.org/chapter/{chapterId}";
|
||||
|
||||
string? title = attributes.ContainsKey("title") && attributes["title"] is not null
|
||||
? attributes["title"]!.GetValue<string>()
|
||||
: null;
|
||||
|
||||
int? volume = attributes.ContainsKey("volume") && attributes["volume"] is not null
|
||||
? int.Parse(attributes["volume"]!.GetValue<string>())
|
||||
: null;
|
||||
|
||||
string? chapterNumStr = attributes.ContainsKey("chapter") && attributes["chapter"] is not null
|
||||
? attributes["chapter"]!.GetValue<string>()
|
||||
: null;
|
||||
|
||||
string chapterNumber = new(chapterNumStr);
|
||||
|
||||
|
||||
if (attributes.ContainsKey("pages") && attributes["pages"] is not null &&
|
||||
attributes["pages"]!.GetValue<int>() < 1)
|
||||
{
|
||||
Log.Info($"No pages: {chapterId}");
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Chapter newChapter = new(manga, url, chapterNumber, volume, title);
|
||||
if(!chapters.Contains(newChapter))
|
||||
chapters.Add(newChapter);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log.Debug(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
string requestUrl =
|
||||
$"https://api.mangadex.org/manga/{manga.IdOnConnectorSite}/feed?limit={Limit}&offset={offset}&" +
|
||||
$"translatedLanguage%5B%5D={language}&" +
|
||||
$"contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&includeFutureUpdates=0&includes%5B%5D=";
|
||||
offset += Limit;
|
||||
|
||||
//Return Chapters ordered by Chapter-Number
|
||||
RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed);
|
||||
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
|
||||
{
|
||||
Log.Error("Request failed");
|
||||
return [];
|
||||
}
|
||||
|
||||
using StreamReader sr = new (result.result);
|
||||
JObject jObject = JObject.Parse(sr.ReadToEnd());
|
||||
|
||||
if (jObject.Value<string>("result") != "ok")
|
||||
{
|
||||
JArray? errors = jObject["errors"] as JArray;
|
||||
Log.Error($"Request failed: {string.Join(',', errors?.Select(e => e.Value<string>("title")) ?? [])}");
|
||||
return [];
|
||||
}
|
||||
|
||||
total = jObject.Value<int>("total");
|
||||
|
||||
JArray? data = jObject.Value<JArray>("data");
|
||||
if (data is null)
|
||||
{
|
||||
Log.Error("Data was null");
|
||||
return [];
|
||||
}
|
||||
|
||||
chapters.AddRange(data.Select(d => ParseChapterFromJToken(manga, d)));
|
||||
}
|
||||
|
||||
Log.Info($"Request for chapters for {manga.Name} yielded {chapters.Count} results.");
|
||||
return chapters.ToArray();
|
||||
}
|
||||
|
||||
private static readonly Regex GetChapterIdFromUrl = new(@"https?:\/\/mangadex\.org\/chapter\/([a-z0-9-]+)\/?.*");
|
||||
internal override string[] GetChapterImageUrls(Chapter chapter)
|
||||
{//Request URLs for Chapter-Images
|
||||
Match m = Regex.Match(chapter.Url, @"https?:\/\/mangadex.org\/chapter\/([0-9\-a-z]+)");
|
||||
if (!m.Success)
|
||||
{
|
||||
Log.Info($"Getting Chapter Image-Urls: {chapter.Url}");
|
||||
if (!UrlMatchesConnector(chapter.Url))
|
||||
{
|
||||
Log.Error($"Could not parse Chapter ID: {chapter.Url}");
|
||||
Log.Debug($"Url is not for Connector. {chapter.Url}");
|
||||
return [];
|
||||
}
|
||||
|
||||
string url = $"https://api.mangadex.org/at-home/server/{m.Groups[1].Value}?forcePort443=false";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(url, RequestType.MangaDexImage);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
Match match = GetChapterIdFromUrl.Match(chapter.Url);
|
||||
if (!match.Success || !match.Groups[1].Success)
|
||||
{
|
||||
Log.Info($"{requestResult.statusCode}: {url}");
|
||||
Log.Debug($"Url is not for Connector (Could not retrieve id). {chapter.Url}");
|
||||
return [];
|
||||
}
|
||||
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
|
||||
if (result is null)
|
||||
|
||||
string id = match.Groups[1].Value;
|
||||
string requestUrl = $"https://api.mangadex.org/at-home/server/{id}";
|
||||
|
||||
RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
|
||||
{
|
||||
Log.Info($"Result was null: {url}");
|
||||
Log.Error("Request failed");
|
||||
return [];
|
||||
}
|
||||
string baseUrl = result["baseUrl"]!.GetValue<string>();
|
||||
string hash = result["chapter"]!["hash"]!.GetValue<string>();
|
||||
JsonArray imageFileNames = result["chapter"]!["data"]!.AsArray();
|
||||
//Loop through all imageNames and construct urls (imageUrl)
|
||||
List<string> imageUrls = new();
|
||||
foreach (JsonNode? image in imageFileNames)
|
||||
imageUrls.Add($"{baseUrl}/data/{hash}/{image!.GetValue<string>()}");
|
||||
return imageUrls.ToArray();
|
||||
|
||||
using StreamReader sr = new (result.result);
|
||||
JObject jObject = JObject.Parse(sr.ReadToEnd());
|
||||
|
||||
if (jObject.Value<string>("result") != "ok")
|
||||
{
|
||||
JArray? errors = jObject["errors"] as JArray;
|
||||
Log.Error($"Request failed: {string.Join(',', errors?.Select(e => e.Value<string>("title")) ?? [])}");
|
||||
return [];
|
||||
}
|
||||
|
||||
string? baseUrl = jObject.Value<string>("baseUrl");
|
||||
JToken? chapterToken = jObject["chapter"];
|
||||
string? hash = chapterToken?.Value<string>("hash");
|
||||
JArray? data = chapterToken?["data"] as JArray;
|
||||
|
||||
if (baseUrl is null || hash is null || data is null)
|
||||
{
|
||||
Log.Error("Data was null");
|
||||
return [];
|
||||
}
|
||||
|
||||
IEnumerable<string> urls = data.Select(t => $"{baseUrl}/data/{hash}/{t.Value<string>()}");
|
||||
|
||||
return urls.ToArray();
|
||||
}
|
||||
|
||||
private Manga ParseMangaFromJToken(JToken jToken)
|
||||
{
|
||||
string? id = jToken.Value<string>("id");
|
||||
|
||||
JObject? attributes = jToken["attributes"] as JObject;
|
||||
string? name = attributes?["title"]?.Value<string>("en");
|
||||
string? description = attributes?["description"]?.Value<string>("en");
|
||||
string? status = attributes?["status"]?.Value<string>();
|
||||
uint? year = attributes?["year"]?.Value<uint>();
|
||||
string? originalLanguage = attributes?["originalLanguage"]?.Value<string>();
|
||||
JArray? altTitlesJArray = attributes?["altTitles"] as JArray;
|
||||
JArray? tagsJArray = attributes?["tags"] as JArray;
|
||||
|
||||
JArray? relationships = jToken["relationships"] as JArray;
|
||||
string? coverFileName =
|
||||
relationships?.FirstOrDefault(r => r["type"]?.Value<string>() == "cover_art")?["attributes"]?.Value<string>("fileName");
|
||||
|
||||
if (id is null || attributes is null || name is null || description is null || status is null ||
|
||||
altTitlesJArray is null || tagsJArray is null || relationships is null || coverFileName is null)
|
||||
throw new Exception("jToken was not in expected format");
|
||||
|
||||
List<Link> links = attributes["links"]?
|
||||
.ToObject<Dictionary<string,string>>()?
|
||||
.Select(kv =>
|
||||
{
|
||||
//https://api.mangadex.org/docs/3-enumerations/#manga-links-data
|
||||
string url = kv.Key switch
|
||||
{
|
||||
"al" => $"https://anilist.co/manga/{kv.Value}",
|
||||
"ap" => $"https://www.anime-planet.com/manga/{kv.Value}",
|
||||
"bw" => $"https://bookwalker.jp/{kv.Value}",
|
||||
"mu" => $"https://www.mangaupdates.com/series.html?id={kv.Value}",
|
||||
"nu" => $"https://www.novelupdates.com/series/{kv.Value}",
|
||||
"mal" => $"https://myanimelist.net/manga/{kv.Value}",
|
||||
_ => kv.Value
|
||||
};
|
||||
string key = kv.Key switch
|
||||
{
|
||||
"al" => "AniList",
|
||||
"ap" => "Anime Planet",
|
||||
"bw" => "BookWalker",
|
||||
"mu" => "Manga Updates",
|
||||
"nu" => "Novel Updates",
|
||||
"kt" => "Kitsu.io",
|
||||
"amz" => "Amazon",
|
||||
"ebj" => "eBookJapan",
|
||||
"mal" => "MyAnimeList",
|
||||
"cdj" => "CDJapan",
|
||||
_ => kv.Key
|
||||
};
|
||||
return new Link(key, url);
|
||||
}).ToList()!;
|
||||
|
||||
List<MangaAltTitle> altTitles = altTitlesJArray
|
||||
.Select(t =>
|
||||
{
|
||||
JObject? j = t as JObject;
|
||||
JProperty? p = j?.Properties().First();
|
||||
if (p is null)
|
||||
return null;
|
||||
return new MangaAltTitle(p.Name, p.Value.ToString());
|
||||
}).Where(x => x is not null).ToList()!;
|
||||
|
||||
List<MangaTag> tags = tagsJArray
|
||||
.Where(t => t.Value<string>("type") == "tag")
|
||||
.Select(t => t["attributes"]?["name"]?.Value<string>("en"))
|
||||
.Select(str => str is not null ? new MangaTag(str) : null)
|
||||
.Where(x => x is not null).ToList()!;
|
||||
|
||||
List<Author> authors = relationships
|
||||
.Where(r => r["type"]?.Value<string>() == "author")
|
||||
.Select(t => t["attributes"]?.Value<string>("name"))
|
||||
.Select(str => str is not null ? new Author(str) : null)
|
||||
.Where(x => x is not null).ToList()!;
|
||||
|
||||
|
||||
MangaReleaseStatus releaseStatus = status switch
|
||||
{
|
||||
"completed" => MangaReleaseStatus.Completed,
|
||||
"ongoing" => MangaReleaseStatus.Continuing,
|
||||
"cancelled" => MangaReleaseStatus.Cancelled,
|
||||
"hiatus" => MangaReleaseStatus.OnHiatus,
|
||||
_ => MangaReleaseStatus.Unreleased
|
||||
};
|
||||
string websiteUrl = $"https://mangadex.org/title/{id}";
|
||||
string coverUrl = $"https://uploads.mangadex.org/covers/{id}/{coverFileName}";
|
||||
|
||||
return new Manga(id, name, description, websiteUrl, coverUrl, releaseStatus, this,
|
||||
authors, tags, links,altTitles,
|
||||
null, 0f, year, originalLanguage);
|
||||
}
|
||||
|
||||
private Chapter ParseChapterFromJToken(Manga parentManga, JToken jToken)
|
||||
{
|
||||
string? id = jToken.Value<string>("id");
|
||||
JToken? attributes = jToken["attributes"];
|
||||
string? chapter = attributes?.Value<string>("chapter");
|
||||
string? volumeStr = attributes?.Value<string>("volume");
|
||||
int? volume = null;
|
||||
string? title = attributes?.Value<string>("title");
|
||||
|
||||
if(id is null || chapter is null)
|
||||
throw new Exception("jToken was not in expected format");
|
||||
if(volumeStr is not null)
|
||||
volume = int.Parse(volumeStr);
|
||||
|
||||
string url = $"https://mangadex.org/chapter/{id}";
|
||||
return new Chapter(parentManga, url, chapter, volume, title);
|
||||
}
|
||||
}
|
||||
@@ -1,183 +0,0 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using API.MangaDownloadClients;
|
||||
using HtmlAgilityPack;
|
||||
|
||||
namespace API.Schema.MangaConnectors;
|
||||
|
||||
public class MangaHere : MangaConnector
|
||||
{
|
||||
public MangaHere() : base("MangaHere", ["en"], ["www.mangahere.cc"], "http://www.mangahere.cc/favicon.ico")
|
||||
{
|
||||
this.downloadClient = new ChromiumDownloadClient();
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(string publicationTitle = "")
|
||||
{
|
||||
string sanitizedTitle = string.Join('+', Regex.Matches(publicationTitle, "[A-z]*").Where(str => str.Length > 0)).ToLower();
|
||||
string requestUrl = $"https://www.mangahere.cc/search?title={sanitizedTitle}";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.htmlDocument is null)
|
||||
return [];
|
||||
|
||||
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument);
|
||||
return publications;
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] ParsePublicationsFromHtml(HtmlDocument document)
|
||||
{
|
||||
if (document.DocumentNode.SelectNodes("//div[contains(concat(' ',normalize-space(@class),' '),' container ')]").Any(node => node.ChildNodes.Any(cNode => cNode.HasClass("search-keywords"))))
|
||||
return [];
|
||||
|
||||
List<string> urls = document.DocumentNode
|
||||
.SelectNodes("//a[contains(@href, '/manga/') and not(contains(@href, '.html'))]")
|
||||
.Select(thumb => $"https://www.mangahere.cc{thumb.GetAttributeValue("href", "")}").Distinct().ToList();
|
||||
|
||||
HashSet<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)> ret = new();
|
||||
foreach (string url in urls)
|
||||
{
|
||||
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? manga = GetMangaFromUrl(url);
|
||||
if (manga is { } x)
|
||||
ret.Add(x);
|
||||
}
|
||||
|
||||
return ret.ToArray();
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(string publicationId)
|
||||
{
|
||||
return GetMangaFromUrl($"https://www.mangahere.cc/manga/{publicationId}");
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromUrl(string url)
|
||||
{
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.htmlDocument is null)
|
||||
return null;
|
||||
|
||||
Regex idRex = new (@"https:\/\/www\.mangahere\.[a-z]{0,63}\/manga\/([0-9A-z\-]+).*");
|
||||
string id = idRex.Match(url).Groups[1].Value;
|
||||
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, id, url);
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?) ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl)
|
||||
{
|
||||
string originalLanguage = "", status = "";
|
||||
Dictionary<string, string> altTitles = new(), links = new();
|
||||
MangaReleaseStatus releaseStatus = MangaReleaseStatus.Unreleased;
|
||||
|
||||
//We dont get posters, because same origin bs HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//img[contains(concat(' ',normalize-space(@class),' '),' detail-info-cover-img ')]");
|
||||
string coverUrl = "http://static.mangahere.cc/v20230914/mangahere/images/nopicture.jpg";
|
||||
|
||||
HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//span[contains(concat(' ',normalize-space(@class),' '),' detail-info-right-title-font ')]");
|
||||
string sortName = titleNode.InnerText;
|
||||
|
||||
List<string> authorNames = document.DocumentNode
|
||||
.SelectNodes("//p[contains(concat(' ',normalize-space(@class),' '),' detail-info-right-say ')]/a")
|
||||
.Select(node => node.InnerText)
|
||||
.ToList();
|
||||
List<Author> authors = authorNames.Select(n => new Author(n)).ToList();
|
||||
|
||||
HashSet<string> tags = document.DocumentNode
|
||||
.SelectNodes("//p[contains(concat(' ',normalize-space(@class),' '),' detail-info-right-tag-list ')]/a")
|
||||
.Select(node => node.InnerText)
|
||||
.ToHashSet();
|
||||
List<MangaTag> mangaTags = tags.Select(n => new MangaTag(n)).ToList();
|
||||
|
||||
status = document.DocumentNode.SelectSingleNode("//span[contains(concat(' ',normalize-space(@class),' '),' detail-info-right-title-tip ')]").InnerText;
|
||||
switch (status.ToLower())
|
||||
{
|
||||
case "cancelled": releaseStatus = MangaReleaseStatus.Cancelled; break;
|
||||
case "hiatus": releaseStatus = MangaReleaseStatus.OnHiatus; break;
|
||||
case "discontinued": releaseStatus = MangaReleaseStatus.Cancelled; break;
|
||||
case "complete": releaseStatus = MangaReleaseStatus.Completed; break;
|
||||
case "ongoing": releaseStatus = MangaReleaseStatus.Continuing; break;
|
||||
}
|
||||
|
||||
HtmlNode descriptionNode = document.DocumentNode
|
||||
.SelectSingleNode("//p[contains(concat(' ',normalize-space(@class),' '),' fullcontent ')]");
|
||||
string description = descriptionNode.InnerText;
|
||||
|
||||
Manga manga = new (publicationId, sortName, description, websiteUrl, coverUrl, null, 0,
|
||||
originalLanguage, releaseStatus, -1,
|
||||
this,
|
||||
authors,
|
||||
mangaTags,
|
||||
[],
|
||||
[]);
|
||||
|
||||
return (manga, authors, mangaTags, [], []);
|
||||
}
|
||||
|
||||
public override Chapter[] GetChapters(Manga manga, string language="en")
|
||||
{
|
||||
string requestUrl = $"https://www.mangahere.cc/manga/{manga.MangaId}";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.htmlDocument is null)
|
||||
return Array.Empty<Chapter>();
|
||||
|
||||
List<string> urls = requestResult.htmlDocument.DocumentNode.SelectNodes("//div[@id='list-1']/ul//li//a[contains(@href, '/manga/')]")
|
||||
.Select(node => node.GetAttributeValue("href", "")).ToList();
|
||||
Regex chapterRex = new(@".*\/manga\/[a-zA-Z0-9\-\._\~\!\$\&\'\(\)\*\+\,\;\=\:\@]+\/v([0-9(TBD)]+)\/c([0-9\.]+)\/.*");
|
||||
|
||||
List<Chapter> chapters = new();
|
||||
foreach (string url in urls)
|
||||
{
|
||||
Match rexMatch = chapterRex.Match(url);
|
||||
|
||||
int? volumeNumber = rexMatch.Groups[1].Value == "TBD" ? null : int.Parse(rexMatch.Groups[1].Value);
|
||||
string chapterNumber = new(rexMatch.Groups[2].Value);
|
||||
string fullUrl = $"https://www.mangahere.cc{url}";
|
||||
|
||||
try
|
||||
{
|
||||
chapters.Add(new Chapter(manga, fullUrl, chapterNumber, volumeNumber, null));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
}
|
||||
}
|
||||
//Return Chapters ordered by Chapter-Number
|
||||
return chapters.Order().ToArray();
|
||||
}
|
||||
|
||||
internal override string[] GetChapterImageUrls(Chapter chapter)
|
||||
{
|
||||
List<string> imageUrls = new();
|
||||
|
||||
int downloaded = 1;
|
||||
int images = 1;
|
||||
string url = string.Join('/', chapter.Url.Split('/')[..^1]);
|
||||
do
|
||||
{
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest($"{url}/{downloaded}.html", RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.htmlDocument is null)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
imageUrls.AddRange(ParseImageUrlsFromHtml(requestResult.htmlDocument));
|
||||
|
||||
images = requestResult.htmlDocument.DocumentNode
|
||||
.SelectNodes("//a[contains(@href, '/manga/')]")
|
||||
.MaxBy(node => node.GetAttributeValue("data-page", 0))!.GetAttributeValue("data-page", 0);
|
||||
} while (downloaded++ <= images);
|
||||
|
||||
return imageUrls.ToArray();
|
||||
}
|
||||
|
||||
private string[] ParseImageUrlsFromHtml(HtmlDocument document)
|
||||
{
|
||||
return document.DocumentNode
|
||||
.SelectNodes("//img[contains(concat(' ',normalize-space(@class),' '),' reader-main-img ')]")
|
||||
.Select(node =>
|
||||
{
|
||||
string url = node.GetAttributeValue("src", "");
|
||||
return url.StartsWith("//") ? $"https:{url}" : url;
|
||||
})
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -1,233 +0,0 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using API.MangaDownloadClients;
|
||||
using HtmlAgilityPack;
|
||||
|
||||
namespace API.Schema.MangaConnectors;
|
||||
|
||||
public class MangaKatana : MangaConnector
|
||||
{
|
||||
public MangaKatana() : base("MangaKatana", ["en"], ["mangakatana.com"], "https://mangakatana.com/static/img/fav.png")
|
||||
{
|
||||
this.downloadClient = new HttpDownloadClient();
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(string publicationTitle = "")
|
||||
{
|
||||
string sanitizedTitle = string.Join("%20", Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower();
|
||||
string requestUrl = $"https://mangakatana.com/?search={sanitizedTitle}&search_by=book_name";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return [];
|
||||
|
||||
// ReSharper disable once MergeIntoPattern
|
||||
// If a single result is found, the user will be redirected to the results directly instead of a result page
|
||||
if(requestResult.hasBeenRedirected
|
||||
&& requestResult.redirectedToUrl is not null
|
||||
&& requestResult.redirectedToUrl.Contains("mangakatana.com/manga"))
|
||||
{
|
||||
return new [] { ParseSinglePublicationFromHtml(requestResult.result, requestResult.redirectedToUrl.Split('/')[^1], requestResult.redirectedToUrl) };
|
||||
}
|
||||
|
||||
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] publications = ParsePublicationsFromHtml(requestResult.result);
|
||||
return publications;
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(string publicationId)
|
||||
{
|
||||
return GetMangaFromUrl($"https://mangakatana.com/manga/{publicationId}");
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromUrl(string url)
|
||||
{
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return null;
|
||||
return ParseSinglePublicationFromHtml(requestResult.result, url.Split('/')[^1], url);
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] ParsePublicationsFromHtml(Stream html)
|
||||
{
|
||||
StreamReader reader = new(html);
|
||||
string htmlString = reader.ReadToEnd();
|
||||
HtmlDocument document = new();
|
||||
document.LoadHtml(htmlString);
|
||||
IEnumerable<HtmlNode> searchResults = document.DocumentNode.SelectNodes("//*[@id='book_list']/div");
|
||||
if (searchResults is null || !searchResults.Any())
|
||||
return [];
|
||||
List<string> urls = new();
|
||||
foreach (HtmlNode mangaResult in searchResults)
|
||||
{
|
||||
urls.Add(mangaResult.Descendants("a").First().GetAttributes()
|
||||
.First(a => a.Name == "href").Value);
|
||||
}
|
||||
|
||||
HashSet<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)> ret = new();
|
||||
foreach (string url in urls)
|
||||
{
|
||||
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? manga = GetMangaFromUrl(url);
|
||||
if (manga is { } x)
|
||||
ret.Add(x);
|
||||
}
|
||||
|
||||
return ret.ToArray();
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?) ParseSinglePublicationFromHtml(Stream html, string publicationId, string websiteUrl)
|
||||
{
|
||||
StreamReader reader = new(html);
|
||||
string htmlString = reader.ReadToEnd();
|
||||
HtmlDocument document = new();
|
||||
document.LoadHtml(htmlString);
|
||||
Dictionary<string, string> altTitlesDict = new();
|
||||
Dictionary<string, string>? links = null;
|
||||
HashSet<string> tags = new();
|
||||
string[] authorNames = [];
|
||||
string originalLanguage = "";
|
||||
MangaReleaseStatus releaseStatus = MangaReleaseStatus.Unreleased;
|
||||
|
||||
HtmlNode infoNode = document.DocumentNode.SelectSingleNode("//*[@id='single_book']");
|
||||
string sortName = infoNode.Descendants("h1").First(n => n.HasClass("heading")).InnerText;
|
||||
HtmlNode infoTable = infoNode.SelectSingleNode("//*[@id='single_book']/div[2]/div/ul");
|
||||
|
||||
foreach (HtmlNode row in infoTable.Descendants("li"))
|
||||
{
|
||||
string key = row.SelectNodes("div").First().InnerText.ToLower();
|
||||
string value = row.SelectNodes("div").Last().InnerText;
|
||||
string keySanitized = string.Concat(Regex.Matches(key, "[a-z]"));
|
||||
|
||||
switch (keySanitized)
|
||||
{
|
||||
case "altnames":
|
||||
string[] alts = value.Split(" ; ");
|
||||
for (int i = 0; i < alts.Length; i++)
|
||||
altTitlesDict.Add(i.ToString(), alts[i]);
|
||||
break;
|
||||
case "authorsartists":
|
||||
authorNames = value.Split(',');
|
||||
break;
|
||||
case "status":
|
||||
switch (value.ToLower())
|
||||
{
|
||||
case "ongoing": releaseStatus = MangaReleaseStatus.Continuing; break;
|
||||
case "completed": releaseStatus = MangaReleaseStatus.Completed; break;
|
||||
}
|
||||
break;
|
||||
case "genres":
|
||||
tags = row.SelectNodes("div").Last().Descendants("a").Select(a => a.InnerText).ToHashSet();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
string coverUrl = document.DocumentNode.SelectSingleNode("//*[@id='single_book']/div[1]/div").Descendants("img").First()
|
||||
.GetAttributes().First(a => a.Name == "src").Value;
|
||||
|
||||
string description = document.DocumentNode.SelectSingleNode("//*[@id='single_book']/div[3]/p").InnerText;
|
||||
while (description.StartsWith('\n'))
|
||||
description = description.Substring(1);
|
||||
|
||||
uint year = (uint)DateTime.UtcNow.Year;
|
||||
string yearString = infoTable.Descendants("div").First(d => d.HasClass("updateAt"))
|
||||
.InnerText.Split('-')[^1];
|
||||
|
||||
if(yearString.Contains("ago") == false)
|
||||
{
|
||||
year = uint.Parse(yearString);
|
||||
}
|
||||
List<Author> authors = authorNames.Select(n => new Author(n)).ToList();
|
||||
List<MangaTag> mangaTags = tags.Select(n => new MangaTag(n)).ToList();
|
||||
List<MangaAltTitle> altTitles = altTitlesDict.Select(x => new MangaAltTitle(x.Key, x.Value)).ToList();
|
||||
|
||||
Manga manga = new (publicationId, sortName, description, websiteUrl, coverUrl, null, year,
|
||||
originalLanguage, releaseStatus, -1,
|
||||
this,
|
||||
authors,
|
||||
mangaTags,
|
||||
[],
|
||||
altTitles);
|
||||
|
||||
return (manga, authors, mangaTags, [], altTitles);
|
||||
}
|
||||
|
||||
public override Chapter[] GetChapters(Manga manga, string language="en")
|
||||
{
|
||||
string requestUrl = $"https://mangakatana.com/manga/{manga.MangaId}";
|
||||
// Leaving this in for verification if the page exists
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return Array.Empty<Chapter>();
|
||||
|
||||
//Return Chapters ordered by Chapter-Number
|
||||
List<Chapter> chapters = ParseChaptersFromHtml(manga, requestUrl);
|
||||
return chapters.Order().ToArray();
|
||||
}
|
||||
|
||||
private List<Chapter> ParseChaptersFromHtml(Manga manga, string mangaUrl)
|
||||
{
|
||||
// Using HtmlWeb will include the chapters since they are loaded with js
|
||||
HtmlWeb web = new();
|
||||
HtmlDocument document = web.Load(mangaUrl);
|
||||
|
||||
List<Chapter> ret = new();
|
||||
|
||||
HtmlNode chapterList = document.DocumentNode.SelectSingleNode("//div[contains(@class, 'chapters')]/table/tbody");
|
||||
|
||||
Regex volumeRex = new(@"[0-9a-z\-\.]+\/[0-9a-z\-]*v([0-9\.]+)");
|
||||
Regex chapterNumRex = new(@"[0-9a-z\-\.]+\/[0-9a-z\-]*c([0-9\.]+)");
|
||||
Regex chapterNameRex = new(@"Chapter [0-9\.]+:? (.*)");
|
||||
|
||||
foreach (HtmlNode chapterInfo in chapterList.Descendants("tr"))
|
||||
{
|
||||
string fullString = chapterInfo.Descendants("a").First().InnerText;
|
||||
string url = chapterInfo.Descendants("a").First()
|
||||
.GetAttributeValue("href", "");
|
||||
|
||||
int? volumeNumber = volumeRex.IsMatch(url) ? int.Parse(volumeRex.Match(url).Groups[1].Value) : null;
|
||||
|
||||
string chapterNumber = new(chapterNumRex.Match(url).Groups[1].Value);
|
||||
string chapterName = chapterNameRex.Match(fullString).Groups[1].Value;
|
||||
try
|
||||
{
|
||||
ret.Add(new Chapter(manga, url, chapterNumber, volumeNumber, chapterName));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
internal override string[] GetChapterImageUrls(Chapter chapter)
|
||||
{
|
||||
string requestUrl = chapter.Url;
|
||||
// Leaving this in to check if the page exists
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.htmlDocument is null)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
string[] imageUrls = ParseImageUrlsFromHtml(requestResult.htmlDocument);
|
||||
return imageUrls;
|
||||
}
|
||||
|
||||
private string[] ParseImageUrlsFromHtml(HtmlDocument document)
|
||||
{
|
||||
// Images are loaded dynamically, but the urls are present in a piece of js code on the page
|
||||
string js = document.DocumentNode.SelectSingleNode("//script[contains(., 'data-src')]").InnerText
|
||||
.Replace("\r", "")
|
||||
.Replace("\n", "")
|
||||
.Replace("\t", "");
|
||||
|
||||
// ReSharper disable once StringLiteralTypo
|
||||
string regexPat = @"(var thzq=\[')(.*)(,];function)";
|
||||
var group = Regex.Matches(js, regexPat).First().Groups[2].Value.Replace("'", "");
|
||||
var urls = group.Split(',');
|
||||
|
||||
return urls;
|
||||
}
|
||||
}
|
||||
@@ -1,219 +0,0 @@
|
||||
using System.Globalization;
|
||||
using System.Net;
|
||||
using System.Text.RegularExpressions;
|
||||
using API.MangaDownloadClients;
|
||||
using HtmlAgilityPack;
|
||||
|
||||
namespace API.Schema.MangaConnectors;
|
||||
|
||||
public class Manganato : MangaConnector
|
||||
{
|
||||
public Manganato() : base("Manganato", ["en"],
|
||||
["natomanga.com", "manganato.gg", "mangakakalot.gg", "nelomanga.com"],
|
||||
"https://www.manganato.gg/images/favicon-manganato.webp")
|
||||
{
|
||||
this.downloadClient = new HttpDownloadClient();
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(
|
||||
string publicationTitle = "")
|
||||
{
|
||||
string sanitizedTitle = string.Join('_', Regex.Matches(publicationTitle, "[A-z]*").Where(str => str.Length > 0))
|
||||
.ToLower();
|
||||
string requestUrl = $"https://manganato.gg/search/story/{sanitizedTitle}";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return [];
|
||||
|
||||
if (requestResult.htmlDocument is null)
|
||||
return [];
|
||||
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] publications =
|
||||
ParsePublicationsFromHtml(requestResult.htmlDocument);
|
||||
return publications;
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] ParsePublicationsFromHtml(
|
||||
HtmlDocument document)
|
||||
{
|
||||
List<HtmlNode> searchResults =
|
||||
document.DocumentNode.Descendants("div").Where(n => n.HasClass("story_item")).ToList();
|
||||
List<string> urls = new();
|
||||
foreach (HtmlNode mangaResult in searchResults)
|
||||
{
|
||||
try
|
||||
{
|
||||
urls.Add(mangaResult.Descendants("h3").First(n => n.HasClass("story_name"))
|
||||
.Descendants("a").First().GetAttributeValue("href", ""));
|
||||
}
|
||||
catch
|
||||
{
|
||||
//failed to get a url, send it to the void
|
||||
}
|
||||
}
|
||||
|
||||
List<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)> ret = new();
|
||||
foreach (string url in urls)
|
||||
{
|
||||
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? manga = GetMangaFromUrl(url);
|
||||
if (manga is { } m)
|
||||
ret.Add(m);
|
||||
}
|
||||
|
||||
return ret.ToArray();
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(
|
||||
string publicationId)
|
||||
{
|
||||
return GetMangaFromUrl($"https://chapmanganato.com/{publicationId}");
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)?
|
||||
GetMangaFromUrl(string url)
|
||||
{
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return null;
|
||||
|
||||
if (requestResult.htmlDocument is null)
|
||||
return null;
|
||||
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, url.Split('/')[^1], url);
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?) ParseSinglePublicationFromHtml(
|
||||
HtmlDocument document, string publicationId, string websiteUrl)
|
||||
{
|
||||
Dictionary<string, string> altTitles = new();
|
||||
List<MangaTag> tags = new();
|
||||
List<Author> authors = new();
|
||||
MangaReleaseStatus releaseStatus = MangaReleaseStatus.Unreleased;
|
||||
|
||||
HtmlNode infoNode = document.DocumentNode.Descendants("ul").First(d => d.HasClass("manga-info-text"));
|
||||
|
||||
string sortName = infoNode.Descendants("h1").First().InnerText;
|
||||
|
||||
foreach (HtmlNode li in infoNode.Descendants("li"))
|
||||
{
|
||||
string text = li.InnerText.Trim().ToLower();
|
||||
|
||||
if (text.StartsWith("author(s) :"))
|
||||
{
|
||||
authors = li.Descendants("a").Select(a => a.InnerText.Trim()).Select(a => new Author(a)).ToList();
|
||||
}
|
||||
else if (text.StartsWith("status :"))
|
||||
{
|
||||
string status = text.Replace("status :", "").Trim().ToLower();
|
||||
if (string.IsNullOrWhiteSpace(status))
|
||||
releaseStatus = MangaReleaseStatus.Continuing;
|
||||
else if (status == "ongoing")
|
||||
releaseStatus = MangaReleaseStatus.Continuing;
|
||||
else
|
||||
releaseStatus = Enum.Parse<MangaReleaseStatus>(status, true);
|
||||
}
|
||||
else if (li.HasClass("genres"))
|
||||
{
|
||||
tags = li.Descendants("a").Select(a => new MangaTag(a.InnerText.Trim())).ToList();
|
||||
}
|
||||
}
|
||||
|
||||
string posterUrl = document.DocumentNode.Descendants("div").First(s => s.HasClass("manga-info-pic"))
|
||||
.Descendants("img").First()
|
||||
.GetAttributes().First(a => a.Name == "src").Value;
|
||||
|
||||
string description = document.DocumentNode.SelectSingleNode("//div[@id='contentBox']")
|
||||
.InnerText.Replace("Description :", "");
|
||||
while (description.StartsWith('\n'))
|
||||
description = description.Substring(1);
|
||||
|
||||
string pattern = "MMM-dd-yyyy HH:mm";
|
||||
|
||||
HtmlNode? oldestChapter = document.DocumentNode
|
||||
.SelectNodes("//div[contains(concat(' ',normalize-space(@class),' '),' row ')]/span[@title]").MaxBy(
|
||||
node => DateTime.ParseExact(node.GetAttributeValue("title", "Dec-31-2400 23:59"), pattern,
|
||||
CultureInfo.InvariantCulture).Millisecond);
|
||||
|
||||
|
||||
uint year = Convert.ToUInt32(DateTime.ParseExact(
|
||||
oldestChapter?.GetAttributeValue("title", "Dec 31 2400, 23:59") ?? "Dec 31 2400, 23:59", pattern,
|
||||
CultureInfo.InvariantCulture).Year);
|
||||
|
||||
Manga manga = new(publicationId, sortName, description, websiteUrl, posterUrl, null, year, null, releaseStatus,
|
||||
-1, this, authors, tags, [], []);
|
||||
return (manga, authors, tags, [], []);
|
||||
}
|
||||
|
||||
public override Chapter[] GetChapters(Manga manga, string language = "en")
|
||||
{
|
||||
string requestUrl = manga.WebsiteUrl;
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return Array.Empty<Chapter>();
|
||||
|
||||
//Return Chapters ordered by Chapter-Number
|
||||
if (requestResult.htmlDocument is null)
|
||||
return Array.Empty<Chapter>();
|
||||
List<Chapter> chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument);
|
||||
return chapters.Order().ToArray();
|
||||
}
|
||||
|
||||
internal override string[] GetChapterImageUrls(Chapter chapter)
|
||||
{
|
||||
string requestUrl = chapter.Url;
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 ||
|
||||
requestResult.htmlDocument is null)
|
||||
return [];
|
||||
|
||||
string[] imageUrls = ParseImageUrlsFromHtml(requestResult.htmlDocument);
|
||||
|
||||
return imageUrls;
|
||||
}
|
||||
|
||||
private List<Chapter> ParseChaptersFromHtml(Manga manga, HtmlDocument document)
|
||||
{
|
||||
List<Chapter> ret = new();
|
||||
|
||||
HtmlNode chapterList = document.DocumentNode.Descendants("div").First(l => l.HasClass("chapter-list"));
|
||||
|
||||
Regex volRex = new(@"Vol\.([0-9]+).*");
|
||||
Regex chapterRex = new(@"https:\/\/chapmanganato.[A-z]+\/manga-[A-z0-9]+\/chapter-([0-9\.]+)");
|
||||
Regex nameRex = new(@"Chapter ([0-9]+(\.[0-9]+)*){1}:? (.*)");
|
||||
|
||||
foreach (HtmlNode chapterInfo in chapterList.Descendants("div").Where(x => x.HasClass("row")))
|
||||
{
|
||||
string url = chapterInfo.Descendants("a").First().GetAttributeValue("href", "");
|
||||
var name = chapterInfo.Descendants("a").First().InnerText.Trim();
|
||||
string chapterName = nameRex.Match(name).Groups[3].Value;
|
||||
string chapterNumber = Regex.Match(name, @"Chapter ([0-9]+(\.[0-9]+)*)").Groups[1].Value;
|
||||
string? volumeNumber = Regex.Match(chapterName, @"Vol\.([0-9]+)").Groups[1].Value;
|
||||
if (string.IsNullOrWhiteSpace(volumeNumber))
|
||||
volumeNumber = "0";
|
||||
try
|
||||
{
|
||||
ret.Add(new Chapter(manga, url, chapterNumber, int.Parse(volumeNumber), chapterName));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
ret.Reverse();
|
||||
return ret;
|
||||
}
|
||||
|
||||
private string[] ParseImageUrlsFromHtml(HtmlDocument document)
|
||||
{
|
||||
List<string> ret = new();
|
||||
|
||||
HtmlNode imageContainer =
|
||||
document.DocumentNode.Descendants("div").First(i => i.HasClass("container-chapter-reader"));
|
||||
foreach (HtmlNode imageNode in imageContainer.Descendants("img"))
|
||||
ret.Add(imageNode.GetAttributeValue("src", ""));
|
||||
|
||||
return ret.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -1,223 +0,0 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using API.MangaDownloadClients;
|
||||
using HtmlAgilityPack;
|
||||
|
||||
namespace API.Schema.MangaConnectors;
|
||||
|
||||
public class Mangaworld : MangaConnector
|
||||
{
|
||||
public Mangaworld() : base("Mangaworld", ["it"], ["www.mangaworld.ac", "www.mangaworld.nz"], "https://www.mangaworld.nz/public/assets/seo/android-icon-192x192.png")
|
||||
{
|
||||
this.downloadClient = new ChromiumDownloadClient();
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(string publicationTitle = "")
|
||||
{
|
||||
string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(str => str.Length > 0)).ToLower();
|
||||
string requestUrl = $"https://www.mangaworld.ac/archive?keyword={sanitizedTitle}";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return [];
|
||||
|
||||
if (requestResult.htmlDocument is null)
|
||||
return [];
|
||||
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument);
|
||||
return publications;
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] ParsePublicationsFromHtml(HtmlDocument document)
|
||||
{
|
||||
if (!document.DocumentNode.SelectSingleNode("//div[@class='comics-grid']").ChildNodes
|
||||
.Any(node => node.HasClass("entry")))
|
||||
return [];
|
||||
|
||||
List<string> urls = document.DocumentNode
|
||||
.SelectNodes(
|
||||
"//div[@class='comics-grid']//div[@class='entry']//a[contains(concat(' ',normalize-space(@class),' '),'thumb')]")
|
||||
.Select(thumb => thumb.GetAttributeValue("href", "")).ToList();
|
||||
|
||||
List<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)> ret = new();
|
||||
foreach (string url in urls)
|
||||
{
|
||||
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? manga = GetMangaFromUrl(url);
|
||||
if (manga is { } x)
|
||||
ret.Add(x);
|
||||
}
|
||||
|
||||
return ret.ToArray();
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(string publicationId)
|
||||
{
|
||||
return GetMangaFromUrl($"https://www.mangaworld.ac/manga/{publicationId}");
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromUrl(string url)
|
||||
{
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return null;
|
||||
|
||||
if (requestResult.htmlDocument is null)
|
||||
return null;
|
||||
|
||||
Regex idRex = new (@"https:\/\/www\.mangaworld\.[a-z]{0,63}\/manga\/([0-9]+\/[0-9A-z\-]+).*");
|
||||
string id = idRex.Match(url).Groups[1].Value;
|
||||
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, id, url);
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?) ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl)
|
||||
{
|
||||
Dictionary<string, string> altTitlesDict = new();
|
||||
string originalLanguage = "";
|
||||
MangaReleaseStatus releaseStatus = MangaReleaseStatus.Unreleased;
|
||||
|
||||
HtmlNode infoNode = document.DocumentNode.Descendants("div").First(d => d.HasClass("info"));
|
||||
|
||||
string sortName = infoNode.Descendants("h1").First().InnerText;
|
||||
|
||||
HtmlNode metadata = infoNode.Descendants().First(d => d.HasClass("meta-data"));
|
||||
|
||||
HtmlNode altTitlesNode = metadata.SelectSingleNode("//span[text()='Titoli alternativi: ' or text()='Titolo alternativo: ']/..").ChildNodes[1];
|
||||
string[] alts = altTitlesNode.InnerText.Split(", ");
|
||||
for(int i = 0; i < alts.Length; i++)
|
||||
altTitlesDict.Add(i.ToString(), alts[i]);
|
||||
List<MangaAltTitle> altTitles = altTitlesDict.Select(a => new MangaAltTitle(a.Key, a.Value)).ToList();
|
||||
|
||||
HtmlNode genresNode =
|
||||
metadata.SelectSingleNode("//span[text()='Generi: ' or text()='Genero: ']/..");
|
||||
HashSet<string> tags = genresNode.SelectNodes("a").Select(node => node.InnerText).ToHashSet();
|
||||
List<MangaTag> mangaTags = tags.Select(t => new MangaTag(t)).ToList();
|
||||
|
||||
HtmlNode authorsNode =
|
||||
metadata.SelectSingleNode("//span[text()='Autore: ' or text()='Autori: ']/..");
|
||||
string[] authorNames = authorsNode.SelectNodes("a").Select(node => node.InnerText).ToArray();
|
||||
List<Author> authors = authorNames.Select(n => new Author(n)).ToList();
|
||||
|
||||
string status = metadata.SelectSingleNode("//span[text()='Stato: ']/..").SelectNodes("a").First().InnerText;
|
||||
// ReSharper disable 5 times StringLiteralTypo
|
||||
switch (status.ToLower())
|
||||
{
|
||||
case "cancellato": releaseStatus = MangaReleaseStatus.Cancelled; break;
|
||||
case "in pausa": releaseStatus = MangaReleaseStatus.OnHiatus; break;
|
||||
case "droppato": releaseStatus = MangaReleaseStatus.Cancelled; break;
|
||||
case "finito": releaseStatus = MangaReleaseStatus.Completed; break;
|
||||
case "in corso": releaseStatus = MangaReleaseStatus.Continuing; break;
|
||||
}
|
||||
|
||||
string coverUrl = document.DocumentNode.SelectSingleNode("//img[@class='rounded']").GetAttributeValue("src", "");
|
||||
|
||||
string description = document.DocumentNode.SelectSingleNode("//div[@id='noidungm']").InnerText;
|
||||
|
||||
string yearString = metadata.SelectSingleNode("//span[text()='Anno di uscita: ']/..").SelectNodes("a").First().InnerText;
|
||||
uint year = uint.Parse(yearString);
|
||||
|
||||
Manga manga = new (publicationId, sortName, description, websiteUrl, coverUrl, null, year,
|
||||
originalLanguage, releaseStatus, -1,
|
||||
this,
|
||||
authors,
|
||||
mangaTags,
|
||||
[],
|
||||
altTitles);
|
||||
|
||||
return (manga, authors, mangaTags, [], altTitles);
|
||||
}
|
||||
|
||||
public override Chapter[] GetChapters(Manga manga, string language="en")
|
||||
{
|
||||
string requestUrl = $"https://www.mangaworld.ac/manga/{manga.MangaId}";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.htmlDocument is null)
|
||||
return [];
|
||||
|
||||
List<Chapter> chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument);
|
||||
return chapters.Order().ToArray();
|
||||
}
|
||||
|
||||
private List<Chapter> ParseChaptersFromHtml(Manga manga, HtmlDocument document)
|
||||
{
|
||||
List<Chapter> ret = new();
|
||||
|
||||
HtmlNode chaptersWrapper =
|
||||
document.DocumentNode.SelectSingleNode(
|
||||
"//div[contains(concat(' ',normalize-space(@class),' '),'chapters-wrapper')]");
|
||||
|
||||
Regex volumeRex = new(@"[Vv]olume ([0-9]+).*");
|
||||
Regex chapterRex = new(@"[Cc]apitolo ([0-9]+(?:\.[0-9]+)?).*");
|
||||
Regex idRex = new(@".*\/read\/([a-z0-9]+)(?:[?\/].*)?");
|
||||
if (chaptersWrapper.Descendants("div").Any(descendant => descendant.HasClass("volume-element")))
|
||||
{
|
||||
foreach (HtmlNode volNode in document.DocumentNode.SelectNodes("//div[contains(concat(' ',normalize-space(@class),' '),'volume-element')]"))
|
||||
{
|
||||
string volumeStr = volumeRex.Match(volNode.SelectNodes("div").First(node => node.HasClass("volume")).SelectSingleNode("p").InnerText).Groups[1].Value;
|
||||
int volume = int.Parse(volumeStr);
|
||||
foreach (HtmlNode chNode in volNode.SelectNodes("div").First(node => node.HasClass("volume-chapters")).SelectNodes("div"))
|
||||
{
|
||||
|
||||
string numberStr = chapterRex.Match(chNode.SelectSingleNode("a").SelectSingleNode("span").InnerText).Groups[1].Value;
|
||||
|
||||
string chapterNumber = new(numberStr);
|
||||
string url = chNode.SelectSingleNode("a").GetAttributeValue("href", "");
|
||||
string id = idRex.Match(chNode.SelectSingleNode("a").GetAttributeValue("href", "")).Groups[1].Value;
|
||||
try
|
||||
{
|
||||
ret.Add(new Chapter(manga, url, chapterNumber, volume, null));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach (HtmlNode chNode in chaptersWrapper.SelectNodes("div").Where(node => node.HasClass("chapter")))
|
||||
{
|
||||
string numberStr = chapterRex.Match(chNode.SelectSingleNode("a").SelectSingleNode("span").InnerText).Groups[1].Value;
|
||||
|
||||
string chapterNumber = new(numberStr);
|
||||
string url = chNode.SelectSingleNode("a").GetAttributeValue("href", "");
|
||||
string id = idRex.Match(chNode.SelectSingleNode("a").GetAttributeValue("href", "")).Groups[1].Value;
|
||||
try
|
||||
{
|
||||
ret.Add(new Chapter(manga, url, chapterNumber, null, null));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ret.Reverse();
|
||||
return ret;
|
||||
}
|
||||
|
||||
internal override string[] GetChapterImageUrls(Chapter chapter)
|
||||
{
|
||||
string requestUrl = $"{chapter.Url}?style=list";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.htmlDocument is null)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
string[] imageUrls = ParseImageUrlsFromHtml(requestResult.htmlDocument);
|
||||
return imageUrls;
|
||||
}
|
||||
|
||||
private string[] ParseImageUrlsFromHtml(HtmlDocument document)
|
||||
{
|
||||
List<string> ret = new();
|
||||
|
||||
HtmlNode imageContainer =
|
||||
document.DocumentNode.SelectSingleNode("//div[@id='page']");
|
||||
foreach(HtmlNode imageNode in imageContainer.Descendants("img"))
|
||||
ret.Add(imageNode.GetAttributeValue("src", ""));
|
||||
|
||||
return ret.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -1,179 +0,0 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using API.MangaDownloadClients;
|
||||
using HtmlAgilityPack;
|
||||
|
||||
namespace API.Schema.MangaConnectors;
|
||||
|
||||
public class ManhuaPlus : MangaConnector
|
||||
{
|
||||
public ManhuaPlus() : base("ManhuaPlus", ["en"], ["manhuaplus.org"], "https://manhuaplus.org/uploads/images/favicon.png")
|
||||
{
|
||||
this.downloadClient = new ChromiumDownloadClient();
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(string publicationTitle = "")
|
||||
{
|
||||
string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(str => str.Length > 0)).ToLower();
|
||||
string requestUrl = $"https://manhuaplus.org/search?keyword={sanitizedTitle}";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.htmlDocument is null)
|
||||
return [];
|
||||
|
||||
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument);
|
||||
return publications;
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] ParsePublicationsFromHtml(HtmlDocument document)
|
||||
{
|
||||
if (document.DocumentNode.SelectSingleNode("//h1/../..").ChildNodes//I already want to not.
|
||||
.Any(node => node.InnerText.Contains("No manga found")))
|
||||
return [];
|
||||
|
||||
List<string> urls = document.DocumentNode
|
||||
.SelectNodes("//h1/../..//a[contains(@href, 'https://manhuaplus.org/manga/') and contains(concat(' ',normalize-space(@class),' '),' clamp ') and not(contains(@href, '/chapter'))]")
|
||||
.Select(mangaNode => mangaNode.GetAttributeValue("href", "")).ToList();
|
||||
|
||||
List<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)> ret = new();
|
||||
foreach (string url in urls)
|
||||
{
|
||||
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? manga = GetMangaFromUrl(url);
|
||||
if (manga is { } x)
|
||||
ret.Add(x);
|
||||
}
|
||||
|
||||
return ret.ToArray();
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(string publicationId)
|
||||
{
|
||||
return GetMangaFromUrl($"https://manhuaplus.org/manga/{publicationId}");
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromUrl(string url)
|
||||
{
|
||||
Regex publicationIdRex = new(@"https:\/\/manhuaplus.org\/manga\/(.*)(\/.*)*");
|
||||
string publicationId = publicationIdRex.Match(url).Groups[1].Value;
|
||||
|
||||
RequestResult requestResult = this.downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||
if((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 && requestResult.htmlDocument is not null && requestResult.redirectedToUrl != "https://manhuaplus.org/home") //When manga doesnt exists it redirects to home
|
||||
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url);
|
||||
return null;
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?) ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl)
|
||||
{
|
||||
string originalLanguage = "", status = "";
|
||||
Dictionary<string, string> altTitles = new(), links = new();
|
||||
HashSet<string> tags = new();
|
||||
MangaReleaseStatus releaseStatus = MangaReleaseStatus.Unreleased;
|
||||
|
||||
HtmlNode posterNode = document.DocumentNode.SelectSingleNode("/html/body/main/div/div/div[2]/div[1]/figure/a/img");//BRUH
|
||||
Regex posterRex = new(@".*(\/uploads/covers/[a-zA-Z0-9\-\._\~\!\$\&\'\(\)\*\+\,\;\=\:\@]+).*");
|
||||
string coverUrl = $"https://manhuaplus.org/{posterRex.Match(posterNode.GetAttributeValue("src", "")).Groups[1].Value}";
|
||||
|
||||
HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//h1");
|
||||
string sortName = titleNode.InnerText.Replace("\n", "");
|
||||
|
||||
List<string> authorNames = new();
|
||||
try
|
||||
{
|
||||
HtmlNode[] authorsNodes = document.DocumentNode
|
||||
.SelectNodes("//a[contains(@href, 'https://manhuaplus.org/authors/')]")
|
||||
.ToArray();
|
||||
foreach (HtmlNode authorNode in authorsNodes)
|
||||
authorNames.Add(authorNode.InnerText);
|
||||
}
|
||||
catch (ArgumentNullException e)
|
||||
{
|
||||
}
|
||||
List<Author> authors = authorNames.Select(a => new Author(a)).ToList();
|
||||
|
||||
try
|
||||
{
|
||||
HtmlNode[] genreNodes = document.DocumentNode
|
||||
.SelectNodes("//a[contains(@href, 'https://manhuaplus.org/genres/')]").ToArray();
|
||||
foreach (HtmlNode genreNode in genreNodes)
|
||||
tags.Add(genreNode.InnerText.Replace("\n", ""));
|
||||
}
|
||||
catch (ArgumentNullException e)
|
||||
{
|
||||
}
|
||||
List<MangaTag> mangaTags = tags.Select(t => new MangaTag(t)).ToList();
|
||||
|
||||
Regex yearRex = new(@"(?:[0-9]{1,2}\/){2}([0-9]{2,4}) [0-9]{1,2}:[0-9]{1,2}");
|
||||
HtmlNode yearNode = document.DocumentNode.SelectSingleNode("//aside//i[contains(concat(' ',normalize-space(@class),' '),' fa-clock ')]/../span");
|
||||
Match match = yearRex.Match(yearNode.InnerText);
|
||||
uint year = match.Success && match.Groups[1].Success ? uint.Parse(match.Groups[1].Value) : 0;
|
||||
|
||||
status = document.DocumentNode.SelectSingleNode("//aside//i[contains(concat(' ',normalize-space(@class),' '),' fa-rss ')]/../span").InnerText.Replace("\n", "");
|
||||
switch (status.ToLower())
|
||||
{
|
||||
case "cancelled": releaseStatus = MangaReleaseStatus.Cancelled; break;
|
||||
case "hiatus": releaseStatus = MangaReleaseStatus.OnHiatus; break;
|
||||
case "discontinued": releaseStatus = MangaReleaseStatus.Cancelled; break;
|
||||
case "complete": releaseStatus = MangaReleaseStatus.Completed; break;
|
||||
case "ongoing": releaseStatus = MangaReleaseStatus.Continuing; break;
|
||||
}
|
||||
|
||||
HtmlNode descriptionNode = document.DocumentNode
|
||||
.SelectSingleNode("//div[@id='syn-target']");
|
||||
string description = descriptionNode.InnerText;
|
||||
|
||||
Manga manga = new (publicationId, sortName, description, websiteUrl, coverUrl, null, year,
|
||||
originalLanguage, releaseStatus, -1,
|
||||
this,
|
||||
authors,
|
||||
mangaTags,
|
||||
[],
|
||||
[]);
|
||||
|
||||
return (manga, authors, mangaTags, [], []);
|
||||
}
|
||||
|
||||
public override Chapter[] GetChapters(Manga manga, string language="en")
|
||||
{
|
||||
RequestResult result = downloadClient.MakeRequest($"https://manhuaplus.org/manga/{manga.MangaId}", RequestType.Default);
|
||||
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null)
|
||||
{
|
||||
return Array.Empty<Chapter>();
|
||||
}
|
||||
|
||||
HtmlNodeCollection chapterNodes = result.htmlDocument.DocumentNode.SelectNodes("//li[contains(concat(' ',normalize-space(@class),' '),' chapter ')]//a");
|
||||
string[] urls = chapterNodes.Select(node => node.GetAttributeValue("href", "")).ToArray();
|
||||
Regex urlRex = new (@".*\/chapter-([0-9\-]+).*");
|
||||
|
||||
List<Chapter> chapters = new();
|
||||
foreach (string url in urls)
|
||||
{
|
||||
Match rexMatch = urlRex.Match(url);
|
||||
|
||||
string chapterNumber = new(rexMatch.Groups[1].Value);
|
||||
string fullUrl = url;
|
||||
try
|
||||
{
|
||||
chapters.Add(new Chapter(manga, fullUrl, chapterNumber, null, null));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
}
|
||||
}
|
||||
//Return Chapters ordered by Chapter-Number
|
||||
return chapters.Order().ToArray();
|
||||
}
|
||||
|
||||
internal override string[] GetChapterImageUrls(Chapter chapter)
|
||||
{
|
||||
RequestResult requestResult = this.downloadClient.MakeRequest(chapter.Url, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.htmlDocument is null)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
HtmlDocument document = requestResult.htmlDocument;
|
||||
|
||||
HtmlNode[] images = document.DocumentNode.SelectNodes("//a[contains(concat(' ',normalize-space(@class),' '),' readImg ')]/img").ToArray();
|
||||
List<string> urls = images.Select(node => node.GetAttributeValue("src", "")).ToList();
|
||||
return urls.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -1,259 +0,0 @@
|
||||
using System.Net;
|
||||
using System.Text.RegularExpressions;
|
||||
using API.MangaDownloadClients;
|
||||
using HtmlAgilityPack;
|
||||
|
||||
namespace API.Schema.MangaConnectors;
|
||||
|
||||
public class Webtoons : MangaConnector
|
||||
{
|
||||
|
||||
public Webtoons() : base("Webtoons", ["en"], ["www.webtoons.com"], "https://webtoons-static.pstatic.net/image/favicon/favicon.ico")
|
||||
{
|
||||
this.downloadClient = new HttpDownloadClient();
|
||||
}
|
||||
|
||||
// Done
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(string publicationTitle = "")
|
||||
{
|
||||
string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower();
|
||||
string requestUrl = $"https://www.webtoons.com/en/search?keyword={sanitizedTitle}&searchType=WEBTOON";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (requestResult.htmlDocument is null)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
(Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>)[] publications =
|
||||
ParsePublicationsFromHtml(requestResult.htmlDocument);
|
||||
return publications;
|
||||
}
|
||||
|
||||
// Done
|
||||
public override (Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>)? GetMangaFromId(string publicationId)
|
||||
{
|
||||
PublicationManager pb = new PublicationManager(publicationId);
|
||||
return GetMangaFromUrl($"https://www.webtoons.com/en/{pb.Category}/{pb.Title}/list?title_no={pb.Id}");
|
||||
}
|
||||
|
||||
// Done
|
||||
public override (Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>)? GetMangaFromUrl(string url)
|
||||
{
|
||||
RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) {
|
||||
return null;
|
||||
}
|
||||
if (requestResult.htmlDocument is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
Regex regex = new Regex(@".*webtoons\.com/en/(?<category>[^/]+)/(?<title>[^/]+)/list\?title_no=(?<id>\d+).*");
|
||||
Match match = regex.Match(url);
|
||||
|
||||
if(match.Success) {
|
||||
PublicationManager pm = new PublicationManager(match.Groups["title"].Value, match.Groups["category"].Value, match.Groups["id"].Value);
|
||||
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, pm.getPublicationId(), url);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Done
|
||||
private (Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>)[] ParsePublicationsFromHtml(HtmlDocument document)
|
||||
{
|
||||
HtmlNode mangaList = document.DocumentNode.SelectSingleNode("//ul[contains(@class, 'card_lst')]");
|
||||
if (!mangaList.ChildNodes.Any(node => node.Name == "li")) {
|
||||
return [];
|
||||
}
|
||||
|
||||
List<string> urls = document.DocumentNode
|
||||
.SelectNodes("//ul[contains(@class, 'card_lst')]/li/a")
|
||||
.Select(node => node.GetAttributeValue("href", "https://www.webtoons.com"))
|
||||
.ToList();
|
||||
|
||||
List<(Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>)> ret = new();
|
||||
foreach (string url in urls)
|
||||
{
|
||||
(Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>)? manga = GetMangaFromUrl(url);
|
||||
if(manga is { } m)
|
||||
ret.Add(m);
|
||||
}
|
||||
|
||||
return ret.ToArray();
|
||||
}
|
||||
|
||||
private string capitalizeString(string str = "") {
|
||||
if(str.Length == 0) return "";
|
||||
if(str.Length == 1) return str.ToUpper();
|
||||
return char.ToUpper(str[0]) + str.Substring(1).ToLower();
|
||||
}
|
||||
|
||||
// Done
|
||||
private (Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>) ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl)
|
||||
{
|
||||
HtmlNode infoNode1 = document.DocumentNode.SelectSingleNode("//*[@id='content']/div[2]/div[1]/div[1]");
|
||||
HtmlNode infoNode2 = document.DocumentNode.SelectSingleNode("//*[@id='content']/div[2]/div[2]/div[2]");
|
||||
|
||||
string sortName = infoNode1.SelectSingleNode(".//h1[contains(@class, 'subj')]").InnerText;
|
||||
string description = infoNode2.SelectSingleNode(".//p[contains(@class, 'summary')]")
|
||||
.InnerText.Trim();
|
||||
|
||||
HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//div[contains(@class, 'detail_body') and contains(@class, 'banner')]");
|
||||
|
||||
Regex regex = new Regex(@"url\((?<url>.*?)\)");
|
||||
Match match = regex.Match(posterNode.GetAttributeValue("style", ""));
|
||||
|
||||
string coverUrl = match.Groups["url"].Value;
|
||||
|
||||
string genre = infoNode1.SelectSingleNode(".//h2[contains(@class, 'genre')]")
|
||||
.InnerText.Trim();
|
||||
List<MangaTag> mangaTags = [new MangaTag(genre)];
|
||||
|
||||
List<HtmlNode> authorsNodes = infoNode1.SelectSingleNode(".//div[contains(@class, 'author_area')]").Descendants("a").ToList();
|
||||
List<Author> authors = authorsNodes.Select(node => new Author(node.InnerText.Trim())).ToList();
|
||||
|
||||
string originalLanguage = "";
|
||||
|
||||
uint year = 0;
|
||||
|
||||
string status1 = infoNode2.SelectSingleNode(".//p").InnerText;
|
||||
string status2 = infoNode2.SelectSingleNode(".//p/span").InnerText;
|
||||
MangaReleaseStatus releaseStatus = MangaReleaseStatus.Unreleased;
|
||||
if(status2.Length == 0 || status1.ToLower() == "completed") {
|
||||
releaseStatus = MangaReleaseStatus.Completed;
|
||||
} else if(status2.ToLower() == "up") {
|
||||
releaseStatus = MangaReleaseStatus.Continuing;
|
||||
}
|
||||
|
||||
Manga manga = new (publicationId, sortName, description, websiteUrl, coverUrl, null, year,
|
||||
originalLanguage, releaseStatus, -1,
|
||||
this,
|
||||
authors,
|
||||
mangaTags,
|
||||
[],
|
||||
[]);
|
||||
|
||||
return (manga, authors, mangaTags, [], []);
|
||||
}
|
||||
|
||||
// Done
|
||||
public override Chapter[] GetChapters(Manga manga, string language = "en")
|
||||
{
|
||||
PublicationManager pm = new(manga.MangaId);
|
||||
string requestUrl = $"https://www.webtoons.com/en/{pm.Category}/{pm.Title}/list?title_no={pm.Id}";
|
||||
// Leaving this in for verification if the page exists
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return Array.Empty<Chapter>();
|
||||
|
||||
// Get number of pages
|
||||
int pages = requestResult.htmlDocument.DocumentNode
|
||||
.SelectNodes("//div[contains(@class, 'paginate')]/a")
|
||||
.ToList()
|
||||
.Count;
|
||||
List<Chapter> chapters = new List<Chapter>();
|
||||
|
||||
for(int page = 1; page <= pages; page++) {
|
||||
string pageRequestUrl = $"{requestUrl}&page={page}";
|
||||
chapters.AddRange(ParseChaptersFromHtml(manga, pageRequestUrl));
|
||||
}
|
||||
|
||||
return chapters.Order().ToArray();
|
||||
}
|
||||
|
||||
// Done
|
||||
private List<Chapter> ParseChaptersFromHtml(Manga manga, string mangaUrl)
|
||||
{
|
||||
RequestResult result = downloadClient.MakeRequest(mangaUrl, RequestType.Default);
|
||||
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null)
|
||||
{
|
||||
return new List<Chapter>();
|
||||
}
|
||||
|
||||
List<Chapter> ret = new();
|
||||
|
||||
foreach (HtmlNode chapterInfo in result.htmlDocument.DocumentNode.SelectNodes("//ul/li[contains(@class, '_episodeItem')]"))
|
||||
{
|
||||
HtmlNode infoNode = chapterInfo.SelectSingleNode(".//a");
|
||||
string url = infoNode.GetAttributeValue("href", "");
|
||||
|
||||
string id = chapterInfo.GetAttributeValue("id", "");
|
||||
if(id == "") continue;
|
||||
string chapterNumber = chapterInfo.GetAttributeValue("data-episode-no", "");
|
||||
if(chapterNumber == "") continue;
|
||||
string chapterName = infoNode.SelectSingleNode(".//span[contains(@class, 'subj')]/span").InnerText.Trim();
|
||||
ret.Add(new Chapter(manga, url, chapterNumber, null, chapterName));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
internal override string[] GetChapterImageUrls(Chapter chapter)
|
||||
{
|
||||
string requestUrl = chapter.Url;
|
||||
// Leaving this in to check if the page exists
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
string[] imageUrls = ParseImageUrlsFromHtml(requestUrl);
|
||||
return imageUrls;
|
||||
}
|
||||
|
||||
private string[] ParseImageUrlsFromHtml(string mangaUrl)
|
||||
{
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(mangaUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
if (requestResult.htmlDocument is null)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
return requestResult.htmlDocument.DocumentNode
|
||||
.SelectNodes("//*[@id='_imageList']/img")
|
||||
.Select(node =>
|
||||
node.GetAttributeValue("data-url", ""))
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
internal class PublicationManager {
|
||||
public PublicationManager(string title = "", string category = "", string id = "") {
|
||||
this.Title = title;
|
||||
this.Category = category;
|
||||
this.Id = id;
|
||||
}
|
||||
|
||||
public PublicationManager(string publicationId) {
|
||||
string[] parts = publicationId.Split("|");
|
||||
if(parts.Length == 3) {
|
||||
this.Title = parts[0];
|
||||
this.Category = parts[1];
|
||||
this.Id = parts[2];
|
||||
} else {
|
||||
this.Title = "";
|
||||
this.Category = "";
|
||||
this.Id = "";
|
||||
}
|
||||
}
|
||||
|
||||
public string getPublicationId() {
|
||||
return $"{this.Title}|{this.Category}|{this.Id}";
|
||||
}
|
||||
|
||||
public string Title { get; set; }
|
||||
public string Category { get; set; }
|
||||
public string Id { get; set; }
|
||||
}
|
||||
@@ -1,175 +0,0 @@
|
||||
using System.Net;
|
||||
using System.Text.RegularExpressions;
|
||||
using API.MangaDownloadClients;
|
||||
using HtmlAgilityPack;
|
||||
|
||||
namespace API.Schema.MangaConnectors;
|
||||
|
||||
public class Weebcentral : MangaConnector
|
||||
{
|
||||
private readonly string[] _filterWords =
|
||||
{ "a", "the", "of", "as", "to", "no", "for", "on", "with", "be", "and", "in", "wa", "at", "be", "ni" };
|
||||
|
||||
public Weebcentral() : base("Weebcentral", ["en"], ["weebcentral.com"], "https://weebcentral.com/favicon.ico")
|
||||
{
|
||||
downloadClient = new ChromiumDownloadClient();
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(string publicationTitle = "")
|
||||
{
|
||||
const int limit = 32; //How many values we want returned at once
|
||||
var offset = 0; //"Page"
|
||||
var requestUrl =
|
||||
$"https://{BaseUris[0]}/search/data?limit={limit}&offset={offset}&text={publicationTitle}&sort=Best+Match&order=Ascending&official=Any&display_mode=Minimal%20Display";
|
||||
var requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 ||
|
||||
requestResult.htmlDocument == null)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var publications = ParsePublicationsFromHtml(requestResult.htmlDocument);
|
||||
|
||||
return publications;
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] ParsePublicationsFromHtml(HtmlDocument document)
|
||||
{
|
||||
if (document.DocumentNode.SelectNodes("//article").Count < 1)
|
||||
return [];
|
||||
|
||||
var urls = document.DocumentNode.SelectNodes("/html/body/article/a[contains(concat(' ',normalize-space(@class),' '),' link ')]")
|
||||
.Select(elem => elem.GetAttributeValue("href", "")).ToList();
|
||||
|
||||
List<(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)> ret = new();
|
||||
foreach (var url in urls)
|
||||
{
|
||||
(Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? manga = GetMangaFromUrl(url);
|
||||
if (manga is { })
|
||||
ret.Add(((Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?))manga);
|
||||
}
|
||||
|
||||
return ret.ToArray();
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromUrl(string url)
|
||||
{
|
||||
Regex publicationIdRex = new(@"https:\/\/weebcentral\.com\/series\/(\w*)\/(.*)");
|
||||
var publicationId = publicationIdRex.Match(url).Groups[1].Value;
|
||||
|
||||
var requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||
if ((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 &&
|
||||
requestResult.htmlDocument is not null)
|
||||
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url);
|
||||
return null;
|
||||
}
|
||||
|
||||
private (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?) ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl)
|
||||
{
|
||||
HtmlNode posterNode =
|
||||
document.DocumentNode.SelectSingleNode("//section[@class='flex items-center justify-center']/picture/img");
|
||||
string posterUrl = posterNode?.GetAttributeValue("src", "") ?? "";
|
||||
|
||||
HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//section/h1");
|
||||
string sortName = titleNode?.InnerText ?? "Undefined";
|
||||
|
||||
HtmlNode[] authorsNodes =
|
||||
document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Author(s): ']/span").ToArray();
|
||||
List<Author> authors = authorsNodes.Select(n => new Author(n.InnerText)).ToList();
|
||||
|
||||
HtmlNode[] genreNodes =
|
||||
document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Tags(s): ']/span").ToArray();
|
||||
List<MangaTag> tags = genreNodes.Select(n => new MangaTag(n.InnerText.EndsWith(',') ? n.InnerText.Substring(0,n.InnerText.Length-1) : n.InnerText)).ToList();
|
||||
|
||||
HtmlNode statusNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Status: ']/a");
|
||||
string statusText = statusNode?.InnerText ?? "";
|
||||
MangaReleaseStatus releaseStatus = statusText.ToLower() switch
|
||||
{
|
||||
"cancelled" => MangaReleaseStatus.Cancelled,
|
||||
"hiatus" => MangaReleaseStatus.OnHiatus,
|
||||
"complete" => MangaReleaseStatus.Completed,
|
||||
"ongoing" => MangaReleaseStatus.Continuing,
|
||||
_ => MangaReleaseStatus.Unreleased
|
||||
};
|
||||
|
||||
HtmlNode yearNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Released: ']/span");
|
||||
uint year = Convert.ToUInt32(yearNode?.InnerText ?? "0");
|
||||
|
||||
HtmlNode descriptionNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Description']/p");
|
||||
string description = descriptionNode?.InnerText ?? "Undefined";
|
||||
|
||||
HtmlNode[] altTitleNodes = document.DocumentNode
|
||||
.SelectNodes("//ul/li[strong/text() = 'Associated Name(s)']/ul/li")?.ToArray() ?? [];
|
||||
List<MangaAltTitle> altTitles = altTitleNodes.Select(n => new MangaAltTitle("", n.InnerText)).ToList();
|
||||
|
||||
Manga m = new(publicationId, sortName, description, websiteUrl, posterUrl, null, year, null, releaseStatus, -1,
|
||||
this, authors, tags, [], altTitles);
|
||||
return (m, authors, tags, [], altTitles);
|
||||
}
|
||||
|
||||
public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)? GetMangaFromId(string publicationId)
|
||||
{
|
||||
return GetMangaFromUrl($"https://{BaseUris[0]}/series/{publicationId}");
|
||||
}
|
||||
|
||||
public override Chapter[] GetChapters(Manga manga, string language = "en")
|
||||
{
|
||||
var requestUrl = $"https://{BaseUris[0]}/series/{manga.MangaConnectorId}/full-chapter-list";
|
||||
var requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return [];
|
||||
|
||||
//Return Chapters ordered by Chapter-Number
|
||||
if (requestResult.htmlDocument is null)
|
||||
return [];
|
||||
var chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument);
|
||||
return chapters.Order().ToArray();
|
||||
}
|
||||
|
||||
internal override string[] GetChapterImageUrls(Chapter chapter)
|
||||
{
|
||||
var requestResult = downloadClient.MakeRequest(chapter.Url, RequestType.Default);
|
||||
if (requestResult.htmlDocument is null)
|
||||
return [];
|
||||
|
||||
var document = requestResult.htmlDocument;
|
||||
|
||||
var imageNodes =
|
||||
document.DocumentNode.SelectNodes($"//section[@hx-get='{chapter.Url}/images']/img")?.ToArray() ?? [];
|
||||
var urls = imageNodes.Select(imgNode => imgNode.GetAttributeValue("src", "")).ToArray();
|
||||
|
||||
return urls;
|
||||
}
|
||||
|
||||
private List<Chapter> ParseChaptersFromHtml(Manga manga, HtmlDocument document)
|
||||
{
|
||||
var chaptersWrapper = document.DocumentNode.SelectSingleNode("/html/body");
|
||||
|
||||
Regex chapterRex = new(@"(\d+(?:\.\d+)*)");
|
||||
Regex idRex = new(@"https:\/\/weebcentral\.com\/chapters\/(\w*)");
|
||||
|
||||
var ret = chaptersWrapper.Descendants("a").Select(elem =>
|
||||
{
|
||||
var url = elem.GetAttributeValue("href", "") ?? "Undefined";
|
||||
|
||||
if (!url.StartsWith("https://") && !url.StartsWith("http://"))
|
||||
return new Chapter(manga, "", "");
|
||||
|
||||
var idMatch = idRex.Match(url);
|
||||
var id = idMatch.Success ? idMatch.Groups[1].Value : null;
|
||||
|
||||
var chapterNode = elem.SelectSingleNode("span[@class='grow flex items-center gap-2']/span")?.InnerText ??
|
||||
"Undefined";
|
||||
|
||||
var chapterNumberMatch = chapterRex.Match(chapterNode);
|
||||
var chapterNumber = chapterNumberMatch.Success ? chapterNumberMatch.Groups[1].Value : "-1";
|
||||
|
||||
return new Chapter(manga, url, chapterNumber);
|
||||
}).Where(elem => elem.ChapterNumber != String.Empty && elem.Url != string.Empty).ToList();
|
||||
|
||||
ret.Reverse();
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user