glax/Tranga
1
0
Fork 0
mirror of https://github.com/C9Glax/tranga.git synced 2025-02-23 07:40:13 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Webtoons

Browse Source
This commit is contained in:
Glax 2025-02-09 17:53:22 +01:00
parent ebb034e0c7
commit 9928abb674
1 changed files with 42 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
API/Schema/MangaConnectors/Webtoons.cs
View File

@ -1,51 +1,48 @@
using System.Net; using System.Net;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using API.MangaDownloadClients;
using HtmlAgilityPack; using HtmlAgilityPack;
using Tranga.Jobs;
namespace Tranga.MangaConnectors; namespace API.Schema.MangaConnectors;
public class Webtoons : MangaConnector public class Webtoons : MangaConnector
{ {
public Webtoons(GlobalBase clone) : base(clone, "Webtoons", ["en"]) public Webtoons() : base("Webtoons", ["en"], ["https://www.webtoons.com"])
{ {
this.downloadClient = new HttpDownloadClient(clone); this.downloadClient = new HttpDownloadClient();
} }
// Done // Done
public override Manga[] GetManga(string publicationTitle = "") public override (Manga, List<Author>?, List<MangaTag>?, List<Link>?, List<MangaAltTitle>?)[] GetManga(string publicationTitle = "")
{ {
string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower(); string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower();
Log($"Searching Publications. Term=\"{publicationTitle}\"");
string requestUrl = $"https://www.webtoons.com/en/search?keyword={sanitizedTitle}&searchType=WEBTOON"; string requestUrl = $"https://www.webtoons.com/en/search?keyword={sanitizedTitle}&searchType=WEBTOON";
RequestResult requestResult = RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, RequestType.Default); downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) { if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) {
Log($"Failed to retrieve site"); return [];
return Array.Empty<Manga>();
} }
if (requestResult.htmlDocument is null) if (requestResult.htmlDocument is null)
{ {
Log($"Failed to retrieve site"); return [];
return Array.Empty<Manga>();
} }
Manga[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument); (Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>)[] publications =
Log($"Retrieved {publications.Length} publications. Term=\"{publicationTitle}\""); ParsePublicationsFromHtml(requestResult.htmlDocument);
return publications; return publications;
} }
// Done // Done
public override Manga? GetMangaFromId(string publicationId) public override (Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>)? GetMangaFromId(string publicationId)
{ {
PublicationManager pb = new PublicationManager(publicationId); PublicationManager pb = new PublicationManager(publicationId);
return GetMangaFromUrl($"https://www.webtoons.com/en/{pb.Category}/{pb.Title}/list?title_no={pb.Id}"); return GetMangaFromUrl($"https://www.webtoons.com/en/{pb.Category}/{pb.Title}/list?title_no={pb.Id}");
} }
// Done // Done
public override Manga? GetMangaFromUrl(string url) public override (Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>)? GetMangaFromUrl(string url)
{ {
RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo); RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) { if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) {
@ -53,7 +50,6 @@ public class Webtoons : MangaConnector
} }
if (requestResult.htmlDocument is null) if (requestResult.htmlDocument is null)
{ {
Log($"Failed to retrieve site");
return null; return null;
} }
Regex regex = new Regex(@".*webtoons\.com/en/(?<category>[^/]+)/(?<title>[^/]+)/list\?title_no=(?<id>\d+).*"); Regex regex = new Regex(@".*webtoons\.com/en/(?<category>[^/]+)/(?<title>[^/]+)/list\?title_no=(?<id>\d+).*");
@ -63,17 +59,15 @@ public class Webtoons : MangaConnector
PublicationManager pm = new PublicationManager(match.Groups["title"].Value, match.Groups["category"].Value, match.Groups["id"].Value); PublicationManager pm = new PublicationManager(match.Groups["title"].Value, match.Groups["category"].Value, match.Groups["id"].Value);
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, pm.getPublicationId(), url); return ParseSinglePublicationFromHtml(requestResult.htmlDocument, pm.getPublicationId(), url);
} }
Log($"Failed match Regex ID");
return null; return null;
} }
// Done // Done
private Manga[] ParsePublicationsFromHtml(HtmlDocument document) private (Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>)[] ParsePublicationsFromHtml(HtmlDocument document)
{ {
HtmlNode mangaList = document.DocumentNode.SelectSingleNode("//ul[contains(@class, 'card_lst')]"); HtmlNode mangaList = document.DocumentNode.SelectSingleNode("//ul[contains(@class, 'card_lst')]");
if (!mangaList.ChildNodes.Any(node => node.Name == "li")) { if (!mangaList.ChildNodes.Any(node => node.Name == "li")) {
Log($"Failed to parse publication"); return [];
return Array.Empty<Manga>();
} }
List<string> urls = document.DocumentNode List<string> urls = document.DocumentNode
@ -81,12 +75,12 @@ public class Webtoons : MangaConnector
.Select(node => node.GetAttributeValue("href", "https://www.webtoons.com")) .Select(node => node.GetAttributeValue("href", "https://www.webtoons.com"))
.ToList(); .ToList();
HashSet<Manga> ret = new(); List<(Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>)> ret = new();
foreach (string url in urls) foreach (string url in urls)
{ {
Manga? manga = GetMangaFromUrl(url); (Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>)? manga = GetMangaFromUrl(url);
if (manga is not null) if(manga is { } m)
ret.Add((Manga)manga); ret.Add(m);
} }
return ret.ToArray(); return ret.ToArray();
@ -99,7 +93,7 @@ public class Webtoons : MangaConnector
} }
// Done // Done
private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl) private (Manga, List<Author>, List<MangaTag>, List<Link>, List<MangaAltTitle>) ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl)
{ {
HtmlNode infoNode1 = document.DocumentNode.SelectSingleNode("//*[@id='content']/div[2]/div[1]/div[1]"); HtmlNode infoNode1 = document.DocumentNode.SelectSingleNode("//*[@id='content']/div[2]/div[1]/div[1]");
HtmlNode infoNode2 = document.DocumentNode.SelectSingleNode("//*[@id='content']/div[2]/div[2]/div[2]"); HtmlNode infoNode2 = document.DocumentNode.SelectSingleNode("//*[@id='content']/div[2]/div[2]/div[2]");
@ -113,39 +107,43 @@ public class Webtoons : MangaConnector
Regex regex = new Regex(@"url\((?<url>.*?)\)"); Regex regex = new Regex(@"url\((?<url>.*?)\)");
Match match = regex.Match(posterNode.GetAttributeValue("style", "")); Match match = regex.Match(posterNode.GetAttributeValue("style", ""));
string posterUrl = match.Groups["url"].Value; string coverUrl = match.Groups["url"].Value;
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover, websiteUrl);
string genre = infoNode1.SelectSingleNode(".//h2[contains(@class, 'genre')]") string genre = infoNode1.SelectSingleNode(".//h2[contains(@class, 'genre')]")
.InnerText.Trim(); .InnerText.Trim();
string[] tags = [ genre ]; List<MangaTag> mangaTags = [new MangaTag(genre)];
List<HtmlNode> authorsNodes = infoNode1.SelectSingleNode(".//div[contains(@class, 'author_area')]").Descendants("a").ToList(); List<HtmlNode> authorsNodes = infoNode1.SelectSingleNode(".//div[contains(@class, 'author_area')]").Descendants("a").ToList();
List<string> authors = authorsNodes.Select(node => node.InnerText.Trim()).ToList(); List<Author> authors = authorsNodes.Select(node => new Author(node.InnerText.Trim())).ToList();
string originalLanguage = ""; string originalLanguage = "";
int year = DateTime.Now.Year; uint year = 0;
string status1 = infoNode2.SelectSingleNode(".//p").InnerText; string status1 = infoNode2.SelectSingleNode(".//p").InnerText;
string status2 = infoNode2.SelectSingleNode(".//p/span").InnerText; string status2 = infoNode2.SelectSingleNode(".//p/span").InnerText;
Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased; MangaReleaseStatus releaseStatus = MangaReleaseStatus.Unreleased;
if(status2.Length == 0 || status1.ToLower() == "completed") { if(status2.Length == 0 || status1.ToLower() == "completed") {
releaseStatus = Manga.ReleaseStatusByte.Completed; releaseStatus = MangaReleaseStatus.Completed;
} else if(status2.ToLower() == "up") { } else if(status2.ToLower() == "up") {
releaseStatus = Manga.ReleaseStatusByte.Continuing; releaseStatus = MangaReleaseStatus.Continuing;
} }
Manga manga = new(sortName, authors, description, new Dictionary<string, string>(), tags, posterUrl, coverFileNameInCache, new Dictionary<string, string>(), Manga manga = new (publicationId, sortName, description, websiteUrl, coverUrl, null, year,
year, originalLanguage, publicationId, releaseStatus, websiteUrl: websiteUrl); originalLanguage, releaseStatus, -1,
AddMangaToCache(manga); this,
return manga; authors,
mangaTags,
[],
[]);
return (manga, authors, mangaTags, [], []);
} }
// Done // Done
public override Chapter[] GetChapters(Manga manga, string language = "en") public override Chapter[] GetChapters(Manga manga, string language = "en")
{ {
PublicationManager pm = new PublicationManager(manga.publicationId); PublicationManager pm = new(manga.MangaId);
string requestUrl = $"https://www.webtoons.com/en/{pm.Category}/{pm.Title}/list?title_no={pm.Id}"; string requestUrl = $"https://www.webtoons.com/en/{pm.Category}/{pm.Title}/list?title_no={pm.Id}";
// Leaving this in for verification if the page exists // Leaving this in for verification if the page exists
RequestResult requestResult = RequestResult requestResult =
@ -163,7 +161,6 @@ public class Webtoons : MangaConnector
chapters.AddRange(ParseChaptersFromHtml(manga, pageRequestUrl)); chapters.AddRange(ParseChaptersFromHtml(manga, pageRequestUrl));
} }
Log($"Got {chapters.Count} chapters. {manga}");
return chapters.Order().ToArray(); return chapters.Order().ToArray();
} }
@ -173,7 +170,6 @@ public class Webtoons : MangaConnector
RequestResult result = downloadClient.MakeRequest(mangaUrl, RequestType.Default); RequestResult result = downloadClient.MakeRequest(mangaUrl, RequestType.Default);
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null) if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null)
{ {
Log("Failed to load site");
return new List<Chapter>(); return new List<Chapter>();
} }
@ -186,38 +182,28 @@ public class Webtoons : MangaConnector
string id = chapterInfo.GetAttributeValue("id", ""); string id = chapterInfo.GetAttributeValue("id", "");
if(id == "") continue; if(id == "") continue;
string? volumeNumber = null;
string chapterNumber = chapterInfo.GetAttributeValue("data-episode-no", ""); string chapterNumber = chapterInfo.GetAttributeValue("data-episode-no", "");
if(chapterNumber == "") continue; if(chapterNumber == "") continue;
string chapterName = infoNode.SelectSingleNode(".//span[contains(@class, 'subj')]/span").InnerText.Trim(); string chapterName = infoNode.SelectSingleNode(".//span[contains(@class, 'subj')]/span").InnerText.Trim();
ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url)); ret.Add(new Chapter(manga, url, chapterNumber, null, chapterName));
} }
return ret; return ret;
} }
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) internal override string[] GetChapterImageUrls(Chapter chapter)
{ {
if (progressToken?.cancellationRequested ?? false) string requestUrl = chapter.Url;
{
progressToken.Cancel();
return HttpStatusCode.RequestTimeout;
}
Manga chapterParentManga = chapter.parentManga;
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
string requestUrl = chapter.url;
// Leaving this in to check if the page exists // Leaving this in to check if the page exists
RequestResult requestResult = RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, RequestType.Default); downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{ {
progressToken?.Cancel(); return [];
return requestResult.statusCode;
} }
string[] imageUrls = ParseImageUrlsFromHtml(requestUrl); string[] imageUrls = ParseImageUrlsFromHtml(requestUrl);
return DownloadChapterImages(imageUrls, chapter, RequestType.MangaImage, progressToken:progressToken, referrer: requestUrl); return imageUrls;
} }
private string[] ParseImageUrlsFromHtml(string mangaUrl) private string[] ParseImageUrlsFromHtml(string mangaUrl)
@ -226,12 +212,11 @@ public class Webtoons : MangaConnector
downloadClient.MakeRequest(mangaUrl, RequestType.Default); downloadClient.MakeRequest(mangaUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{ {
return Array.Empty<string>(); return [];
} }
if (requestResult.htmlDocument is null) if (requestResult.htmlDocument is null)
{ {
Log($"Failed to retrieve site"); return [];
return Array.Empty<string>();
} }
return requestResult.htmlDocument.DocumentNode return requestResult.htmlDocument.DocumentNode