diff --git a/Tranga.sln.DotSettings b/Tranga.sln.DotSettings index 05f49b5..4d3085b 100644 --- a/Tranga.sln.DotSettings +++ b/Tranga.sln.DotSettings @@ -1,4 +1,5 @@  True + True True True \ No newline at end of file diff --git a/Tranga/Connector.cs b/Tranga/Connector.cs index 932bfb5..24e6ad2 100644 --- a/Tranga/Connector.cs +++ b/Tranga/Connector.cs @@ -126,9 +126,9 @@ public abstract class Connector /// /// /// RequestType for Rate-Limit - private void DownloadImage(string imageUrl, string fullPath, byte requestType) + private void DownloadImage(string imageUrl, string fullPath, byte requestType, string? referrer = null) { - DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType); + DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType, referrer); byte[] buffer = new byte[requestResult.result.Length]; requestResult.result.ReadExactly(buffer, 0, buffer.Length); File.WriteAllBytes(fullPath, buffer); @@ -141,7 +141,7 @@ public abstract class Connector /// Full path to save archive to (without file ending .cbz) /// Path of the generate Chapter ComicInfo.xml, if it was generated /// RequestType for RateLimits - protected void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, string? comicInfoPath = null) + protected void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, string? comicInfoPath = null, string? referrer = null) { logger?.WriteLine("Connector", $"Downloading Images for {saveArchiveFilePath}"); //Check if Publication Directory already exists @@ -162,7 +162,7 @@ public abstract class Connector string[] split = imageUrl.Split('.'); string extension = split[^1]; logger?.WriteLine("Connector", $"Downloading Image {chapter + 1}/{imageUrls.Length}"); - DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType); + DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer); } if(comicInfoPath is not null) @@ -174,6 +174,23 @@ public abstract class Connector Directory.Delete(tempFolder, true); //Cleanup } + protected string SaveCoverImageToCache(string url, byte requestType) + { + string[] split = url.Split('/'); + string filename = split[^1]; + string saveImagePath = Path.Join(imageCachePath, filename); + + if (File.Exists(saveImagePath)) + return filename; + + DownloadClient.RequestResult coverResult = downloadClient.MakeRequest(url, requestType); + using MemoryStream ms = new(); + coverResult.result.CopyTo(ms); + File.WriteAllBytes(saveImagePath, ms.ToArray()); + logger?.WriteLine(this.GetType().ToString(), $"Saving image to {saveImagePath}"); + return filename; + } + protected class DownloadClient { private static readonly HttpClient Client = new(); @@ -202,7 +219,7 @@ public abstract class Connector /// /// For RateLimits: Same Endpoints use same type /// RequestResult with StatusCode and Stream of received data - public RequestResult MakeRequest(string url, byte requestType) + public RequestResult MakeRequest(string url, byte requestType, string? referrer = null) { if (_rateLimit.TryGetValue(requestType, out TimeSpan value)) _lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value)); @@ -224,6 +241,8 @@ public abstract class Connector try { HttpRequestMessage requestMessage = new(HttpMethod.Get, url); + if(referrer is not null) + requestMessage.Headers.Referrer = new Uri(referrer); _lastExecutedRateLimit[requestType] = DateTime.Now; response = Client.Send(requestMessage); } diff --git a/Tranga/Connectors/MangaDex.cs b/Tranga/Connectors/MangaDex.cs index b3bad9b..dd2cdb4 100644 --- a/Tranga/Connectors/MangaDex.cs +++ b/Tranga/Connectors/MangaDex.cs @@ -102,7 +102,7 @@ public class MangaDex : Connector string? coverUrl = GetCoverUrl(publicationId, posterId); string? coverCacheName = null; if (coverUrl is not null) - coverCacheName = SaveImage(coverUrl); + coverCacheName = SaveCoverImageToCache(coverUrl, (byte)RequestType.AtHomeServer); string? author = GetAuthor(authorId); @@ -273,21 +273,4 @@ public class MangaDex : Connector logger?.WriteLine(this.GetType().ToString(), $"Got author {authorId} -> {author}"); return author; } - - private string SaveImage(string url) - { - string[] split = url.Split('/'); - string filename = split[^1]; - string saveImagePath = Path.Join(imageCachePath, filename); - - if (File.Exists(saveImagePath)) - return filename; - - DownloadClient.RequestResult coverResult = downloadClient.MakeRequest(url, (byte)RequestType.AtHomeServer); - using MemoryStream ms = new(); - coverResult.result.CopyTo(ms); - File.WriteAllBytes(saveImagePath, ms.ToArray()); - logger?.WriteLine(this.GetType().ToString(), $"Saving image to {saveImagePath}"); - return filename; - } } \ No newline at end of file diff --git a/Tranga/Connectors/Manganato.cs b/Tranga/Connectors/Manganato.cs new file mode 100644 index 0000000..4123ec2 --- /dev/null +++ b/Tranga/Connectors/Manganato.cs @@ -0,0 +1,187 @@ +using System.Collections; +using System.Net; +using System.Text.RegularExpressions; +using HtmlAgilityPack; +using Logging; + +namespace Tranga.Connectors; + +public class Manganato : Connector +{ + public override string name { get; } + + public Manganato(string downloadLocation, string imageCachePath, Logger? logger) : base(downloadLocation, imageCachePath, logger) + { + this.name = "Manganato"; + this.downloadClient = new DownloadClient(new Dictionary() + { + {(byte)1, 100} + }, logger); + } + + public override Publication[] GetPublications(string publicationTitle = "") + { + string sanitizedTitle = publicationTitle.ToLower().Replace(' ', '_'); + logger?.WriteLine(this.GetType().ToString(), $"Getting Publications (title={sanitizedTitle})"); + string requestUrl = $"https://manganato.com/search/story/{sanitizedTitle}"; + DownloadClient.RequestResult requestResult = + downloadClient.MakeRequest(requestUrl, (byte)1); + if (requestResult.statusCode != HttpStatusCode.OK) + return Array.Empty(); + + return ParsePublicationsFromHtml(requestResult.result); + } + + private Publication[] ParsePublicationsFromHtml(Stream html) + { + StreamReader reader = new (html); + string htmlString = reader.ReadToEnd(); + HtmlDocument document = new (); + document.LoadHtml(htmlString); + IEnumerable searchResults = document.DocumentNode.Descendants("div").Where(n => n.HasClass("search-story-item")); + List urls = new(); + foreach (HtmlNode mangaResult in searchResults) + { + urls.Add(mangaResult.Descendants("a").First(n => n.HasClass("item-title")).GetAttributes() + .First(a => a.Name == "href").Value); + } + + HashSet ret = new(); + foreach (string url in urls) + { + DownloadClient.RequestResult requestResult = + downloadClient.MakeRequest(url, (byte)1); + if (requestResult.statusCode != HttpStatusCode.OK) + return Array.Empty(); + + ret.Add(ParseSinglePublicationFromHtml(requestResult.result, url.Split('/')[^1])); + } + + return ret.ToArray(); + } + + private Publication ParseSinglePublicationFromHtml(Stream html, string publicationId) + { + StreamReader reader = new (html); + string htmlString = reader.ReadToEnd(); + HtmlDocument document = new (); + document.LoadHtml(htmlString); + string status = ""; + Dictionary altTitles = new(); + Dictionary? links = null; + HashSet tags = new(); + string? author = null, originalLanguage = null; + int? year = null; + + HtmlNode infoNode = document.DocumentNode.Descendants("div").First(d => d.HasClass("story-info-right")); + + string sortName = infoNode.Descendants("h1").First().InnerText; + + HtmlNode infoTable = infoNode.Descendants().First(d => d.Name == "table"); + + foreach (HtmlNode row in infoTable.Descendants("tr")) + { + string key = row.SelectNodes("td").First().InnerText.ToLower(); + string value = row.SelectNodes("td").Last().InnerText; + string keySanitized = string.Concat(Regex.Matches(key, "[a-z]")); + + switch (keySanitized) + { + case "alternative": + altTitles.Add("",value); + break; + case "authors": + author = value; + break; + case "status": + status = value; + break; + case "genres": + string[] genres = value.Split(" - "); + tags = genres.ToHashSet(); + break; + default: break; + } + } + + string posterUrl = document.DocumentNode.Descendants("span").First(s => s.HasClass("info-image")).Descendants("img").First() + .GetAttributes().First(a => a.Name == "src").Value; + + string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1); + + string description = document.DocumentNode.Descendants("div").First(d => d.HasClass("panel-story-info-description")) + .InnerText; + + + return new Publication(sortName, author, description, altTitles, tags.ToArray(), posterUrl, coverFileNameInCache, links, + year, originalLanguage, status, publicationId); + } + + public override Chapter[] GetChapters(Publication publication, string language = "") + { + string requestUrl = $"https://manganato.com/{publication.publicationId}"; + DownloadClient.RequestResult requestResult = + downloadClient.MakeRequest(requestUrl, (byte)1); + if (requestResult.statusCode != HttpStatusCode.OK) + return Array.Empty(); + + return ParseChaptersFromHtml(requestResult.result); + } + + private Chapter[] ParseChaptersFromHtml(Stream html) + { + StreamReader reader = new (html); + string htmlString = reader.ReadToEnd(); + HtmlDocument document = new (); + document.LoadHtml(htmlString); + List ret = new(); + + HtmlNode chapterList = document.DocumentNode.Descendants("ul").First(l => l.HasClass("row-content-chapter")); + + foreach (HtmlNode chapterInfo in chapterList.Descendants("li")) + { + string fullString = chapterInfo.Descendants("a").First(d => d.HasClass("chapter-name")).InnerText; + + string? volumeNumber = fullString.Contains("Vol.") ? fullString.Replace("Vol.", "").Split(' ')[0] : null; + string? chapterNumber = fullString.Split(':')[0].Split(' ')[^1]; + string chapterName = string.Concat(fullString.Split(':')[1..]); + string url = chapterInfo.Descendants("a").First(d => d.HasClass("chapter-name")) + .GetAttributeValue("href", ""); + ret.Add(new Chapter(chapterName, volumeNumber, chapterNumber, url)); + } + + return ret.ToArray(); + } + + public override void DownloadChapter(Publication publication, Chapter chapter) + { + string requestUrl = chapter.url; + DownloadClient.RequestResult requestResult = + downloadClient.MakeRequest(requestUrl, (byte)1); + if (requestResult.statusCode != HttpStatusCode.OK) + return; + + string[] imageUrls = ParseImageUrlsFromHtml(requestResult.result); + + string comicInfoPath = Path.GetTempFileName(); + File.WriteAllText(comicInfoPath, GetComicInfoXmlString(publication, chapter, logger)); + + DownloadChapterImages(imageUrls, GetArchiveFilePath(publication, chapter), (byte)1, comicInfoPath, "https://chapmanganato.com/"); + } + + private string[] ParseImageUrlsFromHtml(Stream html) + { + StreamReader reader = new (html); + string htmlString = reader.ReadToEnd(); + HtmlDocument document = new (); + document.LoadHtml(htmlString); + List ret = new(); + + HtmlNode imageContainer = + document.DocumentNode.Descendants("div").First(i => i.HasClass("container-chapter-reader")); + foreach(HtmlNode imageNode in imageContainer.Descendants("img")) + ret.Add(imageNode.GetAttributeValue("src", "")); + + return ret.ToArray(); + } +} \ No newline at end of file diff --git a/Tranga/TaskManager.cs b/Tranga/TaskManager.cs index e7cfaf2..705062e 100644 --- a/Tranga/TaskManager.cs +++ b/Tranga/TaskManager.cs @@ -39,7 +39,11 @@ public class TaskManager this.settings = new TrangaSettings(downloadFolderPath, workingDirectory, newKomga); ExportDataAndSettings(); - this._connectors = new Connector[]{ new MangaDex(downloadFolderPath, imageCachePath, logger) }; + this._connectors = new Connector[] + { + new MangaDex(downloadFolderPath, imageCachePath, logger), + new Manganato(downloadFolderPath, imageCachePath, logger) + }; foreach(Connector cConnector in this._connectors) _taskQueue.Add(cConnector, new List()); @@ -59,7 +63,11 @@ public class TaskManager public TaskManager(TrangaSettings settings, Logger? logger = null) { this.logger = logger; - this._connectors = new Connector[]{ new MangaDex(settings.downloadLocation, settings.coverImageCache, logger) }; + this._connectors = new Connector[] + { + new MangaDex(settings.downloadLocation, settings.coverImageCache, logger), + new Manganato(settings.downloadLocation, settings.coverImageCache, logger) + }; foreach(Connector cConnector in this._connectors) _taskQueue.Add(cConnector, new List()); _allTasks = new HashSet(); diff --git a/Tranga/Tranga.csproj b/Tranga/Tranga.csproj index 6ddc3bb..d735d48 100644 --- a/Tranga/Tranga.csproj +++ b/Tranga/Tranga.csproj @@ -7,6 +7,7 @@ +