glax/Tranga
2
0
Fork 0
Code

#2 First Attempt

Browse Source
This commit is contained in:
glax 2023-06-01 13:13:53 +02:00
parent b24032d124
commit 8381951168
6 changed files with 224 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Tranga.sln.DotSettings
View File

@ -1,4 +1,5 @@
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
<s:Boolean x:Key="/Default/UserDictionary/Words/=Komga/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Manganato/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Taskmanager/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Tranga/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

29
Tranga/Connector.cs
View File

@ -126,9 +126,9 @@ public abstract class Connector
/// <param name="imageUrl"></param>
/// <param name="fullPath"></param>
/// <param name="requestType">RequestType for Rate-Limit</param>
private void DownloadImage(string imageUrl, string fullPath, byte requestType)
private void DownloadImage(string imageUrl, string fullPath, byte requestType, string? referrer = null)
{
DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType);
DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType, referrer);
byte[] buffer = new byte[requestResult.result.Length];
requestResult.result.ReadExactly(buffer, 0, buffer.Length);
File.WriteAllBytes(fullPath, buffer);
@ -141,7 +141,7 @@ public abstract class Connector
/// <param name="saveArchiveFilePath">Full path to save archive to (without file ending .cbz)</param>
/// <param name="comicInfoPath">Path of the generate Chapter ComicInfo.xml, if it was generated</param>
/// <param name="requestType">RequestType for RateLimits</param>
protected void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, string? comicInfoPath = null)
protected void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, string? comicInfoPath = null, string? referrer = null)
{
logger?.WriteLine("Connector", $"Downloading Images for {saveArchiveFilePath}");
//Check if Publication Directory already exists
@ -162,7 +162,7 @@ public abstract class Connector
string[] split = imageUrl.Split('.');
string extension = split[^1];
logger?.WriteLine("Connector", $"Downloading Image {chapter + 1}/{imageUrls.Length}");
DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType);
DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer);
}
if(comicInfoPath is not null)
@ -174,6 +174,23 @@ public abstract class Connector
Directory.Delete(tempFolder, true); //Cleanup
}
protected string SaveCoverImageToCache(string url, byte requestType)
{
string[] split = url.Split('/');
string filename = split[^1];
string saveImagePath = Path.Join(imageCachePath, filename);
if (File.Exists(saveImagePath))
return filename;
DownloadClient.RequestResult coverResult = downloadClient.MakeRequest(url, requestType);
using MemoryStream ms = new();
coverResult.result.CopyTo(ms);
File.WriteAllBytes(saveImagePath, ms.ToArray());
logger?.WriteLine(this.GetType().ToString(), $"Saving image to {saveImagePath}");
return filename;
}
protected class DownloadClient
{
private static readonly HttpClient Client = new();
@ -202,7 +219,7 @@ public abstract class Connector
/// <param name="url"></param>
/// <param name="requestType">For RateLimits: Same Endpoints use same type</param>
/// <returns>RequestResult with StatusCode and Stream of received data</returns>
public RequestResult MakeRequest(string url, byte requestType)
public RequestResult MakeRequest(string url, byte requestType, string? referrer = null)
{
if (_rateLimit.TryGetValue(requestType, out TimeSpan value))
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
@ -224,6 +241,8 @@ public abstract class Connector
try
{
HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
if(referrer is not null)
requestMessage.Headers.Referrer = new Uri(referrer);
_lastExecutedRateLimit[requestType] = DateTime.Now;
response = Client.Send(requestMessage);
}

19
Tranga/Connectors/MangaDex.cs
View File

@ -102,7 +102,7 @@ public class MangaDex : Connector
string? coverUrl = GetCoverUrl(publicationId, posterId);
string? coverCacheName = null;
if (coverUrl is not null)
coverCacheName = SaveImage(coverUrl);
coverCacheName = SaveCoverImageToCache(coverUrl, (byte)RequestType.AtHomeServer);
string? author = GetAuthor(authorId);
@ -273,21 +273,4 @@ public class MangaDex : Connector
logger?.WriteLine(this.GetType().ToString(), $"Got author {authorId} -> {author}");
return author;
}
private string SaveImage(string url)
{
string[] split = url.Split('/');
string filename = split[^1];
string saveImagePath = Path.Join(imageCachePath, filename);
if (File.Exists(saveImagePath))
return filename;
DownloadClient.RequestResult coverResult = downloadClient.MakeRequest(url, (byte)RequestType.AtHomeServer);
using MemoryStream ms = new();
coverResult.result.CopyTo(ms);
File.WriteAllBytes(saveImagePath, ms.ToArray());
logger?.WriteLine(this.GetType().ToString(), $"Saving image to {saveImagePath}");
return filename;
}
}

187
Tranga/Connectors/Manganato.cs Normal file
View File

@ -0,0 +1,187 @@
using System.Collections;
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using Logging;
namespace Tranga.Connectors;
public class Manganato : Connector
{
public override string name { get; }
public Manganato(string downloadLocation, string imageCachePath, Logger? logger) : base(downloadLocation, imageCachePath, logger)
{
this.name = "Manganato";
this.downloadClient = new DownloadClient(new Dictionary<byte, int>()
{
{(byte)1, 100}
}, logger);
}
public override Publication[] GetPublications(string publicationTitle = "")
{
string sanitizedTitle = publicationTitle.ToLower().Replace(' ', '_');
logger?.WriteLine(this.GetType().ToString(), $"Getting Publications (title={sanitizedTitle})");
string requestUrl = $"https://manganato.com/search/story/{sanitizedTitle}";
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, (byte)1);
if (requestResult.statusCode != HttpStatusCode.OK)
return Array.Empty<Publication>();
return ParsePublicationsFromHtml(requestResult.result);
}
private Publication[] ParsePublicationsFromHtml(Stream html)
{
StreamReader reader = new (html);
string htmlString = reader.ReadToEnd();
HtmlDocument document = new ();
document.LoadHtml(htmlString);
IEnumerable<HtmlNode> searchResults = document.DocumentNode.Descendants("div").Where(n => n.HasClass("search-story-item"));
List<string> urls = new();
foreach (HtmlNode mangaResult in searchResults)
{
urls.Add(mangaResult.Descendants("a").First(n => n.HasClass("item-title")).GetAttributes()
.First(a => a.Name == "href").Value);
}
HashSet<Publication> ret = new();
foreach (string url in urls)
{
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(url, (byte)1);
if (requestResult.statusCode != HttpStatusCode.OK)
return Array.Empty<Publication>();
ret.Add(ParseSinglePublicationFromHtml(requestResult.result, url.Split('/')[^1]));
}
return ret.ToArray();
}
private Publication ParseSinglePublicationFromHtml(Stream html, string publicationId)
{
StreamReader reader = new (html);
string htmlString = reader.ReadToEnd();
HtmlDocument document = new ();
document.LoadHtml(htmlString);
string status = "";
Dictionary<string, string> altTitles = new();
Dictionary<string, string>? links = null;
HashSet<string> tags = new();
string? author = null, originalLanguage = null;
int? year = null;
HtmlNode infoNode = document.DocumentNode.Descendants("div").First(d => d.HasClass("story-info-right"));
string sortName = infoNode.Descendants("h1").First().InnerText;
HtmlNode infoTable = infoNode.Descendants().First(d => d.Name == "table");
foreach (HtmlNode row in infoTable.Descendants("tr"))
{
string key = row.SelectNodes("td").First().InnerText.ToLower();
string value = row.SelectNodes("td").Last().InnerText;
string keySanitized = string.Concat(Regex.Matches(key, "[a-z]"));
switch (keySanitized)
{
case "alternative":
altTitles.Add("",value);
break;
case "authors":
author = value;
break;
case "status":
status = value;
break;
case "genres":
string[] genres = value.Split(" - ");
tags = genres.ToHashSet();
break;
default: break;
}
}
string posterUrl = document.DocumentNode.Descendants("span").First(s => s.HasClass("info-image")).Descendants("img").First()
.GetAttributes().First(a => a.Name == "src").Value;
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1);
string description = document.DocumentNode.Descendants("div").First(d => d.HasClass("panel-story-info-description"))
.InnerText;
return new Publication(sortName, author, description, altTitles, tags.ToArray(), posterUrl, coverFileNameInCache, links,
year, originalLanguage, status, publicationId);
}
public override Chapter[] GetChapters(Publication publication, string language = "")
{
string requestUrl = $"https://manganato.com/{publication.publicationId}";
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, (byte)1);
if (requestResult.statusCode != HttpStatusCode.OK)
return Array.Empty<Chapter>();
return ParseChaptersFromHtml(requestResult.result);
}
private Chapter[] ParseChaptersFromHtml(Stream html)
{
StreamReader reader = new (html);
string htmlString = reader.ReadToEnd();
HtmlDocument document = new ();
document.LoadHtml(htmlString);
List<Chapter> ret = new();
HtmlNode chapterList = document.DocumentNode.Descendants("ul").First(l => l.HasClass("row-content-chapter"));
foreach (HtmlNode chapterInfo in chapterList.Descendants("li"))
{
string fullString = chapterInfo.Descendants("a").First(d => d.HasClass("chapter-name")).InnerText;
string? volumeNumber = fullString.Contains("Vol.") ? fullString.Replace("Vol.", "").Split(' ')[0] : null;
string? chapterNumber = fullString.Split(':')[0].Split(' ')[^1];
string chapterName = string.Concat(fullString.Split(':')[1..]);
string url = chapterInfo.Descendants("a").First(d => d.HasClass("chapter-name"))
.GetAttributeValue("href", "");
ret.Add(new Chapter(chapterName, volumeNumber, chapterNumber, url));
}
return ret.ToArray();
}
public override void DownloadChapter(Publication publication, Chapter chapter)
{
string requestUrl = chapter.url;
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, (byte)1);
if (requestResult.statusCode != HttpStatusCode.OK)
return;
string[] imageUrls = ParseImageUrlsFromHtml(requestResult.result);
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, GetComicInfoXmlString(publication, chapter, logger));
DownloadChapterImages(imageUrls, GetArchiveFilePath(publication, chapter), (byte)1, comicInfoPath, "https://chapmanganato.com/");
}
private string[] ParseImageUrlsFromHtml(Stream html)
{
StreamReader reader = new (html);
string htmlString = reader.ReadToEnd();
HtmlDocument document = new ();
document.LoadHtml(htmlString);
List<string> ret = new();
HtmlNode imageContainer =
document.DocumentNode.Descendants("div").First(i => i.HasClass("container-chapter-reader"));
foreach(HtmlNode imageNode in imageContainer.Descendants("img"))
ret.Add(imageNode.GetAttributeValue("src", ""));
return ret.ToArray();
}
}

12
Tranga/TaskManager.cs
View File

@ -39,7 +39,11 @@ public class TaskManager
this.settings = new TrangaSettings(downloadFolderPath, workingDirectory, newKomga);
ExportDataAndSettings();
this._connectors = new Connector[]{ new MangaDex(downloadFolderPath, imageCachePath, logger) };
this._connectors = new Connector[]
{
new MangaDex(downloadFolderPath, imageCachePath, logger),
new Manganato(downloadFolderPath, imageCachePath, logger)
};
foreach(Connector cConnector in this._connectors)
_taskQueue.Add(cConnector, new List<TrangaTask>());
@ -59,7 +63,11 @@ public class TaskManager
public TaskManager(TrangaSettings settings, Logger? logger = null)
{
this.logger = logger;
this._connectors = new Connector[]{ new MangaDex(settings.downloadLocation, settings.coverImageCache, logger) };
this._connectors = new Connector[]
{
new MangaDex(settings.downloadLocation, settings.coverImageCache, logger),
new Manganato(settings.downloadLocation, settings.coverImageCache, logger)
};
foreach(Connector cConnector in this._connectors)
_taskQueue.Add(cConnector, new List<TrangaTask>());
_allTasks = new HashSet<TrangaTask>();

1
Tranga/Tranga.csproj
View File

@ -7,6 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
</ItemGroup>