glax/Tranga
2
0
Fork 0
Code

Merge pull request #133 from C9Glax/cuttingedge

Browse Source 
RateLimits, FileNames, Volume/Chapter Numbers
This commit is contained in:
Glax 2024-02-28 02:49:48 +01:00 committed by GitHub
commit 9b2a6de841
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 150 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
Tranga/Chapter.cs
View File

@ -20,20 +20,28 @@ public readonly struct Chapter : IComparable
private static readonly Regex LegalCharacters = new (@"([A-z]*[0-9]* *\.*-*,*\]*\[*'*\'*\)*\(*~*!*)*");
private static readonly Regex IllegalStrings = new(@"Vol(ume)?.?", RegexOptions.IgnoreCase);
private static readonly Regex Digits = new(@"[0-9\.]*");
public Chapter(Manga parentManga, string? name, string? volumeNumber, string chapterNumber, string url)
{
this.parentManga = parentManga;
this.name = name;
this.volumeNumber = volumeNumber ?? "0";
this.chapterNumber = chapterNumber;
this.volumeNumber = volumeNumber is not null ? string.Concat(Digits.Matches(volumeNumber).Select(x => x.Value)) : "0";
this.chapterNumber = string.Concat(Digits.Matches(chapterNumber).Select(x => x.Value));
this.url = url;
string chapterName = string.Concat(LegalCharacters.Matches(name ?? ""));
string volStr = volumeNumber is not null ? $"Vol.{volumeNumber} " : "";
string chNumberStr = $"Ch.{chapterNumber} ";
string chNameStr = chapterName.Length > 0 ? $"- {chapterName}" : "";
chNameStr = IllegalStrings.Replace(chNameStr, "");
this.fileName = $"{volStr}{chNumberStr}{chNameStr}";
string chapterVolNumStr;
if (volumeNumber is not null && volumeNumber.Length > 0)
chapterVolNumStr = $"Vol.{volumeNumber} Ch.{chapterNumber}";
else
chapterVolNumStr = $"Ch.{chapterNumber}";
if (name is not null && name.Length > 0)
{
string chapterName = IllegalStrings.Replace(string.Concat(LegalCharacters.Matches(name)), "");
this.fileName = $"{chapterVolNumStr} - {chapterName}";
}
else
this.fileName = chapterVolNumStr;
}
public override string ToString()
@ -81,23 +89,20 @@ public readonly struct Chapter : IComparable
/// <returns>true if chapter is present</returns>
internal bool CheckChapterIsDownloaded(string downloadLocation)
{
string newFilePath = GetArchiveFilePath(downloadLocation);
if (!Directory.Exists(Path.Join(downloadLocation, parentManga.folderName)))
return false;
FileInfo[] archives = new DirectoryInfo(Path.Join(downloadLocation, parentManga.folderName)).GetFiles();
Regex chapterInfoRex = new(@"Ch\.[0-9.]+");
Regex chapterRex = new(@"[0-9]+(\.[0-9]+)?");
Regex volChRex = new(@"(?:Vol(?:ume)?\.([0-9]+)\D*)?Ch(?:apter)?\.([0-9]+(?:\.[0-9]+)*)");
if (File.Exists(newFilePath))
return true;
string cn = this.chapterNumber;
if (archives.FirstOrDefault(archive => chapterRex.Match(chapterInfoRex.Match(archive.Name).Value).Value == cn) is { } path)
Chapter t = this;
return archives.Select(archive => archive.Name).Any(archiveFileName =>
{
File.Move(path.FullName, newFilePath);
return true;
}
return false;
Match m = volChRex.Match(archiveFileName);
string archiveVolNum = m.Groups[1].Success ? m.Groups[1].Value : "0";
string archiveChNum = m.Groups[2].Value;
return archiveVolNum == t.volumeNumber &&
archiveChNum == t.chapterNumber;
});
}
/// <summary>
/// Creates full file path of chapter-archive

5
Tranga/Jobs/JobJsonConverter.cs
View File

@ -38,6 +38,9 @@ public class JobJsonConverter : JsonConverter
jo.GetValue("parentJobId")!.Value<string?>());
}else if ((jo.ContainsKey("jobType") && jo["jobType"]!.Value<byte>() == (byte)Job.JobType.DownloadNewChaptersJob) || jo.ContainsKey("translatedLanguage"))//TODO change to jobType
{
DateTime lastExecution = jo.GetValue("lastExecution") is {} le
? le.ToObject<DateTime>()
: DateTime.UnixEpoch; //TODO do null checks on all variables
return new DownloadNewChapters(this._clone,
jo.GetValue("mangaConnector")!.ToObject<MangaConnector>(JsonSerializer.Create(new JsonSerializerSettings()
{
@ -47,7 +50,7 @@ public class JobJsonConverter : JsonConverter
}
}))!,
jo.GetValue("manga")!.ToObject<Manga>(),
jo.GetValue("lastExecution")!.ToObject<DateTime>(),
lastExecution,
jo.GetValue("recurring")!.Value<bool>(),
jo.GetValue("recurrenceTime")!.ToObject<TimeSpan?>(),
jo.GetValue("parentJobId")!.Value<string?>());

23
Tranga/MangaConnectors/Bato.cs
View File

@ -7,12 +7,10 @@ namespace Tranga.MangaConnectors;
public class Bato : MangaConnector
{
public Bato(GlobalBase clone) : base(clone, "Bato")
{
this.downloadClient = new HttpDownloadClient(clone, new Dictionary<byte, int>()
{
{1, 60}
});
this.downloadClient = new HttpDownloadClient(clone);
}
public override Manga[] GetManga(string publicationTitle = "")
@ -21,7 +19,7 @@ public class Bato : MangaConnector
string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower();
string requestUrl = $"https://bato.to/v3x-search?word={sanitizedTitle}&lang=en";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Manga>();
@ -43,8 +41,7 @@ public class Bato : MangaConnector
public override Manga? GetMangaFromUrl(string url)
{
RequestResult requestResult =
downloadClient.MakeRequest(url, 1);
RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
if (requestResult.htmlDocument is null)
@ -89,7 +86,7 @@ public class Bato : MangaConnector
string posterUrl = document.DocumentNode.SelectNodes("//img")
.First(child => child.GetAttributeValue("data-hk", "") == "0-1-0").GetAttributeValue("src", "").Replace("&amp;", "&");
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1);
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover);
List<HtmlNode> genreNodes = document.DocumentNode.SelectSingleNode("//b[text()='Genres:']/..").SelectNodes("span").ToList();
string[] tags = genreNodes.Select(node => node.FirstChild.InnerText).ToArray();
@ -129,7 +126,7 @@ public class Bato : MangaConnector
string requestUrl = $"https://bato.to/title/{manga.publicationId}";
// Leaving this in for verification if the page exists
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Chapter>();
@ -141,7 +138,7 @@ public class Bato : MangaConnector
private List<Chapter> ParseChaptersFromHtml(Manga manga, string mangaUrl)
{
RequestResult result = downloadClient.MakeRequest(mangaUrl, 1);
RequestResult result = downloadClient.MakeRequest(mangaUrl, RequestType.Default);
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null)
{
Log("Failed to load site");
@ -184,7 +181,7 @@ public class Bato : MangaConnector
string requestUrl = chapter.url;
// Leaving this in to check if the page exists
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
progressToken?.Cancel();
@ -196,13 +193,13 @@ public class Bato : MangaConnector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken);
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken);
}
private string[] ParseImageUrlsFromHtml(string mangaUrl)
{
RequestResult requestResult =
downloadClient.MakeRequest(mangaUrl, 1);
downloadClient.MakeRequest(mangaUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
return Array.Empty<string>();

2
Tranga/MangaConnectors/ChromiumDownloadClient.cs
View File

@ -53,7 +53,7 @@ internal class ChromiumDownloadClient : DownloadClient
});
}
public ChromiumDownloadClient(GlobalBase clone, Dictionary<byte, int> rateLimitRequestsPerMinute) : base(clone, rateLimitRequestsPerMinute)
public ChromiumDownloadClient(GlobalBase clone) : base(clone)
{
this.browser = DownloadBrowser().Result;
}

32
Tranga/MangaConnectors/DownloadClient.cs
View File

@ -5,37 +5,29 @@ namespace Tranga.MangaConnectors;
internal abstract class DownloadClient : GlobalBase
{
private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit;
private readonly Dictionary<byte, TimeSpan> _rateLimit;
private readonly Dictionary<RequestType, DateTime> _lastExecutedRateLimit;
protected DownloadClient(GlobalBase clone, Dictionary<byte, int> rateLimitRequestsPerMinute) : base(clone)
protected DownloadClient(GlobalBase clone) : base(clone)
{
this._lastExecutedRateLimit = new();
_rateLimit = new();
foreach (KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute)
_rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value));
}
internal void SetCustomRequestLimit(byte requestType, int limit)
public RequestResult MakeRequest(string url, RequestType requestType, string? referrer = null, string? clickButton = null)
{
if (_rateLimit.ContainsKey(requestType))
_rateLimit[requestType] = TimeSpan.FromMinutes(1).Divide(limit);
else
_rateLimit.Add(requestType, TimeSpan.FromMinutes(1).Divide(limit));
}
public RequestResult MakeRequest(string url, byte requestType, string? referrer = null, string? clickButton = null)
{
if (_rateLimit.TryGetValue(requestType, out TimeSpan value))
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
else
if (!settings.requestLimits.ContainsKey(requestType))
{
Log("RequestType not configured for rate-limit.");
return new RequestResult(HttpStatusCode.NotAcceptable, null, Stream.Null);
}
TimeSpan rateLimitTimeout = _rateLimit[requestType]
.Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]));
int rateLimit = settings.userAgent == TrangaSettings.DefaultUserAgent
? TrangaSettings.DefaultRequestLimits[requestType]
: settings.requestLimits[requestType];
TimeSpan timeBetweenRequests = TimeSpan.FromMinutes(1).Divide(rateLimit);
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(timeBetweenRequests));
TimeSpan rateLimitTimeout = timeBetweenRequests.Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]));
if (rateLimitTimeout > TimeSpan.Zero)
{

3
Tranga/MangaConnectors/HttpDownloadClient.cs
View File

@ -11,8 +11,7 @@ internal class HttpDownloadClient : DownloadClient
Timeout = TimeSpan.FromSeconds(10)
};
public HttpDownloadClient(GlobalBase clone, Dictionary<byte, int> rateLimitRequestsPerMinute) : base(clone, rateLimitRequestsPerMinute)
public HttpDownloadClient(GlobalBase clone) : base(clone)
{
Client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", settings.userAgent);
}

6
Tranga/MangaConnectors/MangaConnector.cs
View File

@ -202,7 +202,7 @@ public abstract class MangaConnector : GlobalBase
/// <param name="fullPath"></param>
/// <param name="requestType">RequestType for Rate-Limit</param>
/// <param name="referrer">referrer used in html request header</param>
private HttpStatusCode DownloadImage(string imageUrl, string fullPath, byte requestType, string? referrer = null)
private HttpStatusCode DownloadImage(string imageUrl, string fullPath, RequestType requestType, string? referrer = null)
{
RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType, referrer);
@ -217,7 +217,7 @@ public abstract class MangaConnector : GlobalBase
return requestResult.statusCode;
}
protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, string? comicInfoPath = null, string? referrer = null, ProgressToken? progressToken = null)
protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, RequestType requestType, string? comicInfoPath = null, string? referrer = null, ProgressToken? progressToken = null)
{
if (progressToken?.cancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
@ -274,7 +274,7 @@ public abstract class MangaConnector : GlobalBase
return HttpStatusCode.OK;
}
protected string SaveCoverImageToCache(string url, byte requestType)
protected string SaveCoverImageToCache(string url, RequestType requestType)
{
string filetype = url.Split('/')[^1].Split('?')[0].Split('.')[^1];
string filename = $"{DateTime.Now.Ticks.ToString()}.{filetype}";

34
Tranga/MangaConnectors/MangaDex.cs
View File

@ -7,25 +7,9 @@ using JsonSerializer = System.Text.Json.JsonSerializer;
namespace Tranga.MangaConnectors;
public class MangaDex : MangaConnector
{
private enum RequestType : byte
{
Manga,
Feed,
AtHomeServer,
CoverUrl,
Author,
}
public MangaDex(GlobalBase clone) : base(clone, "MangaDex")
{
this.downloadClient = new HttpDownloadClient(clone, new Dictionary<byte, int>()
{
{(byte)RequestType.Manga, 250},
{(byte)RequestType.Feed, 250},
{(byte)RequestType.AtHomeServer, 40},
{(byte)RequestType.CoverUrl, 250},
{(byte)RequestType.Author, 250}
});
this.downloadClient = new HttpDownloadClient(clone);
}
public override Manga[] GetManga(string publicationTitle = "")
@ -41,7 +25,7 @@ public class MangaDex : MangaConnector
//Request next Page
RequestResult requestResult =
downloadClient.MakeRequest(
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", (byte)RequestType.Manga);
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -75,7 +59,7 @@ public class MangaDex : MangaConnector
public override Manga? GetMangaFromId(string publicationId)
{
RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/manga/{publicationId}", (byte)RequestType.Manga);
downloadClient.MakeRequest($"https://api.mangadex.org/manga/{publicationId}", RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -149,7 +133,7 @@ public class MangaDex : MangaConnector
string? coverUrl = GetCoverUrl(publicationId, posterId);
string? coverCacheName = null;
if (coverUrl is not null)
coverCacheName = SaveCoverImageToCache(coverUrl, (byte)RequestType.AtHomeServer);
coverCacheName = SaveCoverImageToCache(coverUrl, RequestType.MangaCover);
List<string> authors = GetAuthors(authorIds);
@ -216,7 +200,7 @@ public class MangaDex : MangaConnector
//Request next "Page"
RequestResult requestResult =
downloadClient.MakeRequest(
$"https://api.mangadex.org/manga/{manga.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", (byte)RequestType.Feed);
$"https://api.mangadex.org/manga/{manga.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", RequestType.MangaDexFeed);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -268,7 +252,7 @@ public class MangaDex : MangaConnector
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
//Request URLs for Chapter-Images
RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer);
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", RequestType.MangaDexImage);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
progressToken?.Cancel();
@ -293,7 +277,7 @@ public class MangaDex : MangaConnector
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
//Download Chapter-Images
return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), (byte)RequestType.AtHomeServer, comicInfoPath, progressToken:progressToken);
return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, progressToken:progressToken);
}
private string? GetCoverUrl(string publicationId, string? posterId)
@ -307,7 +291,7 @@ public class MangaDex : MangaConnector
//Request information where to download Cover
RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", (byte)RequestType.CoverUrl);
downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", RequestType.MangaCover);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -328,7 +312,7 @@ public class MangaDex : MangaConnector
foreach (string authorId in authorIds)
{
RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", (byte)RequestType.Author);
downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", RequestType.MangaDexAuthor);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return ret;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);

17
Tranga/MangaConnectors/MangaKatana.cs
View File

@ -9,10 +9,7 @@ public class MangaKatana : MangaConnector
{
public MangaKatana(GlobalBase clone) : base(clone, "MangaKatana")
{
this.downloadClient = new HttpDownloadClient(clone, new Dictionary<byte, int>()
{
{1, 60}
});
this.downloadClient = new HttpDownloadClient(clone);
}
public override Manga[] GetManga(string publicationTitle = "")
@ -21,7 +18,7 @@ public class MangaKatana : MangaConnector
string sanitizedTitle = string.Join('_', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower();
string requestUrl = $"https://mangakatana.com/?search={sanitizedTitle}&search_by=book_name";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Manga>();
@ -47,7 +44,7 @@ public class MangaKatana : MangaConnector
public override Manga? GetMangaFromUrl(string url)
{
RequestResult requestResult =
downloadClient.MakeRequest(url, 1);
downloadClient.MakeRequest(url, RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
return ParseSinglePublicationFromHtml(requestResult.result, url.Split('/')[^1]);
@ -131,7 +128,7 @@ public class MangaKatana : MangaConnector
string posterUrl = document.DocumentNode.SelectSingleNode("//*[@id='single_book']/div[1]/div").Descendants("img").First()
.GetAttributes().First(a => a.Name == "src").Value;
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1);
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover);
string description = document.DocumentNode.SelectSingleNode("//*[@id='single_book']/div[3]/p").InnerText;
while (description.StartsWith('\n'))
@ -158,7 +155,7 @@ public class MangaKatana : MangaConnector
string requestUrl = $"https://mangakatana.com/manga/{manga.publicationId}";
// Leaving this in for verification if the page exists
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Chapter>();
@ -210,7 +207,7 @@ public class MangaKatana : MangaConnector
string requestUrl = chapter.url;
// Leaving this in to check if the page exists
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
progressToken?.Cancel();
@ -222,7 +219,7 @@ public class MangaKatana : MangaConnector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken);
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken);
}
private string[] ParseImageUrlsFromHtml(string mangaUrl)

23
Tranga/MangaConnectors/MangaLife.cs
View File

@ -9,10 +9,7 @@ public class MangaLife : MangaConnector
{
public MangaLife(GlobalBase clone) : base(clone, "Manga4Life")
{
this.downloadClient = new ChromiumDownloadClient(clone, new Dictionary<byte, int>()
{
{ 1, 60 }
});
this.downloadClient = new ChromiumDownloadClient(clone);
}
public override Manga[] GetManga(string publicationTitle = "")
@ -21,7 +18,7 @@ public class MangaLife : MangaConnector
string sanitizedTitle = WebUtility.UrlEncode(publicationTitle);
string requestUrl = $"https://manga4life.com/search/?name={sanitizedTitle}";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Manga>();
@ -39,10 +36,10 @@ public class MangaLife : MangaConnector
public override Manga? GetMangaFromUrl(string url)
{
Regex publicationIdRex = new(@"https:\/\/manga4life.com\/manga\/(.*)(\/.*)*");
string publicationId = publicationIdRex.Match(url).Groups[1].Value;
Regex publicationIdRex = new(@"https:\/\/(www\.)?manga4life.com\/manga\/(.*)(\/.*)*");
string publicationId = publicationIdRex.Match(url).Groups[2].Value;
RequestResult requestResult = this.downloadClient.MakeRequest(url, 1);
RequestResult requestResult = this.downloadClient.MakeRequest(url, RequestType.MangaInfo);
if(requestResult.htmlDocument is not null)
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId);
return null;
@ -81,7 +78,7 @@ public class MangaLife : MangaConnector
HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//img");
string posterUrl = posterNode.GetAttributeValue("src", "");
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1);
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover);
HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//h1");
string sortName = titleNode.InnerText;
@ -133,7 +130,7 @@ public class MangaLife : MangaConnector
public override Chapter[] GetChapters(Manga manga, string language="en")
{
Log($"Getting chapters {manga}");
RequestResult result = downloadClient.MakeRequest($"https://manga4life.com/manga/{manga.publicationId}", 1, clickButton:"[class*='ShowAllChapters']");
RequestResult result = downloadClient.MakeRequest($"https://manga4life.com/manga/{manga.publicationId}", RequestType.Default, clickButton:"[class*='ShowAllChapters']");
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null)
{
return Array.Empty<Chapter>();
@ -142,7 +139,7 @@ public class MangaLife : MangaConnector
HtmlNodeCollection chapterNodes = result.htmlDocument.DocumentNode.SelectNodes(
"//a[contains(concat(' ',normalize-space(@class),' '),' ChapterLink ')]");
string[] urls = chapterNodes.Select(node => node.GetAttributeValue("href", "")).ToArray();
Regex urlRex = new Regex(@"-chapter-([0-9\\.]+)(-index-([0-9\\.]+))?");
Regex urlRex = new (@"-chapter-([0-9\\.]+)(-index-([0-9\\.]+))?");
List<Chapter> chapters = new();
foreach (string url in urls)
@ -179,7 +176,7 @@ public class MangaLife : MangaConnector
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, 1);
RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, RequestType.Default);
if (requestResult.htmlDocument is null)
{
progressToken?.Cancel();
@ -197,6 +194,6 @@ public class MangaLife : MangaConnector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, progressToken:progressToken);
return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, progressToken:progressToken);
}
}

17
Tranga/MangaConnectors/Manganato.cs
View File

@ -9,10 +9,7 @@ public class Manganato : MangaConnector
{
public Manganato(GlobalBase clone) : base(clone, "Manganato")
{
this.downloadClient = new HttpDownloadClient(clone, new Dictionary<byte, int>()
{
{1, 60}
});
this.downloadClient = new HttpDownloadClient(clone);
}
public override Manga[] GetManga(string publicationTitle = "")
@ -21,7 +18,7 @@ public class Manganato : MangaConnector
string sanitizedTitle = string.Join('_', Regex.Matches(publicationTitle, "[A-z]*").Where(str => str.Length > 0)).ToLower();
string requestUrl = $"https://manganato.com/search/story/{sanitizedTitle}";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Manga>();
@ -62,7 +59,7 @@ public class Manganato : MangaConnector
public override Manga? GetMangaFromUrl(string url)
{
RequestResult requestResult =
downloadClient.MakeRequest(url, 1);
downloadClient.MakeRequest(url, RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
@ -121,7 +118,7 @@ public class Manganato : MangaConnector
string posterUrl = document.DocumentNode.Descendants("span").First(s => s.HasClass("info-image")).Descendants("img").First()
.GetAttributes().First(a => a.Name == "src").Value;
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1);
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover);
string description = document.DocumentNode.Descendants("div").First(d => d.HasClass("panel-story-info-description"))
.InnerText.Replace("Description :", "");
@ -143,7 +140,7 @@ public class Manganato : MangaConnector
Log($"Getting chapters {manga}");
string requestUrl = $"https://chapmanganato.com/{manga.publicationId}";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Chapter>();
@ -192,7 +189,7 @@ public class Manganato : MangaConnector
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
string requestUrl = chapter.url;
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
progressToken?.Cancel();
@ -210,7 +207,7 @@ public class Manganato : MangaConnector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://chapmanganato.com/", progressToken:progressToken);
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, "https://chapmanganato.com/", progressToken:progressToken);
}
private string[] ParseImageUrlsFromHtml(HtmlDocument document)

25
Tranga/MangaConnectors/Mangasee.cs
View File

@ -12,10 +12,7 @@ public class Mangasee : MangaConnector
{
public Mangasee(GlobalBase clone) : base(clone, "Mangasee")
{
this.downloadClient = new ChromiumDownloadClient(clone, new Dictionary<byte, int>()
{
{ 1, 60 }
});
this.downloadClient = new ChromiumDownloadClient(clone);
}
private struct SearchResult
@ -30,7 +27,7 @@ public class Mangasee : MangaConnector
Log($"Searching Publications. Term=\"{publicationTitle}\"");
string requestUrl = "https://mangasee123.com/_search.php";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
Log($"Failed to retrieve search: {requestResult.statusCode}");
@ -121,8 +118,8 @@ public class Mangasee : MangaConnector
Regex publicationIdRex = new(@"https:\/\/mangasee123.com\/manga\/(.*)(\/.*)*");
string publicationId = publicationIdRex.Match(url).Groups[1].Value;
RequestResult requestResult = this.downloadClient.MakeRequest(url, 1);
if(requestResult.htmlDocument is not null)
RequestResult requestResult = this.downloadClient.MakeRequest(url, RequestType.MangaInfo);
if((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 && requestResult.htmlDocument is not null)
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId);
return null;
}
@ -136,7 +133,7 @@ public class Mangasee : MangaConnector
HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//img");
string posterUrl = posterNode.GetAttributeValue("src", "");
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1);
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover);
HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//h1");
string sortName = titleNode.InnerText;
@ -194,13 +191,15 @@ public class Mangasee : MangaConnector
XDocument doc = XDocument.Load($"https://mangasee123.com/rss/{manga.publicationId}.xml");
XElement[] chapterItems = doc.Descendants("item").ToArray();
List<Chapter> chapters = new();
Regex chVolRex = new(@".*chapter-([0-9\.]+)(?:-index-([0-9\.]+))?.*");
foreach (XElement chapter in chapterItems)
{
string volumeNumber = "1";
string url = chapter.Descendants("link").First().Value;
string chapterNumber = Regex.Match(url, @"-chapter-([0-9\.]+)").Groups[1].ToString();
Match m = chVolRex.Match(url);
string? volumeNumber = m.Groups[2].Success ? m.Groups[2].Value : "1";
string chapterNumber = m.Groups[1].Value;
url = url.Replace(Regex.Match(url,"(-page-[0-9])").Value,"");
url = string.Concat(Regex.Match(url, @"(.*)-page-[0-9]+(\.html)").Groups.Values.Select(v => v.Value));
chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, url));
}
@ -232,7 +231,7 @@ public class Mangasee : MangaConnector
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, 1);
RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, RequestType.Default);
if (requestResult.htmlDocument is null)
{
progressToken?.Cancel();
@ -250,6 +249,6 @@ public class Mangasee : MangaConnector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, progressToken:progressToken);
return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, progressToken:progressToken);
}
}

27
Tranga/MangaConnectors/Mangaworld.cs
View File

@ -9,10 +9,7 @@ public class Mangaworld: MangaConnector
{
public Mangaworld(GlobalBase clone) : base(clone, "Mangaworld")
{
this.downloadClient = new HttpDownloadClient(clone, new Dictionary<byte, int>()
{
{1, 60}
});
this.downloadClient = new HttpDownloadClient(clone);
}
public override Manga[] GetManga(string publicationTitle = "")
@ -21,7 +18,7 @@ public class Mangaworld: MangaConnector
string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(str => str.Length > 0)).ToLower();
string requestUrl = $"https://www.mangaworld.ac/archive?keyword={sanitizedTitle}";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Manga>();
@ -62,14 +59,14 @@ public class Mangaworld: MangaConnector
public override Manga? GetMangaFromUrl(string url)
{
RequestResult requestResult =
downloadClient.MakeRequest(url, 1);
downloadClient.MakeRequest(url, RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
if (requestResult.htmlDocument is null)
return null;
Regex idRex = new (@"https:\/\/www\.mangaworld\.[a-z]{0,63}\/manga\/([0-9]+\/[0-9A-z\-]+)");
Regex idRex = new (@"https:\/\/www\.mangaworld\.[a-z]{0,63}\/manga\/([0-9]+\/[0-9A-z\-]+).*");
string id = idRex.Match(url).Groups[1].Value;
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, id);
}
@ -87,19 +84,19 @@ public class Mangaworld: MangaConnector
HtmlNode metadata = infoNode.Descendants().First(d => d.HasClass("meta-data"));
HtmlNode altTitlesNode = metadata.SelectSingleNode("//span[text()='Titoli alternativi: ']/..").ChildNodes[1];
HtmlNode altTitlesNode = metadata.SelectSingleNode("//span[text()='Titoli alternativi: ' or text()='Titolo alternativo: ']/..").ChildNodes[1];
string[] alts = altTitlesNode.InnerText.Split(", ");
for(int i = 0; i < alts.Length; i++)
altTitles.Add(i.ToString(), alts[i]);
HtmlNode genresNode =
metadata.SelectSingleNode("//span[text()='Generi: ']/..");
metadata.SelectSingleNode("//span[text()='Generi: ' or text()='Genero: ']/..");
HashSet<string> tags = genresNode.SelectNodes("a").Select(node => node.InnerText).ToHashSet();
HtmlNode authorsNode =
metadata.SelectSingleNode("//span[text()='Autore: ']/..");
string[] authors = new[] { authorsNode.SelectNodes("a").First().InnerText };
metadata.SelectSingleNode("//span[text()='Autore: ' or text()='Autori: ']/..");
string[] authors = authorsNode.SelectNodes("a").Select(node => node.InnerText).ToArray();
string status = metadata.SelectSingleNode("//span[text()='Stato: ']/..").SelectNodes("a").First().InnerText;
// ReSharper disable 5 times StringLiteralTypo
@ -114,7 +111,7 @@ public class Mangaworld: MangaConnector
string posterUrl = document.DocumentNode.SelectSingleNode("//img[@class='rounded']").GetAttributeValue("src", "");
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1);
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover);
string description = document.DocumentNode.SelectSingleNode("//div[@id='noidungm']").InnerText;
@ -132,7 +129,7 @@ public class Mangaworld: MangaConnector
Log($"Getting chapters {manga}");
string requestUrl = $"https://www.mangaworld.ac/manga/{manga.publicationId}";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Chapter>();
@ -191,7 +188,7 @@ public class Mangaworld: MangaConnector
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
string requestUrl = $"{chapter.url}?style=list";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
progressToken?.Cancel();
@ -209,7 +206,7 @@ public class Mangaworld: MangaConnector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://www.mangaworld.bz/", progressToken:progressToken);
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, "https://www.mangaworld.bz/", progressToken:progressToken);
}
private string[] ParseImageUrlsFromHtml(HtmlDocument document)

12
Tranga/MangaConnectors/RequestType.cs Normal file
View File

@ -0,0 +1,12 @@
namespace Tranga.MangaConnectors;
public enum RequestType : byte
{
Default = 0,
MangaDexFeed = 1,
MangaImage = 2,
MangaCover = 3,
MangaDexAuthor = 4,
MangaDexImage = 5,
MangaInfo = 6
}

15
Tranga/Server.cs
View File

@ -399,16 +399,19 @@ public class Server : GlobalBase
case "Settings/customRequestLimit":
if (!requestVariables.TryGetValue("requestType", out string? requestTypeStr) ||
!requestVariables.TryGetValue("requestsPerMinute", out string? requestsPerMinuteStr) ||
!requestVariables.TryGetValue("connector", out connectorName) ||
!byte.TryParse(requestTypeStr, out byte requestType) ||
!int.TryParse(requestsPerMinuteStr, out int requestsPerMinute) ||
!_parent.TryGetConnector(connectorName, out connector))
!Enum.TryParse(requestTypeStr, out RequestType requestType) ||
!int.TryParse(requestsPerMinuteStr, out int requestsPerMinute))
{
SendResponse(HttpStatusCode.BadRequest, response);
break;
}
connector!.downloadClient.SetCustomRequestLimit(requestType, requestsPerMinute);
SendResponse(HttpStatusCode.Accepted, response);
if (settings.requestLimits.ContainsKey(requestType))
{
settings.requestLimits[requestType] = requestsPerMinute;
SendResponse(HttpStatusCode.Accepted, response);
}else
SendResponse(HttpStatusCode.BadRequest, response);
break;
case "NotificationConnectors/Update":
if (!requestVariables.TryGetValue("notificationConnector", out string? notificationConnectorStr) ||

20
Tranga/TrangaSettings.cs
View File

@ -2,6 +2,7 @@
using System.Runtime.InteropServices;
using Newtonsoft.Json;
using Tranga.LibraryConnectors;
using Tranga.MangaConnectors;
using Tranga.NotificationConnectors;
using static System.IO.UnixFileMode;
@ -13,15 +14,26 @@ public class TrangaSettings
public string workingDirectory { get; private set; }
public int apiPortNumber { get; init; }
public string styleSheet { get; private set; }
public string userAgent { get; set; } =
$"Tranga ({Enum.GetName(Environment.OSVersion.Platform)}; {(Environment.Is64BitOperatingSystem ? "x64" : "")}) / 1.0";
public string userAgent { get; set; } = DefaultUserAgent;
[JsonIgnore] public string settingsFilePath => Path.Join(workingDirectory, "settings.json");
[JsonIgnore] public string libraryConnectorsFilePath => Path.Join(workingDirectory, "libraryConnectors.json");
[JsonIgnore] public string notificationConnectorsFilePath => Path.Join(workingDirectory, "notificationConnectors.json");
[JsonIgnore] public string jobsFolderPath => Path.Join(workingDirectory, "jobs");
[JsonIgnore] public string coverImageCache => Path.Join(workingDirectory, "imageCache");
public ushort? version { get; set; }
[JsonIgnore] internal static readonly string DefaultUserAgent = $"Tranga ({Enum.GetName(Environment.OSVersion.Platform)}; {(Environment.Is64BitOperatingSystem ? "x64" : "")}) / 1.0";
public ushort? version { get; set; } = 1;
[JsonIgnore]internal static readonly Dictionary<RequestType, int> DefaultRequestLimits = new ()
{
{RequestType.MangaInfo, 250},
{RequestType.MangaDexFeed, 250},
{RequestType.MangaDexImage, 40},
{RequestType.MangaImage, 60},
{RequestType.MangaCover, 250},
{RequestType.MangaDexAuthor, 250},
{RequestType.Default, 60}
};
public Dictionary<RequestType, int> requestLimits { get; set; } = DefaultRequestLimits;
public TrangaSettings(string? downloadLocation = null, string? workingDirectory = null, int? apiPortNumber = null)
{