glax/Tranga
2
0
Fork 0
Code

#109 Rate Limits

Browse Source 
Moved Config for RateLimits to TrangaSettings
Updated API: Settings/customRequestLimit
requestType in RequestType.cs
requestsPerMinute as int
This commit is contained in:
Glax 2024-02-11 00:35:33 +01:00
parent c008d55f26
commit 0b4461265c
15 changed files with 110 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Tranga/Chapter.cs
View File

@ -20,12 +20,13 @@ public readonly struct Chapter : IComparable
private static readonly Regex LegalCharacters = new (@"([A-z]*[0-9]* *\.*-*,*\]*\[*'*\'*\)*\(*~*!*)*");
private static readonly Regex IllegalStrings = new(@"Vol(ume)?.?", RegexOptions.IgnoreCase);
private static readonly Regex Digits = new(@"[0-9]*");
public Chapter(Manga parentManga, string? name, string? volumeNumber, string chapterNumber, string url)
{
this.parentManga = parentManga;
this.name = name;
this.volumeNumber = volumeNumber ?? "0";
this.chapterNumber = chapterNumber;
this.volumeNumber = volumeNumber is not null ? string.Concat(Digits.Matches(volumeNumber).Select(x => x.Value)) : "0";
this.chapterNumber = string.Concat(Digits.Matches(chapterNumber).Select(x => x.Value));
this.url = url;
string chapterVolNumStr;

23
Tranga/MangaConnectors/Bato.cs
View File

@ -7,12 +7,10 @@ namespace Tranga.MangaConnectors;
public class Bato : MangaConnector
{
public Bato(GlobalBase clone) : base(clone, "Bato")
{
this.downloadClient = new HttpDownloadClient(clone, new Dictionary<byte, int>()
{
{1, 60}
});
this.downloadClient = new HttpDownloadClient(clone);
}
public override Manga[] GetManga(string publicationTitle = "")
@ -21,7 +19,7 @@ public class Bato : MangaConnector
string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower();
string requestUrl = $"https://bato.to/v3x-search?word={sanitizedTitle}&lang=en";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Manga>();
@ -43,8 +41,7 @@ public class Bato : MangaConnector
public override Manga? GetMangaFromUrl(string url)
{
RequestResult requestResult =
downloadClient.MakeRequest(url, 1);
RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
if (requestResult.htmlDocument is null)
@ -89,7 +86,7 @@ public class Bato : MangaConnector
string posterUrl = document.DocumentNode.SelectNodes("//img")
.First(child => child.GetAttributeValue("data-hk", "") == "0-1-0").GetAttributeValue("src", "").Replace("&amp;", "&");
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1);
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover);
List<HtmlNode> genreNodes = document.DocumentNode.SelectSingleNode("//b[text()='Genres:']/..").SelectNodes("span").ToList();
string[] tags = genreNodes.Select(node => node.FirstChild.InnerText).ToArray();
@ -129,7 +126,7 @@ public class Bato : MangaConnector
string requestUrl = $"https://bato.to/title/{manga.publicationId}";
// Leaving this in for verification if the page exists
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Chapter>();
@ -141,7 +138,7 @@ public class Bato : MangaConnector
private List<Chapter> ParseChaptersFromHtml(Manga manga, string mangaUrl)
{
RequestResult result = downloadClient.MakeRequest(mangaUrl, 1);
RequestResult result = downloadClient.MakeRequest(mangaUrl, RequestType.Default);
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null)
{
Log("Failed to load site");
@ -184,7 +181,7 @@ public class Bato : MangaConnector
string requestUrl = chapter.url;
// Leaving this in to check if the page exists
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
progressToken?.Cancel();
@ -196,13 +193,13 @@ public class Bato : MangaConnector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken);
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken);
}
private string[] ParseImageUrlsFromHtml(string mangaUrl)
{
RequestResult requestResult =
downloadClient.MakeRequest(mangaUrl, 1);
downloadClient.MakeRequest(mangaUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
return Array.Empty<string>();

2
Tranga/MangaConnectors/ChromiumDownloadClient.cs
View File

@ -53,7 +53,7 @@ internal class ChromiumDownloadClient : DownloadClient
});
}
public ChromiumDownloadClient(GlobalBase clone, Dictionary<byte, int> rateLimitRequestsPerMinute) : base(clone, rateLimitRequestsPerMinute)
public ChromiumDownloadClient(GlobalBase clone) : base(clone)
{
this.browser = DownloadBrowser().Result;
}

32
Tranga/MangaConnectors/DownloadClient.cs
View File

@ -5,37 +5,29 @@ namespace Tranga.MangaConnectors;
internal abstract class DownloadClient : GlobalBase
{
private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit;
private readonly Dictionary<byte, TimeSpan> _rateLimit;
private readonly Dictionary<RequestType, DateTime> _lastExecutedRateLimit;
protected DownloadClient(GlobalBase clone, Dictionary<byte, int> rateLimitRequestsPerMinute) : base(clone)
protected DownloadClient(GlobalBase clone) : base(clone)
{
this._lastExecutedRateLimit = new();
_rateLimit = new();
foreach (KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute)
_rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value));
}
internal void SetCustomRequestLimit(byte requestType, int limit)
public RequestResult MakeRequest(string url, RequestType requestType, string? referrer = null, string? clickButton = null)
{
if (_rateLimit.ContainsKey(requestType))
_rateLimit[requestType] = TimeSpan.FromMinutes(1).Divide(limit);
else
_rateLimit.Add(requestType, TimeSpan.FromMinutes(1).Divide(limit));
}
public RequestResult MakeRequest(string url, byte requestType, string? referrer = null, string? clickButton = null)
{
if (_rateLimit.TryGetValue(requestType, out TimeSpan value))
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
else
if (!settings.requestLimits.ContainsKey(requestType))
{
Log("RequestType not configured for rate-limit.");
return new RequestResult(HttpStatusCode.NotAcceptable, null, Stream.Null);
}
TimeSpan rateLimitTimeout = _rateLimit[requestType]
.Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]));
int rateLimit = settings.userAgent == TrangaSettings.DefaultUserAgent
? TrangaSettings.DefaultRequestLimits[requestType]
: settings.requestLimits[requestType];
TimeSpan timeBetweenRequests = TimeSpan.FromMinutes(1).Divide(rateLimit);
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(timeBetweenRequests));
TimeSpan rateLimitTimeout = timeBetweenRequests.Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]));
if (rateLimitTimeout > TimeSpan.Zero)
{

3
Tranga/MangaConnectors/HttpDownloadClient.cs
View File

@ -11,8 +11,7 @@ internal class HttpDownloadClient : DownloadClient
Timeout = TimeSpan.FromSeconds(10)
};
public HttpDownloadClient(GlobalBase clone, Dictionary<byte, int> rateLimitRequestsPerMinute) : base(clone, rateLimitRequestsPerMinute)
public HttpDownloadClient(GlobalBase clone) : base(clone)
{
Client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", settings.userAgent);
}

6
Tranga/MangaConnectors/MangaConnector.cs
View File

@ -202,7 +202,7 @@ public abstract class MangaConnector : GlobalBase
/// <param name="fullPath"></param>
/// <param name="requestType">RequestType for Rate-Limit</param>
/// <param name="referrer">referrer used in html request header</param>
private HttpStatusCode DownloadImage(string imageUrl, string fullPath, byte requestType, string? referrer = null)
private HttpStatusCode DownloadImage(string imageUrl, string fullPath, RequestType requestType, string? referrer = null)
{
RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType, referrer);
@ -217,7 +217,7 @@ public abstract class MangaConnector : GlobalBase
return requestResult.statusCode;
}
protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, string? comicInfoPath = null, string? referrer = null, ProgressToken? progressToken = null)
protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, RequestType requestType, string? comicInfoPath = null, string? referrer = null, ProgressToken? progressToken = null)
{
if (progressToken?.cancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
@ -274,7 +274,7 @@ public abstract class MangaConnector : GlobalBase
return HttpStatusCode.OK;
}
protected string SaveCoverImageToCache(string url, byte requestType)
protected string SaveCoverImageToCache(string url, RequestType requestType)
{
string filetype = url.Split('/')[^1].Split('?')[0].Split('.')[^1];
string filename = $"{DateTime.Now.Ticks.ToString()}.{filetype}";

34
Tranga/MangaConnectors/MangaDex.cs
View File

@ -7,25 +7,9 @@ using JsonSerializer = System.Text.Json.JsonSerializer;
namespace Tranga.MangaConnectors;
public class MangaDex : MangaConnector
{
private enum RequestType : byte
{
Manga,
Feed,
AtHomeServer,
CoverUrl,
Author,
}
public MangaDex(GlobalBase clone) : base(clone, "MangaDex")
{
this.downloadClient = new HttpDownloadClient(clone, new Dictionary<byte, int>()
{
{(byte)RequestType.Manga, 250},
{(byte)RequestType.Feed, 250},
{(byte)RequestType.AtHomeServer, 40},
{(byte)RequestType.CoverUrl, 250},
{(byte)RequestType.Author, 250}
});
this.downloadClient = new HttpDownloadClient(clone);
}
public override Manga[] GetManga(string publicationTitle = "")
@ -41,7 +25,7 @@ public class MangaDex : MangaConnector
//Request next Page
RequestResult requestResult =
downloadClient.MakeRequest(
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", (byte)RequestType.Manga);
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -75,7 +59,7 @@ public class MangaDex : MangaConnector
public override Manga? GetMangaFromId(string publicationId)
{
RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/manga/{publicationId}", (byte)RequestType.Manga);
downloadClient.MakeRequest($"https://api.mangadex.org/manga/{publicationId}", RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -149,7 +133,7 @@ public class MangaDex : MangaConnector
string? coverUrl = GetCoverUrl(publicationId, posterId);
string? coverCacheName = null;
if (coverUrl is not null)
coverCacheName = SaveCoverImageToCache(coverUrl, (byte)RequestType.AtHomeServer);
coverCacheName = SaveCoverImageToCache(coverUrl, RequestType.MangaCover);
List<string> authors = GetAuthors(authorIds);
@ -216,7 +200,7 @@ public class MangaDex : MangaConnector
//Request next "Page"
RequestResult requestResult =
downloadClient.MakeRequest(
$"https://api.mangadex.org/manga/{manga.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", (byte)RequestType.Feed);
$"https://api.mangadex.org/manga/{manga.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", RequestType.MangaDexFeed);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -268,7 +252,7 @@ public class MangaDex : MangaConnector
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
//Request URLs for Chapter-Images
RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer);
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", RequestType.MangaDexImage);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
progressToken?.Cancel();
@ -293,7 +277,7 @@ public class MangaDex : MangaConnector
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
//Download Chapter-Images
return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), (byte)RequestType.AtHomeServer, comicInfoPath, progressToken:progressToken);
return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, progressToken:progressToken);
}
private string? GetCoverUrl(string publicationId, string? posterId)
@ -307,7 +291,7 @@ public class MangaDex : MangaConnector
//Request information where to download Cover
RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", (byte)RequestType.CoverUrl);
downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", RequestType.MangaCover);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -328,7 +312,7 @@ public class MangaDex : MangaConnector
foreach (string authorId in authorIds)
{
RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", (byte)RequestType.Author);
downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", RequestType.MangaDexAuthor);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return ret;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);

17
Tranga/MangaConnectors/MangaKatana.cs
View File

@ -9,10 +9,7 @@ public class MangaKatana : MangaConnector
{
public MangaKatana(GlobalBase clone) : base(clone, "MangaKatana")
{
this.downloadClient = new HttpDownloadClient(clone, new Dictionary<byte, int>()
{
{1, 60}
});
this.downloadClient = new HttpDownloadClient(clone);
}
public override Manga[] GetManga(string publicationTitle = "")
@ -21,7 +18,7 @@ public class MangaKatana : MangaConnector
string sanitizedTitle = string.Join('_', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower();
string requestUrl = $"https://mangakatana.com/?search={sanitizedTitle}&search_by=book_name";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Manga>();
@ -47,7 +44,7 @@ public class MangaKatana : MangaConnector
public override Manga? GetMangaFromUrl(string url)
{
RequestResult requestResult =
downloadClient.MakeRequest(url, 1);
downloadClient.MakeRequest(url, RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
return ParseSinglePublicationFromHtml(requestResult.result, url.Split('/')[^1]);
@ -131,7 +128,7 @@ public class MangaKatana : MangaConnector
string posterUrl = document.DocumentNode.SelectSingleNode("//*[@id='single_book']/div[1]/div").Descendants("img").First()
.GetAttributes().First(a => a.Name == "src").Value;
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1);
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover);
string description = document.DocumentNode.SelectSingleNode("//*[@id='single_book']/div[3]/p").InnerText;
while (description.StartsWith('\n'))
@ -158,7 +155,7 @@ public class MangaKatana : MangaConnector
string requestUrl = $"https://mangakatana.com/manga/{manga.publicationId}";
// Leaving this in for verification if the page exists
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Chapter>();
@ -210,7 +207,7 @@ public class MangaKatana : MangaConnector
string requestUrl = chapter.url;
// Leaving this in to check if the page exists
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
progressToken?.Cancel();
@ -222,7 +219,7 @@ public class MangaKatana : MangaConnector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken);
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken);
}
private string[] ParseImageUrlsFromHtml(string mangaUrl)

17
Tranga/MangaConnectors/MangaLife.cs
View File

@ -9,10 +9,7 @@ public class MangaLife : MangaConnector
{
public MangaLife(GlobalBase clone) : base(clone, "Manga4Life")
{
this.downloadClient = new ChromiumDownloadClient(clone, new Dictionary<byte, int>()
{
{ 1, 60 }
});
this.downloadClient = new ChromiumDownloadClient(clone);
}
public override Manga[] GetManga(string publicationTitle = "")
@ -21,7 +18,7 @@ public class MangaLife : MangaConnector
string sanitizedTitle = WebUtility.UrlEncode(publicationTitle);
string requestUrl = $"https://manga4life.com/search/?name={sanitizedTitle}";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Manga>();
@ -42,7 +39,7 @@ public class MangaLife : MangaConnector
Regex publicationIdRex = new(@"https:\/\/(www\.)?manga4life.com\/manga\/(.*)(\/.*)*");
string publicationId = publicationIdRex.Match(url).Groups[2].Value;
RequestResult requestResult = this.downloadClient.MakeRequest(url, 1);
RequestResult requestResult = this.downloadClient.MakeRequest(url, RequestType.MangaInfo);
if(requestResult.htmlDocument is not null)
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId);
return null;
@ -81,7 +78,7 @@ public class MangaLife : MangaConnector
HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//img");
string posterUrl = posterNode.GetAttributeValue("src", "");
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1);
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover);
HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//h1");
string sortName = titleNode.InnerText;
@ -133,7 +130,7 @@ public class MangaLife : MangaConnector
public override Chapter[] GetChapters(Manga manga, string language="en")
{
Log($"Getting chapters {manga}");
RequestResult result = downloadClient.MakeRequest($"https://manga4life.com/manga/{manga.publicationId}", 1, clickButton:"[class*='ShowAllChapters']");
RequestResult result = downloadClient.MakeRequest($"https://manga4life.com/manga/{manga.publicationId}", RequestType.Default, clickButton:"[class*='ShowAllChapters']");
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null)
{
return Array.Empty<Chapter>();
@ -179,7 +176,7 @@ public class MangaLife : MangaConnector
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, 1);
RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, RequestType.Default);
if (requestResult.htmlDocument is null)
{
progressToken?.Cancel();
@ -197,6 +194,6 @@ public class MangaLife : MangaConnector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, progressToken:progressToken);
return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, progressToken:progressToken);
}
}

17
Tranga/MangaConnectors/Manganato.cs
View File

@ -9,10 +9,7 @@ public class Manganato : MangaConnector
{
public Manganato(GlobalBase clone) : base(clone, "Manganato")
{
this.downloadClient = new HttpDownloadClient(clone, new Dictionary<byte, int>()
{
{1, 60}
});
this.downloadClient = new HttpDownloadClient(clone);
}
public override Manga[] GetManga(string publicationTitle = "")
@ -21,7 +18,7 @@ public class Manganato : MangaConnector
string sanitizedTitle = string.Join('_', Regex.Matches(publicationTitle, "[A-z]*").Where(str => str.Length > 0)).ToLower();
string requestUrl = $"https://manganato.com/search/story/{sanitizedTitle}";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Manga>();
@ -62,7 +59,7 @@ public class Manganato : MangaConnector
public override Manga? GetMangaFromUrl(string url)
{
RequestResult requestResult =
downloadClient.MakeRequest(url, 1);
downloadClient.MakeRequest(url, RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
@ -121,7 +118,7 @@ public class Manganato : MangaConnector
string posterUrl = document.DocumentNode.Descendants("span").First(s => s.HasClass("info-image")).Descendants("img").First()
.GetAttributes().First(a => a.Name == "src").Value;
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1);
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover);
string description = document.DocumentNode.Descendants("div").First(d => d.HasClass("panel-story-info-description"))
.InnerText.Replace("Description :", "");
@ -143,7 +140,7 @@ public class Manganato : MangaConnector
Log($"Getting chapters {manga}");
string requestUrl = $"https://chapmanganato.com/{manga.publicationId}";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Chapter>();
@ -192,7 +189,7 @@ public class Manganato : MangaConnector
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
string requestUrl = chapter.url;
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
progressToken?.Cancel();
@ -210,7 +207,7 @@ public class Manganato : MangaConnector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://chapmanganato.com/", progressToken:progressToken);
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, "https://chapmanganato.com/", progressToken:progressToken);
}
private string[] ParseImageUrlsFromHtml(HtmlDocument document)

15
Tranga/MangaConnectors/Mangasee.cs
View File

@ -12,10 +12,7 @@ public class Mangasee : MangaConnector
{
public Mangasee(GlobalBase clone) : base(clone, "Mangasee")
{
this.downloadClient = new ChromiumDownloadClient(clone, new Dictionary<byte, int>()
{
{ 1, 60 }
});
this.downloadClient = new ChromiumDownloadClient(clone);
}
private struct SearchResult
@ -30,7 +27,7 @@ public class Mangasee : MangaConnector
Log($"Searching Publications. Term=\"{publicationTitle}\"");
string requestUrl = "https://mangasee123.com/_search.php";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
Log($"Failed to retrieve search: {requestResult.statusCode}");
@ -121,7 +118,7 @@ public class Mangasee : MangaConnector
Regex publicationIdRex = new(@"https:\/\/mangasee123.com\/manga\/(.*)(\/.*)*");
string publicationId = publicationIdRex.Match(url).Groups[1].Value;
RequestResult requestResult = this.downloadClient.MakeRequest(url, 1);
RequestResult requestResult = this.downloadClient.MakeRequest(url, RequestType.MangaInfo);
if((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 && requestResult.htmlDocument is not null)
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId);
return null;
@ -136,7 +133,7 @@ public class Mangasee : MangaConnector
HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//img");
string posterUrl = posterNode.GetAttributeValue("src", "");
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1);
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover);
HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//h1");
string sortName = titleNode.InnerText;
@ -232,7 +229,7 @@ public class Mangasee : MangaConnector
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, 1);
RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, RequestType.Default);
if (requestResult.htmlDocument is null)
{
progressToken?.Cancel();
@ -250,6 +247,6 @@ public class Mangasee : MangaConnector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, progressToken:progressToken);
return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, progressToken:progressToken);
}
}

17
Tranga/MangaConnectors/Mangaworld.cs
View File

@ -9,10 +9,7 @@ public class Mangaworld: MangaConnector
{
public Mangaworld(GlobalBase clone) : base(clone, "Mangaworld")
{
this.downloadClient = new HttpDownloadClient(clone, new Dictionary<byte, int>()
{
{1, 60}
});
this.downloadClient = new HttpDownloadClient(clone);
}
public override Manga[] GetManga(string publicationTitle = "")
@ -21,7 +18,7 @@ public class Mangaworld: MangaConnector
string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(str => str.Length > 0)).ToLower();
string requestUrl = $"https://www.mangaworld.ac/archive?keyword={sanitizedTitle}";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Manga>();
@ -62,7 +59,7 @@ public class Mangaworld: MangaConnector
public override Manga? GetMangaFromUrl(string url)
{
RequestResult requestResult =
downloadClient.MakeRequest(url, 1);
downloadClient.MakeRequest(url, RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
@ -114,7 +111,7 @@ public class Mangaworld: MangaConnector
string posterUrl = document.DocumentNode.SelectSingleNode("//img[@class='rounded']").GetAttributeValue("src", "");
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1);
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover);
string description = document.DocumentNode.SelectSingleNode("//div[@id='noidungm']").InnerText;
@ -132,7 +129,7 @@ public class Mangaworld: MangaConnector
Log($"Getting chapters {manga}");
string requestUrl = $"https://www.mangaworld.ac/manga/{manga.publicationId}";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Chapter>();
@ -191,7 +188,7 @@ public class Mangaworld: MangaConnector
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
string requestUrl = $"{chapter.url}?style=list";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
progressToken?.Cancel();
@ -209,7 +206,7 @@ public class Mangaworld: MangaConnector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://www.mangaworld.bz/", progressToken:progressToken);
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, "https://www.mangaworld.bz/", progressToken:progressToken);
}
private string[] ParseImageUrlsFromHtml(HtmlDocument document)

12
Tranga/MangaConnectors/RequestType.cs Normal file
View File

@ -0,0 +1,12 @@
namespace Tranga.MangaConnectors;
public enum RequestType : byte
{
Default = 0,
MangaDexFeed = 1,
MangaImage = 2,
MangaCover = 3,
MangaDexAuthor = 4,
MangaDexImage = 5,
MangaInfo = 6
}

13
Tranga/Server.cs
View File

@ -399,16 +399,19 @@ public class Server : GlobalBase
case "Settings/customRequestLimit":
if (!requestVariables.TryGetValue("requestType", out string? requestTypeStr) ||
!requestVariables.TryGetValue("requestsPerMinute", out string? requestsPerMinuteStr) ||
!requestVariables.TryGetValue("connector", out connectorName) ||
!byte.TryParse(requestTypeStr, out byte requestType) ||
!int.TryParse(requestsPerMinuteStr, out int requestsPerMinute) ||
!_parent.TryGetConnector(connectorName, out connector))
!Enum.TryParse(requestTypeStr, out RequestType requestType) ||
!int.TryParse(requestsPerMinuteStr, out int requestsPerMinute))
{
SendResponse(HttpStatusCode.BadRequest, response);
break;
}
connector!.downloadClient.SetCustomRequestLimit(requestType, requestsPerMinute);
if (settings.requestLimits.ContainsKey(requestType))
{
settings.requestLimits[requestType] = requestsPerMinute;
SendResponse(HttpStatusCode.Accepted, response);
}else
SendResponse(HttpStatusCode.BadRequest, response);
break;
case "NotificationConnectors/Update":
if (!requestVariables.TryGetValue("notificationConnector", out string? notificationConnectorStr) ||

20
Tranga/TrangaSettings.cs
View File

@ -2,6 +2,7 @@
using System.Runtime.InteropServices;
using Newtonsoft.Json;
using Tranga.LibraryConnectors;
using Tranga.MangaConnectors;
using Tranga.NotificationConnectors;
using static System.IO.UnixFileMode;
@ -13,15 +14,26 @@ public class TrangaSettings
public string workingDirectory { get; private set; }
public int apiPortNumber { get; init; }
public string styleSheet { get; private set; }
public string userAgent { get; set; } =
$"Tranga ({Enum.GetName(Environment.OSVersion.Platform)}; {(Environment.Is64BitOperatingSystem ? "x64" : "")}) / 1.0";
public string userAgent { get; set; } = DefaultUserAgent;
[JsonIgnore] public string settingsFilePath => Path.Join(workingDirectory, "settings.json");
[JsonIgnore] public string libraryConnectorsFilePath => Path.Join(workingDirectory, "libraryConnectors.json");
[JsonIgnore] public string notificationConnectorsFilePath => Path.Join(workingDirectory, "notificationConnectors.json");
[JsonIgnore] public string jobsFolderPath => Path.Join(workingDirectory, "jobs");
[JsonIgnore] public string coverImageCache => Path.Join(workingDirectory, "imageCache");
public ushort? version { get; set; }
[JsonIgnore] internal static readonly string DefaultUserAgent = $"Tranga ({Enum.GetName(Environment.OSVersion.Platform)}; {(Environment.Is64BitOperatingSystem ? "x64" : "")}) / 1.0";
public ushort? version { get; set; } = 1;
[JsonIgnore]internal static readonly Dictionary<RequestType, int> DefaultRequestLimits = new ()
{
{RequestType.MangaInfo, 250},
{RequestType.MangaDexFeed, 250},
{RequestType.MangaDexImage, 40},
{RequestType.MangaImage, 60},
{RequestType.MangaCover, 250},
{RequestType.MangaDexAuthor, 250},
{RequestType.Default, 60}
};
public Dictionary<RequestType, int> requestLimits { get; set; } = DefaultRequestLimits;
public TrangaSettings(string? downloadLocation = null, string? workingDirectory = null, int? apiPortNumber = null)
{