Compare commits
4 Commits
9b70994f71
...
9a640aed27
Author | SHA1 | Date | |
---|---|---|---|
9a640aed27 | |||
30b6c4680b | |||
7b6253de0f | |||
5aa3214ce5 |
@ -52,7 +52,7 @@ public abstract class LoggerBase : TextWriter
|
|||||||
public override string ToString()
|
public override string ToString()
|
||||||
{
|
{
|
||||||
string dateTimeString = $"{logTime.ToShortDateString()} {logTime.ToLongTimeString()}";
|
string dateTimeString = $"{logTime.ToShortDateString()} {logTime.ToLongTimeString()}";
|
||||||
return $"[{dateTimeString}] {caller,30} | {value}";
|
return $"[{dateTimeString}] {caller.Split(new char[]{'.','+'}).Last(),15} | {value}";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -210,7 +210,11 @@ public static class Tranga_Cli
|
|||||||
Console.WriteLine(header);
|
Console.WriteLine(header);
|
||||||
Console.WriteLine(new string('-', header.Length));
|
Console.WriteLine(new string('-', header.Length));
|
||||||
foreach (TrangaTask trangaTask in tasks)
|
foreach (TrangaTask trangaTask in tasks)
|
||||||
Console.WriteLine($"{tIndex++:000}: {trangaTask}");
|
{
|
||||||
|
string[] taskSplit = trangaTask.ToString().Split(", ");
|
||||||
|
Console.WriteLine($"{tIndex++:000}: {taskSplit[0],-20} | {taskSplit[1],-20} | {taskSplit[2],-12} | {taskSplit[3],-10} | {(taskSplit.Length > 4 ? taskSplit[4] : ""),-15} | {(taskSplit.Length > 5 ? taskSplit[5] : "")}");
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static TrangaTask? SelectTask(TrangaTask[] tasks, Logger logger)
|
private static TrangaTask? SelectTask(TrangaTask[] tasks, Logger logger)
|
||||||
|
@ -169,19 +169,21 @@ public abstract class Connector
|
|||||||
private static readonly HttpClient Client = new();
|
private static readonly HttpClient Client = new();
|
||||||
|
|
||||||
private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit;
|
private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit;
|
||||||
private readonly Dictionary<byte, TimeSpan> _RateLimit;
|
private readonly Dictionary<byte, TimeSpan> _rateLimit;
|
||||||
|
private Logger? logger;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a httpClient
|
/// Creates a httpClient
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="delay">minimum delay between requests (to avoid spam)</param>
|
/// <param name="delay">minimum delay between requests (to avoid spam)</param>
|
||||||
/// <param name="rateLimitRequestsPerMinute">Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType</param>
|
/// <param name="rateLimitRequestsPerMinute">Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType</param>
|
||||||
public DownloadClient(Dictionary<byte, int> rateLimitRequestsPerMinute)
|
public DownloadClient(Dictionary<byte, int> rateLimitRequestsPerMinute, Logger? logger)
|
||||||
{
|
{
|
||||||
|
this.logger = logger;
|
||||||
_lastExecutedRateLimit = new();
|
_lastExecutedRateLimit = new();
|
||||||
_RateLimit = new();
|
_rateLimit = new();
|
||||||
foreach(KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute)
|
foreach(KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute)
|
||||||
_RateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value));
|
_rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@ -192,19 +194,36 @@ public abstract class Connector
|
|||||||
/// <returns>RequestResult with StatusCode and Stream of received data</returns>
|
/// <returns>RequestResult with StatusCode and Stream of received data</returns>
|
||||||
public RequestResult MakeRequest(string url, byte requestType)
|
public RequestResult MakeRequest(string url, byte requestType)
|
||||||
{
|
{
|
||||||
if (_RateLimit.TryGetValue(requestType, out TimeSpan value))
|
if (_rateLimit.TryGetValue(requestType, out TimeSpan value))
|
||||||
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
|
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
|
||||||
else
|
else
|
||||||
|
{
|
||||||
|
logger?.WriteLine(this.GetType().ToString(), "RequestType not configured for rate-limit.");
|
||||||
return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null);
|
return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null);
|
||||||
|
}
|
||||||
|
|
||||||
|
TimeSpan rateLimitTimeout = _rateLimit[requestType]
|
||||||
|
.Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]));
|
||||||
|
|
||||||
while(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]) < _RateLimit[requestType])
|
Thread.Sleep(rateLimitTimeout);
|
||||||
Thread.Sleep(10);
|
|
||||||
_lastExecutedRateLimit[requestType] = DateTime.Now;
|
|
||||||
|
|
||||||
|
HttpResponseMessage? response = null;
|
||||||
|
while (response is null)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
|
HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
|
||||||
HttpResponseMessage response = Client.Send(requestMessage);
|
_lastExecutedRateLimit[requestType] = DateTime.Now;
|
||||||
|
response = Client.Send(requestMessage);
|
||||||
|
}
|
||||||
|
catch (HttpRequestException e)
|
||||||
|
{
|
||||||
|
Thread.Sleep(_rateLimit[requestType] * 2);
|
||||||
|
}
|
||||||
|
}
|
||||||
Stream resultString = response.IsSuccessStatusCode ? response.Content.ReadAsStream() : Stream.Null;
|
Stream resultString = response.IsSuccessStatusCode ? response.Content.ReadAsStream() : Stream.Null;
|
||||||
|
if (!response.IsSuccessStatusCode)
|
||||||
|
logger?.WriteLine(this.GetType().ToString(), $"Request-Error {response.StatusCode}: {response.ReasonPhrase}");
|
||||||
return new RequestResult(response.StatusCode, resultString);
|
return new RequestResult(response.StatusCode, resultString);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -28,7 +28,7 @@ public class MangaDex : Connector
|
|||||||
{(byte)RequestType.AtHomeServer, 40},
|
{(byte)RequestType.AtHomeServer, 40},
|
||||||
{(byte)RequestType.Cover, 250},
|
{(byte)RequestType.Cover, 250},
|
||||||
{(byte)RequestType.Author, 250}
|
{(byte)RequestType.Author, 250}
|
||||||
});
|
}, logger);
|
||||||
}
|
}
|
||||||
|
|
||||||
public override Publication[] GetPublications(string publicationTitle = "")
|
public override Publication[] GetPublications(string publicationTitle = "")
|
||||||
@ -270,29 +270,26 @@ public class MangaDex : Connector
|
|||||||
if(!Directory.Exists(publicationFolder))
|
if(!Directory.Exists(publicationFolder))
|
||||||
Directory.CreateDirectory(publicationFolder);
|
Directory.CreateDirectory(publicationFolder);
|
||||||
DirectoryInfo dirInfo = new (publicationFolder);
|
DirectoryInfo dirInfo = new (publicationFolder);
|
||||||
foreach(FileInfo fileInfo in dirInfo.EnumerateFiles())
|
if (dirInfo.EnumerateFiles().Any(info => info.Name.Contains("cover.")))
|
||||||
if (fileInfo.Name.Contains("cover."))
|
|
||||||
{
|
{
|
||||||
logger?.WriteLine(this.GetType().ToString(), $"Cover exists {publication.sortName}");
|
logger?.WriteLine(this.GetType().ToString(), $"Cover exists {publication.sortName}");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (publication.posterUrl is null)
|
if (publication.posterUrl is null || !(bool)publication.posterUrl?.Contains("http"))
|
||||||
{
|
{
|
||||||
logger?.WriteLine(this.GetType().ToString(), $"No posterurl in publication");
|
logger?.WriteLine(this.GetType().ToString(), $"No Poster-URL in publication");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
string coverUrl = publication.posterUrl;
|
|
||||||
|
|
||||||
//Get file-extension (jpg, png)
|
//Get file-extension (jpg, png)
|
||||||
string[] split = coverUrl.Split('.');
|
string[] split = publication.posterUrl.Split('.');
|
||||||
string extension = split[^1];
|
string extension = split[^1];
|
||||||
|
|
||||||
string outFolderPath = Path.Join(downloadLocation, publication.folderName);
|
string outFolderPath = Path.Join(downloadLocation, publication.folderName);
|
||||||
Directory.CreateDirectory(outFolderPath);
|
Directory.CreateDirectory(outFolderPath);
|
||||||
|
|
||||||
//Download cover-Image
|
//Download cover-Image
|
||||||
DownloadImage(coverUrl, Path.Join(downloadLocation, publication.folderName, $"cover.{extension}"), this.downloadClient, (byte)RequestType.AtHomeServer);
|
DownloadImage(publication.posterUrl, Path.Join(downloadLocation, publication.folderName, $"cover.{extension}"), this.downloadClient, (byte)RequestType.AtHomeServer);
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -92,13 +92,13 @@ public static class TaskExecutor
|
|||||||
/// <param name="chapterCollection"></param>
|
/// <param name="chapterCollection"></param>
|
||||||
private static void DownloadNewChapters(Connector connector, Publication publication, string language, ref Dictionary<Publication, List<Chapter>> chapterCollection)
|
private static void DownloadNewChapters(Connector connector, Publication publication, string language, ref Dictionary<Publication, List<Chapter>> chapterCollection)
|
||||||
{
|
{
|
||||||
List<Chapter> newChapters = UpdateChapters(connector, publication, language, ref chapterCollection);
|
//Check if Publication already has a Folder
|
||||||
connector.DownloadCover(publication);
|
|
||||||
|
|
||||||
//Check if Publication already has a Folder and a series.json
|
|
||||||
string publicationFolder = Path.Join(connector.downloadLocation, publication.folderName);
|
string publicationFolder = Path.Join(connector.downloadLocation, publication.folderName);
|
||||||
if(!Directory.Exists(publicationFolder))
|
if(!Directory.Exists(publicationFolder))
|
||||||
Directory.CreateDirectory(publicationFolder);
|
Directory.CreateDirectory(publicationFolder);
|
||||||
|
List<Chapter> newChapters = UpdateChapters(connector, publication, language, ref chapterCollection);
|
||||||
|
|
||||||
|
connector.DownloadCover(publication);
|
||||||
|
|
||||||
string seriesInfoPath = Path.Join(publicationFolder, "series.json");
|
string seriesInfoPath = Path.Join(publicationFolder, "series.json");
|
||||||
if(!File.Exists(seriesInfoPath))
|
if(!File.Exists(seriesInfoPath))
|
||||||
|
@ -56,6 +56,6 @@ public class TrangaTask
|
|||||||
|
|
||||||
public override string ToString()
|
public override string ToString()
|
||||||
{
|
{
|
||||||
return $"{task,-20} | {lastExecuted,-20} | {reoccurrence,-12} | {state,-10} | {connectorName,-15} | {publication?.sortName}";
|
return $"{task}, {lastExecuted}, {reoccurrence}, {state} {(connectorName is not null ? $", {connectorName}" : "" )} {(publication is not null ? $", {publication?.sortName}": "")}";
|
||||||
}
|
}
|
||||||
}
|
}
|
Loading…
Reference in New Issue
Block a user