Compare commits

..

4 Commits

Author SHA1 Message Date
9a640aed27 Rewrote CoverDownload check if exists. 2023-05-22 21:38:44 +02:00
30b6c4680b Better Rate-Limits
Added Logger to DownloadClient
2023-05-22 21:38:23 +02:00
7b6253de0f Create Publication Folder at start of DownloadNewChapters 2023-05-22 21:37:30 +02:00
5aa3214ce5 TrangaTask.ToString() rewrite for logs-readability.
LogMessages only include class-name without path
2023-05-22 21:37:02 +02:00
6 changed files with 53 additions and 33 deletions

View File

@ -52,7 +52,7 @@ public abstract class LoggerBase : TextWriter
public override string ToString() public override string ToString()
{ {
string dateTimeString = $"{logTime.ToShortDateString()} {logTime.ToLongTimeString()}"; string dateTimeString = $"{logTime.ToShortDateString()} {logTime.ToLongTimeString()}";
return $"[{dateTimeString}] {caller,30} | {value}"; return $"[{dateTimeString}] {caller.Split(new char[]{'.','+'}).Last(),15} | {value}";
} }
} }
} }

View File

@ -209,8 +209,12 @@ public static class Tranga_Cli
$"{"",-5}{"Task",-20} | {"Last Executed",-20} | {"Reoccurrence",-12} | {"State",-10} | {"Connector",-15} | Publication/Manga"; $"{"",-5}{"Task",-20} | {"Last Executed",-20} | {"Reoccurrence",-12} | {"State",-10} | {"Connector",-15} | Publication/Manga";
Console.WriteLine(header); Console.WriteLine(header);
Console.WriteLine(new string('-', header.Length)); Console.WriteLine(new string('-', header.Length));
foreach(TrangaTask trangaTask in tasks) foreach (TrangaTask trangaTask in tasks)
Console.WriteLine($"{tIndex++:000}: {trangaTask}"); {
string[] taskSplit = trangaTask.ToString().Split(", ");
Console.WriteLine($"{tIndex++:000}: {taskSplit[0],-20} | {taskSplit[1],-20} | {taskSplit[2],-12} | {taskSplit[3],-10} | {(taskSplit.Length > 4 ? taskSplit[4] : ""),-15} | {(taskSplit.Length > 5 ? taskSplit[5] : "")}");
}
} }
private static TrangaTask? SelectTask(TrangaTask[] tasks, Logger logger) private static TrangaTask? SelectTask(TrangaTask[] tasks, Logger logger)

View File

@ -169,19 +169,21 @@ public abstract class Connector
private static readonly HttpClient Client = new(); private static readonly HttpClient Client = new();
private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit; private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit;
private readonly Dictionary<byte, TimeSpan> _RateLimit; private readonly Dictionary<byte, TimeSpan> _rateLimit;
private Logger? logger;
/// <summary> /// <summary>
/// Creates a httpClient /// Creates a httpClient
/// </summary> /// </summary>
/// <param name="delay">minimum delay between requests (to avoid spam)</param> /// <param name="delay">minimum delay between requests (to avoid spam)</param>
/// <param name="rateLimitRequestsPerMinute">Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType</param> /// <param name="rateLimitRequestsPerMinute">Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType</param>
public DownloadClient(Dictionary<byte, int> rateLimitRequestsPerMinute) public DownloadClient(Dictionary<byte, int> rateLimitRequestsPerMinute, Logger? logger)
{ {
this.logger = logger;
_lastExecutedRateLimit = new(); _lastExecutedRateLimit = new();
_RateLimit = new(); _rateLimit = new();
foreach(KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute) foreach(KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute)
_RateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value)); _rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value));
} }
/// <summary> /// <summary>
@ -192,19 +194,36 @@ public abstract class Connector
/// <returns>RequestResult with StatusCode and Stream of received data</returns> /// <returns>RequestResult with StatusCode and Stream of received data</returns>
public RequestResult MakeRequest(string url, byte requestType) public RequestResult MakeRequest(string url, byte requestType)
{ {
if (_RateLimit.TryGetValue(requestType, out TimeSpan value)) if (_rateLimit.TryGetValue(requestType, out TimeSpan value))
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value)); _lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
else else
{
logger?.WriteLine(this.GetType().ToString(), "RequestType not configured for rate-limit.");
return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null); return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null);
}
TimeSpan rateLimitTimeout = _rateLimit[requestType]
.Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]));
while(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]) < _RateLimit[requestType]) Thread.Sleep(rateLimitTimeout);
Thread.Sleep(10);
_lastExecutedRateLimit[requestType] = DateTime.Now;
HttpResponseMessage? response = null;
while (response is null)
{
try
{
HttpRequestMessage requestMessage = new(HttpMethod.Get, url); HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
HttpResponseMessage response = Client.Send(requestMessage); _lastExecutedRateLimit[requestType] = DateTime.Now;
response = Client.Send(requestMessage);
}
catch (HttpRequestException e)
{
Thread.Sleep(_rateLimit[requestType] * 2);
}
}
Stream resultString = response.IsSuccessStatusCode ? response.Content.ReadAsStream() : Stream.Null; Stream resultString = response.IsSuccessStatusCode ? response.Content.ReadAsStream() : Stream.Null;
if (!response.IsSuccessStatusCode)
logger?.WriteLine(this.GetType().ToString(), $"Request-Error {response.StatusCode}: {response.ReasonPhrase}");
return new RequestResult(response.StatusCode, resultString); return new RequestResult(response.StatusCode, resultString);
} }

View File

@ -28,7 +28,7 @@ public class MangaDex : Connector
{(byte)RequestType.AtHomeServer, 40}, {(byte)RequestType.AtHomeServer, 40},
{(byte)RequestType.Cover, 250}, {(byte)RequestType.Cover, 250},
{(byte)RequestType.Author, 250} {(byte)RequestType.Author, 250}
}); }, logger);
} }
public override Publication[] GetPublications(string publicationTitle = "") public override Publication[] GetPublications(string publicationTitle = "")
@ -270,29 +270,26 @@ public class MangaDex : Connector
if(!Directory.Exists(publicationFolder)) if(!Directory.Exists(publicationFolder))
Directory.CreateDirectory(publicationFolder); Directory.CreateDirectory(publicationFolder);
DirectoryInfo dirInfo = new (publicationFolder); DirectoryInfo dirInfo = new (publicationFolder);
foreach(FileInfo fileInfo in dirInfo.EnumerateFiles()) if (dirInfo.EnumerateFiles().Any(info => info.Name.Contains("cover.")))
if (fileInfo.Name.Contains("cover."))
{ {
logger?.WriteLine(this.GetType().ToString(), $"Cover exists {publication.sortName}"); logger?.WriteLine(this.GetType().ToString(), $"Cover exists {publication.sortName}");
return; return;
} }
if (publication.posterUrl is null) if (publication.posterUrl is null || !(bool)publication.posterUrl?.Contains("http"))
{ {
logger?.WriteLine(this.GetType().ToString(), $"No posterurl in publication"); logger?.WriteLine(this.GetType().ToString(), $"No Poster-URL in publication");
return; return;
} }
string coverUrl = publication.posterUrl;
//Get file-extension (jpg, png) //Get file-extension (jpg, png)
string[] split = coverUrl.Split('.'); string[] split = publication.posterUrl.Split('.');
string extension = split[^1]; string extension = split[^1];
string outFolderPath = Path.Join(downloadLocation, publication.folderName); string outFolderPath = Path.Join(downloadLocation, publication.folderName);
Directory.CreateDirectory(outFolderPath); Directory.CreateDirectory(outFolderPath);
//Download cover-Image //Download cover-Image
DownloadImage(coverUrl, Path.Join(downloadLocation, publication.folderName, $"cover.{extension}"), this.downloadClient, (byte)RequestType.AtHomeServer); DownloadImage(publication.posterUrl, Path.Join(downloadLocation, publication.folderName, $"cover.{extension}"), this.downloadClient, (byte)RequestType.AtHomeServer);
} }
} }

View File

@ -92,13 +92,13 @@ public static class TaskExecutor
/// <param name="chapterCollection"></param> /// <param name="chapterCollection"></param>
private static void DownloadNewChapters(Connector connector, Publication publication, string language, ref Dictionary<Publication, List<Chapter>> chapterCollection) private static void DownloadNewChapters(Connector connector, Publication publication, string language, ref Dictionary<Publication, List<Chapter>> chapterCollection)
{ {
List<Chapter> newChapters = UpdateChapters(connector, publication, language, ref chapterCollection); //Check if Publication already has a Folder
connector.DownloadCover(publication);
//Check if Publication already has a Folder and a series.json
string publicationFolder = Path.Join(connector.downloadLocation, publication.folderName); string publicationFolder = Path.Join(connector.downloadLocation, publication.folderName);
if(!Directory.Exists(publicationFolder)) if(!Directory.Exists(publicationFolder))
Directory.CreateDirectory(publicationFolder); Directory.CreateDirectory(publicationFolder);
List<Chapter> newChapters = UpdateChapters(connector, publication, language, ref chapterCollection);
connector.DownloadCover(publication);
string seriesInfoPath = Path.Join(publicationFolder, "series.json"); string seriesInfoPath = Path.Join(publicationFolder, "series.json");
if(!File.Exists(seriesInfoPath)) if(!File.Exists(seriesInfoPath))

View File

@ -56,6 +56,6 @@ public class TrangaTask
public override string ToString() public override string ToString()
{ {
return $"{task,-20} | {lastExecuted,-20} | {reoccurrence,-12} | {state,-10} | {connectorName,-15} | {publication?.sortName}"; return $"{task}, {lastExecuted}, {reoccurrence}, {state} {(connectorName is not null ? $", {connectorName}" : "" )} {(publication is not null ? $", {publication?.sortName}": "")}";
} }
} }