glax/Tranga
2
0
Fork 0
Code

Using HttpStatusCode to signify Task-Success

Browse Source 
When DownloadChapterTask returns notfound, do not retry.
This commit is contained in:
glax 2023-06-20 15:46:54 +02:00
parent 1035939309
commit 9af5c1603e
9 changed files with 93 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
Tranga/Connector.cs
View File

@ -1,4 +1,5 @@
using System.IO.Compression;
using System.Net;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using System.Xml.Linq;
@ -133,7 +134,7 @@ public abstract class Connector
/// <param name="chapter">Chapter with Images to retrieve</param>
/// <param name="parentTask">Will be used for progress-tracking</param>
/// <param name="cancellationToken"></param>
public abstract bool DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null);
public abstract HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null);
/// <summary>
/// Copies the already downloaded cover from cache to downloadLocation
@ -212,15 +213,15 @@ public abstract class Connector
/// <param name="fullPath"></param>
/// <param name="requestType">RequestType for Rate-Limit</param>
/// <param name="referrer">referrer used in html request header</param>
private bool DownloadImage(string imageUrl, string fullPath, byte requestType, string? referrer = null)
private HttpStatusCode DownloadImage(string imageUrl, string fullPath, byte requestType, string? referrer = null)
{
DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType, referrer);
if (!requestResult.success || requestResult.result == Stream.Null)
return false;
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.result == Stream.Null)
return requestResult.statusCode;
byte[] buffer = new byte[requestResult.result.Length];
requestResult.result.ReadExactly(buffer, 0, buffer.Length);
File.WriteAllBytes(fullPath, buffer);
return true;
return requestResult.statusCode;
}
/// <summary>
@ -233,10 +234,10 @@ public abstract class Connector
/// <param name="requestType">RequestType for RateLimits</param>
/// <param name="referrer">Used in http request header</param>
/// <param name="cancellationToken"></param>
protected bool DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, DownloadChapterTask parentTask, string? comicInfoPath = null, string? referrer = null, CancellationToken? cancellationToken = null)
protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, DownloadChapterTask parentTask, string? comicInfoPath = null, string? referrer = null, CancellationToken? cancellationToken = null)
{
if (cancellationToken?.IsCancellationRequested??false)
return false;
if (cancellationToken?.IsCancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
logger?.WriteLine("Connector", $"Downloading Images for {saveArchiveFilePath}");
//Check if Publication Directory already exists
string directoryPath = Path.GetDirectoryName(saveArchiveFilePath)!;
@ -244,7 +245,7 @@ public abstract class Connector
Directory.CreateDirectory(directoryPath);
if (File.Exists(saveArchiveFilePath)) //Don't download twice.
return false;
return HttpStatusCode.OK;
//Create a temporary folder to store images
string tempFolder = Directory.CreateTempSubdirectory().FullName;
@ -256,11 +257,12 @@ public abstract class Connector
string[] split = imageUrl.Split('.');
string extension = split[^1];
logger?.WriteLine("Connector", $"Downloading Image {chapter + 1:000}/{imageUrls.Length:000} {parentTask.publication.sortName} {parentTask.publication.internalId} Vol.{parentTask.chapter.volumeNumber} Ch.{parentTask.chapter.chapterNumber} {parentTask.progress:P2}");
if (!DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer))
return false;
HttpStatusCode status = DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer);
if ((int)status < 200 || (int)status >= 300)
return status;
parentTask.IncrementProgress(1.0 / imageUrls.Length);
if (cancellationToken?.IsCancellationRequested??false)
return false;
if (cancellationToken?.IsCancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
}
if(comicInfoPath is not null)
@ -272,7 +274,7 @@ public abstract class Connector
if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
File.SetUnixFileMode(saveArchiveFilePath, GroupRead | GroupWrite | OtherRead | OtherWrite | UserRead | UserWrite);
Directory.Delete(tempFolder, true); //Cleanup
return true;
return HttpStatusCode.OK;
}
protected string SaveCoverImageToCache(string url, byte requestType)
@ -332,7 +334,7 @@ public abstract class Connector
else
{
logger?.WriteLine(this.GetType().ToString(), "RequestType not configured for rate-limit.");
return new RequestResult(false, Stream.Null);
return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null);
}
TimeSpan rateLimitTimeout = _rateLimit[requestType]
@ -362,19 +364,19 @@ public abstract class Connector
if (!response.IsSuccessStatusCode)
{
logger?.WriteLine(this.GetType().ToString(), $"Request-Error {response.StatusCode}: {response.ReasonPhrase}");
return new RequestResult(false, Stream.Null);
return new RequestResult(response.StatusCode, Stream.Null);
}
return new RequestResult(true, response.Content.ReadAsStream());
return new RequestResult(response.StatusCode, response.Content.ReadAsStream());
}
public struct RequestResult
{
public bool success { get; }
public HttpStatusCode statusCode { get; }
public Stream result { get; }
public RequestResult(bool success, Stream result)
public RequestResult(HttpStatusCode statusCode, Stream result)
{
this.success = success;
this.statusCode = statusCode;
this.result = result;
}
}

20
Tranga/Connectors/MangaDex.cs
View File

@ -46,7 +46,7 @@ public class MangaDex : Connector
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", (byte)RequestType.Manga);
if (!requestResult.success)
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -165,7 +165,7 @@ public class MangaDex : Connector
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(
$"https://api.mangadex.org/manga/{publication.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", (byte)RequestType.Feed);
if (!requestResult.success)
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -207,19 +207,19 @@ public class MangaDex : Connector
return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray();
}
public override bool DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null)
public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null)
{
if (cancellationToken?.IsCancellationRequested??false)
return false;
if (cancellationToken?.IsCancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
//Request URLs for Chapter-Images
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer);
if (!requestResult.success)
return false;
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return requestResult.statusCode;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
if (result is null)
return false;
return HttpStatusCode.NoContent;
string baseUrl = result["baseUrl"]!.GetValue<string>();
string hash = result["chapter"]!["hash"]!.GetValue<string>();
@ -248,7 +248,7 @@ public class MangaDex : Connector
//Request information where to download Cover
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", (byte)RequestType.CoverUrl);
if (!requestResult.success)
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
if (result is null)
@ -268,7 +268,7 @@ public class MangaDex : Connector
{
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", (byte)RequestType.Author);
if (!requestResult.success)
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return ret;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
if (result is null)

16
Tranga/Connectors/Manganato.cs
View File

@ -27,7 +27,7 @@ public class Manganato : Connector
string requestUrl = $"https://manganato.com/search/story/{sanitizedTitle}";
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, (byte)1);
if (!requestResult.success)
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Publication>();
return ParsePublicationsFromHtml(requestResult.result);
@ -52,7 +52,7 @@ public class Manganato : Connector
{
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(url, (byte)1);
if (!requestResult.success)
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Publication>();
ret.Add(ParseSinglePublicationFromHtml(requestResult.result, url.Split('/')[^1]));
@ -131,7 +131,7 @@ public class Manganato : Connector
string requestUrl = $"https://chapmanganato.com/{publication.publicationId}";
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, (byte)1);
if (!requestResult.success)
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Chapter>();
//Return Chapters ordered by Chapter-Number
@ -169,16 +169,16 @@ public class Manganato : Connector
return ret;
}
public override bool DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null)
public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null)
{
if (cancellationToken?.IsCancellationRequested??false)
return false;
if (cancellationToken?.IsCancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
string requestUrl = chapter.url;
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, (byte)1);
if (!requestResult.success)
return false;
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return requestResult.statusCode;
string[] imageUrls = ParseImageUrlsFromHtml(requestResult.result);

13
Tranga/Connectors/Mangasee.cs
View File

@ -80,7 +80,7 @@ public class Mangasee : Connector
string requestUrl = $"https://mangasee123.com/_search.php";
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, (byte)1);
if (!requestResult.success)
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Publication>();
return ParsePublicationsFromHtml(requestResult.result, publicationTitle);
@ -110,7 +110,7 @@ public class Mangasee : Connector
{
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://mangasee123.com/manga/{orderedItem.i}", (byte)1);
if (!requestResult.success)
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Publication>();
ret.Add(ParseSinglePublicationFromHtml(requestResult.result, orderedItem.s, orderedItem.i, orderedItem.a));
}
@ -209,17 +209,17 @@ public class Mangasee : Connector
return ret.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray();
}
public override bool DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null)
public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null)
{
if (cancellationToken?.IsCancellationRequested ?? false)
return false;
return HttpStatusCode.RequestTimeout;
while (this._browser is null && !(cancellationToken?.IsCancellationRequested??false))
{
logger?.WriteLine(this.GetType().ToString(), "Waiting for headless browser to download...");
Thread.Sleep(1000);
}
if (cancellationToken?.IsCancellationRequested??false)
return false;
return HttpStatusCode.RequestTimeout;
logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
IPage page = _browser!.NewPageAsync().Result;
@ -240,7 +240,6 @@ public class Mangasee : Connector
return DownloadChapterImages(urls.ToArray(), GetArchiveFilePath(publication, chapter), (byte)1, parentTask, comicInfoPath, cancellationToken:cancellationToken);
}
return false;
return response.Status;
}
}

25
Tranga/TaskManager.cs
View File

@ -82,18 +82,21 @@ public class TaskManager
}
}
foreach (TrangaTask failedTask in _allTasks.Where(taskQuery =>
taskQuery.state is TrangaTask.ExecutionState.Failed))
TrangaTask[] failedDownloadChapterTasks = _allTasks.Where(taskQuery =>
taskQuery.state is TrangaTask.ExecutionState.Failed && taskQuery is DownloadChapterTask).ToArray();
foreach (TrangaTask failedDownloadChapterTask in failedDownloadChapterTasks)
{
switch (failedTask.task)
{
case TrangaTask.Task.DownloadChapter:
DeleteTask(failedTask);
TrangaTask newTask = failedTask.Clone();
failedTask.parentTask?.AddChildTask(newTask);
AddTask(newTask);
break;
}
DeleteTask(failedDownloadChapterTask);
TrangaTask newTask = failedDownloadChapterTask.Clone();
failedDownloadChapterTask.parentTask?.AddChildTask(newTask);
AddTask(newTask);
}
TrangaTask[] successfulDownloadChapterTasks = _allTasks.Where(taskQuery =>
taskQuery.state is TrangaTask.ExecutionState.Success && taskQuery is DownloadChapterTask).ToArray();
foreach(TrangaTask successfulDownloadChapterTask in successfulDownloadChapterTasks)
{
DeleteTask(successfulDownloadChapterTask);
}
if(waitingTasksCount != _allTasks.Count(task => task.state is TrangaTask.ExecutionState.Waiting))

21
Tranga/TrangaTask.cs
View File

@ -1,4 +1,5 @@
using System.Text.Json.Serialization;
using System.Net;
using System.Text.Json.Serialization;
using Logging;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
@ -32,7 +33,7 @@ public abstract class TrangaTask
[Newtonsoft.Json.JsonIgnore]public TimeSpan executionApproximatelyRemaining => executionApproximatelyFinished.Subtract(DateTime.Now);
[Newtonsoft.Json.JsonIgnore]public DateTime nextExecution => lastExecuted.Add(reoccurrence);
public enum ExecutionState { Waiting, Enqueued, Running, Failed }
public enum ExecutionState { Waiting, Enqueued, Running, Failed, Success }
protected TrangaTask(Task task, TimeSpan reoccurrence, TrangaTask? parentTask = null)
{
@ -53,7 +54,7 @@ public abstract class TrangaTask
/// <param name="taskManager"></param>
/// <param name="logger"></param>
/// <param name="cancellationToken"></param>
protected abstract bool ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null);
protected abstract HttpStatusCode ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null);
public abstract TrangaTask Clone();
@ -73,18 +74,24 @@ public abstract class TrangaTask
this.parentTask.state = ExecutionState.Running;
this.executionStarted = DateTime.Now;
this.lastChange = DateTime.Now;
bool success = ExecuteTask(taskManager, logger, cancellationToken);
HttpStatusCode statusCode = ExecuteTask(taskManager, logger, cancellationToken);
while(childTasks.Any(ct => ct.state is ExecutionState.Enqueued or ExecutionState.Running))
Thread.Sleep(1000);
if (success)
if ((int)statusCode >= 200 && (int)statusCode < 300)
{
this.lastExecuted = DateTime.Now;
this.state = ExecutionState.Waiting;
if (this is DownloadChapterTask)
this.state = ExecutionState.Success;
else
this.state = ExecutionState.Waiting;
}
else
{
if (this is DownloadChapterTask && statusCode == HttpStatusCode.NotFound)
this.state = ExecutionState.Success;
else
this.state = ExecutionState.Failed;
this.lastExecuted = DateTime.MaxValue;
this.state = ExecutionState.Failed;
}
if(this.parentTask is not null)
this.parentTask.state = ExecutionState.Waiting;

14
Tranga/TrangaTasks/DownloadChapterTask.cs
View File

@ -1,4 +1,5 @@
using Logging;
using System.Net;
using Logging;
namespace Tranga.TrangaTasks;
@ -19,15 +20,14 @@ public class DownloadChapterTask : TrangaTask
this.language = language;
}
protected override bool ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null)
protected override HttpStatusCode ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null)
{
if (cancellationToken?.IsCancellationRequested??false)
return false;
if (cancellationToken?.IsCancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
Connector connector = taskManager.GetConnector(this.connectorName);
connector.CopyCoverFromCacheToDownloadLocation(this.publication, taskManager.settings);
bool downloadSuccess = connector.DownloadChapter(this.publication, this.chapter, this, cancellationToken);
taskManager.DeleteTask(this);
if(downloadSuccess && parentTask is not null)
HttpStatusCode downloadSuccess = connector.DownloadChapter(this.publication, this.chapter, this, cancellationToken);
if((int)downloadSuccess >= 200 && (int)downloadSuccess < 300 && parentTask is not null)
foreach(NotificationManager nm in taskManager.settings.notificationManagers)
nm.SendNotification("New Chapter downloaded", $"{this.publication.sortName} {this.chapter.chapterNumber} {this.chapter.name}");
return downloadSuccess;

11
Tranga/TrangaTasks/MonitorPublicationTask.cs
View File

@ -1,4 +1,5 @@
using Logging;
using System.Net;
using Logging;
namespace Tranga.TrangaTasks;
@ -14,10 +15,10 @@ public class MonitorPublicationTask : TrangaTask
this.language = language;
}
protected override bool ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null)
protected override HttpStatusCode ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null)
{
if (cancellationToken?.IsCancellationRequested??false)
return false;
if (cancellationToken?.IsCancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
Connector connector = taskManager.GetConnector(this.connectorName);
//Check if Publication already has a Folder
@ -36,7 +37,7 @@ public class MonitorPublicationTask : TrangaTask
taskManager.AddTask(newTask);
}
return true;
return HttpStatusCode.OK;
}
public override TrangaTask Clone()

11
Tranga/TrangaTasks/UpdateLibrariesTask.cs
View File

@ -1,4 +1,5 @@
using Logging;
using System.Net;
using Logging;
namespace Tranga.TrangaTasks;
@ -8,13 +9,13 @@ public class UpdateLibrariesTask : TrangaTask
{
}
protected override bool ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null)
protected override HttpStatusCode ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null)
{
if (cancellationToken?.IsCancellationRequested??false)
return false;
if (cancellationToken?.IsCancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
foreach(LibraryManager lm in taskManager.settings.libraryManagers)
lm.UpdateLibrary();
return true;
return HttpStatusCode.OK;
}
public override TrangaTask Clone()