Connector DownloadChapter, DownloadImage, DownloadChapterImages returns successState.

RequestResult replace HttpStatusCode with success-status boolean.
DownloadChapterTask: Only send Notification when Chapter download successful
This commit is contained in:
glax 2023-06-19 22:45:33 +02:00
parent edc24fff5b
commit 23dfdc0933
5 changed files with 52 additions and 47 deletions

View File

@ -134,7 +134,7 @@ public abstract class Connector
/// <param name="chapter">Chapter with Images to retrieve</param> /// <param name="chapter">Chapter with Images to retrieve</param>
/// <param name="parentTask">Will be used for progress-tracking</param> /// <param name="parentTask">Will be used for progress-tracking</param>
/// <param name="cancellationToken"></param> /// <param name="cancellationToken"></param>
public abstract void DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null); public abstract bool DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null);
/// <summary> /// <summary>
/// Copies the already downloaded cover from cache to downloadLocation /// Copies the already downloaded cover from cache to downloadLocation
@ -213,16 +213,15 @@ public abstract class Connector
/// <param name="fullPath"></param> /// <param name="fullPath"></param>
/// <param name="requestType">RequestType for Rate-Limit</param> /// <param name="requestType">RequestType for Rate-Limit</param>
/// <param name="referrer">referrer used in html request header</param> /// <param name="referrer">referrer used in html request header</param>
private void DownloadImage(string imageUrl, string fullPath, byte requestType, string? referrer = null) private bool DownloadImage(string imageUrl, string fullPath, byte requestType, string? referrer = null)
{ {
DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType, referrer); DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType, referrer);
if (requestResult.result != Stream.Null) if (!requestResult.success || requestResult.result == Stream.Null)
{ return false;
byte[] buffer = new byte[requestResult.result.Length]; byte[] buffer = new byte[requestResult.result.Length];
requestResult.result.ReadExactly(buffer, 0, buffer.Length); requestResult.result.ReadExactly(buffer, 0, buffer.Length);
File.WriteAllBytes(fullPath, buffer); File.WriteAllBytes(fullPath, buffer);
}else return true;
logger?.WriteLine(this.GetType().ToString(), "No Stream-Content in result.");
} }
/// <summary> /// <summary>
@ -234,10 +233,10 @@ public abstract class Connector
/// <param name="comicInfoPath">Path of the generate Chapter ComicInfo.xml, if it was generated</param> /// <param name="comicInfoPath">Path of the generate Chapter ComicInfo.xml, if it was generated</param>
/// <param name="requestType">RequestType for RateLimits</param> /// <param name="requestType">RequestType for RateLimits</param>
/// <param name="referrer">Used in http request header</param> /// <param name="referrer">Used in http request header</param>
protected void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, DownloadChapterTask parentTask, string? comicInfoPath = null, string? referrer = null, CancellationToken? cancellationToken = null) protected bool DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, DownloadChapterTask parentTask, string? comicInfoPath = null, string? referrer = null, CancellationToken? cancellationToken = null)
{ {
if (cancellationToken?.IsCancellationRequested??false) if (cancellationToken?.IsCancellationRequested??false)
return; return false;
logger?.WriteLine("Connector", $"Downloading Images for {saveArchiveFilePath}"); logger?.WriteLine("Connector", $"Downloading Images for {saveArchiveFilePath}");
//Check if Publication Directory already exists //Check if Publication Directory already exists
string directoryPath = Path.GetDirectoryName(saveArchiveFilePath)!; string directoryPath = Path.GetDirectoryName(saveArchiveFilePath)!;
@ -245,7 +244,7 @@ public abstract class Connector
Directory.CreateDirectory(directoryPath); Directory.CreateDirectory(directoryPath);
if (File.Exists(saveArchiveFilePath)) //Don't download twice. if (File.Exists(saveArchiveFilePath)) //Don't download twice.
return; return false;
//Create a temporary folder to store images //Create a temporary folder to store images
string tempFolder = Directory.CreateTempSubdirectory().FullName; string tempFolder = Directory.CreateTempSubdirectory().FullName;
@ -257,10 +256,11 @@ public abstract class Connector
string[] split = imageUrl.Split('.'); string[] split = imageUrl.Split('.');
string extension = split[^1]; string extension = split[^1];
logger?.WriteLine("Connector", $"Downloading Image {chapter + 1:000}/{imageUrls.Length:000} {parentTask.publication.sortName} {parentTask.publication.internalId} Vol.{parentTask.chapter.volumeNumber} Ch.{parentTask.chapter.chapterNumber} {parentTask.progress:P2}"); logger?.WriteLine("Connector", $"Downloading Image {chapter + 1:000}/{imageUrls.Length:000} {parentTask.publication.sortName} {parentTask.publication.internalId} Vol.{parentTask.chapter.volumeNumber} Ch.{parentTask.chapter.chapterNumber} {parentTask.progress:P2}");
DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer); if (!DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer))
return false;
parentTask.IncrementProgress(1.0 / imageUrls.Length); parentTask.IncrementProgress(1.0 / imageUrls.Length);
if (cancellationToken?.IsCancellationRequested??false) if (cancellationToken?.IsCancellationRequested??false)
return; return false;
} }
if(comicInfoPath is not null) if(comicInfoPath is not null)
@ -272,6 +272,7 @@ public abstract class Connector
if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
File.SetUnixFileMode(saveArchiveFilePath, GroupRead | GroupWrite | OtherRead | OtherWrite | UserRead | UserWrite); File.SetUnixFileMode(saveArchiveFilePath, GroupRead | GroupWrite | OtherRead | OtherWrite | UserRead | UserWrite);
Directory.Delete(tempFolder, true); //Cleanup Directory.Delete(tempFolder, true); //Cleanup
return true;
} }
protected string SaveCoverImageToCache(string url, byte requestType) protected string SaveCoverImageToCache(string url, byte requestType)
@ -327,7 +328,7 @@ public abstract class Connector
else else
{ {
logger?.WriteLine(this.GetType().ToString(), "RequestType not configured for rate-limit."); logger?.WriteLine(this.GetType().ToString(), "RequestType not configured for rate-limit.");
return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null); return new RequestResult(false, Stream.Null);
} }
TimeSpan rateLimitTimeout = _rateLimit[requestType] TimeSpan rateLimitTimeout = _rateLimit[requestType]
@ -354,20 +355,22 @@ public abstract class Connector
Thread.Sleep(_rateLimit[requestType] * 2); Thread.Sleep(_rateLimit[requestType] * 2);
} }
} }
Stream resultString = response.IsSuccessStatusCode ? response.Content.ReadAsStream() : Stream.Null;
if (!response.IsSuccessStatusCode) if (!response.IsSuccessStatusCode)
{
logger?.WriteLine(this.GetType().ToString(), $"Request-Error {response.StatusCode}: {response.ReasonPhrase}"); logger?.WriteLine(this.GetType().ToString(), $"Request-Error {response.StatusCode}: {response.ReasonPhrase}");
return new RequestResult(response.StatusCode, resultString); return new RequestResult(false, Stream.Null);
}
return new RequestResult(true, response.Content.ReadAsStream());
} }
public struct RequestResult public struct RequestResult
{ {
public HttpStatusCode statusCode { get; } public bool success { get; }
public Stream result { get; } public Stream result { get; }
public RequestResult(HttpStatusCode statusCode, Stream result) public RequestResult(bool success, Stream result)
{ {
this.statusCode = statusCode; this.success = success;
this.result = result; this.result = result;
} }
} }

View File

@ -46,7 +46,7 @@ public class MangaDex : Connector
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest( downloadClient.MakeRequest(
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", (byte)RequestType.Manga); $"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", (byte)RequestType.Manga);
if (requestResult.statusCode != HttpStatusCode.OK) if (!requestResult.success)
break; break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result); JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -165,7 +165,7 @@ public class MangaDex : Connector
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest( downloadClient.MakeRequest(
$"https://api.mangadex.org/manga/{publication.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", (byte)RequestType.Feed); $"https://api.mangadex.org/manga/{publication.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", (byte)RequestType.Feed);
if (requestResult.statusCode != HttpStatusCode.OK) if (!requestResult.success)
break; break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result); JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -207,19 +207,19 @@ public class MangaDex : Connector
return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray(); return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray();
} }
public override void DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null) public override bool DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null)
{ {
if (cancellationToken?.IsCancellationRequested??false) if (cancellationToken?.IsCancellationRequested??false)
return; return false;
logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}"); logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
//Request URLs for Chapter-Images //Request URLs for Chapter-Images
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer); downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer);
if (requestResult.statusCode != HttpStatusCode.OK) if (!requestResult.success)
return; return false;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result); JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
if (result is null) if (result is null)
return; return false;
string baseUrl = result["baseUrl"]!.GetValue<string>(); string baseUrl = result["baseUrl"]!.GetValue<string>();
string hash = result["chapter"]!["hash"]!.GetValue<string>(); string hash = result["chapter"]!["hash"]!.GetValue<string>();
@ -233,7 +233,7 @@ public class MangaDex : Connector
File.WriteAllText(comicInfoPath, GetComicInfoXmlString(publication, chapter, logger)); File.WriteAllText(comicInfoPath, GetComicInfoXmlString(publication, chapter, logger));
//Download Chapter-Images //Download Chapter-Images
DownloadChapterImages(imageUrls.ToArray(), GetArchiveFilePath(publication, chapter), (byte)RequestType.AtHomeServer, parentTask, comicInfoPath, cancellationToken:cancellationToken); return DownloadChapterImages(imageUrls.ToArray(), GetArchiveFilePath(publication, chapter), (byte)RequestType.AtHomeServer, parentTask, comicInfoPath, cancellationToken:cancellationToken);
} }
private string? GetCoverUrl(string publicationId, string? posterId) private string? GetCoverUrl(string publicationId, string? posterId)
@ -248,7 +248,7 @@ public class MangaDex : Connector
//Request information where to download Cover //Request information where to download Cover
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", (byte)RequestType.CoverUrl); downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", (byte)RequestType.CoverUrl);
if (requestResult.statusCode != HttpStatusCode.OK) if (!requestResult.success)
return null; return null;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result); JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
if (result is null) if (result is null)
@ -268,7 +268,7 @@ public class MangaDex : Connector
{ {
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", (byte)RequestType.Author); downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", (byte)RequestType.Author);
if (requestResult.statusCode != HttpStatusCode.OK) if (!requestResult.success)
return ret; return ret;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result); JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
if (result is null) if (result is null)

View File

@ -27,7 +27,7 @@ public class Manganato : Connector
string requestUrl = $"https://manganato.com/search/story/{sanitizedTitle}"; string requestUrl = $"https://manganato.com/search/story/{sanitizedTitle}";
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, (byte)1); downloadClient.MakeRequest(requestUrl, (byte)1);
if (requestResult.statusCode != HttpStatusCode.OK) if (!requestResult.success)
return Array.Empty<Publication>(); return Array.Empty<Publication>();
return ParsePublicationsFromHtml(requestResult.result); return ParsePublicationsFromHtml(requestResult.result);
@ -52,7 +52,7 @@ public class Manganato : Connector
{ {
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(url, (byte)1); downloadClient.MakeRequest(url, (byte)1);
if (requestResult.statusCode != HttpStatusCode.OK) if (!requestResult.success)
return Array.Empty<Publication>(); return Array.Empty<Publication>();
ret.Add(ParseSinglePublicationFromHtml(requestResult.result, url.Split('/')[^1])); ret.Add(ParseSinglePublicationFromHtml(requestResult.result, url.Split('/')[^1]));
@ -131,7 +131,7 @@ public class Manganato : Connector
string requestUrl = $"https://chapmanganato.com/{publication.publicationId}"; string requestUrl = $"https://chapmanganato.com/{publication.publicationId}";
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, (byte)1); downloadClient.MakeRequest(requestUrl, (byte)1);
if (requestResult.statusCode != HttpStatusCode.OK) if (!requestResult.success)
return Array.Empty<Chapter>(); return Array.Empty<Chapter>();
//Return Chapters ordered by Chapter-Number //Return Chapters ordered by Chapter-Number
@ -169,23 +169,23 @@ public class Manganato : Connector
return ret; return ret;
} }
public override void DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null) public override bool DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null)
{ {
if (cancellationToken?.IsCancellationRequested??false) if (cancellationToken?.IsCancellationRequested??false)
return; return false;
logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}"); logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
string requestUrl = chapter.url; string requestUrl = chapter.url;
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, (byte)1); downloadClient.MakeRequest(requestUrl, (byte)1);
if (requestResult.statusCode != HttpStatusCode.OK) if (!requestResult.success)
return; return false;
string[] imageUrls = ParseImageUrlsFromHtml(requestResult.result); string[] imageUrls = ParseImageUrlsFromHtml(requestResult.result);
string comicInfoPath = Path.GetTempFileName(); string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, GetComicInfoXmlString(publication, chapter, logger)); File.WriteAllText(comicInfoPath, GetComicInfoXmlString(publication, chapter, logger));
DownloadChapterImages(imageUrls, GetArchiveFilePath(publication, chapter), (byte)1, parentTask, comicInfoPath, "https://chapmanganato.com/", cancellationToken); return DownloadChapterImages(imageUrls, GetArchiveFilePath(publication, chapter), (byte)1, parentTask, comicInfoPath, "https://chapmanganato.com/", cancellationToken);
} }
private string[] ParseImageUrlsFromHtml(Stream html) private string[] ParseImageUrlsFromHtml(Stream html)

View File

@ -80,7 +80,7 @@ public class Mangasee : Connector
string requestUrl = $"https://mangasee123.com/_search.php"; string requestUrl = $"https://mangasee123.com/_search.php";
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, (byte)1); downloadClient.MakeRequest(requestUrl, (byte)1);
if (requestResult.statusCode != HttpStatusCode.OK) if (!requestResult.success)
return Array.Empty<Publication>(); return Array.Empty<Publication>();
return ParsePublicationsFromHtml(requestResult.result, publicationTitle); return ParsePublicationsFromHtml(requestResult.result, publicationTitle);
@ -110,7 +110,7 @@ public class Mangasee : Connector
{ {
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://mangasee123.com/manga/{orderedItem.i}", (byte)1); downloadClient.MakeRequest($"https://mangasee123.com/manga/{orderedItem.i}", (byte)1);
if (requestResult.statusCode != HttpStatusCode.OK) if (!requestResult.success)
return Array.Empty<Publication>(); return Array.Empty<Publication>();
ret.Add(ParseSinglePublicationFromHtml(requestResult.result, orderedItem.s, orderedItem.i, orderedItem.a)); ret.Add(ParseSinglePublicationFromHtml(requestResult.result, orderedItem.s, orderedItem.i, orderedItem.a));
} }
@ -209,17 +209,17 @@ public class Mangasee : Connector
return ret.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray(); return ret.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray();
} }
public override void DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null) public override bool DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null)
{ {
if (cancellationToken?.IsCancellationRequested??false) if (cancellationToken?.IsCancellationRequested ?? false)
return; return false;
while (this._browser is null && !(cancellationToken?.IsCancellationRequested??false)) while (this._browser is null && !(cancellationToken?.IsCancellationRequested??false))
{ {
logger?.WriteLine(this.GetType().ToString(), "Waiting for headless browser to download..."); logger?.WriteLine(this.GetType().ToString(), "Waiting for headless browser to download...");
Thread.Sleep(1000); Thread.Sleep(1000);
} }
if (cancellationToken?.IsCancellationRequested??false) if (cancellationToken?.IsCancellationRequested??false)
return; return false;
logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}"); logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
IPage page = _browser!.NewPageAsync().Result; IPage page = _browser!.NewPageAsync().Result;
@ -238,7 +238,9 @@ public class Mangasee : Connector
string comicInfoPath = Path.GetTempFileName(); string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, GetComicInfoXmlString(publication, chapter, logger)); File.WriteAllText(comicInfoPath, GetComicInfoXmlString(publication, chapter, logger));
DownloadChapterImages(urls.ToArray(), GetArchiveFilePath(publication, chapter), (byte)1, parentTask, comicInfoPath, cancellationToken:cancellationToken); return DownloadChapterImages(urls.ToArray(), GetArchiveFilePath(publication, chapter), (byte)1, parentTask, comicInfoPath, cancellationToken:cancellationToken);
} }
return false;
} }
} }

View File

@ -26,11 +26,11 @@ public class DownloadChapterTask : TrangaTask
this.parentTask.state = ExecutionState.Running; this.parentTask.state = ExecutionState.Running;
Connector connector = taskManager.GetConnector(this.connectorName); Connector connector = taskManager.GetConnector(this.connectorName);
connector.CopyCoverFromCacheToDownloadLocation(this.publication, taskManager.settings); connector.CopyCoverFromCacheToDownloadLocation(this.publication, taskManager.settings);
connector.DownloadChapter(this.publication, this.chapter, this, cancellationToken); bool downloadSuccess = connector.DownloadChapter(this.publication, this.chapter, this, cancellationToken);
if(this.parentTask is not null) if(this.parentTask is not null)
this.parentTask.state = ExecutionState.Waiting; this.parentTask.state = ExecutionState.Waiting;
taskManager.DeleteTask(this); taskManager.DeleteTask(this);
if(parentTask is not null) if(downloadSuccess && parentTask is not null)
foreach(NotificationManager nm in taskManager.settings.notificationManagers) foreach(NotificationManager nm in taskManager.settings.notificationManagers)
nm.SendNotification("New Chapter downloaded", $"{this.publication.sortName} {this.chapter.chapterNumber} {this.chapter.name}"); nm.SendNotification("New Chapter downloaded", $"{this.publication.sortName} {this.chapter.chapterNumber} {this.chapter.name}");
} }