glax/Tranga
1
0
Fork 0
mirror of https://github.com/C9Glax/tranga.git synced 2025-07-07 03:14:17 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added Jobs and ProgressToken

Browse Source
This commit is contained in:
glax
2023-08-04 14:51:40 +02:00
parent e4086a8892
commit a4aa571870
11 changed files with 284 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
Tranga/MangaConnectors/Connector.cs → Tranga/MangaConnectors/MangaConnector.cs
View File

@ -3,6 +3,7 @@ using System.IO.Compression;
using System.Net;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using Tranga.Jobs;
using static System.IO.UnixFileMode;
namespace Tranga.MangaConnectors;
@ -11,11 +12,11 @@ namespace Tranga.MangaConnectors;
/// Base-Class for all Connectors
/// Provides some methods to be used by all Connectors, as well as a DownloadClient
/// </summary>
public abstract class Connector : GlobalBase
public abstract class MangaConnector : GlobalBase
{
internal DownloadClient downloadClient { get; init; } = null!;
protected Connector(GlobalBase clone) : base(clone)
protected MangaConnector(GlobalBase clone) : base(clone)
{
if (!Directory.Exists(settings.coverImageCache))
Directory.CreateDirectory(settings.coverImageCache);
@ -45,13 +46,11 @@ public abstract class Connector : GlobalBase
/// </summary>
/// <param name="publication">Publication to check</param>
/// <param name="language">Language to receive chapters for</param>
/// <param name="collection"></param>
/// <returns>List of Chapters that were previously not in collection</returns>
public List<Chapter> GetNewChaptersList(Publication publication, string language, ref HashSet<Publication> collection)
public Chapter[] GetNewChapters(Publication publication, string language = "en")
{
Log($"Getting new Chapters for {publication}");
Chapter[] newChapters = this.GetChapters(publication, language);
collection.Add(publication);
NumberFormatInfo decimalPoint = new (){ NumberDecimalSeparator = "." };
Log($"Checking for duplicates {publication}");
List<Chapter> newChaptersList = newChapters.Where(nChapter =>
@ -59,7 +58,7 @@ public abstract class Connector : GlobalBase
!nChapter.CheckChapterIsDownloaded(settings.downloadLocation)).ToList();
Log($"{newChaptersList.Count} new chapters. {publication}");
return newChaptersList;
return newChaptersList.ToArray();
}
public Chapter[] SelectChapters(Publication publication, string searchTerm, string? language = null)
@ -135,14 +134,7 @@ public abstract class Connector : GlobalBase
return Array.Empty<Chapter>();
}
/// <summary>
/// Retrieves the Chapter (+Images) from the website.
/// Should later call DownloadChapterImages to retrieve the individual Images of the Chapter and create .cbz archive.
/// </summary>
/// <param name="publication">Publication that contains Chapter</param>
/// <param name="chapter">Chapter with Images to retrieve</param>
/// <param name="cancellationToken"></param>
public abstract HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, CancellationToken? cancellationToken = null);
public abstract HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null);
/// <summary>
/// Copies the already downloaded cover from cache to downloadLocation
@ -186,20 +178,13 @@ public abstract class Connector : GlobalBase
return requestResult.statusCode;
}
/// <summary>
/// Downloads all Images from URLs, Compresses to zip(cbz) and saves.
/// </summary>
/// <param name="imageUrls">List of URLs to download Images from</param>
/// <param name="saveArchiveFilePath">Full path to save archive to (without file ending .cbz)</param>
/// <param name="comicInfoPath">Path of the generate Chapter ComicInfo.xml, if it was generated</param>
/// <param name="requestType">RequestType for RateLimits</param>
/// <param name="referrer">Used in http request header</param>
/// <param name="cancellationToken"></param>
protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, string? comicInfoPath = null, string? referrer = null, CancellationToken? cancellationToken = null)
protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, string? comicInfoPath = null, string? referrer = null, ProgressToken? progressToken = null)
{
if (cancellationToken?.IsCancellationRequested ?? false)
if (progressToken?.cancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
Log($"Downloading Images for {saveArchiveFilePath}");
if(progressToken is not null)
progressToken.increments = imageUrls.Length;
//Check if Publication Directory already exists
string directoryPath = Path.GetDirectoryName(saveArchiveFilePath)!;
if (!Directory.Exists(directoryPath))
@ -220,9 +205,16 @@ public abstract class Connector : GlobalBase
Log($"Downloading image {chapter + 1:000}/{imageUrls.Length:000}"); //TODO
HttpStatusCode status = DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer);
if ((int)status < 200 || (int)status >= 300)
{
progressToken?.Complete();
return status;
if (cancellationToken?.IsCancellationRequested ?? false)
}
if (progressToken?.cancellationRequested ?? false)
{
progressToken?.Complete();
return HttpStatusCode.RequestTimeout;
}
progressToken?.Increment();
}
if(comicInfoPath is not null)
@ -234,6 +226,8 @@ public abstract class Connector : GlobalBase
if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
File.SetUnixFileMode(saveArchiveFilePath, GroupRead | GroupWrite | OtherRead | OtherWrite | UserRead | UserWrite);
Directory.Delete(tempFolder, true); //Cleanup
progressToken?.Complete();
return HttpStatusCode.OK;
}

11
Tranga/MangaConnectors/MangaDex.cs
View File

@ -2,9 +2,10 @@
using System.Net;
using System.Text.Json;
using System.Text.Json.Nodes;
using Tranga.Jobs;
namespace Tranga.MangaConnectors;
public class MangaDex : Connector
public class MangaDex : MangaConnector
{
public override string name { get; }
@ -203,11 +204,11 @@ public class MangaDex : Connector
return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray();
}
public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, CancellationToken? cancellationToken = null)
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (cancellationToken?.IsCancellationRequested ?? false)
if (progressToken?.cancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
Log($"Retrieving chapter-info {chapter} {publication}");
Log($"Retrieving chapter-info {chapter} {chapter.parentPublication}");
//Request URLs for Chapter-Images
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer);
@ -229,7 +230,7 @@ public class MangaDex : Connector
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
//Download Chapter-Images
return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), (byte)RequestType.AtHomeServer, comicInfoPath, cancellationToken:cancellationToken);
return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), (byte)RequestType.AtHomeServer, comicInfoPath, progressToken:progressToken);
}
private string? GetCoverUrl(string publicationId, string? posterId)

11
Tranga/MangaConnectors/MangaKatana.cs
View File

@ -2,10 +2,11 @@
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using Tranga.Jobs;
namespace Tranga.MangaConnectors;
public class MangaKatana : Connector
public class MangaKatana : MangaConnector
{
public override string name { get; }
@ -181,11 +182,11 @@ public class MangaKatana : Connector
return ret;
}
public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, CancellationToken? cancellationToken = null)
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (cancellationToken?.IsCancellationRequested ?? false)
if (progressToken?.cancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
Log($"Retrieving chapter-info {chapter} {publication}");
Log($"Retrieving chapter-info {chapter} {chapter.parentPublication}");
string requestUrl = chapter.url;
// Leaving this in to check if the page exists
DownloadClient.RequestResult requestResult =
@ -198,7 +199,7 @@ public class MangaKatana : Connector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://mangakatana.com/", cancellationToken);
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken);
}
private string[] ParseImageUrlsFromHtml(string mangaUrl)

11
Tranga/MangaConnectors/Manganato.cs
View File

@ -2,10 +2,11 @@
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using Tranga.Jobs;
namespace Tranga.MangaConnectors;
public class Manganato : Connector
public class Manganato : MangaConnector
{
public override string name { get; }
@ -168,11 +169,11 @@ public class Manganato : Connector
return ret;
}
public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, CancellationToken? cancellationToken = null)
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (cancellationToken?.IsCancellationRequested ?? false)
if (progressToken?.cancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
Log($"Retrieving chapter-info {chapter} {publication}");
Log($"Retrieving chapter-info {chapter} {chapter.parentPublication}");
string requestUrl = chapter.url;
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
@ -184,7 +185,7 @@ public class Manganato : Connector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://chapmanganato.com/", cancellationToken);
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://chapmanganato.com/", progressToken:progressToken);
}
private string[] ParseImageUrlsFromHtml(Stream html)

15
Tranga/MangaConnectors/Mangasee.cs
View File

@ -5,10 +5,11 @@ using System.Xml.Linq;
using HtmlAgilityPack;
using Newtonsoft.Json;
using PuppeteerSharp;
using Tranga.Jobs;
namespace Tranga.MangaConnectors;
public class Mangasee : Connector
public class Mangasee : MangaConnector
{
public override string name { get; }
private IBrowser? _browser;
@ -237,19 +238,19 @@ public class Mangasee : Connector
return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray();
}
public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, CancellationToken? cancellationToken = null)
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (cancellationToken?.IsCancellationRequested ?? false)
if (progressToken?.cancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
while (this._browser is null && !(cancellationToken?.IsCancellationRequested??false))
while (this._browser is null && !(progressToken?.cancellationRequested??false))
{
Log("Waiting for headless browser to download...");
Thread.Sleep(1000);
}
if (cancellationToken?.IsCancellationRequested??false)
if (progressToken?.cancellationRequested??false)
return HttpStatusCode.RequestTimeout;
Log($"Retrieving chapter-info {chapter} {publication}");
Log($"Retrieving chapter-info {chapter} {chapter.parentPublication}");
IPage page = _browser!.NewPageAsync().Result;
IResponse response = page.GoToAsync(chapter.url).Result;
if (response.Ok)
@ -266,7 +267,7 @@ public class Mangasee : Connector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, cancellationToken:cancellationToken);
return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, progressToken:progressToken);
}
return response.Status;
}