glax/Tranga
2
0
Fork 0
Code Issues Pull Requests Releases 25 Activity

Added Jobs and ProgressToken

Browse Source
This commit is contained in:
glax 2023-08-04 14:51:40 +02:00
parent e4086a8892
commit a4aa571870
11 changed files with 284 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Tranga/GlobalBase.cs
View File

@ -2,7 +2,7 @@
namespace Tranga;
public class GlobalBase
public abstract class GlobalBase
{
protected Logger? logger { get; init; }
protected TrangaSettings settings { get; init; }

23
Tranga/Jobs/DownloadChapter.cs Normal file
View File

@ -0,0 +1,23 @@
using Tranga.MangaConnectors;
namespace Tranga.Jobs;
public class DownloadChapter : Job
{
public Chapter chapter { get; init; }
public DownloadChapter(MangaConnector connector, Chapter chapter) : base(connector)
{
this.chapter = chapter;
}
protected override IEnumerable<Job> ExecuteReturnSubTasksInternal()
{
Task downloadTask = new(delegate
{
mangaConnector.DownloadChapter(chapter, this.progressToken);
});
downloadTask.Start();
return Array.Empty<Job>();
}
}

27
Tranga/Jobs/DownloadNewChapters.cs Normal file
View File

@ -0,0 +1,27 @@
using Tranga.MangaConnectors;
namespace Tranga.Jobs;
public class DownloadNewChapters : Job
{
public Publication publication { get; init; }
public DownloadNewChapters(MangaConnector connector, Publication publication, bool recurring = false) : base (connector, recurring)
{
this.publication = publication;
}
protected override IEnumerable<Job> ExecuteReturnSubTasksInternal()
{
Chapter[] chapters = mangaConnector.GetNewChapters(publication);
this.progressToken.increments = chapters.Length;
List<Job> subJobs = new();
foreach (Chapter chapter in chapters)
{
DownloadChapter downloadChapterJob = new(this.mangaConnector, chapter);
subJobs.Add(downloadChapterJob);
}
progressToken.Complete();
return subJobs;
}
}

73
Tranga/Jobs/Job.cs Normal file
View File

@ -0,0 +1,73 @@
using Tranga.MangaConnectors;
namespace Tranga.Jobs;
public abstract class Job
{
public MangaConnector mangaConnector { get; init; }
public ProgressToken progressToken { get; private set; }
public bool recurring { get; init; }
public TimeSpan? recurrenceTime { get; set; }
public DateTime? lastExecution { get; private set; }
public DateTime nextExecution => NextExecution();
public Job(MangaConnector connector, bool recurring = false, TimeSpan? recurrenceTime = null)
{
this.mangaConnector = connector;
this.progressToken = new ProgressToken(0);
this.recurring = recurring;
if (recurring && recurrenceTime is null)
throw new ArgumentException("If recurrence is set to true, a recurrence time has to be provided.");
this.recurrenceTime = recurrenceTime;
}
public Job(MangaConnector connector, ProgressToken progressToken, bool recurring = false, TimeSpan? recurrenceTime = null)
{
this.mangaConnector = connector;
this.progressToken = progressToken;
this.recurring = recurring;
if (recurring && recurrenceTime is null)
throw new ArgumentException("If recurrence is set to true, a recurrence time has to be provided.");
this.recurrenceTime = recurrenceTime;
}
public Job(MangaConnector connector, int taskIncrements, bool recurring = false, TimeSpan? recurrenceTime = null)
{
this.mangaConnector = connector;
this.progressToken = new ProgressToken(taskIncrements);
this.recurring = recurring;
if (recurring && recurrenceTime is null)
throw new ArgumentException("If recurrence is set to true, a recurrence time has to be provided.");
this.recurrenceTime = recurrenceTime;
}
private DateTime NextExecution()
{
if(recurring && recurrenceTime.HasValue && lastExecution.HasValue)
return lastExecution.Value.Add(recurrenceTime.Value);
if(recurring && recurrenceTime.HasValue && !lastExecution.HasValue)
return DateTime.Now;
return DateTime.MaxValue;
}
public void Reset()
{
this.progressToken = new ProgressToken(this.progressToken.increments);
}
public void Cancel()
{
this.progressToken.cancellationRequested = true;
this.progressToken.Complete();
}
public IEnumerable<Job> ExecuteReturnSubTasks()
{
progressToken.Start();
IEnumerable<Job> ret = ExecuteReturnSubTasksInternal();
lastExecution = DateTime.Now;
return ret;
}
protected abstract IEnumerable<Job> ExecuteReturnSubTasksInternal();
}

70
Tranga/Jobs/JobBoss.cs Normal file
View File

@ -0,0 +1,70 @@
using System.Runtime.CompilerServices;
using Tranga.MangaConnectors;
namespace Tranga.Jobs;
public class JobBoss : GlobalBase
{
private HashSet<Job> jobs { get; init; }
private Dictionary<MangaConnector, Queue<Job>> mangaConnectorJobQueue { get; init; }
public JobBoss(GlobalBase clone) : base(clone)
{
this.jobs = new();
this.mangaConnectorJobQueue = new();
}
public void MonitorJobs()
{
foreach (Job job in jobs.Where(job => job.nextExecution < DateTime.Now && !QueueContainsJob(job)).OrderBy(job => job.nextExecution))
AddJobToQueue(job);
CheckJobQueue();
}
public void AddJob(Job job)
{
this.jobs.Add(job);
}
public void RemoveJob(Job job)
{
job.Cancel();
this.jobs.Remove(job);
}
private bool QueueContainsJob(Job job)
{
mangaConnectorJobQueue.TryAdd(job.mangaConnector, new Queue<Job>());
return mangaConnectorJobQueue[job.mangaConnector].Contains(job);
}
private void AddJobToQueue(Job job)
{
Log($"Adding Job to Queue. {job}");
mangaConnectorJobQueue.TryAdd(job.mangaConnector, new Queue<Job>());
Queue<Job> connectorJobQueue = mangaConnectorJobQueue[job.mangaConnector];
if(!connectorJobQueue.Contains(job))
connectorJobQueue.Enqueue(job);
}
public void AddJobsToQueue(IEnumerable<Job> jobs)
{
foreach(Job job in jobs)
AddJobToQueue(job);
}
private void CheckJobQueue()
{
foreach (Queue<Job> jobQueue in mangaConnectorJobQueue.Values)
{
Job queueHead = jobQueue.Peek();
if (queueHead.progressToken.state == ProgressToken.State.Complete)
{
if(queueHead.recurring)
queueHead.Reset();
jobQueue.Dequeue();
AddJobsToQueue(jobQueue.Peek().ExecuteReturnSubTasks());
}
}
}
}

44
Tranga/Jobs/ProgressToken.cs Normal file
View File

@ -0,0 +1,44 @@
namespace Tranga.Jobs;
public class ProgressToken
{
public bool cancellationRequested { get; set; }
public int increments { get; set; }
public int incrementsCompleted { get; set; }
public float progress => GetProgress();
public enum State { Running, Complete, Standby }
public State state { get; private set; }
public ProgressToken(int increments)
{
this.cancellationRequested = false;
this.increments = increments;
this.incrementsCompleted = 0;
this.state = State.Standby;
}
private float GetProgress()
{
if(increments > 0 && incrementsCompleted > 0)
return (float)incrementsCompleted / (float)increments;
return 0;
}
public void Increment()
{
this.incrementsCompleted++;
if (incrementsCompleted > increments)
state = State.Complete;
}
public void Start()
{
state = State.Running;
}
public void Complete()
{
state = State.Complete;
}
}

46
Tranga/MangaConnectors/Connector.cs → Tranga/MangaConnectors/MangaConnector.cs
View File

@ -3,6 +3,7 @@ using System.IO.Compression;
using System.Net;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using Tranga.Jobs;
using static System.IO.UnixFileMode;
namespace Tranga.MangaConnectors;
@ -11,11 +12,11 @@ namespace Tranga.MangaConnectors;
/// Base-Class for all Connectors
/// Provides some methods to be used by all Connectors, as well as a DownloadClient
/// </summary>
public abstract class Connector : GlobalBase
public abstract class MangaConnector : GlobalBase
{
internal DownloadClient downloadClient { get; init; } = null!;
protected Connector(GlobalBase clone) : base(clone)
protected MangaConnector(GlobalBase clone) : base(clone)
{
if (!Directory.Exists(settings.coverImageCache))
Directory.CreateDirectory(settings.coverImageCache);
@ -45,13 +46,11 @@ public abstract class Connector : GlobalBase
/// </summary>
/// <param name="publication">Publication to check</param>
/// <param name="language">Language to receive chapters for</param>
/// <param name="collection"></param>
/// <returns>List of Chapters that were previously not in collection</returns>
public List<Chapter> GetNewChaptersList(Publication publication, string language, ref HashSet<Publication> collection)
public Chapter[] GetNewChapters(Publication publication, string language = "en")
{
Log($"Getting new Chapters for {publication}");
Chapter[] newChapters = this.GetChapters(publication, language);
collection.Add(publication);
NumberFormatInfo decimalPoint = new (){ NumberDecimalSeparator = "." };
Log($"Checking for duplicates {publication}");
List<Chapter> newChaptersList = newChapters.Where(nChapter =>
@ -59,7 +58,7 @@ public abstract class Connector : GlobalBase
!nChapter.CheckChapterIsDownloaded(settings.downloadLocation)).ToList();
Log($"{newChaptersList.Count} new chapters. {publication}");
return newChaptersList;
return newChaptersList.ToArray();
}
public Chapter[] SelectChapters(Publication publication, string searchTerm, string? language = null)
@ -135,14 +134,7 @@ public abstract class Connector : GlobalBase
return Array.Empty<Chapter>();
}
/// <summary>
/// Retrieves the Chapter (+Images) from the website.
/// Should later call DownloadChapterImages to retrieve the individual Images of the Chapter and create .cbz archive.
/// </summary>
/// <param name="publication">Publication that contains Chapter</param>
/// <param name="chapter">Chapter with Images to retrieve</param>
/// <param name="cancellationToken"></param>
public abstract HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, CancellationToken? cancellationToken = null);
public abstract HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null);
/// <summary>
/// Copies the already downloaded cover from cache to downloadLocation
@ -186,20 +178,13 @@ public abstract class Connector : GlobalBase
return requestResult.statusCode;
}
/// <summary>
/// Downloads all Images from URLs, Compresses to zip(cbz) and saves.
/// </summary>
/// <param name="imageUrls">List of URLs to download Images from</param>
/// <param name="saveArchiveFilePath">Full path to save archive to (without file ending .cbz)</param>
/// <param name="comicInfoPath">Path of the generate Chapter ComicInfo.xml, if it was generated</param>
/// <param name="requestType">RequestType for RateLimits</param>
/// <param name="referrer">Used in http request header</param>
/// <param name="cancellationToken"></param>
protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, string? comicInfoPath = null, string? referrer = null, CancellationToken? cancellationToken = null)
protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, string? comicInfoPath = null, string? referrer = null, ProgressToken? progressToken = null)
{
if (cancellationToken?.IsCancellationRequested ?? false)
if (progressToken?.cancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
Log($"Downloading Images for {saveArchiveFilePath}");
if(progressToken is not null)
progressToken.increments = imageUrls.Length;
//Check if Publication Directory already exists
string directoryPath = Path.GetDirectoryName(saveArchiveFilePath)!;
if (!Directory.Exists(directoryPath))
@ -220,9 +205,16 @@ public abstract class Connector : GlobalBase
Log($"Downloading image {chapter + 1:000}/{imageUrls.Length:000}"); //TODO
HttpStatusCode status = DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer);
if ((int)status < 200 || (int)status >= 300)
{
progressToken?.Complete();
return status;
if (cancellationToken?.IsCancellationRequested ?? false)
}
if (progressToken?.cancellationRequested ?? false)
{
progressToken?.Complete();
return HttpStatusCode.RequestTimeout;
}
progressToken?.Increment();
}
if(comicInfoPath is not null)
@ -234,6 +226,8 @@ public abstract class Connector : GlobalBase
if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
File.SetUnixFileMode(saveArchiveFilePath, GroupRead | GroupWrite | OtherRead | OtherWrite | UserRead | UserWrite);
Directory.Delete(tempFolder, true); //Cleanup
progressToken?.Complete();
return HttpStatusCode.OK;
}

11
Tranga/MangaConnectors/MangaDex.cs
View File

@ -2,9 +2,10 @@
using System.Net;
using System.Text.Json;
using System.Text.Json.Nodes;
using Tranga.Jobs;
namespace Tranga.MangaConnectors;
public class MangaDex : Connector
public class MangaDex : MangaConnector
{
public override string name { get; }
@ -203,11 +204,11 @@ public class MangaDex : Connector
return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray();
}
public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, CancellationToken? cancellationToken = null)
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (cancellationToken?.IsCancellationRequested ?? false)
if (progressToken?.cancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
Log($"Retrieving chapter-info {chapter} {publication}");
Log($"Retrieving chapter-info {chapter} {chapter.parentPublication}");
//Request URLs for Chapter-Images
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer);
@ -229,7 +230,7 @@ public class MangaDex : Connector
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
//Download Chapter-Images
return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), (byte)RequestType.AtHomeServer, comicInfoPath, cancellationToken:cancellationToken);
return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), (byte)RequestType.AtHomeServer, comicInfoPath, progressToken:progressToken);
}
private string? GetCoverUrl(string publicationId, string? posterId)

11
Tranga/MangaConnectors/MangaKatana.cs
View File

@ -2,10 +2,11 @@
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using Tranga.Jobs;
namespace Tranga.MangaConnectors;
public class MangaKatana : Connector
public class MangaKatana : MangaConnector
{
public override string name { get; }
@ -181,11 +182,11 @@ public class MangaKatana : Connector
return ret;
}
public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, CancellationToken? cancellationToken = null)
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (cancellationToken?.IsCancellationRequested ?? false)
if (progressToken?.cancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
Log($"Retrieving chapter-info {chapter} {publication}");
Log($"Retrieving chapter-info {chapter} {chapter.parentPublication}");
string requestUrl = chapter.url;
// Leaving this in to check if the page exists
DownloadClient.RequestResult requestResult =
@ -198,7 +199,7 @@ public class MangaKatana : Connector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://mangakatana.com/", cancellationToken);
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken);
}
private string[] ParseImageUrlsFromHtml(string mangaUrl)

11
Tranga/MangaConnectors/Manganato.cs
View File

@ -2,10 +2,11 @@
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using Tranga.Jobs;
namespace Tranga.MangaConnectors;
public class Manganato : Connector
public class Manganato : MangaConnector
{
public override string name { get; }
@ -168,11 +169,11 @@ public class Manganato : Connector
return ret;
}
public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, CancellationToken? cancellationToken = null)
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (cancellationToken?.IsCancellationRequested ?? false)
if (progressToken?.cancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
Log($"Retrieving chapter-info {chapter} {publication}");
Log($"Retrieving chapter-info {chapter} {chapter.parentPublication}");
string requestUrl = chapter.url;
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
@ -184,7 +185,7 @@ public class Manganato : Connector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://chapmanganato.com/", cancellationToken);
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://chapmanganato.com/", progressToken:progressToken);
}
private string[] ParseImageUrlsFromHtml(Stream html)

15
Tranga/MangaConnectors/Mangasee.cs
View File

@ -5,10 +5,11 @@ using System.Xml.Linq;
using HtmlAgilityPack;
using Newtonsoft.Json;
using PuppeteerSharp;
using Tranga.Jobs;
namespace Tranga.MangaConnectors;
public class Mangasee : Connector
public class Mangasee : MangaConnector
{
public override string name { get; }
private IBrowser? _browser;
@ -237,19 +238,19 @@ public class Mangasee : Connector
return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray();
}
public override HttpStatusCode DownloadChapter(Publication publication, Chapter chapter, CancellationToken? cancellationToken = null)
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (cancellationToken?.IsCancellationRequested ?? false)
if (progressToken?.cancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
while (this._browser is null && !(cancellationToken?.IsCancellationRequested??false))
while (this._browser is null && !(progressToken?.cancellationRequested??false))
{
Log("Waiting for headless browser to download...");
Thread.Sleep(1000);
}
if (cancellationToken?.IsCancellationRequested??false)
if (progressToken?.cancellationRequested??false)
return HttpStatusCode.RequestTimeout;
Log($"Retrieving chapter-info {chapter} {publication}");
Log($"Retrieving chapter-info {chapter} {chapter.parentPublication}");
IPage page = _browser!.NewPageAsync().Result;
IResponse response = page.GoToAsync(chapter.url).Result;
if (response.Ok)
@ -266,7 +267,7 @@ public class Mangasee : Connector
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, cancellationToken:cancellationToken);
return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, progressToken:progressToken);
}
return response.Status;
}