From 61df23ea291656774115495e9e115269cf803777 Mon Sep 17 00:00:00 2001 From: Glax Date: Wed, 28 Feb 2024 00:44:42 +0100 Subject: [PATCH] Jobqueue changes untested --- CLI/CLI.csproj | 1 + CLI/Program.cs | 48 +--- Tranga.sln | 6 - Tranga.sln.DotSettings.user | 2 + Tranga/GlobalBase.cs | 39 +-- Tranga/Jobs/DownloadChapter.cs | 53 ++-- Tranga/Jobs/DownloadNewChapters.cs | 60 ++--- Tranga/Jobs/Job.cs | 93 +------ Tranga/Jobs/JobBoss.cs | 258 ------------------- Tranga/Jobs/JobJsonConverter.cs | 97 +++---- Tranga/Jobs/ProgressToken.cs | 78 ------ Tranga/Jobs/UpdateMetadata.cs | 49 ++-- Tranga/LibraryConnectors/LibraryConnector.cs | 13 +- Tranga/MangaConnectors/Bato.cs | 5 +- Tranga/MangaConnectors/MangaConnector.cs | 15 +- Tranga/MangaConnectors/MangaDex.cs | 6 +- Tranga/MangaConnectors/MangaKatana.cs | 5 +- Tranga/MangaConnectors/MangaLife.cs | 9 +- Tranga/MangaConnectors/Manganato.cs | 5 +- Tranga/MangaConnectors/Mangasee.cs | 9 +- Tranga/MangaConnectors/Mangaworld.cs | 5 +- Tranga/Server.cs | 117 ++++----- Tranga/Tranga.cs | 23 +- Tranga/Tranga.csproj | 11 +- Tranga/TrangaArgs.cs | 22 +- Tranga/TrangaSettings.cs | 1 + 26 files changed, 259 insertions(+), 771 deletions(-) create mode 100644 Tranga.sln.DotSettings.user delete mode 100644 Tranga/Jobs/JobBoss.cs delete mode 100644 Tranga/Jobs/ProgressToken.cs diff --git a/CLI/CLI.csproj b/CLI/CLI.csproj index abf151f..b7368e8 100644 --- a/CLI/CLI.csproj +++ b/CLI/CLI.csproj @@ -8,6 +8,7 @@ + diff --git a/CLI/Program.cs b/CLI/Program.cs index 9543773..403dabb 100644 --- a/CLI/Program.cs +++ b/CLI/Program.cs @@ -1,6 +1,7 @@ using System.ComponentModel; using System.Diagnostics.CodeAnalysis; -using Logging; +using GlaxLogger; +using Microsoft.Extensions.Logging; using Spectre.Console; using Spectre.Console.Cli; using Tranga; @@ -22,16 +23,16 @@ internal sealed class TrangaCli : Command [DefaultValue(null)] public string? workingDirectory { get; init; } - [Description("Enables the file-logger")] - [CommandOption("-f")] - [DefaultValue(null)] - public bool? fileLogger { get; init; } - [Description("Path to save logfile to")] - [CommandOption("-l|--fPath")] + [CommandOption("-f|--fileLogger")] [DefaultValue(null)] public string? fileLoggerPath { get; init; } + [Description("LogLevel")] + [CommandOption("-l|--loglevel")] + [DefaultValue(LogLevel.Information)] + public LogLevel level { get; init; } + [Description("Port on which to run API on")] [CommandOption("-p|--port")] [DefaultValue(null)] @@ -40,12 +41,7 @@ internal sealed class TrangaCli : Command public override int Execute([NotNull] CommandContext context, [NotNull] Settings settings) { - List enabledLoggers = new(); - if(settings.fileLogger is true) - enabledLoggers.Add(Logger.LoggerType.FileLogger); - - string? logFilePath = settings.fileLoggerPath ?? ""; - Logger logger = new(enabledLoggers.ToArray(), Console.Out, Console.OutputEncoding, logFilePath); + Logger logger = new (settings.level, settings.fileLoggerPath, Console.Out); TrangaSettings trangaSettings = new (settings.downloadLocation, settings.workingDirectory, settings.apiPort); @@ -73,7 +69,6 @@ internal sealed class TrangaCli : Command .AddChoices(new[] { "CustomRequest", - "Log", "Exit" })); @@ -116,31 +111,6 @@ internal sealed class TrangaCli : Command AnsiConsole.WriteLine($"Response: {(int)response.StatusCode} {response.StatusCode}"); AnsiConsole.WriteLine(response.Content.ReadAsStringAsync().Result); break; - case "Log": - List lines = logger.Tail(10).ToList(); - Rows rows = new Rows(lines.Select(line => new Text(line))); - - AnsiConsole.Live(rows).Start(context => - { - bool running = true; - while (running) - { - string[] newLines = logger.GetNewLines(); - if (newLines.Length > 0) - { - lines.AddRange(newLines); - rows = new Rows(lines.Select(line => new Text(line))); - context.UpdateTarget(rows); - } - Thread.Sleep(100); - if (AnsiConsole.Console.Input.IsKeyAvailable()) - { - AnsiConsole.Console.Input.ReadKey(true); //Do not process input - running = false; - } - } - }); - break; case "Exit": exit = true; break; diff --git a/Tranga.sln b/Tranga.sln index 501c3c3..98cd9d2 100644 --- a/Tranga.sln +++ b/Tranga.sln @@ -2,8 +2,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tranga", ".\Tranga\Tranga.csproj", "{545E81B9-D96B-4C8F-A97F-2C02414DE566}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Logging", "Logging\Logging.csproj", "{415BE889-BB7D-426F-976F-8D977876A462}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CLI", "CLI\CLI.csproj", "{4324C816-F9D2-468F-8ED6-397FE2F0DCB3}" EndProject Global @@ -16,10 +14,6 @@ Global {545E81B9-D96B-4C8F-A97F-2C02414DE566}.Debug|Any CPU.Build.0 = Debug|Any CPU {545E81B9-D96B-4C8F-A97F-2C02414DE566}.Release|Any CPU.ActiveCfg = Release|Any CPU {545E81B9-D96B-4C8F-A97F-2C02414DE566}.Release|Any CPU.Build.0 = Release|Any CPU - {415BE889-BB7D-426F-976F-8D977876A462}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {415BE889-BB7D-426F-976F-8D977876A462}.Debug|Any CPU.Build.0 = Debug|Any CPU - {415BE889-BB7D-426F-976F-8D977876A462}.Release|Any CPU.ActiveCfg = Release|Any CPU - {415BE889-BB7D-426F-976F-8D977876A462}.Release|Any CPU.Build.0 = Release|Any CPU {4324C816-F9D2-468F-8ED6-397FE2F0DCB3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {4324C816-F9D2-468F-8ED6-397FE2F0DCB3}.Debug|Any CPU.Build.0 = Debug|Any CPU {4324C816-F9D2-468F-8ED6-397FE2F0DCB3}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/Tranga.sln.DotSettings.user b/Tranga.sln.DotSettings.user new file mode 100644 index 0000000..4a9c6e9 --- /dev/null +++ b/Tranga.sln.DotSettings.user @@ -0,0 +1,2 @@ + + True \ No newline at end of file diff --git a/Tranga/GlobalBase.cs b/Tranga/GlobalBase.cs index e9c68d0..6497b40 100644 --- a/Tranga/GlobalBase.cs +++ b/Tranga/GlobalBase.cs @@ -1,6 +1,7 @@ using System.Globalization; using System.Text.RegularExpressions; -using Logging; +using GlaxLogger; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Tranga.LibraryConnectors; using Tranga.NotificationConnectors; @@ -9,13 +10,13 @@ namespace Tranga; public abstract class GlobalBase { - protected Logger? logger { get; init; } - protected TrangaSettings settings { get; init; } - protected HashSet notificationConnectors { get; init; } - protected HashSet libraryConnectors { get; init; } - protected List cachedPublications { get; init; } - public static readonly NumberFormatInfo numberFormatDecimalPoint = new (){ NumberDecimalSeparator = "." }; - protected static readonly Regex baseUrlRex = new(@"https?:\/\/[0-9A-z\.-]+(:[0-9]+)?"); + internal Logger? logger { get; init; } + internal TrangaSettings settings { get; init; } + internal HashSet notificationConnectors { get; init; } + internal HashSet libraryConnectors { get; init; } + internal List cachedPublications { get; init; } + internal static readonly NumberFormatInfo numberFormatDecimalPoint = new (){ NumberDecimalSeparator = "." }; + internal static readonly Regex baseUrlRex = new(@"https?:\/\/[0-9A-z\.-]+(:[0-9]+)?"); protected GlobalBase(GlobalBase clone) { @@ -35,23 +36,23 @@ public abstract class GlobalBase this.cachedPublications = new(); } - protected void Log(string message) + internal void Log(string message) { - logger?.WriteLine(this.GetType().Name, message); + Log(this.GetType().Name, message); } - protected void Log(string fStr, params object?[] replace) + internal void Log(string objectType, string message) { - Log(string.Format(fStr, replace)); + logger?.LogInformation($"{objectType} | {message}"); } - protected void SendNotifications(string title, string text) + internal void SendNotifications(string title, string text) { foreach (NotificationConnector nc in notificationConnectors) nc.SendNotification(title, text); } - protected void AddNotificationConnector(NotificationConnector notificationConnector) + internal void AddNotificationConnector(NotificationConnector notificationConnector) { Log($"Adding {notificationConnector}"); notificationConnectors.RemoveWhere(nc => nc.notificationConnectorType == notificationConnector.notificationConnectorType); @@ -63,7 +64,7 @@ public abstract class GlobalBase File.WriteAllText(settings.notificationConnectorsFilePath, JsonConvert.SerializeObject(notificationConnectors)); } - protected void DeleteNotificationConnector(NotificationConnector.NotificationConnectorType notificationConnectorType) + internal void DeleteNotificationConnector(NotificationConnector.NotificationConnectorType notificationConnectorType) { Log($"Removing {notificationConnectorType}"); notificationConnectors.RemoveWhere(nc => nc.notificationConnectorType == notificationConnectorType); @@ -73,13 +74,13 @@ public abstract class GlobalBase File.WriteAllText(settings.notificationConnectorsFilePath, JsonConvert.SerializeObject(notificationConnectors)); } - protected void UpdateLibraries() + internal void UpdateLibraries() { foreach(LibraryConnector lc in libraryConnectors) lc.UpdateLibrary(); } - protected void AddLibraryConnector(LibraryConnector libraryConnector) + internal void AddLibraryConnector(LibraryConnector libraryConnector) { Log($"Adding {libraryConnector}"); libraryConnectors.RemoveWhere(lc => lc.libraryType == libraryConnector.libraryType); @@ -91,7 +92,7 @@ public abstract class GlobalBase File.WriteAllText(settings.libraryConnectorsFilePath, JsonConvert.SerializeObject(libraryConnectors)); } - protected void DeleteLibraryConnector(LibraryConnector.LibraryType libraryType) + internal void DeleteLibraryConnector(LibraryConnector.LibraryType libraryType) { Log($"Removing {libraryType}"); libraryConnectors.RemoveWhere(lc => lc.libraryType == libraryType); @@ -101,7 +102,7 @@ public abstract class GlobalBase File.WriteAllText(settings.libraryConnectorsFilePath, JsonConvert.SerializeObject(libraryConnectors)); } - protected bool IsFileInUse(string filePath) + internal bool IsFileInUse(string filePath) { if (!File.Exists(filePath)) return false; diff --git a/Tranga/Jobs/DownloadChapter.cs b/Tranga/Jobs/DownloadChapter.cs index 434736e..3d1261c 100644 --- a/Tranga/Jobs/DownloadChapter.cs +++ b/Tranga/Jobs/DownloadChapter.cs @@ -1,4 +1,6 @@ using System.Net; +using JobQueue; +using Microsoft.Extensions.Logging; using Tranga.MangaConnectors; namespace Tranga.Jobs; @@ -6,42 +8,27 @@ namespace Tranga.Jobs; public class DownloadChapter : Job { public Chapter chapter { get; init; } - - public DownloadChapter(GlobalBase clone, MangaConnector connector, Chapter chapter, DateTime lastExecution, string? parentJobId = null) : base(clone, JobType.DownloadChapterJob, connector, lastExecution, parentJobId: parentJobId) + + public DownloadChapter(GlobalBase clone, JobQueue queue, MangaConnector connector, Chapter chapter, int steps, string? jobId = null, string? parentJobId = null, ILogger? logger = null) : base (clone, queue, connector, JobType.DownloadChapterJob, TimeSpan.Zero, TimeSpan.FromSeconds(clone.settings.jobTimeout), steps, jobId, parentJobId, logger) { this.chapter = chapter; - } - - public DownloadChapter(GlobalBase clone, MangaConnector connector, Chapter chapter, string? parentJobId = null) : base(clone, JobType.DownloadChapterJob, connector, parentJobId: parentJobId) - { - this.chapter = chapter; - } - - protected override string GetId() - { - return $"{GetType()}-{chapter.parentManga.internalId}-{chapter.chapterNumber}"; - } - - public override string ToString() - { - return $"{id} Chapter: {chapter}"; - } - - protected override IEnumerable ExecuteReturnSubTasksInternal(JobBoss jobBoss) - { - Task downloadTask = new(delegate + if (jobId is null) { - mangaConnector.CopyCoverFromCacheToDownloadLocation(chapter.parentManga); - HttpStatusCode success = mangaConnector.DownloadChapter(chapter, this.progressToken); - chapter.parentManga.UpdateLatestDownloadedChapter(chapter); - if (success == HttpStatusCode.OK) - { - UpdateLibraries(); - SendNotifications("Chapter downloaded", $"{chapter.parentManga.sortName} - {chapter.chapterNumber}"); - } - }); - downloadTask.Start(); - return Array.Empty(); + const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + this.JobId = $"{this.GetType().Name}-{connector.name}-{chapter}-{new string(Enumerable.Repeat(chars, 4).Select(s => s[Random.Shared.Next(s.Length)]).ToArray())}"; + } + } + + protected override void Execute(CancellationToken cancellationToken) + { + mangaConnector.CopyCoverFromCacheToDownloadLocation(chapter.parentManga); + HttpStatusCode success = mangaConnector.DownloadChapter(chapter, this.ProgressToken); + chapter.parentManga.UpdateLatestDownloadedChapter(chapter); + if (success == HttpStatusCode.OK) + { + this.GlobalBase.UpdateLibraries(); + this.GlobalBase.SendNotifications("Chapter downloaded", $"{chapter.parentManga.sortName} - {chapter.chapterNumber}"); + } } public override bool Equals(object? obj) diff --git a/Tranga/Jobs/DownloadNewChapters.cs b/Tranga/Jobs/DownloadNewChapters.cs index ebcbde7..ec2b2a1 100644 --- a/Tranga/Jobs/DownloadNewChapters.cs +++ b/Tranga/Jobs/DownloadNewChapters.cs @@ -1,59 +1,39 @@ -using Tranga.MangaConnectors; +using JobQueue; +using Microsoft.Extensions.Logging; +using Tranga.MangaConnectors; namespace Tranga.Jobs; -public class DownloadNewChapters : Job +public class DownloadNewChapter : Job { public Manga manga { get; set; } public string translatedLanguage { get; init; } - - public DownloadNewChapters(GlobalBase clone, MangaConnector connector, Manga manga, DateTime lastExecution, - bool recurring = false, TimeSpan? recurrence = null, string? parentJobId = null, string translatedLanguage = "en") : base(clone, JobType.DownloadNewChaptersJob, connector, lastExecution, recurring, - recurrence, parentJobId) - { - this.manga = manga; - this.translatedLanguage = translatedLanguage; - } - - public DownloadNewChapters(GlobalBase clone, MangaConnector connector, Manga manga, bool recurring = false, TimeSpan? recurrence = null, string? parentJobId = null, string translatedLanguage = "en") : base (clone, JobType.DownloadNewChaptersJob, connector, recurring, recurrence, parentJobId) + + public DownloadNewChapter(GlobalBase clone, JobQueue queue, MangaConnector connector, Manga manga, TimeSpan interval, int steps, string? jobId = null, string? parentJobId = null, ILogger? logger = null, string translatedLanguage = "en") : base (clone, queue, connector, JobType.DownloadNewChaptersJob, interval, TimeSpan.FromSeconds(clone.settings.jobTimeout), steps, jobId, parentJobId, logger) { this.manga = manga; this.translatedLanguage = translatedLanguage; + if (jobId is null) + { + const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + this.JobId = $"{this.GetType().Name}-{connector.name}-{manga.sortName}-{new string(Enumerable.Repeat(chars, 4).Select(s => s[Random.Shared.Next(s.Length)]).ToArray())}"; + } } - protected override string GetId() + protected override void Execute(CancellationToken cancellationToken) { - return $"{GetType()}-{manga.internalId}"; - } - - public override string ToString() - { - return $"{id} Manga: {manga}"; - } - - protected override IEnumerable ExecuteReturnSubTasksInternal(JobBoss jobBoss) - { - manga.SaveSeriesInfoJson(settings.downloadLocation); + manga.SaveSeriesInfoJson(GlobalBase.settings.downloadLocation); Chapter[] chapters = mangaConnector.GetNewChapters(manga, this.translatedLanguage); - this.progressToken.increments = chapters.Length; - List jobs = new(); + this.ProgressToken.SetSteps(chapters.Length); mangaConnector.CopyCoverFromCacheToDownloadLocation(manga); foreach (Chapter chapter in chapters) { - DownloadChapter downloadChapterJob = new(this, this.mangaConnector, chapter, parentJobId: this.id); - jobs.Add(downloadChapterJob); + DownloadChapter downloadChapterJob = new(this.GlobalBase, Queue, mangaConnector, chapter, 0, null, + this.JobId, this.logger); + Queue.AddJob(mangaConnector, downloadChapterJob); } - UpdateMetadata updateMetadataJob = new(this, this.mangaConnector, this.manga, parentJobId: this.id); - jobs.Add(updateMetadataJob); - progressToken.Complete(); - return jobs; - } - - public override bool Equals(object? obj) - { - if (obj is not DownloadNewChapters otherJob) - return false; - return otherJob.mangaConnector == this.mangaConnector && - otherJob.manga.Equals(this.manga); + UpdateMetadata updateMetadataJob = new(this.GlobalBase, Queue, mangaConnector, manga, null, this.JobId, this.logger); + Queue.AddJob(mangaConnector, updateMetadataJob); + this.ProgressToken.MarkFinished(); } } \ No newline at end of file diff --git a/Tranga/Jobs/Job.cs b/Tranga/Jobs/Job.cs index deefd77..56dde22 100644 --- a/Tranga/Jobs/Job.cs +++ b/Tranga/Jobs/Job.cs @@ -1,98 +1,23 @@ -using Tranga.MangaConnectors; +using JobQueue; +using Tranga.MangaConnectors; +using Microsoft.Extensions.Logging; namespace Tranga.Jobs; -public abstract class Job : GlobalBase +public abstract class Job : Job { + protected readonly GlobalBase GlobalBase; public MangaConnector mangaConnector { get; init; } - public ProgressToken progressToken { get; private set; } - public bool recurring { get; init; } - public TimeSpan? recurrenceTime { get; set; } - public DateTime? lastExecution { get; private set; } - public DateTime nextExecution => NextExecution(); - public string id => GetId(); - internal IEnumerable? subJobs { get; private set; } - public string? parentJobId { get; init; } public enum JobType : byte { DownloadChapterJob, DownloadNewChaptersJob, UpdateMetaDataJob } public JobType jobType; + protected ILogger? logger; - internal Job(GlobalBase clone, JobType jobType, MangaConnector connector, bool recurring = false, TimeSpan? recurrenceTime = null, string? parentJobId = null) : base(clone) + public Job(GlobalBase clone, JobQueue queue, MangaConnector connector, JobType jobType, TimeSpan interval, TimeSpan maximumTimeBetweenUpdates, int steps, string? jobId = null, string? parentJobId = null, ILogger? logger = null) : base(queue, interval, maximumTimeBetweenUpdates, steps, jobId, parentJobId, logger) { - this.jobType = jobType; + this.GlobalBase = clone; + this.logger = logger; this.mangaConnector = connector; - this.progressToken = new ProgressToken(0); - this.recurring = recurring; - if (recurring && recurrenceTime is null) - throw new ArgumentException("If recurrence is set to true, a recurrence time has to be provided."); - else if(recurring && recurrenceTime is not null) - this.lastExecution = DateTime.Now.Subtract((TimeSpan)recurrenceTime); - this.recurrenceTime = recurrenceTime ?? TimeSpan.Zero; - this.parentJobId = parentJobId; - } - - internal Job(GlobalBase clone, JobType jobType, MangaConnector connector, DateTime lastExecution, bool recurring = false, - TimeSpan? recurrenceTime = null, string? parentJobId = null) : base(clone) - { this.jobType = jobType; - this.mangaConnector = connector; - this.progressToken = new ProgressToken(0); - this.recurring = recurring; - if (recurring && recurrenceTime is null) - throw new ArgumentException("If recurrence is set to true, a recurrence time has to be provided."); - this.lastExecution = lastExecution; - this.recurrenceTime = recurrenceTime ?? TimeSpan.Zero; - this.parentJobId = parentJobId; } - - protected abstract string GetId(); - - public void AddSubJob(Job job) - { - subJobs ??= new List(); - subJobs = subJobs.Append(job); - } - - private DateTime NextExecution() - { - if(recurrenceTime.HasValue && lastExecution.HasValue) - return lastExecution.Value.Add(recurrenceTime.Value); - if(recurrenceTime.HasValue && !lastExecution.HasValue) - return DateTime.Now; - return DateTime.MaxValue; - } - - public void ResetProgress() - { - this.progressToken.increments -= progressToken.incrementsCompleted; - this.lastExecution = DateTime.Now; - this.progressToken.Waiting(); - } - - public void ExecutionEnqueue() - { - this.progressToken.increments -= progressToken.incrementsCompleted; - this.progressToken.Standby(); - } - - public void Cancel() - { - Log($"Cancelling {this}"); - this.progressToken.cancellationRequested = true; - this.progressToken.Cancel(); - this.lastExecution = DateTime.Now; - if(subJobs is not null) - foreach(Job subJob in subJobs) - subJob.Cancel(); - } - - public IEnumerable ExecuteReturnSubTasks(JobBoss jobBoss) - { - progressToken.Start(); - subJobs = ExecuteReturnSubTasksInternal(jobBoss); - lastExecution = DateTime.Now; - return subJobs; - } - - protected abstract IEnumerable ExecuteReturnSubTasksInternal(JobBoss jobBoss); } \ No newline at end of file diff --git a/Tranga/Jobs/JobBoss.cs b/Tranga/Jobs/JobBoss.cs deleted file mode 100644 index f603313..0000000 --- a/Tranga/Jobs/JobBoss.cs +++ /dev/null @@ -1,258 +0,0 @@ -using System.Text.RegularExpressions; -using Newtonsoft.Json; -using Tranga.MangaConnectors; - -namespace Tranga.Jobs; - -public class JobBoss : GlobalBase -{ - public HashSet jobs { get; init; } - private Dictionary> mangaConnectorJobQueue { get; init; } - - public JobBoss(GlobalBase clone, HashSet connectors) : base(clone) - { - this.jobs = new(); - LoadJobsList(connectors); - this.mangaConnectorJobQueue = new(); - Log($"Next job in {jobs.MinBy(job => job.nextExecution)?.nextExecution.Subtract(DateTime.Now)} {jobs.MinBy(job => job.nextExecution)?.id}"); - } - - public void AddJob(Job job) - { - if (ContainsJobLike(job)) - { - Log($"Already Contains Job {job}"); - } - else - { - Log($"Added {job}"); - this.jobs.Add(job); - UpdateJobFile(job); - } - } - - public void AddJobs(IEnumerable jobsToAdd) - { - foreach (Job job in jobsToAdd) - AddJob(job); - } - - /// - /// Compares contents of the provided job and all current jobs - /// Does not check if objects are the same - /// - public bool ContainsJobLike(Job job) - { - return this.jobs.Any(existingJob => existingJob.Equals(job)); - } - - public void RemoveJob(Job job) - { - Log($"Removing {job}"); - job.Cancel(); - this.jobs.Remove(job); - if(job.subJobs is not null && job.subJobs.Any()) - RemoveJobs(job.subJobs); - UpdateJobFile(job); - } - - public void RemoveJobs(IEnumerable jobsToRemove) - { - List toRemove = jobsToRemove.ToList(); //Prevent multiple enumeration - Log($"Removing {toRemove.Count()} jobs."); - foreach (Job? job in toRemove) - if(job is not null) - RemoveJob(job); - } - - public IEnumerable GetJobsLike(string? connectorName = null, string? internalId = null, string? chapterNumber = null) - { - IEnumerable ret = this.jobs; - if (connectorName is not null) - ret = ret.Where(job => job.mangaConnector.name == connectorName); - - if (internalId is not null && chapterNumber is not null) - ret = ret.Where(jjob => - { - if (jjob is not DownloadChapter job) - return false; - return job.chapter.parentManga.internalId == internalId && - job.chapter.chapterNumber == chapterNumber; - }); - else if (internalId is not null) - ret = ret.Where(jjob => - { - if (jjob is not DownloadNewChapters job) - return false; - return job.manga.internalId == internalId; - }); - return ret; - } - - public IEnumerable GetJobsLike(MangaConnector? mangaConnector = null, Manga? publication = null, - Chapter? chapter = null) - { - if (chapter is not null) - return GetJobsLike(mangaConnector?.name, chapter.Value.parentManga.internalId, chapter.Value.chapterNumber); - else - return GetJobsLike(mangaConnector?.name, publication?.internalId); - } - - public Job? GetJobById(string jobId) - { - if (this.jobs.FirstOrDefault(jjob => jjob.id == jobId) is { } job) - return job; - return null; - } - - public bool TryGetJobById(string jobId, out Job? job) - { - if (this.jobs.FirstOrDefault(jjob => jjob.id == jobId) is { } ret) - { - job = ret; - return true; - } - - job = null; - return false; - } - - private bool QueueContainsJob(Job job) - { - if (mangaConnectorJobQueue.TryAdd(job.mangaConnector, new Queue()))//If we can add the queue, there is certainly no job in it - return true; - return mangaConnectorJobQueue[job.mangaConnector].Contains(job); - } - - public void AddJobToQueue(Job job) - { - Log($"Adding Job to Queue. {job}"); - if(!QueueContainsJob(job)) - mangaConnectorJobQueue[job.mangaConnector].Enqueue(job); - job.ExecutionEnqueue(); - } - - private void AddJobsToQueue(IEnumerable newJobs) - { - foreach(Job job in newJobs) - AddJobToQueue(job); - } - - private void LoadJobsList(HashSet connectors) - { - if (!Directory.Exists(settings.jobsFolderPath)) //No jobs to load - { - Directory.CreateDirectory(settings.jobsFolderPath); - return; - } - Regex idRex = new (@"(.*)\.json"); - - //Load json-job-files - foreach (FileInfo file in new DirectoryInfo(settings.jobsFolderPath).EnumerateFiles().Where(fileInfo => idRex.IsMatch(fileInfo.Name))) - { - Job job = JsonConvert.DeserializeObject(File.ReadAllText(file.FullName), - new JobJsonConverter(this, new MangaConnectorJsonConverter(this, connectors)))!; - this.jobs.Add(job); - } - - //Connect jobs to parent-jobs and add Publications to cache - foreach (Job job in this.jobs) - { - this.jobs.FirstOrDefault(jjob => jjob.id == job.parentJobId)?.AddSubJob(job); - if (job is DownloadNewChapters dncJob) - cachedPublications.Add(dncJob.manga); - } - - HashSet coverFileNames = cachedPublications.Select(manga => manga.coverFileNameInCache!).ToHashSet(); - foreach (string fileName in Directory.GetFiles(settings.coverImageCache)) - { - if(!coverFileNames.Any(existingManga => fileName.Contains(existingManga))) - File.Delete(fileName); - } - } - - private void UpdateJobFile(Job job) - { - string jobFilePath = Path.Join(settings.jobsFolderPath, $"{job.id}.json"); - - if (!this.jobs.Any(jjob => jjob.id == job.id)) - { - try - { - Log($"Deleting Job-file {jobFilePath}"); - while(IsFileInUse(jobFilePath)) - Thread.Sleep(10); - File.Delete(jobFilePath); - } - catch (Exception e) - { - Log(e.ToString()); - } - } - else - { - Log($"Exporting Job {jobFilePath}"); - string jobStr = JsonConvert.SerializeObject(job); - while(IsFileInUse(jobFilePath)) - Thread.Sleep(10); - File.WriteAllText(jobFilePath, jobStr); - } - } - - private void UpdateAllJobFiles() - { - Log("Exporting Jobs"); - foreach (Job job in this.jobs) - UpdateJobFile(job); - - //Remove files with jobs not in this.jobs-list - Regex idRex = new (@"(.*)\.json"); - foreach (FileInfo file in new DirectoryInfo(settings.jobsFolderPath).EnumerateFiles()) - { - if (idRex.IsMatch(file.Name)) - { - string id = idRex.Match(file.Name).Groups[1].Value; - if (!this.jobs.Any(job => job.id == id)) - { - try - { - file.Delete(); - } - catch (Exception e) - { - Log(e.ToString()); - } - } - } - } - } - - public void CheckJobs() - { - AddJobsToQueue(jobs.Where(job => job.progressToken.state == ProgressToken.State.Waiting && job.nextExecution < DateTime.Now && !QueueContainsJob(job)).OrderBy(job => job.nextExecution)); - foreach (Queue jobQueue in mangaConnectorJobQueue.Values) - { - if(jobQueue.Count < 1) - continue; - Job queueHead = jobQueue.Peek(); - if (queueHead.progressToken.state is ProgressToken.State.Complete or ProgressToken.State.Cancelled) - { - if(!queueHead.recurring) - RemoveJob(queueHead); - else - queueHead.ResetProgress(); - jobQueue.Dequeue(); - Log($"Next job in {jobs.MinBy(job => job.nextExecution)?.nextExecution.Subtract(DateTime.Now)} {jobs.MinBy(job => job.nextExecution)?.id}"); - }else if (queueHead.progressToken.state is ProgressToken.State.Standby) - { - Job[] subJobs = jobQueue.Peek().ExecuteReturnSubTasks(this).ToArray(); - AddJobs(subJobs); - AddJobsToQueue(subJobs); - }else if (queueHead.progressToken.state is ProgressToken.State.Running && DateTime.Now.Subtract(queueHead.progressToken.lastUpdate) > TimeSpan.FromMinutes(5)) - { - Log($"{queueHead} inactive for more than 5 minutes. Cancelling."); - queueHead.Cancel(); - } - } - } - } \ No newline at end of file diff --git a/Tranga/Jobs/JobJsonConverter.cs b/Tranga/Jobs/JobJsonConverter.cs index 5b12143..13b3ccb 100644 --- a/Tranga/Jobs/JobJsonConverter.cs +++ b/Tranga/Jobs/JobJsonConverter.cs @@ -1,4 +1,6 @@ -using Newtonsoft.Json; +using JobQueue; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Tranga.MangaConnectors; @@ -8,11 +10,15 @@ public class JobJsonConverter : JsonConverter { private GlobalBase _clone; private MangaConnectorJsonConverter _mangaConnectorJsonConverter; + private JobQueue queue; + private ILogger? logger; - internal JobJsonConverter(GlobalBase clone, MangaConnectorJsonConverter mangaConnectorJsonConverter) + internal JobJsonConverter(GlobalBase clone, MangaConnectorJsonConverter mangaConnectorJsonConverter, JobQueue queue, ILogger? logger = null) { this._clone = clone; this._mangaConnectorJsonConverter = mangaConnectorJsonConverter; + this.queue = queue; + this.logger = logger; } public override bool CanConvert(Type objectType) @@ -23,53 +29,52 @@ public class JobJsonConverter : JsonConverter public override object ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) { JObject jo = JObject.Load(reader); - - if (jo.ContainsKey("jobType") && jo["jobType"]!.Value() == (byte)Job.JobType.UpdateMetaDataJob) - { - return new UpdateMetadata(this._clone, - jo.GetValue("mangaConnector")!.ToObject(JsonSerializer.Create(new JsonSerializerSettings() + Job.JobType? jobType = (Job.JobType?)jo["jobType"]?.Value(); + MangaConnector? mangaConnector = jo.GetValue("mangaConnector")?.ToObject(JsonSerializer.Create( + new JsonSerializerSettings() + { + Converters = { - Converters = - { - this._mangaConnectorJsonConverter - } - }))!, - jo.GetValue("manga")!.ToObject(), - jo.GetValue("parentJobId")!.Value()); - }else if ((jo.ContainsKey("jobType") && jo["jobType"]!.Value() == (byte)Job.JobType.DownloadNewChaptersJob) || jo.ContainsKey("translatedLanguage"))//TODO change to jobType + this._mangaConnectorJsonConverter + } + })); + if(mangaConnector is null) + throw new JsonException("Could not deserialize this type"); + + switch (jobType) { - DateTime lastExecution = jo.GetValue("lastExecution") is {} le - ? le.ToObject() - : DateTime.UnixEpoch; //TODO do null checks on all variables - return new DownloadNewChapters(this._clone, - jo.GetValue("mangaConnector")!.ToObject(JsonSerializer.Create(new JsonSerializerSettings() - { - Converters = - { - this._mangaConnectorJsonConverter - } - }))!, - jo.GetValue("manga")!.ToObject(), - lastExecution, - jo.GetValue("recurring")!.Value(), - jo.GetValue("recurrenceTime")!.ToObject(), - jo.GetValue("parentJobId")!.Value()); - }else if ((jo.ContainsKey("jobType") && jo["jobType"]!.Value() == (byte)Job.JobType.DownloadChapterJob) || jo.ContainsKey("chapter"))//TODO change to jobType - { - return new DownloadChapter(this._clone, - jo.GetValue("mangaConnector")!.ToObject(JsonSerializer.Create(new JsonSerializerSettings() - { - Converters = - { - this._mangaConnectorJsonConverter - } - }))!, - jo.GetValue("chapter")!.ToObject(), - DateTime.UnixEpoch, - jo.GetValue("parentJobId")!.Value()); + case Job.JobType.UpdateMetaDataJob: + return new UpdateMetadata(_clone, + queue, + mangaConnector, + jo.GetValue("manga")!.ToObject(), + jo.GetValue("jobId")!.Value(), + jo.GetValue("parentJobId")!.Value(), + logger); + case Job.JobType.DownloadChapterJob: + return new DownloadChapter(_clone, + queue, + mangaConnector, + jo.GetValue("chapter")!.ToObject(), + jo.GetValue("steps")!.Value(), + jo.GetValue("jobId")!.Value(), + jo.GetValue("parentJobId")!.Value(), + logger); + break; + case Job.JobType.DownloadNewChaptersJob: + return new DownloadNewChapter(_clone, + queue, + mangaConnector, + jo.GetValue("manga")!.ToObject(), + jo.GetValue("interval")!.ToObject(), + jo.GetValue("steps")!.Value(), + jo.GetValue("jobId")!.Value(), + jo.GetValue("parentJobId")!.Value(), + logger); + break; + default: + throw new JsonException("Could not deserialize this type"); } - - throw new Exception(); } public override bool CanWrite => false; diff --git a/Tranga/Jobs/ProgressToken.cs b/Tranga/Jobs/ProgressToken.cs deleted file mode 100644 index e718c7d..0000000 --- a/Tranga/Jobs/ProgressToken.cs +++ /dev/null @@ -1,78 +0,0 @@ -namespace Tranga.Jobs; - -public class ProgressToken -{ - public bool cancellationRequested { get; set; } - public int increments { get; set; } - public int incrementsCompleted { get; set; } - public float progress => GetProgress(); - public DateTime lastUpdate { get; private set; } - public DateTime executionStarted { get; private set; } - public TimeSpan timeRemaining => GetTimeRemaining(); - - public enum State { Running, Complete, Standby, Cancelled, Waiting } - public State state { get; private set; } - - public ProgressToken(int increments) - { - this.cancellationRequested = false; - this.increments = increments; - this.incrementsCompleted = 0; - this.state = State.Waiting; - this.executionStarted = DateTime.UnixEpoch; - this.lastUpdate = DateTime.UnixEpoch; - } - - private float GetProgress() - { - if(increments > 0 && incrementsCompleted > 0) - return incrementsCompleted / (float)increments; - return 0; - } - - private TimeSpan GetTimeRemaining() - { - if (increments > 0 && incrementsCompleted > 0) - return DateTime.Now.Subtract(this.executionStarted).Divide(incrementsCompleted).Multiply(increments - incrementsCompleted); - return TimeSpan.MaxValue; - } - - public void Increment() - { - this.lastUpdate = DateTime.Now; - this.incrementsCompleted++; - if (incrementsCompleted > increments) - state = State.Complete; - } - - public void Standby() - { - this.lastUpdate = DateTime.Now; - state = State.Standby; - } - - public void Start() - { - this.lastUpdate = DateTime.Now; - state = State.Running; - this.executionStarted = DateTime.Now; - } - - public void Complete() - { - this.lastUpdate = DateTime.Now; - state = State.Complete; - } - - public void Cancel() - { - this.lastUpdate = DateTime.Now; - state = State.Cancelled; - } - - public void Waiting() - { - this.lastUpdate = DateTime.Now; - state = State.Waiting; - } -} \ No newline at end of file diff --git a/Tranga/Jobs/UpdateMetadata.cs b/Tranga/Jobs/UpdateMetadata.cs index e9107ce..1533fb4 100644 --- a/Tranga/Jobs/UpdateMetadata.cs +++ b/Tranga/Jobs/UpdateMetadata.cs @@ -1,4 +1,7 @@ -using Tranga.MangaConnectors; +using System.Runtime.CompilerServices; +using JobQueue; +using Microsoft.Extensions.Logging; +using Tranga.MangaConnectors; namespace Tranga.Jobs; @@ -6,22 +9,17 @@ public class UpdateMetadata : Job { public Manga manga { get; set; } - public UpdateMetadata(GlobalBase clone, MangaConnector connector, Manga manga, string? parentJobId = null) : base(clone, JobType.UpdateMetaDataJob, connector, parentJobId: parentJobId) + public UpdateMetadata(GlobalBase clone, JobQueue queue, MangaConnector connector, Manga manga, string? jobId = null, string? parentJobId = null, ILogger? logger = null) : base (clone, queue, connector, JobType.UpdateMetaDataJob, TimeSpan.Zero, TimeSpan.FromSeconds(clone.settings.jobTimeout), 1, jobId, parentJobId, logger) { this.manga = manga; - } - - protected override string GetId() - { - return $"{GetType()}-{manga.internalId}"; + if (jobId is null) + { + const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + this.JobId = $"{this.GetType().Name}-{connector.name}-{manga.sortName}-{new string(Enumerable.Repeat(chars, 4).Select(s => s[Random.Shared.Next(s.Length)]).ToArray())}"; + } } - public override string ToString() - { - return $"{id} Manga: {manga}"; - } - - protected override IEnumerable ExecuteReturnSubTasksInternal(JobBoss jobBoss) + protected override void Execute(CancellationToken cancellationToken) { //Retrieve new Metadata Manga? possibleUpdatedManga = mangaConnector.GetMangaFromId(manga.publicationId); @@ -29,30 +27,19 @@ public class UpdateMetadata : Job { if (updatedManga.Equals(this.manga)) //Check if anything changed { - this.progressToken.Complete(); - return Array.Empty(); + this.ProgressToken.MarkFinished(); } this.manga.UpdateMetadata(updatedManga); - this.manga.SaveSeriesInfoJson(settings.downloadLocation, true); - this.progressToken.Complete(); + this.manga.SaveSeriesInfoJson(this.GlobalBase.settings.downloadLocation, true); + this.ProgressToken.MarkFinished(); } else { - Log($"Could not find Manga {manga}"); - this.progressToken.Cancel(); - return Array.Empty(); + this.GlobalBase.Log($"Could not find Manga {manga}"); + this.ProgressToken.MarkFailed(); } - this.progressToken.Cancel(); - return Array.Empty(); - } - - public override bool Equals(object? obj) - { - - if (obj is not UpdateMetadata otherJob) - return false; - return otherJob.mangaConnector == this.mangaConnector && - otherJob.manga.Equals(this.manga); + this.ProgressToken.Cancel(); + return ; } } \ No newline at end of file diff --git a/Tranga/LibraryConnectors/LibraryConnector.cs b/Tranga/LibraryConnectors/LibraryConnector.cs index 5bd7097..9634827 100644 --- a/Tranga/LibraryConnectors/LibraryConnector.cs +++ b/Tranga/LibraryConnectors/LibraryConnector.cs @@ -1,6 +1,6 @@ using System.Net; using System.Net.Http.Headers; -using Logging; +using Microsoft.Extensions.Logging; namespace Tranga.LibraryConnectors; @@ -31,7 +31,7 @@ public abstract class LibraryConnector : GlobalBase protected static class NetClient { - public static Stream MakeRequest(string url, string authScheme, string auth, Logger? logger) + public static Stream MakeRequest(string url, string authScheme, string auth, ILogger? logger) { HttpClient client = new(); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(authScheme, auth); @@ -45,8 +45,7 @@ public abstract class LibraryConnector : GlobalBase { HttpResponseMessage response = client.Send(requestMessage); - logger?.WriteLine("LibraryManager.NetClient", - $"GET {url} -> {(int)response.StatusCode}: {response.ReasonPhrase}"); + logger?.LogInformation($"LibraryManager.NetClient | GET {url} -> {(int)response.StatusCode}: {response.ReasonPhrase}"); if (response.StatusCode is HttpStatusCode.Unauthorized && response.RequestMessage!.RequestUri!.AbsoluteUri != url) @@ -61,7 +60,7 @@ public abstract class LibraryConnector : GlobalBase switch (e) { case HttpRequestException: - logger?.WriteLine("LibraryManager.NetClient", $"Failed to make Request:\n\r{e}\n\rContinuing."); + logger?.LogInformation($"LibraryManager.NetClient | Failed to make Request:\n\r{e}\n\rContinuing."); break; default: throw; @@ -70,7 +69,7 @@ public abstract class LibraryConnector : GlobalBase } } - public static bool MakePost(string url, string authScheme, string auth, Logger? logger) + public static bool MakePost(string url, string authScheme, string auth, ILogger? logger) { HttpClient client = new() { @@ -86,7 +85,7 @@ public abstract class LibraryConnector : GlobalBase RequestUri = new Uri(url) }; HttpResponseMessage response = client.Send(requestMessage); - logger?.WriteLine("LibraryManager.NetClient", $"POST {url} -> {(int)response.StatusCode}: {response.ReasonPhrase}"); + logger?.LogInformation($"LibraryManager.NetClient | POST {url} -> {(int)response.StatusCode}: {response.ReasonPhrase}"); if(response.StatusCode is HttpStatusCode.Unauthorized && response.RequestMessage!.RequestUri!.AbsoluteUri != url) return MakePost(response.RequestMessage!.RequestUri!.AbsoluteUri, authScheme, auth, logger); diff --git a/Tranga/MangaConnectors/Bato.cs b/Tranga/MangaConnectors/Bato.cs index 96d97c0..1b49583 100644 --- a/Tranga/MangaConnectors/Bato.cs +++ b/Tranga/MangaConnectors/Bato.cs @@ -1,6 +1,7 @@ using System.Net; using System.Text.RegularExpressions; using HtmlAgilityPack; +using JobQueue; using Tranga.Jobs; namespace Tranga.MangaConnectors; @@ -170,9 +171,9 @@ public class Bato : MangaConnector public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) { - if (progressToken?.cancellationRequested ?? false) + if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false) { - progressToken.Cancel(); + progressToken.Value.Cancel(); return HttpStatusCode.RequestTimeout; } diff --git a/Tranga/MangaConnectors/MangaConnector.cs b/Tranga/MangaConnectors/MangaConnector.cs index bd80aed..7c9e3d4 100644 --- a/Tranga/MangaConnectors/MangaConnector.cs +++ b/Tranga/MangaConnectors/MangaConnector.cs @@ -2,6 +2,7 @@ using System.Net; using System.Runtime.InteropServices; using System.Text.RegularExpressions; +using JobQueue; using Tranga.Jobs; using static System.IO.UnixFileMode; @@ -219,11 +220,11 @@ public abstract class MangaConnector : GlobalBase protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, RequestType requestType, string? comicInfoPath = null, string? referrer = null, ProgressToken? progressToken = null) { - if (progressToken?.cancellationRequested ?? false) + if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false) return HttpStatusCode.RequestTimeout; Log($"Downloading Images for {saveArchiveFilePath}"); if(progressToken is not null) - progressToken.increments = imageUrls.Length; + progressToken.Value.SetSteps(imageUrls.Length); //Check if Publication Directory already exists string directoryPath = Path.GetDirectoryName(saveArchiveFilePath)!; if (!Directory.Exists(directoryPath)) @@ -249,15 +250,15 @@ public abstract class MangaConnector : GlobalBase Log($"{saveArchiveFilePath} {chapter + 1:000}/{imageUrls.Length:000} {status}"); if ((int)status < 200 || (int)status >= 300) { - progressToken?.Complete(); + progressToken?.MarkFinished(); return status; } - if (progressToken?.cancellationRequested ?? false) + if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false) { - progressToken.Complete(); + progressToken.Value.MarkFinished(); return HttpStatusCode.RequestTimeout; } - progressToken?.Increment(); + progressToken?.UpdateProgress(1); } if(comicInfoPath is not null) @@ -270,7 +271,7 @@ public abstract class MangaConnector : GlobalBase File.SetUnixFileMode(saveArchiveFilePath, UserRead | UserWrite | UserExecute | GroupRead | GroupWrite | GroupExecute); Directory.Delete(tempFolder, true); //Cleanup - progressToken?.Complete(); + progressToken?.MarkFinished(); return HttpStatusCode.OK; } diff --git a/Tranga/MangaConnectors/MangaDex.cs b/Tranga/MangaConnectors/MangaDex.cs index 08c4bac..b44570e 100644 --- a/Tranga/MangaConnectors/MangaDex.cs +++ b/Tranga/MangaConnectors/MangaDex.cs @@ -1,7 +1,7 @@ using System.Net; using System.Text.Json.Nodes; using System.Text.RegularExpressions; -using Tranga.Jobs; +using JobQueue; using JsonSerializer = System.Text.Json.JsonSerializer; namespace Tranga.MangaConnectors; @@ -242,9 +242,9 @@ public class MangaDex : MangaConnector public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) { - if (progressToken?.cancellationRequested ?? false) + if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false) { - progressToken.Cancel(); + progressToken.Value.Cancel(); return HttpStatusCode.RequestTimeout; } diff --git a/Tranga/MangaConnectors/MangaKatana.cs b/Tranga/MangaConnectors/MangaKatana.cs index a62315b..471265c 100644 --- a/Tranga/MangaConnectors/MangaKatana.cs +++ b/Tranga/MangaConnectors/MangaKatana.cs @@ -1,6 +1,7 @@ using System.Net; using System.Text.RegularExpressions; using HtmlAgilityPack; +using JobQueue; using Tranga.Jobs; namespace Tranga.MangaConnectors; @@ -196,9 +197,9 @@ public class MangaKatana : MangaConnector public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) { - if (progressToken?.cancellationRequested ?? false) + if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false) { - progressToken.Cancel(); + progressToken.Value.Cancel(); return HttpStatusCode.RequestTimeout; } diff --git a/Tranga/MangaConnectors/MangaLife.cs b/Tranga/MangaConnectors/MangaLife.cs index fd92377..3442234 100644 --- a/Tranga/MangaConnectors/MangaLife.cs +++ b/Tranga/MangaConnectors/MangaLife.cs @@ -1,6 +1,7 @@ using System.Net; using System.Text.RegularExpressions; using HtmlAgilityPack; +using JobQueue; using Tranga.Jobs; namespace Tranga.MangaConnectors; @@ -161,16 +162,16 @@ public class MangaLife : MangaConnector public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) { - if (progressToken?.cancellationRequested ?? false) + if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false) { - progressToken.Cancel(); + progressToken.Value.Cancel(); return HttpStatusCode.RequestTimeout; } Manga chapterParentManga = chapter.parentManga; - if (progressToken?.cancellationRequested ?? false) + if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false) { - progressToken.Cancel(); + progressToken.Value.Cancel(); return HttpStatusCode.RequestTimeout; } diff --git a/Tranga/MangaConnectors/Manganato.cs b/Tranga/MangaConnectors/Manganato.cs index d7bbd79..8e884b7 100644 --- a/Tranga/MangaConnectors/Manganato.cs +++ b/Tranga/MangaConnectors/Manganato.cs @@ -1,6 +1,7 @@ using System.Net; using System.Text.RegularExpressions; using HtmlAgilityPack; +using JobQueue; using Tranga.Jobs; namespace Tranga.MangaConnectors; @@ -179,9 +180,9 @@ public class Manganato : MangaConnector public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) { - if (progressToken?.cancellationRequested ?? false) + if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false) { - progressToken.Cancel(); + progressToken.Value.Cancel(); return HttpStatusCode.RequestTimeout; } diff --git a/Tranga/MangaConnectors/Mangasee.cs b/Tranga/MangaConnectors/Mangasee.cs index 3a0efde..29fca73 100644 --- a/Tranga/MangaConnectors/Mangasee.cs +++ b/Tranga/MangaConnectors/Mangasee.cs @@ -3,6 +3,7 @@ using System.Net; using System.Text.RegularExpressions; using System.Xml.Linq; using HtmlAgilityPack; +using JobQueue; using Newtonsoft.Json; using Tranga.Jobs; @@ -216,16 +217,16 @@ public class Mangasee : MangaConnector public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) { - if (progressToken?.cancellationRequested ?? false) + if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false) { - progressToken.Cancel(); + progressToken.Value.Cancel(); return HttpStatusCode.RequestTimeout; } Manga chapterParentManga = chapter.parentManga; - if (progressToken?.cancellationRequested ?? false) + if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false) { - progressToken.Cancel(); + progressToken.Value.Cancel(); return HttpStatusCode.RequestTimeout; } diff --git a/Tranga/MangaConnectors/Mangaworld.cs b/Tranga/MangaConnectors/Mangaworld.cs index a457999..1ba3285 100644 --- a/Tranga/MangaConnectors/Mangaworld.cs +++ b/Tranga/MangaConnectors/Mangaworld.cs @@ -1,6 +1,7 @@ using System.Net; using System.Text.RegularExpressions; using HtmlAgilityPack; +using JobQueue; using Tranga.Jobs; namespace Tranga.MangaConnectors; @@ -178,9 +179,9 @@ public class Mangaworld: MangaConnector public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null) { - if (progressToken?.cancellationRequested ?? false) + if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false) { - progressToken.Cancel(); + progressToken.Value.Cancel(); return HttpStatusCode.RequestTimeout; } diff --git a/Tranga/Server.cs b/Tranga/Server.cs index c12f1fa..75b1eb2 100644 --- a/Tranga/Server.cs +++ b/Tranga/Server.cs @@ -2,6 +2,8 @@ using System.Runtime.InteropServices; using System.Text; using System.Text.RegularExpressions; +using JobQueue; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Tranga.Jobs; using Tranga.LibraryConnectors; @@ -167,35 +169,47 @@ public class Server : GlobalBase SendResponse(HttpStatusCode.OK, response, connector!.GetChapters((Manga)manga!, translatedLanguage??"en")); break; case "Jobs": - if (!requestVariables.TryGetValue("jobId", out jobId)) + if (requestVariables.TryGetValue("jobId", out jobId)) { - if(!_parent.jobBoss.jobs.Any(jjob => jjob.id == jobId)) - SendResponse(HttpStatusCode.BadRequest, response); + Job? job = _parent.JobQueue.JobWithId(jobId); + if (job is null) + { + SendResponse(HttpStatusCode.NotFound, response); + break; + } else - SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.First(jjob => jjob.id == jobId)); - break; + { + SendResponse(HttpStatusCode.OK, response, job); + break; + } } - SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs); + SendResponse(HttpStatusCode.OK, response, _parent.JobQueue.AllJobs()); break; case "Jobs/Progress": if (requestVariables.TryGetValue("jobId", out jobId)) { - if(!_parent.jobBoss.jobs.Any(jjob => jjob.id == jobId)) - SendResponse(HttpStatusCode.BadRequest, response); + Job? job = _parent.JobQueue.JobWithId(jobId); + if (job is null) + { + SendResponse(HttpStatusCode.NotFound, response); + break; + } else - SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.First(jjob => jjob.id == jobId).progressToken); - break; + { + SendResponse(HttpStatusCode.OK, response, job.ProgressToken); + break; + } } - SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.Select(jjob => jjob.progressToken)); + SendResponse(HttpStatusCode.OK, response, _parent.JobQueue.AllJobs().Select(jjob => jjob.ProgressToken)); break; case "Jobs/Running": - SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.Where(jjob => jjob.progressToken.state is ProgressToken.State.Running)); + SendResponse(HttpStatusCode.OK, response, _parent.JobQueue.RunningJobs()); break; case "Jobs/Waiting": - SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.Where(jjob => jjob.progressToken.state is ProgressToken.State.Standby).OrderBy(jjob => jjob.nextExecution)); + SendResponse(HttpStatusCode.OK, response, _parent.JobQueue.WaitingJobs().Order()); break; case "Jobs/MonitorJobs": - SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.Where(jjob => jjob is DownloadNewChapters).OrderBy(jjob => ((DownloadNewChapters)jjob).manga.sortName)); + SendResponse(HttpStatusCode.OK, response, _parent.JobQueue.AllJobs().Cast().Where(job => job.jobType is Job.JobType.DownloadNewChaptersJob)); break; case "Settings": SendResponse(HttpStatusCode.OK, response, settings); @@ -233,7 +247,7 @@ public class Server : GlobalBase MangaConnector? connector; Manga? tmpManga; Manga manga; - Job? job; + Job? job; string path = Regex.Match(request.Url!.LocalPath, @"[A-z0-9]+(\/[A-z0-9]+)*").Value; switch (path) { @@ -275,7 +289,8 @@ public class Server : GlobalBase manga.MovePublicationFolder(settings.downloadLocation, customFolderName); requestVariables.TryGetValue("translatedLanguage", out translatedLanguage); - _parent.jobBoss.AddJob(new DownloadNewChapters(this, connector!, manga, true, interval, translatedLanguage: translatedLanguage??"en")); + _parent.JobQueue.AddJob(connector!, new DownloadNewChapter(this, _parent.JobQueue, connector!, manga, interval, 1, null, null, this.logger, translatedLanguage??"en")); + SendResponse(HttpStatusCode.Accepted, response); break; case "Jobs/DownloadNewChapters": @@ -304,32 +319,33 @@ public class Server : GlobalBase manga.MovePublicationFolder(settings.downloadLocation, customFolderName); requestVariables.TryGetValue("translatedLanguage", out translatedLanguage); - _parent.jobBoss.AddJob(new DownloadNewChapters(this, connector!, manga, false, translatedLanguage: translatedLanguage??"en")); + _parent.JobQueue.AddJob(connector!, new DownloadNewChapter(this, _parent.JobQueue, connector!, manga, TimeSpan.FromHours(1), 1, null, null, this.logger, translatedLanguage??"en")); + SendResponse(HttpStatusCode.Accepted, response); break; case "Jobs/UpdateMetadata": if (!requestVariables.TryGetValue("internalId", out internalId)) { - foreach (Job pJob in _parent.jobBoss.jobs.Where(possibleDncJob => - possibleDncJob.jobType is Job.JobType.DownloadNewChaptersJob).ToArray())//ToArray to avoid modyifying while adding new jobs + foreach (DownloadNewChapter dncJob in _parent.JobQueue.AllJobs().Cast().Where(j => j.jobType is Job.JobType.DownloadNewChaptersJob).Cast().ToArray())//ToArray to avoid modyifying while adding new jobs { - DownloadNewChapters dncJob = pJob as DownloadNewChapters ?? - throw new Exception("Has to be DownloadNewChapters Job"); - _parent.jobBoss.AddJob(new UpdateMetadata(this, dncJob.mangaConnector, dncJob.manga)); + _parent.JobQueue.AddJob(dncJob.mangaConnector, new UpdateMetadata(this, _parent.JobQueue, dncJob.mangaConnector, dncJob.manga, null, dncJob.JobId, logger)); } SendResponse(HttpStatusCode.Accepted, response); } else { - Job[] possibleDncJobs = _parent.jobBoss.GetJobsLike(internalId: internalId).ToArray(); + Job[] possibleDncJobs = _parent.JobQueue.AllJobs().Cast() + .Where(j => j.jobType is Job.JobType.DownloadNewChaptersJob).Cast() + .ToArray(); switch (possibleDncJobs.Length) { case <1: SendResponse(HttpStatusCode.BadRequest, response, "Could not find matching release"); break; case >1: SendResponse(HttpStatusCode.BadRequest, response, "Multiple releases??"); break; default: - DownloadNewChapters dncJob = possibleDncJobs[0] as DownloadNewChapters ?? + DownloadNewChapter dncJob = possibleDncJobs[0] as DownloadNewChapter ?? throw new Exception("Has to be DownloadNewChapters Job"); - _parent.jobBoss.AddJob(new UpdateMetadata(this, dncJob.mangaConnector, dncJob.manga)); + + _parent.JobQueue.AddJob(dncJob.mangaConnector, new UpdateMetadata(this, _parent.JobQueue, dncJob.mangaConnector, dncJob.manga, null, dncJob.JobId, logger)); SendResponse(HttpStatusCode.Accepted, response); break; } @@ -337,17 +353,17 @@ public class Server : GlobalBase break; case "Jobs/StartNow": if (!requestVariables.TryGetValue("jobId", out jobId) || - !_parent.jobBoss.TryGetJobById(jobId, out job)) + !_parent.JobQueue.TryJobWithId(jobId, out job)) { SendResponse(HttpStatusCode.BadRequest, response); break; } - _parent.jobBoss.AddJobToQueue(job!); + job!.Start(); SendResponse(HttpStatusCode.Accepted, response); break; case "Jobs/Cancel": if (!requestVariables.TryGetValue("jobId", out jobId) || - !_parent.jobBoss.TryGetJobById(jobId, out job)) + !_parent.JobQueue.TryJobWithId(jobId, out job)) { SendResponse(HttpStatusCode.BadRequest, response); break; @@ -491,37 +507,16 @@ public class Server : GlobalBase SendResponse(HttpStatusCode.BadRequest, response); } break; - case "LogMessages": - if (logger is null || !File.Exists(logger?.logFilePath)) - { - SendResponse(HttpStatusCode.NotFound, response); - break; - } - - if (requestVariables.TryGetValue("count", out string? count)) - { - try - { - uint messageCount = uint.Parse(count); - SendResponse(HttpStatusCode.OK, response, logger.Tail(messageCount)); - } - catch (FormatException f) - { - SendResponse(HttpStatusCode.InternalServerError, response, f); - } - }else - SendResponse(HttpStatusCode.OK, response, logger.GetLog()); - break; case "LogFile": - if (logger is null || !File.Exists(logger?.logFilePath)) + if (logger is null || !File.Exists(logger.LogFilePath)) { SendResponse(HttpStatusCode.NotFound, response); break; } - string logDir = new FileInfo(logger.logFilePath).DirectoryName!; + string logDir = new FileInfo(logger.LogFilePath).DirectoryName!; string tmpFilePath = Path.Join(logDir, "Tranga.log"); - File.Copy(logger.logFilePath, tmpFilePath); + File.Copy(logger.LogFilePath, tmpFilePath); SendResponse(HttpStatusCode.OK, response, new FileStream(tmpFilePath, FileMode.Open)); File.Delete(tmpFilePath); break; @@ -542,26 +537,12 @@ public class Server : GlobalBase { case "Jobs": if (!requestVariables.TryGetValue("jobId", out string? jobId) || - !_parent.jobBoss.TryGetJobById(jobId, out Job? job)) + !_parent.JobQueue.TryJobWithId(jobId, out Job? job)) { SendResponse(HttpStatusCode.BadRequest, response); break; } - _parent.jobBoss.RemoveJob(job!); - SendResponse(HttpStatusCode.Accepted, response); - break; - case "Jobs/DownloadNewChapters": - if(!requestVariables.TryGetValue("connector", out connectorName) || - !requestVariables.TryGetValue("internalId", out internalId) || - _parent.GetConnector(connectorName) is null || - _parent.GetPublicationById(internalId) is null) - { - SendResponse(HttpStatusCode.BadRequest, response); - break; - } - connector = _parent.GetConnector(connectorName)!; - manga = (Manga)_parent.GetPublicationById(internalId)!; - _parent.jobBoss.RemoveJobs(_parent.jobBoss.GetJobsLike(connector, manga)); + _parent.JobQueue.RemoveJob(job!); SendResponse(HttpStatusCode.Accepted, response); break; case "NotificationConnectors": diff --git a/Tranga/Tranga.cs b/Tranga/Tranga.cs index e146f6f..6867520 100644 --- a/Tranga/Tranga.cs +++ b/Tranga/Tranga.cs @@ -1,13 +1,14 @@ -using Logging; -using Tranga.Jobs; +using GlaxLogger; using Tranga.MangaConnectors; +using JobQueue; +using Microsoft.Extensions.Logging; namespace Tranga; public partial class Tranga : GlobalBase { public bool keepRunning; - public JobBoss jobBoss; + public JobQueue JobQueue; private Server _server; private HashSet _connectors; @@ -26,8 +27,7 @@ public partial class Tranga : GlobalBase new Bato(this), new MangaLife(this) }; - jobBoss = new(this, this._connectors); - StartJobBoss(); + this.JobQueue = new JobQueue(100, logger); this._server = new Server(this); string[] emojis = { "(•‿•)", "(づ \u25d5‿\u25d5 )づ", "( \u02d8\u25bd\u02d8)っ\u2668", "=\uff3e\u25cf \u22cf \u25cf\uff3e=", "（ΦωΦ）", "(\u272a\u3268\u272a)", "( ﾉ･o･ )ﾉ", "（〜^\u2207^ )〜", "~(\u2267ω\u2266)~","૮ \u00b4• ﻌ \u00b4• ა", "(\u02c3ᆺ\u02c2)", "(=\ud83d\udf66 \u0f1d \ud83d\udf66=)"}; SendNotifications("Tranga Started", emojis[Random.Shared.Next(0,emojis.Length-1)]); @@ -64,17 +64,4 @@ public partial class Tranga : GlobalBase manga = GetPublicationById(internalId); return manga is not null; } - - private void StartJobBoss() - { - Thread t = new (() => - { - while (keepRunning) - { - jobBoss.CheckJobs(); - Thread.Sleep(100); - } - }); - t.Start(); - } } \ No newline at end of file diff --git a/Tranga/Tranga.csproj b/Tranga/Tranga.csproj index 2e4242b..1fe6077 100644 --- a/Tranga/Tranga.csproj +++ b/Tranga/Tranga.csproj @@ -8,15 +8,12 @@ + - - - - .dockerignore @@ -24,4 +21,10 @@ + + + ..\..\TaskQueue\JobQueue\bin\Debug\net7.0\JobQueue.dll + + + diff --git a/Tranga/TrangaArgs.cs b/Tranga/TrangaArgs.cs index 86d9028..0ec2a80 100644 --- a/Tranga/TrangaArgs.cs +++ b/Tranga/TrangaArgs.cs @@ -1,4 +1,5 @@ -using Logging; +using GlaxLogger; +using Microsoft.Extensions.Logging; namespace Tranga; @@ -14,17 +15,11 @@ public partial class Tranga : GlobalBase return; } - string[]? consoleLogger = GetArg(args, ArgEnum.ConsoleLogger); string[]? fileLogger = GetArg(args, ArgEnum.FileLogger); string? filePath = fileLogger?[0];//TODO validate path + string[]? logLevel = GetArg(args, ArgEnum.LogLevel); + LogLevel? level = logLevel is null || logLevel.Length < 1 ? null : Enum.Parse(logLevel[0]); - List enabledLoggers = new(); - if(consoleLogger is not null) - enabledLoggers.Add(Logger.LoggerType.ConsoleLogger); - if (fileLogger is not null) - enabledLoggers.Add(Logger.LoggerType.FileLogger); - Logger logger = new(enabledLoggers.ToArray(), Console.Out, Console.OutputEncoding, filePath); - TrangaSettings? settings = null; string[]? downloadLocationPath = GetArg(args, ArgEnum.DownloadLocation); string[]? workingDirectory = GetArg(args, ArgEnum.WorkingDirectory); @@ -53,7 +48,7 @@ public partial class Tranga : GlobalBase Directory.CreateDirectory(settings.downloadLocation);//TODO validate path Directory.CreateDirectory(settings.workingDirectory);//TODO validate path - Tranga _ = new (logger, settings); + Tranga _ = new (new Logger(outputFolderPath: filePath, consoleOut: Console.Out), settings); } private static void PrintHelp() @@ -104,18 +99,17 @@ public partial class Tranga : GlobalBase { { ArgEnum.DownloadLocation, new(new []{"-d", "--downloadLocation"}, 1, "Directory to which downloaded Manga are saved") }, { ArgEnum.WorkingDirectory, new(new []{"-w", "--workingDirectory"}, 1, "Directory in which application-data is saved") }, - { ArgEnum.ConsoleLogger, new(new []{"-c", "--consoleLogger"}, 0, "Enables the consoleLogger") }, - { ArgEnum.FileLogger, new(new []{"-f", "--fileLogger"}, 1, "Enables the fileLogger, Directory where logfiles are saved") }, + { ArgEnum.FileLogger, new(new []{"-f", "--fileLogger"}, 1, "Directory where logfiles are saved") }, + { ArgEnum.LogLevel, new(new []{"-l", "--loglevel"}, 1, "Log-Level") }, { ArgEnum.Help, new(new []{"-h", "--help"}, 0, "Print this") } //{ ArgEnum., new(new []{""}, 1, "") } }; internal enum ArgEnum { - TrangaSettings, + LogLevel, DownloadLocation, WorkingDirectory, - ConsoleLogger, FileLogger, Help } diff --git a/Tranga/TrangaSettings.cs b/Tranga/TrangaSettings.cs index 4058188..4cca386 100644 --- a/Tranga/TrangaSettings.cs +++ b/Tranga/TrangaSettings.cs @@ -13,6 +13,7 @@ public class TrangaSettings public string downloadLocation { get; private set; } public string workingDirectory { get; private set; } public int apiPortNumber { get; init; } + public int jobTimeout { get; init; } = 180; public string styleSheet { get; private set; } public string userAgent { get; set; } = DefaultUserAgent; [JsonIgnore] public string settingsFilePath => Path.Join(workingDirectory, "settings.json");