glax/Tranga
2
0
Fork 0
Code Issues Pull Requests Releases 25 Activity

Jobqueue changes untested

Browse Source
This commit is contained in:
Glax 2024-02-28 00:44:42 +01:00
parent 597abde115
commit 61df23ea29
26 changed files with 259 additions and 771 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CLI/CLI.csproj
View File

@ -8,6 +8,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="GlaxLogger" Version="1.0.7.2" />
<PackageReference Include="Spectre.Console.Cli" Version="0.47.1-preview.0.11" />
</ItemGroup>

48
CLI/Program.cs
View File

@ -1,6 +1,7 @@
using System.ComponentModel;
using System.Diagnostics.CodeAnalysis;
using Logging;
using GlaxLogger;
using Microsoft.Extensions.Logging;
using Spectre.Console;
using Spectre.Console.Cli;
using Tranga;
@ -22,16 +23,16 @@ internal sealed class TrangaCli : Command<TrangaCli.Settings>
[DefaultValue(null)]
public string? workingDirectory { get; init; }
[Description("Enables the file-logger")]
[CommandOption("-f")]
[DefaultValue(null)]
public bool? fileLogger { get; init; }
[Description("Path to save logfile to")]
[CommandOption("-l|--fPath")]
[CommandOption("-f|--fileLogger")]
[DefaultValue(null)]
public string? fileLoggerPath { get; init; }
[Description("LogLevel")]
[CommandOption("-l|--loglevel")]
[DefaultValue(LogLevel.Information)]
public LogLevel level { get; init; }
[Description("Port on which to run API on")]
[CommandOption("-p|--port")]
[DefaultValue(null)]
@ -40,12 +41,7 @@ internal sealed class TrangaCli : Command<TrangaCli.Settings>
public override int Execute([NotNull] CommandContext context, [NotNull] Settings settings)
{
List<Logger.LoggerType> enabledLoggers = new();
if(settings.fileLogger is true)
enabledLoggers.Add(Logger.LoggerType.FileLogger);
string? logFilePath = settings.fileLoggerPath ?? "";
Logger logger = new(enabledLoggers.ToArray(), Console.Out, Console.OutputEncoding, logFilePath);
Logger logger = new (settings.level, settings.fileLoggerPath, Console.Out);
TrangaSettings trangaSettings = new (settings.downloadLocation, settings.workingDirectory, settings.apiPort);
@ -73,7 +69,6 @@ internal sealed class TrangaCli : Command<TrangaCli.Settings>
.AddChoices(new[]
{
"CustomRequest",
"Log",
"Exit"
}));
@ -116,31 +111,6 @@ internal sealed class TrangaCli : Command<TrangaCli.Settings>
AnsiConsole.WriteLine($"Response: {(int)response.StatusCode} {response.StatusCode}");
AnsiConsole.WriteLine(response.Content.ReadAsStringAsync().Result);
break;
case "Log":
List<string> lines = logger.Tail(10).ToList();
Rows rows = new Rows(lines.Select(line => new Text(line)));
AnsiConsole.Live(rows).Start(context =>
{
bool running = true;
while (running)
{
string[] newLines = logger.GetNewLines();
if (newLines.Length > 0)
{
lines.AddRange(newLines);
rows = new Rows(lines.Select(line => new Text(line)));
context.UpdateTarget(rows);
}
Thread.Sleep(100);
if (AnsiConsole.Console.Input.IsKeyAvailable())
{
AnsiConsole.Console.Input.ReadKey(true); //Do not process input
running = false;
}
}
});
break;
case "Exit":
exit = true;
break;

6
Tranga.sln
View File

@ -2,8 +2,6 @@
Microsoft Visual Studio Solution File, Format Version 12.00
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tranga", ".\Tranga\Tranga.csproj", "{545E81B9-D96B-4C8F-A97F-2C02414DE566}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Logging", "Logging\Logging.csproj", "{415BE889-BB7D-426F-976F-8D977876A462}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CLI", "CLI\CLI.csproj", "{4324C816-F9D2-468F-8ED6-397FE2F0DCB3}"
EndProject
Global
@ -16,10 +14,6 @@ Global
{545E81B9-D96B-4C8F-A97F-2C02414DE566}.Debug|Any CPU.Build.0 = Debug|Any CPU
{545E81B9-D96B-4C8F-A97F-2C02414DE566}.Release|Any CPU.ActiveCfg = Release|Any CPU
{545E81B9-D96B-4C8F-A97F-2C02414DE566}.Release|Any CPU.Build.0 = Release|Any CPU
{415BE889-BB7D-426F-976F-8D977876A462}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{415BE889-BB7D-426F-976F-8D977876A462}.Debug|Any CPU.Build.0 = Debug|Any CPU
{415BE889-BB7D-426F-976F-8D977876A462}.Release|Any CPU.ActiveCfg = Release|Any CPU
{415BE889-BB7D-426F-976F-8D977876A462}.Release|Any CPU.Build.0 = Release|Any CPU
{4324C816-F9D2-468F-8ED6-397FE2F0DCB3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4324C816-F9D2-468F-8ED6-397FE2F0DCB3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4324C816-F9D2-468F-8ED6-397FE2F0DCB3}.Release|Any CPU.ActiveCfg = Release|Any CPU

2
Tranga.sln.DotSettings.user Normal file
View File

@ -0,0 +1,2 @@
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
<s:Boolean x:Key="/Default/AddReferences/RecentPaths/=C_003A_005CUsers_005CGlax_005CRiderProjects_005CTaskQueue_005CJobQueue_005Cbin_005CDebug_005Cnet7_002E0_005CJobQueue_002Edll/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

39
Tranga/GlobalBase.cs
View File

@ -1,6 +1,7 @@
using System.Globalization;
using System.Text.RegularExpressions;
using Logging;
using GlaxLogger;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Tranga.LibraryConnectors;
using Tranga.NotificationConnectors;
@ -9,13 +10,13 @@ namespace Tranga;
public abstract class GlobalBase
{
protected Logger? logger { get; init; }
protected TrangaSettings settings { get; init; }
protected HashSet<NotificationConnector> notificationConnectors { get; init; }
protected HashSet<LibraryConnector> libraryConnectors { get; init; }
protected List<Manga> cachedPublications { get; init; }
public static readonly NumberFormatInfo numberFormatDecimalPoint = new (){ NumberDecimalSeparator = "." };
protected static readonly Regex baseUrlRex = new(@"https?:\/\/[0-9A-z\.-]+(:[0-9]+)?");
internal Logger? logger { get; init; }
internal TrangaSettings settings { get; init; }
internal HashSet<NotificationConnector> notificationConnectors { get; init; }
internal HashSet<LibraryConnector> libraryConnectors { get; init; }
internal List<Manga> cachedPublications { get; init; }
internal static readonly NumberFormatInfo numberFormatDecimalPoint = new (){ NumberDecimalSeparator = "." };
internal static readonly Regex baseUrlRex = new(@"https?:\/\/[0-9A-z\.-]+(:[0-9]+)?");
protected GlobalBase(GlobalBase clone)
{
@ -35,23 +36,23 @@ public abstract class GlobalBase
this.cachedPublications = new();
}
protected void Log(string message)
internal void Log(string message)
{
logger?.WriteLine(this.GetType().Name, message);
Log(this.GetType().Name, message);
}
protected void Log(string fStr, params object?[] replace)
internal void Log(string objectType, string message)
{
Log(string.Format(fStr, replace));
logger?.LogInformation($"{objectType} | {message}");
}
protected void SendNotifications(string title, string text)
internal void SendNotifications(string title, string text)
{
foreach (NotificationConnector nc in notificationConnectors)
nc.SendNotification(title, text);
}
protected void AddNotificationConnector(NotificationConnector notificationConnector)
internal void AddNotificationConnector(NotificationConnector notificationConnector)
{
Log($"Adding {notificationConnector}");
notificationConnectors.RemoveWhere(nc => nc.notificationConnectorType == notificationConnector.notificationConnectorType);
@ -63,7 +64,7 @@ public abstract class GlobalBase
File.WriteAllText(settings.notificationConnectorsFilePath, JsonConvert.SerializeObject(notificationConnectors));
}
protected void DeleteNotificationConnector(NotificationConnector.NotificationConnectorType notificationConnectorType)
internal void DeleteNotificationConnector(NotificationConnector.NotificationConnectorType notificationConnectorType)
{
Log($"Removing {notificationConnectorType}");
notificationConnectors.RemoveWhere(nc => nc.notificationConnectorType == notificationConnectorType);
@ -73,13 +74,13 @@ public abstract class GlobalBase
File.WriteAllText(settings.notificationConnectorsFilePath, JsonConvert.SerializeObject(notificationConnectors));
}
protected void UpdateLibraries()
internal void UpdateLibraries()
{
foreach(LibraryConnector lc in libraryConnectors)
lc.UpdateLibrary();
}
protected void AddLibraryConnector(LibraryConnector libraryConnector)
internal void AddLibraryConnector(LibraryConnector libraryConnector)
{
Log($"Adding {libraryConnector}");
libraryConnectors.RemoveWhere(lc => lc.libraryType == libraryConnector.libraryType);
@ -91,7 +92,7 @@ public abstract class GlobalBase
File.WriteAllText(settings.libraryConnectorsFilePath, JsonConvert.SerializeObject(libraryConnectors));
}
protected void DeleteLibraryConnector(LibraryConnector.LibraryType libraryType)
internal void DeleteLibraryConnector(LibraryConnector.LibraryType libraryType)
{
Log($"Removing {libraryType}");
libraryConnectors.RemoveWhere(lc => lc.libraryType == libraryType);
@ -101,7 +102,7 @@ public abstract class GlobalBase
File.WriteAllText(settings.libraryConnectorsFilePath, JsonConvert.SerializeObject(libraryConnectors));
}
protected bool IsFileInUse(string filePath)
internal bool IsFileInUse(string filePath)
{
if (!File.Exists(filePath))
return false;

53
Tranga/Jobs/DownloadChapter.cs
View File

@ -1,4 +1,6 @@
using System.Net;
using JobQueue;
using Microsoft.Extensions.Logging;
using Tranga.MangaConnectors;
namespace Tranga.Jobs;
@ -6,42 +8,27 @@ namespace Tranga.Jobs;
public class DownloadChapter : Job
{
public Chapter chapter { get; init; }
public DownloadChapter(GlobalBase clone, MangaConnector connector, Chapter chapter, DateTime lastExecution, string? parentJobId = null) : base(clone, JobType.DownloadChapterJob, connector, lastExecution, parentJobId: parentJobId)
public DownloadChapter(GlobalBase clone, JobQueue<MangaConnector> queue, MangaConnector connector, Chapter chapter, int steps, string? jobId = null, string? parentJobId = null, ILogger? logger = null) : base (clone, queue, connector, JobType.DownloadChapterJob, TimeSpan.Zero, TimeSpan.FromSeconds(clone.settings.jobTimeout), steps, jobId, parentJobId, logger)
{
this.chapter = chapter;
}
public DownloadChapter(GlobalBase clone, MangaConnector connector, Chapter chapter, string? parentJobId = null) : base(clone, JobType.DownloadChapterJob, connector, parentJobId: parentJobId)
{
this.chapter = chapter;
}
protected override string GetId()
{
return $"{GetType()}-{chapter.parentManga.internalId}-{chapter.chapterNumber}";
}
public override string ToString()
{
return $"{id} Chapter: {chapter}";
}
protected override IEnumerable<Job> ExecuteReturnSubTasksInternal(JobBoss jobBoss)
{
Task downloadTask = new(delegate
if (jobId is null)
{
mangaConnector.CopyCoverFromCacheToDownloadLocation(chapter.parentManga);
HttpStatusCode success = mangaConnector.DownloadChapter(chapter, this.progressToken);
chapter.parentManga.UpdateLatestDownloadedChapter(chapter);
if (success == HttpStatusCode.OK)
{
UpdateLibraries();
SendNotifications("Chapter downloaded", $"{chapter.parentManga.sortName} - {chapter.chapterNumber}");
}
});
downloadTask.Start();
return Array.Empty<Job>();
const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
this.JobId = $"{this.GetType().Name}-{connector.name}-{chapter}-{new string(Enumerable.Repeat(chars, 4).Select(s => s[Random.Shared.Next(s.Length)]).ToArray())}";
}
}
protected override void Execute(CancellationToken cancellationToken)
{
mangaConnector.CopyCoverFromCacheToDownloadLocation(chapter.parentManga);
HttpStatusCode success = mangaConnector.DownloadChapter(chapter, this.ProgressToken);
chapter.parentManga.UpdateLatestDownloadedChapter(chapter);
if (success == HttpStatusCode.OK)
{
this.GlobalBase.UpdateLibraries();
this.GlobalBase.SendNotifications("Chapter downloaded", $"{chapter.parentManga.sortName} - {chapter.chapterNumber}");
}
}
public override bool Equals(object? obj)

60
Tranga/Jobs/DownloadNewChapters.cs
View File

@ -1,59 +1,39 @@
using Tranga.MangaConnectors;
using JobQueue;
using Microsoft.Extensions.Logging;
using Tranga.MangaConnectors;
namespace Tranga.Jobs;
public class DownloadNewChapters : Job
public class DownloadNewChapter : Job
{
public Manga manga { get; set; }
public string translatedLanguage { get; init; }
public DownloadNewChapters(GlobalBase clone, MangaConnector connector, Manga manga, DateTime lastExecution,
bool recurring = false, TimeSpan? recurrence = null, string? parentJobId = null, string translatedLanguage = "en") : base(clone, JobType.DownloadNewChaptersJob, connector, lastExecution, recurring,
recurrence, parentJobId)
{
this.manga = manga;
this.translatedLanguage = translatedLanguage;
}
public DownloadNewChapters(GlobalBase clone, MangaConnector connector, Manga manga, bool recurring = false, TimeSpan? recurrence = null, string? parentJobId = null, string translatedLanguage = "en") : base (clone, JobType.DownloadNewChaptersJob, connector, recurring, recurrence, parentJobId)
public DownloadNewChapter(GlobalBase clone, JobQueue<MangaConnector> queue, MangaConnector connector, Manga manga, TimeSpan interval, int steps, string? jobId = null, string? parentJobId = null, ILogger? logger = null, string translatedLanguage = "en") : base (clone, queue, connector, JobType.DownloadNewChaptersJob, interval, TimeSpan.FromSeconds(clone.settings.jobTimeout), steps, jobId, parentJobId, logger)
{
this.manga = manga;
this.translatedLanguage = translatedLanguage;
if (jobId is null)
{
const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
this.JobId = $"{this.GetType().Name}-{connector.name}-{manga.sortName}-{new string(Enumerable.Repeat(chars, 4).Select(s => s[Random.Shared.Next(s.Length)]).ToArray())}";
}
}
protected override string GetId()
protected override void Execute(CancellationToken cancellationToken)
{
return $"{GetType()}-{manga.internalId}";
}
public override string ToString()
{
return $"{id} Manga: {manga}";
}
protected override IEnumerable<Job> ExecuteReturnSubTasksInternal(JobBoss jobBoss)
{
manga.SaveSeriesInfoJson(settings.downloadLocation);
manga.SaveSeriesInfoJson(GlobalBase.settings.downloadLocation);
Chapter[] chapters = mangaConnector.GetNewChapters(manga, this.translatedLanguage);
this.progressToken.increments = chapters.Length;
List<Job> jobs = new();
this.ProgressToken.SetSteps(chapters.Length);
mangaConnector.CopyCoverFromCacheToDownloadLocation(manga);
foreach (Chapter chapter in chapters)
{
DownloadChapter downloadChapterJob = new(this, this.mangaConnector, chapter, parentJobId: this.id);
jobs.Add(downloadChapterJob);
DownloadChapter downloadChapterJob = new(this.GlobalBase, Queue, mangaConnector, chapter, 0, null,
this.JobId, this.logger);
Queue.AddJob(mangaConnector, downloadChapterJob);
}
UpdateMetadata updateMetadataJob = new(this, this.mangaConnector, this.manga, parentJobId: this.id);
jobs.Add(updateMetadataJob);
progressToken.Complete();
return jobs;
}
public override bool Equals(object? obj)
{
if (obj is not DownloadNewChapters otherJob)
return false;
return otherJob.mangaConnector == this.mangaConnector &&
otherJob.manga.Equals(this.manga);
UpdateMetadata updateMetadataJob = new(this.GlobalBase, Queue, mangaConnector, manga, null, this.JobId, this.logger);
Queue.AddJob(mangaConnector, updateMetadataJob);
this.ProgressToken.MarkFinished();
}
}

93
Tranga/Jobs/Job.cs
View File

@ -1,98 +1,23 @@
using Tranga.MangaConnectors;
using JobQueue;
using Tranga.MangaConnectors;
using Microsoft.Extensions.Logging;
namespace Tranga.Jobs;
public abstract class Job : GlobalBase
public abstract class Job : Job<MangaConnector>
{
protected readonly GlobalBase GlobalBase;
public MangaConnector mangaConnector { get; init; }
public ProgressToken progressToken { get; private set; }
public bool recurring { get; init; }
public TimeSpan? recurrenceTime { get; set; }
public DateTime? lastExecution { get; private set; }
public DateTime nextExecution => NextExecution();
public string id => GetId();
internal IEnumerable<Job>? subJobs { get; private set; }
public string? parentJobId { get; init; }
public enum JobType : byte { DownloadChapterJob, DownloadNewChaptersJob, UpdateMetaDataJob }
public JobType jobType;
protected ILogger? logger;
internal Job(GlobalBase clone, JobType jobType, MangaConnector connector, bool recurring = false, TimeSpan? recurrenceTime = null, string? parentJobId = null) : base(clone)
public Job(GlobalBase clone, JobQueue<MangaConnector> queue, MangaConnector connector, JobType jobType, TimeSpan interval, TimeSpan maximumTimeBetweenUpdates, int steps, string? jobId = null, string? parentJobId = null, ILogger? logger = null) : base(queue, interval, maximumTimeBetweenUpdates, steps, jobId, parentJobId, logger)
{
this.jobType = jobType;
this.GlobalBase = clone;
this.logger = logger;
this.mangaConnector = connector;
this.progressToken = new ProgressToken(0);
this.recurring = recurring;
if (recurring && recurrenceTime is null)
throw new ArgumentException("If recurrence is set to true, a recurrence time has to be provided.");
else if(recurring && recurrenceTime is not null)
this.lastExecution = DateTime.Now.Subtract((TimeSpan)recurrenceTime);
this.recurrenceTime = recurrenceTime ?? TimeSpan.Zero;
this.parentJobId = parentJobId;
}
internal Job(GlobalBase clone, JobType jobType, MangaConnector connector, DateTime lastExecution, bool recurring = false,
TimeSpan? recurrenceTime = null, string? parentJobId = null) : base(clone)
{
this.jobType = jobType;
this.mangaConnector = connector;
this.progressToken = new ProgressToken(0);
this.recurring = recurring;
if (recurring && recurrenceTime is null)
throw new ArgumentException("If recurrence is set to true, a recurrence time has to be provided.");
this.lastExecution = lastExecution;
this.recurrenceTime = recurrenceTime ?? TimeSpan.Zero;
this.parentJobId = parentJobId;
}
protected abstract string GetId();
public void AddSubJob(Job job)
{
subJobs ??= new List<Job>();
subJobs = subJobs.Append(job);
}
private DateTime NextExecution()
{
if(recurrenceTime.HasValue && lastExecution.HasValue)
return lastExecution.Value.Add(recurrenceTime.Value);
if(recurrenceTime.HasValue && !lastExecution.HasValue)
return DateTime.Now;
return DateTime.MaxValue;
}
public void ResetProgress()
{
this.progressToken.increments -= progressToken.incrementsCompleted;
this.lastExecution = DateTime.Now;
this.progressToken.Waiting();
}
public void ExecutionEnqueue()
{
this.progressToken.increments -= progressToken.incrementsCompleted;
this.progressToken.Standby();
}
public void Cancel()
{
Log($"Cancelling {this}");
this.progressToken.cancellationRequested = true;
this.progressToken.Cancel();
this.lastExecution = DateTime.Now;
if(subJobs is not null)
foreach(Job subJob in subJobs)
subJob.Cancel();
}
public IEnumerable<Job> ExecuteReturnSubTasks(JobBoss jobBoss)
{
progressToken.Start();
subJobs = ExecuteReturnSubTasksInternal(jobBoss);
lastExecution = DateTime.Now;
return subJobs;
}
protected abstract IEnumerable<Job> ExecuteReturnSubTasksInternal(JobBoss jobBoss);
}

258
Tranga/Jobs/JobBoss.cs
View File

@ -1,258 +0,0 @@
using System.Text.RegularExpressions;
using Newtonsoft.Json;
using Tranga.MangaConnectors;
namespace Tranga.Jobs;
public class JobBoss : GlobalBase
{
public HashSet<Job> jobs { get; init; }
private Dictionary<MangaConnector, Queue<Job>> mangaConnectorJobQueue { get; init; }
public JobBoss(GlobalBase clone, HashSet<MangaConnector> connectors) : base(clone)
{
this.jobs = new();
LoadJobsList(connectors);
this.mangaConnectorJobQueue = new();
Log($"Next job in {jobs.MinBy(job => job.nextExecution)?.nextExecution.Subtract(DateTime.Now)} {jobs.MinBy(job => job.nextExecution)?.id}");
}
public void AddJob(Job job)
{
if (ContainsJobLike(job))
{
Log($"Already Contains Job {job}");
}
else
{
Log($"Added {job}");
this.jobs.Add(job);
UpdateJobFile(job);
}
}
public void AddJobs(IEnumerable<Job> jobsToAdd)
{
foreach (Job job in jobsToAdd)
AddJob(job);
}
/// <summary>
/// Compares contents of the provided job and all current jobs
/// Does not check if objects are the same
/// </summary>
public bool ContainsJobLike(Job job)
{
return this.jobs.Any(existingJob => existingJob.Equals(job));
}
public void RemoveJob(Job job)
{
Log($"Removing {job}");
job.Cancel();
this.jobs.Remove(job);
if(job.subJobs is not null && job.subJobs.Any())
RemoveJobs(job.subJobs);
UpdateJobFile(job);
}
public void RemoveJobs(IEnumerable<Job?> jobsToRemove)
{
List<Job?> toRemove = jobsToRemove.ToList(); //Prevent multiple enumeration
Log($"Removing {toRemove.Count()} jobs.");
foreach (Job? job in toRemove)
if(job is not null)
RemoveJob(job);
}
public IEnumerable<Job> GetJobsLike(string? connectorName = null, string? internalId = null, string? chapterNumber = null)
{
IEnumerable<Job> ret = this.jobs;
if (connectorName is not null)
ret = ret.Where(job => job.mangaConnector.name == connectorName);
if (internalId is not null && chapterNumber is not null)
ret = ret.Where(jjob =>
{
if (jjob is not DownloadChapter job)
return false;
return job.chapter.parentManga.internalId == internalId &&
job.chapter.chapterNumber == chapterNumber;
});
else if (internalId is not null)
ret = ret.Where(jjob =>
{
if (jjob is not DownloadNewChapters job)
return false;
return job.manga.internalId == internalId;
});
return ret;
}
public IEnumerable<Job> GetJobsLike(MangaConnector? mangaConnector = null, Manga? publication = null,
Chapter? chapter = null)
{
if (chapter is not null)
return GetJobsLike(mangaConnector?.name, chapter.Value.parentManga.internalId, chapter.Value.chapterNumber);
else
return GetJobsLike(mangaConnector?.name, publication?.internalId);
}
public Job? GetJobById(string jobId)
{
if (this.jobs.FirstOrDefault(jjob => jjob.id == jobId) is { } job)
return job;
return null;
}
public bool TryGetJobById(string jobId, out Job? job)
{
if (this.jobs.FirstOrDefault(jjob => jjob.id == jobId) is { } ret)
{
job = ret;
return true;
}
job = null;
return false;
}
private bool QueueContainsJob(Job job)
{
if (mangaConnectorJobQueue.TryAdd(job.mangaConnector, new Queue<Job>()))//If we can add the queue, there is certainly no job in it
return true;
return mangaConnectorJobQueue[job.mangaConnector].Contains(job);
}
public void AddJobToQueue(Job job)
{
Log($"Adding Job to Queue. {job}");
if(!QueueContainsJob(job))
mangaConnectorJobQueue[job.mangaConnector].Enqueue(job);
job.ExecutionEnqueue();
}
private void AddJobsToQueue(IEnumerable<Job> newJobs)
{
foreach(Job job in newJobs)
AddJobToQueue(job);
}
private void LoadJobsList(HashSet<MangaConnector> connectors)
{
if (!Directory.Exists(settings.jobsFolderPath)) //No jobs to load
{
Directory.CreateDirectory(settings.jobsFolderPath);
return;
}
Regex idRex = new (@"(.*)\.json");
//Load json-job-files
foreach (FileInfo file in new DirectoryInfo(settings.jobsFolderPath).EnumerateFiles().Where(fileInfo => idRex.IsMatch(fileInfo.Name)))
{
Job job = JsonConvert.DeserializeObject<Job>(File.ReadAllText(file.FullName),
new JobJsonConverter(this, new MangaConnectorJsonConverter(this, connectors)))!;
this.jobs.Add(job);
}
//Connect jobs to parent-jobs and add Publications to cache
foreach (Job job in this.jobs)
{
this.jobs.FirstOrDefault(jjob => jjob.id == job.parentJobId)?.AddSubJob(job);
if (job is DownloadNewChapters dncJob)
cachedPublications.Add(dncJob.manga);
}
HashSet<string> coverFileNames = cachedPublications.Select(manga => manga.coverFileNameInCache!).ToHashSet();
foreach (string fileName in Directory.GetFiles(settings.coverImageCache))
{
if(!coverFileNames.Any(existingManga => fileName.Contains(existingManga)))
File.Delete(fileName);
}
}
private void UpdateJobFile(Job job)
{
string jobFilePath = Path.Join(settings.jobsFolderPath, $"{job.id}.json");
if (!this.jobs.Any(jjob => jjob.id == job.id))
{
try
{
Log($"Deleting Job-file {jobFilePath}");
while(IsFileInUse(jobFilePath))
Thread.Sleep(10);
File.Delete(jobFilePath);
}
catch (Exception e)
{
Log(e.ToString());
}
}
else
{
Log($"Exporting Job {jobFilePath}");
string jobStr = JsonConvert.SerializeObject(job);
while(IsFileInUse(jobFilePath))
Thread.Sleep(10);
File.WriteAllText(jobFilePath, jobStr);
}
}
private void UpdateAllJobFiles()
{
Log("Exporting Jobs");
foreach (Job job in this.jobs)
UpdateJobFile(job);
//Remove files with jobs not in this.jobs-list
Regex idRex = new (@"(.*)\.json");
foreach (FileInfo file in new DirectoryInfo(settings.jobsFolderPath).EnumerateFiles())
{
if (idRex.IsMatch(file.Name))
{
string id = idRex.Match(file.Name).Groups[1].Value;
if (!this.jobs.Any(job => job.id == id))
{
try
{
file.Delete();
}
catch (Exception e)
{
Log(e.ToString());
}
}
}
}
}
public void CheckJobs()
{
AddJobsToQueue(jobs.Where(job => job.progressToken.state == ProgressToken.State.Waiting && job.nextExecution < DateTime.Now && !QueueContainsJob(job)).OrderBy(job => job.nextExecution));
foreach (Queue<Job> jobQueue in mangaConnectorJobQueue.Values)
{
if(jobQueue.Count < 1)
continue;
Job queueHead = jobQueue.Peek();
if (queueHead.progressToken.state is ProgressToken.State.Complete or ProgressToken.State.Cancelled)
{
if(!queueHead.recurring)
RemoveJob(queueHead);
else
queueHead.ResetProgress();
jobQueue.Dequeue();
Log($"Next job in {jobs.MinBy(job => job.nextExecution)?.nextExecution.Subtract(DateTime.Now)} {jobs.MinBy(job => job.nextExecution)?.id}");
}else if (queueHead.progressToken.state is ProgressToken.State.Standby)
{
Job[] subJobs = jobQueue.Peek().ExecuteReturnSubTasks(this).ToArray();
AddJobs(subJobs);
AddJobsToQueue(subJobs);
}else if (queueHead.progressToken.state is ProgressToken.State.Running && DateTime.Now.Subtract(queueHead.progressToken.lastUpdate) > TimeSpan.FromMinutes(5))
{
Log($"{queueHead} inactive for more than 5 minutes. Cancelling.");
queueHead.Cancel();
}
}
}
}

97
Tranga/Jobs/JobJsonConverter.cs
View File

@ -1,4 +1,6 @@
using Newtonsoft.Json;
using JobQueue;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Tranga.MangaConnectors;
@ -8,11 +10,15 @@ public class JobJsonConverter : JsonConverter
{
private GlobalBase _clone;
private MangaConnectorJsonConverter _mangaConnectorJsonConverter;
private JobQueue<MangaConnector> queue;
private ILogger? logger;
internal JobJsonConverter(GlobalBase clone, MangaConnectorJsonConverter mangaConnectorJsonConverter)
internal JobJsonConverter(GlobalBase clone, MangaConnectorJsonConverter mangaConnectorJsonConverter, JobQueue<MangaConnector> queue, ILogger? logger = null)
{
this._clone = clone;
this._mangaConnectorJsonConverter = mangaConnectorJsonConverter;
this.queue = queue;
this.logger = logger;
}
public override bool CanConvert(Type objectType)
@ -23,53 +29,52 @@ public class JobJsonConverter : JsonConverter
public override object ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer)
{
JObject jo = JObject.Load(reader);
if (jo.ContainsKey("jobType") && jo["jobType"]!.Value<byte>() == (byte)Job.JobType.UpdateMetaDataJob)
{
return new UpdateMetadata(this._clone,
jo.GetValue("mangaConnector")!.ToObject<MangaConnector>(JsonSerializer.Create(new JsonSerializerSettings()
Job.JobType? jobType = (Job.JobType?)jo["jobType"]?.Value<byte>();
MangaConnector? mangaConnector = jo.GetValue("mangaConnector")?.ToObject<MangaConnector>(JsonSerializer.Create(
new JsonSerializerSettings()
{
Converters =
{
Converters =
{
this._mangaConnectorJsonConverter
}
}))!,
jo.GetValue("manga")!.ToObject<Manga>(),
jo.GetValue("parentJobId")!.Value<string?>());
}else if ((jo.ContainsKey("jobType") && jo["jobType"]!.Value<byte>() == (byte)Job.JobType.DownloadNewChaptersJob) || jo.ContainsKey("translatedLanguage"))//TODO change to jobType
this._mangaConnectorJsonConverter
}
}));
if(mangaConnector is null)
throw new JsonException("Could not deserialize this type");
switch (jobType)
{
DateTime lastExecution = jo.GetValue("lastExecution") is {} le
? le.ToObject<DateTime>()
: DateTime.UnixEpoch; //TODO do null checks on all variables
return new DownloadNewChapters(this._clone,
jo.GetValue("mangaConnector")!.ToObject<MangaConnector>(JsonSerializer.Create(new JsonSerializerSettings()
{
Converters =
{
this._mangaConnectorJsonConverter
}
}))!,
jo.GetValue("manga")!.ToObject<Manga>(),
lastExecution,
jo.GetValue("recurring")!.Value<bool>(),
jo.GetValue("recurrenceTime")!.ToObject<TimeSpan?>(),
jo.GetValue("parentJobId")!.Value<string?>());
}else if ((jo.ContainsKey("jobType") && jo["jobType"]!.Value<byte>() == (byte)Job.JobType.DownloadChapterJob) || jo.ContainsKey("chapter"))//TODO change to jobType
{
return new DownloadChapter(this._clone,
jo.GetValue("mangaConnector")!.ToObject<MangaConnector>(JsonSerializer.Create(new JsonSerializerSettings()
{
Converters =
{
this._mangaConnectorJsonConverter
}
}))!,
jo.GetValue("chapter")!.ToObject<Chapter>(),
DateTime.UnixEpoch,
jo.GetValue("parentJobId")!.Value<string?>());
case Job.JobType.UpdateMetaDataJob:
return new UpdateMetadata(_clone,
queue,
mangaConnector,
jo.GetValue("manga")!.ToObject<Manga>(),
jo.GetValue("jobId")!.Value<string>(),
jo.GetValue("parentJobId")!.Value<string?>(),
logger);
case Job.JobType.DownloadChapterJob:
return new DownloadChapter(_clone,
queue,
mangaConnector,
jo.GetValue("chapter")!.ToObject<Chapter>(),
jo.GetValue("steps")!.Value<int>(),
jo.GetValue("jobId")!.Value<string>(),
jo.GetValue("parentJobId")!.Value<string?>(),
logger);
break;
case Job.JobType.DownloadNewChaptersJob:
return new DownloadNewChapter(_clone,
queue,
mangaConnector,
jo.GetValue("manga")!.ToObject<Manga>(),
jo.GetValue("interval")!.ToObject<TimeSpan>(),
jo.GetValue("steps")!.Value<int>(),
jo.GetValue("jobId")!.Value<string>(),
jo.GetValue("parentJobId")!.Value<string?>(),
logger);
break;
default:
throw new JsonException("Could not deserialize this type");
}
throw new Exception();
}
public override bool CanWrite => false;

78
Tranga/Jobs/ProgressToken.cs
View File

@ -1,78 +0,0 @@
namespace Tranga.Jobs;
public class ProgressToken
{
public bool cancellationRequested { get; set; }
public int increments { get; set; }
public int incrementsCompleted { get; set; }
public float progress => GetProgress();
public DateTime lastUpdate { get; private set; }
public DateTime executionStarted { get; private set; }
public TimeSpan timeRemaining => GetTimeRemaining();
public enum State { Running, Complete, Standby, Cancelled, Waiting }
public State state { get; private set; }
public ProgressToken(int increments)
{
this.cancellationRequested = false;
this.increments = increments;
this.incrementsCompleted = 0;
this.state = State.Waiting;
this.executionStarted = DateTime.UnixEpoch;
this.lastUpdate = DateTime.UnixEpoch;
}
private float GetProgress()
{
if(increments > 0 && incrementsCompleted > 0)
return incrementsCompleted / (float)increments;
return 0;
}
private TimeSpan GetTimeRemaining()
{
if (increments > 0 && incrementsCompleted > 0)
return DateTime.Now.Subtract(this.executionStarted).Divide(incrementsCompleted).Multiply(increments - incrementsCompleted);
return TimeSpan.MaxValue;
}
public void Increment()
{
this.lastUpdate = DateTime.Now;
this.incrementsCompleted++;
if (incrementsCompleted > increments)
state = State.Complete;
}
public void Standby()
{
this.lastUpdate = DateTime.Now;
state = State.Standby;
}
public void Start()
{
this.lastUpdate = DateTime.Now;
state = State.Running;
this.executionStarted = DateTime.Now;
}
public void Complete()
{
this.lastUpdate = DateTime.Now;
state = State.Complete;
}
public void Cancel()
{
this.lastUpdate = DateTime.Now;
state = State.Cancelled;
}
public void Waiting()
{
this.lastUpdate = DateTime.Now;
state = State.Waiting;
}
}

49
Tranga/Jobs/UpdateMetadata.cs
View File

@ -1,4 +1,7 @@
using Tranga.MangaConnectors;
using System.Runtime.CompilerServices;
using JobQueue;
using Microsoft.Extensions.Logging;
using Tranga.MangaConnectors;
namespace Tranga.Jobs;
@ -6,22 +9,17 @@ public class UpdateMetadata : Job
{
public Manga manga { get; set; }
public UpdateMetadata(GlobalBase clone, MangaConnector connector, Manga manga, string? parentJobId = null) : base(clone, JobType.UpdateMetaDataJob, connector, parentJobId: parentJobId)
public UpdateMetadata(GlobalBase clone, JobQueue<MangaConnector> queue, MangaConnector connector, Manga manga, string? jobId = null, string? parentJobId = null, ILogger? logger = null) : base (clone, queue, connector, JobType.UpdateMetaDataJob, TimeSpan.Zero, TimeSpan.FromSeconds(clone.settings.jobTimeout), 1, jobId, parentJobId, logger)
{
this.manga = manga;
}
protected override string GetId()
{
return $"{GetType()}-{manga.internalId}";
if (jobId is null)
{
const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
this.JobId = $"{this.GetType().Name}-{connector.name}-{manga.sortName}-{new string(Enumerable.Repeat(chars, 4).Select(s => s[Random.Shared.Next(s.Length)]).ToArray())}";
}
}
public override string ToString()
{
return $"{id} Manga: {manga}";
}
protected override IEnumerable<Job> ExecuteReturnSubTasksInternal(JobBoss jobBoss)
protected override void Execute(CancellationToken cancellationToken)
{
//Retrieve new Metadata
Manga? possibleUpdatedManga = mangaConnector.GetMangaFromId(manga.publicationId);
@ -29,30 +27,19 @@ public class UpdateMetadata : Job
{
if (updatedManga.Equals(this.manga)) //Check if anything changed
{
this.progressToken.Complete();
return Array.Empty<Job>();
this.ProgressToken.MarkFinished();
}
this.manga.UpdateMetadata(updatedManga);
this.manga.SaveSeriesInfoJson(settings.downloadLocation, true);
this.progressToken.Complete();
this.manga.SaveSeriesInfoJson(this.GlobalBase.settings.downloadLocation, true);
this.ProgressToken.MarkFinished();
}
else
{
Log($"Could not find Manga {manga}");
this.progressToken.Cancel();
return Array.Empty<Job>();
this.GlobalBase.Log($"Could not find Manga {manga}");
this.ProgressToken.MarkFailed();
}
this.progressToken.Cancel();
return Array.Empty<Job>();
}
public override bool Equals(object? obj)
{
if (obj is not UpdateMetadata otherJob)
return false;
return otherJob.mangaConnector == this.mangaConnector &&
otherJob.manga.Equals(this.manga);
this.ProgressToken.Cancel();
return ;
}
}

13
Tranga/LibraryConnectors/LibraryConnector.cs
View File

@ -1,6 +1,6 @@
using System.Net;
using System.Net.Http.Headers;
using Logging;
using Microsoft.Extensions.Logging;
namespace Tranga.LibraryConnectors;
@ -31,7 +31,7 @@ public abstract class LibraryConnector : GlobalBase
protected static class NetClient
{
public static Stream MakeRequest(string url, string authScheme, string auth, Logger? logger)
public static Stream MakeRequest(string url, string authScheme, string auth, ILogger? logger)
{
HttpClient client = new();
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(authScheme, auth);
@ -45,8 +45,7 @@ public abstract class LibraryConnector : GlobalBase
{
HttpResponseMessage response = client.Send(requestMessage);
logger?.WriteLine("LibraryManager.NetClient",
$"GET {url} -> {(int)response.StatusCode}: {response.ReasonPhrase}");
logger?.LogInformation($"LibraryManager.NetClient | GET {url} -> {(int)response.StatusCode}: {response.ReasonPhrase}");
if (response.StatusCode is HttpStatusCode.Unauthorized &&
response.RequestMessage!.RequestUri!.AbsoluteUri != url)
@ -61,7 +60,7 @@ public abstract class LibraryConnector : GlobalBase
switch (e)
{
case HttpRequestException:
logger?.WriteLine("LibraryManager.NetClient", $"Failed to make Request:\n\r{e}\n\rContinuing.");
logger?.LogInformation($"LibraryManager.NetClient | Failed to make Request:\n\r{e}\n\rContinuing.");
break;
default:
throw;
@ -70,7 +69,7 @@ public abstract class LibraryConnector : GlobalBase
}
}
public static bool MakePost(string url, string authScheme, string auth, Logger? logger)
public static bool MakePost(string url, string authScheme, string auth, ILogger? logger)
{
HttpClient client = new()
{
@ -86,7 +85,7 @@ public abstract class LibraryConnector : GlobalBase
RequestUri = new Uri(url)
};
HttpResponseMessage response = client.Send(requestMessage);
logger?.WriteLine("LibraryManager.NetClient", $"POST {url} -> {(int)response.StatusCode}: {response.ReasonPhrase}");
logger?.LogInformation($"LibraryManager.NetClient | POST {url} -> {(int)response.StatusCode}: {response.ReasonPhrase}");
if(response.StatusCode is HttpStatusCode.Unauthorized && response.RequestMessage!.RequestUri!.AbsoluteUri != url)
return MakePost(response.RequestMessage!.RequestUri!.AbsoluteUri, authScheme, auth, logger);

5
Tranga/MangaConnectors/Bato.cs
View File

@ -1,6 +1,7 @@
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using JobQueue;
using Tranga.Jobs;
namespace Tranga.MangaConnectors;
@ -170,9 +171,9 @@ public class Bato : MangaConnector
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (progressToken?.cancellationRequested ?? false)
if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false)
{
progressToken.Cancel();
progressToken.Value.Cancel();
return HttpStatusCode.RequestTimeout;
}

15
Tranga/MangaConnectors/MangaConnector.cs
View File

@ -2,6 +2,7 @@
using System.Net;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using JobQueue;
using Tranga.Jobs;
using static System.IO.UnixFileMode;
@ -219,11 +220,11 @@ public abstract class MangaConnector : GlobalBase
protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, RequestType requestType, string? comicInfoPath = null, string? referrer = null, ProgressToken? progressToken = null)
{
if (progressToken?.cancellationRequested ?? false)
if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
Log($"Downloading Images for {saveArchiveFilePath}");
if(progressToken is not null)
progressToken.increments = imageUrls.Length;
progressToken.Value.SetSteps(imageUrls.Length);
//Check if Publication Directory already exists
string directoryPath = Path.GetDirectoryName(saveArchiveFilePath)!;
if (!Directory.Exists(directoryPath))
@ -249,15 +250,15 @@ public abstract class MangaConnector : GlobalBase
Log($"{saveArchiveFilePath} {chapter + 1:000}/{imageUrls.Length:000} {status}");
if ((int)status < 200 || (int)status >= 300)
{
progressToken?.Complete();
progressToken?.MarkFinished();
return status;
}
if (progressToken?.cancellationRequested ?? false)
if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false)
{
progressToken.Complete();
progressToken.Value.MarkFinished();
return HttpStatusCode.RequestTimeout;
}
progressToken?.Increment();
progressToken?.UpdateProgress(1);
}
if(comicInfoPath is not null)
@ -270,7 +271,7 @@ public abstract class MangaConnector : GlobalBase
File.SetUnixFileMode(saveArchiveFilePath, UserRead | UserWrite | UserExecute | GroupRead | GroupWrite | GroupExecute);
Directory.Delete(tempFolder, true); //Cleanup
progressToken?.Complete();
progressToken?.MarkFinished();
return HttpStatusCode.OK;
}

6
Tranga/MangaConnectors/MangaDex.cs
View File

@ -1,7 +1,7 @@
using System.Net;
using System.Text.Json.Nodes;
using System.Text.RegularExpressions;
using Tranga.Jobs;
using JobQueue;
using JsonSerializer = System.Text.Json.JsonSerializer;
namespace Tranga.MangaConnectors;
@ -242,9 +242,9 @@ public class MangaDex : MangaConnector
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (progressToken?.cancellationRequested ?? false)
if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false)
{
progressToken.Cancel();
progressToken.Value.Cancel();
return HttpStatusCode.RequestTimeout;
}

5
Tranga/MangaConnectors/MangaKatana.cs
View File

@ -1,6 +1,7 @@
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using JobQueue;
using Tranga.Jobs;
namespace Tranga.MangaConnectors;
@ -196,9 +197,9 @@ public class MangaKatana : MangaConnector
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (progressToken?.cancellationRequested ?? false)
if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false)
{
progressToken.Cancel();
progressToken.Value.Cancel();
return HttpStatusCode.RequestTimeout;
}

9
Tranga/MangaConnectors/MangaLife.cs
View File

@ -1,6 +1,7 @@
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using JobQueue;
using Tranga.Jobs;
namespace Tranga.MangaConnectors;
@ -161,16 +162,16 @@ public class MangaLife : MangaConnector
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (progressToken?.cancellationRequested ?? false)
if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false)
{
progressToken.Cancel();
progressToken.Value.Cancel();
return HttpStatusCode.RequestTimeout;
}
Manga chapterParentManga = chapter.parentManga;
if (progressToken?.cancellationRequested ?? false)
if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false)
{
progressToken.Cancel();
progressToken.Value.Cancel();
return HttpStatusCode.RequestTimeout;
}

5
Tranga/MangaConnectors/Manganato.cs
View File

@ -1,6 +1,7 @@
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using JobQueue;
using Tranga.Jobs;
namespace Tranga.MangaConnectors;
@ -179,9 +180,9 @@ public class Manganato : MangaConnector
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (progressToken?.cancellationRequested ?? false)
if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false)
{
progressToken.Cancel();
progressToken.Value.Cancel();
return HttpStatusCode.RequestTimeout;
}

9
Tranga/MangaConnectors/Mangasee.cs
View File

@ -3,6 +3,7 @@ using System.Net;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using HtmlAgilityPack;
using JobQueue;
using Newtonsoft.Json;
using Tranga.Jobs;
@ -216,16 +217,16 @@ public class Mangasee : MangaConnector
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (progressToken?.cancellationRequested ?? false)
if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false)
{
progressToken.Cancel();
progressToken.Value.Cancel();
return HttpStatusCode.RequestTimeout;
}
Manga chapterParentManga = chapter.parentManga;
if (progressToken?.cancellationRequested ?? false)
if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false)
{
progressToken.Cancel();
progressToken.Value.Cancel();
return HttpStatusCode.RequestTimeout;
}

5
Tranga/MangaConnectors/Mangaworld.cs
View File

@ -1,6 +1,7 @@
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using JobQueue;
using Tranga.Jobs;
namespace Tranga.MangaConnectors;
@ -178,9 +179,9 @@ public class Mangaworld: MangaConnector
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (progressToken?.cancellationRequested ?? false)
if (progressToken?.CancellationTokenSource.IsCancellationRequested ?? false)
{
progressToken.Cancel();
progressToken.Value.Cancel();
return HttpStatusCode.RequestTimeout;
}

117
Tranga/Server.cs
View File

@ -2,6 +2,8 @@
using System.Runtime.InteropServices;
using System.Text;
using System.Text.RegularExpressions;
using JobQueue;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Tranga.Jobs;
using Tranga.LibraryConnectors;
@ -167,35 +169,47 @@ public class Server : GlobalBase
SendResponse(HttpStatusCode.OK, response, connector!.GetChapters((Manga)manga!, translatedLanguage??"en"));
break;
case "Jobs":
if (!requestVariables.TryGetValue("jobId", out jobId))
if (requestVariables.TryGetValue("jobId", out jobId))
{
if(!_parent.jobBoss.jobs.Any(jjob => jjob.id == jobId))
SendResponse(HttpStatusCode.BadRequest, response);
Job<MangaConnector>? job = _parent.JobQueue.JobWithId(jobId);
if (job is null)
{
SendResponse(HttpStatusCode.NotFound, response);
break;
}
else
SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.First(jjob => jjob.id == jobId));
break;
{
SendResponse(HttpStatusCode.OK, response, job);
break;
}
}
SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs);
SendResponse(HttpStatusCode.OK, response, _parent.JobQueue.AllJobs());
break;
case "Jobs/Progress":
if (requestVariables.TryGetValue("jobId", out jobId))
{
if(!_parent.jobBoss.jobs.Any(jjob => jjob.id == jobId))
SendResponse(HttpStatusCode.BadRequest, response);
Job<MangaConnector>? job = _parent.JobQueue.JobWithId(jobId);
if (job is null)
{
SendResponse(HttpStatusCode.NotFound, response);
break;
}
else
SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.First(jjob => jjob.id == jobId).progressToken);
break;
{
SendResponse(HttpStatusCode.OK, response, job.ProgressToken);
break;
}
}
SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.Select(jjob => jjob.progressToken));
SendResponse(HttpStatusCode.OK, response, _parent.JobQueue.AllJobs().Select(jjob => jjob.ProgressToken));
break;
case "Jobs/Running":
SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.Where(jjob => jjob.progressToken.state is ProgressToken.State.Running));
SendResponse(HttpStatusCode.OK, response, _parent.JobQueue.RunningJobs());
break;
case "Jobs/Waiting":
SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.Where(jjob => jjob.progressToken.state is ProgressToken.State.Standby).OrderBy(jjob => jjob.nextExecution));
SendResponse(HttpStatusCode.OK, response, _parent.JobQueue.WaitingJobs().Order());
break;
case "Jobs/MonitorJobs":
SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.Where(jjob => jjob is DownloadNewChapters).OrderBy(jjob => ((DownloadNewChapters)jjob).manga.sortName));
SendResponse(HttpStatusCode.OK, response, _parent.JobQueue.AllJobs().Cast<Job>().Where(job => job.jobType is Job.JobType.DownloadNewChaptersJob));
break;
case "Settings":
SendResponse(HttpStatusCode.OK, response, settings);
@ -233,7 +247,7 @@ public class Server : GlobalBase
MangaConnector? connector;
Manga? tmpManga;
Manga manga;
Job? job;
Job<MangaConnector>? job;
string path = Regex.Match(request.Url!.LocalPath, @"[A-z0-9]+(\/[A-z0-9]+)*").Value;
switch (path)
{
@ -275,7 +289,8 @@ public class Server : GlobalBase
manga.MovePublicationFolder(settings.downloadLocation, customFolderName);
requestVariables.TryGetValue("translatedLanguage", out translatedLanguage);
_parent.jobBoss.AddJob(new DownloadNewChapters(this, connector!, manga, true, interval, translatedLanguage: translatedLanguage??"en"));
_parent.JobQueue.AddJob(connector!, new DownloadNewChapter(this, _parent.JobQueue, connector!, manga, interval, 1, null, null, this.logger, translatedLanguage??"en"));
SendResponse(HttpStatusCode.Accepted, response);
break;
case "Jobs/DownloadNewChapters":
@ -304,32 +319,33 @@ public class Server : GlobalBase
manga.MovePublicationFolder(settings.downloadLocation, customFolderName);
requestVariables.TryGetValue("translatedLanguage", out translatedLanguage);
_parent.jobBoss.AddJob(new DownloadNewChapters(this, connector!, manga, false, translatedLanguage: translatedLanguage??"en"));
_parent.JobQueue.AddJob(connector!, new DownloadNewChapter(this, _parent.JobQueue, connector!, manga, TimeSpan.FromHours(1), 1, null, null, this.logger, translatedLanguage??"en"));
SendResponse(HttpStatusCode.Accepted, response);
break;
case "Jobs/UpdateMetadata":
if (!requestVariables.TryGetValue("internalId", out internalId))
{
foreach (Job pJob in _parent.jobBoss.jobs.Where(possibleDncJob =>
possibleDncJob.jobType is Job.JobType.DownloadNewChaptersJob).ToArray())//ToArray to avoid modyifying while adding new jobs
foreach (DownloadNewChapter dncJob in _parent.JobQueue.AllJobs().Cast<Job>().Where(j => j.jobType is Job.JobType.DownloadNewChaptersJob).Cast<DownloadNewChapter>().ToArray())//ToArray to avoid modyifying while adding new jobs
{
DownloadNewChapters dncJob = pJob as DownloadNewChapters ??
throw new Exception("Has to be DownloadNewChapters Job");
_parent.jobBoss.AddJob(new UpdateMetadata(this, dncJob.mangaConnector, dncJob.manga));
_parent.JobQueue.AddJob(dncJob.mangaConnector, new UpdateMetadata(this, _parent.JobQueue, dncJob.mangaConnector, dncJob.manga, null, dncJob.JobId, logger));
}
SendResponse(HttpStatusCode.Accepted, response);
}
else
{
Job[] possibleDncJobs = _parent.jobBoss.GetJobsLike(internalId: internalId).ToArray();
Job[] possibleDncJobs = _parent.JobQueue.AllJobs().Cast<Job>()
.Where(j => j.jobType is Job.JobType.DownloadNewChaptersJob).Cast<DownloadNewChapter>()
.ToArray();
switch (possibleDncJobs.Length)
{
case <1: SendResponse(HttpStatusCode.BadRequest, response, "Could not find matching release"); break;
case >1: SendResponse(HttpStatusCode.BadRequest, response, "Multiple releases??"); break;
default:
DownloadNewChapters dncJob = possibleDncJobs[0] as DownloadNewChapters ??
DownloadNewChapter dncJob = possibleDncJobs[0] as DownloadNewChapter ??
throw new Exception("Has to be DownloadNewChapters Job");
_parent.jobBoss.AddJob(new UpdateMetadata(this, dncJob.mangaConnector, dncJob.manga));
_parent.JobQueue.AddJob(dncJob.mangaConnector, new UpdateMetadata(this, _parent.JobQueue, dncJob.mangaConnector, dncJob.manga, null, dncJob.JobId, logger));
SendResponse(HttpStatusCode.Accepted, response);
break;
}
@ -337,17 +353,17 @@ public class Server : GlobalBase
break;
case "Jobs/StartNow":
if (!requestVariables.TryGetValue("jobId", out jobId) ||
!_parent.jobBoss.TryGetJobById(jobId, out job))
!_parent.JobQueue.TryJobWithId(jobId, out job))
{
SendResponse(HttpStatusCode.BadRequest, response);
break;
}
_parent.jobBoss.AddJobToQueue(job!);
job!.Start();
SendResponse(HttpStatusCode.Accepted, response);
break;
case "Jobs/Cancel":
if (!requestVariables.TryGetValue("jobId", out jobId) ||
!_parent.jobBoss.TryGetJobById(jobId, out job))
!_parent.JobQueue.TryJobWithId(jobId, out job))
{
SendResponse(HttpStatusCode.BadRequest, response);
break;
@ -491,37 +507,16 @@ public class Server : GlobalBase
SendResponse(HttpStatusCode.BadRequest, response);
}
break;
case "LogMessages":
if (logger is null || !File.Exists(logger?.logFilePath))
{
SendResponse(HttpStatusCode.NotFound, response);
break;
}
if (requestVariables.TryGetValue("count", out string? count))
{
try
{
uint messageCount = uint.Parse(count);
SendResponse(HttpStatusCode.OK, response, logger.Tail(messageCount));
}
catch (FormatException f)
{
SendResponse(HttpStatusCode.InternalServerError, response, f);
}
}else
SendResponse(HttpStatusCode.OK, response, logger.GetLog());
break;
case "LogFile":
if (logger is null || !File.Exists(logger?.logFilePath))
if (logger is null || !File.Exists(logger.LogFilePath))
{
SendResponse(HttpStatusCode.NotFound, response);
break;
}
string logDir = new FileInfo(logger.logFilePath).DirectoryName!;
string logDir = new FileInfo(logger.LogFilePath).DirectoryName!;
string tmpFilePath = Path.Join(logDir, "Tranga.log");
File.Copy(logger.logFilePath, tmpFilePath);
File.Copy(logger.LogFilePath, tmpFilePath);
SendResponse(HttpStatusCode.OK, response, new FileStream(tmpFilePath, FileMode.Open));
File.Delete(tmpFilePath);
break;
@ -542,26 +537,12 @@ public class Server : GlobalBase
{
case "Jobs":
if (!requestVariables.TryGetValue("jobId", out string? jobId) ||
!_parent.jobBoss.TryGetJobById(jobId, out Job? job))
!_parent.JobQueue.TryJobWithId(jobId, out Job<MangaConnector>? job))
{
SendResponse(HttpStatusCode.BadRequest, response);
break;
}
_parent.jobBoss.RemoveJob(job!);
SendResponse(HttpStatusCode.Accepted, response);
break;
case "Jobs/DownloadNewChapters":
if(!requestVariables.TryGetValue("connector", out connectorName) ||
!requestVariables.TryGetValue("internalId", out internalId) ||
_parent.GetConnector(connectorName) is null ||
_parent.GetPublicationById(internalId) is null)
{
SendResponse(HttpStatusCode.BadRequest, response);
break;
}
connector = _parent.GetConnector(connectorName)!;
manga = (Manga)_parent.GetPublicationById(internalId)!;
_parent.jobBoss.RemoveJobs(_parent.jobBoss.GetJobsLike(connector, manga));
_parent.JobQueue.RemoveJob(job!);
SendResponse(HttpStatusCode.Accepted, response);
break;
case "NotificationConnectors":

23
Tranga/Tranga.cs
View File

@ -1,13 +1,14 @@
using Logging;
using Tranga.Jobs;
using GlaxLogger;
using Tranga.MangaConnectors;
using JobQueue;
using Microsoft.Extensions.Logging;
namespace Tranga;
public partial class Tranga : GlobalBase
{
public bool keepRunning;
public JobBoss jobBoss;
public JobQueue<MangaConnector> JobQueue;
private Server _server;
private HashSet<MangaConnector> _connectors;
@ -26,8 +27,7 @@ public partial class Tranga : GlobalBase
new Bato(this),
new MangaLife(this)
};
jobBoss = new(this, this._connectors);
StartJobBoss();
this.JobQueue = new JobQueue<MangaConnector>(100, logger);
this._server = new Server(this);
string[] emojis = { "(•‿•)", "(づ \u25d5‿\u25d5 )づ", "( \u02d8\u25bd\u02d8)っ\u2668", "=\uff3e\u25cf \u22cf \u25cf\uff3e=", "（ΦωΦ）", "(\u272a\u3268\u272a)", "( ﾉ･o･ )ﾉ", "（〜^\u2207^ )〜", "~(\u2267ω\u2266)~","૮ \u00b4• ﻌ \u00b4• ა", "(\u02c3ᆺ\u02c2)", "(=\ud83d\udf66 \u0f1d \ud83d\udf66=)"};
SendNotifications("Tranga Started", emojis[Random.Shared.Next(0,emojis.Length-1)]);
@ -64,17 +64,4 @@ public partial class Tranga : GlobalBase
manga = GetPublicationById(internalId);
return manga is not null;
}
private void StartJobBoss()
{
Thread t = new (() =>
{
while (keepRunning)
{
jobBoss.CheckJobs();
Thread.Sleep(100);
}
});
t.Start();
}
}

11
Tranga/Tranga.csproj
View File

@ -8,15 +8,12 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="GlaxLogger" Version="1.0.7.2" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="PuppeteerSharp" Version="10.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Logging\Logging.csproj" />
</ItemGroup>
<ItemGroup>
<Content Include="..\.dockerignore">
<Link>.dockerignore</Link>
@ -24,4 +21,10 @@
</Content>
</ItemGroup>
<ItemGroup>
<Reference Include="JobQueue">
<HintPath>..\..\TaskQueue\JobQueue\bin\Debug\net7.0\JobQueue.dll</HintPath>
</Reference>
</ItemGroup>
</Project>

22
Tranga/TrangaArgs.cs
View File

@ -1,4 +1,5 @@
using Logging;
using GlaxLogger;
using Microsoft.Extensions.Logging;
namespace Tranga;
@ -14,17 +15,11 @@ public partial class Tranga : GlobalBase
return;
}
string[]? consoleLogger = GetArg(args, ArgEnum.ConsoleLogger);
string[]? fileLogger = GetArg(args, ArgEnum.FileLogger);
string? filePath = fileLogger?[0];//TODO validate path
string[]? logLevel = GetArg(args, ArgEnum.LogLevel);
LogLevel? level = logLevel is null || logLevel.Length < 1 ? null : Enum.Parse<LogLevel>(logLevel[0]);
List<Logger.LoggerType> enabledLoggers = new();
if(consoleLogger is not null)
enabledLoggers.Add(Logger.LoggerType.ConsoleLogger);
if (fileLogger is not null)
enabledLoggers.Add(Logger.LoggerType.FileLogger);
Logger logger = new(enabledLoggers.ToArray(), Console.Out, Console.OutputEncoding, filePath);
TrangaSettings? settings = null;
string[]? downloadLocationPath = GetArg(args, ArgEnum.DownloadLocation);
string[]? workingDirectory = GetArg(args, ArgEnum.WorkingDirectory);
@ -53,7 +48,7 @@ public partial class Tranga : GlobalBase
Directory.CreateDirectory(settings.downloadLocation);//TODO validate path
Directory.CreateDirectory(settings.workingDirectory);//TODO validate path
Tranga _ = new (logger, settings);
Tranga _ = new (new Logger(outputFolderPath: filePath, consoleOut: Console.Out), settings);
}
private static void PrintHelp()
@ -104,18 +99,17 @@ public partial class Tranga : GlobalBase
{
{ ArgEnum.DownloadLocation, new(new []{"-d", "--downloadLocation"}, 1, "Directory to which downloaded Manga are saved") },
{ ArgEnum.WorkingDirectory, new(new []{"-w", "--workingDirectory"}, 1, "Directory in which application-data is saved") },
{ ArgEnum.ConsoleLogger, new(new []{"-c", "--consoleLogger"}, 0, "Enables the consoleLogger") },
{ ArgEnum.FileLogger, new(new []{"-f", "--fileLogger"}, 1, "Enables the fileLogger, Directory where logfiles are saved") },
{ ArgEnum.FileLogger, new(new []{"-f", "--fileLogger"}, 1, "Directory where logfiles are saved") },
{ ArgEnum.LogLevel, new(new []{"-l", "--loglevel"}, 1, "Log-Level") },
{ ArgEnum.Help, new(new []{"-h", "--help"}, 0, "Print this") }
//{ ArgEnum., new(new []{""}, 1, "") }
};
internal enum ArgEnum
{
TrangaSettings,
LogLevel,
DownloadLocation,
WorkingDirectory,
ConsoleLogger,
FileLogger,
Help
}

1
Tranga/TrangaSettings.cs
View File

@ -13,6 +13,7 @@ public class TrangaSettings
public string downloadLocation { get; private set; }
public string workingDirectory { get; private set; }
public int apiPortNumber { get; init; }
public int jobTimeout { get; init; } = 180;
public string styleSheet { get; private set; }
public string userAgent { get; set; } = DefaultUserAgent;
[JsonIgnore] public string settingsFilePath => Path.Join(workingDirectory, "settings.json");