glax/Tranga
1
0
Fork 0
mirror of https://github.com/C9Glax/tranga.git synced 2025-06-13 06:47:54 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

MangaConnectors do not have to return an Object with 6 Parameters.

Browse Source 
Job-Start Logic readable and optimized
More robust Database design
This commit is contained in:
Glax
2025-05-09 06:28:44 +02:00
parent 7477f4d04d
commit 7d4a6be569
56 changed files with 2924 additions and 5855 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
API/Schema/Jobs/DownloadAvailableChaptersJob.cs
View File

@ -1,17 +1,31 @@
using System.ComponentModel.DataAnnotations;
using Newtonsoft.Json;
namespace API.Schema.Jobs;
public class DownloadAvailableChaptersJob(ulong recurrenceMs, string mangaId, string? parentJobId = null, ICollection<string>? dependsOnJobsIds = null)
: Job(TokenGen.CreateToken(typeof(DownloadAvailableChaptersJob)), JobType.DownloadAvailableChaptersJob, recurrenceMs, parentJobId, dependsOnJobsIds)
public class DownloadAvailableChaptersJob : Job
{
[StringLength(64)]
[Required]
public string MangaId { get; init; } = mangaId;
[StringLength(64)] [Required] public string MangaId { get; init; }
[JsonIgnore] public Manga Manga { get; init; } = null!;
public DownloadAvailableChaptersJob(Manga manga, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(DownloadAvailableChaptersJob)), JobType.DownloadAvailableChaptersJob, recurrenceMs, parentJob, dependsOnJobs)
{
this.MangaId = manga.MangaId;
this.Manga = manga;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
public DownloadAvailableChaptersJob(string mangaId, ulong recurrenceMs, string? parentJobId = null)
: base(TokenGen.CreateToken(typeof(DownloadAvailableChaptersJob)), JobType.DownloadAvailableChaptersJob, recurrenceMs, parentJobId)
{
this.MangaId = mangaId;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
return context.Chapters.Where(c => c.ParentMangaId == MangaId).AsEnumerable()
.Select(chapter => new DownloadSingleChapterJob(chapter.ChapterId, this.JobId));
return Manga.Chapters.Select(chapter => new DownloadSingleChapterJob(chapter, this));
}
}

41
API/Schema/Jobs/DownloadMangaCoverJob.cs
View File

@ -1,26 +1,41 @@
using System.ComponentModel.DataAnnotations;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
namespace API.Schema.Jobs;
public class DownloadMangaCoverJob(string mangaId, string? parentJobId = null, ICollection<string>? dependsOnJobsIds = null)
: Job(TokenGen.CreateToken(typeof(DownloadMangaCoverJob)), JobType.DownloadMangaCoverJob, 0, parentJobId, dependsOnJobsIds)
public class DownloadMangaCoverJob : Job
{
[StringLength(64)]
[Required]
public string MangaId { get; init; } = mangaId;
[StringLength(64)] [Required] public string MangaId { get; init; }
[JsonIgnore] public Manga Manga { get; init; } = null!;
public DownloadMangaCoverJob(Manga manga, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(DownloadMangaCoverJob)), JobType.DownloadMangaCoverJob, 0, parentJob, dependsOnJobs)
{
this.MangaId = manga.MangaId;
this.Manga = manga;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
public DownloadMangaCoverJob(string mangaId, string? parentJobId = null)
: base(TokenGen.CreateToken(typeof(DownloadMangaCoverJob)), JobType.DownloadMangaCoverJob, 0, parentJobId)
{
this.MangaId = mangaId;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
Manga? manga = context.Mangas.Find(this.MangaId);
if (manga is null)
try
{
Log.Error($"Manga {this.MangaId} not found.");
return [];
Manga.CoverFileNameInCache = Manga.MangaConnector.SaveCoverImageToCache(Manga);
context.SaveChanges();
}
catch (DbUpdateException e)
{
Log.Error(e);
}
manga.CoverFileNameInCache = manga.SaveCoverImageToCache();
context.SaveChanges();
Log.Info($"Saved cover for Manga {this.MangaId} to cache at {manga.CoverFileNameInCache}.");
return [];
}
}

64
API/Schema/Jobs/DownloadSingleChapterJob.cs
View File

@ -2,7 +2,7 @@
using System.IO.Compression;
using System.Runtime.InteropServices;
using API.MangaDownloadClients;
using API.Schema.MangaConnectors;
using Newtonsoft.Json;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.Formats.Jpeg;
using SixLabors.ImageSharp.Processing;
@ -11,45 +11,37 @@ using static System.IO.UnixFileMode;
namespace API.Schema.Jobs;
public class DownloadSingleChapterJob(string chapterId, string? parentJobId = null, ICollection<string>? dependsOnJobsIds = null)
: Job(TokenGen.CreateToken(typeof(DownloadSingleChapterJob)), JobType.DownloadSingleChapterJob, 0, parentJobId, dependsOnJobsIds)
public class DownloadSingleChapterJob : Job
{
[StringLength(64)]
[Required]
public string ChapterId { get; init; } = chapterId;
[StringLength(64)] [Required] public string ChapterId { get; init; }
[JsonIgnore] public Chapter Chapter { get; init; } = null!;
public DownloadSingleChapterJob(Chapter chapter, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(DownloadSingleChapterJob)), JobType.DownloadSingleChapterJob, 0, parentJob, dependsOnJobs)
{
this.ChapterId = chapter.ChapterId;
this.Chapter = chapter;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
public DownloadSingleChapterJob(string chapterId, string? parentJobId = null)
: base(TokenGen.CreateToken(typeof(DownloadSingleChapterJob)), JobType.DownloadSingleChapterJob, 0)
{
this.ChapterId = chapterId;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
Chapter? chapter = context.Chapters.Find(ChapterId);
if (chapter is null)
{
Log.Error("Chapter is null.");
return [];
}
Manga? manga = context.Mangas.Find(chapter.ParentMangaId) ?? chapter.ParentManga;
if (manga is null)
{
Log.Error("Manga is null.");
return [];
}
MangaConnector? connector = context.MangaConnectors.Find(manga.MangaConnectorId) ?? manga.MangaConnector;
if (connector is null)
{
Log.Error("Connector is null.");
return [];
}
string[] imageUrls = connector.GetChapterImageUrls(chapter);
string[] imageUrls = Chapter.ParentManga.MangaConnector.GetChapterImageUrls(Chapter);
if (imageUrls.Length < 1)
{
Log.Info($"No imageUrls for chapter {ChapterId}");
return [];
}
string? saveArchiveFilePath = chapter.FullArchiveFilePath;
if (saveArchiveFilePath is null)
{
Log.Error("saveArchiveFilePath is null.");
return [];
}
string saveArchiveFilePath = Chapter.FullArchiveFilePath;
//Check if Publication Directory already exists
string directoryPath = Path.GetDirectoryName(saveArchiveFilePath)!;
@ -88,10 +80,10 @@ public class DownloadSingleChapterJob(string chapterId, string? parentJobId = nu
}
}
CopyCoverFromCacheToDownloadLocation(manga);
CopyCoverFromCacheToDownloadLocation(Chapter.ParentManga);
Log.Debug($"Creating ComicInfo.xml {ChapterId}");
File.WriteAllText(Path.Join(tempFolder, "ComicInfo.xml"), chapter.GetComicInfoXmlString());
File.WriteAllText(Path.Join(tempFolder, "ComicInfo.xml"), Chapter.GetComicInfoXmlString());
Log.Debug($"Packaging images to archive {ChapterId}");
//ZIP-it and ship-it
@ -100,10 +92,10 @@ public class DownloadSingleChapterJob(string chapterId, string? parentJobId = nu
File.SetUnixFileMode(saveArchiveFilePath, UserRead | UserWrite | UserExecute | GroupRead | GroupWrite | GroupExecute | OtherRead | OtherExecute);
Directory.Delete(tempFolder, true); //Cleanup
chapter.Downloaded = true;
Chapter.Downloaded = true;
context.SaveChanges();
return [new UpdateFilesDownloadedJob(0, manga.MangaId, this.JobId)];
return [new UpdateFilesDownloadedJob(Chapter.ParentManga, 0, this)];
}
private void ProcessImage(string imagePath)
@ -138,7 +130,7 @@ public class DownloadSingleChapterJob(string chapterId, string? parentJobId = nu
}
Log.Info($"Copying cover to {publicationFolder}");
string? fileInCache = manga.CoverFileNameInCache ?? manga.SaveCoverImageToCache();
string? fileInCache = manga.CoverFileNameInCache ?? manga.MangaConnector.SaveCoverImageToCache(manga);
if (fileInCache is null)
{
Log.Error($"File {fileInCache} does not exist");

75
API/Schema/Jobs/Job.cs
View File

@ -12,49 +12,50 @@ public abstract class Job
[StringLength(64)]
[Required]
public string JobId { get; init; }
[StringLength(64)]
public string? ParentJobId { get; init; }
[JsonIgnore]
public Job? ParentJob { get; init; }
[StringLength(64)]
public ICollection<string>? DependsOnJobsIds { get; init; }
[JsonIgnore]
public ICollection<Job>? DependsOnJobs { get; init; }
[Required]
public JobType JobType { get; init; }
[Required]
public ulong RecurrenceMs { get; set; }
[Required]
public DateTime LastExecution { get; internal set; } = DateTime.UnixEpoch;
[NotMapped]
[Required]
public DateTime NextExecution => LastExecution.AddMilliseconds(RecurrenceMs);
[Required]
public JobState state { get; internal set; } = JobState.Waiting;
[Required]
public bool Enabled { get; internal set; } = true;
[NotMapped]
[JsonIgnore]
protected ILog Log { get; init; }
public Job(string jobId, JobType jobType, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: this(jobId, jobType, recurrenceMs, parentJob?.JobId, dependsOnJobs?.Select(j => j.JobId).ToList())
[StringLength(64)] public string? ParentJobId { get; init; }
[JsonIgnore] public Job? ParentJob { get; init; }
[JsonIgnore] public ICollection<Job> DependsOnJobs { get; init; }
[Required] public JobType JobType { get; init; }
[Required] public ulong RecurrenceMs { get; set; }
[Required] public DateTime LastExecution { get; internal set; } = DateTime.UnixEpoch;
[NotMapped] [Required] public DateTime NextExecution => LastExecution.AddMilliseconds(RecurrenceMs);
[Required] public JobState state { get; internal set; } = JobState.FirstExecution;
[Required] public bool Enabled { get; internal set; } = true;
[JsonIgnore] [NotMapped] internal bool IsCompleted => state is >= (JobState)128 and < (JobState)192;
[JsonIgnore] [NotMapped] internal bool DependenciesFulfilled => DependsOnJobs.All(j => j.IsCompleted);
[NotMapped] [JsonIgnore] protected ILog Log { get; init; }
protected Job(string jobId, JobType jobType, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
{
this.JobId = jobId;
this.JobType = jobType;
this.RecurrenceMs = recurrenceMs;
this.ParentJobId = parentJob?.JobId;
this.ParentJob = parentJob;
this.DependsOnJobs = dependsOnJobs;
this.DependsOnJobs = dependsOnJobs ?? [];
this.Log = LogManager.GetLogger(this.GetType());
}
public Job(string jobId, JobType jobType, ulong recurrenceMs, string? parentJobId = null, ICollection<string>? dependsOnJobsIds = null)
/// <summary>
/// EF ONLY!!!
/// </summary>
protected Job(string jobId, JobType jobType, ulong recurrenceMs, string? parentJobId)
{
Log = LogManager.GetLogger(GetType());
JobId = jobId;
ParentJobId = parentJobId;
DependsOnJobsIds = dependsOnJobsIds;
JobType = jobType;
RecurrenceMs = recurrenceMs;
this.JobId = jobId;
this.JobType = jobType;
this.RecurrenceMs = recurrenceMs;
this.ParentJobId = parentJobId;
this.DependsOnJobs = [];
this.Log = LogManager.GetLogger(this.GetType());
}
public IEnumerable<Job> Run(IServiceProvider serviceProvider)

3
API/Schema/Jobs/JobState.cs
View File

@ -3,11 +3,12 @@
public enum JobState : byte
{
//Values 0-63 Preparation Stages
Waiting = 0,
FirstExecution = 0,
//64-127 Running Stages
Running = 64,
//128-191 Completion Stages
Completed = 128,
CompletedWaiting = 159,
//192-255 Error stages
Failed = 192
}

24
API/Schema/Jobs/MoveFileOrFolderJob.cs
View File

@ -2,15 +2,31 @@
namespace API.Schema.Jobs;
public class MoveFileOrFolderJob(string fromLocation, string toLocation, string? parentJobId = null, ICollection<string>? dependsOnJobsIds = null)
: Job(TokenGen.CreateToken(typeof(MoveFileOrFolderJob)), JobType.MoveFileOrFolderJob, 0, parentJobId, dependsOnJobsIds)
public class MoveFileOrFolderJob : Job
{
[StringLength(256)]
[Required]
public string FromLocation { get; init; } = fromLocation;
public string FromLocation { get; init; }
[StringLength(256)]
[Required]
public string ToLocation { get; init; } = toLocation;
public string ToLocation { get; init; }
public MoveFileOrFolderJob(string fromLocation, string toLocation, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(MoveFileOrFolderJob)), JobType.MoveFileOrFolderJob, 0, parentJob, dependsOnJobs)
{
this.FromLocation = fromLocation;
this.ToLocation = toLocation;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
public MoveFileOrFolderJob(string jobId, string fromLocation, string toLocation, string? parentJobId = null)
: base(jobId, JobType.MoveFileOrFolderJob, 0, parentJobId)
{
this.FromLocation = fromLocation;
this.ToLocation = toLocation;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{

52
API/Schema/Jobs/MoveMangaLibraryJob.cs
View File

@ -1,35 +1,39 @@
using System.ComponentModel.DataAnnotations;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
namespace API.Schema.Jobs;
public class MoveMangaLibraryJob(string mangaId, string toLibraryId, string? parentJobId = null, ICollection<string>? dependsOnJobsIds = null)
: Job(TokenGen.CreateToken(typeof(MoveMangaLibraryJob)), JobType.MoveMangaLibraryJob, 0, parentJobId, dependsOnJobsIds)
public class MoveMangaLibraryJob : Job
{
[StringLength(64)]
[Required]
public string MangaId { get; init; } = mangaId;
[StringLength(64)]
[Required]
public string ToLibraryId { get; init; } = toLibraryId;
[StringLength(64)] [Required] public string MangaId { get; init; }
[JsonIgnore] public Manga Manga { get; init; } = null!;
[StringLength(64)] [Required] public string ToLibraryId { get; init; }
public LocalLibrary ToLibrary { get; init; } = null!;
public MoveMangaLibraryJob(Manga manga, LocalLibrary toLibrary, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(MoveMangaLibraryJob)), JobType.MoveMangaLibraryJob, 0, parentJob, dependsOnJobs)
{
this.MangaId = manga.MangaId;
this.Manga = manga;
this.ToLibraryId = toLibrary.LocalLibraryId;
this.ToLibrary = toLibrary;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
public MoveMangaLibraryJob(string mangaId, string toLibraryId, string? parentJobId = null)
: base(TokenGen.CreateToken(typeof(MoveMangaLibraryJob)), JobType.MoveMangaLibraryJob, 0, parentJobId)
{
this.MangaId = mangaId;
this.ToLibraryId = toLibraryId;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
Manga? manga = context.Mangas.Find(MangaId);
if (manga is null)
{
Log.Error("Manga not found");
return [];
}
LocalLibrary? library = context.LocalLibraries.Find(ToLibraryId);
if (library is null)
{
Log.Error("LocalLibrary not found");
return [];
}
Chapter[] chapters = context.Chapters.Where(c => c.ParentMangaId == MangaId).ToArray();
Dictionary<Chapter, string> oldPath = chapters.ToDictionary(c => c, c => c.FullArchiveFilePath!);
manga.Library = library;
Dictionary<Chapter, string> oldPath = Manga.Chapters.ToDictionary(c => c, c => c.FullArchiveFilePath);
Manga.Library = ToLibrary;
try
{
context.SaveChanges();
@ -40,6 +44,6 @@ public class MoveMangaLibraryJob(string mangaId, string toLibraryId, string? par
return [];
}
return chapters.Select(c => new MoveFileOrFolderJob(oldPath[c], c.FullArchiveFilePath!));
return Manga.Chapters.Select(c => new MoveFileOrFolderJob(oldPath[c], c.FullArchiveFilePath));
}
}

49
API/Schema/Jobs/RetrieveChaptersJob.cs
View File

@ -1,40 +1,43 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.MangaConnectors;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
namespace API.Schema.Jobs;
public class RetrieveChaptersJob(ulong recurrenceMs, string mangaId, string? parentJobId = null, ICollection<string>? dependsOnJobsIds = null)
: Job(TokenGen.CreateToken(typeof(RetrieveChaptersJob)), JobType.RetrieveChaptersJob, recurrenceMs, parentJobId, dependsOnJobsIds)
public class RetrieveChaptersJob : Job
{
[StringLength(64)]
[Required]
public string MangaId { get; init; } = mangaId;
[StringLength(64)] [Required] public string MangaId { get; init; }
[JsonIgnore] public Manga Manga { get; init; } = null!;
[StringLength(8)] [Required] public string Language { get; private set; }
public RetrieveChaptersJob(Manga manga, string language, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(RetrieveChaptersJob)), JobType.RetrieveChaptersJob, recurrenceMs, parentJob, dependsOnJobs)
{
this.MangaId = manga.MangaId;
this.Manga = manga;
this.Language = language;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
public RetrieveChaptersJob(string mangaId, string language, ulong recurrenceMs, string? parentJobId = null)
: base(TokenGen.CreateToken(typeof(RetrieveChaptersJob)), JobType.RetrieveChaptersJob, recurrenceMs, parentJobId)
{
this.MangaId = mangaId;
this.Language = language;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
Manga? manga = context.Mangas.Find(MangaId);
if (manga is null)
{
Log.Error("Manga is null.");
return [];
}
MangaConnector? connector = manga.MangaConnector ?? context.MangaConnectors.Find(manga.MangaConnectorId);
if (connector is null)
{
Log.Error("Connector is null.");
return [];
}
// This gets all chapters that are not downloaded
Chapter[] allNewChapters = connector.GetNewChapters(manga).DistinctBy(c => c.ChapterId).ToArray();
Log.Info($"{allNewChapters.Length} new chapters.");
Chapter[] allChapters = Manga.MangaConnector.GetChapters(Manga, Language);
Chapter[] newChapters = allChapters.Where(chapter => context.Chapters.Contains(chapter) == false).ToArray();
Log.Info($"{newChapters.Length} new chapters.");
try
{
// This filters out chapters that are not downloaded but already exist in the DB
string[] chapterIds = context.Chapters.Where(chapter => chapter.ParentMangaId == manga.MangaId)
.Select(chapter => chapter.ChapterId).ToArray();
Chapter[] newChapters = allNewChapters.Where(chapter => !chapterIds.Contains(chapter.ChapterId)).ToArray();
context.Chapters.AddRange(newChapters);
context.SaveChanges();
}

39
API/Schema/Jobs/UpdateFilesDownloadedJob.cs
View File

@ -1,22 +1,43 @@
using System.ComponentModel.DataAnnotations;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
namespace API.Schema.Jobs;
public class UpdateFilesDownloadedJob(ulong recurrenceMs, string mangaId, string? parentJobId = null, ICollection<string>? dependsOnJobsIds = null)
: Job(TokenGen.CreateToken(typeof(UpdateFilesDownloadedJob)), JobType.UpdateFilesDownloadedJob, recurrenceMs, parentJobId, dependsOnJobsIds)
public class UpdateFilesDownloadedJob : Job
{
[StringLength(64)]
[Required]
public string MangaId { get; init; } = mangaId;
[StringLength(64)] [Required] public string MangaId { get; init; }
[JsonIgnore] public Manga Manga { get; init; } = null!;
public UpdateFilesDownloadedJob(Manga manga, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(UpdateFilesDownloadedJob)), JobType.UpdateFilesDownloadedJob, recurrenceMs, parentJob, dependsOnJobs)
{
this.MangaId = manga.MangaId;
this.Manga = manga;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
public UpdateFilesDownloadedJob(string mangaId, ulong recurrenceMs, string? parentJobId = null)
: base(TokenGen.CreateToken(typeof(UpdateFilesDownloadedJob)), JobType.UpdateFilesDownloadedJob, recurrenceMs, parentJobId)
{
this.MangaId = mangaId;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
IQueryable<Chapter> chapters = context.Chapters.Where(c => c.ParentMangaId == MangaId);
foreach (Chapter chapter in chapters)
chapter.Downloaded = chapter.IsDownloaded();
foreach (Chapter chapter in Manga.Chapters)
chapter.Downloaded = chapter.CheckDownloaded();
context.SaveChanges();
try
{
context.SaveChanges();
}
catch (DbUpdateException e)
{
Log.Error(e);
}
return [];
}
}

27
API/Schema/Jobs/UpdateMetadataJob.cs
View File

@ -1,27 +0,0 @@
using System.ComponentModel.DataAnnotations;
using Newtonsoft.Json;
namespace API.Schema.Jobs;
public class UpdateMetadataJob(ulong recurrenceMs, string mangaId, string? parentJobId = null, ICollection<string>? dependsOnJobsIds = null)
: Job(TokenGen.CreateToken(typeof(UpdateMetadataJob)), JobType.UpdateMetaDataJob, recurrenceMs, parentJobId, dependsOnJobsIds)
{
[StringLength(64)]
[Required]
public string MangaId { get; init; } = mangaId;
[JsonIgnore]
public virtual Manga? Manga { get; init; }
/// <summary>
/// Updates all data related to Manga.
/// Retrieves data from Mangaconnector
/// Updates Chapter-info
/// </summary>
/// <param name="context"></param>
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
Log.Warn("NOT IMPLEMENTED.");
return [];//TODO
}
}