This commit is contained in:
2025-07-02 02:26:02 +02:00
parent 07880fedb5
commit 57bb87120a
96 changed files with 814 additions and 13216 deletions

View File

@ -1,19 +0,0 @@
using log4net;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Diagnostics;
namespace API.Schema.Contexts;
public abstract class TrangaBaseContext<T>(DbContextOptions<T> options) : DbContext(options) where T : DbContext
{
private ILog Log => LogManager.GetLogger(GetType());
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
base.OnConfiguring(optionsBuilder);
optionsBuilder.LogTo(s =>
{
Log.Debug(s);
}, Array.Empty<string>(), LogLevel.Warning, DbContextLoggerOptions.Level | DbContextLoggerOptions.Category | DbContextLoggerOptions.UtcTime);
}
}

View File

@ -3,9 +3,19 @@ using Microsoft.EntityFrameworkCore;
namespace API.Schema;
[PrimaryKey("Key")]
public abstract class Identifiable(string key)
public abstract class Identifiable
{
public string Key { get; init; } = key;
public Identifiable()
{
this.Key = TokenGen.CreateToken(this.GetType());
}
public Identifiable(string key)
{
this.Key = key;
}
public string Key { get; init; }
public override string ToString() => Key;
}

View File

@ -1,50 +0,0 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
namespace API.Schema.Jobs;
public class DownloadAvailableChaptersJob : JobWithDownloading
{
[StringLength(64)] [Required] public string MangaId { get; init; } = null!;
private Manga? _manga;
[JsonIgnore]
public Manga Manga
{
get => LazyLoader.Load(this, ref _manga) ?? throw new InvalidOperationException();
init
{
MangaId = value.Key;
_manga = value;
}
}
public DownloadAvailableChaptersJob(Manga manga, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(DownloadAvailableChaptersJob)), JobType.DownloadAvailableChaptersJob, recurrenceMs, parentJob, dependsOnJobs)
{
this.Manga = manga;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
internal DownloadAvailableChaptersJob(ILazyLoader lazyLoader, string key, string mangaId, ulong recurrenceMs, string? parentJobId)
: base(lazyLoader, key, JobType.DownloadAvailableChaptersJob, recurrenceMs, parentJobId)
{
this.MangaId = mangaId;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
// Chapters that aren't downloaded and for which no downloading-Job exists
IEnumerable<Chapter> newChapters = Manga.Chapters
.Where(c =>
c.Downloaded == false &&
context.Jobs.Any(j =>
j.JobType == JobType.DownloadSingleChapterJob &&
((DownloadSingleChapterJob)j).Chapter.ParentMangaId == MangaId) == false);
return newChapters.Select(c => new DownloadSingleChapterJob(c, this));
}
}

View File

@ -1,55 +0,0 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
namespace API.Schema.Jobs;
public class DownloadMangaCoverJob : JobWithDownloading
{
[StringLength(64)] [Required] public string MangaId { get; init; } = null!;
private Manga? _manga;
[JsonIgnore]
public Manga Manga
{
get => LazyLoader.Load(this, ref _manga) ?? throw new InvalidOperationException();
init
{
MangaId = value.Key;
_manga = value;
}
}
public DownloadMangaCoverJob(Manga manga, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(DownloadMangaCoverJob)), JobType.DownloadMangaCoverJob, 0, parentJob, dependsOnJobs)
{
this.Manga = manga;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
internal DownloadMangaCoverJob(ILazyLoader lazyLoader, string key, string mangaId, ulong recurrenceMs, string? parentJobId)
: base(lazyLoader, key, JobType.DownloadMangaCoverJob, recurrenceMs, parentJobId)
{
this.MangaId = mangaId;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
//TODO MangaConnector Selection
MangaConnectorId<Manga> mcId = Manga.MangaConnectorIds.First();
try
{
Manga.CoverFileNameInCache = mcId.MangaConnector.SaveCoverImageToCache(mcId);
context.SaveChanges();
}
catch (DbUpdateException e)
{
Log.Error(e);
}
return [];
}
}

View File

@ -1,220 +0,0 @@
using System.ComponentModel.DataAnnotations;
using System.IO.Compression;
using System.Runtime.InteropServices;
using API.MangaDownloadClients;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.Formats.Jpeg;
using SixLabors.ImageSharp.Processing;
using SixLabors.ImageSharp.Processing.Processors.Binarization;
using static System.IO.UnixFileMode;
namespace API.Schema.Jobs;
public class DownloadSingleChapterJob : JobWithDownloading
{
[StringLength(64)] [Required] public string ChapterId { get; init; } = null!;
private Chapter? _chapter;
[JsonIgnore]
public Chapter Chapter
{
get => LazyLoader.Load(this, ref _chapter) ?? throw new InvalidOperationException();
init
{
ChapterId = value.Key;
_chapter = value;
}
}
public DownloadSingleChapterJob(Chapter chapter, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(DownloadSingleChapterJob)), JobType.DownloadSingleChapterJob, 0, parentJob, dependsOnJobs)
{
this.Chapter = chapter;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
internal DownloadSingleChapterJob(ILazyLoader lazyLoader, string key, string chapterId, ulong recurrenceMs, string? parentJobId)
: base(lazyLoader, key, JobType.DownloadSingleChapterJob, recurrenceMs, parentJobId)
{
this.ChapterId = chapterId;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
if (Chapter.Downloaded)
{
Log.Info("Chapter was already downloaded.");
return [];
}
//TODO MangaConnector Selection
MangaConnectorId<Chapter> mcId = Chapter.MangaConnectorIds.First();
string[] imageUrls = mcId.MangaConnector.GetChapterImageUrls(mcId);
if (imageUrls.Length < 1)
{
Log.Info($"No imageUrls for chapter {Chapter}");
return [];
}
string saveArchiveFilePath = Chapter.FullArchiveFilePath;
Log.Debug($"Chapter path: {saveArchiveFilePath}");
//Check if Publication Directory already exists
string? directoryPath = Path.GetDirectoryName(saveArchiveFilePath);
if (directoryPath is null)
{
Log.Error($"Directory path could not be found: {saveArchiveFilePath}");
this.state = JobState.Failed;
return [];
}
if (!Directory.Exists(directoryPath))
{
Log.Info($"Creating publication Directory: {directoryPath}");
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
Directory.CreateDirectory(directoryPath,
UserRead | UserWrite | UserExecute | GroupRead | GroupWrite | GroupExecute );
else
Directory.CreateDirectory(directoryPath);
}
if (File.Exists(saveArchiveFilePath)) //Don't download twice. Redownload
{
Log.Info($"Archive {saveArchiveFilePath} already existed, but deleting and re-downloading.");
File.Delete(saveArchiveFilePath);
}
//Create a temporary folder to store images
string tempFolder = Directory.CreateTempSubdirectory("trangatemp").FullName;
Log.Debug($"Created temp folder: {tempFolder}");
Log.Info($"Downloading images: {Chapter}");
int chapterNum = 0;
//Download all Images to temporary Folder
foreach (string imageUrl in imageUrls)
{
string extension = imageUrl.Split('.')[^1].Split('?')[0];
string imagePath = Path.Join(tempFolder, $"{chapterNum++}.{extension}");
bool status = DownloadImage(imageUrl, imagePath);
if (status is false)
{
Log.Error($"Failed to download image: {imageUrl}");
return [];
}
}
CopyCoverFromCacheToDownloadLocation(Chapter.ParentManga);
Log.Debug($"Creating ComicInfo.xml {Chapter}");
File.WriteAllText(Path.Join(tempFolder, "ComicInfo.xml"), Chapter.GetComicInfoXmlString());
Log.Debug($"Packaging images to archive {Chapter}");
//ZIP-it and ship-it
ZipFile.CreateFromDirectory(tempFolder, saveArchiveFilePath);
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
File.SetUnixFileMode(saveArchiveFilePath, UserRead | UserWrite | UserExecute | GroupRead | GroupWrite | GroupExecute | OtherRead | OtherExecute);
Directory.Delete(tempFolder, true); //Cleanup
Chapter.Downloaded = true;
context.SaveChanges();
if (context.Jobs.ToList().Any(j =>
{
if (j.JobType != JobType.UpdateChaptersDownloadedJob)
return false;
UpdateChaptersDownloadedJob job = (UpdateChaptersDownloadedJob)j;
return job.MangaId == Chapter.ParentMangaId;
}))
return [];
return [new UpdateChaptersDownloadedJob(Chapter.ParentManga, 0, this.ParentJob)];
}
private void ProcessImage(string imagePath)
{
if (!TrangaSettings.bwImages && TrangaSettings.compression == 100)
{
Log.Debug("No processing requested for image");
return;
}
Log.Debug($"Processing image: {imagePath}");
try
{
using Image image = Image.Load(imagePath);
if (TrangaSettings.bwImages)
image.Mutate(i => i.ApplyProcessor(new AdaptiveThresholdProcessor()));
File.Delete(imagePath);
image.SaveAsJpeg(imagePath, new JpegEncoder()
{
Quality = TrangaSettings.compression
});
}
catch (Exception e)
{
if (e is UnknownImageFormatException or NotSupportedException)
{
//If the Image-Format is not processable by ImageSharp, we can't modify it.
Log.Debug($"Unable to process {imagePath}: Not supported image format");
}else if (e is InvalidImageContentException)
{
Log.Debug($"Unable to process {imagePath}: Invalid Content");
}
else
{
Log.Error(e);
}
}
}
private void CopyCoverFromCacheToDownloadLocation(Manga manga)
{
//Check if Publication already has a Folder and cover
string publicationFolder = manga.CreatePublicationFolder();
DirectoryInfo dirInfo = new (publicationFolder);
if (dirInfo.EnumerateFiles().Any(info => info.Name.Contains("cover", StringComparison.InvariantCultureIgnoreCase)))
{
Log.Debug($"Cover already exists at {publicationFolder}");
return;
}
//TODO MangaConnector Selection
MangaConnectorId<Manga> mcId = manga.MangaConnectorIds.First();
Log.Info($"Copying cover to {publicationFolder}");
string? fileInCache = manga.CoverFileNameInCache ?? mcId.MangaConnector.SaveCoverImageToCache(mcId);
if (fileInCache is null)
{
Log.Error($"File {fileInCache} does not exist");
return;
}
string newFilePath = Path.Join(publicationFolder, $"cover.{Path.GetFileName(fileInCache).Split('.')[^1]}" );
File.Copy(fileInCache, newFilePath, true);
if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
File.SetUnixFileMode(newFilePath, GroupRead | GroupWrite | UserRead | UserWrite | OtherRead | OtherWrite);
Log.Debug($"Copied cover from {fileInCache} to {newFilePath}");
}
private bool DownloadImage(string imageUrl, string savePath)
{
HttpDownloadClient downloadClient = new();
RequestResult requestResult = downloadClient.MakeRequest(imageUrl, RequestType.MangaImage);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return false;
if (requestResult.result == Stream.Null)
return false;
FileStream fs = new (savePath, FileMode.Create, FileAccess.Write, FileShare.None);
requestResult.result.CopyTo(fs);
fs.Close();
ProcessImage(savePath);
return true;
}
}

View File

@ -1,149 +0,0 @@
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using API.Schema.Contexts;
using log4net;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
namespace API.Schema.Jobs;
[PrimaryKey("Key")]
public abstract class Job : Identifiable, IComparable<Job>
{
[StringLength(64)] public string? ParentJobId { get; private set; }
[JsonIgnore] public Job? ParentJob { get; internal set; }
private ICollection<Job>? _dependsOnJobs;
[JsonIgnore] public ICollection<Job> DependsOnJobs
{
get => LazyLoader.Load(this, ref _dependsOnJobs) ?? throw new InvalidOperationException();
init => _dependsOnJobs = value;
}
[Required] public JobType JobType { get; init; }
[Required] public ulong RecurrenceMs { get; set; }
[Required] public DateTime LastExecution { get; internal set; } = DateTime.UnixEpoch;
[NotMapped] [Required] public DateTime NextExecution => LastExecution.AddMilliseconds(RecurrenceMs);
[Required] public JobState state { get; internal set; } = JobState.FirstExecution;
[Required] public bool Enabled { get; internal set; } = true;
[JsonIgnore] [NotMapped] internal bool IsCompleted => state is >= (JobState)128 and < (JobState)192;
[NotMapped] [JsonIgnore] protected ILog Log { get; init; }
[NotMapped] [JsonIgnore] protected ILazyLoader LazyLoader { get; init; } = null!;
protected Job(string key, JobType jobType, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(key)
{
this.JobType = jobType;
this.RecurrenceMs = recurrenceMs;
this.ParentJobId = parentJob?.Key;
this.ParentJob = parentJob;
this.DependsOnJobs = dependsOnJobs ?? [];
this.Log = LogManager.GetLogger(this.GetType());
}
/// <summary>
/// EF ONLY!!!
/// </summary>
protected internal Job(ILazyLoader lazyLoader, string key, JobType jobType, ulong recurrenceMs, string? parentJobId)
: base(key)
{
this.LazyLoader = lazyLoader;
this.JobType = jobType;
this.RecurrenceMs = recurrenceMs;
this.ParentJobId = parentJobId;
this.DependsOnJobs = [];
this.Log = LogManager.GetLogger(this.GetType());
}
public IEnumerable<Job> Run(PgsqlContext context, ref bool running)
{
Log.Info($"Running job {this}");
DateTime jobStart = DateTime.UtcNow;
Job[]? ret = null;
try
{
this.state = JobState.Running;
context.SaveChanges();
running = true;
ret = RunInternal(context).ToArray();
Log.Info($"Job {this} completed. Generated {ret.Length} new jobs.");
this.state = this.RecurrenceMs > 0 ? JobState.CompletedWaiting : JobState.Completed;
this.LastExecution = DateTime.UtcNow;
context.SaveChanges();
}
catch (Exception e)
{
if (e is not DbUpdateException)
{
Log.Error($"Failed to run job {this}", e);
this.state = JobState.Failed;
this.Enabled = false;
this.LastExecution = DateTime.UtcNow;
context.SaveChanges();
}
else
{
Log.Error($"Failed to update Database {this}", e);
}
}
try
{
if (ret != null)
{
context.Jobs.AddRange(ret);
context.SaveChanges();
}
}
catch (DbUpdateException e)
{
Log.Error($"Failed to update Database {this}", e);
}
Log.Info($"Finished Job {this}! (took {DateTime.UtcNow.Subtract(jobStart).TotalMilliseconds}ms)");
return ret ?? [];
}
protected abstract IEnumerable<Job> RunInternal(PgsqlContext context);
public List<Job> GetDependenciesAndSelf()
{
List<Job> ret = GetDependencies();
ret.Add(this);
return ret;
}
public List<Job> GetDependencies()
{
List<Job> ret = new ();
foreach (Job job in DependsOnJobs)
{
ret.AddRange(job.GetDependenciesAndSelf());
}
return ret;
}
public int CompareTo(Job? other)
{
if (other is null)
return -1;
// Sort by missing dependencies
if (this.GetDependencies().Count(job => !job.IsCompleted) <
other.GetDependencies().Count(job => !job.IsCompleted))
return -1;
// Sort by NextExecution-time
if (this.NextExecution < other.NextExecution)
return -1;
return 1;
}
public override string ToString() => base.ToString();
}

View File

@ -1,14 +0,0 @@
namespace API.Schema.Jobs;
public enum JobState : byte
{
//Values 0-63 Preparation Stages
FirstExecution = 0,
//64-127 Running Stages
Running = 64,
//128-191 Completion Stages
Completed = 128,
CompletedWaiting = 159,
//192-255 Error stages
Failed = 192
}

View File

@ -1,14 +0,0 @@
namespace API.Schema.Jobs;
public enum JobType : byte
{
DownloadSingleChapterJob = 0,
DownloadAvailableChaptersJob = 1,
MoveFileOrFolderJob = 3,
DownloadMangaCoverJob = 4,
RetrieveChaptersJob = 5,
UpdateChaptersDownloadedJob = 6,
MoveMangaLibraryJob = 7,
UpdateCoverJob = 9,
}

View File

@ -1,18 +0,0 @@
using Microsoft.EntityFrameworkCore.Infrastructure;
namespace API.Schema.Jobs;
public abstract class JobWithDownloading : Job
{
public JobWithDownloading(string key, JobType jobType, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(key, jobType, recurrenceMs, parentJob, dependsOnJobs)
{
}
public JobWithDownloading(ILazyLoader lazyLoader, string key, JobType jobType, ulong recurrenceMs, string? parentJobId)
: base(lazyLoader, key, jobType, recurrenceMs, parentJobId)
{
}
}

View File

@ -1,71 +0,0 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore.Infrastructure;
namespace API.Schema.Jobs;
public class MoveFileOrFolderJob : Job
{
[StringLength(256)]
[Required]
public string FromLocation { get; init; }
[StringLength(256)]
[Required]
public string ToLocation { get; init; }
public MoveFileOrFolderJob(string fromLocation, string toLocation, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(MoveFileOrFolderJob)), JobType.MoveFileOrFolderJob, 0, parentJob, dependsOnJobs)
{
this.FromLocation = fromLocation;
this.ToLocation = toLocation;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
internal MoveFileOrFolderJob(ILazyLoader lazyLoader, string key, ulong recurrenceMs, string fromLocation, string toLocation, string? parentJobId)
: base(lazyLoader, key, JobType.MoveFileOrFolderJob, recurrenceMs, parentJobId)
{
this.FromLocation = fromLocation;
this.ToLocation = toLocation;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
try
{
FileInfo fi = new (FromLocation);
if (!fi.Exists)
{
Log.Error($"File does not exist at {FromLocation}");
return [];
}
if (File.Exists(ToLocation))//Do not override existing
{
Log.Error($"File already exists at {ToLocation}");
return [];
}
if(fi.Attributes.HasFlag(FileAttributes.Directory))
MoveDirectory(fi, ToLocation);
else
MoveFile(fi, ToLocation);
}
catch (Exception e)
{
Log.Error(e);
}
return [];
}
private void MoveDirectory(FileInfo from, string toLocation)
{
Directory.Move(from.FullName, toLocation);
}
private void MoveFile(FileInfo from, string toLocation)
{
File.Move(from.FullName, toLocation);
}
}

View File

@ -1,72 +0,0 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
namespace API.Schema.Jobs;
public class MoveMangaLibraryJob : Job
{
[StringLength(64)] [Required] public string MangaId { get; init; } = null!;
private Manga? _manga;
[JsonIgnore]
public Manga Manga
{
get => LazyLoader.Load(this, ref _manga) ?? throw new InvalidOperationException();
init
{
MangaId = value.Key;
_manga = value;
}
}
[StringLength(64)] [Required] public string ToLibraryId { get; private set; } = null!;
private FileLibrary? _toFileLibrary;
[JsonIgnore]
public FileLibrary ToFileLibrary
{
get => LazyLoader.Load(this, ref _toFileLibrary) ?? throw new InvalidOperationException();
init
{
ToLibraryId = value.Key;
_toFileLibrary = value;
}
}
public MoveMangaLibraryJob(Manga manga, FileLibrary toFileLibrary, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(MoveMangaLibraryJob)), JobType.MoveMangaLibraryJob, 0, parentJob, dependsOnJobs)
{
this.Manga = manga;
this.ToFileLibrary = toFileLibrary;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
internal MoveMangaLibraryJob(ILazyLoader lazyLoader, string key, ulong recurrenceMs, string mangaId, string toLibraryId, string? parentJobId)
: base(lazyLoader, key, JobType.MoveMangaLibraryJob, recurrenceMs, parentJobId)
{
this.MangaId = mangaId;
this.ToLibraryId = toLibraryId;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
context.Entry(Manga).Reference<FileLibrary>(m => m.Library).Load();
Dictionary<Chapter, string> oldPath = Manga.Chapters.ToDictionary(c => c, c => c.FullArchiveFilePath);
Manga.Library = ToFileLibrary;
try
{
context.SaveChanges();
}
catch (DbUpdateException e)
{
Log.Error(e);
return [];
}
return Manga.Chapters.Select(c => new MoveFileOrFolderJob(oldPath[c], c.FullArchiveFilePath));
}
}

View File

@ -1,70 +0,0 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
namespace API.Schema.Jobs;
public class RetrieveChaptersJob : JobWithDownloading
{
[StringLength(64)] [Required] public string MangaId { get; init; } = null!;
private Manga? _manga;
[JsonIgnore]
public Manga Manga
{
get => LazyLoader.Load(this, ref _manga) ?? throw new InvalidOperationException();
init
{
MangaId = value.Key;
_manga = value;
}
}
[StringLength(8)] [Required] public string Language { get; private set; }
public RetrieveChaptersJob(Manga manga, string language, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(RetrieveChaptersJob)), JobType.RetrieveChaptersJob, recurrenceMs, parentJob, dependsOnJobs)
{
this.Manga = manga;
this.Language = language;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
internal RetrieveChaptersJob(ILazyLoader lazyLoader, string key, string mangaId, ulong recurrenceMs, string language, string? parentJobId)
: base(lazyLoader, key, JobType.RetrieveChaptersJob, recurrenceMs, parentJobId)
{
this.MangaId = mangaId;
this.Language = language;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
//TODO MangaConnector Selection
MangaConnectorId<Manga> mcId = Manga.MangaConnectorIds.First();
// This gets all chapters that are not downloaded
(Chapter, MangaConnectorId<Chapter>)[] allChapters = mcId.MangaConnector.GetChapters(mcId, Language).DistinctBy(c => c.Item1.Key).ToArray();
(Chapter, MangaConnectorId<Chapter>)[] newChapters = allChapters.Where(chapter => Manga.Chapters.Any(ch => chapter.Item1.Key == ch.Key && ch.Downloaded) == false).ToArray();
Log.Info($"{Manga.Chapters.Count} existing + {newChapters.Length} new chapters.");
try
{
foreach ((Chapter chapter, MangaConnectorId<Chapter> mcId) newChapter in newChapters)
{
Manga.Chapters.Add(newChapter.chapter);
context.MangaConnectorToChapter.Add(newChapter.mcId);
}
context.SaveChanges();
}
catch (DbUpdateException e)
{
Log.Error(e);
}
return [];
}
}

View File

@ -1,58 +0,0 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
namespace API.Schema.Jobs;
public class UpdateChaptersDownloadedJob : Job
{
[StringLength(64)] [Required] public string MangaId { get; init; } = null!;
private Manga? _manga;
[JsonIgnore]
public Manga Manga
{
get => LazyLoader.Load(this, ref _manga) ?? throw new InvalidOperationException();
init
{
MangaId = value.Key;
_manga = value;
}
}
public UpdateChaptersDownloadedJob(Manga manga, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(UpdateChaptersDownloadedJob)), JobType.UpdateChaptersDownloadedJob, recurrenceMs, parentJob, dependsOnJobs)
{
this.Manga = manga;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
internal UpdateChaptersDownloadedJob(ILazyLoader lazyLoader, string key, ulong recurrenceMs, string mangaId, string? parentJobId)
: base(lazyLoader, key, JobType.UpdateChaptersDownloadedJob, recurrenceMs, parentJobId)
{
this.MangaId = mangaId;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
context.Entry(Manga).Reference<FileLibrary>(m => m.Library).Load();
foreach (Chapter mangaChapter in Manga.Chapters)
{
mangaChapter.Downloaded = mangaChapter.CheckDownloaded();
}
try
{
context.SaveChanges();
}
catch (DbUpdateException e)
{
Log.Error(e);
}
return [];
}
}

View File

@ -1,67 +0,0 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
namespace API.Schema.Jobs;
public class UpdateCoverJob : Job
{
[StringLength(64)] [Required] public string MangaId { get; init; } = null!;
private Manga? _manga;
[JsonIgnore]
public Manga Manga
{
get => LazyLoader.Load(this, ref _manga) ?? throw new InvalidOperationException();
init
{
MangaId = value.Key;
_manga = value;
}
}
public UpdateCoverJob(Manga manga, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(UpdateCoverJob)), JobType.UpdateCoverJob, recurrenceMs, parentJob, dependsOnJobs)
{
this.Manga = manga;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
internal UpdateCoverJob(ILazyLoader lazyLoader, string key, string mangaId, ulong recurrenceMs, string? parentJobId)
: base(lazyLoader, key, JobType.UpdateCoverJob, recurrenceMs, parentJobId)
{
this.MangaId = mangaId;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
bool keepCover = context.Jobs
.Any(job => job.JobType == JobType.DownloadAvailableChaptersJob
&& ((DownloadAvailableChaptersJob)job).MangaId == MangaId);
if (!keepCover)
{
if(File.Exists(Manga.CoverFileNameInCache))
File.Delete(Manga.CoverFileNameInCache);
try
{
Manga.CoverFileNameInCache = null;
context.Jobs.Remove(this);
context.SaveChanges();
}
catch (DbUpdateException e)
{
Log.Error(e);
}
}
else
{
return [new DownloadMangaCoverJob(Manga, this)];
}
return [];
}
}

View File

@ -1,7 +1,7 @@
using System.Text.Json;
using System.Text.Json.Nodes;
namespace API.Schema.LibraryConnectors;
namespace API.Schema.LibraryContext.LibraryConnectors;
public class Kavita : LibraryConnector
{

View File

@ -1,7 +1,7 @@
using System.Text.Json;
using System.Text.Json.Nodes;
namespace API.Schema.LibraryConnectors;
namespace API.Schema.LibraryContext.LibraryConnectors;
public class Komga : LibraryConnector
{

View File

@ -4,7 +4,7 @@ using log4net;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
namespace API.Schema.LibraryConnectors;
namespace API.Schema.LibraryContext.LibraryConnectors;
[PrimaryKey("LibraryConnectorId")]
public abstract class LibraryConnector(string libraryConnectorId, LibraryType libraryType, string baseUrl, string auth)

View File

@ -1,4 +1,4 @@
namespace API.Schema.LibraryConnectors;
namespace API.Schema.LibraryContext.LibraryConnectors;
public enum LibraryType : byte
{

View File

@ -2,7 +2,7 @@
using System.Net.Http.Headers;
using log4net;
namespace API.Schema.LibraryConnectors;
namespace API.Schema.LibraryContext.LibraryConnectors;
public class NetClient
{

View File

@ -1,9 +1,7 @@
using API.Schema.LibraryConnectors;
using log4net;
using API.Schema.LibraryContext.LibraryConnectors;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Diagnostics;
namespace API.Schema.Contexts;
namespace API.Schema.LibraryContext;
public class LibraryContext(DbContextOptions<LibraryContext> options) : TrangaBaseContext<LibraryContext>(options)
{

View File

@ -1,7 +1,7 @@
using System.ComponentModel.DataAnnotations;
using Microsoft.EntityFrameworkCore;
namespace API.Schema;
namespace API.Schema.MangaContext;
[PrimaryKey("Key")]
public class AltTitle(string language, string title) : Identifiable(TokenGen.CreateToken("AltTitle"))

View File

@ -1,7 +1,7 @@
using System.ComponentModel.DataAnnotations;
using Microsoft.EntityFrameworkCore;
namespace API.Schema;
namespace API.Schema.MangaContext;
[PrimaryKey("Key")]
public class Author(string authorName) : Identifiable(TokenGen.CreateToken(typeof(Author), authorName))

View File

@ -7,7 +7,7 @@ using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
namespace API.Schema;
namespace API.Schema.MangaContext;
[PrimaryKey("Key")]
public class Chapter : Identifiable, IComparable<Chapter>

View File

@ -1,7 +1,7 @@
using System.ComponentModel.DataAnnotations;
using Microsoft.EntityFrameworkCore;
namespace API.Schema;
namespace API.Schema.MangaContext;
[PrimaryKey("Key")]
public class FileLibrary(string basePath, string libraryName)

View File

@ -1,7 +1,7 @@
using System.ComponentModel.DataAnnotations;
using Microsoft.EntityFrameworkCore;
namespace API.Schema;
namespace API.Schema.MangaContext;
[PrimaryKey("Key")]
public class Link(string linkProvider, string linkUrl) : Identifiable(TokenGen.CreateToken(typeof(Link), linkProvider, linkUrl))

View File

@ -2,14 +2,13 @@ using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Runtime.InteropServices;
using System.Text;
using API.Schema.Contexts;
using API.Schema.Jobs;
using API.Workers;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
using static System.IO.UnixFileMode;
namespace API.Schema;
namespace API.Schema.MangaContext;
[PrimaryKey("Key")]
public class Manga : Identifiable
@ -157,42 +156,31 @@ public class Manga : Identifiable
}
/// <summary>
///
/// Merges another Manga (MangaConnectorIds and Chapters)
/// </summary>
/// <param name="other"></param>
/// <param name="context"></param>
/// <exception cref="DbUpdateException"></exception>
public void MergeFrom(Manga other, PgsqlContext context)
/// <param name="other">The other <see cref="Manga" /> to merge</param>
/// <param name="context"><see cref="MangaContext"/> to use for Database operations</param>
/// <returns>An array of <see cref="MoveFileOrFolderWorker"/> for moving <see cref="Chapter"/> to new Directory</returns>
public BaseWorker[] MergeFrom(Manga other, MangaContext context)
{
try
context.Mangas.Remove(other);
List<BaseWorker> newJobs = new();
this.MangaConnectorIds = this.MangaConnectorIds
.UnionBy(other.MangaConnectorIds, id => id.MangaConnectorName)
.ToList();
foreach (Chapter otherChapter in other.Chapters)
{
context.Mangas.Remove(other);
List<Job> newJobs = new();
this.MangaConnectorIds = this.MangaConnectorIds
.UnionBy(other.MangaConnectorIds, id => id.MangaConnectorName)
.ToList();
foreach (Chapter otherChapter in other.Chapters)
{
string oldPath = otherChapter.FullArchiveFilePath;
Chapter newChapter = new(this, otherChapter.ChapterNumber, otherChapter.VolumeNumber,
otherChapter.Title);
this.Chapters.Add(newChapter);
string newPath = newChapter.FullArchiveFilePath;
newJobs.Add(new MoveFileOrFolderJob(oldPath, newPath));
}
if (other.Chapters.Count > 0)
newJobs.Add(new UpdateChaptersDownloadedJob(this, 0, null, newJobs));
context.Jobs.AddRange(newJobs);
context.SaveChanges();
}
catch (DbUpdateException e)
{
throw new DbUpdateException(e.Message, e.InnerException, e.Entries);
string oldPath = otherChapter.FullArchiveFilePath;
Chapter newChapter = new(this, otherChapter.ChapterNumber, otherChapter.VolumeNumber,
otherChapter.Title);
this.Chapters.Add(newChapter);
string newPath = newChapter.FullArchiveFilePath;
newJobs.Add(new MoveFileOrFolderWorker(newPath, oldPath));
}
return newJobs.ToArray();
}
public override string ToString() => $"{base.ToString()} {Name}";

View File

@ -1,10 +1,10 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.MangaConnectors;
using API.Schema.MangaContext.MangaConnectors;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
namespace API.Schema;
namespace API.Schema.MangaContext;
[PrimaryKey("Key")]
public class MangaConnectorId<T> : Identifiable where T : Identifiable

View File

@ -2,7 +2,7 @@
using API.MangaDownloadClients;
using Newtonsoft.Json.Linq;
namespace API.Schema.MangaConnectors;
namespace API.Schema.MangaContext.MangaConnectors;
public class ComickIo : MangaConnector
{

View File

@ -1,11 +1,9 @@
using API.Schema.Contexts;
namespace API.Schema.MangaConnectors;
namespace API.Schema.MangaContext.MangaConnectors;
public class Global : MangaConnector
{
private PgsqlContext context { get; init; }
public Global(PgsqlContext context) : base("Global", ["all"], [""], "")
private MangaContext context { get; init; }
public Global(MangaContext context) : base("Global", ["all"], [""], "")
{
this.context = context;
}

View File

@ -6,7 +6,7 @@ using log4net;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
namespace API.Schema.MangaConnectors;
namespace API.Schema.MangaContext.MangaConnectors;
[PrimaryKey("Name")]
public abstract class MangaConnector(string name, string[] supportedLanguages, string[] baseUris, string iconUrl)

View File

@ -2,7 +2,7 @@
using API.MangaDownloadClients;
using Newtonsoft.Json.Linq;
namespace API.Schema.MangaConnectors;
namespace API.Schema.MangaContext.MangaConnectors;
public class MangaDex : MangaConnector
{

View File

@ -1,15 +1,11 @@
using API.Schema.Jobs;
using API.Schema.MangaConnectors;
using API.Schema.MetadataFetchers;
using log4net;
using API.Schema.MangaContext.MangaConnectors;
using API.Schema.MangaContext.MetadataFetchers;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Diagnostics;
namespace API.Schema.Contexts;
namespace API.Schema.MangaContext;
public class PgsqlContext(DbContextOptions<PgsqlContext> options) : TrangaBaseContext<PgsqlContext>(options)
public class MangaContext(DbContextOptions<MangaContext> options) : TrangaBaseContext<MangaContext>(options)
{
public DbSet<Job> Jobs { get; set; }
public DbSet<MangaConnector> MangaConnectors { get; set; }
public DbSet<Manga> Mangas { get; set; }
public DbSet<FileLibrary> LocalLibraries { get; set; }
@ -22,92 +18,6 @@ public class PgsqlContext(DbContextOptions<PgsqlContext> options) : TrangaBaseCo
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
//Job Types
modelBuilder.Entity<Job>()
.HasDiscriminator(j => j.JobType)
.HasValue<MoveFileOrFolderJob>(JobType.MoveFileOrFolderJob)
.HasValue<MoveMangaLibraryJob>(JobType.MoveMangaLibraryJob)
.HasValue<DownloadAvailableChaptersJob>(JobType.DownloadAvailableChaptersJob)
.HasValue<DownloadSingleChapterJob>(JobType.DownloadSingleChapterJob)
.HasValue<DownloadMangaCoverJob>(JobType.DownloadMangaCoverJob)
.HasValue<RetrieveChaptersJob>(JobType.RetrieveChaptersJob)
.HasValue<UpdateCoverJob>(JobType.UpdateCoverJob)
.HasValue<UpdateChaptersDownloadedJob>(JobType.UpdateChaptersDownloadedJob);
modelBuilder.Entity<DownloadAvailableChaptersJob>()
.HasOne<Manga>(j => j.Manga)
.WithMany()
.HasForeignKey(j => j.MangaId)
.OnDelete(DeleteBehavior.Cascade);
modelBuilder.Entity<DownloadAvailableChaptersJob>()
.Navigation(j => j.Manga)
.EnableLazyLoading();
modelBuilder.Entity<DownloadMangaCoverJob>()
.HasOne<Manga>(j => j.Manga)
.WithMany()
.HasForeignKey(j => j.MangaId)
.OnDelete(DeleteBehavior.Cascade);
modelBuilder.Entity<DownloadMangaCoverJob>()
.Navigation(j => j.Manga)
.EnableLazyLoading();
modelBuilder.Entity<DownloadSingleChapterJob>()
.HasOne<Chapter>(j => j.Chapter)
.WithMany()
.HasForeignKey(j => j.ChapterId)
.OnDelete(DeleteBehavior.Cascade);
modelBuilder.Entity<DownloadSingleChapterJob>()
.Navigation(j => j.Chapter)
.EnableLazyLoading();
modelBuilder.Entity<MoveMangaLibraryJob>()
.HasOne<Manga>(j => j.Manga)
.WithMany()
.HasForeignKey(j => j.MangaId)
.OnDelete(DeleteBehavior.Cascade);
modelBuilder.Entity<MoveMangaLibraryJob>()
.Navigation(j => j.Manga)
.EnableLazyLoading();
modelBuilder.Entity<MoveMangaLibraryJob>()
.HasOne<FileLibrary>(j => j.ToFileLibrary)
.WithMany()
.HasForeignKey(j => j.ToLibraryId)
.OnDelete(DeleteBehavior.Cascade);
modelBuilder.Entity<MoveMangaLibraryJob>()
.Navigation(j => j.ToFileLibrary)
.EnableLazyLoading();
modelBuilder.Entity<RetrieveChaptersJob>()
.HasOne<Manga>(j => j.Manga)
.WithMany()
.HasForeignKey(j => j.MangaId)
.OnDelete(DeleteBehavior.Cascade);
modelBuilder.Entity<RetrieveChaptersJob>()
.Navigation(j => j.Manga)
.EnableLazyLoading();
modelBuilder.Entity<UpdateChaptersDownloadedJob>()
.HasOne<Manga>(j => j.Manga)
.WithMany()
.HasForeignKey(j => j.MangaId)
.OnDelete(DeleteBehavior.Cascade);
modelBuilder.Entity<UpdateChaptersDownloadedJob>()
.Navigation(j => j.Manga)
.EnableLazyLoading();
//Job has possible ParentJob
modelBuilder.Entity<Job>()
.HasOne<Job>(childJob => childJob.ParentJob)
.WithMany()
.HasForeignKey(childJob => childJob.ParentJobId)
.OnDelete(DeleteBehavior.Cascade);
modelBuilder.Entity<Job>()
.Navigation(childJob => childJob.ParentJob)
.EnableLazyLoading();
//Job might be dependent on other Jobs
modelBuilder.Entity<Job>()
.HasMany<Job>(root => root.DependsOnJobs)
.WithMany();
modelBuilder.Entity<Job>()
.Navigation(j => j.DependsOnJobs)
.EnableLazyLoading();
//MangaConnector Types
modelBuilder.Entity<MangaConnector>()
.HasDiscriminator(c => c.Name)

View File

@ -1,7 +1,7 @@
using System.ComponentModel.DataAnnotations;
using Microsoft.EntityFrameworkCore;
namespace API.Schema;
namespace API.Schema.MangaContext;
[PrimaryKey("Tag")]
public class MangaTag(string tag)

View File

@ -1,7 +1,7 @@
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
namespace API.Schema.MetadataFetchers;
namespace API.Schema.MangaContext.MetadataFetchers;
[PrimaryKey("MetadataFetcherName", "Identifier")]
public class MetadataEntry

View File

@ -1,8 +1,6 @@
using System.Diagnostics.CodeAnalysis;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore;
namespace API.Schema.MetadataFetchers;
namespace API.Schema.MangaContext.MetadataFetchers;
[PrimaryKey("MetadataFetcherName")]
public abstract class MetadataFetcher
@ -33,5 +31,5 @@ public abstract class MetadataFetcher
/// <summary>
/// Updates the Manga linked in the MetadataEntry
/// </summary>
public abstract void UpdateMetadata(MetadataEntry metadataEntry, PgsqlContext dbContext);
public abstract void UpdateMetadata(MetadataEntry metadataEntry, MangaContext dbContext);
}

View File

@ -1,3 +1,3 @@
namespace API.Schema.MetadataFetchers;
namespace API.Schema.MangaContext.MetadataFetchers;
public record MetadataSearchResult(string Identifier, string Name, string Url, string? Description = null, string? CoverUrl = null);

View File

@ -1,9 +1,8 @@
using System.Text.RegularExpressions;
using API.Schema.Contexts;
using JikanDotNet;
using Microsoft.EntityFrameworkCore;
namespace API.Schema.MetadataFetchers;
namespace API.Schema.MangaContext.MetadataFetchers;
public class MyAnimeList : MetadataFetcher
{
@ -45,7 +44,7 @@ public class MyAnimeList : MetadataFetcher
/// <param name="dbContext"></param>
/// <exception cref="FormatException"></exception>
/// <exception cref="DbUpdateException"></exception>
public override void UpdateMetadata(MetadataEntry metadataEntry, PgsqlContext dbContext)
public override void UpdateMetadata(MetadataEntry metadataEntry, MangaContext dbContext)
{
Manga dbManga = dbContext.Mangas.Find(metadataEntry.MangaId)!;
MangaFull resultData;

View File

@ -1,7 +1,7 @@
using System.ComponentModel.DataAnnotations;
using Microsoft.EntityFrameworkCore;
namespace API.Schema;
namespace API.Schema.NotificationsContext;
[PrimaryKey(nameof(Key))]
public class Notification : Identifiable

View File

@ -5,7 +5,7 @@ using log4net;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
namespace API.Schema.NotificationConnectors;
namespace API.Schema.NotificationsContext.NotificationConnectors;
[PrimaryKey("Name")]
public class NotificationConnector(string name, string url, Dictionary<string, string> headers, string httpMethod, string body)

View File

@ -1,9 +1,7 @@
using API.Schema.NotificationConnectors;
using log4net;
using API.Schema.NotificationsContext.NotificationConnectors;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Diagnostics;
namespace API.Schema.Contexts;
namespace API.Schema.NotificationsContext;
public class NotificationsContext(DbContextOptions<NotificationsContext> options) : TrangaBaseContext<NotificationsContext>(options)
{

View File

@ -0,0 +1,38 @@
using log4net;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Diagnostics;
namespace API.Schema;
public abstract class TrangaBaseContext<T> : DbContext where T : DbContext
{
private ILog Log { get; init; }
protected TrangaBaseContext(DbContextOptions<T> options) : base(options)
{
this.Log = LogManager.GetLogger(GetType());
}
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
base.OnConfiguring(optionsBuilder);
optionsBuilder.LogTo(s =>
{
Log.Debug(s);
}, Array.Empty<string>(), LogLevel.Warning, DbContextLoggerOptions.Level | DbContextLoggerOptions.Category | DbContextLoggerOptions.UtcTime);
}
internal async Task<string?> Sync()
{
try
{
await this.SaveChangesAsync();
return null;
}
catch (Exception e)
{
Log.Error(null, e);
return e.Message;
}
}
}