WIP: Manga can be linked to multiple Connectors

This commit is contained in:
2025-06-30 14:24:17 +02:00
parent e5937d2654
commit e9d9bebcd7
15 changed files with 349 additions and 194 deletions

47
API/JobQueueSortable.cs Normal file
View File

@ -0,0 +1,47 @@
using API.Schema.Jobs;
namespace API;
internal static class JobQueueSorter
{
public static readonly Dictionary<JobType, byte> JobTypePriority = new()
{
{ JobType.DownloadSingleChapterJob, 50 },
{ JobType.DownloadAvailableChaptersJob, 51 },
{ JobType.MoveFileOrFolderJob, 102 },
{ JobType.DownloadMangaCoverJob, 10 },
{ JobType.RetrieveChaptersJob, 52 },
{ JobType.UpdateChaptersDownloadedJob, 90 },
{ JobType.MoveMangaLibraryJob, 101 },
{ JobType.UpdateCoverJob, 11 },
};
public static byte GetPriority(Job job)
{
return JobTypePriority[job.JobType];
}
public static byte GetPriority(JobType jobType)
{
return JobTypePriority[jobType];
}
public static IEnumerable<Job> Sort(this IEnumerable<Job> jobQueueSortables)
{
return jobQueueSortables.Order();
}
public static IEnumerable<Job> GetStartableJobs(this IEnumerable<Job> jobQueueSortables)
{
Job[] sorted = jobQueueSortables.Order().ToArray();
// Job has to be due, no missing dependenices
// Index - 1, Index is first job that does not match requirements
IEnumerable<(int Index, Job Item)> index = sorted.Index();
(int i, Job? item) = index.FirstOrDefault(job =>
job.Item.NextExecution > DateTime.UtcNow || job.Item.GetDependencies().Any(j => !j.IsCompleted));
if (item is null)
return sorted;
index.
}
}

View File

@ -4,6 +4,7 @@ using System.Text;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
namespace API.Schema;
@ -14,8 +15,14 @@ public class Chapter : IComparable<Chapter>
[StringLength(64)] [Required] public string ChapterId { get; init; }
[StringLength(256)]public string? IdOnConnectorSite { get; init; }
public string ParentMangaId { get; init; }
[JsonIgnore] public Manga ParentManga { get; init; } = null!;
private MangaConnectorMangaEntry? _mangaConnectorMangaEntry = null!;
[JsonIgnore]
public MangaConnectorMangaEntry MangaConnectorMangaEntry
{
get => _lazyLoader.Load(this, ref _mangaConnectorMangaEntry) ?? throw new InvalidOperationException();
init => _mangaConnectorMangaEntry = value;
}
public int? VolumeNumber { get; private set; }
[StringLength(10)] [Required] public string ChapterNumber { get; private set; }
@ -27,14 +34,15 @@ public class Chapter : IComparable<Chapter>
[StringLength(256)] [Required] public string FileName { get; private set; }
[Required] public bool Downloaded { get; internal set; }
[NotMapped] public string FullArchiveFilePath => Path.Join(ParentManga.FullDirectoryPath, FileName);
[NotMapped] public string FullArchiveFilePath => Path.Join(MangaConnectorMangaEntry.Manga.FullDirectoryPath, FileName);
public Chapter(Manga parentManga, string url, string chapterNumber, int? volumeNumber = null, string? idOnConnectorSite = null, string? title = null)
private readonly ILazyLoader _lazyLoader = null!;
public Chapter(MangaConnectorMangaEntry mangaConnectorMangaEntry, string url, string chapterNumber, int? volumeNumber = null, string? idOnConnectorSite = null, string? title = null)
{
this.ChapterId = TokenGen.CreateToken(typeof(Chapter), parentManga.MangaId, chapterNumber);
this.ChapterId = TokenGen.CreateToken(typeof(Chapter), mangaConnectorMangaEntry.MangaId, chapterNumber);
this.MangaConnectorMangaEntry = mangaConnectorMangaEntry;
this.IdOnConnectorSite = idOnConnectorSite;
this.ParentMangaId = parentManga.MangaId;
this.ParentManga = parentManga;
this.VolumeNumber = volumeNumber;
this.ChapterNumber = chapterNumber;
this.Url = url;
@ -46,11 +54,11 @@ public class Chapter : IComparable<Chapter>
/// <summary>
/// EF ONLY!!!
/// </summary>
internal Chapter(string chapterId, string parentMangaId, int? volumeNumber, string chapterNumber, string url, string? idOnConnectorSite, string? title, string fileName, bool downloaded)
internal Chapter(ILazyLoader lazyLoader, string chapterId, int? volumeNumber, string chapterNumber, string url, string? idOnConnectorSite, string? title, string fileName, bool downloaded)
{
this._lazyLoader = lazyLoader;
this.ChapterId = chapterId;
this.IdOnConnectorSite = idOnConnectorSite;
this.ParentMangaId = parentMangaId;
this.VolumeNumber = volumeNumber;
this.ChapterNumber = chapterNumber;
this.Url = url;
@ -103,14 +111,14 @@ public class Chapter : IComparable<Chapter>
char placeholder = nullable.Groups[1].Value[0];
bool isNull = placeholder switch
{
'M' => ParentManga?.Name is null,
'M' => MangaConnectorMangaEntry.Manga?.Name is null,
'V' => VolumeNumber is null,
'C' => ChapterNumber is null,
'T' => Title is null,
'A' => ParentManga?.Authors?.FirstOrDefault()?.AuthorName is null,
'A' => MangaConnectorMangaEntry.Manga?.Authors?.FirstOrDefault()?.AuthorName is null,
'I' => ChapterId is null,
'i' => ParentManga?.MangaId is null,
'Y' => ParentManga?.Year is null,
'i' => MangaConnectorMangaEntry.Manga?.MangaId is null,
'Y' => MangaConnectorMangaEntry.Manga?.Year is null,
_ => true
};
if(!isNull)
@ -131,14 +139,14 @@ public class Chapter : IComparable<Chapter>
char placeholder = replace.Groups[1].Value[0];
string? value = placeholder switch
{
'M' => ParentManga?.Name,
'M' => MangaConnectorMangaEntry.Manga?.Name,
'V' => VolumeNumber?.ToString(),
'C' => ChapterNumber,
'T' => Title,
'A' => ParentManga?.Authors?.FirstOrDefault()?.AuthorName,
'A' => MangaConnectorMangaEntry.Manga?.Authors?.FirstOrDefault()?.AuthorName,
'I' => ChapterId,
'i' => ParentManga?.MangaId,
'Y' => ParentManga?.Year.ToString(),
'i' => MangaConnectorMangaEntry.Manga?.MangaId,
'Y' => MangaConnectorMangaEntry.Manga?.Year.ToString(),
_ => null
};
stringBuilder.Append(value);
@ -179,16 +187,16 @@ public class Chapter : IComparable<Chapter>
);
if(Title is not null)
comicInfo.Add(new XElement("Title", Title));
if(ParentManga.MangaTags.Count > 0)
comicInfo.Add(new XElement("Tags", string.Join(',', ParentManga.MangaTags.Select(tag => tag.Tag))));
if(MangaConnectorMangaEntry.Manga.MangaTags.Count > 0)
comicInfo.Add(new XElement("Tags", string.Join(',', MangaConnectorMangaEntry.Manga.MangaTags.Select(tag => tag.Tag))));
if(VolumeNumber is not null)
comicInfo.Add(new XElement("Volume", VolumeNumber));
if(ParentManga.Authors.Count > 0)
comicInfo.Add(new XElement("Writer", string.Join(',', ParentManga.Authors.Select(author => author.AuthorName))));
if(ParentManga.OriginalLanguage is not null)
comicInfo.Add(new XElement("LanguageISO", ParentManga.OriginalLanguage));
if(ParentManga.Description != string.Empty)
comicInfo.Add(new XElement("Summary", ParentManga.Description));
if(MangaConnectorMangaEntry.Manga.Authors.Count > 0)
comicInfo.Add(new XElement("Writer", string.Join(',', MangaConnectorMangaEntry.Manga.Authors.Select(author => author.AuthorName))));
if(MangaConnectorMangaEntry.Manga.OriginalLanguage is not null)
comicInfo.Add(new XElement("LanguageISO", MangaConnectorMangaEntry.Manga.OriginalLanguage));
if(MangaConnectorMangaEntry.Manga.Description != string.Empty)
comicInfo.Add(new XElement("Summary", MangaConnectorMangaEntry.Manga.Description));
return comicInfo.ToString();
}

View File

@ -1,42 +1,36 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.Contexts;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
namespace API.Schema.Jobs;
public class DownloadAvailableChaptersJob : Job
public class DownloadAvailableChaptersJob : JobWithDownloading
{
[StringLength(64)] [Required] public string MangaId { get; init; }
private Manga _manga = null!;
private MangaConnectorMangaEntry? _mangaConnectorMangaEntry = null!;
[JsonIgnore]
public Manga Manga
public MangaConnectorMangaEntry MangaConnectorMangaEntry
{
get => LazyLoader.Load(this, ref _manga);
init => _manga = value;
get => LazyLoader.Load(this, ref _mangaConnectorMangaEntry) ?? throw new InvalidOperationException();
init => _mangaConnectorMangaEntry = value;
}
public DownloadAvailableChaptersJob(Manga manga, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(DownloadAvailableChaptersJob)), JobType.DownloadAvailableChaptersJob, recurrenceMs, parentJob, dependsOnJobs)
public DownloadAvailableChaptersJob(MangaConnectorMangaEntry mangaConnectorMangaEntry, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(DownloadAvailableChaptersJob)), JobType.DownloadAvailableChaptersJob, recurrenceMs, mangaConnectorMangaEntry.MangaConnector, parentJob, dependsOnJobs)
{
this.MangaId = manga.MangaId;
this.Manga = manga;
this.MangaConnectorMangaEntry = mangaConnectorMangaEntry;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
internal DownloadAvailableChaptersJob(ILazyLoader lazyLoader, string jobId, ulong recurrenceMs, string mangaId, string? parentJobId)
: base(lazyLoader, jobId, JobType.DownloadAvailableChaptersJob, recurrenceMs, parentJobId)
internal DownloadAvailableChaptersJob(ILazyLoader lazyLoader, string jobId, ulong recurrenceMs, string mangaConnectorName, string? parentJobId)
: base(lazyLoader, jobId, JobType.DownloadAvailableChaptersJob, recurrenceMs, mangaConnectorName, parentJobId)
{
this.MangaId = mangaId;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
context.Entry(Manga).Reference<LocalLibrary>(m => m.Library).Load();
return Manga.Chapters.Where(c => c.Downloaded == false).Select(chapter => new DownloadSingleChapterJob(chapter, this));
return MangaConnectorMangaEntry.Manga.Chapters.Where(c => c.Downloaded == false).Select(chapter => new DownloadSingleChapterJob(chapter, this.MangaConnectorMangaEntry));
}
}

View File

@ -6,40 +6,36 @@ using Newtonsoft.Json;
namespace API.Schema.Jobs;
public class DownloadMangaCoverJob : Job
public class DownloadMangaCoverJob : JobWithDownloading
{
[StringLength(64)] [Required] public string MangaId { get; init; }
private Manga _manga = null!;
private MangaConnectorMangaEntry? _mangaConnectorMangaEntry = null!;
[JsonIgnore]
public Manga Manga
public MangaConnectorMangaEntry MangaConnectorMangaEntry
{
get => LazyLoader.Load(this, ref _manga);
init => _manga = value;
get => LazyLoader.Load(this, ref _mangaConnectorMangaEntry) ?? throw new InvalidOperationException();
init => _mangaConnectorMangaEntry = value;
}
public DownloadMangaCoverJob(Manga manga, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(DownloadMangaCoverJob)), JobType.DownloadMangaCoverJob, 0, parentJob, dependsOnJobs)
public DownloadMangaCoverJob(MangaConnectorMangaEntry mangaConnectorEntry, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(DownloadMangaCoverJob)), JobType.DownloadMangaCoverJob, 0, mangaConnectorEntry.MangaConnector, parentJob, dependsOnJobs)
{
this.MangaId = manga.MangaId;
this.Manga = manga;
this.MangaConnectorMangaEntry = mangaConnectorEntry;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
internal DownloadMangaCoverJob(ILazyLoader lazyLoader, string jobId, ulong recurrenceMs, string mangaId, string? parentJobId)
: base(lazyLoader, jobId, JobType.DownloadMangaCoverJob, recurrenceMs, parentJobId)
internal DownloadMangaCoverJob(ILazyLoader lazyLoader, string jobId, ulong recurrenceMs, string mangaConnectorName, string? parentJobId)
: base(lazyLoader, jobId, JobType.DownloadMangaCoverJob, recurrenceMs, mangaConnectorName, parentJobId)
{
this.MangaId = mangaId;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
try
{
Manga.CoverFileNameInCache = Manga.MangaConnector.SaveCoverImageToCache(Manga);
MangaConnectorMangaEntry.Manga.CoverFileNameInCache = MangaConnectorMangaEntry.MangaConnector.SaveCoverImageToCache(MangaConnectorMangaEntry.Manga);
context.SaveChanges();
}
catch (DbUpdateException e)

View File

@ -3,7 +3,6 @@ using System.IO.Compression;
using System.Runtime.InteropServices;
using API.MangaDownloadClients;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
using SixLabors.ImageSharp;
@ -14,31 +13,42 @@ using static System.IO.UnixFileMode;
namespace API.Schema.Jobs;
public class DownloadSingleChapterJob : Job
public class DownloadSingleChapterJob : JobWithDownloading
{
[StringLength(64)] [Required] public string ChapterId { get; init; }
private Chapter _chapter = null!;
[StringLength(64)] [Required] public string ChapterId { get; init; } = null!;
private Chapter? _chapter = null!;
[JsonIgnore]
public Chapter Chapter
{
get => LazyLoader.Load(this, ref _chapter);
init => _chapter = value;
get => LazyLoader.Load(this, ref _chapter) ?? throw new InvalidOperationException();
init
{
ChapterId = value.ChapterId;
_chapter = value;
}
}
public DownloadSingleChapterJob(Chapter chapter, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(DownloadSingleChapterJob)), JobType.DownloadSingleChapterJob, 0, parentJob, dependsOnJobs)
private MangaConnectorMangaEntry? _mangaConnectorMangaEntry = null!;
[JsonIgnore]
public MangaConnectorMangaEntry MangaConnectorMangaEntry
{
get => LazyLoader.Load(this, ref _mangaConnectorMangaEntry) ?? throw new InvalidOperationException();
init => _mangaConnectorMangaEntry = value;
}
public DownloadSingleChapterJob(Chapter chapter, MangaConnectorMangaEntry mangaConnectorMangaEntry, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(DownloadSingleChapterJob)), JobType.DownloadSingleChapterJob, 0, mangaConnectorMangaEntry.MangaConnector, parentJob, dependsOnJobs)
{
this.ChapterId = chapter.ChapterId;
this.Chapter = chapter;
this.MangaConnectorMangaEntry = mangaConnectorMangaEntry;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
internal DownloadSingleChapterJob(ILazyLoader lazyLoader, string jobId, ulong recurrenceMs, string chapterId, string? parentJobId)
: base(lazyLoader, jobId, JobType.DownloadSingleChapterJob, recurrenceMs, parentJobId)
internal DownloadSingleChapterJob(ILazyLoader lazyLoader, string jobId, ulong recurrenceMs, string mangaConnectorName, string chapterId, string? parentJobId)
: base(lazyLoader, jobId, JobType.DownloadSingleChapterJob, recurrenceMs, mangaConnectorName, parentJobId)
{
this.ChapterId = chapterId;
}
@ -50,13 +60,13 @@ public class DownloadSingleChapterJob : Job
Log.Info("Chapter was already downloaded.");
return [];
}
string[] imageUrls = Chapter.ParentManga.MangaConnector.GetChapterImageUrls(Chapter);
string[] imageUrls = MangaConnectorMangaEntry.MangaConnector.GetChapterImageUrls(Chapter);
if (imageUrls.Length < 1)
{
Log.Info($"No imageUrls for chapter {ChapterId}");
return [];
}
context.Entry(Chapter.ParentManga).Reference<LocalLibrary>(m => m.Library).Load(); //Need to explicitly load, because we are not accessing navigation directly...
context.Entry(Chapter.MangaConnectorMangaEntry.Manga).Reference<LocalLibrary>(m => m.Library).Load(); //Need to explicitly load, because we are not accessing navigation directly...
string saveArchiveFilePath = Chapter.FullArchiveFilePath;
Log.Debug($"Chapter path: {saveArchiveFilePath}");
@ -103,7 +113,7 @@ public class DownloadSingleChapterJob : Job
}
}
CopyCoverFromCacheToDownloadLocation(Chapter.ParentManga);
CopyCoverFromCacheToDownloadLocation(Chapter.MangaConnectorMangaEntry.Manga);
Log.Debug($"Creating ComicInfo.xml {ChapterId}");
File.WriteAllText(Path.Join(tempFolder, "ComicInfo.xml"), Chapter.GetComicInfoXmlString());
@ -123,11 +133,11 @@ public class DownloadSingleChapterJob : Job
if (j.JobType != JobType.UpdateChaptersDownloadedJob)
return false;
UpdateChaptersDownloadedJob job = (UpdateChaptersDownloadedJob)j;
return job.MangaId == this.Chapter.ParentMangaId;
return job.MangaId == this.Chapter.MangaConnectorMangaEntry.MangaId;
}))
return [];
return [new UpdateChaptersDownloadedJob(Chapter.ParentManga, 0, this.ParentJob)];
return [new UpdateChaptersDownloadedJob(Chapter.MangaConnectorMangaEntry.Manga, 0, this.ParentJob)];
}
private void ProcessImage(string imagePath)
@ -180,7 +190,7 @@ public class DownloadSingleChapterJob : Job
}
Log.Info($"Copying cover to {publicationFolder}");
string? fileInCache = manga.CoverFileNameInCache ?? manga.MangaConnector.SaveCoverImageToCache(manga);
string? fileInCache = manga.CoverFileNameInCache ?? MangaConnectorMangaEntry.MangaConnector.SaveCoverImageToCache(manga);
if (fileInCache is null)
{
Log.Error($"File {fileInCache} does not exist");

View File

@ -0,0 +1,37 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.MangaConnectors;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
namespace API.Schema.Jobs;
public abstract class JobWithDownloading : Job
{
[StringLength(32)] [Required] public string MangaConnectorName { get; private set; } = null!;
[JsonIgnore] private MangaConnector? _mangaConnector;
[JsonIgnore]
public MangaConnector MangaConnector
{
get => LazyLoader.Load(this, ref _mangaConnector) ?? throw new InvalidOperationException();
init
{
MangaConnectorName = value.Name;
_mangaConnector = value;
}
}
protected JobWithDownloading(string jobId, JobType jobType, ulong recurrenceMs, MangaConnector mangaConnector, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(jobId, jobType, recurrenceMs, parentJob, dependsOnJobs)
{
this.MangaConnector = mangaConnector;
}
/// <summary>
/// EF CORE ONLY!!!
/// </summary>
internal JobWithDownloading(ILazyLoader lazyLoader, string jobId, JobType jobType, ulong recurrenceMs, string mangaConnectorName, string? parentJobId)
: base(lazyLoader, jobId, jobType, recurrenceMs, parentJobId)
{
this.MangaConnectorName = mangaConnectorName;
}
}

View File

@ -10,23 +10,33 @@ public class MoveMangaLibraryJob : Job
{
[StringLength(64)] [Required] public string MangaId { get; init; }
private Manga _manga = null!;
private Manga? _manga = null!;
[JsonIgnore]
public Manga Manga
{
get => LazyLoader.Load(this, ref _manga);
get => LazyLoader.Load(this, ref _manga) ?? throw new InvalidOperationException();
init => _manga = value;
}
[StringLength(64)] [Required] public string ToLibraryId { get; init; }
public LocalLibrary ToLibrary { get; init; } = null!;
[StringLength(64)] [Required] public string ToLibraryId { get; private set; } = null!;
private LocalLibrary? _toLibrary = null!;
[JsonIgnore]
public LocalLibrary ToLibrary
{
get => LazyLoader.Load(this, ref _toLibrary) ?? throw new InvalidOperationException();
init
{
ToLibraryId = value.LocalLibraryId;
_toLibrary = value;
}
}
public MoveMangaLibraryJob(Manga manga, LocalLibrary toLibrary, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(MoveMangaLibraryJob)), JobType.MoveMangaLibraryJob, 0, parentJob, dependsOnJobs)
{
this.MangaId = manga.MangaId;
this.Manga = manga;
this.ToLibraryId = toLibrary.LocalLibraryId;
this.ToLibrary = toLibrary;
}

View File

@ -6,49 +6,45 @@ using Newtonsoft.Json;
namespace API.Schema.Jobs;
public class RetrieveChaptersJob : Job
public class RetrieveChaptersJob : JobWithDownloading
{
[StringLength(64)] [Required] public string MangaId { get; init; }
private Manga _manga = null!;
private MangaConnectorMangaEntry? _mangaConnectorMangaEntry = null!;
[JsonIgnore]
public Manga Manga
public MangaConnectorMangaEntry MangaConnectorMangaEntry
{
get => LazyLoader.Load(this, ref _manga);
init => _manga = value;
get => LazyLoader.Load(this, ref _mangaConnectorMangaEntry) ?? throw new InvalidOperationException();
init => _mangaConnectorMangaEntry = value;
}
[StringLength(8)] [Required] public string Language { get; private set; }
public RetrieveChaptersJob(Manga manga, string language, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(RetrieveChaptersJob)), JobType.RetrieveChaptersJob, recurrenceMs, parentJob, dependsOnJobs)
public RetrieveChaptersJob(MangaConnectorMangaEntry mangaConnectorMangaEntry, string language, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(RetrieveChaptersJob)), JobType.RetrieveChaptersJob, recurrenceMs, mangaConnectorMangaEntry.MangaConnector, parentJob, dependsOnJobs)
{
this.MangaId = manga.MangaId;
this.Manga = manga;
this.MangaConnectorMangaEntry = mangaConnectorMangaEntry;
this.Language = language;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
internal RetrieveChaptersJob(ILazyLoader lazyLoader, string jobId, ulong recurrenceMs, string mangaId, string language, string? parentJobId)
: base(lazyLoader, jobId, JobType.RetrieveChaptersJob, recurrenceMs, parentJobId)
internal RetrieveChaptersJob(ILazyLoader lazyLoader, string jobId, ulong recurrenceMs, string mangaConnectorName, string language, string? parentJobId)
: base(lazyLoader, jobId, JobType.RetrieveChaptersJob, recurrenceMs, mangaConnectorName, parentJobId)
{
this.MangaId = mangaId;
this.Language = language;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
// This gets all chapters that are not downloaded
Chapter[] allChapters = Manga.MangaConnector.GetChapters(Manga, Language).DistinctBy(c => c.ChapterId).ToArray();
Chapter[] newChapters = allChapters.Where(chapter => Manga.Chapters.Select(c => c.ChapterId).Contains(chapter.ChapterId) == false).ToArray();
Log.Info($"{Manga.Chapters.Count} existing + {newChapters.Length} new chapters.");
Chapter[] allChapters = MangaConnectorMangaEntry.MangaConnector.GetChapters(MangaConnectorMangaEntry.Manga, Language).DistinctBy(c => c.ChapterId).ToArray();
Chapter[] newChapters = allChapters.Where(chapter => MangaConnectorMangaEntry.Manga.Chapters.Select(c => c.ChapterId).Contains(chapter.ChapterId) == false).ToArray();
Log.Info($"{MangaConnectorMangaEntry.Manga.Chapters.Count} existing + {newChapters.Length} new chapters.");
try
{
foreach (Chapter newChapter in newChapters)
Manga.Chapters.Add(newChapter);
MangaConnectorMangaEntry.Manga.Chapters.Add(newChapter);
context.SaveChanges();
}
catch (DbUpdateException e)

View File

@ -8,46 +8,41 @@ namespace API.Schema.Jobs;
public class UpdateCoverJob : Job
{
[StringLength(64)] [Required] public string MangaId { get; init; }
private Manga _manga = null!;
private MangaConnectorMangaEntry? _mangaConnectorMangaEntry = null!;
[JsonIgnore]
public Manga Manga
public MangaConnectorMangaEntry MangaConnectorMangaEntry
{
get => LazyLoader.Load(this, ref _manga);
init => _manga = value;
get => LazyLoader.Load(this, ref _mangaConnectorMangaEntry) ?? throw new InvalidOperationException();
init => _mangaConnectorMangaEntry = value;
}
public UpdateCoverJob(Manga manga, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
public UpdateCoverJob(MangaConnectorMangaEntry mangaConnectorMangaEntry, ulong recurrenceMs, Job? parentJob = null, ICollection<Job>? dependsOnJobs = null)
: base(TokenGen.CreateToken(typeof(UpdateCoverJob)), JobType.UpdateCoverJob, recurrenceMs, parentJob, dependsOnJobs)
{
this.MangaId = manga.MangaId;
this.Manga = manga;
this.MangaConnectorMangaEntry = mangaConnectorMangaEntry;
}
/// <summary>
/// EF ONLY!!!
/// </summary>
internal UpdateCoverJob(ILazyLoader lazyLoader, string jobId, ulong recurrenceMs, string mangaId, string? parentJobId)
internal UpdateCoverJob(ILazyLoader lazyLoader, string jobId, ulong recurrenceMs, string? parentJobId)
: base(lazyLoader, jobId, JobType.UpdateCoverJob, recurrenceMs, parentJobId)
{
this.MangaId = mangaId;
}
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
bool keepCover = context.Jobs
.Any(job => job.JobType == JobType.DownloadAvailableChaptersJob
&& ((DownloadAvailableChaptersJob)job).MangaId == MangaId);
&& ((DownloadAvailableChaptersJob)job).MangaConnectorMangaEntry.MangaId == MangaConnectorMangaEntry.MangaId);
if (!keepCover)
{
if(File.Exists(Manga.CoverFileNameInCache))
File.Delete(Manga.CoverFileNameInCache);
if(File.Exists(MangaConnectorMangaEntry.Manga.CoverFileNameInCache))
File.Delete(MangaConnectorMangaEntry.Manga.CoverFileNameInCache);
try
{
Manga.CoverFileNameInCache = null;
MangaConnectorMangaEntry.Manga.CoverFileNameInCache = null;
context.Jobs.Remove(this);
context.SaveChanges();
}
@ -58,7 +53,7 @@ public class UpdateCoverJob : Job
}
else
{
return [new DownloadMangaCoverJob(Manga, this)];
return [new DownloadMangaCoverJob(MangaConnectorMangaEntry, this)];
}
return [];
}

View File

@ -16,21 +16,22 @@ public class Manga
[StringLength(64)]
[Required]
public string MangaId { get; init; }
[StringLength(256)] [Required] public string IdOnConnectorSite { get; init; }
[StringLength(512)] [Required] public string Name { get; internal set; }
[Required] public string Description { get; internal set; }
[Url] [StringLength(512)] [Required] public string WebsiteUrl { get; internal init; }
[JsonIgnore] [Url] [StringLength(512)] public string CoverUrl { get; internal set; }
[Required] public MangaReleaseStatus ReleaseStatus { get; internal set; }
[StringLength(64)]
public string? LibraryId { get; init; }
[JsonIgnore] public LocalLibrary? Library { get; internal set; }
[StringLength(32)]
[Required]
public string MangaConnectorName { get; init; }
[JsonIgnore] public MangaConnector MangaConnector { get; init; } = null!;
[StringLength(64)] public string? LibraryId { get; private set; }
private LocalLibrary? _library = null!;
[JsonIgnore]
public LocalLibrary? Library
{
get => _lazyLoader.Load(this, ref _library);
set
{
LibraryId = value?.LocalLibraryId;
_library = value;
}
}
public ICollection<Author> Authors { get; internal set; }= null!;
public ICollection<MangaTag> MangaTags { get; internal set; }= null!;
@ -38,7 +39,6 @@ public class Manga
public ICollection<MangaAltTitle> AltTitles { get; internal set; } = null!;
[Required] public float IgnoreChaptersBefore { get; internal set; }
[StringLength(1024)] [Required] public string DirectoryName { get; private set; }
[JsonIgnore] [StringLength(512)] public string? CoverFileNameInCache { get; internal set; } = null;
public uint? Year { get; internal init; }
[StringLength(8)] public string? OriginalLanguage { get; internal init; }
@ -48,30 +48,37 @@ public class Manga
public string? FullDirectoryPath => Library is not null ? Path.Join(Library.BasePath, DirectoryName) : null;
[NotMapped] public ICollection<string> ChapterIds => Chapters.Select(c => c.ChapterId).ToList();
private readonly ILazyLoader _lazyLoader = null!;
private ICollection<Chapter> _chapters = null!;
private ICollection<Chapter>? _chapters = null!;
[JsonIgnore]
public ICollection<Chapter> Chapters
{
get => _lazyLoader.Load(this, ref _chapters);
get => _lazyLoader.Load(this, ref _chapters) ?? throw new InvalidOperationException();
init => _chapters = value;
}
public Manga(string idOnConnector, string name, string description, string websiteUrl, string coverUrl, MangaReleaseStatus releaseStatus,
MangaConnector mangaConnector, ICollection<Author> authors, ICollection<MangaTag> mangaTags, ICollection<Link> links, ICollection<MangaAltTitle> altTitles,
[NotMapped]
public ICollection<string> LinkedMangaConnectors =>
MangaConnectorLinkedToManga.Select(l => l.MangaConnectorName).ToList();
private ICollection<MangaConnectorMangaEntry>? _mangaConnectorLinkedToManga = null!;
[JsonIgnore]
public ICollection<MangaConnectorMangaEntry> MangaConnectorLinkedToManga
{
get => _lazyLoader.Load(this, ref _mangaConnectorLinkedToManga) ?? throw new InvalidOperationException();
init => _mangaConnectorLinkedToManga = value;
}
private readonly ILazyLoader _lazyLoader = null!;
public Manga(string name, string description, string coverUrl, MangaReleaseStatus releaseStatus,
ICollection<Author> authors, ICollection<MangaTag> mangaTags, ICollection<Link> links, ICollection<MangaAltTitle> altTitles,
LocalLibrary? library = null, float ignoreChaptersBefore = 0f, uint? year = null, string? originalLanguage = null)
{
this.MangaId = TokenGen.CreateToken(typeof(Manga), mangaConnector.Name, idOnConnector);
this.IdOnConnectorSite = idOnConnector;
this.MangaId = TokenGen.CreateToken(typeof(Manga), name);
this.Name = name;
this.Description = description;
this.WebsiteUrl = websiteUrl;
this.CoverUrl = coverUrl;
this.ReleaseStatus = releaseStatus;
this.LibraryId = library?.LocalLibraryId;
this.Library = library;
this.MangaConnectorName = mangaConnector.Name;
this.MangaConnector = mangaConnector;
this.Authors = authors;
this.MangaTags = mangaTags;
this.Links = links;
@ -86,18 +93,15 @@ public class Manga
/// <summary>
/// EF ONLY!!!
/// </summary>
public Manga(ILazyLoader lazyLoader, string mangaId, string idOnConnectorSite, string name, string description, string websiteUrl, string coverUrl, MangaReleaseStatus releaseStatus,
string mangaConnectorName, string directoryName, float ignoreChaptersBefore, string? libraryId, uint? year, string? originalLanguage)
public Manga(ILazyLoader lazyLoader, string mangaId, string name, string description, string coverUrl, MangaReleaseStatus releaseStatus,
string directoryName, float ignoreChaptersBefore, string? libraryId, uint? year, string? originalLanguage)
{
this._lazyLoader = lazyLoader;
this.MangaId = mangaId;
this.IdOnConnectorSite = idOnConnectorSite;
this.Name = name;
this.Description = description;
this.WebsiteUrl = websiteUrl;
this.CoverUrl = coverUrl;
this.ReleaseStatus = releaseStatus;
this.MangaConnectorName = mangaConnectorName;
this.DirectoryName = directoryName;
this.LibraryId = libraryId;
this.IgnoreChaptersBefore = ignoreChaptersBefore;

View File

@ -0,0 +1,59 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.MangaConnectors;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
namespace API.Schema;
[PrimaryKey("MangaId", "MangaConnectorName")]
public class MangaConnectorMangaEntry
{
[StringLength(64)] [Required] public string MangaId { get; private set; } = null!;
[JsonIgnore] private Manga? _manga = null!;
[JsonIgnore]
public Manga Manga
{
get => _lazyLoader.Load(this, ref _manga) ?? throw new InvalidOperationException();
init => _manga = value;
}
[StringLength(32)] [Required] public string MangaConnectorName { get; private set; } = null!;
[JsonIgnore] private MangaConnector? _mangaConnector = null!;
[JsonIgnore]
public MangaConnector MangaConnector
{
get => _lazyLoader.Load(this, ref _mangaConnector) ?? throw new InvalidOperationException();
init
{
MangaConnectorName = value.Name;
_mangaConnector = value;
}
}
[StringLength(256)] [Required] public string IdOnConnectorSite { get; init; }
[Url] [StringLength(512)] [Required] public string WebsiteUrl { get; internal init; }
private readonly ILazyLoader _lazyLoader = null!;
public MangaConnectorMangaEntry(Manga manga, MangaConnector mangaConnector, string idOnConnectorSite, string websiteUrl)
{
this.Manga = manga;
this.MangaConnector = mangaConnector;
this.IdOnConnectorSite = idOnConnectorSite;
this.WebsiteUrl = websiteUrl;
}
/// <summary>
/// EF CORE ONLY!!!
/// </summary>
public MangaConnectorMangaEntry(ILazyLoader lazyLoader, string mangaId, string mangaConnectorName, string idOnConnectorSite, string websiteUrl)
{
this._lazyLoader = lazyLoader;
this.MangaId = mangaId;
this.MangaConnectorName = mangaConnectorName;
this.IdOnConnectorSite = idOnConnectorSite;
this.WebsiteUrl = websiteUrl;
}
}

View File

@ -17,7 +17,7 @@ public class ComickIo : MangaConnector
this.downloadClient = new HttpDownloadClient();
}
public override Manga[] SearchManga(string mangaSearchName)
public override MangaConnectorMangaEntry[] SearchManga(string mangaSearchName)
{
Log.Info($"Searching Manga: {mangaSearchName}");
@ -46,20 +46,20 @@ public class ComickIo : MangaConnector
}
Log.Debug($"Search {mangaSearchName} yielded {slugs.Count} slugs. Requesting mangas now...");
List<Manga> mangas = slugs.Select(GetMangaFromId).ToList()!;
List<MangaConnectorMangaEntry> mangas = slugs.Select(GetMangaFromId).ToList()!;
Log.Info($"Search {mangaSearchName} yielded {mangas.Count} results.");
return mangas.ToArray();
}
private readonly Regex _getSlugFromTitleRex = new(@"https?:\/\/comick\.io\/comic\/(.+)(?:\/.*)*");
public override Manga? GetMangaFromUrl(string url)
public override MangaConnectorMangaEntry? GetMangaFromUrl(string url)
{
Match m = _getSlugFromTitleRex.Match(url);
return m.Groups[1].Success ? GetMangaFromId(m.Groups[1].Value) : null;
}
public override Manga? GetMangaFromId(string mangaIdOnSite)
public override MangaConnectorMangaEntry? GetMangaFromId(string mangaIdOnSite)
{
string requestUrl = $"https://api.comick.fun/comic/{mangaIdOnSite}";
@ -75,14 +75,14 @@ public class ComickIo : MangaConnector
return ParseMangaFromJToken(data);
}
public override Chapter[] GetChapters(Manga manga, string? language = null)
public override Chapter[] GetChapters(MangaConnectorMangaEntry mangaConnectorMangaEntry, string? language = null)
{
Log.Info($"Getting Chapters: {manga.IdOnConnectorSite}");
Log.Info($"Getting Chapters: {mangaConnectorMangaEntry.IdOnConnectorSite}");
List<Chapter> chapters = new();
int page = 1;
while(page < 50)
{
string requestUrl = $"https://api.comick.fun/comic/{manga.IdOnConnectorSite}/chapters?limit=100&page={page}&lang={language}";
string requestUrl = $"https://api.comick.fun/comic/{mangaConnectorMangaEntry.IdOnConnectorSite}/chapters?limit=100&page={page}&lang={language}";
RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.MangaInfo);
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
@ -98,7 +98,7 @@ public class ComickIo : MangaConnector
if (chaptersArray is null || chaptersArray.Count < 1)
break;
chapters.AddRange(ParseChapters(manga, chaptersArray));
chapters.AddRange(ParseChapters(mangaConnectorMangaEntry, chaptersArray));
page++;
}
@ -133,7 +133,7 @@ public class ComickIo : MangaConnector
}).ToArray();
}
private Manga ParseMangaFromJToken(JToken json)
private MangaConnectorMangaEntry ParseMangaFromJToken(JToken json)
{
string? hid = json["comic"]?.Value<string>("hid");
string? slug = json["comic"]?.Value<string>("slug");
@ -211,12 +211,12 @@ public class ComickIo : MangaConnector
if(name is null)
throw new Exception("name is null");
return new Manga(hid, name, description??"", url, coverUrl, status, this,
authors, tags, links, altTitles,
Manga manga = new (name, description??"", coverUrl, status, authors, tags, links, altTitles,
year: year, originalLanguage: originalLanguage);
return new MangaConnectorMangaEntry(manga, this, hid, url);
}
private List<Chapter> ParseChapters(Manga parentManga, JArray chaptersArray)
private List<Chapter> ParseChapters(MangaConnectorMangaEntry mangaConnectorMangaEntry, JArray chaptersArray)
{
List<Chapter> chapters = new ();
foreach (JToken chapter in chaptersArray)
@ -226,12 +226,12 @@ public class ComickIo : MangaConnector
int? volumeNum = volumeNumStr is null ? null : int.Parse(volumeNumStr);
string? title = chapter.Value<string>("title");
string? hid = chapter.Value<string>("hid");
string url = $"https://comick.io/comic/{parentManga.IdOnConnectorSite}/{hid}";
string url = $"https://comick.io/comic/{mangaConnectorMangaEntry.IdOnConnectorSite}/{hid}";
if(chapterNum is null || hid is null)
continue;
chapters.Add(new (parentManga, url, chapterNum, volumeNum, hid, title));
chapters.Add(new (mangaConnectorMangaEntry, url, chapterNum, volumeNum, hid, title));
}
return chapters;
}

View File

@ -10,14 +10,14 @@ public class Global : MangaConnector
this.context = context;
}
public override Manga[] SearchManga(string mangaSearchName)
public override MangaConnectorMangaEntry[] SearchManga(string mangaSearchName)
{
//Get all enabled Connectors
MangaConnector[] enabledConnectors = context.MangaConnectors.Where(c => c.Enabled && c.Name != "Global").ToArray();
//Create Task for each MangaConnector to search simulatneously
Task<Manga[]>[] tasks =
enabledConnectors.Select(c => new Task<Manga[]>(() => c.SearchManga(mangaSearchName))).ToArray();
Task<MangaConnectorMangaEntry[]>[] tasks =
enabledConnectors.Select(c => new Task<MangaConnectorMangaEntry[]>(() => c.SearchManga(mangaSearchName))).ToArray();
foreach (var task in tasks)
task.Start();
@ -28,28 +28,28 @@ public class Global : MangaConnector
}while(tasks.Any(t => t.Status < TaskStatus.RanToCompletion));
//Concatenate all results into one
Manga[] ret = tasks.Select(t => t.IsCompletedSuccessfully ? t.Result : []).ToArray().SelectMany(i => i).ToArray();
MangaConnectorMangaEntry[] ret = tasks.Select(t => t.IsCompletedSuccessfully ? t.Result : []).ToArray().SelectMany(i => i).ToArray();
return ret;
}
public override Manga? GetMangaFromUrl(string url)
public override MangaConnectorMangaEntry? GetMangaFromUrl(string url)
{
MangaConnector? mc = context.MangaConnectors.ToArray().FirstOrDefault(c => c.UrlMatchesConnector(url));
return mc?.GetMangaFromUrl(url) ?? null;
}
public override Manga? GetMangaFromId(string mangaIdOnSite)
public override MangaConnectorMangaEntry? GetMangaFromId(string mangaIdOnSite)
{
return null;
}
public override Chapter[] GetChapters(Manga manga, string? language = null)
public override Chapter[] GetChapters(MangaConnectorMangaEntry mangaConnectorMangaEntry, string? language = null)
{
return manga.MangaConnector.GetChapters(manga, language);
return mangaConnectorMangaEntry.MangaConnector.GetChapters(mangaConnectorMangaEntry, language);
}
internal override string[] GetChapterImageUrls(Chapter chapter)
{
return chapter.ParentManga.MangaConnector.GetChapterImageUrls(chapter);
return chapter.MangaConnectorMangaEntry.MangaConnector.GetChapterImageUrls(chapter);
}
}

View File

@ -34,13 +34,13 @@ public abstract class MangaConnector(string name, string[] supportedLanguages, s
[Required]
public bool Enabled { get; internal set; } = true;
public abstract Manga[] SearchManga(string mangaSearchName);
public abstract MangaConnectorMangaEntry[] SearchManga(string mangaSearchName);
public abstract Manga? GetMangaFromUrl(string url);
public abstract MangaConnectorMangaEntry? GetMangaFromUrl(string url);
public abstract Manga? GetMangaFromId(string mangaIdOnSite);
public abstract MangaConnectorMangaEntry? GetMangaFromId(string mangaIdOnSite);
public abstract Chapter[] GetChapters(Manga manga, string? language = null);
public abstract Chapter[] GetChapters(MangaConnectorMangaEntry mangaConnectorMangaEntry, string? language = null);
internal abstract string[] GetChapterImageUrls(Chapter chapter);

View File

@ -18,10 +18,10 @@ public class MangaDex : MangaConnector
}
private const int Limit = 100;
public override Manga[] SearchManga(string mangaSearchName)
public override MangaConnectorMangaEntry[] SearchManga(string mangaSearchName)
{
Log.Info($"Searching Manga: {mangaSearchName}");
List<Manga> mangas = new ();
List<MangaConnectorMangaEntry> mangas = new ();
int offset = 0;
int total = int.MaxValue;
@ -67,7 +67,7 @@ public class MangaDex : MangaConnector
}
private static readonly Regex GetMangaIdFromUrl = new(@"https?:\/\/mangadex\.org\/title\/([a-z0-9-]+)\/?.*");
public override Manga? GetMangaFromUrl(string url)
public override MangaConnectorMangaEntry? GetMangaFromUrl(string url)
{
Log.Info($"Getting Manga: {url}");
if (!UrlMatchesConnector(url))
@ -87,7 +87,7 @@ public class MangaDex : MangaConnector
return GetMangaFromId(id);
}
public override Manga? GetMangaFromId(string mangaIdOnSite)
public override MangaConnectorMangaEntry? GetMangaFromId(string mangaIdOnSite)
{
Log.Info($"Getting Manga: {mangaIdOnSite}");
string requestUrl =
@ -118,13 +118,12 @@ public class MangaDex : MangaConnector
return null;
}
Manga manga = ParseMangaFromJToken(data);
return manga;
return ParseMangaFromJToken(data);
}
public override Chapter[] GetChapters(Manga manga, string? language = null)
public override Chapter[] GetChapters(MangaConnectorMangaEntry mangaConnectorMangaEntry, string? language = null)
{
Log.Info($"Getting Chapters: {manga.IdOnConnectorSite}");
Log.Info($"Getting Chapters: {mangaConnectorMangaEntry.IdOnConnectorSite}");
List<Chapter> chapters = new ();
int offset = 0;
@ -132,7 +131,7 @@ public class MangaDex : MangaConnector
while(offset < total)
{
string requestUrl =
$"https://api.mangadex.org/manga/{manga.IdOnConnectorSite}/feed?limit={Limit}&offset={offset}&" +
$"https://api.mangadex.org/manga/{mangaConnectorMangaEntry.IdOnConnectorSite}/feed?limit={Limit}&offset={offset}&" +
$"translatedLanguage%5B%5D={language}&" +
$"contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&includeFutureUpdates=0&includes%5B%5D=";
offset += Limit;
@ -163,10 +162,10 @@ public class MangaDex : MangaConnector
return [];
}
chapters.AddRange(data.Select(d => ParseChapterFromJToken(manga, d)));
chapters.AddRange(data.Select(d => ParseChapterFromJToken(mangaConnectorMangaEntry, d)));
}
Log.Info($"Request for chapters for {manga.Name} yielded {chapters.Count} results.");
Log.Info($"Request for chapters for {mangaConnectorMangaEntry.Manga.Name} yielded {chapters.Count} results.");
return chapters.ToArray();
}
@ -223,7 +222,7 @@ public class MangaDex : MangaConnector
return urls.ToArray();
}
private Manga ParseMangaFromJToken(JToken jToken)
private MangaConnectorMangaEntry ParseMangaFromJToken(JToken jToken)
{
string? id = jToken.Value<string>("id");
if(id is null)
@ -313,12 +312,12 @@ public class MangaDex : MangaConnector
string websiteUrl = $"https://mangadex.org/title/{id}";
string coverUrl = $"https://uploads.mangadex.org/covers/{id}/{coverFileName}";
return new Manga(id, name, description, websiteUrl, coverUrl, releaseStatus, this,
authors, tags, links,altTitles,
Manga manga = new Manga(name, description, coverUrl, releaseStatus, authors, tags, links,altTitles,
null, 0f, year, originalLanguage);
return new MangaConnectorMangaEntry(manga, this, id, websiteUrl);
}
private Chapter ParseChapterFromJToken(Manga parentManga, JToken jToken)
private Chapter ParseChapterFromJToken(MangaConnectorMangaEntry mangaConnectorMangaEntry, JToken jToken)
{
string? id = jToken.Value<string>("id");
JToken? attributes = jToken["attributes"];
@ -333,6 +332,6 @@ public class MangaDex : MangaConnector
volume = int.Parse(volumeStr);
string url = $"https://mangadex.org/chapter/{id}";
return new Chapter(parentManga, url, chapter, volume, id, title);
return new Chapter(mangaConnectorMangaEntry, url, chapter, volume, id, title);
}
}