glax/Tranga
1
0
Fork 0
mirror of https://github.com/C9Glax/tranga.git synced 2025-05-22 06:03:01 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Attach Entities to Jobs

Browse Source
This commit is contained in:
Glax 2025-05-15 15:13:53 +02:00
parent 694b88d200
commit 475a29b10d
23 changed files with 132 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
API/Controllers/JobController.cs
View File

@ -1,5 +1,6 @@
using API.APIEndpointRecords;
using API.Schema;
using API.Schema.Contexts;
using API.Schema.Jobs;
using Asp.Versioning;
using log4net;

3
API/Controllers/LibraryConnectorController.cs
View File

@ -1,4 +1,5 @@
using API.Schema;
using API.Schema.Contexts;
using API.Schema.LibraryConnectors;
using Asp.Versioning;
using log4net;
@ -10,7 +11,7 @@ namespace API.Controllers;
[ApiVersion(2)]
[ApiController]
[Route("v{v:apiVersion}/[controller]")]
public class LibraryConnectorController(PgsqlContext context, ILog Log) : Controller
public class LibraryConnectorController(LibraryContext context, ILog Log) : Controller
{
/// <summary>
/// Gets all configured ToLibrary-Connectors

1
API/Controllers/LocalLibrariesController.cs
View File

@ -1,5 +1,6 @@
using API.APIEndpointRecords;
using API.Schema;
using API.Schema.Contexts;
using Asp.Versioning;
using log4net;
using Microsoft.AspNetCore.Mvc;

2
API/Controllers/MangaConnectorController.cs
View File

@ -1,4 +1,4 @@
using API.Schema;
using API.Schema.Contexts;
using API.Schema.MangaConnectors;
using Asp.Versioning;
using log4net;

1
API/Controllers/MangaController.cs
View File

@ -1,4 +1,5 @@
using API.Schema;
using API.Schema.Contexts;
using API.Schema.Jobs;
using Asp.Versioning;
using log4net;

4
API/Controllers/NotificationConnectorController.cs
View File

@ -1,6 +1,6 @@
using System.Text;
using API.APIEndpointRecords;
using API.Schema;
using API.Schema.Contexts;
using API.Schema.NotificationConnectors;
using Asp.Versioning;
using log4net;
@ -13,7 +13,7 @@ namespace API.Controllers;
[ApiController]
[Produces("application/json")]
[Route("v{v:apiVersion}/[controller]")]
public class NotificationConnectorController(PgsqlContext context, ILog Log) : Controller
public class NotificationConnectorController(NotificationsContext context, ILog Log) : Controller
{
/// <summary>
/// Gets all configured Notification-Connectors

1
API/Controllers/QueryController.cs
View File

@ -1,4 +1,5 @@
using API.Schema;
using API.Schema.Contexts;
using Asp.Versioning;
using log4net;
using Microsoft.AspNetCore.Mvc;

1
API/Controllers/SearchController.cs
View File

@ -1,4 +1,5 @@
using API.Schema;
using API.Schema.Contexts;
using API.Schema.Jobs;
using API.Schema.MangaConnectors;
using Asp.Versioning;

1
API/Controllers/SettingsController.cs
View File

@ -1,5 +1,6 @@
using API.MangaDownloadClients;
using API.Schema;
using API.Schema.Contexts;
using API.Schema.Jobs;
using Asp.Versioning;
using log4net;

54
API/Program.cs
View File

@ -1,7 +1,7 @@
using System.Reflection;
using System.Text.Json.Serialization;
using API;
using API.Schema;
using API.Schema.Contexts;
using API.Schema.Jobs;
using API.Schema.MangaConnectors;
using Asp.Versioning;
@ -55,11 +55,17 @@ builder.Services.AddSwaggerGen(opt =>
});
builder.Services.ConfigureOptions<NamedSwaggerGenOptions>();
builder.Services.AddDbContext<PgsqlContext>(options =>
options.UseNpgsql($"Host={Environment.GetEnvironmentVariable("POSTGRES_HOST")??"localhost:5432"}; " +
string ConnectionString = $"Host={Environment.GetEnvironmentVariable("POSTGRES_HOST") ?? "localhost:5432"}; " +
$"Database={Environment.GetEnvironmentVariable("POSTGRES_DB") ?? "postgres"}; " +
$"Username={Environment.GetEnvironmentVariable("POSTGRES_USER") ?? "postgres"}; " +
$"Password={Environment.GetEnvironmentVariable("POSTGRES_PASSWORD")??"postgres"}"));
$"Password={Environment.GetEnvironmentVariable("POSTGRES_PASSWORD") ?? "postgres"}";
builder.Services.AddDbContext<PgsqlContext>(options =>
options.UseNpgsql(ConnectionString));
builder.Services.AddDbContext<NotificationsContext>(options =>
options.UseNpgsql(ConnectionString));
builder.Services.AddDbContext<LibraryContext>(options =>
options.UseNpgsql(ConnectionString));
builder.Services.AddControllers(options =>
{
@ -99,15 +105,10 @@ app.UseHttpsRedirection();
app.UseMiddleware<RequestTimeMiddleware>();
using (var scope = app.Services.CreateScope())
using (IServiceScope scope = app.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService<PgsqlContext>();
db.Database.Migrate();
}
using (var scope = app.Services.CreateScope())
{
PgsqlContext context = scope.ServiceProvider.GetService<PgsqlContext>()!;
PgsqlContext context = scope.ServiceProvider.GetRequiredService<PgsqlContext>();
context.Database.Migrate();
MangaConnector[] connectors =
[
@ -116,13 +117,30 @@ using (var scope = app.Services.CreateScope())
];
MangaConnector[] newConnectors = connectors.Where(c => !context.MangaConnectors.Contains(c)).ToArray();
context.MangaConnectors.AddRange(newConnectors);
context.Jobs.AddRange(context.Mangas.AsEnumerable().Select(m => new UpdateFilesDownloadedJob(m, 0)));
context.Jobs.RemoveRange(context.Jobs.Where(j => j.state == JobState.Completed && j.RecurrenceMs < 1));
if (!context.LocalLibraries.Any())
context.LocalLibraries.Add(new LocalLibrary(TrangaSettings.downloadLocation, "Default ToLibrary"));
context.LocalLibraries.Add(new LocalLibrary(TrangaSettings.downloadLocation, "Default Library"));
context.Jobs.AddRange(context.Jobs.Where(j => j.JobType == JobType.DownloadAvailableChaptersJob)
.AsEnumerable()
.Select(dacj =>
{
DownloadAvailableChaptersJob? j = dacj as DownloadAvailableChaptersJob;
return new UpdateFilesDownloadedJob(j!.Manga, 0);
}));
context.Jobs.RemoveRange(context.Jobs.Where(j => j.state == JobState.Completed && j.RecurrenceMs < 1));
foreach (Job job in context.Jobs.Where(j => j.state == JobState.Running))
{
job.state = JobState.FirstExecution;
job.LastExecution = DateTime.UnixEpoch;
}
context.SaveChanges();
}
using (IServiceScope scope = app.Services.CreateScope())
{
NotificationsContext context = scope.ServiceProvider.GetRequiredService<NotificationsContext>();
context.Database.Migrate();
string[] emojis = { "(•‿•)", "(づ \u25d5‿\u25d5 )づ", "( \u02d8\u25bd\u02d8)っ\u2668", "=\uff3e\u25cf \u22cf \u25cf\uff3e=", "（ΦωΦ）", "(\u272a\u3268\u272a)", "( ﾉ･o･ )ﾉ", "（〜^\u2207^ )〜", "~(\u2267ω\u2266)~","૮ \u00b4• ﻌ \u00b4• ა", "(\u02c3ᆺ\u02c2)", "(=\ud83d\udf66 \u0f1d \ud83d\udf66=)"};
context.Notifications.Add(new Notification("Tranga Started", emojis[Random.Shared.Next(0, emojis.Length - 1)], NotificationUrgency.High));

18
API/Schema/Contexts/LibraryContext.cs Normal file
View File

@ -0,0 +1,18 @@
using API.Schema.LibraryConnectors;
using Microsoft.EntityFrameworkCore;
namespace API.Schema.Contexts;
public class LibraryContext(DbContextOptions<LibraryContext> options) : DbContext(options)
{
public DbSet<LibraryConnector> LibraryConnectors { get; set; }
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
//LibraryConnector Types
modelBuilder.Entity<LibraryConnector>()
.HasDiscriminator(l => l.LibraryType)
.HasValue<Komga>(LibraryType.Komga)
.HasValue<Kavita>(LibraryType.Kavita);
}
}

10
API/Schema/Contexts/NotificationsContext.cs Normal file
View File

@ -0,0 +1,10 @@
using API.Schema.NotificationConnectors;
using Microsoft.EntityFrameworkCore;
namespace API.Schema.Contexts;
public class NotificationsContext(DbContextOptions<NotificationsContext> options) : DbContext(options)
{
public DbSet<NotificationConnector> NotificationConnectors { get; set; }
public DbSet<Notification> Notifications { get; set; }
}

32
API/Schema/PgsqlContext.cs → API/Schema/Contexts/PgsqlContext.cs
View File

@ -1,10 +1,9 @@
using API.Schema.Jobs;
using API.Schema.LibraryConnectors;
using API.Schema.MangaConnectors;
using API.Schema.NotificationConnectors;
using Microsoft.EntityFrameworkCore;
namespace API.Schema;
namespace API.Schema.Contexts;
public class PgsqlContext(DbContextOptions<PgsqlContext> options) : DbContext(options)
{
@ -15,9 +14,6 @@ public class PgsqlContext(DbContextOptions<PgsqlContext> options) : DbContext(op
public DbSet<Chapter> Chapters { get; set; }
public DbSet<Author> Authors { get; set; }
public DbSet<MangaTag> Tags { get; set; }
public DbSet<LibraryConnector> LibraryConnectors { get; set; }
public DbSet<NotificationConnector> NotificationConnectors { get; set; }
public DbSet<Notification> Notifications { get; set; }
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
@ -109,7 +105,7 @@ public class PgsqlContext(DbContextOptions<PgsqlContext> options) : DbContext(op
.HasMany<Job>(root => root.DependsOnJobs)
.WithMany();
modelBuilder.Entity<Job>()
.Navigation(root => root.DependsOnJobs)
.Navigation(j => j.DependsOnJobs)
.AutoInclude(false);
//MangaConnector Types
@ -138,14 +134,23 @@ public class PgsqlContext(DbContextOptions<PgsqlContext> options) : DbContext(op
modelBuilder.Entity<Chapter>()
.Navigation(c => c.ParentManga)
.AutoInclude();
modelBuilder.Entity<Manga>()
.Navigation(m => m.Chapters)
.AutoInclude();
//Manga owns MangaAltTitles
modelBuilder.Entity<Manga>()
.OwnsMany<MangaAltTitle>(m => m.AltTitles)
.WithOwner();
modelBuilder.Entity<Manga>()
.Navigation(m => m.AltTitles)
.AutoInclude();
//Manga owns Links
modelBuilder.Entity<Manga>()
.OwnsMany<Link>(m => m.Links)
.WithOwner();
modelBuilder.Entity<Manga>()
.Navigation(m => m.Links)
.AutoInclude();
//Manga has many Tags associated with many Manga
modelBuilder.Entity<Manga>()
.HasMany<MangaTag>(m => m.MangaTags)
@ -155,6 +160,9 @@ public class PgsqlContext(DbContextOptions<PgsqlContext> options) : DbContext(op
r => r.HasOne(typeof(Manga)).WithMany().HasForeignKey("MangaIds").HasPrincipalKey(nameof(Manga.MangaId)),
j => j.HasKey("MangaTagIds", "MangaIds")
);
modelBuilder.Entity<Manga>()
.Navigation(m => m.MangaTags)
.AutoInclude();
//Manga has many Authors associated with many Manga
modelBuilder.Entity<Manga>()
.HasMany<Author>(m => m.Authors)
@ -164,6 +172,9 @@ public class PgsqlContext(DbContextOptions<PgsqlContext> options) : DbContext(op
r => r.HasOne(typeof(Manga)).WithMany().HasForeignKey("MangaIds").HasPrincipalKey(nameof(Manga.MangaId)),
j => j.HasKey("AuthorIds", "MangaIds")
);
modelBuilder.Entity<Manga>()
.Navigation(m => m.Authors)
.AutoInclude();
//LocalLibrary has many Mangas
modelBuilder.Entity<LocalLibrary>()
@ -171,11 +182,8 @@ public class PgsqlContext(DbContextOptions<PgsqlContext> options) : DbContext(op
.WithOne(m => m.Library)
.HasForeignKey(m => m.LibraryId)
.OnDelete(DeleteBehavior.SetNull);
//LibraryConnector Types
modelBuilder.Entity<LibraryConnector>()
.HasDiscriminator(l => l.LibraryType)
.HasValue<Komga>(LibraryType.Komga)
.HasValue<Kavita>(LibraryType.Kavita);
modelBuilder.Entity<Manga>()
.Navigation(m => m.Library)
.AutoInclude();
}
}

2
API/Schema/Jobs/DownloadAvailableChaptersJob.cs
View File

@ -1,4 +1,5 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.Contexts;
using Newtonsoft.Json;
namespace API.Schema.Jobs;
@ -26,6 +27,7 @@ public class DownloadAvailableChaptersJob : Job
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
context.Attach(Manga);
return Manga.Chapters.Select(chapter => new DownloadSingleChapterJob(chapter, this));
}
}

2
API/Schema/Jobs/DownloadMangaCoverJob.cs
View File

@ -1,4 +1,5 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
@ -27,6 +28,7 @@ public class DownloadMangaCoverJob : Job
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
context.Attach(Manga);
try
{
Manga.CoverFileNameInCache = Manga.MangaConnector.SaveCoverImageToCache(Manga);

2
API/Schema/Jobs/DownloadSingleChapterJob.cs
View File

@ -2,6 +2,7 @@
using System.IO.Compression;
using System.Runtime.InteropServices;
using API.MangaDownloadClients;
using API.Schema.Contexts;
using Newtonsoft.Json;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.Formats.Jpeg;
@ -35,6 +36,7 @@ public class DownloadSingleChapterJob : Job
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
context.Attach(Chapter);
string[] imageUrls = Chapter.ParentManga.MangaConnector.GetChapterImageUrls(Chapter);
if (imageUrls.Length < 1)
{

5
API/Schema/Jobs/Job.cs
View File

@ -1,5 +1,6 @@
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using API.Schema.Contexts;
using log4net;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
@ -62,14 +63,16 @@ public abstract class Job
{
Log.Debug($"Running job {JobId}");
using IServiceScope scope = serviceProvider.CreateScope();
PgsqlContext context = scope.ServiceProvider.GetRequiredService<PgsqlContext>();
try
{
PgsqlContext context = scope.ServiceProvider.GetRequiredService<PgsqlContext>();
context.Attach(this);
this.state = JobState.Running;
context.SaveChanges();
Job[] newJobs = RunInternal(context).ToArray();
this.state = JobState.Completed;
context.SaveChanges();
context.Jobs.AddRange(newJobs);
context.SaveChanges();
Log.Info($"Job {JobId} completed. Generated {newJobs.Length} new jobs.");

1
API/Schema/Jobs/MoveFileOrFolderJob.cs
View File

@ -1,4 +1,5 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.Contexts;
namespace API.Schema.Jobs;

2
API/Schema/Jobs/MoveMangaLibraryJob.cs
View File

@ -1,4 +1,5 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
@ -32,6 +33,7 @@ public class MoveMangaLibraryJob : Job
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
context.Attach(Manga);
Dictionary<Chapter, string> oldPath = Manga.Chapters.ToDictionary(c => c, c => c.FullArchiveFilePath);
Manga.Library = ToLibrary;
try

3
API/Schema/Jobs/RetrieveChaptersJob.cs
View File

@ -1,5 +1,5 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.MangaConnectors;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
@ -31,6 +31,7 @@ public class RetrieveChaptersJob : Job
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
context.Attach(Manga);
// This gets all chapters that are not downloaded
Chapter[] allChapters = Manga.MangaConnector.GetChapters(Manga, Language);
Chapter[] newChapters = allChapters.Where(chapter => context.Chapters.Contains(chapter) == false).ToArray();

2
API/Schema/Jobs/UpdateFilesDownloadedJob.cs
View File

@ -1,4 +1,5 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
@ -27,6 +28,7 @@ public class UpdateFilesDownloadedJob : Job
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
context.Attach(Manga);
foreach (Chapter chapter in Manga.Chapters)
chapter.Downloaded = chapter.CheckDownloaded();

4
API/Schema/MangaConnectors/Global.cs
View File

@ -1,4 +1,6 @@
namespace API.Schema.MangaConnectors;
using API.Schema.Contexts;
namespace API.Schema.MangaConnectors;
public class Global : MangaConnector
{

27
API/Tranga.cs
View File

@ -1,4 +1,5 @@
using API.Schema;
using API.Schema.Contexts;
using API.Schema.Jobs;
using API.Schema.MangaConnectors;
using API.Schema.NotificationConnectors;
@ -30,12 +31,7 @@ public static class Tranga
}
IServiceProvider serviceProvider = (IServiceProvider)serviceProviderObj!;
using IServiceScope scope = serviceProvider.CreateScope();
PgsqlContext? context = scope.ServiceProvider.GetService<PgsqlContext>();
if (context is null)
{
Log.Error("PgsqlContext is null");
return;
}
NotificationsContext context = scope.ServiceProvider.GetRequiredService<NotificationsContext>();
try
{
@ -64,12 +60,7 @@ public static class Tranga
{
Log.Info($"Sending notifications for {urgency}");
using IServiceScope scope = serviceProvider.CreateScope();
PgsqlContext? context = scope.ServiceProvider.GetService<PgsqlContext>();
if (context is null)
{
Log.Error("PgsqlContext is null");
return;
}
NotificationsContext context = scope.ServiceProvider.GetRequiredService<NotificationsContext>();
List<Notification> notifications = context.Notifications.Where(n => n.Urgency == urgency).ToList();
if (!notifications.Any())
@ -117,6 +108,8 @@ public static class Tranga
}
Log.Info(TRANGA);
Log.Info("Loading Jobs");
context.Jobs.Load();
Log.Info("JobStarter Thread running.");
while (true)
{
@ -146,7 +139,7 @@ public static class Tranga
List<Job> dueJobs = waitingJobs.Where(j => j.NextExecution < DateTime.UtcNow).ToList();
List<MangaConnector> busyConnectors = GetBusyConnectors(runningJobs);
List<Job> startJobs = FilterJobPreconditions(dueJobs, busyConnectors);
List<Job> startJobs = FilterJobPreconditions(context, dueJobs, busyConnectors);
//Start Jobs that are allowed to run (preconditions match)
foreach (Job job in startJobs)
@ -194,9 +187,13 @@ public static class Tranga
return busyConnectors.ToList();
}
private static List<Job> FilterJobPreconditions(List<Job> dueJobs, List<MangaConnector> busyConnectors) =>
private static List<Job> FilterJobPreconditions(PgsqlContext context, List<Job> dueJobs, List<MangaConnector> busyConnectors) =>
dueJobs
.Where(j => j.DependenciesFulfilled)
.Where(j =>
{
context.Entry(j).Collection(j => j.DependsOnJobs).Load(LoadOptions.ForceIdentityResolution);
return j.DependenciesFulfilled;
})
.Where(j =>
{
//Filter jobs with busy connectors