From 826b0e4b98c2dd4a5e4e63fec0bf8e6e5bbdc5cc Mon Sep 17 00:00:00 2001 From: glax Date: Sat, 28 Jun 2025 22:59:35 +0200 Subject: [PATCH] MetadataFetching: - Jikan (MAL) linking, fetching/updating --- API/API.csproj | 1 + API/Controllers/MetadataFetcherController.cs | 108 +++ .../pgsql/20250628204956_AddMAL.Designer.cs | 788 ++++++++++++++++++ API/Migrations/pgsql/20250628204956_AddMAL.cs | 66 ++ .../pgsql/PgsqlContextModelSnapshot.cs | 64 ++ API/Schema/Contexts/PgsqlContext.cs | 98 +-- API/Schema/MetadataFetchers/MetadataEntry.cs | 35 + .../MetadataFetchers/MetadataFetcher.cs | 38 + API/Schema/MetadataFetchers/MyAnimeList.cs | 70 ++ API/Tranga.cs | 2 + README.md | 2 + Tranga.sln.DotSettings | 1 + 12 files changed, 1219 insertions(+), 54 deletions(-) create mode 100644 API/Controllers/MetadataFetcherController.cs create mode 100644 API/Migrations/pgsql/20250628204956_AddMAL.Designer.cs create mode 100644 API/Migrations/pgsql/20250628204956_AddMAL.cs create mode 100644 API/Schema/MetadataFetchers/MetadataEntry.cs create mode 100644 API/Schema/MetadataFetchers/MetadataFetcher.cs create mode 100644 API/Schema/MetadataFetchers/MyAnimeList.cs diff --git a/API/API.csproj b/API/API.csproj index 1cb720c..9771a05 100644 --- a/API/API.csproj +++ b/API/API.csproj @@ -12,6 +12,7 @@ + diff --git a/API/Controllers/MetadataFetcherController.cs b/API/Controllers/MetadataFetcherController.cs new file mode 100644 index 0000000..51f4854 --- /dev/null +++ b/API/Controllers/MetadataFetcherController.cs @@ -0,0 +1,108 @@ +using API.Schema; +using API.Schema.Contexts; +using API.Schema.MetadataFetchers; +using Asp.Versioning; +using log4net; +using Microsoft.AspNetCore.Mvc; +using static Microsoft.AspNetCore.Http.StatusCodes; +// ReSharper disable InconsistentNaming + +namespace API.Controllers; + +[ApiVersion(2)] +[ApiController] +[Route("v{v:apiVersion}/[controller]")] +public class MetadataFetcherController(PgsqlContext context, ILog Log) : Controller +{ + /// + /// Get all available Connectors (Metadata-Sites) + /// + /// Names of Metadata-Fetchers + [HttpGet] + [ProducesResponseType(Status200OK, "application/json")] + public IActionResult GetConnectors() + { + string[] connectors = Tranga.MetadataFetchers.Select(f => f.MetadataFetcherName).ToArray(); + return Ok(connectors); + } + + /// + /// Returns all Mangas which have a linked Metadata-Provider + /// + /// + [HttpGet("Links")] + [ProducesResponseType(Status200OK, "application/json")] + public IActionResult GetLinkedEntries() + { + return Ok(context.MetadataEntries.ToArray()); + } + + /// + /// Tries linking a Manga to a Metadata-Provider-Site + /// + /// + /// Metadata-fetcher with Name does not exist + /// Manga with ID not found + /// Could not find Entry on Metadata-Provider for Manga + /// Error during Database Operation + [HttpPost("{MetadataFetcherName}/{MangaId}/TryLink")] + [ProducesResponseType(Status200OK, "application/json")] + [ProducesResponseType(Status400BadRequest)] + [ProducesResponseType(Status404NotFound)] + [ProducesResponseType(Status417ExpectationFailed)] + [ProducesResponseType(Status500InternalServerError, "text/plain")] + public IActionResult LinkMangaToMetadataFetcher(string MangaId, string MetadataFetcherName) + { + if(context.Mangas.Find(MangaId) is not { } manga) + return NotFound(); + if(Tranga.MetadataFetchers.FirstOrDefault(f => f.MetadataFetcherName == MetadataFetcherName) is not { } fetcher) + return BadRequest(); + if (!fetcher.TryGetMetadataEntry(manga, out MetadataEntry? entry)) + { + return StatusCode(Status417ExpectationFailed, "Metadata entry not found"); + } + + try + { + //Unlink previous metadata-entries + IQueryable metadataEntries = context.MetadataEntries.Where(e => e.MangaId == MangaId); + context.MetadataEntries.RemoveRange(metadataEntries); + //Add new metadata-entry + context.MetadataEntries.Add(entry); + context.SaveChanges(); + return Ok(entry); + } + catch (Exception e) + { + Log.Error(e); + return StatusCode(500, e.Message); + } + } + + /// + /// Tries linking a Manga to a Metadata-Provider-Site + /// + /// + /// Manga has no linked entry with MetadataFetcher + /// Error during Database Operation + [HttpPost("{MetadataFetcherName}/{MangaId}/UpdateMetadata")] + [ProducesResponseType(Status200OK)] + [ProducesResponseType(Status400BadRequest)] + [ProducesResponseType(Status500InternalServerError, "text/plain")] + public IActionResult UpdateMetadata(string MangaId, string MetadataFetcherName) + { + if(context.MetadataEntries.Find(MangaId, MetadataFetcherName) is not { } entry) + return BadRequest(); + MetadataFetcher fetcher = Tranga.MetadataFetchers.FirstOrDefault(f => f.MetadataFetcherName == MetadataFetcherName)!; + try + { + fetcher.UpdateMetadata(entry, context); + } + catch (Exception e) + { + Log.Error(e); + return StatusCode(500, e.Message); + } + return Ok(); + } +} \ No newline at end of file diff --git a/API/Migrations/pgsql/20250628204956_AddMAL.Designer.cs b/API/Migrations/pgsql/20250628204956_AddMAL.Designer.cs new file mode 100644 index 0000000..e5ada84 --- /dev/null +++ b/API/Migrations/pgsql/20250628204956_AddMAL.Designer.cs @@ -0,0 +1,788 @@ +// +using System; +using API.Schema.Contexts; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace API.Migrations.pgsql +{ + [DbContext(typeof(PgsqlContext))] + [Migration("20250628204956_AddMAL")] + partial class AddMAL + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.5") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("API.Schema.Author", b => + { + b.Property("AuthorId") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("AuthorName") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.HasKey("AuthorId"); + + b.ToTable("Authors"); + }); + + modelBuilder.Entity("API.Schema.Chapter", b => + { + b.Property("ChapterId") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("ChapterNumber") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)"); + + b.Property("Downloaded") + .HasColumnType("boolean"); + + b.Property("FileName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("IdOnConnectorSite") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("ParentMangaId") + .IsRequired() + .HasColumnType("character varying(64)"); + + b.Property("Title") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Url") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("VolumeNumber") + .HasColumnType("integer"); + + b.HasKey("ChapterId"); + + b.HasIndex("ParentMangaId"); + + b.ToTable("Chapters"); + }); + + modelBuilder.Entity("API.Schema.Jobs.Job", b => + { + b.Property("JobId") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("Enabled") + .HasColumnType("boolean"); + + b.Property("JobType") + .HasColumnType("smallint"); + + b.Property("LastExecution") + .HasColumnType("timestamp with time zone"); + + b.Property("ParentJobId") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("RecurrenceMs") + .HasColumnType("numeric(20,0)"); + + b.Property("state") + .HasColumnType("smallint"); + + b.HasKey("JobId"); + + b.HasIndex("ParentJobId"); + + b.ToTable("Jobs"); + + b.HasDiscriminator("JobType"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("API.Schema.LocalLibrary", b => + { + b.Property("LocalLibraryId") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("BasePath") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("LibraryName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.HasKey("LocalLibraryId"); + + b.ToTable("LocalLibraries"); + }); + + modelBuilder.Entity("API.Schema.Manga", b => + { + b.Property("MangaId") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("CoverFileNameInCache") + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.Property("CoverUrl") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("DirectoryName") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("IdOnConnectorSite") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("IgnoreChaptersBefore") + .HasColumnType("real"); + + b.Property("LibraryId") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("MangaConnectorName") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.Property("OriginalLanguage") + .HasMaxLength(8) + .HasColumnType("character varying(8)"); + + b.Property("ReleaseStatus") + .HasColumnType("smallint"); + + b.Property("WebsiteUrl") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("character varying(512)"); + + b.Property("Year") + .HasColumnType("bigint"); + + b.HasKey("MangaId"); + + b.HasIndex("LibraryId"); + + b.HasIndex("MangaConnectorName"); + + b.ToTable("Mangas"); + }); + + modelBuilder.Entity("API.Schema.MangaConnectors.MangaConnector", b => + { + b.Property("Name") + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.PrimitiveCollection("BaseUris") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("text[]"); + + b.Property("Enabled") + .HasColumnType("boolean"); + + b.Property("IconUrl") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.PrimitiveCollection("SupportedLanguages") + .IsRequired() + .HasMaxLength(8) + .HasColumnType("text[]"); + + b.HasKey("Name"); + + b.ToTable("MangaConnectors"); + + b.HasDiscriminator("Name").HasValue("MangaConnector"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("API.Schema.MangaTag", b => + { + b.Property("Tag") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.HasKey("Tag"); + + b.ToTable("Tags"); + }); + + modelBuilder.Entity("API.Schema.MetadataFetchers.MetadataEntry", b => + { + b.Property("MangaId") + .HasColumnType("character varying(64)"); + + b.Property("MetadataFetcherName") + .HasColumnType("text"); + + b.Property("Identifier") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("MangaId", "MetadataFetcherName"); + + b.HasIndex("MetadataFetcherName"); + + b.ToTable("MetadataEntries"); + }); + + modelBuilder.Entity("API.Schema.MetadataFetchers.MetadataFetcher", b => + { + b.Property("MetadataFetcherName") + .HasColumnType("text"); + + b.Property("MetadataEntry") + .IsRequired() + .HasMaxLength(21) + .HasColumnType("character varying(21)"); + + b.HasKey("MetadataFetcherName"); + + b.ToTable("MetadataFetcher"); + + b.HasDiscriminator("MetadataEntry").HasValue("MetadataFetcher"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("AuthorToManga", b => + { + b.Property("AuthorIds") + .HasColumnType("character varying(64)"); + + b.Property("MangaIds") + .HasColumnType("character varying(64)"); + + b.HasKey("AuthorIds", "MangaIds"); + + b.HasIndex("MangaIds"); + + b.ToTable("AuthorToManga"); + }); + + modelBuilder.Entity("JobJob", b => + { + b.Property("DependsOnJobsJobId") + .HasColumnType("character varying(64)"); + + b.Property("JobId") + .HasColumnType("character varying(64)"); + + b.HasKey("DependsOnJobsJobId", "JobId"); + + b.HasIndex("JobId"); + + b.ToTable("JobJob"); + }); + + modelBuilder.Entity("MangaTagToManga", b => + { + b.Property("MangaTagIds") + .HasColumnType("character varying(64)"); + + b.Property("MangaIds") + .HasColumnType("character varying(64)"); + + b.HasKey("MangaTagIds", "MangaIds"); + + b.HasIndex("MangaIds"); + + b.ToTable("MangaTagToManga"); + }); + + modelBuilder.Entity("API.Schema.Jobs.DownloadAvailableChaptersJob", b => + { + b.HasBaseType("API.Schema.Jobs.Job"); + + b.Property("MangaId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.HasIndex("MangaId"); + + b.ToTable("Jobs", t => + { + t.Property("MangaId") + .HasColumnName("DownloadAvailableChaptersJob_MangaId"); + }); + + b.HasDiscriminator().HasValue((byte)1); + }); + + modelBuilder.Entity("API.Schema.Jobs.DownloadMangaCoverJob", b => + { + b.HasBaseType("API.Schema.Jobs.Job"); + + b.Property("MangaId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.HasIndex("MangaId"); + + b.HasDiscriminator().HasValue((byte)4); + }); + + modelBuilder.Entity("API.Schema.Jobs.DownloadSingleChapterJob", b => + { + b.HasBaseType("API.Schema.Jobs.Job"); + + b.Property("ChapterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.HasIndex("ChapterId"); + + b.HasDiscriminator().HasValue((byte)0); + }); + + modelBuilder.Entity("API.Schema.Jobs.MoveFileOrFolderJob", b => + { + b.HasBaseType("API.Schema.Jobs.Job"); + + b.Property("FromLocation") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("ToLocation") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.HasDiscriminator().HasValue((byte)3); + }); + + modelBuilder.Entity("API.Schema.Jobs.MoveMangaLibraryJob", b => + { + b.HasBaseType("API.Schema.Jobs.Job"); + + b.Property("MangaId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("ToLibraryId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.HasIndex("MangaId"); + + b.HasIndex("ToLibraryId"); + + b.ToTable("Jobs", t => + { + t.Property("MangaId") + .HasColumnName("MoveMangaLibraryJob_MangaId"); + }); + + b.HasDiscriminator().HasValue((byte)7); + }); + + modelBuilder.Entity("API.Schema.Jobs.RetrieveChaptersJob", b => + { + b.HasBaseType("API.Schema.Jobs.Job"); + + b.Property("Language") + .IsRequired() + .HasMaxLength(8) + .HasColumnType("character varying(8)"); + + b.Property("MangaId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.HasIndex("MangaId"); + + b.ToTable("Jobs", t => + { + t.Property("MangaId") + .HasColumnName("RetrieveChaptersJob_MangaId"); + }); + + b.HasDiscriminator().HasValue((byte)5); + }); + + modelBuilder.Entity("API.Schema.Jobs.UpdateChaptersDownloadedJob", b => + { + b.HasBaseType("API.Schema.Jobs.Job"); + + b.Property("MangaId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.HasIndex("MangaId"); + + b.ToTable("Jobs", t => + { + t.Property("MangaId") + .HasColumnName("UpdateChaptersDownloadedJob_MangaId"); + }); + + b.HasDiscriminator().HasValue((byte)6); + }); + + modelBuilder.Entity("API.Schema.Jobs.UpdateCoverJob", b => + { + b.HasBaseType("API.Schema.Jobs.Job"); + + b.Property("MangaId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.HasIndex("MangaId"); + + b.ToTable("Jobs", t => + { + t.Property("MangaId") + .HasColumnName("UpdateCoverJob_MangaId"); + }); + + b.HasDiscriminator().HasValue((byte)9); + }); + + modelBuilder.Entity("API.Schema.MangaConnectors.ComickIo", b => + { + b.HasBaseType("API.Schema.MangaConnectors.MangaConnector"); + + b.HasDiscriminator().HasValue("ComickIo"); + }); + + modelBuilder.Entity("API.Schema.MangaConnectors.Global", b => + { + b.HasBaseType("API.Schema.MangaConnectors.MangaConnector"); + + b.HasDiscriminator().HasValue("Global"); + }); + + modelBuilder.Entity("API.Schema.MangaConnectors.MangaDex", b => + { + b.HasBaseType("API.Schema.MangaConnectors.MangaConnector"); + + b.HasDiscriminator().HasValue("MangaDex"); + }); + + modelBuilder.Entity("API.Schema.MetadataFetchers.MyAnimeList", b => + { + b.HasBaseType("API.Schema.MetadataFetchers.MetadataFetcher"); + + b.HasDiscriminator().HasValue("MyAnimeList"); + }); + + modelBuilder.Entity("API.Schema.Chapter", b => + { + b.HasOne("API.Schema.Manga", "ParentManga") + .WithMany("Chapters") + .HasForeignKey("ParentMangaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ParentManga"); + }); + + modelBuilder.Entity("API.Schema.Jobs.Job", b => + { + b.HasOne("API.Schema.Jobs.Job", "ParentJob") + .WithMany() + .HasForeignKey("ParentJobId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("ParentJob"); + }); + + modelBuilder.Entity("API.Schema.Manga", b => + { + b.HasOne("API.Schema.LocalLibrary", "Library") + .WithMany() + .HasForeignKey("LibraryId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("API.Schema.MangaConnectors.MangaConnector", "MangaConnector") + .WithMany() + .HasForeignKey("MangaConnectorName") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsMany("API.Schema.Link", "Links", b1 => + { + b1.Property("LinkId") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b1.Property("LinkProvider") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b1.Property("LinkUrl") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b1.Property("MangaId") + .IsRequired() + .HasColumnType("character varying(64)"); + + b1.HasKey("LinkId"); + + b1.HasIndex("MangaId"); + + b1.ToTable("Link"); + + b1.WithOwner() + .HasForeignKey("MangaId"); + }); + + b.OwnsMany("API.Schema.MangaAltTitle", "AltTitles", b1 => + { + b1.Property("AltTitleId") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b1.Property("Language") + .IsRequired() + .HasMaxLength(8) + .HasColumnType("character varying(8)"); + + b1.Property("MangaId") + .IsRequired() + .HasColumnType("character varying(64)"); + + b1.Property("Title") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b1.HasKey("AltTitleId"); + + b1.HasIndex("MangaId"); + + b1.ToTable("MangaAltTitle"); + + b1.WithOwner() + .HasForeignKey("MangaId"); + }); + + b.Navigation("AltTitles"); + + b.Navigation("Library"); + + b.Navigation("Links"); + + b.Navigation("MangaConnector"); + }); + + modelBuilder.Entity("API.Schema.MetadataFetchers.MetadataEntry", b => + { + b.HasOne("API.Schema.Manga", "Manga") + .WithMany() + .HasForeignKey("MangaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Schema.MetadataFetchers.MetadataFetcher", "MetadataFetcher") + .WithMany() + .HasForeignKey("MetadataFetcherName") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Manga"); + + b.Navigation("MetadataFetcher"); + }); + + modelBuilder.Entity("AuthorToManga", b => + { + b.HasOne("API.Schema.Author", null) + .WithMany() + .HasForeignKey("AuthorIds") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Schema.Manga", null) + .WithMany() + .HasForeignKey("MangaIds") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("JobJob", b => + { + b.HasOne("API.Schema.Jobs.Job", null) + .WithMany() + .HasForeignKey("DependsOnJobsJobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Schema.Jobs.Job", null) + .WithMany() + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("MangaTagToManga", b => + { + b.HasOne("API.Schema.Manga", null) + .WithMany() + .HasForeignKey("MangaIds") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Schema.MangaTag", null) + .WithMany() + .HasForeignKey("MangaTagIds") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("API.Schema.Jobs.DownloadAvailableChaptersJob", b => + { + b.HasOne("API.Schema.Manga", "Manga") + .WithMany() + .HasForeignKey("MangaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Manga"); + }); + + modelBuilder.Entity("API.Schema.Jobs.DownloadMangaCoverJob", b => + { + b.HasOne("API.Schema.Manga", "Manga") + .WithMany() + .HasForeignKey("MangaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Manga"); + }); + + modelBuilder.Entity("API.Schema.Jobs.DownloadSingleChapterJob", b => + { + b.HasOne("API.Schema.Chapter", "Chapter") + .WithMany() + .HasForeignKey("ChapterId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Chapter"); + }); + + modelBuilder.Entity("API.Schema.Jobs.MoveMangaLibraryJob", b => + { + b.HasOne("API.Schema.Manga", "Manga") + .WithMany() + .HasForeignKey("MangaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Schema.LocalLibrary", "ToLibrary") + .WithMany() + .HasForeignKey("ToLibraryId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Manga"); + + b.Navigation("ToLibrary"); + }); + + modelBuilder.Entity("API.Schema.Jobs.RetrieveChaptersJob", b => + { + b.HasOne("API.Schema.Manga", "Manga") + .WithMany() + .HasForeignKey("MangaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Manga"); + }); + + modelBuilder.Entity("API.Schema.Jobs.UpdateChaptersDownloadedJob", b => + { + b.HasOne("API.Schema.Manga", "Manga") + .WithMany() + .HasForeignKey("MangaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Manga"); + }); + + modelBuilder.Entity("API.Schema.Jobs.UpdateCoverJob", b => + { + b.HasOne("API.Schema.Manga", "Manga") + .WithMany() + .HasForeignKey("MangaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Manga"); + }); + + modelBuilder.Entity("API.Schema.Manga", b => + { + b.Navigation("Chapters"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/API/Migrations/pgsql/20250628204956_AddMAL.cs b/API/Migrations/pgsql/20250628204956_AddMAL.cs new file mode 100644 index 0000000..1ddb427 --- /dev/null +++ b/API/Migrations/pgsql/20250628204956_AddMAL.cs @@ -0,0 +1,66 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace API.Migrations.pgsql +{ + /// + public partial class AddMAL : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "MetadataFetcher", + columns: table => new + { + MetadataFetcherName = table.Column(type: "text", nullable: false), + MetadataEntry = table.Column(type: "character varying(21)", maxLength: 21, nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_MetadataFetcher", x => x.MetadataFetcherName); + }); + + migrationBuilder.CreateTable( + name: "MetadataEntries", + columns: table => new + { + MangaId = table.Column(type: "character varying(64)", nullable: false), + MetadataFetcherName = table.Column(type: "text", nullable: false), + Identifier = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_MetadataEntries", x => new { x.MangaId, x.MetadataFetcherName }); + table.ForeignKey( + name: "FK_MetadataEntries_Mangas_MangaId", + column: x => x.MangaId, + principalTable: "Mangas", + principalColumn: "MangaId", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_MetadataEntries_MetadataFetcher_MetadataFetcherName", + column: x => x.MetadataFetcherName, + principalTable: "MetadataFetcher", + principalColumn: "MetadataFetcherName", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_MetadataEntries_MetadataFetcherName", + table: "MetadataEntries", + column: "MetadataFetcherName"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "MetadataEntries"); + + migrationBuilder.DropTable( + name: "MetadataFetcher"); + } + } +} diff --git a/API/Migrations/pgsql/PgsqlContextModelSnapshot.cs b/API/Migrations/pgsql/PgsqlContextModelSnapshot.cs index 1272fa2..3637290 100644 --- a/API/Migrations/pgsql/PgsqlContextModelSnapshot.cs +++ b/API/Migrations/pgsql/PgsqlContextModelSnapshot.cs @@ -291,6 +291,44 @@ namespace API.Migrations.pgsql b.ToTable("Tags"); }); + modelBuilder.Entity("API.Schema.MetadataFetchers.MetadataEntry", b => + { + b.Property("MangaId") + .HasColumnType("character varying(64)"); + + b.Property("MetadataFetcherName") + .HasColumnType("text"); + + b.Property("Identifier") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("MangaId", "MetadataFetcherName"); + + b.HasIndex("MetadataFetcherName"); + + b.ToTable("MetadataEntries"); + }); + + modelBuilder.Entity("API.Schema.MetadataFetchers.MetadataFetcher", b => + { + b.Property("MetadataFetcherName") + .HasColumnType("text"); + + b.Property("MetadataEntry") + .IsRequired() + .HasMaxLength(21) + .HasColumnType("character varying(21)"); + + b.HasKey("MetadataFetcherName"); + + b.ToTable("MetadataFetcher"); + + b.HasDiscriminator("MetadataEntry").HasValue("MetadataFetcher"); + + b.UseTphMappingStrategy(); + }); + modelBuilder.Entity("AuthorToManga", b => { b.Property("AuthorIds") @@ -514,6 +552,13 @@ namespace API.Migrations.pgsql b.HasDiscriminator().HasValue("MangaDex"); }); + modelBuilder.Entity("API.Schema.MetadataFetchers.MyAnimeList", b => + { + b.HasBaseType("API.Schema.MetadataFetchers.MetadataFetcher"); + + b.HasDiscriminator().HasValue("MyAnimeList"); + }); + modelBuilder.Entity("API.Schema.Chapter", b => { b.HasOne("API.Schema.Manga", "ParentManga") @@ -645,6 +690,25 @@ namespace API.Migrations.pgsql b.Navigation("Obj"); }); + modelBuilder.Entity("API.Schema.MetadataFetchers.MetadataEntry", b => + { + b.HasOne("API.Schema.Manga", "Manga") + .WithMany() + .HasForeignKey("MangaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Schema.MetadataFetchers.MetadataFetcher", "MetadataFetcher") + .WithMany() + .HasForeignKey("MetadataFetcherName") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Manga"); + + b.Navigation("MetadataFetcher"); + }); + modelBuilder.Entity("AuthorToManga", b => { b.HasOne("API.Schema.Author", null) diff --git a/API/Schema/Contexts/PgsqlContext.cs b/API/Schema/Contexts/PgsqlContext.cs index 3c2c927..7b99e0f 100644 --- a/API/Schema/Contexts/PgsqlContext.cs +++ b/API/Schema/Contexts/PgsqlContext.cs @@ -1,5 +1,6 @@ using API.Schema.Jobs; using API.Schema.MangaConnectors; +using API.Schema.MetadataFetchers; using log4net; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Diagnostics; @@ -11,12 +12,11 @@ public class PgsqlContext(DbContextOptions options) : DbContext(op public DbSet Jobs { get; set; } public DbSet MangaConnectors { get; set; } public DbSet Mangas { get; set; } - public DbSet LocalLibraries { get; set; } + public DbSet LocalLibraries { get; set; } public DbSet Chapters { get; set; } public DbSet Authors { get; set; } public DbSet Tags { get; set; } - public DbSet> MangaConnectorToManga { get; set; } - public DbSet> MangaConnectorToChapter { get; set; } + public DbSet MetadataEntries { get; set; } private ILog Log => LogManager.GetLogger(GetType()); protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) @@ -42,7 +42,8 @@ public class PgsqlContext(DbContextOptions options) : DbContext(op .HasValue(JobType.RetrieveChaptersJob) .HasValue(JobType.UpdateCoverJob) .HasValue(JobType.UpdateChaptersDownloadedJob); - + + //Job specification modelBuilder.Entity() .HasOne(j => j.Manga) .WithMany() @@ -76,12 +77,12 @@ public class PgsqlContext(DbContextOptions options) : DbContext(op .Navigation(j => j.Manga) .EnableLazyLoading(); modelBuilder.Entity() - .HasOne(j => j.ToFileLibrary) + .HasOne(j => j.ToLibrary) .WithMany() .HasForeignKey(j => j.ToLibraryId) .OnDelete(DeleteBehavior.Cascade); modelBuilder.Entity() - .Navigation(j => j.ToFileLibrary) + .Navigation(j => j.ToLibrary) .EnableLazyLoading(); modelBuilder.Entity() .HasOne(j => j.Manga) @@ -104,17 +105,15 @@ public class PgsqlContext(DbContextOptions options) : DbContext(op modelBuilder.Entity() .HasOne(childJob => childJob.ParentJob) .WithMany() - .HasForeignKey(childJob => childJob.ParentJobId) + .HasForeignKey(childjob => childjob.ParentJobId) .OnDelete(DeleteBehavior.Cascade); - modelBuilder.Entity() - .Navigation(childJob => childJob.ParentJob) - .EnableLazyLoading(); //Job might be dependent on other Jobs modelBuilder.Entity() .HasMany(root => root.DependsOnJobs) .WithMany(); modelBuilder.Entity() .Navigation(j => j.DependsOnJobs) + .AutoInclude(false) .EnableLazyLoading(); //MangaConnector Types @@ -123,6 +122,15 @@ public class PgsqlContext(DbContextOptions options) : DbContext(op .HasValue("Global") .HasValue("MangaDex") .HasValue("ComickIo"); + //MangaConnector is responsible for many Manga + modelBuilder.Entity() + .HasMany() + .WithOne(m => m.MangaConnector) + .HasForeignKey(m => m.MangaConnectorName) + .OnDelete(DeleteBehavior.Cascade); + modelBuilder.Entity() + .Navigation(m => m.MangaConnector) + .AutoInclude(); //Manga has many Chapters modelBuilder.Entity() @@ -130,29 +138,16 @@ public class PgsqlContext(DbContextOptions options) : DbContext(op .WithOne(c => c.ParentManga) .HasForeignKey(c => c.ParentMangaId) .OnDelete(DeleteBehavior.Cascade); - modelBuilder.Entity() - .Navigation(m => m.Chapters) - .EnableLazyLoading(); modelBuilder.Entity() .Navigation(c => c.ParentManga) - .EnableLazyLoading(); - //Chapter has MangaConnectorIds - modelBuilder.Entity() - .HasMany>(c => c.MangaConnectorIds) - .WithOne(id => id.Obj) - .HasForeignKey(id => id.ObjId) - .OnDelete(DeleteBehavior.NoAction); - modelBuilder.Entity>() - .HasOne(id => id.MangaConnector) - .WithMany() - .HasForeignKey(id => id.MangaConnectorName) - .OnDelete(DeleteBehavior.Cascade); - modelBuilder.Entity>() - .Navigation(entry => entry.MangaConnector) + .AutoInclude(); + modelBuilder.Entity() + .Navigation(m => m.Chapters) + .AutoInclude(false) .EnableLazyLoading(); //Manga owns MangaAltTitles modelBuilder.Entity() - .OwnsMany(m => m.AltTitles) + .OwnsMany(m => m.AltTitles) .WithOwner(); modelBuilder.Entity() .Navigation(m => m.AltTitles) @@ -164,57 +159,52 @@ public class PgsqlContext(DbContextOptions options) : DbContext(op modelBuilder.Entity() .Navigation(m => m.Links) .AutoInclude(); - //Manga has many Tags associated with many Obj + //Manga has many Tags associated with many Manga modelBuilder.Entity() .HasMany(m => m.MangaTags) .WithMany() .UsingEntity("MangaTagToManga", l=> l.HasOne(typeof(MangaTag)).WithMany().HasForeignKey("MangaTagIds").HasPrincipalKey(nameof(MangaTag.Tag)), - r => r.HasOne(typeof(Manga)).WithMany().HasForeignKey("MangaIds").HasPrincipalKey(nameof(Manga.Key)), + r => r.HasOne(typeof(Manga)).WithMany().HasForeignKey("MangaIds").HasPrincipalKey(nameof(Manga.MangaId)), j => j.HasKey("MangaTagIds", "MangaIds") ); modelBuilder.Entity() .Navigation(m => m.MangaTags) .AutoInclude(); - //Manga has many Authors associated with many Obj + //Manga has many Authors associated with many Manga modelBuilder.Entity() .HasMany(m => m.Authors) .WithMany() .UsingEntity("AuthorToManga", - l=> l.HasOne(typeof(Author)).WithMany().HasForeignKey("AuthorIds").HasPrincipalKey(nameof(Author.Key)), - r => r.HasOne(typeof(Manga)).WithMany().HasForeignKey("MangaIds").HasPrincipalKey(nameof(Manga.Key)), + l=> l.HasOne(typeof(Author)).WithMany().HasForeignKey("AuthorIds").HasPrincipalKey(nameof(Author.AuthorId)), + r => r.HasOne(typeof(Manga)).WithMany().HasForeignKey("MangaIds").HasPrincipalKey(nameof(Manga.MangaId)), j => j.HasKey("AuthorIds", "MangaIds") ); modelBuilder.Entity() .Navigation(m => m.Authors) .AutoInclude(); - //Manga has many MangaIds - modelBuilder.Entity() - .HasMany>(m => m.MangaConnectorIds) - .WithOne(id => id.Obj) - .HasForeignKey(id => id.ObjId) - .OnDelete(DeleteBehavior.Cascade); - modelBuilder.Entity() - .Navigation(m => m.MangaConnectorIds) - .EnableLazyLoading(); - modelBuilder.Entity>() - .HasOne(id => id.MangaConnector) - .WithMany() - .HasForeignKey(id => id.MangaConnectorName) - .OnDelete(DeleteBehavior.Cascade); - modelBuilder.Entity>() - .Navigation(entry => entry.MangaConnector) - .EnableLazyLoading(); - - //FileLibrary has many Mangas - modelBuilder.Entity() + //LocalLibrary has many Mangas + modelBuilder.Entity() .HasMany() .WithOne(m => m.Library) .HasForeignKey(m => m.LibraryId) .OnDelete(DeleteBehavior.SetNull); modelBuilder.Entity() .Navigation(m => m.Library) - .EnableLazyLoading(); + .AutoInclude(); + + modelBuilder.Entity() + .HasDiscriminator(nameof(MetadataEntry)) + .HasValue(nameof(MyAnimeList)); + //MetadataEntry + modelBuilder.Entity() + .HasOne(entry => entry.Manga) + .WithMany() + .OnDelete(DeleteBehavior.Cascade); + modelBuilder.Entity() + .HasOne(entry => entry.MetadataFetcher) + .WithMany() + .OnDelete(DeleteBehavior.Cascade); } } \ No newline at end of file diff --git a/API/Schema/MetadataFetchers/MetadataEntry.cs b/API/Schema/MetadataFetchers/MetadataEntry.cs new file mode 100644 index 0000000..44614e5 --- /dev/null +++ b/API/Schema/MetadataFetchers/MetadataEntry.cs @@ -0,0 +1,35 @@ +using Microsoft.EntityFrameworkCore; +using Newtonsoft.Json; + +namespace API.Schema.MetadataFetchers; + +[PrimaryKey("MangaId", "MetadataFetcherName")] +public class MetadataEntry +{ + [JsonIgnore] + public Manga Manga { get; init; } = null!; + public string MangaId { get; init; } + [JsonIgnore] + public MetadataFetcher MetadataFetcher { get; init; } = null!; + public string MetadataFetcherName { get; init; } + public string Identifier { get; init; } + + public MetadataEntry(MetadataFetcher fetcher, Manga manga, string identifier) + { + this.Manga = manga; + this.MangaId = manga.MangaId; + this.MetadataFetcher = fetcher; + this.MetadataFetcherName = fetcher.MetadataFetcherName; + this.Identifier = identifier; + } + + /// + /// EFCORE only!!!! + /// + internal MetadataEntry(string mangaId, string identifier, string metadataFetcherName) + { + this.MangaId = mangaId; + this.Identifier = identifier; + this.MetadataFetcherName = metadataFetcherName; + } +} \ No newline at end of file diff --git a/API/Schema/MetadataFetchers/MetadataFetcher.cs b/API/Schema/MetadataFetchers/MetadataFetcher.cs new file mode 100644 index 0000000..48b7c07 --- /dev/null +++ b/API/Schema/MetadataFetchers/MetadataFetcher.cs @@ -0,0 +1,38 @@ +using System.Diagnostics.CodeAnalysis; +using API.Schema.Contexts; +using Microsoft.EntityFrameworkCore; + +namespace API.Schema.MetadataFetchers; + +[PrimaryKey("MetadataFetcherName")] +public abstract class MetadataFetcher +{ + // ReSharper disable once EntityFramework.ModelValidation.UnlimitedStringLength + public string MetadataFetcherName { get; init; } + + protected MetadataFetcher() + { + this.MetadataFetcherName = this.GetType().Name; + } + + /// + /// EFCORE ONLY!!! + /// + internal MetadataFetcher(string metadataFetcherName) + { + this.MetadataFetcherName = metadataFetcherName; + } + + public abstract MetadataEntry? FindLinkedMetadataEntry(Manga manga); + + public bool TryGetMetadataEntry(Manga manga, [NotNullWhen(true)] out MetadataEntry? metadataEntry) + { + metadataEntry = FindLinkedMetadataEntry(manga); + return metadataEntry != null; + } + + /// + /// Updates the Manga linked in the MetadataEntry + /// + public abstract void UpdateMetadata(MetadataEntry metadataEntry, PgsqlContext dbContext); +} \ No newline at end of file diff --git a/API/Schema/MetadataFetchers/MyAnimeList.cs b/API/Schema/MetadataFetchers/MyAnimeList.cs new file mode 100644 index 0000000..2497c98 --- /dev/null +++ b/API/Schema/MetadataFetchers/MyAnimeList.cs @@ -0,0 +1,70 @@ +using System.Text.RegularExpressions; +using API.Schema.Contexts; +using JikanDotNet; +using Microsoft.EntityFrameworkCore; + +namespace API.Schema.MetadataFetchers; + +public class MyAnimeList : MetadataFetcher +{ + private static readonly Jikan Jikan = new (); + private static readonly Regex GetIdFromUrl = new(@"https?:\/\/myanimelist\.net\/manga\/([0-9]+)\/?.*"); + + public override MetadataEntry? FindLinkedMetadataEntry(Manga manga) + { + if (manga.Links.Any(link => link.LinkProvider.Equals("MyAnimeList", StringComparison.InvariantCultureIgnoreCase))) + { + string url = manga.Links.First(link => link.LinkProvider.Equals("MyAnimeList", StringComparison.InvariantCultureIgnoreCase)).LinkUrl; + Match m = GetIdFromUrl.Match(url); + if (m.Success && m.Groups[1].Success) + { + long id = long.Parse(m.Groups[1].Value); + return new MetadataEntry(this, manga, id.ToString()!); + } + } + + ICollection resultData = Jikan.SearchMangaAsync(manga.Name).Result.Data; + if (resultData.Count < 1) + return null; + return new MetadataEntry(this, manga, resultData.First().MalId.ToString()); + } + + /// + /// Updates the Manga linked in the MetadataEntry + /// + /// + /// + /// + /// + public override void UpdateMetadata(MetadataEntry metadataEntry, PgsqlContext dbContext) + { + Manga dbManga = dbContext.Mangas.Find(metadataEntry.MangaId)!; + MangaFull resultData; + try + { + long id = long.Parse(metadataEntry.Identifier); + resultData = Jikan.GetMangaFullDataAsync(id).Result.Data; + } + catch (Exception) + { + throw new FormatException("ID was not in correct format"); + } + + try + { + dbManga.Name = resultData.Titles.First().Title; + dbManga.Description = resultData.Synopsis; + dbManga.AltTitles.Clear(); + dbManga.AltTitles = resultData.Titles.Select(t => new MangaAltTitle(t.Type, t.Title)).ToList(); + dbManga.Authors.Clear(); + dbManga.Authors = resultData.Authors.Select(a => new Author(a.Name)).ToList(); + + dbContext.SaveChanges(); + } + catch (DbUpdateException e) + { + throw; + } + } + +} \ No newline at end of file diff --git a/API/Tranga.cs b/API/Tranga.cs index ecfa0fc..d0c75c3 100644 --- a/API/Tranga.cs +++ b/API/Tranga.cs @@ -2,6 +2,7 @@ using API.Schema.Contexts; using API.Schema.Jobs; using API.Schema.MangaConnectors; +using API.Schema.MetadataFetchers; using API.Schema.NotificationConnectors; using log4net; using log4net.Config; @@ -24,6 +25,7 @@ public static class Tranga public static Thread NotificationSenderThread { get; } = new (NotificationSender); public static Thread JobStarterThread { get; } = new (JobStarter); private static readonly ILog Log = LogManager.GetLogger(typeof(Tranga)); + internal static MetadataFetcher[] MetadataFetchers = [new MyAnimeList()]; internal static void StartLogger() { diff --git a/README.md b/README.md index bc848b6..3599b75 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,8 @@ Endpoints are documented in Swagger. Just spin up an instance, and go to `http:/ - [PuppeteerSharp](https://github.com/hardkoded/puppeteer-sharp/blob/master/LICENSE) - [Html Agility Pack (HAP)](https://github.com/zzzprojects/html-agility-pack/blob/master/LICENSE) - [Soenneker.Utils.String.NeedlemanWunsch](https://github.com/soenneker/soenneker.utils.string.needlemanwunsch/blob/main/LICENSE) +- [Jikan](https://jikan.moe/) + - [Jikan.Net](https://github.com/Ervie/jikan.net) - 💙 Blåhaj 🦈

(back to top)

diff --git a/Tranga.sln.DotSettings b/Tranga.sln.DotSettings index c9d6e11..718e87a 100644 --- a/Tranga.sln.DotSettings +++ b/Tranga.sln.DotSettings @@ -3,6 +3,7 @@ True True True + True True True True