23 Commits

Author SHA1 Message Date
7c9e0eddf9 Metadata-Site Search (Interactive linking) 2025-06-29 21:13:05 +02:00
ae0c6c8240 Change PrimaryKey of MetadataEntry to Fetcher + Identifier 2025-06-29 20:43:21 +02:00
2eb0d941b2 MetadataFetching:
- Jikan (MAL) linking, fetching/updating
2025-06-28 22:59:35 +02:00
8170e1d762 JobCycle Info-Debug list jobs started/running
Some checks failed
Docker Image CI / build (push) Has been cancelled
2025-06-28 20:35:10 +02:00
254383b006 Include Description in ComicInfo.xml 2025-06-28 20:28:28 +02:00
df431e533a Add POST Jobs/Cleaup Endpoint:
Removes failed and completed Jobs (that are not recurring)
2025-06-28 20:18:28 +02:00
9a4cc0cbaf Only log Error on image-processing if we dont know what Exception was thrown 2025-06-28 20:13:09 +02:00
861cf7e166 Fix Image-Processing:
Format is not supported by Imagesharp, throwing exception causing Job to fail.
2025-06-28 20:00:01 +02:00
7e34b3b91e Update readme to contain information on how to test locally 2025-06-28 19:48:47 +02:00
29d36484f9 include logging driver in docker-compose
Remove parameters from start-CMD in Dockerfile
2025-06-28 19:39:19 +02:00
2c6e8e4d16 Default startNewJobTimeoutMs set to 20s
Some checks failed
Docker Image CI / build (push) Has been cancelled
2025-06-18 02:11:03 +02:00
fab2886684 ComickIo Stop double work for retrieving chapters:
We can build the canonical url from the hids
2025-06-18 01:55:19 +02:00
d9ccf71b21 DownloadSingleChapterJob add check if chapter is already downloaded before re-downloading 2025-06-18 01:18:06 +02:00
f36f34f212 We dont need to actually load the MangaConnector to know if two names match.
Some checks failed
Docker Image CI / build (push) Has been cancelled
2025-06-18 00:23:33 +02:00
ff10432c79 Fix FilterJobsWithoutDownloading: Dont check if a job has a connector, that takes forever 2025-06-18 00:11:05 +02:00
776e1e4890 ...use what we coded... 2025-06-17 20:18:10 +02:00
db0643fa19 More Debug 2025-06-17 20:09:49 +02:00
3eeb563ca1 Add Debug Statement to find slow operations in Job-Cycle 2025-06-17 19:55:54 +02:00
7a88b1f7ee Increase default request Limits 2025-06-17 19:55:31 +02:00
b5411e9c6c Better Debugging for HttpDownloadClient 2025-06-17 18:52:27 +02:00
07b260dea6 GC Cleanup 2025-06-17 18:52:14 +02:00
71ad32de31 Fix FlareSolverr IsJson-Check 2025-06-17 18:51:29 +02:00
ecd2c2722f Fix FlareSolverr, Flaresolverrsharp is broken 2025-06-17 18:28:18 +02:00
27 changed files with 2504 additions and 134 deletions

View File

@ -11,8 +11,8 @@
<ItemGroup>
<PackageReference Include="Asp.Versioning.Mvc.ApiExplorer" Version="8.1.0" />
<PackageReference Include="FlareSolverrSharp" Version="3.0.7" />
<PackageReference Include="HtmlAgilityPack" Version="1.12.0" />
<PackageReference Include="JikanDotNet" Version="2.9.1" />
<PackageReference Include="log4net" Version="3.0.4" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="9.0.3" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="9.0.3" />

View File

@ -374,4 +374,25 @@ public class JobController(PgsqlContext context, ILog Log) : Controller
{
return StatusCode(Status501NotImplemented);
}
/// <summary>
/// Removes failed and completed Jobs (that are not recurring)
/// </summary>
/// <response code="202">Job started</response>
/// <response code="500">Error during Database Operation</response>
[HttpPost("Cleanup")]
public IActionResult CleanupJobs()
{
try
{
context.Jobs.RemoveRange(context.Jobs.Where(j => j.state == JobState.Failed || j.state == JobState.Completed));
context.SaveChanges();
return Ok();
}
catch (Exception e)
{
Log.Error(e);
return StatusCode(500, e.Message);
}
}
}

View File

@ -0,0 +1,166 @@
using API.Schema;
using API.Schema.Contexts;
using API.Schema.MetadataFetchers;
using Asp.Versioning;
using log4net;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Mvc.ModelBinding;
using static Microsoft.AspNetCore.Http.StatusCodes;
// ReSharper disable InconsistentNaming
namespace API.Controllers;
[ApiVersion(2)]
[ApiController]
[Route("v{v:apiVersion}/[controller]")]
public class MetadataFetcherController(PgsqlContext context, ILog Log) : Controller
{
/// <summary>
/// Get all available Connectors (Metadata-Sites)
/// </summary>
/// <response code="200">Names of Metadata-Fetchers</response>
[HttpGet]
[ProducesResponseType<string[]>(Status200OK, "application/json")]
public IActionResult GetConnectors()
{
string[] connectors = Tranga.MetadataFetchers.Select(f => f.MetadataFetcherName).ToArray();
return Ok(connectors);
}
/// <summary>
/// Returns all Mangas which have a linked Metadata-Provider
/// </summary>
/// <response code="200"></response>
[HttpGet("Links")]
[ProducesResponseType<MetadataEntry>(Status200OK, "application/json")]
public IActionResult GetLinkedEntries()
{
return Ok(context.MetadataEntries.ToArray());
}
/// <summary>
/// Searches Metadata-Provider for Manga-Metadata
/// </summary>
/// <param name="searchTerm">Instead of using the Manga for search, use a specific term</param>
/// <response code="200"></response>
/// <response code="400">Metadata-fetcher with Name does not exist</response>
/// <response code="404">Manga with ID not found</response>
[HttpPost("{MetadataFetcherName}/SearchManga/{MangaId}")]
[ProducesResponseType<MetadataSearchResult[]>(Status200OK, "application/json")]
[ProducesResponseType(Status400BadRequest)]
[ProducesResponseType(Status404NotFound)]
public IActionResult SearchMangaMetadata(string MangaId, string MetadataFetcherName, [FromBody(EmptyBodyBehavior = EmptyBodyBehavior.Allow)]string? searchTerm = null)
{
if(context.Mangas.Find(MangaId) is not { } manga)
return NotFound();
if(Tranga.MetadataFetchers.FirstOrDefault(f => f.MetadataFetcherName == MetadataFetcherName) is not { } fetcher)
return BadRequest();
MetadataSearchResult[] searchResults = searchTerm is null ? fetcher.SearchMetadataEntry(manga) : fetcher.SearchMetadataEntry(searchTerm);
return Ok(searchResults);
}
/// <summary>
/// Links Metadata-Provider using Provider-Specific Identifier to Manga
/// </summary>
/// <response code="200"></response>
/// <response code="400">Metadata-fetcher with Name does not exist</response>
/// <response code="404">Manga with ID not found</response>
/// <response code="500">Error during Database Operation</response>
[HttpPost("{MetadataFetcherName}/Link/{MangaId}")]
[ProducesResponseType(Status200OK)]
[ProducesResponseType(Status400BadRequest)]
[ProducesResponseType(Status404NotFound)]
[ProducesResponseType<string>(Status500InternalServerError, "text/plain")]
public IActionResult LinkMangaMetadata(string MangaId, string MetadataFetcherName, [FromBody]string Identifier)
{
if(context.Mangas.Find(MangaId) is not { } manga)
return NotFound();
if(Tranga.MetadataFetchers.FirstOrDefault(f => f.MetadataFetcherName == MetadataFetcherName) is not { } fetcher)
return BadRequest();
MetadataEntry entry = fetcher.CreateMetadataEntry(manga, Identifier);
try
{
context.MetadataEntries.Add(entry);
context.SaveChanges();
}
catch (Exception e)
{
Log.Error(e);
return StatusCode(500, e.Message);
}
return Ok();
}
/// <summary>
/// Un-Links Metadata-Provider using Provider-Specific Identifier to Manga
/// </summary>
/// <response code="200"></response>
/// <response code="400">Metadata-fetcher with Name does not exist</response>
/// <response code="404">Manga with ID not found</response>
/// <response code="412">No Entry linking Manga and Metadata-Provider found</response>
/// <response code="500">Error during Database Operation</response>
[HttpPost("{MetadataFetcherName}/Unlink/{MangaId}")]
[ProducesResponseType(Status200OK)]
[ProducesResponseType(Status400BadRequest)]
[ProducesResponseType(Status404NotFound)]
[ProducesResponseType<string>(Status412PreconditionFailed, "text/plain")]
[ProducesResponseType<string>(Status500InternalServerError, "text/plain")]
public IActionResult UnlinkMangaMetadata(string MangaId, string MetadataFetcherName)
{
if(context.Mangas.Find(MangaId) is not { } manga)
return NotFound();
if(Tranga.MetadataFetchers.FirstOrDefault(f => f.MetadataFetcherName == MetadataFetcherName) is not { } fetcher)
return BadRequest();
MetadataEntry? entry = context.MetadataEntries.FirstOrDefault(e => e.MangaId == MangaId && e.MetadataFetcherName == MetadataFetcherName);
if (entry is null)
return StatusCode(Status412PreconditionFailed, "No entry found");
try
{
context.MetadataEntries.Remove(entry);
context.SaveChanges();
}
catch (Exception e)
{
Log.Error(e);
return StatusCode(500, e.Message);
}
return Ok();
}
/// <summary>
/// Tries linking a Manga to a Metadata-Provider-Site
/// </summary>
/// <response code="200"></response>
/// <response code="400">MetadataFetcher Name is invalid</response>
/// <response code="404">Manga has no linked entry with MetadataFetcher</response>
/// <response code="500">Error during Database Operation</response>
[HttpPost("{MetadataFetcherName}/{MangaId}/UpdateMetadata")]
[ProducesResponseType(Status200OK)]
[ProducesResponseType(Status400BadRequest)]
[ProducesResponseType(Status404NotFound)]
[ProducesResponseType<string>(Status500InternalServerError, "text/plain")]
public IActionResult UpdateMetadata(string MangaId, string MetadataFetcherName)
{
if(Tranga.MetadataFetchers
.FirstOrDefault(f =>
f.MetadataFetcherName.Equals(MetadataFetcherName, StringComparison.InvariantCultureIgnoreCase)) is not { } fetcher)
return BadRequest();
MetadataEntry? entry = context.MetadataEntries
.FirstOrDefault(e =>
e.MangaId == MangaId && e.MetadataFetcherName.Equals(MetadataFetcherName, StringComparison.InvariantCultureIgnoreCase));
if (entry is null)
return NotFound();
try
{
fetcher.UpdateMetadata(entry, context);
}
catch (Exception e)
{
Log.Error(e);
return StatusCode(500, e.Message);
}
return Ok();
}
}

View File

@ -4,7 +4,6 @@ using API.Schema;
using API.Schema.Contexts;
using API.Schema.Jobs;
using Asp.Versioning;
using FlareSolverrSharp;
using log4net;
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json;
@ -321,43 +320,18 @@ public class SettingsController(PgsqlContext context, ILog Log) : Controller
}
/// <summary>
///
/// Test FlareSolverr
/// </summary>
/// <response code="200">FlareSolverr is working!</response>
/// <response code="400">FlareSolverr URL is malformed</response>
/// <response code="500">FlareSolverr is not working</response>
/// <response code="503">FlareSolverr could not be reached</response>
[HttpPost("FlareSolverr/Test")]
[ProducesResponseType(Status200OK)]
[ProducesResponseType(Status400BadRequest)]
[ProducesResponseType(Status500InternalServerError)]
[ProducesResponseType(Status503ServiceUnavailable)]
public IActionResult TestFlareSolverrReachable()
{
const string knownProtectedUrl = "https://prowlarr.servarr.com/v1/ping";
HttpClient client = new();
if (!Uri.TryCreate(new(TrangaSettings.flareSolverrUrl), "v1", out Uri? uri))
return BadRequest();
HttpRequestMessage request = new(HttpMethod.Post, uri);
JObject data = new()
{
["cmd"] = "request.get",
["url"] = knownProtectedUrl
};
request.Content = new StringContent(JsonConvert.SerializeObject(data));
request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
HttpResponseMessage response = client.Send(request);
if (!response.IsSuccessStatusCode)
return StatusCode(Status503ServiceUnavailable);
client = new(new ClearanceHandler(TrangaSettings.flareSolverrUrl));
try
{
client.GetStringAsync(knownProtectedUrl).Wait();
return Ok();
}
catch (Exception e)
{
return StatusCode(Status500InternalServerError);
}
FlareSolverrDownloadClient client = new();
RequestResult result = client.MakeRequestInternal(knownProtectedUrl);
return (int)result.statusCode >= 200 && (int)result.statusCode < 300 ? Ok() : StatusCode(500, result.statusCode);
}
}

View File

@ -3,7 +3,7 @@ using log4net;
namespace API.MangaDownloadClients;
internal abstract class DownloadClient
public abstract class DownloadClient
{
private static readonly Dictionary<RequestType, DateTime> LastExecutedRateLimit = new();
protected ILog Log { get; init; }

View File

@ -0,0 +1,180 @@
using System.Diagnostics.CodeAnalysis;
using System.Net;
using System.Text;
using System.Text.Json;
using HtmlAgilityPack;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace API.MangaDownloadClients;
public class FlareSolverrDownloadClient : DownloadClient
{
internal override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null)
{
if (clickButton is not null)
Log.Warn("Client can not click button");
if(referrer is not null)
Log.Warn("Client can not set referrer");
if (TrangaSettings.flareSolverrUrl == string.Empty)
{
Log.Error("FlareSolverr URL is empty");
return new(HttpStatusCode.InternalServerError, null, Stream.Null);
}
Uri flareSolverrUri = new (TrangaSettings.flareSolverrUrl);
if (flareSolverrUri.Segments.Last() != "v1")
flareSolverrUri = new UriBuilder(flareSolverrUri)
{
Path = "v1"
}.Uri;
HttpClient client = new()
{
Timeout = TimeSpan.FromSeconds(10),
DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrHigher,
DefaultRequestHeaders = { { "User-Agent", TrangaSettings.userAgent } }
};
JObject requestObj = new()
{
["cmd"] = "request.get",
["url"] = url
};
HttpRequestMessage requestMessage = new(HttpMethod.Post, flareSolverrUri)
{
Content = new StringContent(JsonConvert.SerializeObject(requestObj)),
};
requestMessage.Content.Headers.ContentType = new ("application/json");
Log.Debug($"Requesting {url}");
HttpResponseMessage? response;
try
{
response = client.Send(requestMessage);
}
catch (HttpRequestException e)
{
Log.Error(e);
return new (HttpStatusCode.Unused, null, Stream.Null);
}
if (!response.IsSuccessStatusCode)
{
Log.Debug($"Request returned status code {(int)response.StatusCode} {response.StatusCode}:\n" +
$"=====\n" +
$"Request:\n" +
$"{requestMessage.Method} {requestMessage.RequestUri}\n" +
$"{requestMessage.Version} {requestMessage.VersionPolicy}\n" +
$"Headers:\n\t{string.Join("\n\t", requestMessage.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
$"{requestMessage.Content?.ReadAsStringAsync().Result}" +
$"=====\n" +
$"Response:\n" +
$"{response.Version}\n" +
$"Headers:\n\t{string.Join("\n\t", response.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
$"{response.Content.ReadAsStringAsync().Result}");
return new (response.StatusCode, null, Stream.Null);
}
string responseString = response.Content.ReadAsStringAsync().Result;
JObject responseObj = JObject.Parse(responseString);
if (!IsInCorrectFormat(responseObj, out string? reason))
{
Log.Error($"Wrong format: {reason}");
return new(HttpStatusCode.Unused, null, Stream.Null);
}
string statusResponse = responseObj["status"]!.Value<string>()!;
if (statusResponse != "ok")
{
Log.Debug($"Status is not ok: {statusResponse}");
return new(HttpStatusCode.Unused, null, Stream.Null);
}
JObject solution = (responseObj["solution"] as JObject)!;
if (!Enum.TryParse(solution["status"]!.Value<int>().ToString(), out HttpStatusCode statusCode))
{
Log.Error($"Wrong format: Cant parse status code: {solution["status"]!.Value<int>()}");
return new(HttpStatusCode.Unused, null, Stream.Null);
}
if (statusCode < HttpStatusCode.OK || statusCode >= HttpStatusCode.MultipleChoices)
{
Log.Debug($"Status is: {statusCode}");
return new(statusCode, null, Stream.Null);
}
if (solution["response"]!.Value<string>() is not { } htmlString)
{
Log.Error("Wrong format: Cant find response in solution");
return new(HttpStatusCode.Unused, null, Stream.Null);
}
if (IsJson(htmlString, out HtmlDocument document, out string? json))
{
MemoryStream ms = new();
ms.Write(Encoding.UTF8.GetBytes(json));
ms.Position = 0;
return new(statusCode, document, ms);
}
else
{
MemoryStream ms = new();
ms.Write(Encoding.UTF8.GetBytes(htmlString));
ms.Position = 0;
return new(statusCode, document, ms);
}
}
private bool IsInCorrectFormat(JObject responseObj, [NotNullWhen(false)]out string? reason)
{
reason = null;
if (!responseObj.ContainsKey("status"))
{
reason = "Cant find status on response";
return false;
}
if (responseObj["solution"] is not JObject solution)
{
reason = "Cant find solution";
return false;
}
if (!solution.ContainsKey("status"))
{
reason = "Wrong format: Cant find status in solution";
return false;
}
if (!solution.ContainsKey("response"))
{
reason = "Wrong format: Cant find response in solution";
return false;
}
return true;
}
private bool IsJson(string htmlString, out HtmlDocument document, [NotNullWhen(true)]out string? jsonString)
{
jsonString = null;
document = new();
document.LoadHtml(htmlString);
HtmlNode pre = document.DocumentNode.SelectSingleNode("//pre");
try
{
using JsonDocument _ = JsonDocument.Parse(pre.InnerText);
jsonString = pre.InnerText;
return true;
}
catch (JsonReaderException)
{
return false;
}
}
}

View File

@ -1,5 +1,4 @@
using System.Net;
using FlareSolverrSharp;
using HtmlAgilityPack;
namespace API.MangaDownloadClients;
@ -10,9 +9,7 @@ internal class HttpDownloadClient : DownloadClient
{
if (clickButton is not null)
Log.Warn("Client can not click button");
HttpClient client = TrangaSettings.flareSolverrUrl == string.Empty
? new ()
: new (new ClearanceHandler(TrangaSettings.flareSolverrUrl));
HttpClient client = new();
client.Timeout = TimeSpan.FromSeconds(10);
client.DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrHigher;
client.DefaultRequestHeaders.Add("User-Agent", TrangaSettings.userAgent);
@ -34,19 +31,28 @@ internal class HttpDownloadClient : DownloadClient
if (!response.IsSuccessStatusCode)
{
Log.Debug($"Request returned status code {(int)response.StatusCode} {response.StatusCode}:\n" +
$"=====\n" +
$"Request:\n" +
$"{requestMessage.Method} {requestMessage.RequestUri}\n" +
$"{requestMessage.Version} {requestMessage.VersionPolicy}\n" +
$"Headers:\n\t{string.Join("\n\t", requestMessage.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
$"{requestMessage.Content?.ReadAsStringAsync().Result}" +
$"=====\n" +
$"Response:\n" +
$"{response.Version}\n" +
$"Headers:\n\t{string.Join("\n\t", response.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
$"{response.Content.ReadAsStringAsync().Result}");
return new (response.StatusCode, null, Stream.Null);
Log.Debug($"Request returned status code {(int)response.StatusCode} {response.StatusCode}");
if (response.Headers.Server.Any(s =>
(s.Product?.Name ?? "").Contains("cloudflare", StringComparison.InvariantCultureIgnoreCase)))
{
Log.Debug("Retrying with FlareSolverr!");
return new FlareSolverrDownloadClient().MakeRequestInternal(url, referrer, clickButton);
}
else
{
Log.Debug($"Request returned status code {(int)response.StatusCode} {response.StatusCode}:\n" +
$"=====\n" +
$"Request:\n" +
$"{requestMessage.Method} {requestMessage.RequestUri}\n" +
$"{requestMessage.Version} {requestMessage.VersionPolicy}\n" +
$"Headers:\n\t{string.Join("\n\t", requestMessage.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
$"{requestMessage.Content?.ReadAsStringAsync().Result}" +
$"=====\n" +
$"Response:\n" +
$"{response.Version}\n" +
$"Headers:\n\t{string.Join("\n\t", response.Headers.Select(h => $"{h.Key}: <{string.Join(">, <", h.Value)}"))}>\n" +
$"{response.Content.ReadAsStringAsync().Result}");
}
}
Stream stream;

View File

@ -0,0 +1,788 @@
// <auto-generated />
using System;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace API.Migrations.pgsql
{
[DbContext(typeof(PgsqlContext))]
[Migration("20250628204956_AddMAL")]
partial class AddMAL
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "9.0.5")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("API.Schema.Author", b =>
{
b.Property<string>("AuthorId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<string>("AuthorName")
.IsRequired()
.HasMaxLength(128)
.HasColumnType("character varying(128)");
b.HasKey("AuthorId");
b.ToTable("Authors");
});
modelBuilder.Entity("API.Schema.Chapter", b =>
{
b.Property<string>("ChapterId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<string>("ChapterNumber")
.IsRequired()
.HasMaxLength(10)
.HasColumnType("character varying(10)");
b.Property<bool>("Downloaded")
.HasColumnType("boolean");
b.Property<string>("FileName")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("IdOnConnectorSite")
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("ParentMangaId")
.IsRequired()
.HasColumnType("character varying(64)");
b.Property<string>("Title")
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("Url")
.IsRequired()
.HasMaxLength(2048)
.HasColumnType("character varying(2048)");
b.Property<int?>("VolumeNumber")
.HasColumnType("integer");
b.HasKey("ChapterId");
b.HasIndex("ParentMangaId");
b.ToTable("Chapters");
});
modelBuilder.Entity("API.Schema.Jobs.Job", b =>
{
b.Property<string>("JobId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<bool>("Enabled")
.HasColumnType("boolean");
b.Property<byte>("JobType")
.HasColumnType("smallint");
b.Property<DateTime>("LastExecution")
.HasColumnType("timestamp with time zone");
b.Property<string>("ParentJobId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<decimal>("RecurrenceMs")
.HasColumnType("numeric(20,0)");
b.Property<byte>("state")
.HasColumnType("smallint");
b.HasKey("JobId");
b.HasIndex("ParentJobId");
b.ToTable("Jobs");
b.HasDiscriminator<byte>("JobType");
b.UseTphMappingStrategy();
});
modelBuilder.Entity("API.Schema.LocalLibrary", b =>
{
b.Property<string>("LocalLibraryId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<string>("BasePath")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("LibraryName")
.IsRequired()
.HasMaxLength(512)
.HasColumnType("character varying(512)");
b.HasKey("LocalLibraryId");
b.ToTable("LocalLibraries");
});
modelBuilder.Entity("API.Schema.Manga", b =>
{
b.Property<string>("MangaId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<string>("CoverFileNameInCache")
.HasMaxLength(512)
.HasColumnType("character varying(512)");
b.Property<string>("CoverUrl")
.IsRequired()
.HasMaxLength(512)
.HasColumnType("character varying(512)");
b.Property<string>("Description")
.IsRequired()
.HasColumnType("text");
b.Property<string>("DirectoryName")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)");
b.Property<string>("IdOnConnectorSite")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<float>("IgnoreChaptersBefore")
.HasColumnType("real");
b.Property<string>("LibraryId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<string>("MangaConnectorName")
.IsRequired()
.HasMaxLength(32)
.HasColumnType("character varying(32)");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(512)
.HasColumnType("character varying(512)");
b.Property<string>("OriginalLanguage")
.HasMaxLength(8)
.HasColumnType("character varying(8)");
b.Property<byte>("ReleaseStatus")
.HasColumnType("smallint");
b.Property<string>("WebsiteUrl")
.IsRequired()
.HasMaxLength(512)
.HasColumnType("character varying(512)");
b.Property<long?>("Year")
.HasColumnType("bigint");
b.HasKey("MangaId");
b.HasIndex("LibraryId");
b.HasIndex("MangaConnectorName");
b.ToTable("Mangas");
});
modelBuilder.Entity("API.Schema.MangaConnectors.MangaConnector", b =>
{
b.Property<string>("Name")
.HasMaxLength(32)
.HasColumnType("character varying(32)");
b.PrimitiveCollection<string[]>("BaseUris")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("text[]");
b.Property<bool>("Enabled")
.HasColumnType("boolean");
b.Property<string>("IconUrl")
.IsRequired()
.HasMaxLength(2048)
.HasColumnType("character varying(2048)");
b.PrimitiveCollection<string[]>("SupportedLanguages")
.IsRequired()
.HasMaxLength(8)
.HasColumnType("text[]");
b.HasKey("Name");
b.ToTable("MangaConnectors");
b.HasDiscriminator<string>("Name").HasValue("MangaConnector");
b.UseTphMappingStrategy();
});
modelBuilder.Entity("API.Schema.MangaTag", b =>
{
b.Property<string>("Tag")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasKey("Tag");
b.ToTable("Tags");
});
modelBuilder.Entity("API.Schema.MetadataFetchers.MetadataEntry", b =>
{
b.Property<string>("MangaId")
.HasColumnType("character varying(64)");
b.Property<string>("MetadataFetcherName")
.HasColumnType("text");
b.Property<string>("Identifier")
.IsRequired()
.HasColumnType("text");
b.HasKey("MangaId", "MetadataFetcherName");
b.HasIndex("MetadataFetcherName");
b.ToTable("MetadataEntries");
});
modelBuilder.Entity("API.Schema.MetadataFetchers.MetadataFetcher", b =>
{
b.Property<string>("MetadataFetcherName")
.HasColumnType("text");
b.Property<string>("MetadataEntry")
.IsRequired()
.HasMaxLength(21)
.HasColumnType("character varying(21)");
b.HasKey("MetadataFetcherName");
b.ToTable("MetadataFetcher");
b.HasDiscriminator<string>("MetadataEntry").HasValue("MetadataFetcher");
b.UseTphMappingStrategy();
});
modelBuilder.Entity("AuthorToManga", b =>
{
b.Property<string>("AuthorIds")
.HasColumnType("character varying(64)");
b.Property<string>("MangaIds")
.HasColumnType("character varying(64)");
b.HasKey("AuthorIds", "MangaIds");
b.HasIndex("MangaIds");
b.ToTable("AuthorToManga");
});
modelBuilder.Entity("JobJob", b =>
{
b.Property<string>("DependsOnJobsJobId")
.HasColumnType("character varying(64)");
b.Property<string>("JobId")
.HasColumnType("character varying(64)");
b.HasKey("DependsOnJobsJobId", "JobId");
b.HasIndex("JobId");
b.ToTable("JobJob");
});
modelBuilder.Entity("MangaTagToManga", b =>
{
b.Property<string>("MangaTagIds")
.HasColumnType("character varying(64)");
b.Property<string>("MangaIds")
.HasColumnType("character varying(64)");
b.HasKey("MangaTagIds", "MangaIds");
b.HasIndex("MangaIds");
b.ToTable("MangaTagToManga");
});
modelBuilder.Entity("API.Schema.Jobs.DownloadAvailableChaptersJob", b =>
{
b.HasBaseType("API.Schema.Jobs.Job");
b.Property<string>("MangaId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasIndex("MangaId");
b.ToTable("Jobs", t =>
{
t.Property("MangaId")
.HasColumnName("DownloadAvailableChaptersJob_MangaId");
});
b.HasDiscriminator().HasValue((byte)1);
});
modelBuilder.Entity("API.Schema.Jobs.DownloadMangaCoverJob", b =>
{
b.HasBaseType("API.Schema.Jobs.Job");
b.Property<string>("MangaId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasIndex("MangaId");
b.HasDiscriminator().HasValue((byte)4);
});
modelBuilder.Entity("API.Schema.Jobs.DownloadSingleChapterJob", b =>
{
b.HasBaseType("API.Schema.Jobs.Job");
b.Property<string>("ChapterId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasIndex("ChapterId");
b.HasDiscriminator().HasValue((byte)0);
});
modelBuilder.Entity("API.Schema.Jobs.MoveFileOrFolderJob", b =>
{
b.HasBaseType("API.Schema.Jobs.Job");
b.Property<string>("FromLocation")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("ToLocation")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.HasDiscriminator().HasValue((byte)3);
});
modelBuilder.Entity("API.Schema.Jobs.MoveMangaLibraryJob", b =>
{
b.HasBaseType("API.Schema.Jobs.Job");
b.Property<string>("MangaId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<string>("ToLibraryId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasIndex("MangaId");
b.HasIndex("ToLibraryId");
b.ToTable("Jobs", t =>
{
t.Property("MangaId")
.HasColumnName("MoveMangaLibraryJob_MangaId");
});
b.HasDiscriminator().HasValue((byte)7);
});
modelBuilder.Entity("API.Schema.Jobs.RetrieveChaptersJob", b =>
{
b.HasBaseType("API.Schema.Jobs.Job");
b.Property<string>("Language")
.IsRequired()
.HasMaxLength(8)
.HasColumnType("character varying(8)");
b.Property<string>("MangaId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasIndex("MangaId");
b.ToTable("Jobs", t =>
{
t.Property("MangaId")
.HasColumnName("RetrieveChaptersJob_MangaId");
});
b.HasDiscriminator().HasValue((byte)5);
});
modelBuilder.Entity("API.Schema.Jobs.UpdateChaptersDownloadedJob", b =>
{
b.HasBaseType("API.Schema.Jobs.Job");
b.Property<string>("MangaId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasIndex("MangaId");
b.ToTable("Jobs", t =>
{
t.Property("MangaId")
.HasColumnName("UpdateChaptersDownloadedJob_MangaId");
});
b.HasDiscriminator().HasValue((byte)6);
});
modelBuilder.Entity("API.Schema.Jobs.UpdateCoverJob", b =>
{
b.HasBaseType("API.Schema.Jobs.Job");
b.Property<string>("MangaId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasIndex("MangaId");
b.ToTable("Jobs", t =>
{
t.Property("MangaId")
.HasColumnName("UpdateCoverJob_MangaId");
});
b.HasDiscriminator().HasValue((byte)9);
});
modelBuilder.Entity("API.Schema.MangaConnectors.ComickIo", b =>
{
b.HasBaseType("API.Schema.MangaConnectors.MangaConnector");
b.HasDiscriminator().HasValue("ComickIo");
});
modelBuilder.Entity("API.Schema.MangaConnectors.Global", b =>
{
b.HasBaseType("API.Schema.MangaConnectors.MangaConnector");
b.HasDiscriminator().HasValue("Global");
});
modelBuilder.Entity("API.Schema.MangaConnectors.MangaDex", b =>
{
b.HasBaseType("API.Schema.MangaConnectors.MangaConnector");
b.HasDiscriminator().HasValue("MangaDex");
});
modelBuilder.Entity("API.Schema.MetadataFetchers.MyAnimeList", b =>
{
b.HasBaseType("API.Schema.MetadataFetchers.MetadataFetcher");
b.HasDiscriminator().HasValue("MyAnimeList");
});
modelBuilder.Entity("API.Schema.Chapter", b =>
{
b.HasOne("API.Schema.Manga", "ParentManga")
.WithMany("Chapters")
.HasForeignKey("ParentMangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("ParentManga");
});
modelBuilder.Entity("API.Schema.Jobs.Job", b =>
{
b.HasOne("API.Schema.Jobs.Job", "ParentJob")
.WithMany()
.HasForeignKey("ParentJobId")
.OnDelete(DeleteBehavior.Cascade);
b.Navigation("ParentJob");
});
modelBuilder.Entity("API.Schema.Manga", b =>
{
b.HasOne("API.Schema.LocalLibrary", "Library")
.WithMany()
.HasForeignKey("LibraryId")
.OnDelete(DeleteBehavior.SetNull);
b.HasOne("API.Schema.MangaConnectors.MangaConnector", "MangaConnector")
.WithMany()
.HasForeignKey("MangaConnectorName")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.OwnsMany("API.Schema.Link", "Links", b1 =>
{
b1.Property<string>("LinkId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b1.Property<string>("LinkProvider")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b1.Property<string>("LinkUrl")
.IsRequired()
.HasMaxLength(2048)
.HasColumnType("character varying(2048)");
b1.Property<string>("MangaId")
.IsRequired()
.HasColumnType("character varying(64)");
b1.HasKey("LinkId");
b1.HasIndex("MangaId");
b1.ToTable("Link");
b1.WithOwner()
.HasForeignKey("MangaId");
});
b.OwnsMany("API.Schema.MangaAltTitle", "AltTitles", b1 =>
{
b1.Property<string>("AltTitleId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b1.Property<string>("Language")
.IsRequired()
.HasMaxLength(8)
.HasColumnType("character varying(8)");
b1.Property<string>("MangaId")
.IsRequired()
.HasColumnType("character varying(64)");
b1.Property<string>("Title")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b1.HasKey("AltTitleId");
b1.HasIndex("MangaId");
b1.ToTable("MangaAltTitle");
b1.WithOwner()
.HasForeignKey("MangaId");
});
b.Navigation("AltTitles");
b.Navigation("Library");
b.Navigation("Links");
b.Navigation("MangaConnector");
});
modelBuilder.Entity("API.Schema.MetadataFetchers.MetadataEntry", b =>
{
b.HasOne("API.Schema.Manga", "Manga")
.WithMany()
.HasForeignKey("MangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("API.Schema.MetadataFetchers.MetadataFetcher", "MetadataFetcher")
.WithMany()
.HasForeignKey("MetadataFetcherName")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Manga");
b.Navigation("MetadataFetcher");
});
modelBuilder.Entity("AuthorToManga", b =>
{
b.HasOne("API.Schema.Author", null)
.WithMany()
.HasForeignKey("AuthorIds")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("API.Schema.Manga", null)
.WithMany()
.HasForeignKey("MangaIds")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("JobJob", b =>
{
b.HasOne("API.Schema.Jobs.Job", null)
.WithMany()
.HasForeignKey("DependsOnJobsJobId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("API.Schema.Jobs.Job", null)
.WithMany()
.HasForeignKey("JobId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("MangaTagToManga", b =>
{
b.HasOne("API.Schema.Manga", null)
.WithMany()
.HasForeignKey("MangaIds")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("API.Schema.MangaTag", null)
.WithMany()
.HasForeignKey("MangaTagIds")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("API.Schema.Jobs.DownloadAvailableChaptersJob", b =>
{
b.HasOne("API.Schema.Manga", "Manga")
.WithMany()
.HasForeignKey("MangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Manga");
});
modelBuilder.Entity("API.Schema.Jobs.DownloadMangaCoverJob", b =>
{
b.HasOne("API.Schema.Manga", "Manga")
.WithMany()
.HasForeignKey("MangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Manga");
});
modelBuilder.Entity("API.Schema.Jobs.DownloadSingleChapterJob", b =>
{
b.HasOne("API.Schema.Chapter", "Chapter")
.WithMany()
.HasForeignKey("ChapterId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Chapter");
});
modelBuilder.Entity("API.Schema.Jobs.MoveMangaLibraryJob", b =>
{
b.HasOne("API.Schema.Manga", "Manga")
.WithMany()
.HasForeignKey("MangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("API.Schema.LocalLibrary", "ToLibrary")
.WithMany()
.HasForeignKey("ToLibraryId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Manga");
b.Navigation("ToLibrary");
});
modelBuilder.Entity("API.Schema.Jobs.RetrieveChaptersJob", b =>
{
b.HasOne("API.Schema.Manga", "Manga")
.WithMany()
.HasForeignKey("MangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Manga");
});
modelBuilder.Entity("API.Schema.Jobs.UpdateChaptersDownloadedJob", b =>
{
b.HasOne("API.Schema.Manga", "Manga")
.WithMany()
.HasForeignKey("MangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Manga");
});
modelBuilder.Entity("API.Schema.Jobs.UpdateCoverJob", b =>
{
b.HasOne("API.Schema.Manga", "Manga")
.WithMany()
.HasForeignKey("MangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Manga");
});
modelBuilder.Entity("API.Schema.Manga", b =>
{
b.Navigation("Chapters");
});
#pragma warning restore 612, 618
}
}
}

View File

@ -0,0 +1,66 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Migrations.pgsql
{
/// <inheritdoc />
public partial class AddMAL : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "MetadataFetcher",
columns: table => new
{
MetadataFetcherName = table.Column<string>(type: "text", nullable: false),
MetadataEntry = table.Column<string>(type: "character varying(21)", maxLength: 21, nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_MetadataFetcher", x => x.MetadataFetcherName);
});
migrationBuilder.CreateTable(
name: "MetadataEntries",
columns: table => new
{
MangaId = table.Column<string>(type: "character varying(64)", nullable: false),
MetadataFetcherName = table.Column<string>(type: "text", nullable: false),
Identifier = table.Column<string>(type: "text", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_MetadataEntries", x => new { x.MangaId, x.MetadataFetcherName });
table.ForeignKey(
name: "FK_MetadataEntries_Mangas_MangaId",
column: x => x.MangaId,
principalTable: "Mangas",
principalColumn: "MangaId",
onDelete: ReferentialAction.Cascade);
table.ForeignKey(
name: "FK_MetadataEntries_MetadataFetcher_MetadataFetcherName",
column: x => x.MetadataFetcherName,
principalTable: "MetadataFetcher",
principalColumn: "MetadataFetcherName",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex(
name: "IX_MetadataEntries_MetadataFetcherName",
table: "MetadataEntries",
column: "MetadataFetcherName");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "MetadataEntries");
migrationBuilder.DropTable(
name: "MetadataFetcher");
}
}
}

View File

@ -0,0 +1,788 @@
// <auto-generated />
using System;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace API.Migrations.pgsql
{
[DbContext(typeof(PgsqlContext))]
[Migration("20250629184056_MetadataEntry-PrimaryKeyChange")]
partial class MetadataEntryPrimaryKeyChange
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "9.0.5")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("API.Schema.Author", b =>
{
b.Property<string>("AuthorId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<string>("AuthorName")
.IsRequired()
.HasMaxLength(128)
.HasColumnType("character varying(128)");
b.HasKey("AuthorId");
b.ToTable("Authors");
});
modelBuilder.Entity("API.Schema.Chapter", b =>
{
b.Property<string>("ChapterId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<string>("ChapterNumber")
.IsRequired()
.HasMaxLength(10)
.HasColumnType("character varying(10)");
b.Property<bool>("Downloaded")
.HasColumnType("boolean");
b.Property<string>("FileName")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("IdOnConnectorSite")
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("ParentMangaId")
.IsRequired()
.HasColumnType("character varying(64)");
b.Property<string>("Title")
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("Url")
.IsRequired()
.HasMaxLength(2048)
.HasColumnType("character varying(2048)");
b.Property<int?>("VolumeNumber")
.HasColumnType("integer");
b.HasKey("ChapterId");
b.HasIndex("ParentMangaId");
b.ToTable("Chapters");
});
modelBuilder.Entity("API.Schema.Jobs.Job", b =>
{
b.Property<string>("JobId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<bool>("Enabled")
.HasColumnType("boolean");
b.Property<byte>("JobType")
.HasColumnType("smallint");
b.Property<DateTime>("LastExecution")
.HasColumnType("timestamp with time zone");
b.Property<string>("ParentJobId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<decimal>("RecurrenceMs")
.HasColumnType("numeric(20,0)");
b.Property<byte>("state")
.HasColumnType("smallint");
b.HasKey("JobId");
b.HasIndex("ParentJobId");
b.ToTable("Jobs");
b.HasDiscriminator<byte>("JobType");
b.UseTphMappingStrategy();
});
modelBuilder.Entity("API.Schema.LocalLibrary", b =>
{
b.Property<string>("LocalLibraryId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<string>("BasePath")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("LibraryName")
.IsRequired()
.HasMaxLength(512)
.HasColumnType("character varying(512)");
b.HasKey("LocalLibraryId");
b.ToTable("LocalLibraries");
});
modelBuilder.Entity("API.Schema.Manga", b =>
{
b.Property<string>("MangaId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<string>("CoverFileNameInCache")
.HasMaxLength(512)
.HasColumnType("character varying(512)");
b.Property<string>("CoverUrl")
.IsRequired()
.HasMaxLength(512)
.HasColumnType("character varying(512)");
b.Property<string>("Description")
.IsRequired()
.HasColumnType("text");
b.Property<string>("DirectoryName")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("character varying(1024)");
b.Property<string>("IdOnConnectorSite")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<float>("IgnoreChaptersBefore")
.HasColumnType("real");
b.Property<string>("LibraryId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<string>("MangaConnectorName")
.IsRequired()
.HasMaxLength(32)
.HasColumnType("character varying(32)");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(512)
.HasColumnType("character varying(512)");
b.Property<string>("OriginalLanguage")
.HasMaxLength(8)
.HasColumnType("character varying(8)");
b.Property<byte>("ReleaseStatus")
.HasColumnType("smallint");
b.Property<string>("WebsiteUrl")
.IsRequired()
.HasMaxLength(512)
.HasColumnType("character varying(512)");
b.Property<long?>("Year")
.HasColumnType("bigint");
b.HasKey("MangaId");
b.HasIndex("LibraryId");
b.HasIndex("MangaConnectorName");
b.ToTable("Mangas");
});
modelBuilder.Entity("API.Schema.MangaConnectors.MangaConnector", b =>
{
b.Property<string>("Name")
.HasMaxLength(32)
.HasColumnType("character varying(32)");
b.PrimitiveCollection<string[]>("BaseUris")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("text[]");
b.Property<bool>("Enabled")
.HasColumnType("boolean");
b.Property<string>("IconUrl")
.IsRequired()
.HasMaxLength(2048)
.HasColumnType("character varying(2048)");
b.PrimitiveCollection<string[]>("SupportedLanguages")
.IsRequired()
.HasMaxLength(8)
.HasColumnType("text[]");
b.HasKey("Name");
b.ToTable("MangaConnectors");
b.HasDiscriminator<string>("Name").HasValue("MangaConnector");
b.UseTphMappingStrategy();
});
modelBuilder.Entity("API.Schema.MangaTag", b =>
{
b.Property<string>("Tag")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasKey("Tag");
b.ToTable("Tags");
});
modelBuilder.Entity("API.Schema.MetadataFetchers.MetadataEntry", b =>
{
b.Property<string>("MetadataFetcherName")
.HasColumnType("text");
b.Property<string>("Identifier")
.HasColumnType("text");
b.Property<string>("MangaId")
.IsRequired()
.HasColumnType("character varying(64)");
b.HasKey("MetadataFetcherName", "Identifier");
b.HasIndex("MangaId");
b.ToTable("MetadataEntries");
});
modelBuilder.Entity("API.Schema.MetadataFetchers.MetadataFetcher", b =>
{
b.Property<string>("MetadataFetcherName")
.HasColumnType("text");
b.Property<string>("MetadataEntry")
.IsRequired()
.HasMaxLength(21)
.HasColumnType("character varying(21)");
b.HasKey("MetadataFetcherName");
b.ToTable("MetadataFetcher");
b.HasDiscriminator<string>("MetadataEntry").HasValue("MetadataFetcher");
b.UseTphMappingStrategy();
});
modelBuilder.Entity("AuthorToManga", b =>
{
b.Property<string>("AuthorIds")
.HasColumnType("character varying(64)");
b.Property<string>("MangaIds")
.HasColumnType("character varying(64)");
b.HasKey("AuthorIds", "MangaIds");
b.HasIndex("MangaIds");
b.ToTable("AuthorToManga");
});
modelBuilder.Entity("JobJob", b =>
{
b.Property<string>("DependsOnJobsJobId")
.HasColumnType("character varying(64)");
b.Property<string>("JobId")
.HasColumnType("character varying(64)");
b.HasKey("DependsOnJobsJobId", "JobId");
b.HasIndex("JobId");
b.ToTable("JobJob");
});
modelBuilder.Entity("MangaTagToManga", b =>
{
b.Property<string>("MangaTagIds")
.HasColumnType("character varying(64)");
b.Property<string>("MangaIds")
.HasColumnType("character varying(64)");
b.HasKey("MangaTagIds", "MangaIds");
b.HasIndex("MangaIds");
b.ToTable("MangaTagToManga");
});
modelBuilder.Entity("API.Schema.Jobs.DownloadAvailableChaptersJob", b =>
{
b.HasBaseType("API.Schema.Jobs.Job");
b.Property<string>("MangaId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasIndex("MangaId");
b.ToTable("Jobs", t =>
{
t.Property("MangaId")
.HasColumnName("DownloadAvailableChaptersJob_MangaId");
});
b.HasDiscriminator().HasValue((byte)1);
});
modelBuilder.Entity("API.Schema.Jobs.DownloadMangaCoverJob", b =>
{
b.HasBaseType("API.Schema.Jobs.Job");
b.Property<string>("MangaId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasIndex("MangaId");
b.HasDiscriminator().HasValue((byte)4);
});
modelBuilder.Entity("API.Schema.Jobs.DownloadSingleChapterJob", b =>
{
b.HasBaseType("API.Schema.Jobs.Job");
b.Property<string>("ChapterId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasIndex("ChapterId");
b.HasDiscriminator().HasValue((byte)0);
});
modelBuilder.Entity("API.Schema.Jobs.MoveFileOrFolderJob", b =>
{
b.HasBaseType("API.Schema.Jobs.Job");
b.Property<string>("FromLocation")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("ToLocation")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.HasDiscriminator().HasValue((byte)3);
});
modelBuilder.Entity("API.Schema.Jobs.MoveMangaLibraryJob", b =>
{
b.HasBaseType("API.Schema.Jobs.Job");
b.Property<string>("MangaId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.Property<string>("ToLibraryId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasIndex("MangaId");
b.HasIndex("ToLibraryId");
b.ToTable("Jobs", t =>
{
t.Property("MangaId")
.HasColumnName("MoveMangaLibraryJob_MangaId");
});
b.HasDiscriminator().HasValue((byte)7);
});
modelBuilder.Entity("API.Schema.Jobs.RetrieveChaptersJob", b =>
{
b.HasBaseType("API.Schema.Jobs.Job");
b.Property<string>("Language")
.IsRequired()
.HasMaxLength(8)
.HasColumnType("character varying(8)");
b.Property<string>("MangaId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasIndex("MangaId");
b.ToTable("Jobs", t =>
{
t.Property("MangaId")
.HasColumnName("RetrieveChaptersJob_MangaId");
});
b.HasDiscriminator().HasValue((byte)5);
});
modelBuilder.Entity("API.Schema.Jobs.UpdateChaptersDownloadedJob", b =>
{
b.HasBaseType("API.Schema.Jobs.Job");
b.Property<string>("MangaId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasIndex("MangaId");
b.ToTable("Jobs", t =>
{
t.Property("MangaId")
.HasColumnName("UpdateChaptersDownloadedJob_MangaId");
});
b.HasDiscriminator().HasValue((byte)6);
});
modelBuilder.Entity("API.Schema.Jobs.UpdateCoverJob", b =>
{
b.HasBaseType("API.Schema.Jobs.Job");
b.Property<string>("MangaId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasIndex("MangaId");
b.ToTable("Jobs", t =>
{
t.Property("MangaId")
.HasColumnName("UpdateCoverJob_MangaId");
});
b.HasDiscriminator().HasValue((byte)9);
});
modelBuilder.Entity("API.Schema.MangaConnectors.ComickIo", b =>
{
b.HasBaseType("API.Schema.MangaConnectors.MangaConnector");
b.HasDiscriminator().HasValue("ComickIo");
});
modelBuilder.Entity("API.Schema.MangaConnectors.Global", b =>
{
b.HasBaseType("API.Schema.MangaConnectors.MangaConnector");
b.HasDiscriminator().HasValue("Global");
});
modelBuilder.Entity("API.Schema.MangaConnectors.MangaDex", b =>
{
b.HasBaseType("API.Schema.MangaConnectors.MangaConnector");
b.HasDiscriminator().HasValue("MangaDex");
});
modelBuilder.Entity("API.Schema.MetadataFetchers.MyAnimeList", b =>
{
b.HasBaseType("API.Schema.MetadataFetchers.MetadataFetcher");
b.HasDiscriminator().HasValue("MyAnimeList");
});
modelBuilder.Entity("API.Schema.Chapter", b =>
{
b.HasOne("API.Schema.Manga", "ParentManga")
.WithMany("Chapters")
.HasForeignKey("ParentMangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("ParentManga");
});
modelBuilder.Entity("API.Schema.Jobs.Job", b =>
{
b.HasOne("API.Schema.Jobs.Job", "ParentJob")
.WithMany()
.HasForeignKey("ParentJobId")
.OnDelete(DeleteBehavior.Cascade);
b.Navigation("ParentJob");
});
modelBuilder.Entity("API.Schema.Manga", b =>
{
b.HasOne("API.Schema.LocalLibrary", "Library")
.WithMany()
.HasForeignKey("LibraryId")
.OnDelete(DeleteBehavior.SetNull);
b.HasOne("API.Schema.MangaConnectors.MangaConnector", "MangaConnector")
.WithMany()
.HasForeignKey("MangaConnectorName")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.OwnsMany("API.Schema.Link", "Links", b1 =>
{
b1.Property<string>("LinkId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b1.Property<string>("LinkProvider")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b1.Property<string>("LinkUrl")
.IsRequired()
.HasMaxLength(2048)
.HasColumnType("character varying(2048)");
b1.Property<string>("MangaId")
.IsRequired()
.HasColumnType("character varying(64)");
b1.HasKey("LinkId");
b1.HasIndex("MangaId");
b1.ToTable("Link");
b1.WithOwner()
.HasForeignKey("MangaId");
});
b.OwnsMany("API.Schema.MangaAltTitle", "AltTitles", b1 =>
{
b1.Property<string>("AltTitleId")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b1.Property<string>("Language")
.IsRequired()
.HasMaxLength(8)
.HasColumnType("character varying(8)");
b1.Property<string>("MangaId")
.IsRequired()
.HasColumnType("character varying(64)");
b1.Property<string>("Title")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b1.HasKey("AltTitleId");
b1.HasIndex("MangaId");
b1.ToTable("MangaAltTitle");
b1.WithOwner()
.HasForeignKey("MangaId");
});
b.Navigation("AltTitles");
b.Navigation("Library");
b.Navigation("Links");
b.Navigation("MangaConnector");
});
modelBuilder.Entity("API.Schema.MetadataFetchers.MetadataEntry", b =>
{
b.HasOne("API.Schema.Manga", "Manga")
.WithMany()
.HasForeignKey("MangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("API.Schema.MetadataFetchers.MetadataFetcher", "MetadataFetcher")
.WithMany()
.HasForeignKey("MetadataFetcherName")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Manga");
b.Navigation("MetadataFetcher");
});
modelBuilder.Entity("AuthorToManga", b =>
{
b.HasOne("API.Schema.Author", null)
.WithMany()
.HasForeignKey("AuthorIds")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("API.Schema.Manga", null)
.WithMany()
.HasForeignKey("MangaIds")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("JobJob", b =>
{
b.HasOne("API.Schema.Jobs.Job", null)
.WithMany()
.HasForeignKey("DependsOnJobsJobId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("API.Schema.Jobs.Job", null)
.WithMany()
.HasForeignKey("JobId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("MangaTagToManga", b =>
{
b.HasOne("API.Schema.Manga", null)
.WithMany()
.HasForeignKey("MangaIds")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("API.Schema.MangaTag", null)
.WithMany()
.HasForeignKey("MangaTagIds")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("API.Schema.Jobs.DownloadAvailableChaptersJob", b =>
{
b.HasOne("API.Schema.Manga", "Manga")
.WithMany()
.HasForeignKey("MangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Manga");
});
modelBuilder.Entity("API.Schema.Jobs.DownloadMangaCoverJob", b =>
{
b.HasOne("API.Schema.Manga", "Manga")
.WithMany()
.HasForeignKey("MangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Manga");
});
modelBuilder.Entity("API.Schema.Jobs.DownloadSingleChapterJob", b =>
{
b.HasOne("API.Schema.Chapter", "Chapter")
.WithMany()
.HasForeignKey("ChapterId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Chapter");
});
modelBuilder.Entity("API.Schema.Jobs.MoveMangaLibraryJob", b =>
{
b.HasOne("API.Schema.Manga", "Manga")
.WithMany()
.HasForeignKey("MangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("API.Schema.LocalLibrary", "ToLibrary")
.WithMany()
.HasForeignKey("ToLibraryId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Manga");
b.Navigation("ToLibrary");
});
modelBuilder.Entity("API.Schema.Jobs.RetrieveChaptersJob", b =>
{
b.HasOne("API.Schema.Manga", "Manga")
.WithMany()
.HasForeignKey("MangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Manga");
});
modelBuilder.Entity("API.Schema.Jobs.UpdateChaptersDownloadedJob", b =>
{
b.HasOne("API.Schema.Manga", "Manga")
.WithMany()
.HasForeignKey("MangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Manga");
});
modelBuilder.Entity("API.Schema.Jobs.UpdateCoverJob", b =>
{
b.HasOne("API.Schema.Manga", "Manga")
.WithMany()
.HasForeignKey("MangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Manga");
});
modelBuilder.Entity("API.Schema.Manga", b =>
{
b.Navigation("Chapters");
});
#pragma warning restore 612, 618
}
}
}

View File

@ -0,0 +1,54 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Migrations.pgsql
{
/// <inheritdoc />
public partial class MetadataEntryPrimaryKeyChange : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropPrimaryKey(
name: "PK_MetadataEntries",
table: "MetadataEntries");
migrationBuilder.DropIndex(
name: "IX_MetadataEntries_MetadataFetcherName",
table: "MetadataEntries");
migrationBuilder.AddPrimaryKey(
name: "PK_MetadataEntries",
table: "MetadataEntries",
columns: new[] { "MetadataFetcherName", "Identifier" });
migrationBuilder.CreateIndex(
name: "IX_MetadataEntries_MangaId",
table: "MetadataEntries",
column: "MangaId");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropPrimaryKey(
name: "PK_MetadataEntries",
table: "MetadataEntries");
migrationBuilder.DropIndex(
name: "IX_MetadataEntries_MangaId",
table: "MetadataEntries");
migrationBuilder.AddPrimaryKey(
name: "PK_MetadataEntries",
table: "MetadataEntries",
columns: new[] { "MangaId", "MetadataFetcherName" });
migrationBuilder.CreateIndex(
name: "IX_MetadataEntries_MetadataFetcherName",
table: "MetadataEntries",
column: "MetadataFetcherName");
}
}
}

View File

@ -255,6 +255,44 @@ namespace API.Migrations.pgsql
b.ToTable("Tags");
});
modelBuilder.Entity("API.Schema.MetadataFetchers.MetadataEntry", b =>
{
b.Property<string>("MetadataFetcherName")
.HasColumnType("text");
b.Property<string>("Identifier")
.HasColumnType("text");
b.Property<string>("MangaId")
.IsRequired()
.HasColumnType("character varying(64)");
b.HasKey("MetadataFetcherName", "Identifier");
b.HasIndex("MangaId");
b.ToTable("MetadataEntries");
});
modelBuilder.Entity("API.Schema.MetadataFetchers.MetadataFetcher", b =>
{
b.Property<string>("MetadataFetcherName")
.HasColumnType("text");
b.Property<string>("MetadataEntry")
.IsRequired()
.HasMaxLength(21)
.HasColumnType("character varying(21)");
b.HasKey("MetadataFetcherName");
b.ToTable("MetadataFetcher");
b.HasDiscriminator<string>("MetadataEntry").HasValue("MetadataFetcher");
b.UseTphMappingStrategy();
});
modelBuilder.Entity("AuthorToManga", b =>
{
b.Property<string>("AuthorIds")
@ -478,6 +516,13 @@ namespace API.Migrations.pgsql
b.HasDiscriminator().HasValue("MangaDex");
});
modelBuilder.Entity("API.Schema.MetadataFetchers.MyAnimeList", b =>
{
b.HasBaseType("API.Schema.MetadataFetchers.MetadataFetcher");
b.HasDiscriminator().HasValue("MyAnimeList");
});
modelBuilder.Entity("API.Schema.Chapter", b =>
{
b.HasOne("API.Schema.Manga", "ParentManga")
@ -581,6 +626,25 @@ namespace API.Migrations.pgsql
b.Navigation("MangaConnector");
});
modelBuilder.Entity("API.Schema.MetadataFetchers.MetadataEntry", b =>
{
b.HasOne("API.Schema.Manga", "Manga")
.WithMany()
.HasForeignKey("MangaId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("API.Schema.MetadataFetchers.MetadataFetcher", "MetadataFetcher")
.WithMany()
.HasForeignKey("MetadataFetcherName")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Manga");
b.Navigation("MetadataFetcher");
});
modelBuilder.Entity("AuthorToManga", b =>
{
b.HasOne("API.Schema.Author", null)

View File

@ -187,6 +187,8 @@ public class Chapter : IComparable<Chapter>
comicInfo.Add(new XElement("Writer", string.Join(',', ParentManga.Authors.Select(author => author.AuthorName))));
if(ParentManga.OriginalLanguage is not null)
comicInfo.Add(new XElement("LanguageISO", ParentManga.OriginalLanguage));
if(ParentManga.Description != string.Empty)
comicInfo.Add(new XElement("Summary", ParentManga.Description));
return comicInfo.ToString();
}

View File

@ -1,5 +1,6 @@
using API.Schema.Jobs;
using API.Schema.MangaConnectors;
using API.Schema.MetadataFetchers;
using log4net;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Diagnostics;
@ -15,6 +16,7 @@ public class PgsqlContext(DbContextOptions<PgsqlContext> options) : DbContext(op
public DbSet<Chapter> Chapters { get; set; }
public DbSet<Author> Authors { get; set; }
public DbSet<MangaTag> Tags { get; set; }
public DbSet<MetadataEntry> MetadataEntries { get; set; }
private ILog Log => LogManager.GetLogger(GetType());
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
@ -191,5 +193,18 @@ public class PgsqlContext(DbContextOptions<PgsqlContext> options) : DbContext(op
modelBuilder.Entity<Manga>()
.Navigation(m => m.Library)
.AutoInclude();
modelBuilder.Entity<MetadataFetcher>()
.HasDiscriminator<string>(nameof(MetadataEntry))
.HasValue<MyAnimeList>(nameof(MyAnimeList));
//MetadataEntry
modelBuilder.Entity<MetadataEntry>()
.HasOne<Manga>(entry => entry.Manga)
.WithMany()
.OnDelete(DeleteBehavior.Cascade);
modelBuilder.Entity<MetadataEntry>()
.HasOne<MetadataFetcher>(entry => entry.MetadataFetcher)
.WithMany()
.OnDelete(DeleteBehavior.Cascade);
}
}

View File

@ -45,6 +45,11 @@ public class DownloadSingleChapterJob : Job
protected override IEnumerable<Job> RunInternal(PgsqlContext context)
{
if (Chapter.Downloaded)
{
Log.Info("Chapter was already downloaded.");
return [];
}
string[] imageUrls = Chapter.ParentManga.MangaConnector.GetChapterImageUrls(Chapter);
if (imageUrls.Length < 1)
{
@ -129,20 +134,38 @@ public class DownloadSingleChapterJob : Job
{
if (!TrangaSettings.bwImages && TrangaSettings.compression == 100)
{
Log.Debug($"No processing requested for image");
Log.Debug("No processing requested for image");
return;
}
Log.Debug($"Processing image: {imagePath}");
using Image image = Image.Load(imagePath);
File.Delete(imagePath);
if(TrangaSettings.bwImages)
image.Mutate(i => i.ApplyProcessor(new AdaptiveThresholdProcessor()));
image.SaveAsJpeg(imagePath, new JpegEncoder()
try
{
Quality = TrangaSettings.compression
});
using Image image = Image.Load(imagePath);
if (TrangaSettings.bwImages)
image.Mutate(i => i.ApplyProcessor(new AdaptiveThresholdProcessor()));
File.Delete(imagePath);
image.SaveAsJpeg(imagePath, new JpegEncoder()
{
Quality = TrangaSettings.compression
});
}
catch (Exception e)
{
if (e is UnknownImageFormatException or NotSupportedException)
{
//If the Image-Format is not processable by ImageSharp, we can't modify it.
Log.Debug($"Unable to process {imagePath}: Not supported image format");
}else if (e is InvalidImageContentException)
{
Log.Debug($"Unable to process {imagePath}: Invalid Content");
}
else
{
Log.Error(e);
}
}
}
private void CopyCoverFromCacheToDownloadLocation(Manga manga)

View File

@ -78,7 +78,7 @@ public class ComickIo : MangaConnector
public override Chapter[] GetChapters(Manga manga, string? language = null)
{
Log.Info($"Getting Chapters: {manga.IdOnConnectorSite}");
List<string> chapterHids = new();
List<Chapter> chapters = new();
int page = 1;
while(page < 50)
{
@ -95,16 +95,13 @@ public class ComickIo : MangaConnector
JToken data = JToken.Parse(sr.ReadToEnd());
JArray? chaptersArray = data["chapters"] as JArray;
if (chaptersArray?.Count < 1)
if (chaptersArray is null || chaptersArray.Count < 1)
break;
chapterHids.AddRange(chaptersArray?.Select(token => token.Value<string>("hid")!)!);
chapters.AddRange(ParseChapters(manga, chaptersArray));
page++;
}
Log.Debug($"Getting chapters for {manga.Name} yielded {chapterHids.Count} hids. Requesting chapters now...");
List<Chapter> chapters = chapterHids.Select(hid => ChapterFromHid(manga, hid)).ToList();
return chapters.ToArray();
}
@ -219,29 +216,23 @@ public class ComickIo : MangaConnector
year: year, originalLanguage: originalLanguage);
}
private Chapter ChapterFromHid(Manga parentManga, string hid)
private List<Chapter> ParseChapters(Manga parentManga, JArray chaptersArray)
{
string requestUrl = $"https://api.comick.fun/chapter/{hid}";
RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
List<Chapter> chapters = new ();
foreach (JToken chapter in chaptersArray)
{
Log.Error("Request failed");
throw new Exception("Request failed");
string? chapterNum = chapter.Value<string>("chap");
string? volumeNumStr = chapter.Value<string>("vol");
int? volumeNum = volumeNumStr is null ? null : int.Parse(volumeNumStr);
string? title = chapter.Value<string>("title");
string? hid = chapter.Value<string>("hid");
string url = $"https://comick.io/comic/{parentManga.IdOnConnectorSite}/{hid}";
if(chapterNum is null || hid is null)
continue;
chapters.Add(new (parentManga, url, chapterNum, volumeNum, hid, title));
}
using StreamReader sr = new (result.result);
JToken data = JToken.Parse(sr.ReadToEnd());
string? canonical = data.Value<string>("canonical");
string? chapterNum = data["chapter"]?.Value<string>("chap");
string? volumeNumStr = data["chapter"]?.Value<string>("vol");
int? volumeNum = volumeNumStr is null ? null : int.Parse(volumeNumStr);
string? title = data["chapter"]?.Value<string>("title");
if(chapterNum is null)
throw new Exception("chapterNum is null");
string url = $"https://comick.io{canonical}";
return new Chapter(parentManga, url, chapterNum, volumeNum, hid, title);
return chapters;
}
}

View File

@ -0,0 +1,35 @@
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
namespace API.Schema.MetadataFetchers;
[PrimaryKey("MetadataFetcherName", "Identifier")]
public class MetadataEntry
{
[JsonIgnore]
public Manga Manga { get; init; } = null!;
public string MangaId { get; init; }
[JsonIgnore]
public MetadataFetcher MetadataFetcher { get; init; } = null!;
public string MetadataFetcherName { get; init; }
public string Identifier { get; init; }
public MetadataEntry(MetadataFetcher fetcher, Manga manga, string identifier)
{
this.Manga = manga;
this.MangaId = manga.MangaId;
this.MetadataFetcher = fetcher;
this.MetadataFetcherName = fetcher.MetadataFetcherName;
this.Identifier = identifier;
}
/// <summary>
/// EFCORE only!!!!
/// </summary>
internal MetadataEntry(string mangaId, string identifier, string metadataFetcherName)
{
this.MangaId = mangaId;
this.Identifier = identifier;
this.MetadataFetcherName = metadataFetcherName;
}
}

View File

@ -0,0 +1,37 @@
using System.Diagnostics.CodeAnalysis;
using API.Schema.Contexts;
using Microsoft.EntityFrameworkCore;
namespace API.Schema.MetadataFetchers;
[PrimaryKey("MetadataFetcherName")]
public abstract class MetadataFetcher
{
// ReSharper disable once EntityFramework.ModelValidation.UnlimitedStringLength
public string MetadataFetcherName { get; init; }
protected MetadataFetcher()
{
this.MetadataFetcherName = this.GetType().Name;
}
/// <summary>
/// EFCORE ONLY!!!
/// </summary>
internal MetadataFetcher(string metadataFetcherName)
{
this.MetadataFetcherName = metadataFetcherName;
}
internal MetadataEntry CreateMetadataEntry(Manga manga, string identifier) =>
new (this, manga, identifier);
public abstract MetadataSearchResult[] SearchMetadataEntry(Manga manga);
public abstract MetadataSearchResult[] SearchMetadataEntry(string searchTerm);
/// <summary>
/// Updates the Manga linked in the MetadataEntry
/// </summary>
public abstract void UpdateMetadata(MetadataEntry metadataEntry, PgsqlContext dbContext);
}

View File

@ -0,0 +1,3 @@
namespace API.Schema.MetadataFetchers;
public record MetadataSearchResult(string Identifier, string Name, string Url, string? Description = null, string? CoverUrl = null);

View File

@ -0,0 +1,79 @@
using System.Text.RegularExpressions;
using API.Schema.Contexts;
using JikanDotNet;
using Microsoft.EntityFrameworkCore;
namespace API.Schema.MetadataFetchers;
public class MyAnimeList : MetadataFetcher
{
private static readonly Jikan Jikan = new ();
private static readonly Regex GetIdFromUrl = new(@"https?:\/\/myanimelist\.net\/manga\/([0-9]+)\/?.*");
public override MetadataSearchResult[] SearchMetadataEntry(Manga manga)
{
if (manga.Links.Any(link => link.LinkProvider.Equals("MyAnimeList", StringComparison.InvariantCultureIgnoreCase)))
{
string url = manga.Links.First(link => link.LinkProvider.Equals("MyAnimeList", StringComparison.InvariantCultureIgnoreCase)).LinkUrl;
Match m = GetIdFromUrl.Match(url);
if (m.Success && m.Groups[1].Success)
{
long id = long.Parse(m.Groups[1].Value);
JikanDotNet.Manga data = Jikan.GetMangaAsync(id).Result.Data;
return [new MetadataSearchResult(id.ToString(), data.Titles.First().Title, data.Url, data.Synopsis)];
}
}
return SearchMetadataEntry(manga.Name);
}
public override MetadataSearchResult[] SearchMetadataEntry(string searchTerm)
{
ICollection<JikanDotNet.Manga> resultData = Jikan.SearchMangaAsync(searchTerm).Result.Data;
if (resultData.Count < 1)
return [];
return resultData.Select(data =>
new MetadataSearchResult(data.MalId.ToString(), data.Titles.First().Title, data.Url, data.Synopsis))
.ToArray();
}
/// <summary>
/// Updates the Manga linked in the MetadataEntry
/// </summary>
/// <param name="metadataEntry"></param>
/// <param name="dbContext"></param>
/// <exception cref="FormatException"></exception>
/// <exception cref="DbUpdateException"></exception>
public override void UpdateMetadata(MetadataEntry metadataEntry, PgsqlContext dbContext)
{
Manga dbManga = dbContext.Mangas.Find(metadataEntry.MangaId)!;
MangaFull resultData;
try
{
long id = long.Parse(metadataEntry.Identifier);
resultData = Jikan.GetMangaFullDataAsync(id).Result.Data;
}
catch (Exception)
{
throw new FormatException("ID was not in correct format");
}
try
{
dbManga.Name = resultData.Titles.First().Title;
dbManga.Description = resultData.Synopsis;
dbManga.AltTitles.Clear();
dbManga.AltTitles = resultData.Titles.Select(t => new MangaAltTitle(t.Type, t.Title)).ToList();
dbManga.Authors.Clear();
dbManga.Authors = resultData.Authors.Select(a => new Author(a.Name)).ToList();
dbContext.SaveChanges();
}
catch (DbUpdateException e)
{
throw;
}
}
}

View File

@ -2,6 +2,7 @@
using API.Schema.Contexts;
using API.Schema.Jobs;
using API.Schema.MangaConnectors;
using API.Schema.MetadataFetchers;
using API.Schema.NotificationConnectors;
using log4net;
using log4net.Config;
@ -24,6 +25,7 @@ public static class Tranga
public static Thread NotificationSenderThread { get; } = new (NotificationSender);
public static Thread JobStarterThread { get; } = new (JobStarter);
private static readonly ILog Log = LogManager.GetLogger(typeof(Tranga));
internal static MetadataFetcher[] MetadataFetchers = [new MyAnimeList()];
internal static void StartLogger()
{
@ -137,12 +139,12 @@ public static class Tranga
List<Job> dueJobs = waitingJobs.FilterDueJobs();
List<Job> jobsWithoutDependencies = dueJobs.FilterJobDependencies();
List<Job> jobsWithoutDownloading = jobsWithoutDependencies.Where(j => GetJobConnector(j) is null).ToList();
List<Job> jobsWithoutDownloading = jobsWithoutDependencies.FilterJobsWithoutDownloading();
//Match running and waiting jobs per Connector
Dictionary<MangaConnector, Dictionary<JobType, List<Job>>> runningJobsPerConnector =
Dictionary<string, Dictionary<JobType, List<Job>>> runningJobsPerConnector =
runningJobs.GetJobsPerJobTypeAndConnector();
Dictionary<MangaConnector, Dictionary<JobType, List<Job>>> waitingJobsPerConnector =
Dictionary<string, Dictionary<JobType, List<Job>>> waitingJobsPerConnector =
jobsWithoutDependencies.GetJobsPerJobTypeAndConnector();
List<Job> jobsNotHeldBackByConnector =
MatchJobsRunningAndWaiting(runningJobsPerConnector, waitingJobsPerConnector);
@ -169,11 +171,15 @@ public static class Tranga
while(!running)
Thread.Sleep(10);
}
Log.Debug($"Running: {runningJobs.Count} Waiting: {waitingJobs.Count} Due: {dueJobs.Count} of which \n" +
$"{jobsWithoutDependencies.Count} without missing dependencies, of which\n" +
Log.Debug($"Running: {runningJobs.Count}\n" +
$"{string.Join("\n", runningJobs.Select(s => "\t- " + s))}\n" +
$"Waiting: {waitingJobs.Count} Due: {dueJobs.Count}\n" +
$"{string.Join("\n", dueJobs.Select(s => "\t- " + s))}\n" +
$"of which {jobsWithoutDependencies.Count} without missing dependencies, of which\n" +
$"\t{jobsWithoutDownloading.Count} without downloading\n" +
$"\t{jobsNotHeldBackByConnector.Count} not held back by Connector\n" +
$"{startJobs.Count} were started.");
$"{startJobs.Count} were started:\n" +
$"{string.Join("\n", startJobs.Select(s => "\t- " + s))}");
if (Log.IsDebugEnabled && dueJobs.Count < 1)
if(waitingJobs.MinBy(j => j.NextExecution) is { } nextJob)
@ -195,32 +201,68 @@ public static class Tranga
{
Log.Error("Failed saving Job changes.", e);
}
Log.Debug($"Job-Cycle over! (took {DateTime.UtcNow.Subtract(cycleStart).TotalMilliseconds}ms)");
Log.Debug($"Job-Cycle over! (took {DateTime.UtcNow.Subtract(cycleStart).TotalMilliseconds}ms");
Thread.Sleep(TrangaSettings.startNewJobTimeoutMs);
}
}
private static List<Job> GetRunningJobs(this IQueryable<Job> jobs) =>
jobs.Where(j => j.state == JobState.Running).ToList();
private static List<Job> GetWaitingJobs(this IQueryable<Job> jobs) =>
jobs.Where(j => j.state == JobState.CompletedWaiting || j.state == JobState.FirstExecution)
.ToList();
private static List<Job> FilterDueJobs(this List<Job> jobs) =>
jobs.Where(j => j.NextExecution < DateTime.UtcNow)
.ToList();
private static List<Job> FilterJobDependencies(this List<Job> jobs) =>
jobs.Where(job => job.DependsOnJobs.All(j => j.IsCompleted))
.ToList();
private static Dictionary<MangaConnector, Dictionary<JobType, List<Job>>> GetJobsPerJobTypeAndConnector(this List<Job> jobs)
private static List<Job> GetRunningJobs(this IQueryable<Job> jobs)
{
Dictionary<MangaConnector, Dictionary<JobType, List<Job>>> ret = new();
DateTime start = DateTime.UtcNow;
List<Job> ret = jobs.Where(j => j.state == JobState.Running).ToList();
DateTime end = DateTime.UtcNow;
Log.Debug($"Getting running Jobs took {end.Subtract(start).TotalMilliseconds}ms");
return ret;
}
private static List<Job> GetWaitingJobs(this IQueryable<Job> jobs)
{
DateTime start = DateTime.UtcNow;
List<Job> ret = jobs.Where(j => j.state == JobState.CompletedWaiting || j.state == JobState.FirstExecution).ToList();
DateTime end = DateTime.UtcNow;
Log.Debug($"Getting waiting Jobs took {end.Subtract(start).TotalMilliseconds}ms");
return ret;
}
private static List<Job> FilterDueJobs(this List<Job> jobs)
{
DateTime start = DateTime.UtcNow;
List<Job> ret = jobs.Where(j => j.NextExecution < DateTime.UtcNow).ToList();
DateTime end = DateTime.UtcNow;
Log.Debug($"Filtering Due Jobs took {end.Subtract(start).TotalMilliseconds}ms");
return ret;
}
private static List<Job> FilterJobDependencies(this List<Job> jobs)
{
DateTime start = DateTime.UtcNow;
List<Job> ret = jobs.Where(job => job.DependsOnJobs.All(j => j.IsCompleted)).ToList();
DateTime end = DateTime.UtcNow;
Log.Debug($"Filtering Dependencies took {end.Subtract(start).TotalMilliseconds}ms");
return ret;
}
private static List<Job> FilterJobsWithoutDownloading(this List<Job> jobs)
{
JobType[] types = [JobType.MoveFileOrFolderJob, JobType.MoveMangaLibraryJob, JobType.UpdateChaptersDownloadedJob];
DateTime start = DateTime.UtcNow;
List<Job> ret = jobs.Where(j => types.Contains(j.JobType)).ToList();
DateTime end = DateTime.UtcNow;
Log.Debug($"Filtering Jobs without Download took {end.Subtract(start).TotalMilliseconds}ms");
return ret;
}
private static Dictionary<string, Dictionary<JobType, List<Job>>> GetJobsPerJobTypeAndConnector(this List<Job> jobs)
{
DateTime start = DateTime.UtcNow;
Dictionary<string, Dictionary<JobType, List<Job>>> ret = new();
foreach (Job job in jobs)
{
if(GetJobConnector(job) is not { } connector)
if(GetJobConnectorName(job) is not { } connector)
continue;
if (!ret.ContainsKey(connector))
ret.Add(connector, new());
@ -228,14 +270,17 @@ public static class Tranga
ret[connector].Add(job.JobType, new());
ret[connector][job.JobType].Add(job);
}
DateTime end = DateTime.UtcNow;
Log.Debug($"Fetching connector per Job for jobs took {end.Subtract(start).TotalMilliseconds}ms");
return ret;
}
private static List<Job> MatchJobsRunningAndWaiting(Dictionary<MangaConnector, Dictionary<JobType, List<Job>>> running,
Dictionary<MangaConnector, Dictionary<JobType, List<Job>>> waiting)
private static List<Job> MatchJobsRunningAndWaiting(Dictionary<string, Dictionary<JobType, List<Job>>> running,
Dictionary<string, Dictionary<JobType, List<Job>>> waiting)
{
DateTime start = DateTime.UtcNow;
List<Job> ret = new();
foreach ((MangaConnector connector, Dictionary<JobType, List<Job>> jobTypeJobsWaiting) in waiting)
foreach ((string connector, Dictionary<JobType, List<Job>> jobTypeJobsWaiting) in waiting)
{
if (running.TryGetValue(connector, out Dictionary<JobType, List<Job>>? jobTypeJobsRunning))
{ //MangaConnector has running Jobs
@ -266,20 +311,22 @@ public static class Tranga
}
}
}
DateTime end = DateTime.UtcNow;
Log.Debug($"Getting eligible jobs (not held back by Connector) took {end.Subtract(start).TotalMilliseconds}ms");
return ret;
}
private static MangaConnector? GetJobConnector(Job job)
private static string? GetJobConnectorName(Job job)
{
if (job is DownloadAvailableChaptersJob dacj)
return dacj.Manga.MangaConnector;
return dacj.Manga.MangaConnectorName;
if (job is DownloadMangaCoverJob dmcj)
return dmcj.Manga.MangaConnector;
return dmcj.Manga.MangaConnectorName;
if (job is DownloadSingleChapterJob dscj)
return dscj.Chapter.ParentManga.MangaConnector;
return dscj.Chapter.ParentManga.MangaConnectorName;
if (job is RetrieveChaptersJob rcj)
return rcj.Manga.MangaConnector;
return rcj.Manga.MangaConnectorName;
return null;
}
}

View File

@ -36,14 +36,14 @@ public static class TrangaSettings
[JsonIgnore]
public static string coverImageCache => Path.Join(workingDirectory, "imageCache");
public static bool aprilFoolsMode { get; private set; } = true;
public static int startNewJobTimeoutMs { get; private set; } = 5000;
public static int startNewJobTimeoutMs { get; private set; } = 20000;
[JsonIgnore]
internal static readonly Dictionary<RequestType, int> DefaultRequestLimits = new ()
{
{RequestType.MangaInfo, 60},
{RequestType.MangaDexFeed, 60},
{RequestType.MangaDexImage, 40},
{RequestType.MangaImage, 60},
{RequestType.MangaDexImage, 60},
{RequestType.MangaImage, 240},
{RequestType.MangaCover, 60},
{RequestType.Default, 60}
};

View File

@ -39,4 +39,4 @@ WORKDIR /publish
COPY --chown=1000:1000 --from=build-env /publish .
USER 0
ENTRYPOINT ["dotnet", "/publish/API.dll"]
CMD ["-f", "-c", "-l", "/usr/share/tranga-api/logs"]
CMD [""]

View File

@ -84,18 +84,18 @@ Endpoints are documented in Swagger. Just spin up an instance, and go to `http:/
## Built With
- .NET
- ASP.NET
- ASP.NET
- Entity Framework Core
- [PostgreSQL](https://www.postgresql.org/about/licence/)
- [Swagger](https://github.com/domaindrivendev/Swashbuckle.AspNetCore/blob/master/LICENSE)
- [Ngpsql](https://github.com/npgsql/npgsql/blob/main/LICENSE)
- [Swagger](https://github.com/domaindrivendev/Swashbuckle.AspNetCore/blob/master/LICENSE)
- [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json/blob/master/LICENSE.md)
- [Sixlabors.ImageSharp](https://docs-v2.sixlabors.com/articles/imagesharp/index.html#license)
- [PuppeteerSharp](https://github.com/hardkoded/puppeteer-sharp/blob/master/LICENSE)
- [FlareSolverrSharp](https://github.com/FlareSolverr/FlareSolverrSharp)
- [Html Agility Pack (HAP)](https://github.com/zzzprojects/html-agility-pack/blob/master/LICENSE)
- [Soenneker.Utils.String.NeedlemanWunsch](https://github.com/soenneker/soenneker.utils.string.needlemanwunsch/blob/main/LICENSE)
- [Sixlabors.ImageSharp](https://docs-v2.sixlabors.com/articles/imagesharp/index.html#license)
- [Jikan](https://jikan.moe/)
- [Jikan.Net](https://github.com/Ervie/jikan.net)
- 💙 Blåhaj 🦈
<p align="right">(<a href="#readme-top">back to top</a>)</p>
@ -127,13 +127,13 @@ access the folder. Permission conflicts with Komga and Kavita should thus be lim
### Bare-Metal
While not supported/currently built, Tranga will also run Bare-Metal without issue.
While not supported/currently built, Tranga should also run Bare-Metal without issue.
Configuration-Files will be stored per OS:
- Linux `/usr/share/tranga-api`
- Windows `%appdata%/tranga-api`
Downloads (default) are stored in - but this can be configured in `settings.json`:
Downloads (default) are stored in - but this can be configured in `settings.json` (which will be generated on first after first launch):
- Linux `/Manga`
- Windows `%currentDirectory%/Downloads`
@ -149,9 +149,10 @@ If you want to contribute, please feel free to fork and create a Pull-Request!
General rules:
- Strongly-type your variables. This improves readability.
```csharp
var xyz = Object.GetSomething(); //Do not do this. What type is xyz?
var xyz = Object.GetSomething(); //Do not do this. What type is xyz (without looking at Method returns etc.)?
Manga[] zyx = Object.GetAnotherThing(); //I can now easily see that zyx is an Array.
```
Tranga is using a code-first Entity-Framework Core approach. If you modify the db-table structure you need to create a migration.
**A broad overview of where is what:**<br />
@ -172,6 +173,10 @@ If you want to add a new Website-Connector: <br />
in the constructor).
4. In `Program.cs` add a new Object to the Array.
### How to test locally
In the Project root a `docker-compose.local.yaml` file will compile the code and create the container(s).
<!-- LICENSE -->
## License

View File

@ -2,6 +2,7 @@
<s:Boolean x:Key="/Default/UserDictionary/Words/=altnames/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=authorsartists/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Gotify/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=jikan/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=jjob/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Komga/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=lunasea/@EntryIndexedValue">True</s:Boolean>

View File

@ -16,6 +16,11 @@ services:
environment:
- POSTGRES_HOST=tranga-pg
restart: unless-stopped
logging:
driver: json-file
options:
max-size: "10m"
max-file: "5"
tranga-pg:
image: postgres:latest
container_name: tranga-pg
@ -30,3 +35,8 @@ services:
retries: 5
start_period: 80s
restart: unless-stopped
logging:
driver: json-file
options:
max-size: "10m"
max-file: "5"

View File

@ -1,7 +1,7 @@
version: '3'
services:
tranga-api:
image: glax/tranga-api:latest
image: glax/tranga-api:Server-V2
container_name: tranga-api
volumes:
- ./Manga:/Manga
@ -14,14 +14,24 @@ services:
environment:
- POSTGRES_HOST=tranga-pg
restart: unless-stopped
logging:
driver: json-file
options:
max-size: "10m"
max-file: "5"
tranga-website:
image: glax/tranga-website:latest
image: glax/tranga-website:Server-V2
container_name: tranga-website
ports:
- "9555:80"
depends_on:
- tranga-api
restart: unless-stopped
logging:
driver: json-file
options:
max-size: "10m"
max-file: "5"
tranga-pg:
image: postgres:latest
container_name: tranga-pg
@ -36,3 +46,8 @@ services:
retries: 5
start_period: 80s
restart: unless-stopped
logging:
driver: json-file
options:
max-size: "10m"
max-file: "5"