glax/Tranga
1
0
Fork 0
mirror of https://github.com/C9Glax/tranga.git synced 2025-09-10 11:58:19 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Disable LazyLoading

Browse Source 
Remove MangaConnectors from Database
This commit is contained in:
glax
2025-07-21 11:42:17 +02:00
parent 394944e11a
commit cae8cde53f
20 changed files with 167 additions and 311 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
API/Schema/MangaContext/Chapter.cs
View File

@@ -13,30 +13,12 @@ namespace API.Schema.MangaContext;
public class Chapter : Identifiable, IComparable<Chapter>
{
[StringLength(64)] [Required] public string ParentMangaId { get; init; } = null!;
private Manga? _parentManga;
[JsonIgnore]
public Manga ParentManga
{
get => _lazyLoader.Load(this, ref _parentManga) ?? throw new InvalidOperationException();
init
{
ParentMangaId = value.Key;
_parentManga = value;
}
}
[JsonIgnore] public Manga ParentManga = null!;
[NotMapped]
public Dictionary<string, string> IdsOnMangaConnectors =>
MangaConnectorIds.ToDictionary(id => id.MangaConnectorName, id => id.IdOnConnectorSite);
private ICollection<MangaConnectorId<Chapter>>? _mangaConnectorIds;
[JsonIgnore]
public ICollection<MangaConnectorId<Chapter>> MangaConnectorIds
{
get => _lazyLoader.Load(this, ref _mangaConnectorIds) ?? throw new InvalidOperationException();
init => _mangaConnectorIds = value;
}
[JsonIgnore] public ICollection<MangaConnectorId<Chapter>> MangaConnectorIds = null!;
public int? VolumeNumber { get; private set; }
[StringLength(10)] [Required] public string ChapterNumber { get; private set; }
@@ -48,8 +30,6 @@ public class Chapter : Identifiable, IComparable<Chapter>
[Required] public bool Downloaded { get; internal set; }
[NotMapped] public string FullArchiveFilePath => Path.Join(ParentManga.FullDirectoryPath, FileName);
private readonly ILazyLoader _lazyLoader = null!;
public Chapter(Manga parentManga, string chapterNumber,
int? volumeNumber, string? title = null)
: base(TokenGen.CreateToken(typeof(Chapter), parentManga.Key, chapterNumber))
@@ -66,10 +46,9 @@ public class Chapter : Identifiable, IComparable<Chapter>
/// <summary>
/// EF ONLY!!!
/// </summary>
internal Chapter(ILazyLoader lazyLoader, string key, int? volumeNumber, string chapterNumber, string? title, string fileName, bool downloaded)
internal Chapter(string key, int? volumeNumber, string chapterNumber, string? title, string fileName, bool downloaded)
: base(key)
{
this._lazyLoader = lazyLoader;
this.VolumeNumber = volumeNumber;
this.ChapterNumber = chapterNumber;
this.Title = title;

33
API/Schema/MangaContext/Manga.cs
View File

@@ -18,17 +18,7 @@ public class Manga : Identifiable
[JsonIgnore] [Url] [StringLength(512)] public string CoverUrl { get; internal set; }
[Required] public MangaReleaseStatus ReleaseStatus { get; internal set; }
[StringLength(64)] public string? LibraryId { get; private set; }
private FileLibrary? _library;
[JsonIgnore]
public FileLibrary? Library
{
get => _lazyLoader.Load(this, ref _library);
set
{
LibraryId = value?.Key;
_library = value;
}
}
[JsonIgnore] public FileLibrary? Library = null!;
public ICollection<Author> Authors { get; internal set; }= null!;
public ICollection<MangaTag> MangaTags { get; internal set; }= null!;
@@ -45,25 +35,11 @@ public class Manga : Identifiable
public string? FullDirectoryPath => Library is not null ? Path.Join(Library.BasePath, DirectoryName) : null;
[NotMapped] public ICollection<string> ChapterIds => Chapters.Select(c => c.Key).ToList();
private ICollection<Chapter>? _chapters;
[JsonIgnore]
public ICollection<Chapter> Chapters
{
get => _lazyLoader.Load(this, ref _chapters) ?? throw new InvalidOperationException();
init => _chapters = value;
}
[JsonIgnore] public ICollection<Chapter> Chapters = null!;
[NotMapped] public Dictionary<string, string> IdsOnMangaConnectors =>
MangaConnectorIds.ToDictionary(id => id.MangaConnectorName, id => id.IdOnConnectorSite);
private ICollection<MangaConnectorId<Manga>>? _mangaConnectorIds;
[JsonIgnore]
public ICollection<MangaConnectorId<Manga>> MangaConnectorIds
{
get => _lazyLoader.Load(this, ref _mangaConnectorIds) ?? throw new InvalidOperationException();
private set => _mangaConnectorIds = value;
}
private readonly ILazyLoader _lazyLoader = null!;
[JsonIgnore] public ICollection<MangaConnectorId<Manga>> MangaConnectorIds = null!;
public Manga(string name, string description, string coverUrl, MangaReleaseStatus releaseStatus,
ICollection<Author> authors, ICollection<MangaTag> mangaTags, ICollection<Link> links, ICollection<AltTitle> altTitles,
@@ -89,12 +65,11 @@ public class Manga : Identifiable
/// <summary>
/// EF ONLY!!!
/// </summary>
public Manga(ILazyLoader lazyLoader, string key, string name, string description, string coverUrl,
public Manga(string key, string name, string description, string coverUrl,
MangaReleaseStatus releaseStatus,
string directoryName, float ignoreChaptersBefore, string? libraryId, uint? year, string? originalLanguage)
: base(key)
{
this._lazyLoader = lazyLoader;
this.Name = name;
this.Description = description;
this.CoverUrl = coverUrl;

36
API/Schema/MangaContext/MangaConnectorId.cs
View File

@@ -1,7 +1,6 @@
using System.ComponentModel.DataAnnotations;
using API.Schema.MangaContext.MangaConnectors;
using API.MangaConnectors;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Newtonsoft.Json;
namespace API.Schema.MangaContext;
@@ -10,43 +9,19 @@ namespace API.Schema.MangaContext;
public class MangaConnectorId<T> : Identifiable where T : Identifiable
{
[StringLength(64)] [Required] public string ObjId { get; private set; } = null!;
[JsonIgnore] private T? _obj;
[JsonIgnore]
public T Obj
{
get => _lazyLoader.Load(this, ref _obj) ?? throw new InvalidOperationException();
internal set
{
ObjId = value.Key;
_obj = value;
}
}
[JsonIgnore] public T Obj = null!;
[StringLength(32)] [Required] public string MangaConnectorName { get; private set; } = null!;
[JsonIgnore] private MangaConnector? _mangaConnector;
[JsonIgnore]
public MangaConnector MangaConnector
{
get => _lazyLoader.Load(this, ref _mangaConnector) ?? throw new InvalidOperationException();
init
{
MangaConnectorName = value.Name;
_mangaConnector = value;
}
}
[StringLength(256)] [Required] public string IdOnConnectorSite { get; init; }
[Url] [StringLength(512)] public string? WebsiteUrl { get; internal init; }
public bool UseForDownload { get; internal set; }
private readonly ILazyLoader _lazyLoader = null!;
public MangaConnectorId(T obj, MangaConnector mangaConnector, string idOnConnectorSite, string? websiteUrl, bool useForDownload = false)
: base(TokenGen.CreateToken(typeof(MangaConnectorId<T>), mangaConnector.Name, idOnConnectorSite))
{
this.Obj = obj;
this.MangaConnector = mangaConnector;
this.MangaConnectorName = mangaConnector.Name;
this.IdOnConnectorSite = idOnConnectorSite;
this.WebsiteUrl = websiteUrl;
this.UseForDownload = useForDownload;
@@ -55,10 +30,9 @@ public class MangaConnectorId<T> : Identifiable where T : Identifiable
/// <summary>
/// EF CORE ONLY!!!
/// </summary>
public MangaConnectorId(ILazyLoader lazyLoader, string key, string objId, string mangaConnectorName, string idOnConnectorSite, bool useForDownload, string? websiteUrl)
public MangaConnectorId(string key, string objId, string mangaConnectorName, string idOnConnectorSite, bool useForDownload, string? websiteUrl)
: base(key)
{
this._lazyLoader = lazyLoader;
this.ObjId = objId;
this.MangaConnectorName = mangaConnectorName;
this.IdOnConnectorSite = idOnConnectorSite;
@@ -66,5 +40,5 @@ public class MangaConnectorId<T> : Identifiable where T : Identifiable
this.UseForDownload = useForDownload;
}
public override string ToString() => $"{base.ToString()} {_obj}";
public override string ToString() => $"{base.ToString()} {Obj}";
}

258
API/Schema/MangaContext/MangaConnectors/ComickIo.cs
View File

@@ -1,258 +0,0 @@
using System.Text.RegularExpressions;
using API.MangaDownloadClients;
using Newtonsoft.Json.Linq;
namespace API.Schema.MangaContext.MangaConnectors;
public class ComickIo : MangaConnector
{
//https://api.comick.io/docs/
//https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes
public ComickIo() : base("ComickIo",
["en","pt","pt-br","it","de","ru","aa","ab","ae","af","ak","am","an","ar-ae","ar-bh","ar-dz","ar-eg","ar-iq","ar-jo","ar-kw","ar-lb","ar-ly","ar-ma","ar-om","ar-qa","ar-sa","ar-sy","ar-tn","ar-ye","ar","as","av","ay","az","ba","be","bg","bh","bi","bm","bn","bo","br","bs","ca","ce","ch","co","cr","cs","cu","cv","cy","da","de-at","de-ch","de-de","de-li","de-lu","div","dv","dz","ee","el","en-au","en-bz","en-ca","en-cb","en-gb","en-ie","en-jm","en-nz","en-ph","en-tt","en-us","en-za","en-zw","eo","es-ar","es-bo","es-cl","es-co","es-cr","es-do","es-ec","es-es","es-gt","es-hn","es-la","es-mx","es-ni","es-pa","es-pe","es-pr","es-py","es-sv","es-us","es-uy","es-ve","es","et","eu","fa","ff","fi","fj","fo","fr-be","fr-ca","fr-ch","fr-fr","fr-lu","fr-mc","fr","fy","ga","gd","gl","gn","gu","gv","ha","he","hi","ho","hr-ba","hr-hr","hr","ht","hu","hy","hz","ia","id","ie","ig","ii","ik","in","io","is","it-ch","it-it","iu","iw","ja","ja-ro","ji","jv","jw","ka","kg","ki","kj","kk","kl","km","kn","ko","ko-ro","kr","ks","ku","kv","kw","ky","kz","la","lb","lg","li","ln","lo","ls","lt","lu","lv","mg","mh","mi","mk","ml","mn","mo","mr","ms-bn","ms-my","ms","mt","my","na","nb","nd","ne","ng","nl-be","nl-nl","nl","nn","no","nr","ns","nv","ny","oc","oj","om","or","os","pa","pi","pl","ps","pt-pt","qu-bo","qu-ec","qu-pe","qu","rm","rn","ro","rw","sa","sb","sc","sd","se-fi","se-no","se-se","se","sg","sh","si","sk","sl","sm","sn","so","sq","sr-ba","sr-sp","sr","ss","st","su","sv-fi","sv-se","sv","sw","sx","syr","ta","te","tg","th","ti","tk","tl","tn","to","tr","ts","tt","tw","ty","ug","uk","ur","us","uz","ve","vi","vo","wa","wo","xh","yi","yo","za","zh-cn","zh-hk","zh-mo","zh-ro","zh-sg","zh-tw","zh","zu"],
["comick.io"],
"https://comick.io/static/icons/unicorn-64.png")
{
this.downloadClient = new HttpDownloadClient();
}
public override (Manga, MangaConnectorId<Manga>)[] SearchManga(string mangaSearchName)
{
Log.Info($"Searching Obj: {mangaSearchName}");
List<string> slugs = new();
int page = 1;
while(page < 50)
{
string requestUrl = $"https://api.comick.fun/v1.0/search/?type=comic&t=false&limit=100&showall=true&" +
$"page={page}&q={mangaSearchName}";
RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
{
Log.Error("Request failed");
return [];
}
using StreamReader sr = new (result.result);
JArray data = JArray.Parse(sr.ReadToEnd());
if (data.Count < 1)
break;
slugs.AddRange(data.Select(token => token.Value<string>("slug")!));
page++;
}
Log.Debug($"Search {mangaSearchName} yielded {slugs.Count} slugs. Requesting mangas now...");
List<(Manga, MangaConnectorId<Manga>)> mangas = new ();
foreach (string slug in slugs)
{
if(GetMangaFromId(slug) is { } entry)
mangas.Add(entry);
}
Log.Info($"Search {mangaSearchName} yielded {mangas.Count} results.");
return mangas.ToArray();
}
private readonly Regex _getSlugFromTitleRex = new(@"https?:\/\/comick\.io\/comic\/(.+)(?:\/.*)*");
public override (Manga, MangaConnectorId<Manga>)? GetMangaFromUrl(string url)
{
Match m = _getSlugFromTitleRex.Match(url);
return m.Groups[1].Success ? GetMangaFromId(m.Groups[1].Value) : null;
}
public override (Manga, MangaConnectorId<Manga>)? GetMangaFromId(string mangaIdOnSite)
{
string requestUrl = $"https://api.comick.fun/comic/{mangaIdOnSite}";
RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.MangaInfo);
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
{
Log.Error("Request failed");
return null;
}
using StreamReader sr = new (result.result);
JToken data = JToken.Parse(sr.ReadToEnd());
return ParseMangaFromJToken(data);
}
public override (Chapter, MangaConnectorId<Chapter>)[] GetChapters(MangaConnectorId<Manga> mangaConnectorId,
string? language = null)
{
Log.Info($"Getting Chapters: {mangaConnectorId.IdOnConnectorSite}");
List<(Chapter, MangaConnectorId<Chapter>)> chapters = new();
int page = 1;
while(page < 50)
{
string requestUrl = $"https://api.comick.fun/comic/{mangaConnectorId.IdOnConnectorSite}/chapters?limit=100&page={page}&lang={language}";
RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.MangaInfo);
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
{
Log.Error("Request failed");
return [];
}
using StreamReader sr = new (result.result);
JToken data = JToken.Parse(sr.ReadToEnd());
JArray? chaptersArray = data["chapters"] as JArray;
if (chaptersArray is null || chaptersArray.Count < 1)
break;
chapters.AddRange(ParseChapters(mangaConnectorId, chaptersArray));
page++;
}
return chapters.ToArray();
}
private readonly Regex _hidFromUrl = new(@"https?:\/\/comick\.io\/comic\/.+\/([^-]+).*");
internal override string[] GetChapterImageUrls(MangaConnectorId<Chapter> chapterId)
{
Log.Info($"Getting Chapter Image-Urls: {chapterId.Obj}");
if (chapterId.WebsiteUrl is null || !UrlMatchesConnector(chapterId.WebsiteUrl))
{
Log.Debug($"Url is not for Connector. {chapterId.WebsiteUrl}");
return [];
}
Match m = _hidFromUrl.Match(chapterId.WebsiteUrl);
if (!m.Groups[1].Success)
{
Log.Debug($"Could not parse hid from url. {chapterId.WebsiteUrl}");
return [];
}
string hid = m.Groups[1].Value;
string requestUrl = $"https://api.comick.fun/chapter/{hid}/get_images";
RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.MangaInfo);
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
{
Log.Error("Request failed");
return [];
}
using StreamReader sr = new (result.result);
JArray data = JArray.Parse(sr.ReadToEnd());
return data.Select(token =>
{
string url = $"https://meo.comick.pictures/{token.Value<string>("b2key")}";
return url;
}).ToArray();
}
private (Manga manga, MangaConnectorId<Manga> id) ParseMangaFromJToken(JToken json)
{
string? hid = json["comic"]?.Value<string>("hid");
string? slug = json["comic"]?.Value<string>("slug");
string? name = json["comic"]?.Value<string>("title");
string? description = json["comic"]?.Value<string>("desc");
string? originalLanguage = json["comic"]?.Value<string>("country");
string url = $"https://comick.io/comic/{slug}";
string? coverName = json["comic"]?["md_covers"]?.First?.Value<string>("b2key");
string coverUrl = $"https://meo.comick.pictures/{coverName}";
int? releaseStatusStr = json["comic"]?.Value<int>("status");
MangaReleaseStatus status = releaseStatusStr switch
{
1 => MangaReleaseStatus.Continuing,
2 => MangaReleaseStatus.Completed,
3 => MangaReleaseStatus.Cancelled,
4 => MangaReleaseStatus.OnHiatus,
_ => MangaReleaseStatus.Unreleased
};
uint? year = json["comic"]?.Value<uint?>("year");
JArray? altTitlesArray = json["comic"]?["md_titles"] as JArray;
//Cant let language be null, so fill with whatever.
byte whatever = 0;
List<AltTitle> altTitles = altTitlesArray?
.Select(token => new AltTitle(token.Value<string>("lang")??whatever++.ToString(), token.Value<string>("title")!))
.ToList()!;
JArray? authorsArray = json["authors"] as JArray;
JArray? artistsArray = json["artists"] as JArray;
List<Author> authors = authorsArray?.Concat(artistsArray!)
.Select(token => new Author(token.Value<string>("name")!))
.DistinctBy(a => a.Key)
.ToList()!;
JArray? genreArray = json["comic"]?["md_comic_md_genres"] as JArray;
List<MangaTag> tags = genreArray?
.Select(token => new MangaTag(token["md_genres"]?.Value<string>("name")!))
.ToList()!;
JArray? linksArray = json["comic"]?["links"] as JArray;
List<Link> links = linksArray?
.ToObject<Dictionary<string,string>>()?
.Select(kv =>
{
string fullUrl = kv.Key switch
{
"al" => $"https://anilist.co/manga/{kv.Value}",
"ap" => $"https://www.anime-planet.com/manga/{kv.Value}",
"bw" => $"https://bookwalker.jp/{kv.Value}",
"mu" => $"https://www.mangaupdates.com/series.html?id={kv.Value}",
"nu" => $"https://www.novelupdates.com/series/{kv.Value}",
"mal" => $"https://myanimelist.net/manga/{kv.Value}",
_ => kv.Value
};
string key = kv.Key switch
{
"al" => "AniList",
"ap" => "Anime Planet",
"bw" => "BookWalker",
"mu" => "Obj Updates",
"nu" => "Novel Updates",
"kt" => "Kitsu.io",
"amz" => "Amazon",
"ebj" => "eBookJapan",
"mal" => "MyAnimeList",
"cdj" => "CDJapan",
_ => kv.Key
};
return new Link(key, fullUrl);
}).ToList()!;
if(hid is null)
throw new Exception("hid is null");
if(slug is null)
throw new Exception("slug is null");
if(name is null)
throw new Exception("name is null");
Manga manga = new (name, description??"", coverUrl, status, authors, tags, links, altTitles,
year: year, originalLanguage: originalLanguage);
return (manga, new MangaConnectorId<Manga>(manga, this, hid, url));
}
private List<(Chapter, MangaConnectorId<Chapter>)> ParseChapters(MangaConnectorId<Manga> mcIdManga, JArray chaptersArray)
{
List<(Chapter, MangaConnectorId<Chapter>)> chapters = new ();
foreach (JToken chapter in chaptersArray)
{
string? chapterNum = chapter.Value<string>("chap");
string? volumeNumStr = chapter.Value<string>("vol");
int? volumeNum = volumeNumStr is null ? null : int.Parse(volumeNumStr);
string? title = chapter.Value<string>("title");
string? hid = chapter.Value<string>("hid");
string url = $"https://comick.io/comic/{mcIdManga.IdOnConnectorSite}/{hid}";
if(chapterNum is null || hid is null)
continue;
Chapter ch = new (mcIdManga.Obj, chapterNum, volumeNum, title);
chapters.Add((ch, new (ch, this, hid, url)));
}
return chapters;
}
}

54
API/Schema/MangaContext/MangaConnectors/Global.cs
View File

@@ -1,54 +0,0 @@
namespace API.Schema.MangaContext.MangaConnectors;
public class Global : MangaConnector
{
private MangaContext context { get; init; }
public Global(MangaContext context) : base("Global", ["all"], [""], "")
{
this.context = context;
}
public override (Manga, MangaConnectorId<Manga>)[] SearchManga(string mangaSearchName)
{
//Get all enabled Connectors
MangaConnector[] enabledConnectors = context.MangaConnectors.Where(c => c.Enabled && c.Name != "Global").ToArray();
//Create Task for each MangaConnector to search simultaneously
Task<(Manga, MangaConnectorId<Manga>)[]>[] tasks =
enabledConnectors.Select(c => new Task<(Manga, MangaConnectorId<Manga>)[]>(() => c.SearchManga(mangaSearchName))).ToArray();
foreach (var task in tasks)
task.Start();
//Wait for all tasks to finish
do
{
Thread.Sleep(50);
}while(tasks.Any(t => t.Status < TaskStatus.RanToCompletion));
//Concatenate all results into one
(Manga, MangaConnectorId<Manga>)[] ret = tasks.Select(t => t.IsCompletedSuccessfully ? t.Result : []).ToArray().SelectMany(i => i).ToArray();
return ret;
}
public override (Manga, MangaConnectorId<Manga>)? GetMangaFromUrl(string url)
{
MangaConnector? mc = context.MangaConnectors.ToArray().FirstOrDefault(c => c.UrlMatchesConnector(url));
return mc?.GetMangaFromUrl(url) ?? null;
}
public override (Manga, MangaConnectorId<Manga>)? GetMangaFromId(string mangaIdOnSite)
{
return null;
}
public override (Chapter, MangaConnectorId<Chapter>)[] GetChapters(MangaConnectorId<Manga> manga,
string? language = null)
{
return manga.MangaConnector.GetChapters(manga, language);
}
internal override string[] GetChapterImageUrls(MangaConnectorId<Chapter> chapterId)
{
return chapterId.MangaConnector.GetChapterImageUrls(chapterId);
}
}

75
API/Schema/MangaContext/MangaConnectors/MangaConnector.cs
View File

@@ -1,75 +0,0 @@
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text.RegularExpressions;
using API.MangaDownloadClients;
using log4net;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
namespace API.Schema.MangaContext.MangaConnectors;
[PrimaryKey("Name")]
public abstract class MangaConnector(string name, string[] supportedLanguages, string[] baseUris, string iconUrl)
{
[JsonIgnore]
[NotMapped]
internal DownloadClient downloadClient { get; init; } = null!;
[JsonIgnore]
[NotMapped]
protected ILog Log { get; init; } = LogManager.GetLogger(name);
[StringLength(32)]
[Required]
public string Name { get; init; } = name;
[StringLength(8)]
[Required]
public string[] SupportedLanguages { get; init; } = supportedLanguages;
[StringLength(2048)]
[Required]
public string IconUrl { get; init; } = iconUrl;
[StringLength(256)]
[Required]
public string[] BaseUris { get; init; } = baseUris;
[Required]
public bool Enabled { get; internal set; } = true;
public abstract (Manga, MangaConnectorId<Manga>)[] SearchManga(string mangaSearchName);
public abstract (Manga, MangaConnectorId<Manga>)? GetMangaFromUrl(string url);
public abstract (Manga, MangaConnectorId<Manga>)? GetMangaFromId(string mangaIdOnSite);
public abstract (Chapter, MangaConnectorId<Chapter>)[] GetChapters(MangaConnectorId<Manga> mangaId,
string? language = null);
internal abstract string[] GetChapterImageUrls(MangaConnectorId<Chapter> chapterId);
public bool UrlMatchesConnector(string url) => BaseUris.Any(baseUri => Regex.IsMatch(url, "https?://" + baseUri + "/.*"));
internal string? SaveCoverImageToCache(MangaConnectorId<Manga> mangaId, int retries = 3)
{
if(retries < 0)
return null;
Regex urlRex = new (@"https?:\/\/((?:[a-zA-Z0-9-]+\.)+[a-zA-Z0-9]+)\/(?:.+\/)*(.+\.([a-zA-Z]+))");
//https?:\/\/[a-zA-Z0-9-]+\.([a-zA-Z0-9-]+\.[a-zA-Z0-9]+)\/(?:.+\/)*(.+\.([a-zA-Z]+)) for only second level domains
Match match = urlRex.Match(mangaId.Obj.CoverUrl);
string filename = $"{match.Groups[1].Value}-{mangaId.Key}.{match.Groups[3].Value}";
string saveImagePath = Path.Join(TrangaSettings.coverImageCache, filename);
if (File.Exists(saveImagePath))
return saveImagePath;
RequestResult coverResult = downloadClient.MakeRequest(mangaId.Obj.CoverUrl, RequestType.MangaCover, $"https://{match.Groups[1].Value}");
if ((int)coverResult.statusCode < 200 || (int)coverResult.statusCode >= 300)
return SaveCoverImageToCache(mangaId, --retries);
using MemoryStream ms = new();
coverResult.result.CopyTo(ms);
Directory.CreateDirectory(TrangaSettings.coverImageCache);
File.WriteAllBytes(saveImagePath, ms.ToArray());
return saveImagePath;
}
}

338
API/Schema/MangaContext/MangaConnectors/MangaDex.cs
View File

@@ -1,338 +0,0 @@
using System.Text.RegularExpressions;
using API.MangaDownloadClients;
using Newtonsoft.Json.Linq;
namespace API.Schema.MangaContext.MangaConnectors;
public class MangaDex : MangaConnector
{
//https://api.mangadex.org/docs/3-enumerations/#language-codes--localization
//https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes
//https://gist.github.com/Josantonius/b455e315bc7f790d14b136d61d9ae469
public MangaDex() : base("MangaDex",
["en","pt","pt-br","it","de","ru","aa","ab","ae","af","ak","am","an","ar-ae","ar-bh","ar-dz","ar-eg","ar-iq","ar-jo","ar-kw","ar-lb","ar-ly","ar-ma","ar-om","ar-qa","ar-sa","ar-sy","ar-tn","ar-ye","ar","as","av","ay","az","ba","be","bg","bh","bi","bm","bn","bo","br","bs","ca","ce","ch","co","cr","cs","cu","cv","cy","da","de-at","de-ch","de-de","de-li","de-lu","div","dv","dz","ee","el","en-au","en-bz","en-ca","en-cb","en-gb","en-ie","en-jm","en-nz","en-ph","en-tt","en-us","en-za","en-zw","eo","es-ar","es-bo","es-cl","es-co","es-cr","es-do","es-ec","es-es","es-gt","es-hn","es-la","es-mx","es-ni","es-pa","es-pe","es-pr","es-py","es-sv","es-us","es-uy","es-ve","es","et","eu","fa","ff","fi","fj","fo","fr-be","fr-ca","fr-ch","fr-fr","fr-lu","fr-mc","fr","fy","ga","gd","gl","gn","gu","gv","ha","he","hi","ho","hr-ba","hr-hr","hr","ht","hu","hy","hz","ia","id","ie","ig","ii","ik","in","io","is","it-ch","it-it","iu","iw","ja","ja-ro","ji","jv","jw","ka","kg","ki","kj","kk","kl","km","kn","ko","ko-ro","kr","ks","ku","kv","kw","ky","kz","la","lb","lg","li","ln","lo","ls","lt","lu","lv","mg","mh","mi","mk","ml","mn","mo","mr","ms-bn","ms-my","ms","mt","my","na","nb","nd","ne","ng","nl-be","nl-nl","nl","nn","no","nr","ns","nv","ny","oc","oj","om","or","os","pa","pi","pl","ps","pt-pt","qu-bo","qu-ec","qu-pe","qu","rm","rn","ro","rw","sa","sb","sc","sd","se-fi","se-no","se-se","se","sg","sh","si","sk","sl","sm","sn","so","sq","sr-ba","sr-sp","sr","ss","st","su","sv-fi","sv-se","sv","sw","sx","syr","ta","te","tg","th","ti","tk","tl","tn","to","tr","ts","tt","tw","ty","ug","uk","ur","us","uz","ve","vi","vo","wa","wo","xh","yi","yo","za","zh-cn","zh-hk","zh-mo","zh-ro","zh-sg","zh-tw","zh","zu"],
["mangadex.org"],
"https://mangadex.org/favicon.ico")
{
this.downloadClient = new HttpDownloadClient();
}
private const int Limit = 100;
public override (Manga, MangaConnectorId<Manga>)[] SearchManga(string mangaSearchName)
{
Log.Info($"Searching Obj: {mangaSearchName}");
List<(Manga, MangaConnectorId<Manga>)> mangas = new ();
int offset = 0;
int total = int.MaxValue;
while(offset < total)
{
string requestUrl =
$"https://api.mangadex.org/manga?limit={Limit}&offset={offset}&title={mangaSearchName}" +
$"&contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica" +
$"&includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author&includes%5B%5D=artist&includes%5B%5D=tag'";
offset += Limit;
RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed);
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
{
Log.Error("Request failed");
return [];
}
using StreamReader sr = new (result.result);
JObject jObject = JObject.Parse(sr.ReadToEnd());
if (jObject.Value<string>("result") != "ok")
{
JArray? errors = jObject["errors"] as JArray;
Log.Error($"Request failed: {string.Join(',', errors?.Select(e => e.Value<string>("title")) ?? [])}");
return [];
}
total = jObject.Value<int>("total");
JArray? data = jObject.Value<JArray>("data");
if (data is null)
{
Log.Error("Data was null");
return [];
}
mangas.AddRange(data.Select(ParseMangaFromJToken));
}
Log.Info($"Search {mangaSearchName} yielded {mangas.Count} results.");
return mangas.ToArray();
}
private static readonly Regex GetMangaIdFromUrl = new(@"https?:\/\/mangadex\.org\/title\/([a-z0-9-]+)\/?.*");
public override (Manga, MangaConnectorId<Manga>)? GetMangaFromUrl(string url)
{
Log.Info($"Getting Obj: {url}");
if (!UrlMatchesConnector(url))
{
Log.Debug($"Url is not for Connector. {url}");
return null;
}
Match match = GetMangaIdFromUrl.Match(url);
if (!match.Success || !match.Groups[1].Success)
{
Log.Debug($"Url is not for Connector (Could not retrieve id). {url}");
return null;
}
string id = match.Groups[1].Value;
return GetMangaFromId(id);
}
public override (Manga, MangaConnectorId<Manga>)? GetMangaFromId(string mangaIdOnSite)
{
Log.Info($"Getting Obj: {mangaIdOnSite}");
string requestUrl =
$"https://api.mangadex.org/manga/{mangaIdOnSite}" +
$"?includes%5B%5D=manga&includes%5B%5D=cover_art&includes%5B%5D=author&includes%5B%5D=artist&includes%5B%5D=tag'";
RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed);
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
{
Log.Error("Request failed");
return null;
}
using StreamReader sr = new (result.result);
JObject jObject = JObject.Parse(sr.ReadToEnd());
if (jObject.Value<string>("result") != "ok")
{
JArray? errors = jObject["errors"] as JArray;
Log.Error($"Request failed: {string.Join(',', errors?.Select(e => e.Value<string>("title")) ?? [])}");
return null;
}
JObject? data = jObject["data"] as JObject;
if (data is null)
{
Log.Error("Data was null");
return null;
}
return ParseMangaFromJToken(data);
}
public override (Chapter, MangaConnectorId<Chapter>)[] GetChapters(MangaConnectorId<Manga> manga, string? language = null)
{
Log.Info($"Getting Chapters: {manga.IdOnConnectorSite}");
List<(Chapter, MangaConnectorId<Chapter>)> chapters = new ();
int offset = 0;
int total = int.MaxValue;
while(offset < total)
{
string requestUrl =
$"https://api.mangadex.org/manga/{manga.IdOnConnectorSite}/feed?limit={Limit}&offset={offset}&" +
$"translatedLanguage%5B%5D={language}&" +
$"contentRating%5B%5D=safe&contentRating%5B%5D=suggestive&contentRating%5B%5D=erotica&includeFutureUpdates=0&includes%5B%5D=";
offset += Limit;
RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.MangaDexFeed);
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
{
Log.Error("Request failed");
return [];
}
using StreamReader sr = new (result.result);
JObject jObject = JObject.Parse(sr.ReadToEnd());
if (jObject.Value<string>("result") != "ok")
{
JArray? errors = jObject["errors"] as JArray;
Log.Error($"Request failed: {string.Join(',', errors?.Select(e => e.Value<string>("title")) ?? [])}");
return [];
}
total = jObject.Value<int>("total");
JArray? data = jObject.Value<JArray>("data");
if (data is null)
{
Log.Error("Data was null");
return [];
}
chapters.AddRange(data.Select(d => ParseChapterFromJToken(manga, d)));
}
Log.Info($"Request for chapters for {manga.Obj.Name} yielded {chapters.Count} results.");
return chapters.ToArray();
}
private static readonly Regex GetChapterIdFromUrl = new(@"https?:\/\/mangadex\.org\/chapter\/([a-z0-9-]+)\/?.*");
internal override string[] GetChapterImageUrls(MangaConnectorId<Chapter> chapterId)
{
Log.Info($"Getting Chapter Image-Urls: {chapterId.Obj}");
if (chapterId.WebsiteUrl is null || !UrlMatchesConnector(chapterId.WebsiteUrl))
{
Log.Debug($"Url is not for Connector. {chapterId.WebsiteUrl}");
return [];
}
Match match = GetChapterIdFromUrl.Match(chapterId.WebsiteUrl);
if (!match.Success || !match.Groups[1].Success)
{
Log.Debug($"Url is not for Connector (Could not retrieve id). {chapterId.WebsiteUrl}");
return [];
}
string id = match.Groups[1].Value;
string requestUrl = $"https://api.mangadex.org/at-home/server/{id}";
RequestResult result = downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300)
{
Log.Error("Request failed");
return [];
}
using StreamReader sr = new (result.result);
JObject jObject = JObject.Parse(sr.ReadToEnd());
if (jObject.Value<string>("result") != "ok")
{
JArray? errors = jObject["errors"] as JArray;
Log.Error($"Request failed: {string.Join(',', errors?.Select(e => e.Value<string>("title")) ?? [])}");
return [];
}
string? baseUrl = jObject.Value<string>("baseUrl");
JToken? chapterToken = jObject["chapter"];
string? hash = chapterToken?.Value<string>("hash");
JArray? data = chapterToken?["data"] as JArray;
if (baseUrl is null || hash is null || data is null)
{
Log.Error("Data was null");
return [];
}
IEnumerable<string> urls = data.Select(t => $"{baseUrl}/data/{hash}/{t.Value<string>()}");
return urls.ToArray();
}
private (Manga manga, MangaConnectorId<Manga> id) ParseMangaFromJToken(JToken jToken)
{
string? id = jToken.Value<string>("id");
if(id is null)
throw new Exception("jToken was not in expected format");
JObject? attributes = jToken["attributes"] as JObject;
if(attributes is null)
throw new Exception("jToken was not in expected format");
string? name = attributes["title"]?.Value<string>("en") ?? attributes["title"]?.First?.First?.Value<string>();
string description = attributes["description"]?.Value<string>("en")??attributes["description"]?.First?.First?.Value<string>()??"";
string? status = attributes["status"]?.Value<string>();
uint? year = attributes["year"]?.Value<uint?>();
string? originalLanguage = attributes["originalLanguage"]?.Value<string>();
JArray? altTitlesJArray = attributes.TryGetValue("altTitles", out JToken? altTitlesArray) ? altTitlesArray as JArray : null;
JArray? tagsJArray = attributes.TryGetValue("tags", out JToken? tagsArray) ? tagsArray as JArray : null;
JArray? relationships = jToken["relationships"] as JArray;
if (name is null || status is null || relationships is null)
throw new Exception("jToken was not in expected format");
string? coverFileName = relationships.FirstOrDefault(r => r["type"]?.Value<string>() == "cover_art")?["attributes"]?.Value<string>("fileName");
if(coverFileName is null)
throw new Exception("jToken was not in expected format");
List<Link> links = attributes["links"]?
.ToObject<Dictionary<string,string>>()?
.Select(kv =>
{
//https://api.mangadex.org/docs/3-enumerations/#manga-links-data
string url = kv.Key switch
{
"al" => $"https://anilist.co/manga/{kv.Value}",
"ap" => $"https://www.anime-planet.com/manga/{kv.Value}",
"bw" => $"https://bookwalker.jp/{kv.Value}",
"mu" => $"https://www.mangaupdates.com/series.html?id={kv.Value}",
"nu" => $"https://www.novelupdates.com/series/{kv.Value}",
"mal" => $"https://myanimelist.net/manga/{kv.Value}",
_ => kv.Value
};
string key = kv.Key switch
{
"al" => "AniList",
"ap" => "Anime Planet",
"bw" => "BookWalker",
"mu" => "Obj Updates",
"nu" => "Novel Updates",
"kt" => "Kitsu.io",
"amz" => "Amazon",
"ebj" => "eBookJapan",
"mal" => "MyAnimeList",
"cdj" => "CDJapan",
_ => kv.Key
};
return new Link(key, url);
}).ToList()!;
List<AltTitle> altTitles = (altTitlesJArray??[])
.Select(t =>
{
JObject? j = t as JObject;
JProperty? p = j?.Properties().First();
if (p is null)
return null;
return new AltTitle(p.Name, p.Value.ToString());
}).Where(x => x is not null).ToList()!;
List<MangaTag> tags = (tagsJArray??[])
.Where(t => t.Value<string>("type") == "tag")
.Select(t => t["attributes"]?["name"]?.Value<string>("en")??t["attributes"]?["name"]?.First?.First?.Value<string>())
.Select(str => str is not null ? new MangaTag(str) : null)
.Where(x => x is not null).ToList()!;
List<Author> authors = relationships
.Where(r => r["type"]?.Value<string>() == "author")
.Select(t => t["attributes"]?.Value<string>("name"))
.Select(str => str is not null ? new Author(str) : null)
.Where(x => x is not null).ToList()!;
MangaReleaseStatus releaseStatus = status switch
{
"completed" => MangaReleaseStatus.Completed,
"ongoing" => MangaReleaseStatus.Continuing,
"cancelled" => MangaReleaseStatus.Cancelled,
"hiatus" => MangaReleaseStatus.OnHiatus,
_ => MangaReleaseStatus.Unreleased
};
string websiteUrl = $"https://mangadex.org/title/{id}";
string coverUrl = $"https://uploads.mangadex.org/covers/{id}/{coverFileName}";
Manga manga = new Manga(name, description, coverUrl, releaseStatus, authors, tags, links,altTitles,
null, 0f, year, originalLanguage);
return (manga, new MangaConnectorId<Manga>(manga, this, id, websiteUrl));
}
private (Chapter chapter, MangaConnectorId<Chapter> id) ParseChapterFromJToken(MangaConnectorId<Manga> mcIdManga, JToken jToken)
{
string? id = jToken.Value<string>("id");
JToken? attributes = jToken["attributes"];
string? chapterStr = attributes?.Value<string>("chapter");
string? volumeStr = attributes?.Value<string>("volume");
int? volumeNumber = null;
string? title = attributes?.Value<string>("title");
if(id is null || chapterStr is null)
throw new Exception("jToken was not in expected format");
if(volumeStr is not null)
volumeNumber = int.Parse(volumeStr);
string websiteUrl = $"https://mangadex.org/chapter/{id}";
Chapter chapter = new (mcIdManga.Obj, chapterStr, volumeNumber, title);
return (chapter, new MangaConnectorId<Chapter>(chapter, this, id, websiteUrl));
}
}

31
API/Schema/MangaContext/MangaContext.cs
View File

@@ -1,4 +1,4 @@
using API.Schema.MangaContext.MangaConnectors;
using API.MangaConnectors;
using API.Schema.MangaContext.MetadataFetchers;
using Microsoft.EntityFrameworkCore;
@@ -6,7 +6,6 @@ namespace API.Schema.MangaContext;
public class MangaContext(DbContextOptions<MangaContext> options) : TrangaBaseContext<MangaContext>(options)
{
public DbSet<MangaConnector> MangaConnectors { get; set; }
public DbSet<Manga> Mangas { get; set; }
public DbSet<FileLibrary> FileLibraries { get; set; }
public DbSet<Chapter> Chapters { get; set; }
@@ -31,26 +30,12 @@ public class MangaContext(DbContextOptions<MangaContext> options) : TrangaBaseCo
.WithOne(c => c.ParentManga)
.HasForeignKey(c => c.ParentMangaId)
.OnDelete(DeleteBehavior.Cascade);
modelBuilder.Entity<Manga>()
.Navigation(m => m.Chapters)
.EnableLazyLoading();
modelBuilder.Entity<Chapter>()
.Navigation(c => c.ParentManga)
.EnableLazyLoading();
//Chapter has MangaConnectorIds
modelBuilder.Entity<Chapter>()
.HasMany<MangaConnectorId<Chapter>>(c => c.MangaConnectorIds)
.WithOne(id => id.Obj)
.HasForeignKey(id => id.ObjId)
.OnDelete(DeleteBehavior.NoAction);
modelBuilder.Entity<MangaConnectorId<Chapter>>()
.HasOne<MangaConnector>(id => id.MangaConnector)
.WithMany()
.HasForeignKey(id => id.MangaConnectorName)
.OnDelete(DeleteBehavior.Cascade);
modelBuilder.Entity<MangaConnectorId<Chapter>>()
.Navigation(entry => entry.MangaConnector)
.EnableLazyLoading();
//Manga owns MangaAltTitles
modelBuilder.Entity<Manga>()
.OwnsMany<AltTitle>(m => m.AltTitles)
@@ -95,17 +80,6 @@ public class MangaContext(DbContextOptions<MangaContext> options) : TrangaBaseCo
.WithOne(id => id.Obj)
.HasForeignKey(id => id.ObjId)
.OnDelete(DeleteBehavior.Cascade);
modelBuilder.Entity<Manga>()
.Navigation(m => m.MangaConnectorIds)
.EnableLazyLoading();
modelBuilder.Entity<MangaConnectorId<Manga>>()
.HasOne<MangaConnector>(id => id.MangaConnector)
.WithMany()
.HasForeignKey(id => id.MangaConnectorName)
.OnDelete(DeleteBehavior.Cascade);
modelBuilder.Entity<MangaConnectorId<Manga>>()
.Navigation(entry => entry.MangaConnector)
.EnableLazyLoading();
//FileLibrary has many Mangas
@@ -114,9 +88,6 @@ public class MangaContext(DbContextOptions<MangaContext> options) : TrangaBaseCo
.WithOne(m => m.Library)
.HasForeignKey(m => m.LibraryId)
.OnDelete(DeleteBehavior.SetNull);
modelBuilder.Entity<Manga>()
.Navigation(m => m.Library)
.EnableLazyLoading();
modelBuilder.Entity<MetadataFetcher>()
.HasDiscriminator<string>(nameof(MetadataEntry))