2
0

Compare commits

...

14 Commits

Author SHA1 Message Date
18edcef1c3 Resolve #247
Modify API call:
`/v2/Connector/Types`
Returns: Dictionary with Connector-Names and supported languages.
2024-09-16 21:25:08 +02:00
73ad881600 Merge branch 'refs/heads/cuttingedge-merge-ServerV2' into Server-V2 2024-09-16 21:19:30 +02:00
5f05ba1049 Make SupportedLanguages public. 2024-09-16 21:19:19 +02:00
c6cfd9eb6c Merge branch 'refs/heads/cuttingedge-merge-ServerV2' into Server-V2
# Conflicts:
#	Tranga/Server.cs
2024-09-16 21:17:32 +02:00
cf5cbba9a8 #247 Add supported languages to Mangaconnectors 2024-09-16 21:17:07 +02:00
600b56033d Upgrade to Dotnet 8.0 LangVer 12 2024-09-16 21:11:50 +02:00
7f3754fb64 Fix startup issue/issue with existing chapters: ProgressToken would not complete 2024-09-16 20:36:40 +02:00
2dac5db4da Create single Chromium Instance that is shared between all Connectors.
Fix pages staying open when page could not be loaded.
2024-09-16 20:30:23 +02:00
35f2625f05 Fix #249 Manhuaplus where author/tags are not set. 2024-09-16 19:52:25 +02:00
96f3dbce65 Throw more readable exceptions if deserialization fails for Mangaconnectors.
#249
2024-09-16 18:32:34 +02:00
895128a462 Merge remote-tracking branch 'origin/cuttingedge-merge-ServerV2' into cuttingedge-merge-ServerV2 2024-09-16 18:24:39 +02:00
9b8b80cd24 Fix response closed on OPTIONS request 2024-09-07 20:44:15 +02:00
15f3e2b8ec Use current time as internalId for Manga instead of BASE64 string of title
#232
Fix #237
2024-09-07 20:33:03 +02:00
2be29e4019 MangaDex only download single release for chapter.
Fix #219
2024-09-07 20:16:05 +02:00
22 changed files with 79 additions and 76 deletions

View File

@ -2,9 +2,10 @@
<PropertyGroup> <PropertyGroup>
<OutputType>Exe</OutputType> <OutputType>Exe</OutputType>
<TargetFramework>net7.0</TargetFramework> <TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<LangVersion>12</LangVersion>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>

View File

@ -1,5 +1,5 @@
# syntax=docker/dockerfile:1 # syntax=docker/dockerfile:1
ARG DOTNET=7.0 ARG DOTNET=8.0
FROM mcr.microsoft.com/dotnet/runtime:$DOTNET AS base FROM mcr.microsoft.com/dotnet/runtime:$DOTNET AS base
WORKDIR /publish WORKDIR /publish

View File

@ -1,9 +1,10 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>net7.0</TargetFramework> <TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<LangVersion>12</LangVersion>
</PropertyGroup> </PropertyGroup>
</Project> </Project>

View File

@ -67,7 +67,7 @@ public struct Manga
while (this.folderName.EndsWith('.')) while (this.folderName.EndsWith('.'))
this.folderName = this.folderName.Substring(0, this.folderName.Length - 1); this.folderName = this.folderName.Substring(0, this.folderName.Length - 1);
string onlyLowerLetters = string.Concat(this.sortName.ToLower().Where(Char.IsLetter)); string onlyLowerLetters = string.Concat(this.sortName.ToLower().Where(Char.IsLetter));
this.internalId = Convert.ToBase64String(Encoding.ASCII.GetBytes($"{onlyLowerLetters}{this.year}")); this.internalId = DateTime.Now.Ticks.ToString();
this.ignoreChaptersBelow = ignoreChaptersBelow ?? 0f; this.ignoreChaptersBelow = ignoreChaptersBelow ?? 0f;
this.latestChapterDownloaded = 0; this.latestChapterDownloaded = 0;
this.latestChapterAvailable = 0; this.latestChapterAvailable = 0;

View File

@ -8,7 +8,7 @@ namespace Tranga.MangaConnectors;
public class Bato : MangaConnector public class Bato : MangaConnector
{ {
public Bato(GlobalBase clone) : base(clone, "Bato") public Bato(GlobalBase clone) : base(clone, "Bato", ["en"])
{ {
this.downloadClient = new HttpDownloadClient(clone); this.downloadClient = new HttpDownloadClient(clone);
} }

View File

@ -8,13 +8,12 @@ namespace Tranga.MangaConnectors;
internal class ChromiumDownloadClient : DownloadClient internal class ChromiumDownloadClient : DownloadClient
{ {
private IBrowser browser { get; set; } private static readonly IBrowser Browser = StartBrowser().Result;
private const int StartTimeoutMs = 30000; private const int StartTimeoutMs = 10000;
private readonly HttpDownloadClient _httpDownloadClient; private readonly HttpDownloadClient _httpDownloadClient;
private async Task<IBrowser> StartBrowser() private static async Task<IBrowser> StartBrowser()
{ {
Log($"Starting Browser. ({StartTimeoutMs}ms timeout)");
return await Puppeteer.LaunchAsync(new LaunchOptions return await Puppeteer.LaunchAsync(new LaunchOptions
{ {
Headless = true, Headless = true,
@ -29,7 +28,6 @@ internal class ChromiumDownloadClient : DownloadClient
public ChromiumDownloadClient(GlobalBase clone) : base(clone) public ChromiumDownloadClient(GlobalBase clone) : base(clone)
{ {
this.browser = StartBrowser().Result;
_httpDownloadClient = new(this); _httpDownloadClient = new(this);
} }
@ -43,7 +41,7 @@ internal class ChromiumDownloadClient : DownloadClient
private RequestResult MakeRequestBrowser(string url, string? referrer = null, string? clickButton = null) private RequestResult MakeRequestBrowser(string url, string? referrer = null, string? clickButton = null)
{ {
IPage page = this.browser.NewPageAsync().Result; IPage page = Browser.NewPageAsync().Result;
page.DefaultTimeout = 10000; page.DefaultTimeout = 10000;
IResponse response; IResponse response;
try try
@ -54,6 +52,7 @@ internal class ChromiumDownloadClient : DownloadClient
catch (Exception e) catch (Exception e)
{ {
Log($"Could not load Page:\n{e.Message}"); Log($"Could not load Page:\n{e.Message}");
page.CloseAsync();
return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null); return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null);
} }
@ -84,9 +83,4 @@ internal class ChromiumDownloadClient : DownloadClient
page.CloseAsync(); page.CloseAsync();
return new RequestResult(response.Status, document, stream, false, ""); return new RequestResult(response.Status, document, stream, false, "");
} }
public override void Close()
{
this.browser.CloseAsync();
}
} }

View File

@ -41,5 +41,4 @@ internal abstract class DownloadClient : GlobalBase
} }
internal abstract RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null); internal abstract RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null);
public abstract void Close();
} }

View File

@ -72,9 +72,4 @@ internal class HttpDownloadClient : DownloadClient
return new RequestResult(response.StatusCode, document, stream); return new RequestResult(response.StatusCode, document, stream);
} }
public override void Close()
{
Log("Closing.");
}
} }

View File

@ -14,15 +14,12 @@ namespace Tranga.MangaConnectors;
public abstract class MangaConnector : GlobalBase public abstract class MangaConnector : GlobalBase
{ {
internal DownloadClient downloadClient { get; init; } = null!; internal DownloadClient downloadClient { get; init; } = null!;
public string[] SupportedLanguages;
public void StopDownloadClient() protected MangaConnector(GlobalBase clone, string name, string[] supportedLanguages) : base(clone)
{
downloadClient.Close();
}
protected MangaConnector(GlobalBase clone, string name) : base(clone)
{ {
this.name = name; this.name = name;
this.SupportedLanguages = supportedLanguages;
Directory.CreateDirectory(TrangaSettings.coverImageCache); Directory.CreateDirectory(TrangaSettings.coverImageCache);
} }
@ -234,7 +231,10 @@ public abstract class MangaConnector : GlobalBase
Directory.CreateDirectory(directoryPath); Directory.CreateDirectory(directoryPath);
if (File.Exists(saveArchiveFilePath)) //Don't download twice. if (File.Exists(saveArchiveFilePath)) //Don't download twice.
{
progressToken?.Complete();
return HttpStatusCode.Created; return HttpStatusCode.Created;
}
//Create a temporary folder to store images //Create a temporary folder to store images
string tempFolder = Directory.CreateTempSubdirectory("trangatemp").FullName; string tempFolder = Directory.CreateTempSubdirectory("trangatemp").FullName;

View File

@ -1,4 +1,6 @@
using Newtonsoft.Json; using System.Data;
using System.Diagnostics;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
namespace Tranga.MangaConnectors; namespace Tranga.MangaConnectors;
@ -22,29 +24,22 @@ public class MangaConnectorJsonConverter : JsonConverter
public override object ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) public override object ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer)
{ {
JObject jo = JObject.Load(reader); JObject jo = JObject.Load(reader);
switch (jo.GetValue("name")!.Value<string>()!) string? connectorName = jo.Value<string>("name");
if (connectorName is null)
throw new ConstraintException("Name can not be null.");
return connectorName switch
{ {
case "MangaDex": "MangaDex" => this._connectors.First(c => c is MangaDex),
return this._connectors.First(c => c is MangaDex); "Manganato" => this._connectors.First(c => c is Manganato),
case "Manganato": "MangaKatana" => this._connectors.First(c => c is MangaKatana),
return this._connectors.First(c => c is Manganato); "Mangasee" => this._connectors.First(c => c is Mangasee),
case "MangaKatana": "Mangaworld" => this._connectors.First(c => c is Mangaworld),
return this._connectors.First(c => c is MangaKatana); "Bato" => this._connectors.First(c => c is Bato),
case "Mangasee": "Manga4Life" => this._connectors.First(c => c is MangaLife),
return this._connectors.First(c => c is Mangasee); "ManhuaPlus" => this._connectors.First(c => c is ManhuaPlus),
case "Mangaworld": "MangaHere" => this._connectors.First(c => c is MangaHere),
return this._connectors.First(c => c is Mangaworld); _ => throw new UnreachableException($"Could not find Connector with name {connectorName}")
case "Bato": };
return this._connectors.First(c => c is Bato);
case "Manga4Life":
return this._connectors.First(c => c is MangaLife);
case "ManhuaPlus":
return this._connectors.First(c => c is ManhuaPlus);
case "MangaHere":
return this._connectors.First(c => c is MangaHere);
}
throw new Exception();
} }
public override bool CanWrite => false; public override bool CanWrite => false;

View File

@ -7,14 +7,17 @@ using JsonSerializer = System.Text.Json.JsonSerializer;
namespace Tranga.MangaConnectors; namespace Tranga.MangaConnectors;
public class MangaDex : MangaConnector public class MangaDex : MangaConnector
{ {
public MangaDex(GlobalBase clone) : base(clone, "MangaDex") //https://api.mangadex.org/docs/3-enumerations/#language-codes--localization
//https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes
//https://gist.github.com/Josantonius/b455e315bc7f790d14b136d61d9ae469
public MangaDex(GlobalBase clone) : base(clone, "MangaDex", ["en","pt","pt-br","it","de","ru","aa","ab","ae","af","ak","am","an","ar-ae","ar-bh","ar-dz","ar-eg","ar-iq","ar-jo","ar-kw","ar-lb","ar-ly","ar-ma","ar-om","ar-qa","ar-sa","ar-sy","ar-tn","ar-ye","ar","as","av","ay","az","ba","be","bg","bh","bi","bm","bn","bo","br","bs","ca","ce","ch","co","cr","cs","cu","cv","cy","da","de-at","de-ch","de-de","de-li","de-lu","div","dv","dz","ee","el","en-au","en-bz","en-ca","en-cb","en-gb","en-ie","en-jm","en-nz","en-ph","en-tt","en-us","en-za","en-zw","eo","es-ar","es-bo","es-cl","es-co","es-cr","es-do","es-ec","es-es","es-gt","es-hn","es-la","es-mx","es-ni","es-pa","es-pe","es-pr","es-py","es-sv","es-us","es-uy","es-ve","es","et","eu","fa","ff","fi","fj","fo","fr-be","fr-ca","fr-ch","fr-fr","fr-lu","fr-mc","fr","fy","ga","gd","gl","gn","gu","gv","ha","he","hi","ho","hr-ba","hr-hr","hr","ht","hu","hy","hz","ia","id","ie","ig","ii","ik","in","io","is","it-ch","it-it","iu","iw","ja","ja-ro","ji","jv","jw","ka","kg","ki","kj","kk","kl","km","kn","ko","ko-ro","kr","ks","ku","kv","kw","ky","kz","la","lb","lg","li","ln","lo","ls","lt","lu","lv","mg","mh","mi","mk","ml","mn","mo","mr","ms-bn","ms-my","ms","mt","my","na","nb","nd","ne","ng","nl-be","nl-nl","nl","nn","no","nr","ns","nv","ny","oc","oj","om","or","os","pa","pi","pl","ps","pt-pt","qu-bo","qu-ec","qu-pe","qu","rm","rn","ro","rw","sa","sb","sc","sd","se-fi","se-no","se-se","se","sg","sh","si","sk","sl","sm","sn","so","sq","sr-ba","sr-sp","sr","ss","st","su","sv-fi","sv-se","sv","sw","sx","syr","ta","te","tg","th","ti","tk","tl","tn","to","tr","ts","tt","tw","ty","ug","uk","ur","us","uz","ve","vi","vo","wa","wo","xh","yi","yo","za","zh-cn","zh-hk","zh-mo","zh-ro","zh-sg","zh-tw","zh","zu"])
{ {
this.downloadClient = new HttpDownloadClient(clone); this.downloadClient = new HttpDownloadClient(clone);
} }
public override Manga[] GetManga(string publicationTitle = "") public override Manga[] GetManga(string publicationTitle = "")
{ {
Log($"Searching Publications. Term=\"{publicationTitle}\""); Log($"Searching Publications. Term={publicationTitle}");
const int limit = 100; //How many values we want returned at once const int limit = 100; //How many values we want returned at once
int offset = 0; //"Page" int offset = 0; //"Page"
int total = int.MaxValue; //How many total results are there, is updated on first request int total = int.MaxValue; //How many total results are there, is updated on first request
@ -54,7 +57,7 @@ public class MangaDex : MangaConnector
if(MangaFromJsonObject(mangaNode.AsObject()) is { } manga) if(MangaFromJsonObject(mangaNode.AsObject()) is { } manga)
retManga.Add(manga); //Add Publication (Manga) to result retManga.Add(manga); //Add Publication (Manga) to result
} }
Log($"Retrieved {retManga.Count} publications. Term=\"{publicationTitle}\""); Log($"Retrieved {retManga.Count} publications. Term={publicationTitle}");
return retManga.ToArray(); return retManga.ToArray();
} }
@ -244,7 +247,7 @@ public class MangaDex : MangaConnector
continue; continue;
} }
if(chapterNum is not "null") if(chapterNum is not "null" && !chapters.Any(chp => chp.volumeNumber.Equals(volume) && chp.chapterNumber.Equals(chapterNum)))
chapters.Add(new Chapter(manga, title, volume, chapterNum, chapterId)); chapters.Add(new Chapter(manga, title, volume, chapterNum, chapterId));
} }
} }

View File

@ -7,7 +7,7 @@ namespace Tranga.MangaConnectors;
public class MangaHere : MangaConnector public class MangaHere : MangaConnector
{ {
public MangaHere(GlobalBase clone) : base(clone, "MangaHere") public MangaHere(GlobalBase clone) : base(clone, "MangaHere", ["en"])
{ {
this.downloadClient = new ChromiumDownloadClient(clone); this.downloadClient = new ChromiumDownloadClient(clone);
} }

View File

@ -7,7 +7,7 @@ namespace Tranga.MangaConnectors;
public class MangaKatana : MangaConnector public class MangaKatana : MangaConnector
{ {
public MangaKatana(GlobalBase clone) : base(clone, "MangaKatana") public MangaKatana(GlobalBase clone) : base(clone, "MangaKatana", ["en"])
{ {
this.downloadClient = new HttpDownloadClient(clone); this.downloadClient = new HttpDownloadClient(clone);
} }

View File

@ -7,7 +7,7 @@ namespace Tranga.MangaConnectors;
public class MangaLife : MangaConnector public class MangaLife : MangaConnector
{ {
public MangaLife(GlobalBase clone) : base(clone, "Manga4Life") public MangaLife(GlobalBase clone) : base(clone, "Manga4Life", ["en"])
{ {
this.downloadClient = new ChromiumDownloadClient(clone); this.downloadClient = new ChromiumDownloadClient(clone);
} }

View File

@ -8,7 +8,7 @@ namespace Tranga.MangaConnectors;
public class Manganato : MangaConnector public class Manganato : MangaConnector
{ {
public Manganato(GlobalBase clone) : base(clone, "Manganato") public Manganato(GlobalBase clone) : base(clone, "Manganato", ["en"])
{ {
this.downloadClient = new HttpDownloadClient(clone); this.downloadClient = new HttpDownloadClient(clone);
} }

View File

@ -11,7 +11,7 @@ namespace Tranga.MangaConnectors;
public class Mangasee : MangaConnector public class Mangasee : MangaConnector
{ {
public Mangasee(GlobalBase clone) : base(clone, "Mangasee") public Mangasee(GlobalBase clone) : base(clone, "Mangasee", ["en"])
{ {
this.downloadClient = new ChromiumDownloadClient(clone); this.downloadClient = new ChromiumDownloadClient(clone);
} }

View File

@ -7,7 +7,7 @@ namespace Tranga.MangaConnectors;
public class Mangaworld: MangaConnector public class Mangaworld: MangaConnector
{ {
public Mangaworld(GlobalBase clone) : base(clone, "Mangaworld") public Mangaworld(GlobalBase clone) : base(clone, "Mangaworld", ["it"])
{ {
this.downloadClient = new HttpDownloadClient(clone); this.downloadClient = new HttpDownloadClient(clone);
} }

View File

@ -7,7 +7,7 @@ namespace Tranga.MangaConnectors;
public class ManhuaPlus : MangaConnector public class ManhuaPlus : MangaConnector
{ {
public ManhuaPlus(GlobalBase clone) : base(clone, "ManhuaPlus") public ManhuaPlus(GlobalBase clone) : base(clone, "ManhuaPlus", ["en"])
{ {
this.downloadClient = new ChromiumDownloadClient(clone); this.downloadClient = new ChromiumDownloadClient(clone);
} }
@ -82,17 +82,31 @@ public class ManhuaPlus : MangaConnector
HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//h1"); HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//h1");
string sortName = titleNode.InnerText.Replace("\n", ""); string sortName = titleNode.InnerText.Replace("\n", "");
List<string> authors = new();
try
{
HtmlNode[] authorsNodes = document.DocumentNode HtmlNode[] authorsNodes = document.DocumentNode
.SelectNodes("//a[contains(@href, 'https://manhuaplus.org/authors/')]") .SelectNodes("//a[contains(@href, 'https://manhuaplus.org/authors/')]")
.ToArray(); .ToArray();
List<string> authors = new();
foreach (HtmlNode authorNode in authorsNodes) foreach (HtmlNode authorNode in authorsNodes)
authors.Add(authorNode.InnerText); authors.Add(authorNode.InnerText);
}
catch (ArgumentNullException e)
{
Log("No authors found.");
}
try
{
HtmlNode[] genreNodes = document.DocumentNode HtmlNode[] genreNodes = document.DocumentNode
.SelectNodes("//a[contains(@href, 'https://manhuaplus.org/genres/')]").ToArray(); .SelectNodes("//a[contains(@href, 'https://manhuaplus.org/genres/')]").ToArray();
foreach (HtmlNode genreNode in genreNodes) foreach (HtmlNode genreNode in genreNodes)
tags.Add(genreNode.InnerText.Replace("\n", "")); tags.Add(genreNode.InnerText.Replace("\n", ""));
}
catch (ArgumentNullException e)
{
Log("No genres found");
}
string yearNodeStr = document.DocumentNode string yearNodeStr = document.DocumentNode
.SelectSingleNode("//aside//i[contains(concat(' ',normalize-space(@class),' '),' fa-clock ')]/../span").InnerText.Replace("\n", ""); .SelectSingleNode("//aside//i[contains(concat(' ',normalize-space(@class),' '),' fa-clock ')]/../span").InnerText.Replace("\n", "");

View File

@ -14,7 +14,7 @@ public partial class Server
private ValueTuple<HttpStatusCode, object?> GetV2ConnectorConnectorNameGetManga(GroupCollection groups, Dictionary<string, string> requestParameters) private ValueTuple<HttpStatusCode, object?> GetV2ConnectorConnectorNameGetManga(GroupCollection groups, Dictionary<string, string> requestParameters)
{ {
if(groups.Count < 1 || if(groups.Count < 1 ||
!_parent.GetConnectors().Contains(groups[1].Value) || !_parent.GetConnectors().Keys.Contains(groups[1].Value) ||
!_parent.TryGetConnector(groups[1].Value, out MangaConnector? connector) || !_parent.TryGetConnector(groups[1].Value, out MangaConnector? connector) ||
connector is null) connector is null)
return new ValueTuple<HttpStatusCode, object?>(HttpStatusCode.BadRequest, $"Connector '{groups[1].Value}' does not exist."); return new ValueTuple<HttpStatusCode, object?>(HttpStatusCode.BadRequest, $"Connector '{groups[1].Value}' does not exist.");

View File

@ -49,9 +49,9 @@ public partial class Tranga : GlobalBase
return connector is not null; return connector is not null;
} }
public IEnumerable<string> GetConnectors() public Dictionary<string, string[]> GetConnectors()
{ {
return _connectors.Select(c => c.name); return _connectors.ToDictionary(c => c.name, c => c.SupportedLanguages);
} }
public Manga? GetPublicationById(string internalId) => GetCachedManga(internalId); public Manga? GetPublicationById(string internalId) => GetCachedManga(internalId);

View File

@ -1,10 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>net7.0</TargetFramework> <TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<OutputType>Exe</OutputType> <OutputType>Exe</OutputType>
<LangVersion>12</LangVersion>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>

View File

@ -34,7 +34,7 @@ Returns available Manga Connectors (Scanlation sites)
<details> <details>
<summary>Returns</summary> <summary>Returns</summary>
List of strings with Names. Dictionary with Connector-Names and supported languages.
</details> </details>
### <sub>![GET](https://img.shields.io/badge/GET-0f0)</sub> `/v2/Connector/<ConnectorName>/GetManga` ### <sub>![GET](https://img.shields.io/badge/GET-0f0)</sub> `/v2/Connector/<ConnectorName>/GetManga`