glax/Tranga
2
0
Fork 0
Code

Merge branch 'cuttingedge'

Browse Source
This commit is contained in:
glax 2023-09-26 18:28:15 +02:00
commit 8c554076b2
18 changed files with 178 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Dockerfile
View File

@ -10,7 +10,20 @@ RUN dotnet restore /src/Tranga/Tranga.csproj
RUN dotnet publish -c Release -o /publish
FROM glax/tranga-base:latest as runtime
EXPOSE 6531
ARG UNAME=tranga
ARG UID=1000
ARG GID=1000
RUN groupadd -g $GID -o $UNAME
RUN useradd -m -u $UID -g $GID -o -s /bin/bash $UNAME
RUN mkdir /usr/share/tranga-api
RUN mkdir /Manga
RUN chown 1000:1000 /usr/share/tranga-api
RUN chown 1000:1000 /Manga
USER $UNAME
WORKDIR /publish
COPY --from=build-env /publish .
EXPOSE 6531
USER 0
RUN chown 1000:1000 /publish
ENTRYPOINT ["dotnet", "/publish/Tranga.dll", "-c"]

16
Tranga/GlobalBase.cs
View File

@ -1,4 +1,5 @@
using System.Globalization;
using System.Text.RegularExpressions;
using Logging;
using Newtonsoft.Json;
using Tranga.LibraryConnectors;
@ -14,6 +15,7 @@ public abstract class GlobalBase
protected HashSet<LibraryConnector> libraryConnectors { get; init; }
protected List<Manga> cachedPublications { get; init; }
protected static readonly NumberFormatInfo numberFormatDecimalPoint = new (){ NumberDecimalSeparator = "." };
protected static readonly Regex baseUrlRex = new(@"https?:\/\/[0-9A-z\.-]*");
protected GlobalBase(GlobalBase clone)
{
@ -52,11 +54,12 @@ public abstract class GlobalBase
protected void AddNotificationConnector(NotificationConnector notificationConnector)
{
Log($"Adding {notificationConnector}");
notificationConnectors.RemoveWhere(nc => nc.GetType() == notificationConnector.GetType());
notificationConnectors.RemoveWhere(nc => nc.notificationConnectorType == notificationConnector.notificationConnectorType);
notificationConnectors.Add(notificationConnector);
while(IsFileInUse(settings.notificationConnectorsFilePath))
Thread.Sleep(100);
Log("Exporting notificationConnectors");
File.WriteAllText(settings.notificationConnectorsFilePath, JsonConvert.SerializeObject(notificationConnectors));
}
@ -64,6 +67,10 @@ public abstract class GlobalBase
{
Log($"Removing {notificationConnectorType}");
notificationConnectors.RemoveWhere(nc => nc.notificationConnectorType == notificationConnectorType);
while(IsFileInUse(settings.notificationConnectorsFilePath))
Thread.Sleep(100);
Log("Exporting notificationConnectors");
File.WriteAllText(settings.notificationConnectorsFilePath, JsonConvert.SerializeObject(notificationConnectors));
}
protected void UpdateLibraries()
@ -75,11 +82,12 @@ public abstract class GlobalBase
protected void AddLibraryConnector(LibraryConnector libraryConnector)
{
Log($"Adding {libraryConnector}");
libraryConnectors.RemoveWhere(lc => lc.GetType() == libraryConnector.GetType());
libraryConnectors.RemoveWhere(lc => lc.libraryType == libraryConnector.libraryType);
libraryConnectors.Add(libraryConnector);
while(IsFileInUse(settings.libraryConnectorsFilePath))
Thread.Sleep(100);
Log("Exporting libraryConnectors");
File.WriteAllText(settings.libraryConnectorsFilePath, JsonConvert.SerializeObject(libraryConnectors));
}
@ -87,6 +95,10 @@ public abstract class GlobalBase
{
Log($"Removing {libraryType}");
libraryConnectors.RemoveWhere(lc => lc.libraryType == libraryType);
while(IsFileInUse(settings.libraryConnectorsFilePath))
Thread.Sleep(100);
Log("Exporting libraryConnectors");
File.WriteAllText(settings.libraryConnectorsFilePath, JsonConvert.SerializeObject(libraryConnectors));
}
protected bool IsFileInUse(string filePath)

12
Tranga/Jobs/DownloadChapter.cs
View File

@ -1,4 +1,4 @@
using System.Text;
using System.Net;
using Tranga.MangaConnectors;
namespace Tranga.Jobs;
@ -31,9 +31,13 @@ public class DownloadChapter : Job
{
Task downloadTask = new(delegate
{
mangaConnector.DownloadChapter(chapter, this.progressToken);
UpdateLibraries();
SendNotifications("Chapter downloaded", $"{chapter.parentManga.sortName} - {chapter.chapterNumber}");
mangaConnector.CopyCoverFromCacheToDownloadLocation(chapter.parentManga);
HttpStatusCode success = mangaConnector.DownloadChapter(chapter, this.progressToken);
if (success == HttpStatusCode.OK)
{
UpdateLibraries();
SendNotifications("Chapter downloaded", $"{chapter.parentManga.sortName} - {chapter.chapterNumber}");
}
});
downloadTask.Start();
return Array.Empty<Job>();

1
Tranga/Jobs/DownloadNewChapters.cs
View File

@ -36,6 +36,7 @@ public class DownloadNewChapters : Job
Chapter[] chapters = mangaConnector.GetNewChapters(manga, this.translatedLanguage);
this.progressToken.increments = chapters.Length;
List<Job> jobs = new();
mangaConnector.CopyCoverFromCacheToDownloadLocation(manga);
foreach (Chapter chapter in chapters)
{
DownloadChapter downloadChapterJob = new(this, this.mangaConnector, chapter, parentJobId: this.id);

6
Tranga/Jobs/Job.cs
View File

@ -59,14 +59,14 @@ public abstract class Job : GlobalBase
public void ResetProgress()
{
this.progressToken.increments = this.progressToken.increments - this.progressToken.incrementsCompleted;
this.progressToken.increments -= progressToken.incrementsCompleted;
this.lastExecution = DateTime.Now;
this.progressToken.Waiting();
}
public void ExecutionEnqueue()
{
this.progressToken.increments = this.progressToken.increments - this.progressToken.incrementsCompleted;
this.lastExecution = recurrenceTime is not null ? DateTime.Now.Subtract((TimeSpan)recurrenceTime) : DateTime.UnixEpoch;
this.progressToken.increments -= progressToken.incrementsCompleted;
this.progressToken.Standby();
}

64
Tranga/Jobs/JobBoss.cs
View File

@ -14,6 +14,7 @@ public class JobBoss : GlobalBase
this.jobs = new();
LoadJobsList(connectors);
this.mangaConnectorJobQueue = new();
Log($"Next job in {jobs.MinBy(job => job.nextExecution)?.nextExecution.Subtract(DateTime.Now)} {jobs.MinBy(job => job.nextExecution)?.id}");
}
public void AddJob(Job job)
@ -26,7 +27,7 @@ public class JobBoss : GlobalBase
{
Log($"Added {job}");
this.jobs.Add(job);
ExportJobsList();
ExportJob(job);
}
}
@ -54,9 +55,9 @@ public class JobBoss : GlobalBase
Log($"Removing {job}");
job.Cancel();
this.jobs.Remove(job);
if(job.subJobs is not null)
if(job.subJobs is not null && job.subJobs.Any())
RemoveJobs(job.subJobs);
ExportJobsList();
ExportJob(job);
}
public void RemoveJobs(IEnumerable<Job?> jobsToRemove)
@ -161,21 +162,39 @@ public class JobBoss : GlobalBase
cachedPublications.Add(ncJob.manga);
}
public void ExportJob(Job job)
{
string jobFilePath = Path.Join(settings.jobsFolderPath, $"{job.id}.json");
if (!this.jobs.Any(jjob => jjob.id == job.id))
{
try
{
Log($"Deleting Job-file {jobFilePath}");
while(IsFileInUse(jobFilePath))
Thread.Sleep(10);
File.Delete(jobFilePath);
}
catch (Exception e)
{
Log(e.ToString());
}
}
else
{
Log($"Exporting Job {jobFilePath}");
string jobStr = JsonConvert.SerializeObject(job);
while(IsFileInUse(jobFilePath))
Thread.Sleep(10);
File.WriteAllText(jobFilePath, jobStr);
}
}
public void ExportJobsList()
{
Log("Exporting Jobs");
foreach (Job job in this.jobs)
{
string jobFilePath = Path.Join(settings.jobsFolderPath, $"{job.id}.json");
if (!File.Exists(jobFilePath))
{
string jobStr = JsonConvert.SerializeObject(job);
while(IsFileInUse(jobFilePath))
Thread.Sleep(10);
Log($"Exporting Job {jobFilePath}");
File.WriteAllText(jobFilePath, jobStr);
}
}
ExportJob(job);
//Remove files with jobs not in this.jobs-list
Regex idRex = new (@"(.*)\.json");
@ -201,8 +220,7 @@ public class JobBoss : GlobalBase
public void CheckJobs()
{
foreach (Job job in jobs.Where(job => job.nextExecution < DateTime.Now && !QueueContainsJob(job)).OrderBy(job => job.nextExecution))
AddJobToQueue(job);
AddJobsToQueue(jobs.Where(job => job.progressToken.state == ProgressToken.State.Waiting && job.nextExecution < DateTime.Now && !QueueContainsJob(job)).OrderBy(job => job.nextExecution));
foreach (Queue<Job> jobQueue in mangaConnectorJobQueue.Values)
{
if(jobQueue.Count < 1)
@ -210,17 +228,11 @@ public class JobBoss : GlobalBase
Job queueHead = jobQueue.Peek();
if (queueHead.progressToken.state is ProgressToken.State.Complete or ProgressToken.State.Cancelled)
{
switch (queueHead)
{
case DownloadChapter:
RemoveJob(queueHead);
break;
case DownloadNewChapters:
if(queueHead.recurring)
queueHead.progressToken.Complete();
break;
}
queueHead.ResetProgress();
if(!queueHead.recurring)
RemoveJob(queueHead);
jobQueue.Dequeue();
Log($"Next job in {jobs.MinBy(job => job.nextExecution)?.nextExecution.Subtract(DateTime.Now)} {jobs.MinBy(job => job.nextExecution)?.id}");
}else if (queueHead.progressToken.state is ProgressToken.State.Standby)
{
Job[] subJobs = jobQueue.Peek().ExecuteReturnSubTasks().ToArray();

9
Tranga/Jobs/ProgressToken.cs
View File

@ -10,7 +10,7 @@ public class ProgressToken
public DateTime executionStarted { get; private set; }
public TimeSpan timeRemaining => GetTimeRemaining();
public enum State { Running, Complete, Standby, Cancelled }
public enum State { Running, Complete, Standby, Cancelled, Waiting }
public State state { get; private set; }
public ProgressToken(int increments)
@ -18,7 +18,7 @@ public class ProgressToken
this.cancellationRequested = false;
this.increments = increments;
this.incrementsCompleted = 0;
this.state = State.Complete;
this.state = State.Waiting;
this.executionStarted = DateTime.UnixEpoch;
}
@ -63,4 +63,9 @@ public class ProgressToken
{
state = State.Cancelled;
}
public void Waiting()
{
state = State.Waiting;
}
}

5
Tranga/LibraryConnectors/LibraryConnector.cs
View File

@ -20,7 +20,10 @@ public abstract class LibraryConnector : GlobalBase
protected LibraryConnector(GlobalBase clone, string baseUrl, string auth, LibraryType libraryType) : base(clone)
{
this.baseUrl = baseUrl;
Log($"Creating libraryConnector {Enum.GetName(libraryType)}");
if (!baseUrlRex.IsMatch(baseUrl))
throw new ArgumentException("Base url does not match pattern");
this.baseUrl = baseUrlRex.Match(baseUrl).Value;
this.auth = auth;
this.libraryType = libraryType;
}

12
Tranga/MangaConnectors/MangaConnector.cs
View File

@ -162,7 +162,7 @@ public abstract class MangaConnector : GlobalBase
Log($"Cloning cover {fileInCache} -> {newFilePath}");
File.Copy(fileInCache, newFilePath, true);
if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
File.SetUnixFileMode(newFilePath, GroupRead | GroupWrite | OtherRead | OtherWrite | UserRead | UserWrite);
File.SetUnixFileMode(newFilePath, GroupRead | GroupWrite | UserRead | UserWrite);
}
/// <summary>
@ -193,10 +193,14 @@ public abstract class MangaConnector : GlobalBase
//Check if Publication Directory already exists
string directoryPath = Path.GetDirectoryName(saveArchiveFilePath)!;
if (!Directory.Exists(directoryPath))
Directory.CreateDirectory(directoryPath);
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
Directory.CreateDirectory(directoryPath,
UserRead | UserWrite | UserExecute | GroupRead | GroupWrite | GroupExecute );
else
Directory.CreateDirectory(directoryPath);
if (File.Exists(saveArchiveFilePath)) //Don't download twice.
return HttpStatusCode.OK;
return HttpStatusCode.Created;
//Create a temporary folder to store images
string tempFolder = Directory.CreateTempSubdirectory().FullName;
@ -229,7 +233,7 @@ public abstract class MangaConnector : GlobalBase
//ZIP-it and ship-it
ZipFile.CreateFromDirectory(tempFolder, saveArchiveFilePath);
if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
File.SetUnixFileMode(saveArchiveFilePath, GroupRead | GroupWrite | OtherRead | OtherWrite | UserRead | UserWrite);
File.SetUnixFileMode(saveArchiveFilePath, UserRead | UserWrite | UserExecute | GroupRead | GroupWrite | GroupExecute);
Directory.Delete(tempFolder, true); //Cleanup
progressToken?.Complete();

10
Tranga/MangaConnectors/MangaDex.cs
View File

@ -225,17 +225,27 @@ public class MangaDex : MangaConnector
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (progressToken?.cancellationRequested ?? false)
{
progressToken?.Cancel();
return HttpStatusCode.RequestTimeout;
}
Manga chapterParentManga = chapter.parentManga;
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
//Request URLs for Chapter-Images
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
progressToken?.Cancel();
return requestResult.statusCode;
}
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
if (result is null)
{
progressToken?.Cancel();
return HttpStatusCode.NoContent;
}
string baseUrl = result["baseUrl"]!.GetValue<string>();
string hash = result["chapter"]!["hash"]!.GetValue<string>();

7
Tranga/MangaConnectors/MangaKatana.cs
View File

@ -186,7 +186,11 @@ public class MangaKatana : MangaConnector
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (progressToken?.cancellationRequested ?? false)
{
progressToken?.Cancel();
return HttpStatusCode.RequestTimeout;
}
Manga chapterParentManga = chapter.parentManga;
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
string requestUrl = chapter.url;
@ -194,7 +198,10 @@ public class MangaKatana : MangaConnector
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
progressToken?.Cancel();
return requestResult.statusCode;
}
string[] imageUrls = ParseImageUrlsFromHtml(requestUrl);

11
Tranga/MangaConnectors/Manganato.cs
View File

@ -172,17 +172,28 @@ public class Manganato : MangaConnector
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (progressToken?.cancellationRequested ?? false)
{
progressToken?.Cancel();
return HttpStatusCode.RequestTimeout;
}
Manga chapterParentManga = chapter.parentManga;
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
string requestUrl = chapter.url;
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
progressToken?.Cancel();
return requestResult.statusCode;
}
if (requestResult.htmlDocument is null)
{
progressToken?.Cancel();
return HttpStatusCode.InternalServerError;
}
string[] imageUrls = ParseImageUrlsFromHtml(requestResult.htmlDocument);
string comicInfoPath = Path.GetTempFileName();

15
Tranga/MangaConnectors/Mangasee.cs
View File

@ -179,16 +179,27 @@ public class Mangasee : MangaConnector
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (progressToken?.cancellationRequested ?? false)
{
progressToken?.Cancel();
return HttpStatusCode.RequestTimeout;
}
Manga chapterParentManga = chapter.parentManga;
if (progressToken?.cancellationRequested??false)
if (progressToken?.cancellationRequested ?? false)
{
progressToken?.Cancel();
return HttpStatusCode.RequestTimeout;
}
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
DownloadClient.RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, 1);
if(requestResult.htmlDocument is null)
if (requestResult.htmlDocument is null)
{
progressToken?.Cancel();
return HttpStatusCode.RequestTimeout;
}
HtmlDocument document = requestResult.htmlDocument;
HtmlNode gallery = document.DocumentNode.Descendants("div").First(div => div.HasClass("ImageGallery"));

36
Tranga/MangaConnectors/Mangaworld.cs
View File

@ -134,15 +134,30 @@ public class Mangaworld: MangaConnector
{
List<Chapter> ret = new();
foreach (HtmlNode volNode in document.DocumentNode.SelectNodes(
"//div[contains(concat(' ',normalize-space(@class),' '),'chapters-wrapper')]//div[contains(concat(' ',normalize-space(@class),' '),'volume-element')]"))
HtmlNode chaptersWrapper =
document.DocumentNode.SelectSingleNode(
"//div[contains(concat(' ',normalize-space(@class),' '),'chapters-wrapper')]");
if (chaptersWrapper.Descendants("div").Any(descendant => descendant.HasClass("volume-element")))
{
string volume = volNode.SelectNodes("div").First(node => node.HasClass("volume")).SelectSingleNode("p").InnerText.Split(' ')[^1];
foreach (HtmlNode chNode in volNode.SelectNodes("div").First(node => node.HasClass("volume-chapters")).SelectNodes("div"))
foreach (HtmlNode volNode in document.DocumentNode.SelectNodes("//div[contains(concat(' ',normalize-space(@class),' '),'volume-element')]"))
{
string volume = volNode.SelectNodes("div").First(node => node.HasClass("volume")).SelectSingleNode("p").InnerText.Split(' ')[^1];
foreach (HtmlNode chNode in volNode.SelectNodes("div").First(node => node.HasClass("volume-chapters")).SelectNodes("div"))
{
string number = chNode.SelectSingleNode("a").SelectSingleNode("span").InnerText.Split(" ")[^1];
string url = chNode.SelectSingleNode("a").GetAttributeValue("href", "");
ret.Add(new Chapter(manga, null, volume, number, url));
}
}
}
else
{
foreach (HtmlNode chNode in chaptersWrapper.SelectNodes("div").Where(node => node.HasClass("chapter")))
{
string number = chNode.SelectSingleNode("a").SelectSingleNode("span").InnerText.Split(" ")[^1];
string url = chNode.SelectSingleNode("a").GetAttributeValue("href", "");
ret.Add(new Chapter(manga, null, volume, number, url));
ret.Add(new Chapter(manga, null, null, number, url));
}
}
@ -153,17 +168,28 @@ public class Mangaworld: MangaConnector
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
{
if (progressToken?.cancellationRequested ?? false)
{
progressToken?.Cancel();
return HttpStatusCode.RequestTimeout;
}
Manga chapterParentManga = chapter.parentManga;
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
string requestUrl = $"{chapter.url}?style=list";
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
progressToken?.Cancel();
return requestResult.statusCode;
}
if (requestResult.htmlDocument is null)
{
progressToken?.Cancel();
return HttpStatusCode.InternalServerError;
}
string[] imageUrls = ParseImageUrlsFromHtml(requestResult.htmlDocument);
string comicInfoPath = Path.GetTempFileName();

4
Tranga/NotificationConnectors/Gotify.cs
View File

@ -13,7 +13,9 @@ public class Gotify : NotificationConnector
[JsonConstructor]
public Gotify(GlobalBase clone, string endpoint, string appToken) : base(clone, NotificationConnectorType.Gotify)
{
this.endpoint = endpoint;
if (!baseUrlRex.IsMatch(endpoint))
throw new ArgumentException("endpoint does not match pattern");
this.endpoint = baseUrlRex.Match(endpoint).Value;;
this.appToken = appToken;
}

1
Tranga/NotificationConnectors/NotificationConnector.cs
View File

@ -6,6 +6,7 @@ public abstract class NotificationConnector : GlobalBase
protected NotificationConnector(GlobalBase clone, NotificationConnectorType notificationConnectorType) : base(clone)
{
Log($"Creating notificationConnector {Enum.GetName(notificationConnectorType)}");
this.notificationConnectorType = notificationConnectorType;
}

4
Tranga/Server.cs
View File

@ -192,10 +192,10 @@ public class Server : GlobalBase
SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.Where(jjob => jjob.progressToken.state is ProgressToken.State.Running));
break;
case "Jobs/Waiting":
SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.Where(jjob => jjob.progressToken.state is ProgressToken.State.Standby));
SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.Where(jjob => jjob.progressToken.state is ProgressToken.State.Standby).OrderBy(jjob => jjob.nextExecution));
break;
case "Jobs/MonitorJobs":
SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.Where(jjob => jjob is DownloadNewChapters));
SendResponse(HttpStatusCode.OK, response, _parent.jobBoss.jobs.Where(jjob => jjob is DownloadNewChapters).OrderBy(jjob => ((DownloadNewChapters)jjob).manga.sortName));
break;
case "Settings":
SendResponse(HttpStatusCode.OK, response, settings);

7
Tranga/Tranga.cs
View File

@ -27,6 +27,8 @@ public partial class Tranga : GlobalBase
jobBoss = new(this, this._connectors);
StartJobBoss();
this._server = new Server(this);
string[] emojis = { "(•‿•)", "(づ \u25d5‿\u25d5 )づ", "( \u02d8\u25bd\u02d8)っ\u2668", "=\uff3e\u25cf \u22cf \u25cf\uff3e=", "（ΦωΦ）", "(\u272a\u3268\u272a)", "( ﾉ･o･ )ﾉ", "（〜^\u2207^ )〜", "~(\u2267ω\u2266)~","૮ \u00b4• ﻌ \u00b4• ა", "(\u02c3ᆺ\u02c2)", "(=\ud83d\udf66 \u0f1d \ud83d\udf66=)"};
SendNotifications("Tranga Started", emojis[Random.Shared.Next(0,emojis.Length-1)]);
}
public MangaConnector? GetConnector(string name)
@ -70,11 +72,6 @@ public partial class Tranga : GlobalBase
jobBoss.CheckJobs();
Thread.Sleep(100);
}
foreach (MangaConnector connector in _connectors)
{
}
});
t.Start();
}