17 Commits

Author SHA1 Message Date
9fd8bf1741 website uses taskId 2023-06-10 16:00:41 +02:00
d5c9c5ba96 Redid progress calcuation on DownloadNewChaptersTask and DownloadChapterTask 2023-06-10 16:00:16 +02:00
c8e27921ab Added taskId to trangaTask and parentTaskId to DownloadChapterTask as unique identifier to attach ChildTasks to ParentTask on deserialization. 2023-06-10 15:59:42 +02:00
6eaba07801 Changed progress type from float to double 2023-06-10 15:58:11 +02:00
41929e0c72 DownloadChapterTask sets execution of parentTask 2023-06-10 15:04:37 +02:00
4fcaca1a6e Multiple authors resolves #7 2023-06-10 14:45:04 +02:00
0e3c7f32d7 Added CancellationToken to TrangaTask #14 2023-06-10 14:34:30 +02:00
1c94625840 Added CancellationToken to TrangaTask #14 2023-06-10 14:27:09 +02:00
32f89f9dce Multiple authors resolves #7 2023-06-10 14:05:23 +02:00
234735a562 Order of tasks closes #15
Also API /Queue/Get orders in order of nextExecution
2023-06-10 00:45:55 +02:00
8b916eb854 invalid Ids 2023-06-10 00:23:23 +02:00
29e1790c93 website tasks-width now max 95vw 2023-06-10 00:10:16 +02:00
ac4c799a74 Better indication if tasks have started. 2023-06-10 00:07:41 +02:00
7c62883c37 invalid id 2023-06-10 00:02:51 +02:00
02018253bf wrong nesting ... 2023-06-10 00:01:38 +02:00
2aec884009 Moved update interval for task-progress to own interval, progress gets continually updated. 2023-06-09 23:58:04 +02:00
b3321ff030 unnecessary log 2023-06-09 23:48:39 +02:00
13 changed files with 212 additions and 140 deletions

View File

@ -196,7 +196,7 @@ app.MapGet("/Tasks/GetRunningTasks",
() => taskManager.GetAllTasks().Where(task => task.state is TrangaTask.ExecutionState.Running)); () => taskManager.GetAllTasks().Where(task => task.state is TrangaTask.ExecutionState.Running));
app.MapGet("/Queue/GetList", app.MapGet("/Queue/GetList",
() => taskManager.GetAllTasks().Where(task => task.state is TrangaTask.ExecutionState.Enqueued)); () => taskManager.GetAllTasks().Where(task => task.state is TrangaTask.ExecutionState.Enqueued).OrderBy(task => task.nextExecution));
app.MapPost("/Queue/Enqueue", (string taskType, string? connectorName, string? publicationId) => app.MapPost("/Queue/Enqueue", (string taskType, string? connectorName, string? publicationId) =>
{ {

View File

@ -127,7 +127,8 @@ public abstract class Connector
/// <param name="publication">Publication that contains Chapter</param> /// <param name="publication">Publication that contains Chapter</param>
/// <param name="chapter">Chapter with Images to retrieve</param> /// <param name="chapter">Chapter with Images to retrieve</param>
/// <param name="parentTask">Will be used for progress-tracking</param> /// <param name="parentTask">Will be used for progress-tracking</param>
public abstract void DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask); /// <param name="cancellationToken"></param>
public abstract void DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null);
/// <summary> /// <summary>
/// Copies the already downloaded cover from cache to downloadLocation /// Copies the already downloaded cover from cache to downloadLocation
@ -166,7 +167,7 @@ public abstract class Connector
new XElement("Tags", string.Join(',',publication.tags)), new XElement("Tags", string.Join(',',publication.tags)),
new XElement("LanguageISO", publication.originalLanguage), new XElement("LanguageISO", publication.originalLanguage),
new XElement("Title", chapter.name), new XElement("Title", chapter.name),
new XElement("Writer", publication.author), new XElement("Writer", string.Join(',', publication.authors)),
new XElement("Volume", chapter.volumeNumber), new XElement("Volume", chapter.volumeNumber),
new XElement("Number", chapter.chapterNumber) new XElement("Number", chapter.chapterNumber)
); );
@ -227,8 +228,10 @@ public abstract class Connector
/// <param name="comicInfoPath">Path of the generate Chapter ComicInfo.xml, if it was generated</param> /// <param name="comicInfoPath">Path of the generate Chapter ComicInfo.xml, if it was generated</param>
/// <param name="requestType">RequestType for RateLimits</param> /// <param name="requestType">RequestType for RateLimits</param>
/// <param name="referrer">Used in http request header</param> /// <param name="referrer">Used in http request header</param>
protected void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, DownloadChapterTask parentTask, string? comicInfoPath = null, string? referrer = null) protected void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, DownloadChapterTask parentTask, string? comicInfoPath = null, string? referrer = null, CancellationToken? cancellationToken = null)
{ {
if (cancellationToken?.IsCancellationRequested??false)
return;
logger?.WriteLine("Connector", $"Downloading Images for {saveArchiveFilePath}"); logger?.WriteLine("Connector", $"Downloading Images for {saveArchiveFilePath}");
//Check if Publication Directory already exists //Check if Publication Directory already exists
string directoryPath = Path.GetDirectoryName(saveArchiveFilePath)!; string directoryPath = Path.GetDirectoryName(saveArchiveFilePath)!;
@ -249,7 +252,9 @@ public abstract class Connector
string extension = split[^1]; string extension = split[^1];
logger?.WriteLine("Connector", $"Downloading Image {chapter + 1:000}/{imageUrls.Length:000} {parentTask.publication.sortName} {parentTask.publication.internalId} Vol.{parentTask.chapter.volumeNumber} Ch.{parentTask.chapter.chapterNumber} {parentTask.progress:P2}"); logger?.WriteLine("Connector", $"Downloading Image {chapter + 1:000}/{imageUrls.Length:000} {parentTask.publication.sortName} {parentTask.publication.internalId} Vol.{parentTask.chapter.volumeNumber} Ch.{parentTask.chapter.chapterNumber} {parentTask.progress:P2}");
DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer); DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer);
parentTask.IncrementProgress(1f / imageUrls.Length); parentTask.IncrementProgress(1.0 / imageUrls.Length);
if (cancellationToken?.IsCancellationRequested??false)
return;
} }
if(comicInfoPath is not null) if(comicInfoPath is not null)

View File

@ -93,19 +93,21 @@ public class MangaDex : Connector
} }
string? posterId = null; string? posterId = null;
string? authorId = null; HashSet<string> authorIds = new();
if (manga.ContainsKey("relationships") && manga["relationships"] is not null) if (manga.ContainsKey("relationships") && manga["relationships"] is not null)
{ {
JsonArray relationships = manga["relationships"]!.AsArray(); JsonArray relationships = manga["relationships"]!.AsArray();
posterId = relationships.FirstOrDefault(relationship => relationship!["type"]!.GetValue<string>() == "cover_art")!["id"]!.GetValue<string>(); posterId = relationships.FirstOrDefault(relationship => relationship!["type"]!.GetValue<string>() == "cover_art")!["id"]!.GetValue<string>();
authorId = relationships.FirstOrDefault(relationship => relationship!["type"]!.GetValue<string>() == "author")!["id"]!.GetValue<string>(); foreach (JsonNode node in relationships.Where(relationship =>
relationship!["type"]!.GetValue<string>() == "author"))
authorIds.Add(node!["id"]!.GetValue<string>());
} }
string? coverUrl = GetCoverUrl(publicationId, posterId); string? coverUrl = GetCoverUrl(publicationId, posterId);
string? coverCacheName = null; string? coverCacheName = null;
if (coverUrl is not null) if (coverUrl is not null)
coverCacheName = SaveCoverImageToCache(coverUrl, (byte)RequestType.AtHomeServer); coverCacheName = SaveCoverImageToCache(coverUrl, (byte)RequestType.AtHomeServer);
string? author = GetAuthor(authorId); List<string> authors = GetAuthors(authorIds);
Dictionary<string, string> linksDict = new(); Dictionary<string, string> linksDict = new();
if (attributes.ContainsKey("links") && attributes["links"] is not null) if (attributes.ContainsKey("links") && attributes["links"] is not null)
@ -129,7 +131,7 @@ public class MangaDex : Connector
Publication pub = new ( Publication pub = new (
title, title,
author, authors,
description, description,
altTitlesDict, altTitlesDict,
tags.ToArray(), tags.ToArray(),
@ -205,8 +207,10 @@ public class MangaDex : Connector
return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray(); return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray();
} }
public override void DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask) public override void DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null)
{ {
if (cancellationToken?.IsCancellationRequested??false)
return;
logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}"); logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
//Request URLs for Chapter-Images //Request URLs for Chapter-Images
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
@ -229,7 +233,7 @@ public class MangaDex : Connector
File.WriteAllText(comicInfoPath, GetComicInfoXmlString(publication, chapter, logger)); File.WriteAllText(comicInfoPath, GetComicInfoXmlString(publication, chapter, logger));
//Download Chapter-Images //Download Chapter-Images
DownloadChapterImages(imageUrls.ToArray(), GetArchiveFilePath(publication, chapter), (byte)RequestType.AtHomeServer, parentTask, comicInfoPath); DownloadChapterImages(imageUrls.ToArray(), GetArchiveFilePath(publication, chapter), (byte)RequestType.AtHomeServer, parentTask, comicInfoPath, cancellationToken:cancellationToken);
} }
private string? GetCoverUrl(string publicationId, string? posterId) private string? GetCoverUrl(string publicationId, string? posterId)
@ -257,21 +261,23 @@ public class MangaDex : Connector
return coverUrl; return coverUrl;
} }
private string? GetAuthor(string? authorId) private List<string> GetAuthors(IEnumerable<string> authorIds)
{ {
if (authorId is null) List<string> ret = new();
return null; foreach (string authorId in authorIds)
{
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", (byte)RequestType.Author); downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", (byte)RequestType.Author);
if (requestResult.statusCode != HttpStatusCode.OK) if (requestResult.statusCode != HttpStatusCode.OK)
return null; return ret;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result); JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
if (result is null) if (result is null)
return null; return ret;
string author = result["data"]!["attributes"]!["name"]!.GetValue<string>(); string authorName = result["data"]!["attributes"]!["name"]!.GetValue<string>();
logger?.WriteLine(this.GetType().ToString(), $"Got author {authorId} -> {author}"); ret.Add(authorName);
return author; logger?.WriteLine(this.GetType().ToString(), $"Got author {authorId} -> {authorName}");
}
return ret;
} }
} }

View File

@ -1,4 +1,5 @@
using System.Net; using System.Globalization;
using System.Net;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using HtmlAgilityPack; using HtmlAgilityPack;
using Logging; using Logging;
@ -70,8 +71,8 @@ public class Manganato : Connector
Dictionary<string, string> altTitles = new(); Dictionary<string, string> altTitles = new();
Dictionary<string, string>? links = null; Dictionary<string, string>? links = null;
HashSet<string> tags = new(); HashSet<string> tags = new();
string? author = null, originalLanguage = null; string[] authors = Array.Empty<string>();
int? year = DateTime.Now.Year; string originalLanguage = "";
HtmlNode infoNode = document.DocumentNode.Descendants("div").First(d => d.HasClass("story-info-right")); HtmlNode infoNode = document.DocumentNode.Descendants("div").First(d => d.HasClass("story-info-right"));
@ -93,7 +94,7 @@ public class Manganato : Connector
altTitles.Add(i.ToString(), alts[i]); altTitles.Add(i.ToString(), alts[i]);
break; break;
case "authors": case "authors":
author = value; authors = value.Split('-');
break; break;
case "status": case "status":
status = value; status = value;
@ -118,9 +119,9 @@ public class Manganato : Connector
string yearString = document.DocumentNode.Descendants("li").Last(li => li.HasClass("a-h")).Descendants("span") string yearString = document.DocumentNode.Descendants("li").Last(li => li.HasClass("a-h")).Descendants("span")
.First(s => s.HasClass("chapter-time")).InnerText; .First(s => s.HasClass("chapter-time")).InnerText;
year = Convert.ToInt32(yearString.Split(',')[^1]) + 2000; int year = Convert.ToInt32(yearString.Split(',')[^1]) + 2000;
return new Publication(sortName, author, description, altTitles, tags.ToArray(), posterUrl, coverFileNameInCache, links, return new Publication(sortName, authors.ToList(), description, altTitles, tags.ToArray(), posterUrl, coverFileNameInCache, links,
year, originalLanguage, status, publicationId); year, originalLanguage, status, publicationId);
} }
@ -132,11 +133,18 @@ public class Manganato : Connector
downloadClient.MakeRequest(requestUrl, (byte)1); downloadClient.MakeRequest(requestUrl, (byte)1);
if (requestResult.statusCode != HttpStatusCode.OK) if (requestResult.statusCode != HttpStatusCode.OK)
return Array.Empty<Chapter>(); return Array.Empty<Chapter>();
return ParseChaptersFromHtml(requestResult.result); //Return Chapters ordered by Chapter-Number
NumberFormatInfo chapterNumberFormatInfo = new()
{
NumberDecimalSeparator = "."
};
List<Chapter> chapters = ParseChaptersFromHtml(requestResult.result);
logger?.WriteLine(this.GetType().ToString(), $"Done getting Chapters for {publication.internalId}");
return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray();
} }
private Chapter[] ParseChaptersFromHtml(Stream html) private List<Chapter> ParseChaptersFromHtml(Stream html)
{ {
StreamReader reader = new (html); StreamReader reader = new (html);
string htmlString = reader.ReadToEnd(); string htmlString = reader.ReadToEnd();
@ -158,11 +166,13 @@ public class Manganato : Connector
ret.Add(new Chapter(chapterName, volumeNumber, chapterNumber, url)); ret.Add(new Chapter(chapterName, volumeNumber, chapterNumber, url));
} }
ret.Reverse(); ret.Reverse();
return ret.ToArray(); return ret;
} }
public override void DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask) public override void DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null)
{ {
if (cancellationToken?.IsCancellationRequested??false)
return;
logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}"); logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
string requestUrl = chapter.url; string requestUrl = chapter.url;
DownloadClient.RequestResult requestResult = DownloadClient.RequestResult requestResult =
@ -175,7 +185,7 @@ public class Manganato : Connector
string comicInfoPath = Path.GetTempFileName(); string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, GetComicInfoXmlString(publication, chapter, logger)); File.WriteAllText(comicInfoPath, GetComicInfoXmlString(publication, chapter, logger));
DownloadChapterImages(imageUrls, GetArchiveFilePath(publication, chapter), (byte)1, parentTask, comicInfoPath, "https://chapmanganato.com/"); DownloadChapterImages(imageUrls, GetArchiveFilePath(publication, chapter), (byte)1, parentTask, comicInfoPath, "https://chapmanganato.com/", cancellationToken);
} }
private string[] ParseImageUrlsFromHtml(Stream html) private string[] ParseImageUrlsFromHtml(Stream html)

View File

@ -1,4 +1,5 @@
using System.Net; using System.Globalization;
using System.Net;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Xml.Linq; using System.Xml.Linq;
using HtmlAgilityPack; using HtmlAgilityPack;
@ -140,10 +141,9 @@ public class Mangasee : Connector
HtmlNode[] authorsNodes = attributes.Descendants("li") HtmlNode[] authorsNodes = attributes.Descendants("li")
.First(node => node.InnerText.Contains("author(s):", StringComparison.CurrentCultureIgnoreCase)) .First(node => node.InnerText.Contains("author(s):", StringComparison.CurrentCultureIgnoreCase))
.Descendants("a").ToArray(); .Descendants("a").ToArray();
string[] authors = new string[authorsNodes.Length]; List<string> authors = new();
for (int j = 0; j < authors.Length; j++) foreach(HtmlNode authorNode in authorsNodes)
authors[j] = authorsNodes[j].InnerText; authors.Add(authorNode.InnerText);
string author = string.Join(" - ", authors);
HtmlNode[] genreNodes = attributes.Descendants("li") HtmlNode[] genreNodes = attributes.Descendants("li")
.First(node => node.InnerText.Contains("genre(s):", StringComparison.CurrentCultureIgnoreCase)) .First(node => node.InnerText.Contains("genre(s):", StringComparison.CurrentCultureIgnoreCase))
@ -170,7 +170,7 @@ public class Mangasee : Connector
foreach(string at in a) foreach(string at in a)
altTitles.Add((i++).ToString(), at); altTitles.Add((i++).ToString(), at);
return new Publication(sortName, author, description, altTitles, tags.ToArray(), posterUrl, coverFileNameInCache, links, return new Publication(sortName, authors, description, altTitles, tags.ToArray(), posterUrl, coverFileNameInCache, links,
year, originalLanguage, status, publicationId); year, originalLanguage, status, publicationId);
} }
@ -200,20 +200,29 @@ public class Mangasee : Connector
ret.Add(new Chapter("", volumeNumber, chapterNumber, url)); ret.Add(new Chapter("", volumeNumber, chapterNumber, url));
} }
ret.Reverse(); //Return Chapters ordered by Chapter-Number
return ret.ToArray(); NumberFormatInfo chapterNumberFormatInfo = new()
{
NumberDecimalSeparator = "."
};
logger?.WriteLine(this.GetType().ToString(), $"Done getting Chapters for {publication.internalId}");
return ret.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray();
} }
public override void DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask) public override void DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask, CancellationToken? cancellationToken = null)
{ {
while (this._browser is null) if (cancellationToken?.IsCancellationRequested??false)
return;
while (this._browser is null && !(cancellationToken?.IsCancellationRequested??false))
{ {
logger?.WriteLine(this.GetType().ToString(), "Waiting for headless browser to download..."); logger?.WriteLine(this.GetType().ToString(), "Waiting for headless browser to download...");
Thread.Sleep(1000); Thread.Sleep(1000);
} }
if (cancellationToken?.IsCancellationRequested??false)
return;
logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}"); logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
IPage page = _browser.NewPageAsync().Result; IPage page = _browser!.NewPageAsync().Result;
IResponse response = page.GoToAsync(chapter.url).Result; IResponse response = page.GoToAsync(chapter.url).Result;
if (response.Ok) if (response.Ok)
{ {
@ -229,7 +238,7 @@ public class Mangasee : Connector
string comicInfoPath = Path.GetTempFileName(); string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, GetComicInfoXmlString(publication, chapter, logger)); File.WriteAllText(comicInfoPath, GetComicInfoXmlString(publication, chapter, logger));
DownloadChapterImages(urls.ToArray(), GetArchiveFilePath(publication, chapter), (byte)1, parentTask, comicInfoPath); DownloadChapterImages(urls.ToArray(), GetArchiveFilePath(publication, chapter), (byte)1, parentTask, comicInfoPath, cancellationToken:cancellationToken);
} }
} }
} }

View File

@ -12,7 +12,7 @@ namespace Tranga;
public readonly struct Publication public readonly struct Publication
{ {
public string sortName { get; } public string sortName { get; }
public string? author { get; } public List<string> authors { get; }
public Dictionary<string,string> altTitles { get; } public Dictionary<string,string> altTitles { get; }
// ReSharper disable trice MemberCanBePrivate.Global, trust // ReSharper disable trice MemberCanBePrivate.Global, trust
public string? description { get; } public string? description { get; }
@ -29,10 +29,22 @@ public readonly struct Publication
private static readonly Regex LegalCharacters = new Regex(@"[A-Z]*[a-z]*[0-9]* *\.*-*,*'*\'*\)*\(*~*!*"); private static readonly Regex LegalCharacters = new Regex(@"[A-Z]*[a-z]*[0-9]* *\.*-*,*'*\'*\)*\(*~*!*");
public Publication(string sortName, string? author, string? description, Dictionary<string,string> altTitles, string[] tags, string? posterUrl, string? coverFileNameInCache, Dictionary<string,string>? links, int? year, string? originalLanguage, string status, string publicationId) [JsonConstructor] //Legacy
public Publication(string sortName, string? author, string? description, Dictionary<string, string> altTitles,
string[] tags, string? posterUrl, string? coverFileNameInCache, Dictionary<string, string>? links, int? year,
string? originalLanguage, string status, string publicationId)
{
List<string> pAuthors = new();
if(author is not null)
pAuthors.Add(author);
this = new Publication(sortName, pAuthors, description, altTitles, tags, posterUrl,
coverFileNameInCache, links, year, originalLanguage, status, publicationId);
}
public Publication(string sortName, List<string> authors, string? description, Dictionary<string,string> altTitles, string[] tags, string? posterUrl, string? coverFileNameInCache, Dictionary<string,string>? links, int? year, string? originalLanguage, string status, string publicationId)
{ {
this.sortName = sortName; this.sortName = sortName;
this.author = author; this.authors = authors;
this.description = description; this.description = description;
this.altTitles = altTitles; this.altTitles = altTitles;
this.tags = tags; this.tags = tags;

View File

@ -19,7 +19,7 @@ public class TaskManager
public TrangaSettings settings { get; } public TrangaSettings settings { get; }
private Logger? logger { get; } private Logger? logger { get; }
private readonly Dictionary<DownloadChapterTask, Task> _runningDownloadChapterTasks = new(); private readonly Dictionary<DownloadChapterTask, CancellationTokenSource> _runningDownloadChapterTasks = new();
/// <param name="downloadFolderPath">Local path to save data (Manga) to</param> /// <param name="downloadFolderPath">Local path to save data (Manga) to</param>
/// <param name="workingDirectory">Path to the working directory</param> /// <param name="workingDirectory">Path to the working directory</param>
@ -89,8 +89,9 @@ public class TaskManager
while (_continueRunning) while (_continueRunning)
{ {
TrangaTask[] tmp = _allTasks.Where(taskQuery => TrangaTask[] tmp = _allTasks.Where(taskQuery =>
taskQuery.nextExecution < DateTime.Now && taskQuery.nextExecution < DateTime.Now &&
taskQuery.state is TrangaTask.ExecutionState.Waiting or TrangaTask.ExecutionState.Enqueued).ToArray(); taskQuery.state is TrangaTask.ExecutionState.Waiting or TrangaTask.ExecutionState.Enqueued)
.OrderBy(tmpTask => tmpTask.nextExecution).ToArray();
foreach (TrangaTask task in tmp) foreach (TrangaTask task in tmp)
{ {
task.state = TrangaTask.ExecutionState.Enqueued; task.state = TrangaTask.ExecutionState.Enqueued;
@ -121,23 +122,25 @@ public class TaskManager
} }
HashSet<DownloadChapterTask> toRemove = new(); HashSet<DownloadChapterTask> toRemove = new();
foreach (KeyValuePair<DownloadChapterTask,Task> removeTask in _runningDownloadChapterTasks) foreach (KeyValuePair<DownloadChapterTask, CancellationTokenSource> removeTask in _runningDownloadChapterTasks)
{ {
if (removeTask.Key.GetType() == typeof(DownloadChapterTask) && if (removeTask.Key.GetType() == typeof(DownloadChapterTask) &&
DateTime.Now.Subtract(removeTask.Key.lastChange) > TimeSpan.FromMinutes(3))//3 Minutes since last update to task -> remove DateTime.Now.Subtract(removeTask.Key.lastChange) > TimeSpan.FromMinutes(3))//3 Minutes since last update to task -> remove
{ {
logger?.WriteLine(this.GetType().ToString(), $"Disposing failed task {removeTask.Key}."); logger?.WriteLine(this.GetType().ToString(), $"Disposing failed task {removeTask.Key}.");
removeTask.Key.parentTask?.DecrementProgress(removeTask.Key.progress); removeTask.Key.parentTask?.DecrementProgress(removeTask.Key.progress);
//removeTask.Value.Dispose(); Currently not available, however since task is removed from _allTasks should work. Memory leak however... removeTask.Value.Cancel();
toRemove.Add(removeTask.Key); toRemove.Add(removeTask.Key);
} }
} }
foreach (DownloadChapterTask taskToRemove in toRemove) foreach (DownloadChapterTask taskToRemove in toRemove)
{ {
DeleteTask(taskToRemove); DeleteTask(taskToRemove);
AddTask(new DownloadChapterTask(taskToRemove.task, taskToRemove.connectorName, DownloadChapterTask newTask = new DownloadChapterTask(taskToRemove.task, taskToRemove.connectorName,
taskToRemove.publication, taskToRemove.chapter, taskToRemove.language, taskToRemove.publication, taskToRemove.chapter, taskToRemove.language,
taskToRemove.parentTask)); taskToRemove.parentTask);
AddTask(newTask);
taskToRemove.parentTask?.ReplaceFailedChildTask(taskToRemove, newTask);
} }
if(allTasksWaitingLength != _allTasks.Count(task => task.state is TrangaTask.ExecutionState.Waiting)) if(allTasksWaitingLength != _allTasks.Count(task => task.state is TrangaTask.ExecutionState.Waiting))
@ -154,12 +157,13 @@ public class TaskManager
public void ExecuteTaskNow(TrangaTask task) public void ExecuteTaskNow(TrangaTask task)
{ {
task.state = TrangaTask.ExecutionState.Running; task.state = TrangaTask.ExecutionState.Running;
CancellationTokenSource cToken = new ();
Task t = new(() => Task t = new(() =>
{ {
task.Execute(this, this.logger); task.Execute(this, this.logger, cToken.Token);
}); }, cToken.Token);
if(task.GetType() == typeof(DownloadChapterTask)) if(task.GetType() == typeof(DownloadChapterTask))
_runningDownloadChapterTasks.Add((DownloadChapterTask)task, t); _runningDownloadChapterTasks.Add((DownloadChapterTask)task, cToken);
t.Start(); t.Start();
} }
@ -424,6 +428,16 @@ public class TaskManager
this._allTasks = JsonConvert.DeserializeObject<HashSet<TrangaTask>>(buffer, new JsonSerializerSettings() { Converters = { new TrangaTask.TrangaTaskJsonConverter() } })!; this._allTasks = JsonConvert.DeserializeObject<HashSet<TrangaTask>>(buffer, new JsonSerializerSettings() { Converters = { new TrangaTask.TrangaTaskJsonConverter() } })!;
} }
foreach (TrangaTask task in this._allTasks.Where(task => task.GetType() == typeof(DownloadChapterTask)))
{
DownloadChapterTask dcTask = (DownloadChapterTask)task;
IEnumerable<TrangaTask> dncTasks = this._allTasks.Where(pTask => pTask.GetType() == typeof(DownloadNewChaptersTask));
DownloadNewChaptersTask? parentTask = (DownloadNewChaptersTask?)dncTasks.FirstOrDefault(pTask => pTask.taskId.Equals(dcTask.parentTaskId));
dcTask.parentTask = parentTask;
parentTask?.AddChildTask(dcTask);
}
if (File.Exists(settings.knownPublicationsPath)) if (File.Exists(settings.knownPublicationsPath))
{ {
logger?.WriteLine(this.GetType().ToString(), $"Importing known publications from {settings.knownPublicationsPath}"); logger?.WriteLine(this.GetType().ToString(), $"Importing known publications from {settings.knownPublicationsPath}");

View File

@ -1,4 +1,5 @@
using System.Text.Json.Serialization; using System.Globalization;
using System.Text.Json.Serialization;
using Logging; using Logging;
using Newtonsoft.Json; using Newtonsoft.Json;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
@ -20,8 +21,9 @@ public abstract class TrangaTask
public TimeSpan reoccurrence { get; } public TimeSpan reoccurrence { get; }
public DateTime lastExecuted { get; set; } public DateTime lastExecuted { get; set; }
public Task task { get; } public Task task { get; }
public string taskId { get; set; }
[Newtonsoft.Json.JsonIgnore]public ExecutionState state { get; set; } [Newtonsoft.Json.JsonIgnore]public ExecutionState state { get; set; }
[Newtonsoft.Json.JsonIgnore]public float progress { get; protected set; } [Newtonsoft.Json.JsonIgnore]public double progress { get; protected set; }
[Newtonsoft.Json.JsonIgnore]public DateTime nextExecution => lastExecuted.Add(reoccurrence); [Newtonsoft.Json.JsonIgnore]public DateTime nextExecution => lastExecuted.Add(reoccurrence);
[Newtonsoft.Json.JsonIgnore]public DateTime executionStarted { get; private set; } [Newtonsoft.Json.JsonIgnore]public DateTime executionStarted { get; private set; }
@ -33,7 +35,7 @@ public abstract class TrangaTask
[Newtonsoft.Json.JsonIgnore] [Newtonsoft.Json.JsonIgnore]
public TimeSpan executionApproximatelyRemaining => this.executionApproximatelyFinished.Subtract(DateTime.Now); public TimeSpan executionApproximatelyRemaining => this.executionApproximatelyFinished.Subtract(DateTime.Now);
[Newtonsoft.Json.JsonIgnore]public DateTime lastChange { get; protected set; } [Newtonsoft.Json.JsonIgnore]public DateTime lastChange { get; private set; }
public enum ExecutionState public enum ExecutionState
{ {
@ -50,16 +52,17 @@ public abstract class TrangaTask
this.progress = 0f; this.progress = 0f;
this.executionStarted = DateTime.Now; this.executionStarted = DateTime.Now;
this.lastChange = DateTime.MaxValue; this.lastChange = DateTime.MaxValue;
this.taskId = Convert.ToBase64String(System.Text.Encoding.ASCII.GetBytes(this.executionStarted.ToString(CultureInfo.InvariantCulture)));
} }
public float IncrementProgress(float amount) public double IncrementProgress(double amount)
{ {
this.progress += amount; this.progress += amount;
this.lastChange = DateTime.Now; this.lastChange = DateTime.Now;
return this.progress; return this.progress;
} }
public float DecrementProgress(float amount) public double DecrementProgress(double amount)
{ {
this.progress -= amount; this.progress -= amount;
this.lastChange = DateTime.Now; this.lastChange = DateTime.Now;
@ -71,20 +74,22 @@ public abstract class TrangaTask
/// </summary> /// </summary>
/// <param name="taskManager"></param> /// <param name="taskManager"></param>
/// <param name="logger"></param> /// <param name="logger"></param>
protected abstract void ExecuteTask(TaskManager taskManager, Logger? logger); /// <param name="cancellationToken"></param>
protected abstract void ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null);
/// <summary> /// <summary>
/// Execute the task /// Execute the task
/// </summary> /// </summary>
/// <param name="taskManager">Should be the parent taskManager</param> /// <param name="taskManager">Should be the parent taskManager</param>
/// <param name="logger"></param> /// <param name="logger"></param>
public void Execute(TaskManager taskManager, Logger? logger) /// <param name="cancellationToken"></param>
public void Execute(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null)
{ {
logger?.WriteLine(this.GetType().ToString(), $"Executing Task {this}"); logger?.WriteLine(this.GetType().ToString(), $"Executing Task {this}");
this.state = ExecutionState.Running; this.state = ExecutionState.Running;
this.executionStarted = DateTime.Now; this.executionStarted = DateTime.Now;
this.lastChange = DateTime.Now; this.lastChange = DateTime.Now;
ExecuteTask(taskManager, logger); ExecuteTask(taskManager, logger, cancellationToken);
this.lastExecuted = DateTime.Now; this.lastExecuted = DateTime.Now;
this.state = ExecutionState.Waiting; this.state = ExecutionState.Waiting;
logger?.WriteLine(this.GetType().ToString(), $"Finished Executing Task {this}"); logger?.WriteLine(this.GetType().ToString(), $"Finished Executing Task {this}");

View File

@ -9,7 +9,9 @@ public class DownloadChapterTask : TrangaTask
public Publication publication { get; } public Publication publication { get; }
public string language { get; } public string language { get; }
public Chapter chapter { get; } public Chapter chapter { get; }
[JsonIgnore]public DownloadNewChaptersTask? parentTask { get; init; } [JsonIgnore]public DownloadNewChaptersTask? parentTask { get; set; }
public string? parentTaskId { get; set; }
public DownloadChapterTask(Task task, string connectorName, Publication publication, Chapter chapter, string language = "en", DownloadNewChaptersTask? parentTask = null) : base(task, TimeSpan.Zero) public DownloadChapterTask(Task task, string connectorName, Publication publication, Chapter chapter, string language = "en", DownloadNewChaptersTask? parentTask = null) : base(task, TimeSpan.Zero)
{ {
@ -18,22 +20,21 @@ public class DownloadChapterTask : TrangaTask
this.publication = publication; this.publication = publication;
this.language = language; this.language = language;
this.parentTask = parentTask; this.parentTask = parentTask;
this.parentTaskId = parentTask?.taskId;
} }
protected override void ExecuteTask(TaskManager taskManager, Logger? logger) protected override void ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null)
{ {
if (cancellationToken?.IsCancellationRequested??false)
return;
if(this.parentTask is not null)
this.parentTask.state = ExecutionState.Running;
Connector connector = taskManager.GetConnector(this.connectorName); Connector connector = taskManager.GetConnector(this.connectorName);
connector.DownloadChapter(this.publication, this.chapter, this); connector.DownloadChapter(this.publication, this.chapter, this, cancellationToken);
if(this.parentTask is not null)
this.parentTask.state = ExecutionState.Waiting;
taskManager.DeleteTask(this); taskManager.DeleteTask(this);
} }
public new float IncrementProgress(float amount)
{
this.progress += amount;
this.lastChange = DateTime.Now;
parentTask?.IncrementProgress(amount);
return this.progress;
}
public override string ToString() public override string ToString()
{ {

View File

@ -8,46 +8,51 @@ public class DownloadNewChaptersTask : TrangaTask
public string connectorName { get; } public string connectorName { get; }
public Publication publication { get; } public Publication publication { get; }
public string language { get; } public string language { get; }
[JsonIgnore]private int childTaskAmount { get; set; } [JsonIgnore]private HashSet<DownloadChapterTask> childTasks { get; }
[JsonIgnore]public new double progress => childTasks.Count > 0 ? childTasks.Sum(childTask => childTask.progress) / childTasks.Count : 0;
public DownloadNewChaptersTask(Task task, string connectorName, Publication publication, TimeSpan reoccurrence, string language = "en") : base(task, reoccurrence) public DownloadNewChaptersTask(Task task, string connectorName, Publication publication, TimeSpan reoccurrence, string language = "en") : base(task, reoccurrence)
{ {
this.connectorName = connectorName; this.connectorName = connectorName;
this.publication = publication; this.publication = publication;
this.language = language; this.language = language;
childTaskAmount = 0; this.childTasks = new();
}
public new float IncrementProgress(float amount)
{
this.progress += amount / this.childTaskAmount;
this.lastChange = DateTime.Now;
return this.progress;
}
public new float DecrementProgress(float amount)
{
this.progress -= amount / this.childTaskAmount;
this.lastChange = DateTime.Now;
return this.progress;
} }
protected override void ExecuteTask(TaskManager taskManager, Logger? logger) protected override void ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null)
{ {
if (cancellationToken?.IsCancellationRequested??false)
return;
Publication pub = publication!; Publication pub = publication!;
Connector connector = taskManager.GetConnector(this.connectorName); Connector connector = taskManager.GetConnector(this.connectorName);
//Check if Publication already has a Folder //Check if Publication already has a Folder
pub.CreatePublicationFolder(taskManager.settings.downloadLocation); pub.CreatePublicationFolder(taskManager.settings.downloadLocation);
List<Chapter> newChapters = GetNewChaptersList(connector, pub, language!, ref taskManager.chapterCollection); List<Chapter> newChapters = GetNewChaptersList(connector, pub, language!, ref taskManager.chapterCollection);
this.childTaskAmount = newChapters.Count;
connector.CopyCoverFromCacheToDownloadLocation(pub, taskManager.settings); connector.CopyCoverFromCacheToDownloadLocation(pub, taskManager.settings);
pub.SaveSeriesInfoJson(connector.downloadLocation); pub.SaveSeriesInfoJson(connector.downloadLocation);
foreach (Chapter newChapter in newChapters) foreach (Chapter newChapter in newChapters)
taskManager.AddTask(new DownloadChapterTask(Task.DownloadChapter, this.connectorName!, pub, newChapter, this.language, this)); {
DownloadChapterTask newTask = new (Task.DownloadChapter, this.connectorName!, pub, newChapter, this.language, this);
taskManager.AddTask(newTask);
this.childTasks.Add(newTask);
}
}
public void ReplaceFailedChildTask(DownloadChapterTask failed, DownloadChapterTask newTask)
{
if (!this.childTasks.Contains(failed))
throw new ArgumentException($"Task {failed} is not childTask of {this}");
this.childTasks.Remove(failed);
this.childTasks.Add(newTask);
}
public void AddChildTask(DownloadChapterTask childTask)
{
this.childTasks.Add(childTask);
} }
/// <summary> /// <summary>

View File

@ -8,8 +8,10 @@ public class UpdateLibrariesTask : TrangaTask
{ {
} }
protected override void ExecuteTask(TaskManager taskManager, Logger? logger) protected override void ExecuteTask(TaskManager taskManager, Logger? logger, CancellationToken? cancellationToken = null)
{ {
if (cancellationToken?.IsCancellationRequested??false)
return;
foreach(LibraryManager lm in taskManager.settings.libraryManagers) foreach(LibraryManager lm in taskManager.settings.libraryManagers)
lm.UpdateLibrary(); lm.UpdateLibrary();
this.progress = 1f; this.progress = 1f;

View File

@ -40,6 +40,7 @@ const settingApiUri = document.querySelector("#settingApiUri");
const tagTasksRunning = document.querySelector("#tasksRunningTag"); const tagTasksRunning = document.querySelector("#tasksRunningTag");
const tagTasksQueued = document.querySelector("#tasksQueuedTag"); const tagTasksQueued = document.querySelector("#tasksQueuedTag");
const downloadTasksPopup = document.querySelector("#downloadTasksPopup"); const downloadTasksPopup = document.querySelector("#downloadTasksPopup");
const downloadTasksOutput = downloadTasksPopup.querySelector("popup-content");
searchbox.addEventListener("keyup", (event) => FilterResults()); searchbox.addEventListener("keyup", (event) => FilterResults());
settingsCog.addEventListener("click", () => OpenSettings()); settingsCog.addEventListener("click", () => OpenSettings());
@ -232,7 +233,7 @@ function ShowPublicationViewerWindow(publicationId, event, add){
publicationViewerName.innerText = publication.sortName; publicationViewerName.innerText = publication.sortName;
publicationViewerTags.innerText = publication.tags.join(", "); publicationViewerTags.innerText = publication.tags.join(", ");
publicationViewerDescription.innerText = publication.description; publicationViewerDescription.innerText = publication.description;
publicationViewerAuthor.innerText = publication.author; publicationViewerAuthor.innerText = publication.authors.join(',');
pubviewcover.src = `imageCache/${publication.coverFileNameInCache}`; pubviewcover.src = `imageCache/${publication.coverFileNameInCache}`;
toEditId = publicationId; toEditId = publicationId;
@ -359,34 +360,26 @@ function FilterResults(){
} }
function ShowTasksQueue(){ function ShowTasksQueue(){
downloadTasksPopup.style.display = "flex";
var outputDom = downloadTasksPopup.querySelector("popup-content");
outputDom.replaceChildren();
GetRunningTasks().then((taskJson) => {
console.log(taskJson);
taskJson.forEach(task => {
outputDom.appendChild(CreateProgressChild(task));
});
});
GetQueue().then((taskJson) => {
taskJson.forEach(task => {
outputDom.appendChild(CreateProgressChild(task));
});
});
setInterval(() => { downloadTasksOutput.replaceChildren();
GetRunningTasks().then((json) => { GetRunningTasks()
json.forEach(task => { .then(json => {
if(task.chapter != undefined){ tagTasksRunning.innerText = json.length;
document.querySelector(`#progress${task.publication.internalId}-${task.chapter.sortNumber}`).value = task.progress; json.forEach(task => {
document.querySelector(`#progressStr${task.publication.internalId}-${task.chapter.sortNumber}`).innerText = task.progress.toLocaleString(undefined,{style: 'percent', minimumFractionDigits:2}); downloadTasksOutput.appendChild(CreateProgressChild(task));
}else{ document.querySelector(`#progress${task.taskId.replaceAll('=','')}`).value = task.progress;
document.querySelector(`#progress${task.publication.internalId}`).value = task.progress; document.querySelector(`#progressStr${task.taskId.replaceAll('=','')}`).innerText = task.progress.toLocaleString(undefined,{style: 'percent', minimumFractionDigits:2});
document.querySelector(`#progressStr${task.publication.internalId}`).innerText = task.progress.toLocaleString(undefined,{style: 'percent', minimumFractionDigits:2}); });
}
});
}); });
},500);
GetQueue()
.then(json => {
tagTasksQueued.innerText = json.length;
json.forEach(task => {
downloadTasksOutput.appendChild(CreateProgressChild(task));
});
});
downloadTasksPopup.style.display = "flex";
} }
function CreateProgressChild(task){ function CreateProgressChild(task){
@ -402,12 +395,13 @@ function CreateProgressChild(task){
var progress = document.createElement("progress"); var progress = document.createElement("progress");
progress.value = 0; progress.id = `progress${task.taskId.replaceAll('=','')}`;
child.appendChild(progress); child.appendChild(progress);
var progressStr = document.createElement("span"); var progressStr = document.createElement("span");
progressStr.innerText = "00.00%"; progressStr.innerText = "00.00%";
progressStr.className = "progressStr"; progressStr.className = "progressStr";
progressStr.id = `progressStr${task.taskId.replaceAll('=','')}`;
child.appendChild(progressStr); child.appendChild(progressStr);
if(task.chapter != undefined){ if(task.chapter != undefined){
@ -420,12 +414,6 @@ function CreateProgressChild(task){
chapterName.className = "chapterName"; chapterName.className = "chapterName";
chapterName.innerText = task.chapter.name; chapterName.innerText = task.chapter.name;
child.appendChild(chapterName); child.appendChild(chapterName);
progress.id = `progress${task.publication.internalId}-${task.chapter.sortNumber}`;
progressStr.id = `progressStr${task.publication.internalId}-${task.chapter.sortNumber}`;
}else{
progress.id = `progress${task.publication.internalId}`;
progressStr.id = `progressStr${task.publication.internalId}`;
} }
@ -434,6 +422,7 @@ function CreateProgressChild(task){
//Resets the tasks shown //Resets the tasks shown
ResetContent(); ResetContent();
downloadTasksOutput.replaceChildren();
//Get Tasks and show them //Get Tasks and show them
GetDownloadTasks() GetDownloadTasks()
.then(json => json.forEach(task => { .then(json => json.forEach(task => {
@ -446,11 +435,17 @@ GetDownloadTasks()
GetRunningTasks() GetRunningTasks()
.then(json => { .then(json => {
tagTasksRunning.innerText = json.length; tagTasksRunning.innerText = json.length;
json.forEach(task => {
downloadTasksOutput.appendChild(CreateProgressChild(task));
});
}); });
GetQueue() GetQueue()
.then(json => { .then(json => {
tagTasksQueued.innerText = json.length; tagTasksQueued.innerText = json.length;
json.forEach(task => {
downloadTasksOutput.appendChild(CreateProgressChild(task));
});
}) })
setInterval(() => { setInterval(() => {
@ -483,6 +478,14 @@ setInterval(() => {
GetQueue() GetQueue()
.then(json => { .then(json => {
tagTasksQueued.innerText = json.length; tagTasksQueued.innerText = json.length;
}) });
}, 1000);
}, 1000);
setInterval(() => {
GetRunningTasks().then((json) => {
json.forEach(task => {
document.querySelector(`#progress${task.taskId.replaceAll('=','')}`).value = task.progress;
document.querySelector(`#progressStr${task.taskId.replaceAll('=','')}`).innerText = task.progress.toLocaleString(undefined,{style: 'percent', minimumFractionDigits:2});
});
});
},500);

View File

@ -374,7 +374,7 @@ popup popup-window popup-content input, select {
margin: 0 0 0 10px; margin: 0 0 0 10px;
height: calc(100vh - 140px); height: calc(100vh - 140px);
width: 400px; width: 400px;
max-width: 50vw; max-width: 95vw;
overflow-y: scroll; overflow-y: scroll;
} }