9 Commits

Author SHA1 Message Date
cf6fc3b8f6 Update readme HtmlAgilityPack 2023-06-06 21:27:12 +02:00
f5141d0f8e removed unnecessary async 2023-06-06 21:27:02 +02:00
5c753e7a7d Added fields to TrangaTask:
executionStarted,
executionApproximatelyFinished,
executionApproximatelyRemaining
to track progress
2023-06-06 21:19:30 +02:00
17ce820cf3 Changed taskProgress increment to call method that updates progress accordingly (with parent-tasks being also updated) 2023-06-06 20:54:21 +02:00
5b4a3b9d7c README update 2023-06-06 15:41:26 +02:00
f73997e563 Fix browser startup 2023-06-05 21:38:32 +02:00
437136804d Also delete downloadChapterTask when delete download new chapter tasks 2023-06-05 21:16:04 +02:00
e14683d21a Merge remote-tracking branch 'origin/master'
# Conflicts:
#	Tranga/TaskManager.cs
2023-06-05 21:11:10 +02:00
5ae02ee0ed Fix Bug where all tasks would be deleted... 2023-06-05 21:10:42 +02:00
8 changed files with 116 additions and 58 deletions

View File

@ -55,8 +55,9 @@ Tranga can download Chapters and Metadata from Scanlation sites such as
- [MangaDex.org](https://mangadex.org/)
- [Manganato.com](https://manganato.com/)
- [Mangasee](https://mangasee123.com/)
and automatically start updates in [Komga](https://komga.org/) to import them.
and automatically start updates in [Komga](https://komga.org/) and [Kavita](https://www.kavitareader.com/) to import them.
### Inspiration:
@ -72,6 +73,8 @@ That is why I wanted to create my own project, in a language I understand, and t
- .NET-Core
- Newtonsoft.JSON
- [PuppeteerSharp](https://www.puppeteersharp.com/)
- [Html Agility Pack (HAP)](https://html-agility-pack.net/)
- Love <3 Blåhaj 🦈
<p align="right">(<a href="#readme-top">back to top</a>)</p>
@ -108,14 +111,12 @@ Wherever you are mounting `/usr/share/Tranga-API` you also need to mount that sa
### Prerequisites
[.NET-Core 7.0](https://dotnet.microsoft.com/en-us/download/dotnet/7.0)
[.NET-Core 7.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/7.0)
<!-- ROADMAP -->
## Roadmap
- [x] Web-UI #1
- [ ] More Connectors
- [x] Manganato #2
- [ ] Docker ARM support
- [ ] ?
See the [open issues](https://git.bernloehr.eu/glax/Tranga/issues) for a full list of proposed features (and known issues).

View File

@ -135,8 +135,12 @@ public static class Tranga_Cli
switch (selection)
{
case ConsoleKey.L:
while (!Console.KeyAvailable)
{
PrintTasks(taskManager.GetAllTasks(), logger);
Console.WriteLine("Press any key.");
Thread.Sleep(500);
}
Console.ReadKey();
break;
case ConsoleKey.C:
@ -160,17 +164,26 @@ public static class Tranga_Cli
Console.ReadKey();
break;
case ConsoleKey.R:
while (!Console.KeyAvailable)
{
PrintTasks(
taskManager.GetAllTasks().Where(eTask => eTask.state == TrangaTask.ExecutionState.Running)
.ToArray(), logger);
Console.WriteLine("Press any key.");
Thread.Sleep(500);
}
Console.ReadKey();
break;
case ConsoleKey.K:
while (!Console.KeyAvailable)
{
PrintTasks(
taskManager.GetAllTasks().Where(qTask => qTask.state is TrangaTask.ExecutionState.Enqueued)
taskManager.GetAllTasks()
.Where(qTask => qTask.state is TrangaTask.ExecutionState.Enqueued)
.ToArray(), logger);
Console.WriteLine("Press any key.");
Thread.Sleep(500);
}
Console.ReadKey();
break;
case ConsoleKey.F:
@ -240,13 +253,13 @@ public static class Tranga_Cli
int tIndex = 0;
Console.WriteLine($"Tasks (Running/Queue/Total): {taskRunningCount}/{taskEnqueuedCount}/{taskCount}");
string header =
$"{"",-5}{"Task",-20} | {"Last Executed",-20} | {"Reoccurrence",-12} | {"State",-10} | {"Progress",-9} | {"Connector",-15} | Publication/Manga ";
$"{"",-5}{"Task",-20} | {"Last Executed",-20} | {"Reoccurrence",-12} | {"State",-10} | {"Progress",-9} | {"Finished",-20} | {"Remaining",-12} | {"Connector",-15} | Publication/Manga ";
Console.WriteLine(header);
Console.WriteLine(new string('-', header.Length));
foreach (TrangaTask trangaTask in tasks)
{
string[] taskSplit = trangaTask.ToString().Split(", ");
Console.WriteLine($"{tIndex++:000}: {taskSplit[0],-20} | {taskSplit[1],-20} | {taskSplit[2],-12} | {taskSplit[3],-10} | {(taskSplit.Length > 4 ? taskSplit[4] : ""),-9} | {(taskSplit.Length > 5 ? taskSplit[5] : ""),-15} | {(taskSplit.Length > 6 ? taskSplit[6] : "")} {(taskSplit.Length > 7 ? taskSplit[7] : "")} {(taskSplit.Length > 8 ? taskSplit[8] : "")}");
Console.WriteLine($"{tIndex++:000}: {taskSplit[0],-20} | {taskSplit[1],-20} | {taskSplit[2],-12} | {taskSplit[3],-10} | {taskSplit[4],-9} | {taskSplit[5],-20} | {taskSplit[6][..12],-12} | {(taskSplit.Length > 7 ? taskSplit[7] : ""),-15} | {(taskSplit.Length > 8 ? taskSplit[8] : "")} {(taskSplit.Length > 9 ? taskSplit[9] : "")} {(taskSplit.Length > 10 ? taskSplit[10] : "")}");
}
}

View File

@ -183,7 +183,7 @@ public abstract class Connector
string extension = split[^1];
logger?.WriteLine("Connector", $"Downloading Image {chapter + 1:000}/{imageUrls.Length:000} {parentTask.publication.sortName} {parentTask.publication.internalId} Vol.{parentTask.chapter.volumeNumber} Ch.{parentTask.chapter.chapterNumber} {parentTask.progress:P2}");
DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer);
parentTask.progress += 1f / imageUrls.Length;
parentTask.IncrementProgress(1f / imageUrls.Length);
}
if(comicInfoPath is not null)

View File

@ -31,24 +31,22 @@ public class Mangasee : Connector
private async void DownloadBrowser()
{
BrowserFetcher browserFetcher = new BrowserFetcher();
if (browserFetcher.LocalRevisions().Contains(ChromiumVersion))
return;
else
foreach(string rev in browserFetcher.LocalRevisions())
foreach(string rev in browserFetcher.LocalRevisions().Where(rev => rev != ChromiumVersion))
browserFetcher.Remove(rev);
if (!browserFetcher.LocalRevisions().Contains(ChromiumVersion))
{
logger?.WriteLine(this.GetType().ToString(), "Downloading headless browser");
DateTime last = DateTime.Now.Subtract(TimeSpan.FromSeconds(5));
browserFetcher.DownloadProgressChanged += async (sender, args) =>
browserFetcher.DownloadProgressChanged += (sender, args) =>
{
double currentBytes = Convert.ToDouble(args.BytesReceived) / Convert.ToDouble(args.TotalBytesToReceive);
if (args.TotalBytesToReceive == args.BytesReceived)
{
logger?.WriteLine(this.GetType().ToString(), "Browser downloaded. Launching...");
logger?.WriteLine(this.GetType().ToString(), "Browser downloaded.");
}
else if (DateTime.Now > last.AddSeconds(5))
{
logger?.WriteLine(this.GetType().ToString(), $"Browser progress: {currentBytes:P2}");
logger?.WriteLine(this.GetType().ToString(), $"Browser download progress: {currentBytes:P2}");
last = DateTime.Now;
}
@ -59,6 +57,9 @@ public class Mangasee : Connector
return;
}
await browserFetcher.DownloadAsync(ChromiumVersion);
}
logger?.WriteLine(this.GetType().ToString(), "Starting browser.");
this._browser = await Puppeteer.LaunchAsync(new LaunchOptions
{
Headless = true,
@ -173,11 +174,14 @@ public class Mangasee : Connector
year, originalLanguage, status, publicationId);
}
// ReSharper disable once ClassNeverInstantiated.Local Will be instantiated during deserialization
private class SearchResultItem
{
#pragma warning disable CS8618 //Will always be set
public string i { get; set; }
public string s { get; set; }
public string[] a { get; set; }
#pragma warning restore CS8618
}
public override Chapter[] GetChapters(Publication publication, string language = "")
@ -208,6 +212,7 @@ public class Mangasee : Connector
Thread.Sleep(1000);
}
logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
IPage page = _browser.NewPageAsync().Result;
IResponse response = page.GoToAsync(chapter.url).Result;
if (response.Ok)

View File

@ -207,7 +207,7 @@ public class TaskManager
/// </summary>
/// <param name="task">TrangaTask.Task type</param>
/// <param name="connectorName">Name of Connector that was used</param>
/// <param name="publication">Publication that was used</param>
/// <param name="publicationId">Publication that was used</param>
public void DeleteTask(TrangaTask.Task task, string? connectorName, string? publicationId)
{
logger?.WriteLine(this.GetType().ToString(), $"Removing Task {task} {publicationId}");
@ -220,13 +220,19 @@ public class TaskManager
_allTasks.RemoveWhere(trangaTask => trangaTask.task is TrangaTask.Task.UpdateLibraries);
break;
case TrangaTask.Task.DownloadNewChapters:
if(connectorName is null || publicationId is null)
if (connectorName is null || publicationId is null)
logger?.WriteLine(this.GetType().ToString(), "connectorName and publication can not be null");
else
{
_allTasks.RemoveWhere(mTask =>
mTask.GetType() == typeof(DownloadNewChaptersTask) &&
((DownloadNewChaptersTask)mTask).publication.internalId != publicationId &&
((DownloadNewChaptersTask)mTask).connectorName != connectorName!);
((DownloadNewChaptersTask)mTask).publication.internalId == publicationId &&
((DownloadNewChaptersTask)mTask).connectorName == connectorName!);
_allTasks.RemoveWhere(mTask =>
mTask.GetType() == typeof(DownloadChapterTask) &&
((DownloadChapterTask)mTask).publication.internalId == publicationId &&
((DownloadChapterTask)mTask).connectorName == connectorName!);
}
break;
}
ExportDataAndSettings();

View File

@ -21,8 +21,17 @@ public abstract class TrangaTask
public DateTime lastExecuted { get; set; }
public Task task { get; }
[Newtonsoft.Json.JsonIgnore]public ExecutionState state { get; set; }
[Newtonsoft.Json.JsonIgnore]public float progress { get; set; }
[Newtonsoft.Json.JsonIgnore]public float progress { get; protected set; }
[Newtonsoft.Json.JsonIgnore]public DateTime nextExecution => lastExecuted.Add(reoccurrence);
[Newtonsoft.Json.JsonIgnore]public DateTime executionStarted { get; protected set; }
[Newtonsoft.Json.JsonIgnore]
public DateTime executionApproximatelyFinished => this.progress != 0
? this.executionStarted.Add(DateTime.Now.Subtract(this.executionStarted) / this.progress)
: DateTime.MaxValue;
[Newtonsoft.Json.JsonIgnore]
public TimeSpan executionApproximatelyRemaining => this.executionApproximatelyFinished.Subtract(DateTime.Now);
public enum ExecutionState
{
@ -37,6 +46,13 @@ public abstract class TrangaTask
this.lastExecuted = DateTime.Now.Subtract(reoccurrence);
this.task = task;
this.progress = 0f;
this.executionStarted = DateTime.Now;
}
public float IncrementProgress(float amount)
{
this.progress += amount;
return this.progress;
}
/// <summary>
@ -55,6 +71,7 @@ public abstract class TrangaTask
{
logger?.WriteLine(this.GetType().ToString(), $"Executing Task {this}");
this.state = ExecutionState.Running;
this.executionStarted = DateTime.Now;
ExecuteTask(taskManager, logger);
this.lastExecuted = DateTime.Now;
this.state = ExecutionState.Waiting;
@ -76,7 +93,7 @@ public abstract class TrangaTask
public override string ToString()
{
return $"{task}, {lastExecuted}, {reoccurrence}, {state}, {progress:P2}";
return $"{task}, {lastExecuted}, {reoccurrence}, {state}, {progress:P2}, {executionApproximatelyFinished}, {executionApproximatelyRemaining}";
}
public class TrangaTaskJsonConverter : JsonConverter

View File

@ -1,4 +1,5 @@
using Logging;
using Newtonsoft.Json;
namespace Tranga.TrangaTasks;
@ -8,12 +9,15 @@ public class DownloadChapterTask : TrangaTask
public Publication publication { get; }
public string language { get; }
public Chapter chapter { get; }
public DownloadChapterTask(Task task, string connectorName, Publication publication, Chapter chapter, string language = "en") : base(task, TimeSpan.Zero)
[JsonIgnore]private DownloadNewChaptersTask? parentTask { get; init; }
public DownloadChapterTask(Task task, string connectorName, Publication publication, Chapter chapter, string language = "en", DownloadNewChaptersTask? parentTask = null) : base(task, TimeSpan.Zero)
{
this.chapter = chapter;
this.connectorName = connectorName;
this.publication = publication;
this.language = language;
this.parentTask = parentTask;
}
protected override void ExecuteTask(TaskManager taskManager, Logger? logger)
@ -24,6 +28,13 @@ public class DownloadChapterTask : TrangaTask
taskManager.DeleteTask(this);
}
public new float IncrementProgress(float amount)
{
this.progress += amount;
parentTask?.IncrementProgress(amount);
return this.progress;
}
public override string ToString()
{
return $"{base.ToString()}, {connectorName}, {publication.sortName} {publication.internalId}, Vol.{chapter.volumeNumber} Ch.{chapter.chapterNumber}";

View File

@ -1,4 +1,5 @@
using Logging;
using Newtonsoft.Json;
namespace Tranga.TrangaTasks;
@ -7,34 +8,38 @@ public class DownloadNewChaptersTask : TrangaTask
public string connectorName { get; }
public Publication publication { get; }
public string language { get; }
[JsonIgnore]private int childTaskAmount { get; set; }
public DownloadNewChaptersTask(Task task, string connectorName, Publication publication, TimeSpan reoccurrence, string language = "en") : base(task, reoccurrence)
{
this.connectorName = connectorName;
this.publication = publication;
this.language = language;
childTaskAmount = 0;
}
public new float IncrementProgress(float amount)
{
this.progress += amount / this.childTaskAmount;
return this.progress;
}
protected override void ExecuteTask(TaskManager taskManager, Logger? logger)
{
Publication pub = publication!;
Connector connector = taskManager.GetConnector(this.connectorName);
this.progress = 0.1f;
//Check if Publication already has a Folder
pub.CreatePublicationFolder(taskManager.settings.downloadLocation);
this.progress = 0.2f;
List<Chapter> newChapters = GetNewChaptersList(connector, pub, language!, ref taskManager.chapterCollection);
this.progress = 0.6f;
this.childTaskAmount = newChapters.Count;
connector.CopyCoverFromCacheToDownloadLocation(pub, taskManager.settings);
this.progress = 0.7f;
pub.SaveSeriesInfoJson(connector.downloadLocation);
this.progress = 0.8f;
foreach (Chapter newChapter in newChapters)
taskManager.AddTask(new DownloadChapterTask(Task.DownloadChapter, this.connectorName!, pub, newChapter, this.language));
this.progress = 1f;
taskManager.AddTask(new DownloadChapterTask(Task.DownloadChapter, this.connectorName!, pub, newChapter, this.language, this));
}
/// <summary>