Compare commits

..

9 Commits

7 changed files with 257 additions and 73 deletions

View File

@ -2,19 +2,20 @@
namespace Tranga;
/// <summary>
/// Has to be Part of a publication
/// Includes the Chapter-Name, -VolumeNumber, -ChapterNumber, the location of the chapter on the internet and the saveName of the local file.
/// </summary>
public struct Chapter
{
public Publication publication { get; }
public string? name { get; }
public string? volumeNumber { get; }
public string? chapterNumber { get; }
public string url { get; }
public string fileName { get; }
public Chapter(Publication publication, string? name, string? volumeNumber, string? chapterNumber, string url)
public Chapter(string? name, string? volumeNumber, string? chapterNumber, string url)
{
this.publication = publication;
this.name = name;
this.volumeNumber = volumeNumber;
this.chapterNumber = chapterNumber;

View File

@ -3,66 +3,137 @@ using System.Net;
namespace Tranga;
/// <summary>
/// Base-Class for all Connectors
/// Provides some methods to be used by all Connectors, as well as a DownloadClient
/// </summary>
public abstract class Connector
{
public Connector(string downloadLocation)
internal string downloadLocation { get; } //Location of local files
protected DownloadClient downloadClient { get; }
protected Connector(string downloadLocation, uint downloadDelay)
{
this.downloadLocation = downloadLocation;
this.downloadClient = new DownloadClient(downloadDelay);
}
internal string downloadLocation { get; }
public abstract string name { get; }
public abstract string name { get; } //Name of the Connector (e.g. Website)
/// <summary>
/// Returns all Publications with the given string.
/// If the string is empty or null, returns all Publication of the Connector
/// </summary>
/// <param name="publicationTitle">Search-Query</param>
/// <returns>Publications matching the query</returns>
public abstract Publication[] GetPublications(string publicationTitle = "");
/// <summary>
/// Returns all Chapters of the publication in the provided language.
/// If the language is empty or null, returns all Chapters in all Languages.
/// </summary>
/// <param name="publication">Publication to get Chapters for</param>
/// <param name="language">Language of the Chapters</param>
/// <returns>Array of Chapters matching Publication and Language</returns>
public abstract Chapter[] GetChapters(Publication publication, string language = "");
public abstract void DownloadChapter(Publication publication, Chapter chapter); //where to?
protected abstract void DownloadImage(string url, string savePath);
/// <summary>
/// Retrieves the Chapter (+Images) from the website.
/// Should later call DownloadChapterImages to retrieve the individual Images of the Chapter.
/// </summary>
/// <param name="publication">Publication that contains Chapter</param>
/// <param name="chapter">Chapter with Images to retrieve</param>
public abstract void DownloadChapter(Publication publication, Chapter chapter);
/// <summary>
/// Retrieves the Cover from the Website
/// </summary>
/// <param name="publication">Publication to retrieve Cover for</param>
public abstract void DownloadCover(Publication publication);
protected void DownloadChapter(string[] imageUrls, string saveArchiveFilePath)
{
string[] splitPath = saveArchiveFilePath.Split(Path.DirectorySeparatorChar);
string directoryPath = Path.Combine(splitPath.Take(splitPath.Length - 1).ToArray());
if (!Directory.Exists(directoryPath))
Directory.CreateDirectory(directoryPath);
string fullPath = $"{saveArchiveFilePath}.cbz";
if (File.Exists(fullPath))
return;
string tempFolder = Path.GetTempFileName();
File.Delete(tempFolder);
Directory.CreateDirectory(tempFolder);
int chapter = 0;
foreach (string imageUrl in imageUrls)
{
string[] split = imageUrl.Split('.');
string extension = split[split.Length - 1];
DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"));
}
ZipFile.CreateFromDirectory(tempFolder, fullPath);
}
/// <summary>
/// Saves the series-info to series.json in the Publication Folder
/// </summary>
/// <param name="publication">Publication to save series.json for</param>
public void SaveSeriesInfo(Publication publication)
{
string seriesInfoPath = Path.Join(downloadLocation, publication.folderName, "series.json");
if(!File.Exists(seriesInfoPath))
File.WriteAllText(seriesInfoPath,publication.GetSeriesInfo());
}
/// <summary>
/// Downloads Image from URL and saves it to the given path(incl. fileName)
/// </summary>
/// <param name="imageUrl"></param>
/// <param name="fullPath"></param>
/// <param name="downloadClient">DownloadClient of the connector</param>
protected static void DownloadImage(string imageUrl, string fullPath, DownloadClient downloadClient)
{
DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl);
byte[] buffer = new byte[requestResult.result.Length];
requestResult.result.ReadExactly(buffer, 0, buffer.Length);
File.WriteAllBytes(fullPath, buffer);
}
/// <summary>
/// Downloads all Images from URLs, Compresses to zip(cbz) and saves.
/// </summary>
/// <param name="imageUrls">List of URLs to download Images from</param>
/// <param name="saveArchiveFilePath">Full path to save archive to (without file ending .cbz)</param>
/// <param name="downloadClient">DownloadClient of the connector</param>
protected static void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, DownloadClient downloadClient)
{
//Check if Publication Directory already exists
string[] splitPath = saveArchiveFilePath.Split(Path.DirectorySeparatorChar);
string directoryPath = Path.Combine(splitPath.Take(splitPath.Length - 1).ToArray());
if (!Directory.Exists(directoryPath))
Directory.CreateDirectory(directoryPath);
string fullPath = $"{saveArchiveFilePath}.cbz";
if (File.Exists(fullPath)) //Don't download twice.
return;
//Create a temporary folder to store images
string tempFolder = Path.GetTempFileName();
File.Delete(tempFolder);
Directory.CreateDirectory(tempFolder);
internal class DownloadClient
int chapter = 0;
//Download all Images to temporary Folder
foreach (string imageUrl in imageUrls)
{
string[] split = imageUrl.Split('.');
string extension = split[split.Length - 1];
DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), downloadClient);
}
//ZIP-it and ship-it
ZipFile.CreateFromDirectory(tempFolder, fullPath);
Directory.Delete(tempFolder); //Cleanup
}
protected class DownloadClient
{
private readonly TimeSpan _requestSpeed;
private DateTime _lastRequest;
private static readonly HttpClient Client = new();
/// <summary>
/// Creates a httpClient
/// </summary>
/// <param name="delay">minimum delay between requests (to avoid spam)</param>
public DownloadClient(uint delay)
{
_requestSpeed = TimeSpan.FromMilliseconds(delay);
_lastRequest = DateTime.Now.Subtract(_requestSpeed);
}
/// <summary>
/// Request Webpage
/// </summary>
/// <param name="url"></param>
/// <returns>RequestResult with StatusCode and Stream of received data</returns>
public RequestResult MakeRequest(string url)
{
while((DateTime.Now - _lastRequest) < _requestSpeed)

View File

@ -7,33 +7,41 @@ namespace Tranga.Connectors;
public class MangaDex : Connector
{
public override string name { get; }
private readonly DownloadClient _downloadClient = new (750);
public MangaDex(string downloadLocation) : base(downloadLocation)
public MangaDex(string downloadLocation, uint downloadDelay) : base(downloadLocation, downloadDelay)
{
name = "MangaDex";
}
public MangaDex(string downloadLocation) : base(downloadLocation, 750)
{
name = "MangaDex";
}
public override Publication[] GetPublications(string publicationTitle = "")
{
const int limit = 100;
int offset = 0;
int total = int.MaxValue;
const int limit = 100; //How many values we want returned at once
int offset = 0; //"Page"
int total = int.MaxValue; //How many total results are there, is updated on first request
HashSet<Publication> publications = new();
while (offset < total)
while (offset < total) //As long as we haven't requested all "Pages"
{
//Request next Page
DownloadClient.RequestResult requestResult =
_downloadClient.MakeRequest(
downloadClient.MakeRequest(
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}");
if (requestResult.statusCode != HttpStatusCode.OK)
break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
offset += limit;
if (result is null)
break;
total = result["total"]!.GetValue<int>();
JsonArray mangaInResult = result["data"]!.AsArray();
total = result["total"]!.GetValue<int>(); //Update the total number of Publications
JsonArray mangaInResult = result["data"]!.AsArray(); //Manga-data-Array
//Loop each Manga and extract information from JSON
foreach (JsonNode? mangeNode in mangaInResult)
{
JsonObject manga = (JsonObject)mangeNode!;
@ -109,7 +117,7 @@ public class MangaDex : Connector
status,
manga["id"]!.GetValue<string>()
);
publications.Add(pub);
publications.Add(pub); //Add Publication (Manga) to result
}
}
@ -118,16 +126,17 @@ public class MangaDex : Connector
public override Chapter[] GetChapters(Publication publication, string language = "")
{
const int limit = 100;
int offset = 0;
string id = publication.downloadUrl;
int total = int.MaxValue;
const int limit = 100; //How many values we want returned at once
int offset = 0; //"Page"
int total = int.MaxValue; //How many total results are there, is updated on first request
List<Chapter> chapters = new();
//As long as we haven't requested all "Pages"
while (offset < total)
{
//Request next "Page"
DownloadClient.RequestResult requestResult =
_downloadClient.MakeRequest(
$"https://api.mangadex.org/manga/{id}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}");
downloadClient.MakeRequest(
$"https://api.mangadex.org/manga/{publication.downloadUrl}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}");
if (requestResult.statusCode != HttpStatusCode.OK)
break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -138,6 +147,7 @@ public class MangaDex : Connector
total = result["total"]!.GetValue<int>();
JsonArray chaptersInResult = result["data"]!.AsArray();
//Loop through all Chapters in result and extract information from JSON
foreach (JsonNode? jsonNode in chaptersInResult)
{
JsonObject chapter = (JsonObject)jsonNode!;
@ -156,10 +166,11 @@ public class MangaDex : Connector
? attributes["chapter"]!.GetValue<string>()
: null;
chapters.Add(new Chapter(publication, title, volume, chapterNum, chapterId));
chapters.Add(new Chapter(title, volume, chapterNum, chapterId));
}
}
//Return Chapters ordered by Chapter-Number
NumberFormatInfo chapterNumberFormatInfo = new()
{
NumberDecimalSeparator = "."
@ -169,8 +180,9 @@ public class MangaDex : Connector
public override void DownloadChapter(Publication publication, Chapter chapter)
{
//Request URLs for Chapter-Images
DownloadClient.RequestResult requestResult =
_downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'");
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'");
if (requestResult.statusCode != HttpStatusCode.OK)
return;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -180,32 +192,28 @@ public class MangaDex : Connector
string baseUrl = result["baseUrl"]!.GetValue<string>();
string hash = result["chapter"]!["hash"]!.GetValue<string>();
JsonArray imageFileNames = result["chapter"]!["data"]!.AsArray();
//Loop through all imageNames and construct urls (imageUrl)
HashSet<string> imageUrls = new();
foreach (JsonNode? image in imageFileNames)
imageUrls.Add($"{baseUrl}/data/{hash}/{image!.GetValue<string>()}");
DownloadChapter(imageUrls.ToArray(), Path.Join(downloadLocation, publication.folderName, chapter.fileName));
}
protected override void DownloadImage(string url, string savePath)
{
DownloadClient.RequestResult requestResult = _downloadClient.MakeRequest(url);
byte[] buffer = new byte[requestResult.result.Length];
requestResult.result.ReadExactly(buffer, 0, buffer.Length);
File.WriteAllBytes(savePath, buffer);
//Download Chapter-Images
DownloadChapterImages(imageUrls.ToArray(), Path.Join(downloadLocation, publication.folderName, chapter.fileName), this.downloadClient);
}
public override void DownloadCover(Publication publication)
{
string publicationPath = Path.Join(downloadLocation, publication.folderName);
Directory.CreateDirectory(publicationPath);
DirectoryInfo dirInfo = new (publicationPath);
//Check if Publication already has a Folder and cover
string publicationFolder = Path.Join(downloadLocation, publication.folderName);
Directory.CreateDirectory(publicationFolder);
DirectoryInfo dirInfo = new (publicationFolder);
foreach(FileInfo fileInfo in dirInfo.EnumerateFiles())
if (fileInfo.Name.Contains("cover."))
return;
//Request information where to download Cover
DownloadClient.RequestResult requestResult =
_downloadClient.MakeRequest($"https://api.mangadex.org/cover/{publication.posterUrl}");
downloadClient.MakeRequest($"https://api.mangadex.org/cover/{publication.posterUrl}");
if (requestResult.statusCode != HttpStatusCode.OK)
return;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
@ -215,11 +223,15 @@ public class MangaDex : Connector
string fileName = result!["data"]!["attributes"]!["fileName"]!.GetValue<string>();
string coverUrl = $"https://uploads.mangadex.org/covers/{publication.downloadUrl}/{fileName}";
//Get file-extension (jpg, png)
string[] split = coverUrl.Split('.');
string extension = split[split.Length - 1];
string outFolderPath = Path.Join(downloadLocation, publication.folderName);
Directory.CreateDirectory(outFolderPath);
DownloadImage(coverUrl, Path.Join(downloadLocation, publication.folderName, $"cover.{extension}"));
//Download cover-Image
DownloadImage(coverUrl, Path.Join(downloadLocation, publication.folderName, $"cover.{extension}"), this.downloadClient);
}
}

View File

@ -2,6 +2,9 @@
namespace Tranga;
/// <summary>
/// Contains information on a Publication (Manga)
/// </summary>
public struct Publication
{
public string sortName { get; }
@ -31,19 +34,25 @@ public struct Publication
this.folderName = string.Concat(sortName.Split(Path.GetInvalidPathChars()));
}
/// <summary>
///
/// </summary>
/// <returns>Serialized JSON String for series.json</returns>
public string GetSeriesInfo()
{
SeriesInfo si = new (new Metadata(this.sortName, this.year.ToString() ?? string.Empty, this.status, this.description ?? ""));
return System.Text.Json.JsonSerializer.Serialize(si);
}
internal struct SeriesInfo
//Only for series.json
private struct SeriesInfo
{
[JsonRequired]public Metadata metadata { get; }
public SeriesInfo(Metadata metadata) => this.metadata = metadata;
}
internal struct Metadata
//Only for series.json
private struct Metadata
{
[JsonRequired]public string name { get; }
[JsonRequired]public string year { get; }

View File

@ -1,9 +1,23 @@
namespace Tranga;
/// <summary>
/// Executes TrangaTasks
/// Based on the TrangaTask.Task a method is called.
/// The chapterCollection is updated with new Publications/Chapters.
/// </summary>
public static class TaskExecutor
{
/// <summary>
/// Executes TrangaTask.
/// </summary>
/// <param name="connectors">List of all available Connectors</param>
/// <param name="trangaTask">Task to execute</param>
/// <param name="chapterCollection">Current chapterCollection to update</param>
/// <exception cref="ArgumentException">Is thrown when there is no Connector available with the name of the TrangaTask.connectorName</exception>
public static void Execute(Connector[] connectors, TrangaTask trangaTask, Dictionary<Publication, List<Chapter>> chapterCollection)
{
//Get Connector from list of available Connectors and the required Connector of the TrangaTask
Connector? connector = connectors.FirstOrDefault(c => c.name == trangaTask.connectorName);
if (connector is null)
throw new ArgumentException($"Connector {trangaTask.connectorName} is not a known connector.");
@ -13,6 +27,7 @@ public static class TaskExecutor
trangaTask.isBeingExecuted = true;
trangaTask.lastExecuted = DateTime.Now;
//Call appropriate Method based on TrangaTask.Task
switch (trangaTask.task)
{
case TrangaTask.Task.DownloadNewChapters:
@ -29,6 +44,11 @@ public static class TaskExecutor
trangaTask.isBeingExecuted = false;
}
/// <summary>
/// Updates the available Publications from a Connector (all of them)
/// </summary>
/// <param name="connector">Connector to receive Publications from</param>
/// <param name="chapterCollection"></param>
private static void UpdatePublications(Connector connector, Dictionary<Publication, List<Chapter>> chapterCollection)
{
Publication[] publications = connector.GetPublications();
@ -36,6 +56,14 @@ public static class TaskExecutor
chapterCollection.TryAdd(publication, new List<Chapter>());
}
/// <summary>
/// Checks for new Chapters and Downloads new ones.
/// If no Chapters had been downloaded previously, download also cover and create series.json
/// </summary>
/// <param name="connector">Connector to use</param>
/// <param name="publication">Publication to check</param>
/// <param name="language">Language to receive chapters for</param>
/// <param name="chapterCollection"></param>
private static void DownloadNewChapters(Connector connector, Publication publication, string language, Dictionary<Publication, List<Chapter>> chapterCollection)
{
List<Chapter> newChapters = UpdateChapters(connector, publication, language, chapterCollection);
@ -45,6 +73,14 @@ public static class TaskExecutor
connector.SaveSeriesInfo(publication);
}
/// <summary>
/// Updates the available Chapters of a Publication
/// </summary>
/// <param name="connector">Connector to use</param>
/// <param name="publication">Publication to check</param>
/// <param name="language">Language to receive chapters for</param>
/// <param name="chapterCollection"></param>
/// <returns>List of Chapters that were previously not in collection</returns>
private static List<Chapter> UpdateChapters(Connector connector, Publication publication, string language, Dictionary<Publication, List<Chapter>> chapterCollection)
{
List<Chapter> newChaptersList = new();

View File

@ -3,6 +3,10 @@ using Tranga.Connectors;
namespace Tranga;
/// <summary>
/// Manages all TrangaTasks.
/// Provides a Threaded environment to execute Tasks, and still manage the Task-Collection
/// </summary>
public class TaskManager
{
private readonly Dictionary<Publication, List<Chapter>> _chapterCollection;
@ -11,6 +15,10 @@ public class TaskManager
private readonly Connector[] connectors;
private readonly string folderPath;
/// <summary>
///
/// </summary>
/// <param name="folderPath">Local path to save data (Manga) to</param>
public TaskManager(string folderPath)
{
this.folderPath = folderPath;
@ -34,8 +42,15 @@ public class TaskManager
}
}
/// <summary>
/// Forces the execution of a given task
/// </summary>
/// <param name="task">Task to execute</param>
public void ExecuteTaskNow(TrangaTask task)
{
if (!this._allTasks.Contains(task))
return;
Task t = new Task(() =>
{
TaskExecutor.Execute(this.connectors, task, this._chapterCollection);
@ -43,13 +58,24 @@ public class TaskManager
t.Start();
}
/// <summary>
/// Creates and adds a new Task to the task-Collection
/// </summary>
/// <param name="task">TrangaTask.Task to later execute</param>
/// <param name="connectorName">Name of the connector to use</param>
/// <param name="publication">Publication to execute Task on, can be null in case of unrelated Task</param>
/// <param name="reoccurrence">Time-Interval between Executions</param>
/// <param name="language">language, should Task require parameter. Can be empty</param>
/// <exception cref="ArgumentException">Is thrown when connectorName is not a available Connector</exception>
public void AddTask(TrangaTask.Task task, string connectorName, Publication? publication, TimeSpan reoccurrence,
string language = "")
{
//Get appropriate Connector from available Connectors for TrangaTask
Connector? connector = connectors.FirstOrDefault(c => c.name == connectorName);
if (connector is null)
throw new ArgumentException($"Connector {connectorName} is not a known connector.");
//Check if same task already exists
if (!_allTasks.Any(trangaTask => trangaTask.task != task && trangaTask.connectorName != connector.name &&
trangaTask.publication?.downloadUrl != publication?.downloadUrl))
{
@ -60,6 +86,12 @@ public class TaskManager
}
}
/// <summary>
/// Removes Task from task-collection
/// </summary>
/// <param name="task">TrangaTask.Task type</param>
/// <param name="connectorName">Name of Connector that was used</param>
/// <param name="publication">Publication that was used</param>
public void RemoveTask(TrangaTask.Task task, string connectorName, Publication? publication)
{
_allTasks.RemoveWhere(trangaTask =>
@ -68,11 +100,19 @@ public class TaskManager
ExportTasks(Directory.GetCurrentDirectory());
}
/// <summary>
///
/// </summary>
/// <returns>All available Connectors</returns>
public Dictionary<string, Connector> GetAvailableConnectors()
{
return this.connectors.ToDictionary(connector => connector.name, connector => connector);
}
/// <summary>
///
/// </summary>
/// <returns>All TrangaTasks in task-collection</returns>
public TrangaTask[] GetAllTasks()
{
TrangaTask[] ret = new TrangaTask[_allTasks.Count];
@ -80,11 +120,19 @@ public class TaskManager
return ret;
}
/// <summary>
///
/// </summary>
/// <returns>All added Publications</returns>
public Publication[] GetAllPublications()
{
return this._chapterCollection.Keys.ToArray();
}
/// <summary>
/// Shuts down the taskManager.
/// </summary>
/// <param name="force">If force is true, tasks are aborted.</param>
public void Shutdown(bool force = false)
{
_continueRunning = false;
@ -96,7 +144,7 @@ public class TaskManager
//Wait for tasks to finish
while(_allTasks.Any(task => task.isBeingExecuted))
Thread.Sleep(10);
Environment.Exit(0);
}
private HashSet<TrangaTask> ImportTasks(string importFolderPath)

View File

@ -2,6 +2,9 @@
namespace Tranga;
/// <summary>
/// Stores information on Task
/// </summary>
public class TrangaTask
{
public TimeSpan reoccurrence { get; }
@ -24,6 +27,10 @@ public class TrangaTask
this.language = language;
}
/// <summary>
///
/// </summary>
/// <returns>True if elapsed time since last execution is greater than set interval</returns>
public bool ShouldExecute()
{
return DateTime.Now.Subtract(this.lastExecuted) > reoccurrence;