Tranga-Website/Tranga/Connector.cs

298 lines
14 KiB
C#
Raw Normal View History

using System.IO.Compression;
using System.Net;
2023-06-01 18:28:58 +02:00
using System.Runtime.InteropServices;
2023-06-03 22:25:24 +02:00
using System.Text.RegularExpressions;
2023-05-20 00:19:04 +02:00
using System.Xml.Linq;
using Logging;
using Tranga.TrangaTasks;
2023-06-01 18:28:58 +02:00
using static System.IO.UnixFileMode;
namespace Tranga;
/// <summary>
/// Base-Class for all Connectors
/// Provides some methods to be used by all Connectors, as well as a DownloadClient
/// </summary>
public abstract class Connector
{
internal string downloadLocation { get; } //Location of local files
2023-05-22 18:15:24 +02:00
protected DownloadClient downloadClient { get; init; }
protected readonly Logger? logger;
protected readonly string imageCachePath;
2023-05-25 14:23:33 +02:00
protected Connector(string downloadLocation, string imageCachePath, Logger? logger)
{
this.downloadLocation = downloadLocation;
this.logger = logger;
2023-05-22 21:44:52 +02:00
this.downloadClient = new DownloadClient(new Dictionary<byte, int>()
{
//RequestTypes for RateLimits
}, logger);
2023-05-25 14:23:33 +02:00
this.imageCachePath = imageCachePath;
2023-05-31 21:39:18 +02:00
if (!Directory.Exists(imageCachePath))
Directory.CreateDirectory(this.imageCachePath);
}
public abstract string name { get; } //Name of the Connector (e.g. Website)
/// <summary>
/// Returns all Publications with the given string.
/// If the string is empty or null, returns all Publication of the Connector
/// </summary>
/// <param name="publicationTitle">Search-Query</param>
/// <returns>Publications matching the query</returns>
public abstract Publication[] GetPublications(string publicationTitle = "");
/// <summary>
/// Returns all Chapters of the publication in the provided language.
/// If the language is empty or null, returns all Chapters in all Languages.
/// </summary>
/// <param name="publication">Publication to get Chapters for</param>
/// <param name="language">Language of the Chapters</param>
/// <returns>Array of Chapters matching Publication and Language</returns>
public abstract Chapter[] GetChapters(Publication publication, string language = "");
/// <summary>
/// Retrieves the Chapter (+Images) from the website.
2023-05-31 20:39:23 +02:00
/// Should later call DownloadChapterImages to retrieve the individual Images of the Chapter and create .cbz archive.
/// </summary>
/// <param name="publication">Publication that contains Chapter</param>
/// <param name="chapter">Chapter with Images to retrieve</param>
/// <param name="parentTask">Will be used for progress-tracking</param>
public abstract void DownloadChapter(Publication publication, Chapter chapter, DownloadChapterTask parentTask);
2023-05-26 14:07:11 +02:00
/// <summary>
/// Copies the already downloaded cover from cache to downloadLocation
/// </summary>
/// <param name="publication">Publication to retrieve Cover for</param>
2023-05-26 14:07:11 +02:00
/// <param name="settings">TrangaSettings</param>
public void CopyCoverFromCacheToDownloadLocation(Publication publication, TrangaSettings settings)
{
logger?.WriteLine(this.GetType().ToString(), $"Cloning cover {publication.sortName} {publication.internalId}");
//Check if Publication already has a Folder and cover
string publicationFolder = publication.CreatePublicationFolder(downloadLocation);
DirectoryInfo dirInfo = new (publicationFolder);
if (dirInfo.EnumerateFiles().Any(info => info.Name.Contains("cover.")))
{
logger?.WriteLine(this.GetType().ToString(), $"Cover exists {publication.sortName}");
return;
}
string fileInCache = Path.Join(settings.coverImageCache, publication.coverFileNameInCache);
string newFilePath = Path.Join(publicationFolder, $"cover.{Path.GetFileName(fileInCache).Split('.')[^1]}" );
logger?.WriteLine(this.GetType().ToString(), $"Cloning cover {fileInCache} -> {newFilePath}");
File.Copy(fileInCache, newFilePath, true);
2023-06-01 22:32:11 +02:00
if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
File.SetUnixFileMode(newFilePath, GroupRead | GroupWrite | OtherRead | OtherWrite | UserRead | UserWrite);
}
2023-05-20 00:19:04 +02:00
2023-05-20 15:05:41 +02:00
/// <summary>
/// Creates a string containing XML of publication and chapter.
/// See ComicInfo.xml
/// </summary>
/// <returns>XML-string</returns>
protected static string GetComicInfoXmlString(Publication publication, Chapter chapter, Logger? logger)
2023-05-20 00:19:04 +02:00
{
2023-05-26 15:09:26 +02:00
logger?.WriteLine("Connector", $"Creating ComicInfo.Xml for {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
2023-05-20 00:19:04 +02:00
XElement comicInfo = new XElement("ComicInfo",
new XElement("Tags", string.Join(',',publication.tags)),
new XElement("LanguageISO", publication.originalLanguage),
new XElement("Title", chapter.name),
2023-05-22 17:20:07 +02:00
new XElement("Writer", publication.author),
new XElement("Volume", chapter.volumeNumber),
2023-06-01 22:06:10 +02:00
new XElement("Number", chapter.chapterNumber)
2023-05-20 00:19:04 +02:00
);
return comicInfo.ToString();
}
2023-05-20 15:05:41 +02:00
/// <summary>
/// Checks if a chapter-archive is already present
/// </summary>
/// <returns>true if chapter is present</returns>
public bool CheckChapterIsDownloaded(Publication publication, Chapter chapter)
{
2023-06-03 22:25:24 +02:00
Regex legalCharacters = new Regex(@"([A-z]*[0-9]* *\.*-*,*\]*\[*'*\'*\)*\(*~*!*)*");
string oldFilePath = Path.Join(downloadLocation, publication.folderName, $"{string.Concat(legalCharacters.Matches(chapter.name ?? ""))} - V{chapter.volumeNumber}C{chapter.chapterNumber} - {chapter.sortNumber}.cbz");
2023-06-03 22:39:27 +02:00
string oldFilePath2 = Path.Join(downloadLocation, publication.folderName, $"{string.Concat(legalCharacters.Matches(chapter.name ?? ""))} - VC{chapter.chapterNumber} - {chapter.chapterNumber}.cbz");
2023-06-03 22:25:24 +02:00
string newFilePath = GetArchiveFilePath(publication, chapter);
if (File.Exists(oldFilePath))
File.Move(oldFilePath, newFilePath);
2023-06-03 22:55:53 +02:00
else if (File.Exists(oldFilePath2))
2023-06-03 22:25:24 +02:00
File.Move(oldFilePath2, newFilePath);
return File.Exists(newFilePath);
}
2023-05-20 15:05:41 +02:00
/// <summary>
/// Creates full file path of chapter-archive
/// </summary>
/// <returns>Filepath</returns>
protected string GetArchiveFilePath(Publication publication, Chapter chapter)
{
2023-06-03 22:25:24 +02:00
return Path.Join(downloadLocation, publication.folderName, $"{publication.folderName} - {chapter.fileName}.cbz");
}
2023-05-22 18:15:24 +02:00
2023-05-19 20:22:13 +02:00
/// <summary>
/// Downloads Image from URL and saves it to the given path(incl. fileName)
/// </summary>
/// <param name="imageUrl"></param>
/// <param name="fullPath"></param>
2023-05-31 21:18:41 +02:00
/// <param name="requestType">RequestType for Rate-Limit</param>
/// <param name="referrer">referrer used in html request header</param>
2023-06-01 13:13:53 +02:00
private void DownloadImage(string imageUrl, string fullPath, byte requestType, string? referrer = null)
{
2023-06-01 13:13:53 +02:00
DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType, referrer);
2023-06-01 21:16:57 +02:00
if (requestResult.result != Stream.Null)
{
byte[] buffer = new byte[requestResult.result.Length];
requestResult.result.ReadExactly(buffer, 0, buffer.Length);
File.WriteAllBytes(fullPath, buffer);
}else
logger?.WriteLine(this.GetType().ToString(), "No Stream-Content in result.");
}
2023-05-22 18:15:24 +02:00
2023-05-19 20:22:13 +02:00
/// <summary>
/// Downloads all Images from URLs, Compresses to zip(cbz) and saves.
/// </summary>
/// <param name="imageUrls">List of URLs to download Images from</param>
/// <param name="saveArchiveFilePath">Full path to save archive to (without file ending .cbz)</param>
/// <param name="parentTask">Used for progress tracking</param>
2023-05-20 01:06:12 +02:00
/// <param name="comicInfoPath">Path of the generate Chapter ComicInfo.xml, if it was generated</param>
2023-05-22 18:15:24 +02:00
/// <param name="requestType">RequestType for RateLimits</param>
/// <param name="referrer">Used in http request header</param>
protected void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, DownloadChapterTask parentTask, string? comicInfoPath = null, string? referrer = null)
{
2023-05-26 15:09:26 +02:00
logger?.WriteLine("Connector", $"Downloading Images for {saveArchiveFilePath}");
2023-05-19 20:22:13 +02:00
//Check if Publication Directory already exists
2023-05-22 17:09:47 +02:00
string directoryPath = Path.GetDirectoryName(saveArchiveFilePath)!;
if (!Directory.Exists(directoryPath))
Directory.CreateDirectory(directoryPath);
2023-05-22 17:09:47 +02:00
if (File.Exists(saveArchiveFilePath)) //Don't download twice.
return;
2023-05-19 20:22:13 +02:00
//Create a temporary folder to store images
string tempFolder = Directory.CreateTempSubdirectory().FullName;
int chapter = 0;
2023-05-19 20:22:13 +02:00
//Download all Images to temporary Folder
foreach (string imageUrl in imageUrls)
{
string[] split = imageUrl.Split('.');
string extension = split[^1];
logger?.WriteLine("Connector", $"Downloading Image {chapter + 1:000}/{imageUrls.Length:000} {parentTask.publication.sortName} {parentTask.publication.internalId} Vol.{parentTask.chapter.volumeNumber} Ch.{parentTask.chapter.chapterNumber} {parentTask.progress:P2}");
2023-06-01 13:13:53 +02:00
DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer);
parentTask.progress += 1f / imageUrls.Length;
}
2023-05-20 00:19:04 +02:00
if(comicInfoPath is not null)
File.Copy(comicInfoPath, Path.Join(tempFolder, "ComicInfo.xml"));
2023-05-26 15:09:26 +02:00
logger?.WriteLine("Connector", $"Creating archive {saveArchiveFilePath}");
2023-05-19 20:22:13 +02:00
//ZIP-it and ship-it
2023-05-22 17:09:47 +02:00
ZipFile.CreateFromDirectory(tempFolder, saveArchiveFilePath);
2023-06-01 18:28:58 +02:00
if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
File.SetUnixFileMode(saveArchiveFilePath, GroupRead | GroupWrite | OtherRead | OtherWrite | UserRead | UserWrite);
Directory.Delete(tempFolder, true); //Cleanup
}
2023-06-01 13:13:53 +02:00
protected string SaveCoverImageToCache(string url, byte requestType)
{
string[] split = url.Split('/');
string filename = split[^1];
string saveImagePath = Path.Join(imageCachePath, filename);
if (File.Exists(saveImagePath))
return filename;
DownloadClient.RequestResult coverResult = downloadClient.MakeRequest(url, requestType);
using MemoryStream ms = new();
coverResult.result.CopyTo(ms);
File.WriteAllBytes(saveImagePath, ms.ToArray());
logger?.WriteLine(this.GetType().ToString(), $"Saving image to {saveImagePath}");
return filename;
}
protected class DownloadClient
{
2023-05-18 18:55:11 +02:00
private static readonly HttpClient Client = new();
2023-05-22 18:15:24 +02:00
private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit;
private readonly Dictionary<byte, TimeSpan> _rateLimit;
private Logger? logger;
2023-05-22 18:15:24 +02:00
2023-05-19 20:22:13 +02:00
/// <summary>
/// Creates a httpClient
/// </summary>
2023-05-22 18:15:24 +02:00
/// <param name="rateLimitRequestsPerMinute">Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType</param>
2023-05-31 21:18:41 +02:00
/// <param name="logger"></param>
public DownloadClient(Dictionary<byte, int> rateLimitRequestsPerMinute, Logger? logger)
{
this.logger = logger;
2023-05-22 18:15:24 +02:00
_lastExecutedRateLimit = new();
_rateLimit = new();
2023-05-22 18:15:24 +02:00
foreach(KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute)
_rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value));
}
2023-05-22 18:15:24 +02:00
2023-05-19 20:22:13 +02:00
/// <summary>
/// Request Webpage
/// </summary>
/// <param name="url"></param>
2023-05-22 18:15:24 +02:00
/// <param name="requestType">For RateLimits: Same Endpoints use same type</param>
/// <param name="referrer">Used in http request header</param>
2023-05-19 20:22:13 +02:00
/// <returns>RequestResult with StatusCode and Stream of received data</returns>
2023-06-01 13:13:53 +02:00
public RequestResult MakeRequest(string url, byte requestType, string? referrer = null)
{
if (_rateLimit.TryGetValue(requestType, out TimeSpan value))
2023-05-22 18:15:24 +02:00
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
else
{
logger?.WriteLine(this.GetType().ToString(), "RequestType not configured for rate-limit.");
2023-05-22 18:15:24 +02:00
return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null);
}
TimeSpan rateLimitTimeout = _rateLimit[requestType]
.Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]));
2023-05-22 18:15:24 +02:00
2023-05-22 21:41:11 +02:00
if(rateLimitTimeout > TimeSpan.Zero)
Thread.Sleep(rateLimitTimeout);
HttpResponseMessage? response = null;
while (response is null)
{
try
{
HttpRequestMessage requestMessage = new(HttpMethod.Get, url);
2023-06-01 13:13:53 +02:00
if(referrer is not null)
requestMessage.Headers.Referrer = new Uri(referrer);
_lastExecutedRateLimit[requestType] = DateTime.Now;
response = Client.Send(requestMessage);
}
catch (HttpRequestException e)
{
2023-05-22 21:44:52 +02:00
logger?.WriteLine(this.GetType().ToString(), e.Message);
2023-06-01 21:16:57 +02:00
logger?.WriteLine(this.GetType().ToString(), $"Waiting {_rateLimit[requestType] * 2}... Retrying.");
Thread.Sleep(_rateLimit[requestType] * 2);
}
}
Stream resultString = response.IsSuccessStatusCode ? response.Content.ReadAsStream() : Stream.Null;
if (!response.IsSuccessStatusCode)
logger?.WriteLine(this.GetType().ToString(), $"Request-Error {response.StatusCode}: {response.ReasonPhrase}");
return new RequestResult(response.StatusCode, resultString);
}
public struct RequestResult
{
public HttpStatusCode statusCode { get; }
public Stream result { get; }
public RequestResult(HttpStatusCode statusCode, Stream result)
{
this.statusCode = statusCode;
this.result = result;
}
}
}
}