using System.IO.Compression; using System.Net; using System.Xml.Linq; using Logging; namespace Tranga; /// /// Base-Class for all Connectors /// Provides some methods to be used by all Connectors, as well as a DownloadClient /// public abstract class Connector { internal string downloadLocation { get; } //Location of local files protected DownloadClient downloadClient { get; init; } protected readonly Logger? logger; protected readonly string imageCachePath; protected Connector(string downloadLocation, string imageCachePath, Logger? logger) { this.downloadLocation = downloadLocation; this.logger = logger; this.downloadClient = new DownloadClient(new Dictionary() { //RequestTypes for RateLimits }, logger); this.imageCachePath = imageCachePath; } public abstract string name { get; } //Name of the Connector (e.g. Website) /// /// Returns all Publications with the given string. /// If the string is empty or null, returns all Publication of the Connector /// /// Search-Query /// Publications matching the query public abstract Publication[] GetPublications(string publicationTitle = ""); /// /// Returns all Chapters of the publication in the provided language. /// If the language is empty or null, returns all Chapters in all Languages. /// /// Publication to get Chapters for /// Language of the Chapters /// Array of Chapters matching Publication and Language public abstract Chapter[] GetChapters(Publication publication, string language = ""); /// /// Retrieves the Chapter (+Images) from the website. /// Should later call DownloadChapterImages to retrieve the individual Images of the Chapter and create .cbz archive. /// /// Publication that contains Chapter /// Chapter with Images to retrieve public abstract void DownloadChapter(Publication publication, Chapter chapter); /// /// Copies the already downloaded cover from cache to downloadLocation /// /// Publication to retrieve Cover for /// TrangaSettings public void CopyCoverFromCacheToDownloadLocation(Publication publication, TrangaSettings settings) { logger?.WriteLine(this.GetType().ToString(), $"Cloning cover {publication.sortName} {publication.internalId}"); //Check if Publication already has a Folder and cover string publicationFolder = Path.Join(downloadLocation, publication.folderName); if(!Directory.Exists(publicationFolder)) Directory.CreateDirectory(publicationFolder); DirectoryInfo dirInfo = new (publicationFolder); if (dirInfo.EnumerateFiles().Any(info => info.Name.Contains("cover."))) { logger?.WriteLine(this.GetType().ToString(), $"Cover exists {publication.sortName}"); return; } string fileInCache = Path.Join(settings.coverImageCache, publication.coverFileNameInCache); string newFilePath = Path.Join(publicationFolder, $"cover.{Path.GetFileName(fileInCache).Split('.')[^1]}" ); logger?.WriteLine(this.GetType().ToString(), $"Cloning cover {fileInCache} -> {newFilePath}"); File.Copy(fileInCache, newFilePath, true); } /// /// Creates a string containing XML of publication and chapter. /// See ComicInfo.xml /// /// XML-string protected static string GetComicInfoXmlString(Publication publication, Chapter chapter, Logger? logger) { logger?.WriteLine("Connector", $"Creating ComicInfo.Xml for {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}"); XElement comicInfo = new XElement("ComicInfo", new XElement("Tags", string.Join(',',publication.tags)), new XElement("LanguageISO", publication.originalLanguage), new XElement("Title", chapter.name), new XElement("Writer", publication.author), new XElement("Volume", chapter.volumeNumber), new XElement("Number", chapter.chapterNumber) //TODO check if this is correct at some point ); return comicInfo.ToString(); } /// /// Checks if a chapter-archive is already present /// /// true if chapter is present public bool CheckChapterIsDownloaded(Publication publication, Chapter chapter) { return File.Exists(GetArchiveFilePath(publication, chapter)); } /// /// Creates full file path of chapter-archive /// /// Filepath protected string GetArchiveFilePath(Publication publication, Chapter chapter) { return Path.Join(downloadLocation, publication.folderName, $"{chapter.fileName}.cbz"); } /// /// Downloads Image from URL and saves it to the given path(incl. fileName) /// /// /// /// Requesttype for ratelimit private void DownloadImage(string imageUrl, string fullPath, byte requestType) { DownloadClient.RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType); byte[] buffer = new byte[requestResult.result.Length]; requestResult.result.ReadExactly(buffer, 0, buffer.Length); File.WriteAllBytes(fullPath, buffer); } /// /// Downloads all Images from URLs, Compresses to zip(cbz) and saves. /// /// List of URLs to download Images from /// Full path to save archive to (without file ending .cbz) /// Path of the generate Chapter ComicInfo.xml, if it was generated /// RequestType for RateLimits protected void DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, string? comicInfoPath = null) { logger?.WriteLine("Connector", $"Downloading Images for {saveArchiveFilePath}"); //Check if Publication Directory already exists string directoryPath = Path.GetDirectoryName(saveArchiveFilePath)!; if (!Directory.Exists(directoryPath)) Directory.CreateDirectory(directoryPath); if (File.Exists(saveArchiveFilePath)) //Don't download twice. return; //Create a temporary folder to store images string tempFolder = Directory.CreateTempSubdirectory().FullName; int chapter = 0; //Download all Images to temporary Folder foreach (string imageUrl in imageUrls) { string[] split = imageUrl.Split('.'); string extension = split[^1]; logger?.WriteLine("Connector", $"Downloading Image {chapter + 1}/{imageUrls.Length}"); DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType); } if(comicInfoPath is not null) File.Copy(comicInfoPath, Path.Join(tempFolder, "ComicInfo.xml")); logger?.WriteLine("Connector", $"Creating archive {saveArchiveFilePath}"); //ZIP-it and ship-it ZipFile.CreateFromDirectory(tempFolder, saveArchiveFilePath); Directory.Delete(tempFolder, true); //Cleanup } protected class DownloadClient { private static readonly HttpClient Client = new(); private readonly Dictionary _lastExecutedRateLimit; private readonly Dictionary _rateLimit; private Logger? logger; /// /// Creates a httpClient /// /// minimum delay between requests (to avoid spam) /// Rate limits for requests. byte is RequestType, int maximum requests per minute for RequestType public DownloadClient(Dictionary rateLimitRequestsPerMinute, Logger? logger) { this.logger = logger; _lastExecutedRateLimit = new(); _rateLimit = new(); foreach(KeyValuePair limit in rateLimitRequestsPerMinute) _rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value)); } /// /// Request Webpage /// /// /// For RateLimits: Same Endpoints use same type /// RequestResult with StatusCode and Stream of received data public RequestResult MakeRequest(string url, byte requestType) { if (_rateLimit.TryGetValue(requestType, out TimeSpan value)) _lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value)); else { logger?.WriteLine(this.GetType().ToString(), "RequestType not configured for rate-limit."); return new RequestResult(HttpStatusCode.NotAcceptable, Stream.Null); } TimeSpan rateLimitTimeout = _rateLimit[requestType] .Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType])); if(rateLimitTimeout > TimeSpan.Zero) Thread.Sleep(rateLimitTimeout); HttpResponseMessage? response = null; while (response is null) { try { HttpRequestMessage requestMessage = new(HttpMethod.Get, url); _lastExecutedRateLimit[requestType] = DateTime.Now; response = Client.Send(requestMessage); } catch (HttpRequestException e) { logger?.WriteLine(this.GetType().ToString(), e.Message); logger?.WriteLine(this.GetType().ToString(), $"Waiting {_rateLimit[requestType] * 2}"); Thread.Sleep(_rateLimit[requestType] * 2); } } Stream resultString = response.IsSuccessStatusCode ? response.Content.ReadAsStream() : Stream.Null; if (!response.IsSuccessStatusCode) logger?.WriteLine(this.GetType().ToString(), $"Request-Error {response.StatusCode}: {response.ReasonPhrase}"); return new RequestResult(response.StatusCode, resultString); } public struct RequestResult { public HttpStatusCode statusCode { get; } public Stream result { get; } public RequestResult(HttpStatusCode statusCode, Stream result) { this.statusCode = statusCode; this.result = result; } } } }