glax/tranga-website
1
0
Fork 0
mirror of https://github.com/C9Glax/tranga-website.git synced 2025-06-21 11:05:36 +02:00
Code Issues Packages Projects Releases Wiki Activity

Moved logger to Trangasettings

Browse Source
This commit is contained in:
glax
2023-07-30 17:08:43 +02:00
parent b4bef25a22
commit 000539d6a6
13 changed files with 107 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
Tranga/Connectors/Connector.cs
View File

@ -3,7 +3,6 @@ using System.IO.Compression;
using System.Net;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using Logging;
using Tranga.TrangaTasks;
using static System.IO.UnixFileMode;
@ -18,13 +17,9 @@ public abstract class Connector
protected TrangaSettings settings { get; }
internal DownloadClient downloadClient { get; init; } = null!;
protected readonly Logger? logger;
protected Connector(TrangaSettings settings, Logger? logger = null)
protected Connector(TrangaSettings settings)
{
this.settings = settings;
this.logger = logger;
if (!Directory.Exists(settings.coverImageCache))
Directory.CreateDirectory(settings.coverImageCache);
}
@ -68,11 +63,11 @@ public abstract class Connector
Chapter[] newChapters = this.GetChapters(publication, language);
collection.Add(publication);
NumberFormatInfo decimalPoint = new (){ NumberDecimalSeparator = "." };
logger?.WriteLine(this.GetType().ToString(), "Checking for duplicates");
settings.logger?.WriteLine(this.GetType().ToString(), "Checking for duplicates");
List<Chapter> newChaptersList = newChapters.Where(nChapter =>
float.Parse(nChapter.chapterNumber, decimalPoint) > publication.ignoreChaptersBelow &&
!nChapter.CheckChapterIsDownloaded(settings.downloadLocation)).ToList();
logger?.WriteLine(this.GetType().ToString(), $"{newChaptersList.Count} new chapters.");
settings.logger?.WriteLine(this.GetType().ToString(), $"{newChaptersList.Count} new chapters.");
return newChaptersList;
}
@ -167,19 +162,19 @@ public abstract class Connector
/// <param name="settings">TrangaSettings</param>
public void CopyCoverFromCacheToDownloadLocation(Publication publication, TrangaSettings settings)
{
logger?.WriteLine(this.GetType().ToString(), $"Cloning cover {publication.sortName} -> {publication.internalId}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Cloning cover {publication.sortName} -> {publication.internalId}");
//Check if Publication already has a Folder and cover
string publicationFolder = publication.CreatePublicationFolder(settings.downloadLocation);
DirectoryInfo dirInfo = new (publicationFolder);
if (dirInfo.EnumerateFiles().Any(info => info.Name.Contains("cover", StringComparison.InvariantCultureIgnoreCase)))
{
logger?.WriteLine(this.GetType().ToString(), $"Cover exists {publication.sortName}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Cover exists {publication.sortName}");
return;
}
string fileInCache = Path.Join(settings.coverImageCache, publication.coverFileNameInCache);
string newFilePath = Path.Join(publicationFolder, $"cover.{Path.GetFileName(fileInCache).Split('.')[^1]}" );
logger?.WriteLine(this.GetType().ToString(), $"Cloning cover {fileInCache} -> {newFilePath}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Cloning cover {fileInCache} -> {newFilePath}");
File.Copy(fileInCache, newFilePath, true);
if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
File.SetUnixFileMode(newFilePath, GroupRead | GroupWrite | OtherRead | OtherWrite | UserRead | UserWrite);
@ -217,7 +212,7 @@ public abstract class Connector
{
if (cancellationToken?.IsCancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
logger?.WriteLine("Connector", $"Downloading Images for {saveArchiveFilePath}");
settings.logger?.WriteLine("Connector", $"Downloading Images for {saveArchiveFilePath}");
//Check if Publication Directory already exists
string directoryPath = Path.GetDirectoryName(saveArchiveFilePath)!;
if (!Directory.Exists(directoryPath))
@ -235,7 +230,7 @@ public abstract class Connector
{
string[] split = imageUrl.Split('.');
string extension = split[^1];
logger?.WriteLine("Connector", $"Downloading Image {chapter + 1:000}/{imageUrls.Length:000} {parentTask.publication.sortName} {parentTask.publication.internalId} Vol.{parentTask.chapter.volumeNumber} Ch.{parentTask.chapter.chapterNumber} {parentTask.progress:P2}");
settings.logger?.WriteLine("Connector", $"Downloading Image {chapter + 1:000}/{imageUrls.Length:000} {parentTask.publication.sortName} {parentTask.publication.internalId} Vol.{parentTask.chapter.volumeNumber} Ch.{parentTask.chapter.chapterNumber} {parentTask.progress:P2}");
HttpStatusCode status = DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer);
if ((int)status < 200 || (int)status >= 300)
return status;
@ -247,7 +242,7 @@ public abstract class Connector
if(comicInfoPath is not null)
File.Copy(comicInfoPath, Path.Join(tempFolder, "ComicInfo.xml"));
logger?.WriteLine("Connector", $"Creating archive {saveArchiveFilePath}");
settings.logger?.WriteLine("Connector", $"Creating archive {saveArchiveFilePath}");
//ZIP-it and ship-it
ZipFile.CreateFromDirectory(tempFolder, saveArchiveFilePath);
if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
@ -269,7 +264,7 @@ public abstract class Connector
using MemoryStream ms = new();
coverResult.result.CopyTo(ms);
File.WriteAllBytes(saveImagePath, ms.ToArray());
logger?.WriteLine(this.GetType().ToString(), $"Saving image to {saveImagePath}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Saving image to {saveImagePath}");
return filename;
}
}

25
Tranga/Connectors/MangaDex.cs
View File

@ -2,7 +2,6 @@
using System.Net;
using System.Text.Json;
using System.Text.Json.Nodes;
using Logging;
using Tranga.TrangaTasks;
namespace Tranga.Connectors;
@ -19,7 +18,7 @@ public class MangaDex : Connector
Author,
}
public MangaDex(TrangaSettings settings, Logger? logger = null) : base(settings, logger)
public MangaDex(TrangaSettings settings) : base(settings)
{
name = "MangaDex";
this.downloadClient = new DownloadClient(new Dictionary<byte, int>()
@ -29,12 +28,12 @@ public class MangaDex : Connector
{(byte)RequestType.AtHomeServer, 40},
{(byte)RequestType.CoverUrl, 250},
{(byte)RequestType.Author, 250}
}, logger);
}, settings.logger);
}
protected override Publication[] GetPublicationsInternal(string publicationTitle = "")
{
logger?.WriteLine(this.GetType().ToString(), $"Getting Publications (title={publicationTitle})");
settings.logger?.WriteLine(this.GetType().ToString(), $"Getting Publications (title={publicationTitle})");
const int limit = 100; //How many values we want returned at once
int offset = 0; //"Page"
int total = int.MaxValue; //How many total results are there, is updated on first request
@ -60,7 +59,7 @@ public class MangaDex : Connector
//Loop each Manga and extract information from JSON
foreach (JsonNode? mangeNode in mangaInResult)
{
logger?.WriteLine(this.GetType().ToString(), $"Getting publication data. {++loadedPublicationData}/{total}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Getting publication data. {++loadedPublicationData}/{total}");
JsonObject manga = (JsonObject)mangeNode!;
JsonObject attributes = manga["attributes"]!.AsObject();
@ -147,13 +146,13 @@ public class MangaDex : Connector
}
}
logger?.WriteLine(this.GetType().ToString(), $"Done getting publications (title={publicationTitle})");
settings.logger?.WriteLine(this.GetType().ToString(), $"Done getting publications (title={publicationTitle})");
return publications.ToArray();
}
public override Chapter[] GetChapters(Publication publication, string language = "")
{
logger?.WriteLine(this.GetType().ToString(), $"Getting Chapters for {publication.sortName} {publication.internalId} (language={language})");
settings.logger?.WriteLine(this.GetType().ToString(), $"Getting Chapters for {publication.sortName} {publication.internalId} (language={language})");
const int limit = 100; //How many values we want returned at once
int offset = 0; //"Page"
int total = int.MaxValue; //How many total results are there, is updated on first request
@ -204,7 +203,7 @@ public class MangaDex : Connector
{
NumberDecimalSeparator = "."
};
logger?.WriteLine(this.GetType().ToString(), $"Done getting {chapters.Count} Chapters for {publication.internalId}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Done getting {chapters.Count} Chapters for {publication.internalId}");
return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray();
}
@ -212,7 +211,7 @@ public class MangaDex : Connector
{
if (cancellationToken?.IsCancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
//Request URLs for Chapter-Images
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer);
@ -239,10 +238,10 @@ public class MangaDex : Connector
private string? GetCoverUrl(string publicationId, string? posterId)
{
logger?.WriteLine(this.GetType().ToString(), $"Getting CoverUrl for {publicationId}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Getting CoverUrl for {publicationId}");
if (posterId is null)
{
logger?.WriteLine(this.GetType().ToString(), $"No posterId, aborting");
settings.logger?.WriteLine(this.GetType().ToString(), $"No posterId, aborting");
return null;
}
@ -258,7 +257,7 @@ public class MangaDex : Connector
string fileName = result["data"]!["attributes"]!["fileName"]!.GetValue<string>();
string coverUrl = $"https://uploads.mangadex.org/covers/{publicationId}/{fileName}";
logger?.WriteLine(this.GetType().ToString(), $"Got Cover-Url for {publicationId} -> {coverUrl}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Got Cover-Url for {publicationId} -> {coverUrl}");
return coverUrl;
}
@ -277,7 +276,7 @@ public class MangaDex : Connector
string authorName = result["data"]!["attributes"]!["name"]!.GetValue<string>();
ret.Add(authorName);
logger?.WriteLine(this.GetType().ToString(), $"Got author {authorId} -> {authorName}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Got author {authorId} -> {authorName}");
}
return ret;
}

13
Tranga/Connectors/MangaKatana.cs
View File

@ -2,7 +2,6 @@
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using Logging;
using Tranga.TrangaTasks;
namespace Tranga.Connectors;
@ -11,18 +10,18 @@ public class MangaKatana : Connector
{
public override string name { get; }
public MangaKatana(TrangaSettings settings, Logger? logger = null) : base(settings, logger)
public MangaKatana(TrangaSettings settings) : base(settings)
{
this.name = "MangaKatana";
this.downloadClient = new DownloadClient(new Dictionary<byte, int>()
{
{(byte)1, 60}
}, logger);
}, settings.logger);
}
protected override Publication[] GetPublicationsInternal(string publicationTitle = "")
{
logger?.WriteLine(this.GetType().ToString(), $"Getting Publications (title={publicationTitle})");
settings.logger?.WriteLine(this.GetType().ToString(), $"Getting Publications (title={publicationTitle})");
string sanitizedTitle = string.Concat(Regex.Matches(publicationTitle, "[A-z]* *")).ToLower().Replace(' ', '_');
string requestUrl = $"https://mangakatana.com/?search={sanitizedTitle}&search_by=book_name";
DownloadClient.RequestResult requestResult =
@ -136,7 +135,7 @@ public class MangaKatana : Connector
public override Chapter[] GetChapters(Publication publication, string language = "")
{
logger?.WriteLine(this.GetType().ToString(), $"Getting Chapters for {publication.sortName} {publication.internalId} (language={language})");
settings.logger?.WriteLine(this.GetType().ToString(), $"Getting Chapters for {publication.sortName} {publication.internalId} (language={language})");
string requestUrl = $"https://mangakatana.com/manga/{publication.publicationId}";
// Leaving this in for verification if the page exists
DownloadClient.RequestResult requestResult =
@ -150,7 +149,7 @@ public class MangaKatana : Connector
NumberDecimalSeparator = "."
};
List<Chapter> chapters = ParseChaptersFromHtml(publication, requestUrl);
logger?.WriteLine(this.GetType().ToString(), $"Done getting Chapters for {publication.internalId}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Done getting Chapters for {publication.internalId}");
return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray();
}
@ -183,7 +182,7 @@ public class MangaKatana : Connector
{
if (cancellationToken?.IsCancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
string requestUrl = chapter.url;
// Leaving this in to check if the page exists
DownloadClient.RequestResult requestResult =

13
Tranga/Connectors/Manganato.cs
View File

@ -2,7 +2,6 @@
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using Logging;
using Tranga.TrangaTasks;
namespace Tranga.Connectors;
@ -11,18 +10,18 @@ public class Manganato : Connector
{
public override string name { get; }
public Manganato(TrangaSettings settings, Logger? logger = null) : base(settings, logger)
public Manganato(TrangaSettings settings) : base(settings)
{
this.name = "Manganato";
this.downloadClient = new DownloadClient(new Dictionary<byte, int>()
{
{(byte)1, 60}
}, logger);
}, settings.logger);
}
protected override Publication[] GetPublicationsInternal(string publicationTitle = "")
{
logger?.WriteLine(this.GetType().ToString(), $"Getting Publications (title={publicationTitle})");
settings.logger?.WriteLine(this.GetType().ToString(), $"Getting Publications (title={publicationTitle})");
string sanitizedTitle = string.Join('_', Regex.Matches(publicationTitle, "[A-z]*")).ToLower();
string requestUrl = $"https://manganato.com/search/story/{sanitizedTitle}";
DownloadClient.RequestResult requestResult =
@ -127,7 +126,7 @@ public class Manganato : Connector
public override Chapter[] GetChapters(Publication publication, string language = "")
{
logger?.WriteLine(this.GetType().ToString(), $"Getting Chapters for {publication.sortName} {publication.internalId} (language={language})");
settings.logger?.WriteLine(this.GetType().ToString(), $"Getting Chapters for {publication.sortName} {publication.internalId} (language={language})");
string requestUrl = $"https://chapmanganato.com/{publication.publicationId}";
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, (byte)1);
@ -140,7 +139,7 @@ public class Manganato : Connector
NumberDecimalSeparator = "."
};
List<Chapter> chapters = ParseChaptersFromHtml(publication, requestResult.result);
logger?.WriteLine(this.GetType().ToString(), $"Done getting Chapters for {publication.internalId}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Done getting Chapters for {publication.internalId}");
return chapters.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray();
}
@ -173,7 +172,7 @@ public class Manganato : Connector
{
if (cancellationToken?.IsCancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
string requestUrl = chapter.url;
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, (byte)1);

27
Tranga/Connectors/Mangasee.cs
View File

@ -3,7 +3,6 @@ using System.Net;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using HtmlAgilityPack;
using Logging;
using Newtonsoft.Json;
using PuppeteerSharp;
using Tranga.TrangaTasks;
@ -16,13 +15,13 @@ public class Mangasee : Connector
private IBrowser? _browser = null;
private const string ChromiumVersion = "1154303";
public Mangasee(TrangaSettings settings, Logger? logger = null) : base(settings, logger)
public Mangasee(TrangaSettings settings) : base(settings)
{
this.name = "Mangasee";
this.downloadClient = new DownloadClient(new Dictionary<byte, int>()
{
{ (byte)1, 60 }
}, logger);
}, settings.logger);
Task d = new Task(DownloadBrowser);
d.Start();
@ -35,31 +34,31 @@ public class Mangasee : Connector
browserFetcher.Remove(rev);
if (!browserFetcher.LocalRevisions().Contains(ChromiumVersion))
{
logger?.WriteLine(this.GetType().ToString(), "Downloading headless browser");
settings.logger?.WriteLine(this.GetType().ToString(), "Downloading headless browser");
DateTime last = DateTime.Now.Subtract(TimeSpan.FromSeconds(5));
browserFetcher.DownloadProgressChanged += (sender, args) =>
{
double currentBytes = Convert.ToDouble(args.BytesReceived) / Convert.ToDouble(args.TotalBytesToReceive);
if (args.TotalBytesToReceive == args.BytesReceived)
{
logger?.WriteLine(this.GetType().ToString(), "Browser downloaded.");
settings.logger?.WriteLine(this.GetType().ToString(), "Browser downloaded.");
}
else if (DateTime.Now > last.AddSeconds(5))
{
logger?.WriteLine(this.GetType().ToString(), $"Browser download progress: {currentBytes:P2}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Browser download progress: {currentBytes:P2}");
last = DateTime.Now;
}
};
if (!browserFetcher.CanDownloadAsync(ChromiumVersion).Result)
{
logger?.WriteLine(this.GetType().ToString(), $"Can't download browser version {ChromiumVersion}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Can't download browser version {ChromiumVersion}");
return;
}
await browserFetcher.DownloadAsync(ChromiumVersion);
}
logger?.WriteLine(this.GetType().ToString(), "Starting browser.");
settings.logger?.WriteLine(this.GetType().ToString(), "Starting browser.");
this._browser = await Puppeteer.LaunchAsync(new LaunchOptions
{
Headless = true,
@ -74,7 +73,7 @@ public class Mangasee : Connector
protected override Publication[] GetPublicationsInternal(string publicationTitle = "")
{
logger?.WriteLine(this.GetType().ToString(), $"Getting Publications (title={publicationTitle})");
settings.logger?.WriteLine(this.GetType().ToString(), $"Getting Publications (title={publicationTitle})");
string requestUrl = $"https://mangasee123.com/_search.php";
DownloadClient.RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, (byte)1);
@ -99,7 +98,7 @@ public class Mangasee : Connector
queryFiltered = queryFiltered.Where(item => item.Value >= publicationTitle.Split(' ').Length - 1)
.ToDictionary(item => item.Key, item => item.Value);
logger?.WriteLine(this.GetType().ToString(), $"Got {queryFiltered.Count} Publications (title={publicationTitle})");
settings.logger?.WriteLine(this.GetType().ToString(), $"Got {queryFiltered.Count} Publications (title={publicationTitle})");
HashSet<Publication> ret = new();
List<SearchResultItem> orderedFiltered =
@ -112,7 +111,7 @@ public class Mangasee : Connector
downloadClient.MakeRequest($"https://mangasee123.com/manga/{orderedItem.i}", (byte)1);
if ((int)requestResult.statusCode >= 200 || (int)requestResult.statusCode < 300)
{
logger?.WriteLine(this.GetType().ToString(), $"Retrieving Publication info: {orderedItem.s} {index++}/{orderedFiltered.Count}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Retrieving Publication info: {orderedItem.s} {index++}/{orderedFiltered.Count}");
ret.Add(ParseSinglePublicationFromHtml(requestResult.result, orderedItem.s, orderedItem.i, orderedItem.a));
}
}
@ -230,7 +229,7 @@ public class Mangasee : Connector
{
NumberDecimalSeparator = "."
};
logger?.WriteLine(this.GetType().ToString(), $"Done getting Chapters for {publication.internalId}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Done getting Chapters for {publication.internalId}");
return ret.OrderBy(chapter => Convert.ToSingle(chapter.chapterNumber, chapterNumberFormatInfo)).ToArray();
}
@ -240,13 +239,13 @@ public class Mangasee : Connector
return HttpStatusCode.RequestTimeout;
while (this._browser is null && !(cancellationToken?.IsCancellationRequested??false))
{
logger?.WriteLine(this.GetType().ToString(), "Waiting for headless browser to download...");
settings.logger?.WriteLine(this.GetType().ToString(), "Waiting for headless browser to download...");
Thread.Sleep(1000);
}
if (cancellationToken?.IsCancellationRequested??false)
return HttpStatusCode.RequestTimeout;
logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
settings.logger?.WriteLine(this.GetType().ToString(), $"Downloading Chapter-Info {publication.sortName} {publication.internalId} {chapter.volumeNumber}-{chapter.chapterNumber}");
IPage page = _browser!.NewPageAsync().Result;
IResponse response = page.GoToAsync(chapter.url).Result;
if (response.Ok)