mirror of
https://github.com/C9Glax/tranga.git
synced 2025-07-01 00:14:17 +02:00
Compare commits
229 Commits
d0f9a4102c
...
cuttingedg
Author | SHA1 | Date | |
---|---|---|---|
12a542da39 | |||
3f5c9d0ca1 | |||
538825f0ef | |||
f0de0a29da | |||
d4227f2b8f | |||
cd00d35f22 | |||
4ef3e877ce | |||
7dba2518f9 | |||
7506a0201e | |||
91fb815153 | |||
6faf8bc733 | |||
bdff5b7aec | |||
5af8060d7b | |||
6ed8ff1d52 | |||
3324ed6e4a | |||
67fd9d284b | |||
08f26dd21d | |||
89ed500751 | |||
b00b0ee030 | |||
e47c52ad48 | |||
293f0af8e3 | |||
ebfa34e386 | |||
14524407f9 | |||
d56f0b383a | |||
70391c83c1 | |||
dc7696ee26 | |||
49dab9a670 | |||
c9bc79fbd5 | |||
83ce315f87 | |||
59511056d0 | |||
ed3ca5dba8 | |||
8df05d7e8a | |||
95d1e37b47 | |||
b6494ab7f9 | |||
1d1d01b6e5 | |||
5bb4977876 | |||
c6bb1c9180 | |||
9a066e7ac7 | |||
4bafffded4 | |||
942b43da67 | |||
ce5538b352 | |||
0cfdf17bd4 | |||
0c48c1e020 | |||
0638e75ed6 | |||
5a4bc1c6de | |||
71f663ca2f | |||
1b61a16061 | |||
db81fdce39 | |||
fdb5451162 | |||
6b7632b071 | |||
06c080dfce | |||
8130e11a9c | |||
659a42d370 | |||
9cef068785 | |||
4ad3149523 | |||
e6d40a7b36 | |||
a95cb90561 | |||
603e1b41d9 | |||
bb8a514830 | |||
edacaaba8a | |||
d97da26994 | |||
8b923d73c4 | |||
814efd3528 | |||
2cd5d8bc4f | |||
5a864ab9b7 | |||
c700974693 | |||
553b5558d3 | |||
c9bbfee26b | |||
6e869eeb0d | |||
be7da69dbd | |||
7f13d9b1e6 | |||
0c9e3205c2 | |||
8c3b70b32e | |||
4f7031ecfc | |||
f7a285aabd | |||
786482398c | |||
7921dcb1cb | |||
d0c9313279 | |||
58cf4cf4e0 | |||
280d715a7c | |||
d0b775444d | |||
b4edcccafe | |||
268441a47d | |||
1701881f4b | |||
78a9322036 | |||
e5be5703f8 | |||
cc32b3dfae | |||
ce217aae4f | |||
123a8b06b2 | |||
2350c5a04b | |||
f532e2ff76 | |||
3abf7224d0 | |||
b39dbd5671 | |||
375fad0c21 | |||
ee0d17c24f | |||
36ab3c3fdb | |||
c3d60c6586 | |||
6aa8413c40 | |||
b96ae4a2d2 | |||
3a25c0b221 | |||
e1f1a05724 | |||
72d9bda0e8 | |||
a40a9c84df | |||
825b945ad1 | |||
b8c624f3ea | |||
93cfdddd19 | |||
4c8d9bfaf2 | |||
dd988658c0 | |||
cf4c84a47f | |||
5d9bfc3adf | |||
5a770c8e9f | |||
e3bd7620aa | |||
428d6e13d1 | |||
1e6a65c0fd | |||
025d43b752 | |||
113c0abba7 | |||
747df0bde5 | |||
463f360808 | |||
85d7c07b13 | |||
553f56ecaf | |||
9cc4f8c090 | |||
204fb7614d | |||
d6e73ffcdf | |||
5a8202f872 | |||
55cc2a2e84 | |||
b619109ea1 | |||
72943330c3 | |||
38bc1e4d53 | |||
47479f7a0d | |||
b2381be860 | |||
657e1b338b | |||
ee265a7519 | |||
5b0624654b | |||
a75549c699 | |||
f46244cb9c | |||
9db3f1b0da | |||
dc9cd4b1dd | |||
3566ad774d | |||
94b81969c7 | |||
bd8cb86c52 | |||
34c5436b33 | |||
4690394437 | |||
02cf8578c9 | |||
067497ddd0 | |||
4b88cdbd90 | |||
420013f07b | |||
8cee11aa22 | |||
198bbdcf94 | |||
c58adf64fa | |||
957debea01 | |||
5186ae66c9 | |||
c35e1ef517 | |||
8f6891142b | |||
b52e6d4908 | |||
30c44760e7 | |||
a3ae3c320d | |||
ea262889e6 | |||
445542b653 | |||
b7718220ef | |||
34c62e8658 | |||
a9fcc93670 | |||
68d7ef258f | |||
fdea4f5ea5 | |||
ac3039e587 | |||
3829a1cf26 | |||
c3daa0b751 | |||
3a072beea3 | |||
8e6f2798a9 | |||
9cbde9a6b4 | |||
0870aa9fdb | |||
172650e644 | |||
52ff2e54a8 | |||
61d80a93cf | |||
7be3ee52e9 | |||
981eb0fd9f | |||
47f3044a6d | |||
6d03cc5f8d | |||
290c405f52 | |||
fcdbd32872 | |||
eb6c37cc53 | |||
d922842186 | |||
69323d6d60 | |||
46a0fb8c48 | |||
ec8eb40941 | |||
d2074fae35 | |||
713bbc230f | |||
32ab9a552f | |||
c11c68d6d7 | |||
09fdb6e5f1 | |||
e86ad03b1e | |||
9dfbe89e87 | |||
98e75af486 | |||
e2f5c3badc | |||
cda07bb9aa | |||
7c18466e95 | |||
ce1c4d3f65 | |||
52d0489a1b | |||
f89aea6ac8 | |||
5f05ba1049 | |||
a20ee01cfa | |||
cf5cbba9a8 | |||
600b56033d | |||
fdea3659f1 | |||
7f3754fb64 | |||
2dac5db4da | |||
3456fc6564 | |||
35f2625f05 | |||
0b9948e367 | |||
96f3dbce65 | |||
895128a462 | |||
a94186455b | |||
7d3deee74c | |||
5980b64caa | |||
cbecb257ef | |||
8316ed08a7 | |||
7ff9ac53ee | |||
6faaaf4139 | |||
9b8b80cd24 | |||
15f3e2b8ec | |||
2be29e4019 | |||
e8dbf7a718 | |||
a968f4328d | |||
398b6fff05 | |||
f5da2f8526 | |||
73093ab86c | |||
fccaf9fcbe | |||
3122aa32e8 | |||
02fad2dd44 | |||
e0a7d1a187 |
@ -22,4 +22,6 @@
|
||||
**/secrets.dev.yaml
|
||||
**/values.dev.yaml
|
||||
LICENSE
|
||||
README.md
|
||||
README.md
|
||||
Manga
|
||||
settings
|
||||
|
4
.github/ISSUE_TEMPLATE/new_connector.yml
vendored
4
.github/ISSUE_TEMPLATE/new_connector.yml
vendored
@ -12,7 +12,7 @@ body:
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Is the Website free to access?
|
||||
description: We can't support pay-to-use sites.
|
||||
description: We can't support pay-to-use sites, or captcha-proxied sites as Cloudflare.
|
||||
options:
|
||||
- label: The Website is freely accessible.
|
||||
required: true
|
||||
@ -20,4 +20,4 @@ body:
|
||||
attributes:
|
||||
label: Anything else?
|
||||
validations:
|
||||
required: false
|
||||
required: false
|
||||
|
43
.github/workflows/docker-base.yml
vendored
43
.github/workflows/docker-base.yml
vendored
@ -1,43 +0,0 @@
|
||||
name: Docker Image CI
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# https://github.com/docker/setup-qemu-action#usage
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3.2.0
|
||||
|
||||
# https://github.com/marketplace/actions/docker-setup-buildx
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3.6.1
|
||||
|
||||
# https://github.com/docker/login-action#docker-hub
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
# https://github.com/docker/build-push-action#multi-platform-image
|
||||
- name: Build and push base
|
||||
uses: docker/build-push-action@v6.6.1
|
||||
with:
|
||||
context: ./
|
||||
file: ./Dockerfile-base
|
||||
#platforms: linux/amd64,linux/arm64,linux/riscv64,linux/ppc64le,linux/s390x,linux/386,linux/mips64le,linux/mips64,linux/arm/v7,linux/arm/v6
|
||||
platforms: linux/amd64
|
||||
pull: true
|
||||
push: true
|
||||
tags: |
|
||||
glax/tranga-base:latest
|
12
.github/workflows/docker-image-cuttingedge.yml
vendored
12
.github/workflows/docker-image-cuttingedge.yml
vendored
@ -11,18 +11,18 @@ jobs:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# https://github.com/docker/setup-qemu-action#usage
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3.2.0
|
||||
uses: docker/setup-qemu-action@v3.6.0
|
||||
|
||||
# https://github.com/marketplace/actions/docker-setup-buildx
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3.6.1
|
||||
uses: docker/setup-buildx-action@v3.11.0
|
||||
|
||||
# https://github.com/docker/login-action#docker-hub
|
||||
- name: Login to Docker Hub
|
||||
@ -33,13 +33,13 @@ jobs:
|
||||
|
||||
# https://github.com/docker/build-push-action#multi-platform-image
|
||||
- name: Build and push API
|
||||
uses: docker/build-push-action@v6.6.1
|
||||
uses: docker/build-push-action@v6.18.0
|
||||
with:
|
||||
context: ./
|
||||
file: ./Dockerfile
|
||||
#platforms: linux/amd64,linux/arm64,linux/riscv64,linux/ppc64le,linux/s390x,linux/386,linux/mips64le,linux/mips64,linux/arm/v7,linux/arm/v6
|
||||
platforms: linux/amd64
|
||||
platforms: linux/amd64,linux/arm64
|
||||
pull: true
|
||||
push: true
|
||||
tags: |
|
||||
glax/tranga-api:cuttingedge
|
||||
glax/tranga-api:cuttingedge
|
||||
|
45
.github/workflows/docker-image-dev.yml
vendored
45
.github/workflows/docker-image-dev.yml
vendored
@ -1,45 +0,0 @@
|
||||
name: Docker Image CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "dev" ]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# https://github.com/docker/setup-qemu-action#usage
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3.2.0
|
||||
|
||||
# https://github.com/marketplace/actions/docker-setup-buildx
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3.6.1
|
||||
|
||||
# https://github.com/docker/login-action#docker-hub
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
# https://github.com/docker/build-push-action#multi-platform-image
|
||||
- name: Build and push API
|
||||
uses: docker/build-push-action@v6.6.1
|
||||
with:
|
||||
context: ./
|
||||
file: ./Dockerfile
|
||||
#platforms: linux/amd64,linux/arm64,linux/riscv64,linux/ppc64le,linux/s390x,linux/386,linux/mips64le,linux/mips64,linux/arm/v7,linux/arm/v6
|
||||
platforms: linux/amd64
|
||||
pull: true
|
||||
push: true
|
||||
tags: |
|
||||
glax/tranga-api:dev
|
8
.github/workflows/docker-image-master.yml
vendored
8
.github/workflows/docker-image-master.yml
vendored
@ -17,12 +17,12 @@ jobs:
|
||||
|
||||
# https://github.com/docker/setup-qemu-action#usage
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3.2.0
|
||||
uses: docker/setup-qemu-action@v3.6.0
|
||||
|
||||
# https://github.com/marketplace/actions/docker-setup-buildx
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3.6.1
|
||||
uses: docker/setup-buildx-action@v3.11.0
|
||||
|
||||
# https://github.com/docker/login-action#docker-hub
|
||||
- name: Login to Docker Hub
|
||||
@ -33,12 +33,12 @@ jobs:
|
||||
|
||||
# https://github.com/docker/build-push-action#multi-platform-image
|
||||
- name: Build and push API
|
||||
uses: docker/build-push-action@v6.6.1
|
||||
uses: docker/build-push-action@v6.18.0
|
||||
with:
|
||||
context: ./
|
||||
file: ./Dockerfile
|
||||
#platforms: linux/amd64,linux/arm64,linux/riscv64,linux/ppc64le,linux/s390x,linux/386,linux/mips64le,linux/mips64,linux/arm/v7,linux/arm/v6
|
||||
platforms: linux/amd64
|
||||
platforms: linux/amd64,linux/arm64
|
||||
pull: true
|
||||
push: true
|
||||
tags: |
|
||||
|
10
.github/workflows/docker-image-serverv2.yml
vendored
10
.github/workflows/docker-image-serverv2.yml
vendored
@ -2,7 +2,7 @@ name: Docker Image CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "Server-V2" ]
|
||||
branches: [ "postgres-Server-V2" ]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
@ -17,12 +17,12 @@ jobs:
|
||||
|
||||
# https://github.com/docker/setup-qemu-action#usage
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3.2.0
|
||||
uses: docker/setup-qemu-action@v3.6.0
|
||||
|
||||
# https://github.com/marketplace/actions/docker-setup-buildx
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3.6.1
|
||||
uses: docker/setup-buildx-action@v3.11.0
|
||||
|
||||
# https://github.com/docker/login-action#docker-hub
|
||||
- name: Login to Docker Hub
|
||||
@ -33,12 +33,12 @@ jobs:
|
||||
|
||||
# https://github.com/docker/build-push-action#multi-platform-image
|
||||
- name: Build and push API
|
||||
uses: docker/build-push-action@v6.6.1
|
||||
uses: docker/build-push-action@v6.18.0
|
||||
with:
|
||||
context: ./
|
||||
file: ./Dockerfile
|
||||
#platforms: linux/amd64,linux/arm64,linux/riscv64,linux/ppc64le,linux/s390x,linux/386,linux/mips64le,linux/mips64,linux/arm/v7,linux/arm/v6
|
||||
platforms: linux/amd64
|
||||
platforms: linux/amd64,linux/arm64
|
||||
pull: true
|
||||
push: true
|
||||
tags: |
|
||||
|
5
.gitignore
vendored
5
.gitignore
vendored
@ -19,4 +19,7 @@ riderModule.iml
|
||||
/.idea
|
||||
cover.jpg
|
||||
cover.png
|
||||
/.vscode
|
||||
/.vscode
|
||||
/Manga
|
||||
/settings
|
||||
*.DotSettings.user
|
@ -2,13 +2,14 @@
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<LangVersion>12</LangVersion>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Spectre.Console.Cli" Version="0.47.1-preview.0.11" />
|
||||
<PackageReference Include="Spectre.Console.Cli" Version="0.49.1" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
49
Dockerfile
49
Dockerfile
@ -1,29 +1,44 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
ARG DOTNET=8.0
|
||||
|
||||
FROM mcr.microsoft.com/dotnet/sdk:7.0 as build-env
|
||||
FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/runtime:$DOTNET AS base
|
||||
WORKDIR /publish
|
||||
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
|
||||
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
|
||||
ENV XDG_CONFIG_HOME=/tmp/.chromium
|
||||
ENV XDG_CACHE_HOME=/tmp/.chromium
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y libx11-6 libx11-xcb1 libatk1.0-0 libgtk-3-0 libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 libxrandr2 libgbm1 libpango-1.0-0 libcairo2 libasound2 libxshmfence1 libnss3 chromium \
|
||||
&& apt-get autopurge -y \
|
||||
&& apt-get autoclean -y
|
||||
|
||||
FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:$DOTNET AS build-env
|
||||
WORKDIR /src
|
||||
COPY CLI /src/CLI
|
||||
COPY Tranga /src/Tranga
|
||||
COPY Logging /src/Logging
|
||||
COPY Tranga.sln /src
|
||||
RUN dotnet restore /src/Tranga/Tranga.csproj
|
||||
RUN dotnet publish -c Release -o /publish
|
||||
|
||||
FROM glax/tranga-base:latest as runtime
|
||||
COPY Tranga.sln /src
|
||||
COPY CLI/CLI.csproj /src/CLI/CLI.csproj
|
||||
COPY Logging/Logging.csproj /src/Logging/Logging.csproj
|
||||
COPY Tranga/Tranga.csproj /src/Tranga/Tranga.csproj
|
||||
RUN dotnet restore /src/Tranga.sln
|
||||
|
||||
COPY . /src/
|
||||
RUN dotnet publish -c Release --property:OutputPath=/publish -maxcpucount:1
|
||||
|
||||
FROM --platform=$TARGETPLATFORM base AS runtime
|
||||
EXPOSE 6531
|
||||
ARG UNAME=tranga
|
||||
ARG UID=1000
|
||||
ARG GID=1000
|
||||
RUN groupadd -g $GID -o $UNAME
|
||||
RUN useradd -m -u $UID -g $GID -o -s /bin/bash $UNAME
|
||||
RUN mkdir /usr/share/tranga-api
|
||||
RUN mkdir /Manga
|
||||
RUN chown 1000:1000 /usr/share/tranga-api
|
||||
RUN chown 1000:1000 /Manga
|
||||
RUN groupadd -g $GID -o $UNAME \
|
||||
&& useradd -m -u $UID -g $GID -o -s /bin/bash $UNAME \
|
||||
&& mkdir /usr/share/tranga-api \
|
||||
&& mkdir /Manga \
|
||||
&& chown 1000:1000 /usr/share/tranga-api \
|
||||
&& chown 1000:1000 /Manga
|
||||
USER $UNAME
|
||||
|
||||
WORKDIR /publish
|
||||
COPY --from=build-env /publish .
|
||||
COPY --chown=1000:1000 --from=build-env /publish .
|
||||
USER 0
|
||||
RUN chown 1000:1000 /publish
|
||||
ENTRYPOINT ["dotnet", "/publish/Tranga.dll", "-f", "-c", "-l", "/usr/share/tranga-api/logs"]
|
||||
ENTRYPOINT ["dotnet", "/publish/Tranga.dll"]
|
||||
CMD ["-f", "-c", "-l", "/usr/share/tranga-api/logs"]
|
@ -1,8 +0,0 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
#FROM mcr.microsoft.com/dotnet/aspnet:7.0 as runtime
|
||||
FROM mcr.microsoft.com/dotnet/runtime:7.0 as runtime
|
||||
WORKDIR /publish
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y libx11-6 libx11-xcb1 libatk1.0-0 libgtk-3-0 libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 libxrandr2 libgbm1 libpango-1.0-0 libcairo2 libasound2 libxshmfence1 libnss3
|
||||
RUN apt-get autopurge -y
|
||||
RUN apt-get autoclean -y
|
@ -1,9 +1,10 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<LangVersion>12</LangVersion>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
|
34
README.md
34
README.md
@ -1,3 +1,7 @@
|
||||
# Testers for V2 wanted!
|
||||
|
||||
[Details](https://github.com/C9Glax/tranga/pull/355#issuecomment-2764217944)
|
||||
|
||||
<!-- PROJECT LOGO -->
|
||||
<br />
|
||||
<div align="center">
|
||||
@ -45,17 +49,18 @@ Tranga can download Chapters and Metadata from "Scanlation" sites such as
|
||||
|
||||
- [MangaDex.org](https://mangadex.org/) (Multilingual)
|
||||
- [Manganato.com](https://manganato.com/) (en)
|
||||
- [Mangasee.com](https://mangasee123.com/) (en)
|
||||
- [MangaKatana.com](https://mangakatana.com) (en)
|
||||
- [Mangaworld.bz](https://www.mangaworld.bz/) (it)
|
||||
- [Bato.to](https://bato.to/v3x) (en)
|
||||
- [Manga4Life](https://manga4life.com) (en)
|
||||
- [ManhuaPlus](https://manhuaplus.org/) (en)
|
||||
- [MangaHere](https://www.mangahere.cc/) (en) (Their covers suck)
|
||||
- ❓ Open an [issue](https://github.com/C9Glax/tranga/issues)
|
||||
- [MangaHere](https://www.mangahere.cc/) (en) (Their covers aren't scrapeable.)
|
||||
- [Weebcentral](https://weebcentral.com) (en)
|
||||
- [Webtoons](https://www.webtoons.com/en/)
|
||||
- ❓ Open an [issue](https://github.com/C9Glax/tranga/issues/new?assignees=&labels=New+Connector&projects=&template=new_connector.yml&title=%5BNew+Connector%5D%3A+)
|
||||
|
||||
and trigger a library-scan with [Komga](https://komga.org/) and [Kavita](https://www.kavitareader.com/).
|
||||
Notifications can be sent to your devices using [Gotify](https://gotify.net/) and [LunaSea](https://www.lunasea.app/).
|
||||
Notifications can be sent to your devices using [Gotify](https://gotify.net/), [LunaSea](https://www.lunasea.app/) or [Ntfy](https://ntfy.sh/
|
||||
).
|
||||
|
||||
### What this does and doesn't do
|
||||
|
||||
@ -92,6 +97,16 @@ That is why I wanted to create my own project, in a language I understand, and t
|
||||
|
||||
<p align="right">(<a href="#readme-top">back to top</a>)</p>
|
||||
|
||||
## Star History
|
||||
|
||||
<a href="https://star-history.com/#c9glax/tranga&Date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=c9glax/tranga&type=Date&theme=dark" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=c9glax/tranga&type=Date" />
|
||||
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=c9glax/tranga&type=Date" />
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
<!-- GETTING STARTED -->
|
||||
## Getting Started
|
||||
|
||||
@ -107,14 +122,9 @@ access the folder.
|
||||
### Prerequisites
|
||||
|
||||
#### To Build
|
||||
[.NET-Core 7.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/7.0)
|
||||
[.NET-Core 8.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/8.0)
|
||||
#### To Run
|
||||
[.NET-Core 7.0 Runtime](https://dotnet.microsoft.com/en-us/download/dotnet/7.0) scroll down a bit, should be on the right the second item.
|
||||
|
||||
<!-- ROADMAP -->
|
||||
## Roadmap
|
||||
|
||||
- [ ] ❓
|
||||
[.NET-Core 8.0 Runtime](https://dotnet.microsoft.com/en-us/download/dotnet/8.0) scroll down a bit, should be on the right the second item.
|
||||
|
||||
See the [open issues](https://github.com/C9Glax/tranga/issues) for a full list of proposed features (and known issues).
|
||||
|
||||
|
@ -1,5 +1,7 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Xml.Linq;
|
||||
using static System.IO.UnixFileMode;
|
||||
|
||||
namespace Tranga;
|
||||
|
||||
@ -12,33 +14,37 @@ public readonly struct Chapter : IComparable
|
||||
// ReSharper disable once MemberCanBePrivate.Global
|
||||
public Manga parentManga { get; }
|
||||
public string? name { get; }
|
||||
public string volumeNumber { get; }
|
||||
public string chapterNumber { get; }
|
||||
public float volumeNumber { get; }
|
||||
public float chapterNumber { get; }
|
||||
public string url { get; }
|
||||
// ReSharper disable once MemberCanBePrivate.Global
|
||||
public string fileName { get; }
|
||||
public string? id { get; }
|
||||
|
||||
private static readonly Regex LegalCharacters = new (@"([A-z]*[0-9]* *\.*-*,*\]*\[*'*\'*\)*\(*~*!*)*");
|
||||
private static readonly Regex IllegalStrings = new(@"(Vol(ume)?|Ch(apter)?)\.?", RegexOptions.IgnoreCase);
|
||||
private static readonly Regex Digits = new(@"[0-9\.]*");
|
||||
public Chapter(Manga parentManga, string? name, string? volumeNumber, string chapterNumber, string url)
|
||||
|
||||
public Chapter(Manga parentManga, string? name, string? volumeNumber, string chapterNumber, string url, string? id = null)
|
||||
: this(parentManga, name, float.Parse(volumeNumber??"0", GlobalBase.numberFormatDecimalPoint),
|
||||
float.Parse(chapterNumber, GlobalBase.numberFormatDecimalPoint), url, id)
|
||||
{
|
||||
}
|
||||
|
||||
public Chapter(Manga parentManga, string? name, float? volumeNumber, float chapterNumber, string url, string? id = null)
|
||||
{
|
||||
this.parentManga = parentManga;
|
||||
this.name = name;
|
||||
this.volumeNumber = volumeNumber is not null ? string.Concat(Digits.Matches(volumeNumber).Select(x => x.Value)) : "0";
|
||||
this.chapterNumber = string.Concat(Digits.Matches(chapterNumber).Select(x => x.Value));
|
||||
this.volumeNumber = volumeNumber??0;
|
||||
this.chapterNumber = chapterNumber;
|
||||
this.url = url;
|
||||
this.id = id;
|
||||
|
||||
string chapterVolNumStr;
|
||||
if (volumeNumber is not null && volumeNumber.Length > 0)
|
||||
chapterVolNumStr = $"Vol.{volumeNumber} Ch.{chapterNumber}";
|
||||
else
|
||||
chapterVolNumStr = $"Ch.{chapterNumber}";
|
||||
string chapterVolNumStr = $"Vol.{this.volumeNumber} Ch.{chapterNumber}";
|
||||
|
||||
if (name is not null && name.Length > 0)
|
||||
{
|
||||
string chapterName = IllegalStrings.Replace(string.Concat(LegalCharacters.Matches(name)), "");
|
||||
this.fileName = $"{chapterVolNumStr} - {chapterName}";
|
||||
this.fileName = chapterName.Length > 0 ? $"{chapterVolNumStr} - {chapterName}" : chapterVolNumStr;
|
||||
}
|
||||
else
|
||||
this.fileName = chapterVolNumStr;
|
||||
@ -58,29 +64,14 @@ public readonly struct Chapter : IComparable
|
||||
|
||||
public int CompareTo(object? obj)
|
||||
{
|
||||
if (obj is Chapter otherChapter)
|
||||
if(obj is not Chapter otherChapter)
|
||||
throw new ArgumentException($"{obj} can not be compared to {this}");
|
||||
return volumeNumber.CompareTo(otherChapter.volumeNumber) switch
|
||||
{
|
||||
if (float.TryParse(volumeNumber, GlobalBase.numberFormatDecimalPoint, out float volumeNumberFloat) &&
|
||||
float.TryParse(chapterNumber, GlobalBase.numberFormatDecimalPoint, out float chapterNumberFloat) &&
|
||||
float.TryParse(otherChapter.volumeNumber, GlobalBase.numberFormatDecimalPoint,
|
||||
out float otherVolumeNumberFloat) &&
|
||||
float.TryParse(otherChapter.chapterNumber, GlobalBase.numberFormatDecimalPoint,
|
||||
out float otherChapterNumberFloat))
|
||||
{
|
||||
|
||||
switch (volumeNumberFloat.CompareTo(otherVolumeNumberFloat))
|
||||
{
|
||||
case < 0:
|
||||
return -1;
|
||||
case > 0:
|
||||
return 1;
|
||||
default:
|
||||
return chapterNumberFloat.CompareTo(otherChapterNumberFloat);
|
||||
}
|
||||
}
|
||||
else throw new FormatException($"Value could not be parsed");
|
||||
}
|
||||
throw new ArgumentException($"{obj} can not be compared to {this}");
|
||||
<0 => -1,
|
||||
>0 => 1,
|
||||
_ => chapterNumber.CompareTo(otherChapter.chapterNumber)
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@ -89,25 +80,56 @@ public readonly struct Chapter : IComparable
|
||||
/// <returns>true if chapter is present</returns>
|
||||
internal bool CheckChapterIsDownloaded()
|
||||
{
|
||||
if (!Directory.Exists(Path.Join(TrangaSettings.downloadLocation, parentManga.folderName)))
|
||||
string mangaDirectory = Path.Join(TrangaSettings.downloadLocation, parentManga.folderName);
|
||||
if (!Directory.Exists(mangaDirectory))
|
||||
return false;
|
||||
FileInfo[] archives = new DirectoryInfo(Path.Join(TrangaSettings.downloadLocation, parentManga.folderName)).GetFiles().Where(file => file.Name.Split('.')[^1] == "cbz").ToArray();
|
||||
Regex volChRex = new(@"(?:Vol(?:ume)?\.([0-9]+)\D*)?Ch(?:apter)?\.([0-9]+(?:\.[0-9]+)*)");
|
||||
|
||||
Chapter t = this;
|
||||
string thisPath = GetArchiveFilePath();
|
||||
FileInfo? archive = archives.FirstOrDefault(archive =>
|
||||
FileInfo? mangaArchive = null;
|
||||
string markerPath = Path.Join(mangaDirectory, $".{id}");
|
||||
if (this.id is not null && File.Exists(markerPath))
|
||||
{
|
||||
Match m = volChRex.Match(archive.Name);
|
||||
string archiveVolNum = m.Groups[1].Success ? m.Groups[1].Value : "0";
|
||||
string archiveChNum = m.Groups[2].Value;
|
||||
return archiveVolNum == t.volumeNumber && archiveChNum == t.chapterNumber ||
|
||||
archiveVolNum == "0" && archiveChNum == t.chapterNumber;
|
||||
});
|
||||
if(archive is not null && thisPath != archive.FullName)
|
||||
archive.MoveTo(thisPath, true);
|
||||
return archive is not null;
|
||||
if(File.Exists(File.ReadAllText(markerPath)))
|
||||
mangaArchive = new FileInfo(File.ReadAllText(markerPath));
|
||||
else
|
||||
File.Delete(markerPath);
|
||||
}
|
||||
|
||||
if(mangaArchive is null)
|
||||
{
|
||||
FileInfo[] archives = new DirectoryInfo(mangaDirectory).GetFiles("*.cbz");
|
||||
Regex volChRex = new(@"(?:Vol(?:ume)?\.([0-9]+)\D*)?Ch(?:apter)?\.([0-9]+(?:\.[0-9]+)*)(?: - (.*))?.cbz");
|
||||
|
||||
Chapter t = this;
|
||||
mangaArchive = archives.FirstOrDefault(archive =>
|
||||
{
|
||||
Match m = volChRex.Match(archive.Name);
|
||||
/*
|
||||
* 1. If the volumeNumber is not present in the filename, it is not checked.
|
||||
* 2. Check the chapterNumber in the chapter against the one in the filename.
|
||||
* 3. The chpaterName has to either be absent both in the chapter and the filename or match.
|
||||
*/
|
||||
return (!m.Groups[1].Success || m.Groups[1].Value == t.volumeNumber.ToString(GlobalBase.numberFormatDecimalPoint)) &&
|
||||
m.Groups[2].Value == t.chapterNumber.ToString(GlobalBase.numberFormatDecimalPoint) &&
|
||||
((!m.Groups[3].Success && string.IsNullOrEmpty(t.name)) || m.Groups[3].Value == t.name);
|
||||
});
|
||||
}
|
||||
|
||||
string correctPath = GetArchiveFilePath();
|
||||
if(mangaArchive is not null && mangaArchive.FullName != correctPath)
|
||||
mangaArchive.MoveTo(correctPath, true);
|
||||
return (mangaArchive is not null);
|
||||
}
|
||||
|
||||
public void CreateChapterMarker()
|
||||
{
|
||||
if (this.id is null)
|
||||
return;
|
||||
string path = Path.Join(TrangaSettings.downloadLocation, parentManga.folderName, $".{id}");
|
||||
File.WriteAllText(path, GetArchiveFilePath());
|
||||
File.SetAttributes(path, FileAttributes.Hidden);
|
||||
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
|
||||
File.SetUnixFileMode(path, UserRead | UserWrite | UserExecute | GroupRead | GroupWrite | GroupExecute | OtherRead | OtherExecute);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates full file path of chapter-archive
|
||||
/// </summary>
|
||||
|
@ -66,10 +66,10 @@ public abstract class GlobalBase
|
||||
Log(string.Format(fStr, replace));
|
||||
}
|
||||
|
||||
protected void SendNotifications(string title, string text)
|
||||
protected void SendNotifications(string title, string text, bool buffer = false)
|
||||
{
|
||||
foreach (NotificationConnector nc in notificationConnectors)
|
||||
nc.SendNotification(title, text);
|
||||
nc.SendNotification(title, text, buffer);
|
||||
}
|
||||
|
||||
protected void AddNotificationConnector(NotificationConnector notificationConnector)
|
||||
|
@ -37,7 +37,7 @@ public class DownloadChapter : Job
|
||||
if (success == HttpStatusCode.OK)
|
||||
{
|
||||
UpdateLibraries();
|
||||
SendNotifications("Chapter downloaded", $"{chapter.parentManga.sortName} - {chapter.chapterNumber}");
|
||||
SendNotifications("Chapter downloaded", $"{chapter.parentManga.sortName} - {chapter.chapterNumber}", true);
|
||||
}
|
||||
});
|
||||
downloadTask.Start();
|
||||
|
@ -54,6 +54,6 @@ public class DownloadNewChapters : Job
|
||||
if (obj is not DownloadNewChapters otherJob)
|
||||
return false;
|
||||
return otherJob.mangaConnector == this.mangaConnector &&
|
||||
otherJob.manga.Equals(this.manga);
|
||||
otherJob.manga.publicationId == this.manga.publicationId;
|
||||
}
|
||||
}
|
@ -1,6 +1,9 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text.RegularExpressions;
|
||||
using Newtonsoft.Json;
|
||||
using Tranga.MangaConnectors;
|
||||
using static System.IO.UnixFileMode;
|
||||
|
||||
namespace Tranga.Jobs;
|
||||
|
||||
@ -17,18 +20,21 @@ public class JobBoss : GlobalBase
|
||||
Log($"Next job in {jobs.MinBy(job => job.nextExecution)?.nextExecution.Subtract(DateTime.Now)} {jobs.MinBy(job => job.nextExecution)?.id}");
|
||||
}
|
||||
|
||||
public void AddJob(Job job)
|
||||
public bool AddJob(Job job, string? jobFile = null)
|
||||
{
|
||||
if (ContainsJobLike(job))
|
||||
{
|
||||
Log($"Already Contains Job {job}");
|
||||
return false;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!this.jobs.Add(job))
|
||||
return false;
|
||||
Log($"Added {job}");
|
||||
this.jobs.Add(job);
|
||||
UpdateJobFile(job);
|
||||
UpdateJobFile(job, jobFile);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public void AddJobs(IEnumerable<Job> jobsToAdd)
|
||||
@ -65,7 +71,7 @@ public class JobBoss : GlobalBase
|
||||
RemoveJob(job);
|
||||
}
|
||||
|
||||
public IEnumerable<Job> GetJobsLike(string? connectorName = null, string? internalId = null, string? chapterNumber = null)
|
||||
public IEnumerable<Job> GetJobsLike(string? connectorName = null, string? internalId = null, float? chapterNumber = null)
|
||||
{
|
||||
IEnumerable<Job> ret = this.jobs;
|
||||
if (connectorName is not null)
|
||||
@ -77,7 +83,7 @@ public class JobBoss : GlobalBase
|
||||
if (jjob is not DownloadChapter job)
|
||||
return false;
|
||||
return job.chapter.parentManga.internalId == internalId &&
|
||||
job.chapter.chapterNumber == chapterNumber;
|
||||
job.chapter.chapterNumber.Equals(chapterNumber);
|
||||
});
|
||||
else if (internalId is not null)
|
||||
ret = ret.Where(jjob =>
|
||||
@ -143,26 +149,40 @@ public class JobBoss : GlobalBase
|
||||
if (!Directory.Exists(TrangaSettings.jobsFolderPath)) //No jobs to load
|
||||
{
|
||||
Directory.CreateDirectory(TrangaSettings.jobsFolderPath);
|
||||
if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
|
||||
File.SetUnixFileMode(TrangaSettings.jobsFolderPath, UserRead | UserWrite | UserExecute | GroupRead | OtherRead);
|
||||
return;
|
||||
}
|
||||
Regex idRex = new (@"(.*)\.json");
|
||||
|
||||
//Load json-job-files
|
||||
foreach (FileInfo file in new DirectoryInfo(TrangaSettings.jobsFolderPath).EnumerateFiles().Where(fileInfo => idRex.IsMatch(fileInfo.Name)))
|
||||
foreach (FileInfo file in Directory.GetFiles(TrangaSettings.jobsFolderPath, "*.json").Select(f => new FileInfo(f)))
|
||||
{
|
||||
Log($"Adding {file.Name}");
|
||||
Job? job = JsonConvert.DeserializeObject<Job>(File.ReadAllText(file.FullName),
|
||||
new JobJsonConverter(this, new MangaConnectorJsonConverter(this, connectors)));
|
||||
if (job is null)
|
||||
{
|
||||
string newName = file.FullName + ".failed";
|
||||
Log($"Failed loading file {file.Name}.\nMoving to {newName}");
|
||||
File.Move(file.FullName, newName);
|
||||
}
|
||||
else
|
||||
try
|
||||
{
|
||||
Job? job = JsonConvert.DeserializeObject<Job>(File.ReadAllText(file.FullName),
|
||||
new JobJsonConverter(this, new MangaConnectorJsonConverter(this, connectors)));
|
||||
if (job is null) throw new NullReferenceException();
|
||||
|
||||
Log($"Adding Job {job}");
|
||||
this.jobs.Add(job);
|
||||
if (!AddJob(job, file.FullName)) //If we detect a duplicate, delete the file.
|
||||
{
|
||||
//string path = string.Concat(file.FullName, ".duplicate");
|
||||
//file.MoveTo(path);
|
||||
//Log($"Duplicate detected or otherwise not able to add job to list.\nMoved job {job} to {path}");
|
||||
Log($"Duplicate detected or otherwise not able to add job to list. Removed the file {file.FullName} {job}");
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
if (e is not UnreachableException or NullReferenceException)
|
||||
throw;
|
||||
Log(e.Message);
|
||||
string newName = file.FullName + ".failed";
|
||||
Log($"Failed loading file {file.Name}.\nMoving to {newName}.\n" +
|
||||
$"If you think this is a bug, upload contents of the file to the Bugreport!");
|
||||
File.Move(file.FullName, newName);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
@ -182,48 +202,42 @@ public class JobBoss : GlobalBase
|
||||
|
||||
string[] coverFiles = Directory.GetFiles(TrangaSettings.coverImageCache);
|
||||
foreach(string fileName in coverFiles.Where(fileName => !GetAllCachedManga().Any(manga => manga.coverFileNameInCache == fileName)))
|
||||
File.Delete(fileName);
|
||||
File.Delete(fileName);
|
||||
}
|
||||
|
||||
internal void UpdateJobFile(Job job, string? oldFile = null)
|
||||
{
|
||||
string newJobFilePath = Path.Join(TrangaSettings.jobsFolderPath, $"{job.id}.json");
|
||||
|
||||
if (!this.jobs.Any(jjob => jjob.id == job.id))
|
||||
string oldFilePath = oldFile??Path.Join(TrangaSettings.jobsFolderPath, $"{job.id}.json");
|
||||
|
||||
//Delete old file
|
||||
if (File.Exists(oldFilePath))
|
||||
{
|
||||
Log($"Deleting Job-file {oldFilePath}");
|
||||
try
|
||||
{
|
||||
Log($"Deleting Job-file {newJobFilePath}");
|
||||
while(IsFileInUse(newJobFilePath))
|
||||
while(IsFileInUse(oldFilePath))
|
||||
Thread.Sleep(10);
|
||||
File.Delete(newJobFilePath);
|
||||
File.Delete(oldFilePath);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log(e.ToString());
|
||||
Log($"Error deleting {oldFilePath} job {job.id}\n{e}");
|
||||
return; //Don't export a new file when we haven't actually deleted the old one
|
||||
}
|
||||
}
|
||||
else
|
||||
|
||||
//Export job (in new file) if it is still in our jobs list
|
||||
if (GetJobById(job.id) is not null)
|
||||
{
|
||||
Log($"Exporting Job {newJobFilePath}");
|
||||
string jobStr = JsonConvert.SerializeObject(job, Formatting.Indented);
|
||||
while(IsFileInUse(newJobFilePath))
|
||||
Thread.Sleep(10);
|
||||
File.WriteAllText(newJobFilePath, jobStr);
|
||||
if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
|
||||
File.SetUnixFileMode(newJobFilePath, UserRead | UserWrite | GroupRead | OtherRead);
|
||||
}
|
||||
|
||||
if(oldFile is not null)
|
||||
try
|
||||
{
|
||||
Log($"Deleting old Job-file {oldFile}");
|
||||
while(IsFileInUse(oldFile))
|
||||
Thread.Sleep(10);
|
||||
File.Delete(oldFile);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log(e.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdateAllJobFiles()
|
||||
|
@ -71,6 +71,6 @@ public class UpdateMetadata : Job
|
||||
if (obj is not UpdateMetadata otherJob)
|
||||
return false;
|
||||
return otherJob.mangaConnector == this.mangaConnector &&
|
||||
otherJob.manga.Equals(this.manga);
|
||||
otherJob.manga.publicationId == this.manga.publicationId;
|
||||
}
|
||||
}
|
@ -61,7 +61,7 @@ public class Kavita : LibraryConnector
|
||||
return "";
|
||||
}
|
||||
|
||||
public override void UpdateLibrary()
|
||||
protected override void UpdateLibraryInternal()
|
||||
{
|
||||
Log("Updating libraries.");
|
||||
foreach (KavitaLibrary lib in GetLibraries())
|
||||
|
@ -25,7 +25,7 @@ public class Komga : LibraryConnector
|
||||
return $"Komga {baseUrl}";
|
||||
}
|
||||
|
||||
public override void UpdateLibrary()
|
||||
protected override void UpdateLibraryInternal()
|
||||
{
|
||||
Log("Updating libraries.");
|
||||
foreach (KomgaLibrary lib in GetLibraries())
|
||||
|
@ -17,6 +17,9 @@ public abstract class LibraryConnector : GlobalBase
|
||||
public string baseUrl { get; }
|
||||
// ReSharper disable once MemberCanBeProtected.Global
|
||||
public string auth { get; } //Base64 encoded, if you use your password everywhere, you have problems
|
||||
private DateTime? _updateLibraryRequested = null;
|
||||
private readonly Thread? _libraryBufferThread = null;
|
||||
private const int NoChangeTimeout = 2, BiggestInterval = 20;
|
||||
|
||||
protected LibraryConnector(GlobalBase clone, string baseUrl, string auth, LibraryType libraryType) : base(clone)
|
||||
{
|
||||
@ -28,8 +31,47 @@ public abstract class LibraryConnector : GlobalBase
|
||||
this.baseUrl = baseUrlRex.Match(baseUrl).Value;
|
||||
this.auth = auth;
|
||||
this.libraryType = libraryType;
|
||||
|
||||
if (TrangaSettings.bufferLibraryUpdates)
|
||||
{
|
||||
_libraryBufferThread = new(CheckLibraryBuffer);
|
||||
_libraryBufferThread.Start();
|
||||
}
|
||||
}
|
||||
public abstract void UpdateLibrary();
|
||||
|
||||
private void CheckLibraryBuffer()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
if (_updateLibraryRequested is not null && DateTime.Now.Subtract((DateTime)_updateLibraryRequested) > TimeSpan.FromMinutes(NoChangeTimeout)) //If no updates have been requested for NoChangeTimeout minutes, update library
|
||||
{
|
||||
UpdateLibraryInternal();
|
||||
_updateLibraryRequested = null;
|
||||
}
|
||||
Thread.Sleep(100);
|
||||
}
|
||||
}
|
||||
|
||||
public void UpdateLibrary()
|
||||
{
|
||||
_updateLibraryRequested ??= DateTime.Now;
|
||||
if (!TrangaSettings.bufferLibraryUpdates)
|
||||
{
|
||||
UpdateLibraryInternal();
|
||||
return;
|
||||
}else if (_updateLibraryRequested is not null &&
|
||||
DateTime.Now.Subtract((DateTime)_updateLibraryRequested) > TimeSpan.FromMinutes(BiggestInterval)) //If the last update has been more than BiggestInterval minutes ago, update library
|
||||
{
|
||||
UpdateLibraryInternal();
|
||||
_updateLibraryRequested = null;
|
||||
}
|
||||
else if(_updateLibraryRequested is not null)
|
||||
{
|
||||
Log($"Buffering Library Updates (Updates in latest {((DateTime)_updateLibraryRequested).Add(TimeSpan.FromMinutes(BiggestInterval)).Subtract(DateTime.Now)} or {((DateTime)_updateLibraryRequested).Add(TimeSpan.FromMinutes(NoChangeTimeout)).Subtract(DateTime.Now)})");
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract void UpdateLibraryInternal();
|
||||
internal abstract bool Test();
|
||||
|
||||
protected static class NetClient
|
||||
|
@ -67,7 +67,7 @@ public struct Manga
|
||||
while (this.folderName.EndsWith('.'))
|
||||
this.folderName = this.folderName.Substring(0, this.folderName.Length - 1);
|
||||
string onlyLowerLetters = string.Concat(this.sortName.ToLower().Where(Char.IsLetter));
|
||||
this.internalId = Convert.ToBase64String(Encoding.ASCII.GetBytes($"{onlyLowerLetters}{this.year}"));
|
||||
this.internalId = DateTime.Now.Ticks.ToString();
|
||||
this.ignoreChaptersBelow = ignoreChaptersBelow ?? 0f;
|
||||
this.latestChapterDownloaded = 0;
|
||||
this.latestChapterAvailable = 0;
|
||||
@ -128,10 +128,19 @@ public struct Manga
|
||||
public void MovePublicationFolder(string downloadDirectory, string newFolderName)
|
||||
{
|
||||
string oldPath = Path.Join(downloadDirectory, this.folderName);
|
||||
this.folderName = newFolderName;
|
||||
this.folderName = newFolderName;//Create new Path with the new folderName
|
||||
string newPath = CreatePublicationFolder(downloadDirectory);
|
||||
if(Directory.Exists(oldPath))
|
||||
Directory.Move(oldPath, newPath);
|
||||
if (Directory.Exists(oldPath))
|
||||
{
|
||||
if (Directory.Exists(newPath)) //Move/Overwrite old Files, Delete old Directory
|
||||
{
|
||||
IEnumerable<string> newPathFileNames = new DirectoryInfo(newPath).GetFiles().Select(fi => fi.Name);
|
||||
foreach(FileInfo fileInfo in new DirectoryInfo(oldPath).GetFiles().Where(fi => newPathFileNames.Contains(fi.Name) == false))
|
||||
File.Move(fileInfo.FullName, Path.Join(newPath, fileInfo.Name), true);
|
||||
Directory.Delete(oldPath);
|
||||
}else
|
||||
Directory.Move(oldPath, newPath);
|
||||
}
|
||||
}
|
||||
|
||||
public void UpdateLatestDownloadedChapter(Chapter chapter)//TODO check files if chapters are all downloaded
|
||||
|
217
Tranga/MangaConnectors/AsuraToon.cs
Normal file
217
Tranga/MangaConnectors/AsuraToon.cs
Normal file
@ -0,0 +1,217 @@
|
||||
using System.Net;
|
||||
using System.Text.RegularExpressions;
|
||||
using HtmlAgilityPack;
|
||||
using Tranga.Jobs;
|
||||
|
||||
namespace Tranga.MangaConnectors;
|
||||
|
||||
public class AsuraToon : MangaConnector
|
||||
{
|
||||
|
||||
public AsuraToon(GlobalBase clone) : base(clone, "AsuraToon", ["en"])
|
||||
{
|
||||
this.downloadClient = new ChromiumDownloadClient(clone);
|
||||
}
|
||||
|
||||
public override Manga[] GetManga(string publicationTitle = "")
|
||||
{
|
||||
Log($"Searching Publications. Term=\"{publicationTitle}\"");
|
||||
string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower();
|
||||
string requestUrl = $"https://asuracomic.net/series?name={sanitizedTitle}";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return Array.Empty<Manga>();
|
||||
|
||||
if (requestResult.htmlDocument is null)
|
||||
{
|
||||
Log($"Failed to retrieve site");
|
||||
return Array.Empty<Manga>();
|
||||
}
|
||||
|
||||
Manga[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument);
|
||||
Log($"Retrieved {publications.Length} publications. Term=\"{publicationTitle}\"");
|
||||
return publications;
|
||||
}
|
||||
|
||||
public override Manga? GetMangaFromId(string publicationId)
|
||||
{
|
||||
return GetMangaFromUrl($"https://asuracomic.net/series/{publicationId}");
|
||||
}
|
||||
|
||||
public override Manga? GetMangaFromUrl(string url)
|
||||
{
|
||||
RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return null;
|
||||
if (requestResult.htmlDocument is null)
|
||||
{
|
||||
Log($"Failed to retrieve site");
|
||||
return null;
|
||||
}
|
||||
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, url.Split('/')[^1], url);
|
||||
}
|
||||
|
||||
private Manga[] ParsePublicationsFromHtml(HtmlDocument document)
|
||||
{
|
||||
HtmlNodeCollection mangaList = document.DocumentNode.SelectNodes("//a[starts-with(@href,'series')]");
|
||||
if (mangaList is null || mangaList.Count < 1)
|
||||
return [];
|
||||
|
||||
IEnumerable<string> urls = mangaList.Select(a => $"https://asuracomic.net/{a.GetAttributeValue("href", "")}");
|
||||
|
||||
List<Manga> ret = new();
|
||||
foreach (string url in urls)
|
||||
{
|
||||
Manga? manga = GetMangaFromUrl(url);
|
||||
if (manga is not null)
|
||||
ret.Add((Manga)manga);
|
||||
}
|
||||
|
||||
return ret.ToArray();
|
||||
}
|
||||
|
||||
private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl)
|
||||
{
|
||||
string? originalLanguage = null;
|
||||
Dictionary<string, string> altTitles = new(), links = new();
|
||||
|
||||
HtmlNodeCollection genreNodes = document.DocumentNode.SelectNodes("//h3[text()='Genres']/../div/button");
|
||||
string[] tags = genreNodes.Select(b => b.InnerText).ToArray();
|
||||
|
||||
HtmlNode statusNode = document.DocumentNode.SelectSingleNode("//h3[text()='Status']/../h3[2]");
|
||||
Manga.ReleaseStatusByte releaseStatus = statusNode.InnerText.ToLower() switch
|
||||
{
|
||||
"ongoing" => Manga.ReleaseStatusByte.Continuing,
|
||||
"hiatus" => Manga.ReleaseStatusByte.OnHiatus,
|
||||
"completed" => Manga.ReleaseStatusByte.Completed,
|
||||
"dropped" => Manga.ReleaseStatusByte.Cancelled,
|
||||
"season end" => Manga.ReleaseStatusByte.Continuing,
|
||||
"coming soon" => Manga.ReleaseStatusByte.Unreleased,
|
||||
_ => Manga.ReleaseStatusByte.Unreleased
|
||||
};
|
||||
|
||||
HtmlNode coverNode =
|
||||
document.DocumentNode.SelectSingleNode("//img[@alt='poster']");
|
||||
string coverUrl = coverNode.GetAttributeValue("src", "");
|
||||
string coverFileNameInCache = SaveCoverImageToCache(coverUrl, publicationId, RequestType.MangaCover);
|
||||
|
||||
HtmlNode titleNode =
|
||||
document.DocumentNode.SelectSingleNode("//title");
|
||||
string sortName = Regex.Match(titleNode.InnerText, @"(.*) - Asura Scans").Groups[1].Value;
|
||||
|
||||
HtmlNode descriptionNode =
|
||||
document.DocumentNode.SelectSingleNode("//h3[starts-with(text(),'Synopsis')]/../span");
|
||||
string description = descriptionNode?.InnerText??"";
|
||||
|
||||
HtmlNodeCollection authorNodes = document.DocumentNode.SelectNodes("//h3[text()='Author']/../h3[not(text()='Author' or text()='_')]");
|
||||
HtmlNodeCollection artistNodes = document.DocumentNode.SelectNodes("//h3[text()='Artist']/../h3[not(text()='Artist' or text()='_')]");
|
||||
IEnumerable<string> authorNames = authorNodes is null ? [] : authorNodes.Select(a => a.InnerText);
|
||||
IEnumerable<string> artistNames = artistNodes is null ? [] : artistNodes.Select(a => a.InnerText);
|
||||
List<string> authors = authorNames.Concat(artistNames).ToList();
|
||||
|
||||
HtmlNode? firstChapterNode = document.DocumentNode.SelectSingleNode("//a[contains(@href, 'chapter/1')]/../following-sibling::h3");
|
||||
int? year = int.Parse(firstChapterNode?.InnerText.Split(' ')[^1] ?? "2000");
|
||||
|
||||
Manga manga = new (sortName, authors, description, altTitles, tags, coverUrl, coverFileNameInCache, links,
|
||||
year, originalLanguage, publicationId, releaseStatus, websiteUrl);
|
||||
AddMangaToCache(manga);
|
||||
return manga;
|
||||
}
|
||||
|
||||
public override Chapter[] GetChapters(Manga manga, string language="en")
|
||||
{
|
||||
Log($"Getting chapters {manga}");
|
||||
string requestUrl = $"https://asuracomic.net/series/{manga.publicationId}";
|
||||
// Leaving this in for verification if the page exists
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return Array.Empty<Chapter>();
|
||||
|
||||
//Return Chapters ordered by Chapter-Number
|
||||
List<Chapter> chapters = ParseChaptersFromHtml(manga, requestUrl);
|
||||
Log($"Got {chapters.Count} chapters. {manga}");
|
||||
return chapters.Order().ToArray();
|
||||
}
|
||||
|
||||
private List<Chapter> ParseChaptersFromHtml(Manga manga, string mangaUrl)
|
||||
{
|
||||
RequestResult result = downloadClient.MakeRequest(mangaUrl, RequestType.Default);
|
||||
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null)
|
||||
{
|
||||
Log("Failed to load site");
|
||||
return new List<Chapter>();
|
||||
}
|
||||
|
||||
List<Chapter> ret = new();
|
||||
|
||||
HtmlNodeCollection chapterURLNodes = result.htmlDocument.DocumentNode.SelectNodes("//a[contains(@href, '/chapter/')]");
|
||||
Regex infoRex = new(@"Chapter ([0-9]+)(.*)?");
|
||||
|
||||
foreach (HtmlNode chapterInfo in chapterURLNodes)
|
||||
{
|
||||
string chapterUrl = chapterInfo.GetAttributeValue("href", "");
|
||||
|
||||
Match match = infoRex.Match(chapterInfo.InnerText);
|
||||
string chapterNumber = match.Groups[1].Value;
|
||||
string? chapterName = match.Groups[2].Success && match.Groups[2].Length > 1 ? match.Groups[2].Value : null;
|
||||
string url = $"https://asuracomic.net/series/{chapterUrl}";
|
||||
try
|
||||
{
|
||||
ret.Add(new Chapter(manga, chapterName, null, chapterNumber, url));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log($"Failed to load chapter {chapterNumber}: {e.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
|
||||
{
|
||||
if (progressToken?.cancellationRequested ?? false)
|
||||
{
|
||||
progressToken.Cancel();
|
||||
return HttpStatusCode.RequestTimeout;
|
||||
}
|
||||
|
||||
Manga chapterParentManga = chapter.parentManga;
|
||||
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
|
||||
string requestUrl = chapter.url;
|
||||
// Leaving this in to check if the page exists
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
{
|
||||
progressToken?.Cancel();
|
||||
return requestResult.statusCode;
|
||||
}
|
||||
|
||||
string[] imageUrls = ParseImageUrlsFromHtml(requestUrl);
|
||||
|
||||
return DownloadChapterImages(imageUrls, chapter, RequestType.MangaImage, progressToken:progressToken);
|
||||
}
|
||||
|
||||
private string[] ParseImageUrlsFromHtml(string mangaUrl)
|
||||
{
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(mangaUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
if (requestResult.htmlDocument is null)
|
||||
{
|
||||
Log($"Failed to retrieve site");
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
HtmlNodeCollection images =
|
||||
requestResult.htmlDocument.DocumentNode.SelectNodes("//img[contains(@alt, 'chapter page')]");
|
||||
|
||||
return images.Select(i => i.GetAttributeValue("src", "")).ToArray();
|
||||
}
|
||||
}
|
@ -8,7 +8,7 @@ namespace Tranga.MangaConnectors;
|
||||
public class Bato : MangaConnector
|
||||
{
|
||||
|
||||
public Bato(GlobalBase clone) : base(clone, "Bato")
|
||||
public Bato(GlobalBase clone) : base(clone, "Bato", ["en"])
|
||||
{
|
||||
this.downloadClient = new HttpDownloadClient(clone);
|
||||
}
|
||||
@ -150,7 +150,7 @@ public class Bato : MangaConnector
|
||||
HtmlNode chapterList =
|
||||
result.htmlDocument.DocumentNode.SelectSingleNode("/html/body/div/main/div[3]/astro-island/div/div[2]/div/div/astro-slot");
|
||||
|
||||
Regex numberRex = new(@"\/title\/.+\/[0-9]+(-vol_([0-9]+))?-ch_([0-9\.]+)");
|
||||
Regex numberRex = new(@"\/title\/.+\/([0-9])+(?:-vol_([0-9]+))?-ch_([0-9\.]+)");
|
||||
|
||||
foreach (HtmlNode chapterInfo in chapterList.SelectNodes("div"))
|
||||
{
|
||||
@ -158,11 +158,19 @@ public class Bato : MangaConnector
|
||||
string chapterUrl = infoNode.GetAttributeValue("href", "");
|
||||
|
||||
Match match = numberRex.Match(chapterUrl);
|
||||
string id = match.Groups[1].Value;
|
||||
string? volumeNumber = match.Groups[2].Success ? match.Groups[2].Value : null;
|
||||
string chapterNumber = match.Groups[3].Value;
|
||||
string chapterName = chapterNumber;
|
||||
string url = $"https://bato.to{chapterUrl}?load=2";
|
||||
ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url));
|
||||
try
|
||||
{
|
||||
ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log($"Failed to load chapter {chapterNumber}: {e.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
@ -189,11 +197,8 @@ public class Bato : MangaConnector
|
||||
}
|
||||
|
||||
string[] imageUrls = ParseImageUrlsFromHtml(requestUrl);
|
||||
|
||||
string comicInfoPath = Path.GetTempFileName();
|
||||
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
|
||||
|
||||
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(), RequestType.MangaImage, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken);
|
||||
|
||||
return DownloadChapterImages(imageUrls, chapter, RequestType.MangaImage, progressToken:progressToken);
|
||||
}
|
||||
|
||||
private string[] ParseImageUrlsFromHtml(string mangaUrl)
|
||||
|
@ -2,64 +2,61 @@
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using HtmlAgilityPack;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using PuppeteerSharp;
|
||||
|
||||
namespace Tranga.MangaConnectors;
|
||||
|
||||
internal class ChromiumDownloadClient : DownloadClient
|
||||
{
|
||||
private IBrowser browser { get; set; }
|
||||
private const string ChromiumVersion = "1154303";
|
||||
private const int StartTimeoutMs = 30000;
|
||||
private static IBrowser? _browser;
|
||||
private readonly HttpDownloadClient _httpDownloadClient;
|
||||
|
||||
private async Task<IBrowser> DownloadBrowser()
|
||||
private static async Task<IBrowser> StartBrowser(Logging.Logger? logger = null)
|
||||
{
|
||||
BrowserFetcher browserFetcher = new ();
|
||||
foreach(string rev in browserFetcher.LocalRevisions().Where(rev => rev != ChromiumVersion))
|
||||
browserFetcher.Remove(rev);
|
||||
if (!browserFetcher.LocalRevisions().Contains(ChromiumVersion))
|
||||
{
|
||||
Log("Downloading headless browser");
|
||||
DateTime last = DateTime.Now.Subtract(TimeSpan.FromSeconds(5));
|
||||
browserFetcher.DownloadProgressChanged += (_, args) =>
|
||||
{
|
||||
double currentBytes = Convert.ToDouble(args.BytesReceived) / Convert.ToDouble(args.TotalBytesToReceive);
|
||||
if (args.TotalBytesToReceive == args.BytesReceived)
|
||||
Log("Browser downloaded.");
|
||||
else if (DateTime.Now > last.AddSeconds(1))
|
||||
{
|
||||
Log($"Browser download progress: {currentBytes:P2}");
|
||||
last = DateTime.Now;
|
||||
}
|
||||
|
||||
};
|
||||
if (!browserFetcher.CanDownloadAsync(ChromiumVersion).Result)
|
||||
{
|
||||
Log($"Can't download browser version {ChromiumVersion}");
|
||||
throw new Exception();
|
||||
}
|
||||
await browserFetcher.DownloadAsync(ChromiumVersion);
|
||||
}
|
||||
|
||||
Log($"Starting Browser. ({StartTimeoutMs}ms timeout)");
|
||||
logger?.WriteLine("Starting ChromiumDownloadClient Puppeteer");
|
||||
return await Puppeteer.LaunchAsync(new LaunchOptions
|
||||
{
|
||||
Headless = true,
|
||||
ExecutablePath = browserFetcher.GetExecutablePath(ChromiumVersion),
|
||||
Args = new [] {
|
||||
"--disable-gpu",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-setuid-sandbox",
|
||||
"--no-sandbox"},
|
||||
Timeout = StartTimeoutMs
|
||||
});
|
||||
Timeout = TrangaSettings.ChromiumStartupTimeoutMs
|
||||
}, new LoggerFactory([new LogProvider(logger)]));
|
||||
}
|
||||
|
||||
private class LogProvider : GlobalBase, ILoggerProvider
|
||||
{
|
||||
public LogProvider(Logging.Logger? logger) : base(logger) { }
|
||||
|
||||
public void Dispose() { }
|
||||
|
||||
public ILogger CreateLogger(string categoryName) => new Logger(logger);
|
||||
}
|
||||
|
||||
private class Logger : GlobalBase, ILogger
|
||||
{
|
||||
public Logger(Logging.Logger? logger) : base(logger) { }
|
||||
|
||||
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
|
||||
{
|
||||
if (logLevel <= LogLevel.Information)
|
||||
return;
|
||||
logger?.WriteLine("Puppeteer", formatter.Invoke(state, exception));
|
||||
}
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
||||
}
|
||||
|
||||
public ChromiumDownloadClient(GlobalBase clone) : base(clone)
|
||||
{
|
||||
this.browser = DownloadBrowser().Result;
|
||||
_httpDownloadClient = new(this);
|
||||
if(_browser is null)
|
||||
_browser = StartBrowser(this.logger).Result;
|
||||
}
|
||||
|
||||
private readonly Regex _imageUrlRex = new(@"https?:\/\/.*\.(?:p?jpe?g|gif|a?png|bmp|avif|webp)(\?.*)?");
|
||||
@ -72,17 +69,21 @@ internal class ChromiumDownloadClient : DownloadClient
|
||||
|
||||
private RequestResult MakeRequestBrowser(string url, string? referrer = null, string? clickButton = null)
|
||||
{
|
||||
IPage page = this.browser.NewPageAsync().Result;
|
||||
page.DefaultTimeout = 10000;
|
||||
if (_browser is null)
|
||||
return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null);
|
||||
IPage page = _browser.NewPageAsync().Result;
|
||||
page.DefaultTimeout = TrangaSettings.ChromiumPageTimeoutMs;
|
||||
page.SetExtraHttpHeadersAsync(new() { { "Referer", referrer } });
|
||||
IResponse response;
|
||||
try
|
||||
{
|
||||
response = page.GoToAsync(url, WaitUntilNavigation.Networkidle0).Result;
|
||||
Log("Page loaded.");
|
||||
Log($"Page loaded. {url}");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log($"Could not load Page:\n{e.Message}");
|
||||
Log($"Could not load Page {url}\n{e.Message}");
|
||||
page.CloseAsync();
|
||||
return new RequestResult(HttpStatusCode.InternalServerError, null, Stream.Null);
|
||||
}
|
||||
|
||||
@ -113,9 +114,4 @@ internal class ChromiumDownloadClient : DownloadClient
|
||||
page.CloseAsync();
|
||||
return new RequestResult(response.Status, document, stream, false, "");
|
||||
}
|
||||
|
||||
public override void Close()
|
||||
{
|
||||
this.browser.CloseAsync();
|
||||
}
|
||||
}
|
@ -41,5 +41,4 @@ internal abstract class DownloadClient : GlobalBase
|
||||
}
|
||||
|
||||
internal abstract RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null);
|
||||
public abstract void Close();
|
||||
}
|
@ -72,9 +72,4 @@ internal class HttpDownloadClient : DownloadClient
|
||||
|
||||
return new RequestResult(response.StatusCode, document, stream);
|
||||
}
|
||||
|
||||
public override void Close()
|
||||
{
|
||||
Log("Closing.");
|
||||
}
|
||||
}
|
@ -14,15 +14,12 @@ namespace Tranga.MangaConnectors;
|
||||
public abstract class MangaConnector : GlobalBase
|
||||
{
|
||||
internal DownloadClient downloadClient { get; init; } = null!;
|
||||
public string[] SupportedLanguages;
|
||||
|
||||
public void StopDownloadClient()
|
||||
{
|
||||
downloadClient.Close();
|
||||
}
|
||||
|
||||
protected MangaConnector(GlobalBase clone, string name) : base(clone)
|
||||
protected MangaConnector(GlobalBase clone, string name, string[] supportedLanguages) : base(clone)
|
||||
{
|
||||
this.name = name;
|
||||
this.SupportedLanguages = supportedLanguages;
|
||||
Directory.CreateDirectory(TrangaSettings.coverImageCache);
|
||||
}
|
||||
|
||||
@ -63,8 +60,7 @@ public abstract class MangaConnector : GlobalBase
|
||||
return Array.Empty<Chapter>();
|
||||
|
||||
Log($"Checking for duplicates {manga}");
|
||||
List<Chapter> newChaptersList = allChapters.Where(nChapter => float.TryParse(nChapter.chapterNumber, numberFormatDecimalPoint, out float chapterNumber)
|
||||
&& chapterNumber > manga.ignoreChaptersBelow
|
||||
List<Chapter> newChaptersList = allChapters.Where(nChapter => nChapter.chapterNumber >= manga.ignoreChaptersBelow
|
||||
&& !nChapter.CheckChapterIsDownloaded()).ToList();
|
||||
Log($"{newChaptersList.Count} new chapters. {manga}");
|
||||
try
|
||||
@ -82,79 +78,6 @@ public abstract class MangaConnector : GlobalBase
|
||||
|
||||
return newChaptersList.ToArray();
|
||||
}
|
||||
|
||||
public Chapter[] SelectChapters(Manga manga, string searchTerm, string? language = null)
|
||||
{
|
||||
Chapter[] availableChapters = this.GetChapters(manga, language??"en");
|
||||
Regex volumeRegex = new ("((v(ol)*(olume)*){1} *([0-9]+(-[0-9]+)?){1})", RegexOptions.IgnoreCase);
|
||||
Regex chapterRegex = new ("((c(h)*(hapter)*){1} *([0-9]+(-[0-9]+)?){1})", RegexOptions.IgnoreCase);
|
||||
Regex singleResultRegex = new("([0-9]+)", RegexOptions.IgnoreCase);
|
||||
Regex rangeResultRegex = new("([0-9]+(-[0-9]+))", RegexOptions.IgnoreCase);
|
||||
Regex allRegex = new("a(ll)?", RegexOptions.IgnoreCase);
|
||||
if (volumeRegex.IsMatch(searchTerm) && chapterRegex.IsMatch(searchTerm))
|
||||
{
|
||||
string volume = singleResultRegex.Match(volumeRegex.Match(searchTerm).Value).Value;
|
||||
string chapter = singleResultRegex.Match(chapterRegex.Match(searchTerm).Value).Value;
|
||||
return availableChapters.Where(aCh => aCh.volumeNumber is not null &&
|
||||
aCh.volumeNumber.Equals(volume, StringComparison.InvariantCultureIgnoreCase) &&
|
||||
aCh.chapterNumber.Equals(chapter, StringComparison.InvariantCultureIgnoreCase))
|
||||
.ToArray();
|
||||
}
|
||||
else if (volumeRegex.IsMatch(searchTerm))
|
||||
{
|
||||
string volume = volumeRegex.Match(searchTerm).Value;
|
||||
if (rangeResultRegex.IsMatch(volume))
|
||||
{
|
||||
string range = rangeResultRegex.Match(volume).Value;
|
||||
int start = Convert.ToInt32(range.Split('-')[0]);
|
||||
int end = Convert.ToInt32(range.Split('-')[1]);
|
||||
return availableChapters.Where(aCh => aCh.volumeNumber is not null &&
|
||||
Convert.ToInt32(aCh.volumeNumber) >= start &&
|
||||
Convert.ToInt32(aCh.volumeNumber) <= end).ToArray();
|
||||
}
|
||||
else if (singleResultRegex.IsMatch(volume))
|
||||
{
|
||||
string volumeNumber = singleResultRegex.Match(volume).Value;
|
||||
return availableChapters.Where(aCh =>
|
||||
aCh.volumeNumber is not null &&
|
||||
aCh.volumeNumber.Equals(volumeNumber, StringComparison.InvariantCultureIgnoreCase)).ToArray();
|
||||
}
|
||||
|
||||
}
|
||||
else if (chapterRegex.IsMatch(searchTerm))
|
||||
{
|
||||
string chapter = chapterRegex.Match(searchTerm).Value;
|
||||
if (rangeResultRegex.IsMatch(chapter))
|
||||
{
|
||||
string range = rangeResultRegex.Match(chapter).Value;
|
||||
int start = Convert.ToInt32(range.Split('-')[0]);
|
||||
int end = Convert.ToInt32(range.Split('-')[1]);
|
||||
return availableChapters.Where(aCh => Convert.ToInt32(aCh.chapterNumber) >= start &&
|
||||
Convert.ToInt32(aCh.chapterNumber) <= end).ToArray();
|
||||
}
|
||||
else if (singleResultRegex.IsMatch(chapter))
|
||||
{
|
||||
string chapterNumber = singleResultRegex.Match(chapter).Value;
|
||||
return availableChapters.Where(aCh =>
|
||||
aCh.chapterNumber.Equals(chapterNumber, StringComparison.InvariantCultureIgnoreCase)).ToArray();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (rangeResultRegex.IsMatch(searchTerm))
|
||||
{
|
||||
int start = Convert.ToInt32(searchTerm.Split('-')[0]);
|
||||
int end = Convert.ToInt32(searchTerm.Split('-')[1]);
|
||||
return availableChapters[start..(end + 1)];
|
||||
}
|
||||
else if(singleResultRegex.IsMatch(searchTerm))
|
||||
return new [] { availableChapters[Convert.ToInt32(searchTerm)] };
|
||||
else if (allRegex.IsMatch(searchTerm))
|
||||
return availableChapters;
|
||||
}
|
||||
|
||||
return Array.Empty<Chapter>();
|
||||
}
|
||||
|
||||
public abstract HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null);
|
||||
|
||||
@ -217,8 +140,10 @@ public abstract class MangaConnector : GlobalBase
|
||||
return requestResult.statusCode;
|
||||
}
|
||||
|
||||
protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, RequestType requestType, string? comicInfoPath = null, string? referrer = null, ProgressToken? progressToken = null)
|
||||
protected HttpStatusCode DownloadChapterImages(string[] imageUrls, Chapter chapter, RequestType requestType, string? referrer = null, ProgressToken? progressToken = null)
|
||||
{
|
||||
string saveArchiveFilePath = chapter.GetArchiveFilePath();
|
||||
|
||||
if (progressToken?.cancellationRequested ?? false)
|
||||
return HttpStatusCode.RequestTimeout;
|
||||
Log($"Downloading Images for {saveArchiveFilePath}");
|
||||
@ -234,12 +159,15 @@ public abstract class MangaConnector : GlobalBase
|
||||
Directory.CreateDirectory(directoryPath);
|
||||
|
||||
if (File.Exists(saveArchiveFilePath)) //Don't download twice.
|
||||
{
|
||||
progressToken?.Complete();
|
||||
return HttpStatusCode.Created;
|
||||
}
|
||||
|
||||
//Create a temporary folder to store images
|
||||
string tempFolder = Directory.CreateTempSubdirectory("trangatemp").FullName;
|
||||
|
||||
int chapter = 0;
|
||||
int chapterNum = 0;
|
||||
//Download all Images to temporary Folder
|
||||
if (imageUrls.Length == 0)
|
||||
{
|
||||
@ -253,9 +181,9 @@ public abstract class MangaConnector : GlobalBase
|
||||
foreach (string imageUrl in imageUrls)
|
||||
{
|
||||
string extension = imageUrl.Split('.')[^1].Split('?')[0];
|
||||
Log($"Downloading image {chapter + 1:000}/{imageUrls.Length:000}"); //TODO
|
||||
HttpStatusCode status = DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapter++}.{extension}"), requestType, referrer);
|
||||
Log($"{saveArchiveFilePath} {chapter + 1:000}/{imageUrls.Length:000} {status}");
|
||||
Log($"Downloading image {chapterNum + 1:000}/{imageUrls.Length:000}"); //TODO
|
||||
HttpStatusCode status = DownloadImage(imageUrl, Path.Join(tempFolder, $"{chapterNum++}.{extension}"), requestType, referrer);
|
||||
Log($"{saveArchiveFilePath} {chapterNum + 1:000}/{imageUrls.Length:000} {status}");
|
||||
if ((int)status < 200 || (int)status >= 300)
|
||||
{
|
||||
progressToken?.Complete();
|
||||
@ -269,23 +197,23 @@ public abstract class MangaConnector : GlobalBase
|
||||
progressToken?.Increment();
|
||||
}
|
||||
|
||||
if(comicInfoPath is not null){
|
||||
File.Copy(comicInfoPath, Path.Join(tempFolder, "ComicInfo.xml"));
|
||||
File.Delete(comicInfoPath); //Delete tmp-file
|
||||
}
|
||||
File.WriteAllText(Path.Join(tempFolder, "ComicInfo.xml"), chapter.GetComicInfoXmlString());
|
||||
|
||||
Log($"Creating archive {saveArchiveFilePath}");
|
||||
//ZIP-it and ship-it
|
||||
ZipFile.CreateFromDirectory(tempFolder, saveArchiveFilePath);
|
||||
if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
|
||||
File.SetUnixFileMode(saveArchiveFilePath, UserRead | UserWrite | UserExecute | GroupRead | GroupWrite | GroupExecute);
|
||||
chapter.CreateChapterMarker();
|
||||
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
|
||||
File.SetUnixFileMode(saveArchiveFilePath, UserRead | UserWrite | UserExecute | GroupRead | GroupWrite | GroupExecute | OtherRead | OtherExecute);
|
||||
Directory.Delete(tempFolder, true); //Cleanup
|
||||
|
||||
Log("Created archive.");
|
||||
progressToken?.Complete();
|
||||
Log("Download complete.");
|
||||
return HttpStatusCode.OK;
|
||||
}
|
||||
|
||||
protected string SaveCoverImageToCache(string url, string mangaInternalId, RequestType requestType)
|
||||
protected string SaveCoverImageToCache(string url, string mangaInternalId, RequestType requestType, string? referrer = null)
|
||||
{
|
||||
Regex urlRex = new (@"https?:\/\/((?:[a-zA-Z0-9-]+\.)+[a-zA-Z0-9]+)\/(?:.+\/)*(.+\.([a-zA-Z]+))");
|
||||
//https?:\/\/[a-zA-Z0-9-]+\.([a-zA-Z0-9-]+\.[a-zA-Z0-9]+)\/(?:.+\/)*(.+\.([a-zA-Z]+)) for only second level domains
|
||||
@ -296,7 +224,7 @@ public abstract class MangaConnector : GlobalBase
|
||||
if (File.Exists(saveImagePath))
|
||||
return saveImagePath;
|
||||
|
||||
RequestResult coverResult = downloadClient.MakeRequest(url, requestType);
|
||||
RequestResult coverResult = downloadClient.MakeRequest(url, requestType, referrer);
|
||||
using MemoryStream ms = new();
|
||||
coverResult.result.CopyTo(ms);
|
||||
Directory.CreateDirectory(TrangaSettings.coverImageCache);
|
||||
|
@ -1,4 +1,6 @@
|
||||
using Newtonsoft.Json;
|
||||
using System.Data;
|
||||
using System.Diagnostics;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace Tranga.MangaConnectors;
|
||||
@ -22,29 +24,23 @@ public class MangaConnectorJsonConverter : JsonConverter
|
||||
public override object ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer)
|
||||
{
|
||||
JObject jo = JObject.Load(reader);
|
||||
switch (jo.GetValue("name")!.Value<string>()!)
|
||||
string? connectorName = jo.Value<string>("name");
|
||||
if (connectorName is null)
|
||||
throw new ConstraintException("Name can not be null.");
|
||||
return connectorName switch
|
||||
{
|
||||
case "MangaDex":
|
||||
return this._connectors.First(c => c is MangaDex);
|
||||
case "Manganato":
|
||||
return this._connectors.First(c => c is Manganato);
|
||||
case "MangaKatana":
|
||||
return this._connectors.First(c => c is MangaKatana);
|
||||
case "Mangasee":
|
||||
return this._connectors.First(c => c is Mangasee);
|
||||
case "Mangaworld":
|
||||
return this._connectors.First(c => c is Mangaworld);
|
||||
case "Bato":
|
||||
return this._connectors.First(c => c is Bato);
|
||||
case "Manga4Life":
|
||||
return this._connectors.First(c => c is MangaLife);
|
||||
case "ManhuaPlus":
|
||||
return this._connectors.First(c => c is ManhuaPlus);
|
||||
case "MangaHere":
|
||||
return this._connectors.First(c => c is MangaHere);
|
||||
}
|
||||
|
||||
throw new Exception();
|
||||
"MangaDex" => this._connectors.First(c => c is MangaDex),
|
||||
"Manganato" => this._connectors.First(c => c is Manganato),
|
||||
"MangaKatana" => this._connectors.First(c => c is MangaKatana),
|
||||
"Mangaworld" => this._connectors.First(c => c is Mangaworld),
|
||||
"Bato" => this._connectors.First(c => c is Bato),
|
||||
"ManhuaPlus" => this._connectors.First(c => c is ManhuaPlus),
|
||||
"MangaHere" => this._connectors.First(c => c is MangaHere),
|
||||
"AsuraToon" => this._connectors.First(c => c is AsuraToon),
|
||||
"Weebcentral" => this._connectors.First(c => c is Weebcentral),
|
||||
"Webtoons" => this._connectors.First(c => c is Webtoons),
|
||||
_ => throw new UnreachableException($"Could not find Connector with name {connectorName}")
|
||||
};
|
||||
}
|
||||
|
||||
public override bool CanWrite => false;
|
||||
|
@ -7,14 +7,17 @@ using JsonSerializer = System.Text.Json.JsonSerializer;
|
||||
namespace Tranga.MangaConnectors;
|
||||
public class MangaDex : MangaConnector
|
||||
{
|
||||
public MangaDex(GlobalBase clone) : base(clone, "MangaDex")
|
||||
//https://api.mangadex.org/docs/3-enumerations/#language-codes--localization
|
||||
//https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes
|
||||
//https://gist.github.com/Josantonius/b455e315bc7f790d14b136d61d9ae469
|
||||
public MangaDex(GlobalBase clone) : base(clone, "MangaDex", ["en","pt","pt-br","it","de","ru","aa","ab","ae","af","ak","am","an","ar-ae","ar-bh","ar-dz","ar-eg","ar-iq","ar-jo","ar-kw","ar-lb","ar-ly","ar-ma","ar-om","ar-qa","ar-sa","ar-sy","ar-tn","ar-ye","ar","as","av","ay","az","ba","be","bg","bh","bi","bm","bn","bo","br","bs","ca","ce","ch","co","cr","cs","cu","cv","cy","da","de-at","de-ch","de-de","de-li","de-lu","div","dv","dz","ee","el","en-au","en-bz","en-ca","en-cb","en-gb","en-ie","en-jm","en-nz","en-ph","en-tt","en-us","en-za","en-zw","eo","es-ar","es-bo","es-cl","es-co","es-cr","es-do","es-ec","es-es","es-gt","es-hn","es-la","es-mx","es-ni","es-pa","es-pe","es-pr","es-py","es-sv","es-us","es-uy","es-ve","es","et","eu","fa","ff","fi","fj","fo","fr-be","fr-ca","fr-ch","fr-fr","fr-lu","fr-mc","fr","fy","ga","gd","gl","gn","gu","gv","ha","he","hi","ho","hr-ba","hr-hr","hr","ht","hu","hy","hz","ia","id","ie","ig","ii","ik","in","io","is","it-ch","it-it","iu","iw","ja","ja-ro","ji","jv","jw","ka","kg","ki","kj","kk","kl","km","kn","ko","ko-ro","kr","ks","ku","kv","kw","ky","kz","la","lb","lg","li","ln","lo","ls","lt","lu","lv","mg","mh","mi","mk","ml","mn","mo","mr","ms-bn","ms-my","ms","mt","my","na","nb","nd","ne","ng","nl-be","nl-nl","nl","nn","no","nr","ns","nv","ny","oc","oj","om","or","os","pa","pi","pl","ps","pt-pt","qu-bo","qu-ec","qu-pe","qu","rm","rn","ro","rw","sa","sb","sc","sd","se-fi","se-no","se-se","se","sg","sh","si","sk","sl","sm","sn","so","sq","sr-ba","sr-sp","sr","ss","st","su","sv-fi","sv-se","sv","sw","sx","syr","ta","te","tg","th","ti","tk","tl","tn","to","tr","ts","tt","tw","ty","ug","uk","ur","us","uz","ve","vi","vo","wa","wo","xh","yi","yo","za","zh-cn","zh-hk","zh-mo","zh-ro","zh-sg","zh-tw","zh","zu"])
|
||||
{
|
||||
this.downloadClient = new HttpDownloadClient(clone);
|
||||
}
|
||||
|
||||
public override Manga[] GetManga(string publicationTitle = "")
|
||||
{
|
||||
Log($"Searching Publications. Term=\"{publicationTitle}\"");
|
||||
Log($"Searching Publications. Term={publicationTitle}");
|
||||
const int limit = 100; //How many values we want returned at once
|
||||
int offset = 0; //"Page"
|
||||
int total = int.MaxValue; //How many total results are there, is updated on first request
|
||||
@ -54,7 +57,7 @@ public class MangaDex : MangaConnector
|
||||
if(MangaFromJsonObject(mangaNode.AsObject()) is { } manga)
|
||||
retManga.Add(manga); //Add Publication (Manga) to result
|
||||
}
|
||||
Log($"Retrieved {retManga.Count} publications. Term=\"{publicationTitle}\"");
|
||||
Log($"Retrieved {retManga.Count} publications. Term={publicationTitle}");
|
||||
return retManga.ToArray();
|
||||
}
|
||||
|
||||
@ -243,8 +246,17 @@ public class MangaDex : MangaConnector
|
||||
continue;
|
||||
}
|
||||
|
||||
if(chapterNum is not "null")
|
||||
chapters.Add(new Chapter(manga, title, volume, chapterNum, chapterId));
|
||||
try
|
||||
{
|
||||
if(!chapters.Any(chp =>
|
||||
chp.volumeNumber.Equals(float.Parse(volume??"0", numberFormatDecimalPoint)) &&
|
||||
chp.chapterNumber.Equals(float.Parse(chapterNum, numberFormatDecimalPoint))))
|
||||
chapters.Add(new Chapter(manga, title, volume, chapterNum, chapterId, chapterId));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log($"Failed to load chapter {chapterNum}: {e.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -285,11 +297,8 @@ public class MangaDex : MangaConnector
|
||||
HashSet<string> imageUrls = new();
|
||||
foreach (JsonNode? image in imageFileNames)
|
||||
imageUrls.Add($"{baseUrl}/data/{hash}/{image!.GetValue<string>()}");
|
||||
|
||||
string comicInfoPath = Path.GetTempFileName();
|
||||
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
|
||||
|
||||
//Download Chapter-Images
|
||||
return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(), RequestType.MangaImage, comicInfoPath, progressToken:progressToken);
|
||||
return DownloadChapterImages(imageUrls.ToArray(), chapter, RequestType.MangaImage, progressToken:progressToken);
|
||||
}
|
||||
}
|
@ -7,7 +7,7 @@ namespace Tranga.MangaConnectors;
|
||||
|
||||
public class MangaHere : MangaConnector
|
||||
{
|
||||
public MangaHere(GlobalBase clone) : base(clone, "MangaHere")
|
||||
public MangaHere(GlobalBase clone) : base(clone, "MangaHere", ["en"])
|
||||
{
|
||||
this.downloadClient = new ChromiumDownloadClient(clone);
|
||||
}
|
||||
@ -117,7 +117,7 @@ public class MangaHere : MangaConnector
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 || requestResult.htmlDocument is null)
|
||||
return Array.Empty<Chapter>();
|
||||
|
||||
List<string> urls = requestResult.htmlDocument.DocumentNode.SelectNodes("//div[@id='list-2']/ul//li//a[contains(@href, '/manga/')]")
|
||||
List<string> urls = requestResult.htmlDocument.DocumentNode.SelectNodes("//div[@id='list-1']/ul//li//a[contains(@href, '/manga/')]")
|
||||
.Select(node => node.GetAttributeValue("href", "")).ToList();
|
||||
Regex chapterRex = new(@".*\/manga\/[a-zA-Z0-9\-\._\~\!\$\&\'\(\)\*\+\,\;\=\:\@]+\/v([0-9(TBD)]+)\/c([0-9\.]+)\/.*");
|
||||
|
||||
@ -129,7 +129,15 @@ public class MangaHere : MangaConnector
|
||||
string volumeNumber = rexMatch.Groups[1].Value == "TBD" ? "0" : rexMatch.Groups[1].Value;
|
||||
string chapterNumber = rexMatch.Groups[2].Value;
|
||||
string fullUrl = $"https://www.mangahere.cc{url}";
|
||||
chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, fullUrl));
|
||||
|
||||
try
|
||||
{
|
||||
chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, fullUrl));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log($"Failed to load chapter {chapterNumber}: {e.Message}");
|
||||
}
|
||||
}
|
||||
//Return Chapters ordered by Chapter-Number
|
||||
Log($"Got {chapters.Count} chapters. {manga}");
|
||||
@ -181,12 +189,9 @@ public class MangaHere : MangaConnector
|
||||
}
|
||||
} while (downloaded++ <= images);
|
||||
|
||||
string comicInfoPath = Path.GetTempFileName();
|
||||
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
|
||||
|
||||
if (progressToken is not null)
|
||||
progressToken.increments = images;//we blip to normal length, in downloadchapterimages it is increasaed by the amount of urls again
|
||||
return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(), RequestType.MangaImage, comicInfoPath, progressToken:progressToken);
|
||||
return DownloadChapterImages(imageUrls.ToArray(), chapter, RequestType.MangaImage, progressToken:progressToken);
|
||||
}
|
||||
|
||||
private string[] ParseImageUrlsFromHtml(HtmlDocument document)
|
||||
|
@ -7,7 +7,7 @@ namespace Tranga.MangaConnectors;
|
||||
|
||||
public class MangaKatana : MangaConnector
|
||||
{
|
||||
public MangaKatana(GlobalBase clone) : base(clone, "MangaKatana")
|
||||
public MangaKatana(GlobalBase clone) : base(clone, "MangaKatana", ["en"])
|
||||
{
|
||||
this.downloadClient = new HttpDownloadClient(clone);
|
||||
}
|
||||
@ -15,7 +15,7 @@ public class MangaKatana : MangaConnector
|
||||
public override Manga[] GetManga(string publicationTitle = "")
|
||||
{
|
||||
Log($"Searching Publications. Term=\"{publicationTitle}\"");
|
||||
string sanitizedTitle = string.Join('_', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower();
|
||||
string sanitizedTitle = string.Join("%20", Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower();
|
||||
string requestUrl = $"https://mangakatana.com/?search={sanitizedTitle}&search_by=book_name";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
@ -186,7 +186,14 @@ public class MangaKatana : MangaConnector
|
||||
string? volumeNumber = volumeRex.IsMatch(url) ? volumeRex.Match(url).Groups[1].Value : null;
|
||||
string chapterNumber = chapterNumRex.Match(url).Groups[1].Value;
|
||||
string chapterName = chapterNameRex.Match(fullString).Groups[1].Value;
|
||||
ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url));
|
||||
try
|
||||
{
|
||||
ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log($"Failed to load chapter {chapterNumber}: {e.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
@ -213,11 +220,8 @@ public class MangaKatana : MangaConnector
|
||||
}
|
||||
|
||||
string[] imageUrls = ParseImageUrlsFromHtml(requestUrl);
|
||||
|
||||
string comicInfoPath = Path.GetTempFileName();
|
||||
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
|
||||
|
||||
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(), RequestType.MangaImage, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken);
|
||||
|
||||
return DownloadChapterImages(imageUrls, chapter, RequestType.MangaImage, progressToken:progressToken);
|
||||
}
|
||||
|
||||
private string[] ParseImageUrlsFromHtml(string mangaUrl)
|
||||
|
@ -1,199 +0,0 @@
|
||||
using System.Net;
|
||||
using System.Text.RegularExpressions;
|
||||
using HtmlAgilityPack;
|
||||
using Tranga.Jobs;
|
||||
|
||||
namespace Tranga.MangaConnectors;
|
||||
|
||||
public class MangaLife : MangaConnector
|
||||
{
|
||||
public MangaLife(GlobalBase clone) : base(clone, "Manga4Life")
|
||||
{
|
||||
this.downloadClient = new ChromiumDownloadClient(clone);
|
||||
}
|
||||
|
||||
public override Manga[] GetManga(string publicationTitle = "")
|
||||
{
|
||||
Log($"Searching Publications. Term=\"{publicationTitle}\"");
|
||||
string sanitizedTitle = WebUtility.UrlEncode(publicationTitle);
|
||||
string requestUrl = $"https://manga4life.com/search/?name={sanitizedTitle}";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return Array.Empty<Manga>();
|
||||
|
||||
if (requestResult.htmlDocument is null)
|
||||
return Array.Empty<Manga>();
|
||||
Manga[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument);
|
||||
Log($"Retrieved {publications.Length} publications. Term=\"{publicationTitle}\"");
|
||||
return publications;
|
||||
}
|
||||
|
||||
public override Manga? GetMangaFromId(string publicationId)
|
||||
{
|
||||
return GetMangaFromUrl($"https://manga4life.com/manga/{publicationId}");
|
||||
}
|
||||
|
||||
public override Manga? GetMangaFromUrl(string url)
|
||||
{
|
||||
Regex publicationIdRex = new(@"https:\/\/(www\.)?manga4life.com\/manga\/(.*)(\/.*)*");
|
||||
string publicationId = publicationIdRex.Match(url).Groups[2].Value;
|
||||
|
||||
RequestResult requestResult = this.downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||
if(requestResult.htmlDocument is not null)
|
||||
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url);
|
||||
return null;
|
||||
}
|
||||
|
||||
private Manga[] ParsePublicationsFromHtml(HtmlDocument document)
|
||||
{
|
||||
HtmlNode resultsNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']/div[last()]/div[1]/div");
|
||||
if (resultsNode.Descendants("div").Count() == 1 && resultsNode.Descendants("div").First().HasClass("NoResults"))
|
||||
{
|
||||
Log("No results.");
|
||||
return Array.Empty<Manga>();
|
||||
}
|
||||
Log($"{resultsNode.SelectNodes("div").Count} items.");
|
||||
|
||||
HashSet<Manga> ret = new();
|
||||
|
||||
foreach (HtmlNode resultNode in resultsNode.SelectNodes("div"))
|
||||
{
|
||||
string url = resultNode.Descendants().First(d => d.HasClass("SeriesName")).GetAttributeValue("href", "");
|
||||
Manga? manga = GetMangaFromUrl($"https://manga4life.com{url}");
|
||||
if (manga is not null)
|
||||
ret.Add((Manga)manga);
|
||||
}
|
||||
|
||||
return ret.ToArray();
|
||||
}
|
||||
|
||||
|
||||
private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl)
|
||||
{
|
||||
string originalLanguage = "", status = "";
|
||||
Dictionary<string, string> altTitles = new(), links = new();
|
||||
HashSet<string> tags = new();
|
||||
Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased;
|
||||
|
||||
HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//img");
|
||||
string posterUrl = posterNode.GetAttributeValue("src", "");
|
||||
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover);
|
||||
|
||||
HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//h1");
|
||||
string sortName = titleNode.InnerText;
|
||||
|
||||
HtmlNode[] authorsNodes = document.DocumentNode
|
||||
.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Author(s):']/..").Descendants("a")
|
||||
.ToArray();
|
||||
List<string> authors = new();
|
||||
foreach (HtmlNode authorNode in authorsNodes)
|
||||
authors.Add(authorNode.InnerText);
|
||||
|
||||
HtmlNode[] genreNodes = document.DocumentNode
|
||||
.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Genre(s):']/..").Descendants("a")
|
||||
.ToArray();
|
||||
foreach (HtmlNode genreNode in genreNodes)
|
||||
tags.Add(genreNode.InnerText);
|
||||
|
||||
HtmlNode yearNode = document.DocumentNode
|
||||
.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Released:']/..").Descendants("a")
|
||||
.First();
|
||||
int year = Convert.ToInt32(yearNode.InnerText);
|
||||
|
||||
HtmlNode[] statusNodes = document.DocumentNode
|
||||
.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Status:']/..").Descendants("a")
|
||||
.ToArray();
|
||||
foreach (HtmlNode statusNode in statusNodes)
|
||||
if (statusNode.InnerText.Contains("publish", StringComparison.CurrentCultureIgnoreCase))
|
||||
status = statusNode.InnerText.Split(' ')[0];
|
||||
switch (status.ToLower())
|
||||
{
|
||||
case "cancelled": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break;
|
||||
case "hiatus": releaseStatus = Manga.ReleaseStatusByte.OnHiatus; break;
|
||||
case "discontinued": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break;
|
||||
case "complete": releaseStatus = Manga.ReleaseStatusByte.Completed; break;
|
||||
case "ongoing": releaseStatus = Manga.ReleaseStatusByte.Continuing; break;
|
||||
}
|
||||
|
||||
HtmlNode descriptionNode = document.DocumentNode
|
||||
.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Description:']/..")
|
||||
.Descendants("div").First();
|
||||
string description = descriptionNode.InnerText;
|
||||
|
||||
Manga manga = new(sortName, authors.ToList(), description, altTitles, tags.ToArray(), posterUrl,
|
||||
coverFileNameInCache, links, year, originalLanguage, publicationId, releaseStatus, websiteUrl: websiteUrl);
|
||||
AddMangaToCache(manga);
|
||||
return manga;
|
||||
}
|
||||
|
||||
public override Chapter[] GetChapters(Manga manga, string language="en")
|
||||
{
|
||||
Log($"Getting chapters {manga}");
|
||||
RequestResult result = downloadClient.MakeRequest($"https://manga4life.com/manga/{manga.publicationId}", RequestType.Default, clickButton:"[class*='ShowAllChapters']");
|
||||
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null)
|
||||
{
|
||||
return Array.Empty<Chapter>();
|
||||
}
|
||||
|
||||
HtmlNodeCollection chapterNodes = result.htmlDocument.DocumentNode.SelectNodes(
|
||||
"//a[contains(concat(' ',normalize-space(@class),' '),' ChapterLink ')]");
|
||||
string[] urls = chapterNodes.Select(node => node.GetAttributeValue("href", "")).ToArray();
|
||||
Regex urlRex = new (@"-chapter-([0-9\\.]+)(-index-([0-9\\.]+))?");
|
||||
|
||||
List<Chapter> chapters = new();
|
||||
foreach (string url in urls)
|
||||
{
|
||||
Match rexMatch = urlRex.Match(url);
|
||||
|
||||
string volumeNumber = "1";
|
||||
if (rexMatch.Groups[3].Value.Length > 0)
|
||||
volumeNumber = rexMatch.Groups[3].Value;
|
||||
string chapterNumber = rexMatch.Groups[1].Value;
|
||||
string fullUrl = $"https://manga4life.com{url}";
|
||||
fullUrl = fullUrl.Replace(Regex.Match(url,"(-page-[0-9])").Value,"");
|
||||
chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, fullUrl));
|
||||
}
|
||||
//Return Chapters ordered by Chapter-Number
|
||||
Log($"Got {chapters.Count} chapters. {manga}");
|
||||
return chapters.Order().ToArray();
|
||||
}
|
||||
|
||||
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
|
||||
{
|
||||
if (progressToken?.cancellationRequested ?? false)
|
||||
{
|
||||
progressToken.Cancel();
|
||||
return HttpStatusCode.RequestTimeout;
|
||||
}
|
||||
|
||||
Manga chapterParentManga = chapter.parentManga;
|
||||
if (progressToken?.cancellationRequested ?? false)
|
||||
{
|
||||
progressToken.Cancel();
|
||||
return HttpStatusCode.RequestTimeout;
|
||||
}
|
||||
|
||||
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
|
||||
|
||||
RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, RequestType.Default);
|
||||
if (requestResult.htmlDocument is null)
|
||||
{
|
||||
progressToken?.Cancel();
|
||||
return HttpStatusCode.RequestTimeout;
|
||||
}
|
||||
|
||||
HtmlDocument document = requestResult.htmlDocument;
|
||||
|
||||
HtmlNode gallery = document.DocumentNode.Descendants("div").First(div => div.HasClass("ImageGallery"));
|
||||
HtmlNode[] images = gallery.Descendants("img").Where(img => img.HasClass("img-fluid")).ToArray();
|
||||
List<string> urls = new();
|
||||
foreach(HtmlNode galleryImage in images)
|
||||
urls.Add(galleryImage.GetAttributeValue("src", ""));
|
||||
|
||||
string comicInfoPath = Path.GetTempFileName();
|
||||
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
|
||||
|
||||
return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(), RequestType.MangaImage, comicInfoPath, progressToken:progressToken);
|
||||
}
|
||||
}
|
@ -8,7 +8,7 @@ namespace Tranga.MangaConnectors;
|
||||
|
||||
public class Manganato : MangaConnector
|
||||
{
|
||||
public Manganato(GlobalBase clone) : base(clone, "Manganato")
|
||||
public Manganato(GlobalBase clone) : base(clone, "Manganato", ["en"])
|
||||
{
|
||||
this.downloadClient = new HttpDownloadClient(clone);
|
||||
}
|
||||
@ -17,7 +17,7 @@ public class Manganato : MangaConnector
|
||||
{
|
||||
Log($"Searching Publications. Term=\"{publicationTitle}\"");
|
||||
string sanitizedTitle = string.Join('_', Regex.Matches(publicationTitle, "[A-z]*").Where(str => str.Length > 0)).ToLower();
|
||||
string requestUrl = $"https://manganato.com/search/story/{sanitizedTitle}";
|
||||
string requestUrl = $"https://manganato.gg/search/story/{sanitizedTitle}";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
@ -32,13 +32,19 @@ public class Manganato : MangaConnector
|
||||
|
||||
private Manga[] ParsePublicationsFromHtml(HtmlDocument document)
|
||||
{
|
||||
List<HtmlNode> searchResults = document.DocumentNode.Descendants("div").Where(n => n.HasClass("search-story-item")).ToList();
|
||||
List<HtmlNode> searchResults = document.DocumentNode.Descendants("div").Where(n => n.HasClass("story_item")).ToList();
|
||||
Log($"{searchResults.Count} items.");
|
||||
List<string> urls = new();
|
||||
foreach (HtmlNode mangaResult in searchResults)
|
||||
{
|
||||
urls.Add(mangaResult.Descendants("a").First(n => n.HasClass("item-title")).GetAttributes()
|
||||
.First(a => a.Name == "href").Value);
|
||||
try
|
||||
{
|
||||
urls.Add(mangaResult.Descendants("h3").First(n => n.HasClass("story_name"))
|
||||
.Descendants("a").First().GetAttributeValue("href", ""));
|
||||
} catch
|
||||
{
|
||||
//failed to get a url, send it to the void
|
||||
}
|
||||
}
|
||||
|
||||
HashSet<Manga> ret = new();
|
||||
@ -78,65 +84,53 @@ public class Manganato : MangaConnector
|
||||
string originalLanguage = "";
|
||||
Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased;
|
||||
|
||||
HtmlNode infoNode = document.DocumentNode.Descendants("div").First(d => d.HasClass("story-info-right"));
|
||||
HtmlNode infoNode = document.DocumentNode.Descendants("ul").First(d => d.HasClass("manga-info-text"));
|
||||
|
||||
string sortName = infoNode.Descendants("h1").First().InnerText;
|
||||
|
||||
HtmlNode infoTable = infoNode.Descendants().First(d => d.Name == "table");
|
||||
|
||||
foreach (HtmlNode row in infoTable.Descendants("tr"))
|
||||
foreach (HtmlNode li in infoNode.Descendants("li"))
|
||||
{
|
||||
string key = row.SelectNodes("td").First().InnerText.ToLower();
|
||||
string value = row.SelectNodes("td").Last().InnerText;
|
||||
string keySanitized = string.Concat(Regex.Matches(key, "[a-z]"));
|
||||
|
||||
switch (keySanitized)
|
||||
string text = li.InnerText.Trim().ToLower();
|
||||
|
||||
if (text.StartsWith("author(s) :"))
|
||||
{
|
||||
case "alternative":
|
||||
string[] alts = value.Split(" ; ");
|
||||
for(int i = 0; i < alts.Length; i++)
|
||||
altTitles.Add(i.ToString(), alts[i]);
|
||||
break;
|
||||
case "authors":
|
||||
authors = value.Split('-');
|
||||
for (int i = 0; i < authors.Length; i++)
|
||||
authors[i] = authors[i].Replace("\r\n", "");
|
||||
break;
|
||||
case "status":
|
||||
switch (value.ToLower())
|
||||
{
|
||||
case "ongoing": releaseStatus = Manga.ReleaseStatusByte.Continuing; break;
|
||||
case "completed": releaseStatus = Manga.ReleaseStatusByte.Completed; break;
|
||||
}
|
||||
break;
|
||||
case "genres":
|
||||
string[] genres = value.Split(" - ");
|
||||
for (int i = 0; i < genres.Length; i++)
|
||||
genres[i] = genres[i].Replace("\r\n", "");
|
||||
tags = genres.ToHashSet();
|
||||
break;
|
||||
authors = li.Descendants("a").Select(a => a.InnerText.Trim()).ToArray();
|
||||
}
|
||||
else if (text.StartsWith("status :"))
|
||||
{
|
||||
string status = text.Replace("status :", "").Trim().ToLower();
|
||||
if (string.IsNullOrWhiteSpace(status))
|
||||
releaseStatus = Manga.ReleaseStatusByte.Continuing;
|
||||
else if (status == "ongoing")
|
||||
releaseStatus = Manga.ReleaseStatusByte.Continuing;
|
||||
else
|
||||
releaseStatus = Enum.Parse<Manga.ReleaseStatusByte>(status, true);
|
||||
}
|
||||
else if (li.HasClass("genres"))
|
||||
{
|
||||
tags = li.Descendants("a").Select(a => a.InnerText.Trim()).ToHashSet();
|
||||
}
|
||||
}
|
||||
|
||||
string posterUrl = document.DocumentNode.Descendants("span").First(s => s.HasClass("info-image")).Descendants("img").First()
|
||||
string posterUrl = document.DocumentNode.Descendants("div").First(s => s.HasClass("manga-info-pic")).Descendants("img").First()
|
||||
.GetAttributes().First(a => a.Name == "src").Value;
|
||||
|
||||
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover);
|
||||
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover, "https://www.manganato.gg/");
|
||||
|
||||
string description = document.DocumentNode.Descendants("div").First(d => d.HasClass("panel-story-info-description"))
|
||||
string description = document.DocumentNode.SelectSingleNode("//div[@id='contentBox']")
|
||||
.InnerText.Replace("Description :", "");
|
||||
while (description.StartsWith('\n'))
|
||||
description = description.Substring(1);
|
||||
|
||||
string pattern = "MMM dd,yyyy HH:mm";
|
||||
string pattern = "MMM-dd-yyyy HH:mm";
|
||||
|
||||
HtmlNode oldestChapter = document.DocumentNode
|
||||
.SelectNodes("//chapter-time[contains(concat(' ',normalize-space(@class),' '),' chapter-time ')]").MaxBy(
|
||||
node => DateTime.ParseExact(node.GetAttributeValue("title", "Dec 31 2400, 23:59"), pattern,
|
||||
CultureInfo.InvariantCulture))!;
|
||||
HtmlNode? oldestChapter = document.DocumentNode
|
||||
.SelectNodes("//div[contains(concat(' ',normalize-space(@class),' '),' row ')]/span[@title]").MaxBy(
|
||||
node => DateTime.ParseExact(node.GetAttributeValue("title", "Dec-31-2400 23:59"), pattern,
|
||||
CultureInfo.InvariantCulture).Millisecond);
|
||||
|
||||
|
||||
int year = DateTime.ParseExact(oldestChapter.GetAttributeValue("title", "Dec 31 2400, 23:59"), pattern,
|
||||
int year = DateTime.ParseExact(oldestChapter?.GetAttributeValue("title", "Dec 31 2400, 23:59")??"Dec 31 2400, 23:59", pattern,
|
||||
CultureInfo.InvariantCulture).Year;
|
||||
|
||||
Manga manga = new (sortName, authors.ToList(), description, altTitles, tags.ToArray(), posterUrl, coverFileNameInCache, links,
|
||||
@ -148,7 +142,7 @@ public class Manganato : MangaConnector
|
||||
public override Chapter[] GetChapters(Manga manga, string language="en")
|
||||
{
|
||||
Log($"Getting chapters {manga}");
|
||||
string requestUrl = $"https://chapmanganato.com/{manga.publicationId}";
|
||||
string requestUrl = manga.websiteUrl;
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
@ -166,22 +160,29 @@ public class Manganato : MangaConnector
|
||||
{
|
||||
List<Chapter> ret = new();
|
||||
|
||||
HtmlNode chapterList = document.DocumentNode.Descendants("ul").First(l => l.HasClass("row-content-chapter"));
|
||||
HtmlNode chapterList = document.DocumentNode.Descendants("div").First(l => l.HasClass("chapter-list"));
|
||||
|
||||
Regex volRex = new(@"Vol\.([0-9]+).*");
|
||||
Regex chapterRex = new(@"https:\/\/chapmanganato.[A-z]+\/manga-[A-z0-9]+\/chapter-([0-9\.]+)");
|
||||
Regex nameRex = new(@"Chapter ([0-9]+(\.[0-9]+)*){1}:? (.*)");
|
||||
|
||||
foreach (HtmlNode chapterInfo in chapterList.Descendants("li"))
|
||||
foreach (HtmlNode chapterInfo in chapterList.Descendants("div").Where(x => x.HasClass("row")))
|
||||
{
|
||||
string fullString = chapterInfo.Descendants("a").First(d => d.HasClass("chapter-name")).InnerText;
|
||||
|
||||
string url = chapterInfo.Descendants("a").First(d => d.HasClass("chapter-name"))
|
||||
.GetAttributeValue("href", "");
|
||||
string? volumeNumber = volRex.IsMatch(fullString) ? volRex.Match(fullString).Groups[1].Value : null;
|
||||
string chapterNumber = chapterRex.Match(url).Groups[1].Value;
|
||||
string chapterName = nameRex.Match(fullString).Groups[3].Value;
|
||||
ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url));
|
||||
string url = chapterInfo.Descendants("a").First().GetAttributeValue("href", "");
|
||||
var name = chapterInfo.Descendants("a").First().InnerText.Trim();
|
||||
string chapterName = nameRex.Match(name).Groups[3].Value;
|
||||
string chapterNumber = Regex.Match(name, @"Chapter ([0-9]+(\.[0-9]+)*)").Groups[1].Value;
|
||||
string? volumeNumber = Regex.Match(chapterName, @"Vol\.([0-9]+)").Groups[1].Value;
|
||||
if (string.IsNullOrWhiteSpace(volumeNumber))
|
||||
volumeNumber = "0";
|
||||
try
|
||||
{
|
||||
ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log($"Failed to load chapter {chapterNumber}: {e.Message}");
|
||||
}
|
||||
}
|
||||
ret.Reverse();
|
||||
return ret;
|
||||
@ -214,10 +215,7 @@ public class Manganato : MangaConnector
|
||||
|
||||
string[] imageUrls = ParseImageUrlsFromHtml(requestResult.htmlDocument);
|
||||
|
||||
string comicInfoPath = Path.GetTempFileName();
|
||||
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
|
||||
|
||||
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(), RequestType.MangaImage, comicInfoPath, "https://chapmanganato.com/", progressToken:progressToken);
|
||||
return DownloadChapterImages(imageUrls, chapter, RequestType.MangaImage, "https://www.manganato.gg", progressToken:progressToken);
|
||||
}
|
||||
|
||||
private string[] ParseImageUrlsFromHtml(HtmlDocument document)
|
||||
|
@ -1,230 +0,0 @@
|
||||
using System.Data;
|
||||
using System.Net;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Xml.Linq;
|
||||
using HtmlAgilityPack;
|
||||
using Newtonsoft.Json;
|
||||
using Soenneker.Utils.String.NeedlemanWunsch;
|
||||
using Tranga.Jobs;
|
||||
|
||||
namespace Tranga.MangaConnectors;
|
||||
|
||||
public class Mangasee : MangaConnector
|
||||
{
|
||||
public Mangasee(GlobalBase clone) : base(clone, "Mangasee")
|
||||
{
|
||||
this.downloadClient = new ChromiumDownloadClient(clone);
|
||||
}
|
||||
|
||||
private struct SearchResult
|
||||
{
|
||||
public string i { get; set; }
|
||||
public string s { get; set; }
|
||||
public string[] a { get; set; }
|
||||
}
|
||||
|
||||
public override Manga[] GetManga(string publicationTitle = "")
|
||||
{
|
||||
Log($"Searching Publications. Term=\"{publicationTitle}\"");
|
||||
string requestUrl = "https://mangasee123.com/_search.php";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
{
|
||||
Log($"Failed to retrieve search: {requestResult.statusCode}");
|
||||
return Array.Empty<Manga>();
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
SearchResult[] searchResults = JsonConvert.DeserializeObject<SearchResult[]>(requestResult.htmlDocument!.DocumentNode.InnerText) ??
|
||||
throw new NoNullAllowedException();
|
||||
SearchResult[] filteredResults = FilteredResults(publicationTitle, searchResults);
|
||||
Log($"Total available manga: {searchResults.Length} Filtered down to: {filteredResults.Length}");
|
||||
|
||||
|
||||
string[] urls = filteredResults.Select(result => $"https://mangasee123.com/manga/{result.i}").ToArray();
|
||||
List<Manga> searchResultManga = new();
|
||||
foreach (string url in urls)
|
||||
{
|
||||
Manga? newManga = GetMangaFromUrl(url);
|
||||
if(newManga is { } manga)
|
||||
searchResultManga.Add(manga);
|
||||
}
|
||||
Log($"Retrieved {searchResultManga.Count} publications. Term=\"{publicationTitle}\"");
|
||||
return searchResultManga.ToArray();
|
||||
}
|
||||
catch (NoNullAllowedException)
|
||||
{
|
||||
Log("Failed to retrieve search");
|
||||
return Array.Empty<Manga>();
|
||||
}
|
||||
}
|
||||
|
||||
private readonly string[] _filterWords = {"a", "the", "of", "as", "to", "no", "for", "on", "with", "be", "and", "in", "wa", "at", "be", "ni"};
|
||||
private string ToFilteredString(string input) => string.Join(' ', input.ToLower().Split(' ').Where(word => _filterWords.Contains(word) == false));
|
||||
private SearchResult[] FilteredResults(string publicationTitle, SearchResult[] unfilteredSearchResults)
|
||||
{
|
||||
Dictionary<SearchResult, int> similarity = new();
|
||||
foreach (SearchResult sr in unfilteredSearchResults)
|
||||
{
|
||||
List<int> scores = new();
|
||||
string filteredPublicationString = ToFilteredString(publicationTitle);
|
||||
string filteredSString = ToFilteredString(sr.s);
|
||||
scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredSString, filteredPublicationString));
|
||||
foreach (string srA in sr.a)
|
||||
{
|
||||
string filteredAString = ToFilteredString(srA);
|
||||
scores.Add(NeedlemanWunschStringUtil.CalculateSimilarity(filteredAString, filteredPublicationString));
|
||||
}
|
||||
similarity.Add(sr, scores.Sum() / scores.Count);
|
||||
}
|
||||
|
||||
List<SearchResult> ret = similarity.OrderBy(s => s.Value).Take(10).Select(s => s.Key).ToList();
|
||||
return ret.ToArray();
|
||||
}
|
||||
|
||||
public override Manga? GetMangaFromId(string publicationId)
|
||||
{
|
||||
return GetMangaFromUrl($"https://mangasee123.com/manga/{publicationId}");
|
||||
}
|
||||
|
||||
public override Manga? GetMangaFromUrl(string url)
|
||||
{
|
||||
Regex publicationIdRex = new(@"https:\/\/mangasee123.com\/manga\/(.*)(\/.*)*");
|
||||
string publicationId = publicationIdRex.Match(url).Groups[1].Value;
|
||||
|
||||
RequestResult requestResult = this.downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||
if((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 && requestResult.htmlDocument is not null)
|
||||
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url);
|
||||
return null;
|
||||
}
|
||||
|
||||
private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl)
|
||||
{
|
||||
string originalLanguage = "", status = "";
|
||||
Dictionary<string, string> altTitles = new(), links = new();
|
||||
HashSet<string> tags = new();
|
||||
Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased;
|
||||
|
||||
HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//img");
|
||||
string posterUrl = posterNode.GetAttributeValue("src", "");
|
||||
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover);
|
||||
|
||||
HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//div[@class='BoxBody']//div[@class='row']//h1");
|
||||
string sortName = titleNode.InnerText;
|
||||
|
||||
HtmlNode[] authorsNodes = document.DocumentNode
|
||||
.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Author(s):']/..").Descendants("a")
|
||||
.ToArray();
|
||||
List<string> authors = new();
|
||||
foreach (HtmlNode authorNode in authorsNodes)
|
||||
authors.Add(authorNode.InnerText);
|
||||
|
||||
HtmlNode[] genreNodes = document.DocumentNode
|
||||
.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Genre(s):']/..").Descendants("a")
|
||||
.ToArray();
|
||||
foreach (HtmlNode genreNode in genreNodes)
|
||||
tags.Add(genreNode.InnerText);
|
||||
|
||||
HtmlNode yearNode = document.DocumentNode
|
||||
.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Released:']/..").Descendants("a")
|
||||
.First();
|
||||
int year = Convert.ToInt32(yearNode.InnerText);
|
||||
|
||||
HtmlNode[] statusNodes = document.DocumentNode
|
||||
.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Status:']/..").Descendants("a")
|
||||
.ToArray();
|
||||
foreach (HtmlNode statusNode in statusNodes)
|
||||
if (statusNode.InnerText.Contains("publish", StringComparison.CurrentCultureIgnoreCase))
|
||||
status = statusNode.InnerText.Split(' ')[0];
|
||||
switch (status.ToLower())
|
||||
{
|
||||
case "cancelled": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break;
|
||||
case "hiatus": releaseStatus = Manga.ReleaseStatusByte.OnHiatus; break;
|
||||
case "discontinued": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break;
|
||||
case "complete": releaseStatus = Manga.ReleaseStatusByte.Completed; break;
|
||||
case "ongoing": releaseStatus = Manga.ReleaseStatusByte.Continuing; break;
|
||||
}
|
||||
|
||||
HtmlNode descriptionNode = document.DocumentNode
|
||||
.SelectNodes("//div[@class='BoxBody']//div[@class='row']//span[text()='Description:']/..")
|
||||
.Descendants("div").First();
|
||||
string description = descriptionNode.InnerText;
|
||||
|
||||
Manga manga = new(sortName, authors.ToList(), description, altTitles, tags.ToArray(), posterUrl,
|
||||
coverFileNameInCache, links,
|
||||
year, originalLanguage, publicationId, releaseStatus, websiteUrl: websiteUrl);
|
||||
AddMangaToCache(manga);
|
||||
return manga;
|
||||
}
|
||||
|
||||
public override Chapter[] GetChapters(Manga manga, string language="en")
|
||||
{
|
||||
Log($"Getting chapters {manga}");
|
||||
try
|
||||
{
|
||||
XDocument doc = XDocument.Load($"https://mangasee123.com/rss/{manga.publicationId}.xml");
|
||||
XElement[] chapterItems = doc.Descendants("item").ToArray();
|
||||
List<Chapter> chapters = new();
|
||||
Regex chVolRex = new(@".*chapter-([0-9\.]+)(?:-index-([0-9\.]+))?.*");
|
||||
foreach (XElement chapter in chapterItems)
|
||||
{
|
||||
string url = chapter.Descendants("link").First().Value;
|
||||
Match m = chVolRex.Match(url);
|
||||
string? volumeNumber = m.Groups[2].Success ? m.Groups[2].Value : "1";
|
||||
string chapterNumber = m.Groups[1].Value;
|
||||
|
||||
string chapterUrl = Regex.Replace(url, @"-page-[0-9]+(\.html)", ".html");
|
||||
chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, chapterUrl));
|
||||
}
|
||||
|
||||
//Return Chapters ordered by Chapter-Number
|
||||
Log($"Got {chapters.Count} chapters. {manga}");
|
||||
return chapters.Order().ToArray();
|
||||
}
|
||||
catch (HttpRequestException e)
|
||||
{
|
||||
Log($"Failed to load https://mangasee123.com/rss/{manga.publicationId}.xml \n\r{e}");
|
||||
return Array.Empty<Chapter>();
|
||||
}
|
||||
}
|
||||
|
||||
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
|
||||
{
|
||||
if (progressToken?.cancellationRequested ?? false)
|
||||
{
|
||||
progressToken.Cancel();
|
||||
return HttpStatusCode.RequestTimeout;
|
||||
}
|
||||
|
||||
Manga chapterParentManga = chapter.parentManga;
|
||||
if (progressToken?.cancellationRequested ?? false)
|
||||
{
|
||||
progressToken.Cancel();
|
||||
return HttpStatusCode.RequestTimeout;
|
||||
}
|
||||
|
||||
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
|
||||
|
||||
RequestResult requestResult = this.downloadClient.MakeRequest(chapter.url, RequestType.Default);
|
||||
if (requestResult.htmlDocument is null)
|
||||
{
|
||||
progressToken?.Cancel();
|
||||
return HttpStatusCode.RequestTimeout;
|
||||
}
|
||||
|
||||
HtmlDocument document = requestResult.htmlDocument;
|
||||
|
||||
HtmlNode gallery = document.DocumentNode.Descendants("div").First(div => div.HasClass("ImageGallery"));
|
||||
HtmlNode[] images = gallery.Descendants("img").Where(img => img.HasClass("img-fluid")).ToArray();
|
||||
List<string> urls = new();
|
||||
foreach(HtmlNode galleryImage in images)
|
||||
urls.Add(galleryImage.GetAttributeValue("src", ""));
|
||||
|
||||
string comicInfoPath = Path.GetTempFileName();
|
||||
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
|
||||
|
||||
return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(), RequestType.MangaImage, comicInfoPath, progressToken:progressToken);
|
||||
}
|
||||
}
|
@ -7,9 +7,9 @@ namespace Tranga.MangaConnectors;
|
||||
|
||||
public class Mangaworld: MangaConnector
|
||||
{
|
||||
public Mangaworld(GlobalBase clone) : base(clone, "Mangaworld")
|
||||
public Mangaworld(GlobalBase clone) : base(clone, "Mangaworld", ["it"])
|
||||
{
|
||||
this.downloadClient = new HttpDownloadClient(clone);
|
||||
this.downloadClient = new ChromiumDownloadClient(clone);
|
||||
}
|
||||
|
||||
public override Manga[] GetManga(string publicationTitle = "")
|
||||
@ -149,19 +149,28 @@ public class Mangaworld: MangaConnector
|
||||
document.DocumentNode.SelectSingleNode(
|
||||
"//div[contains(concat(' ',normalize-space(@class),' '),'chapters-wrapper')]");
|
||||
|
||||
Regex volumeRex = new(@"[Vv]olume ([0-9]+).*");
|
||||
Regex chapterRex = new(@"[Cc]apitolo ([0-9]+(?:\.[0-9]+)?).*");
|
||||
Regex idRex = new(@".*\/read\/([a-z0-9]+)(?:[?\/].*)?");
|
||||
if (chaptersWrapper.Descendants("div").Any(descendant => descendant.HasClass("volume-element")))
|
||||
{
|
||||
foreach (HtmlNode volNode in document.DocumentNode.SelectNodes("//div[contains(concat(' ',normalize-space(@class),' '),'volume-element')]"))
|
||||
{
|
||||
string volume = Regex.Match(volNode.SelectNodes("div").First(node => node.HasClass("volume")).SelectSingleNode("p").InnerText,
|
||||
@"[Vv]olume ([0-9]+).*").Groups[1].Value;
|
||||
string volume = volumeRex.Match(volNode.SelectNodes("div").First(node => node.HasClass("volume")).SelectSingleNode("p").InnerText).Groups[1].Value;
|
||||
foreach (HtmlNode chNode in volNode.SelectNodes("div").First(node => node.HasClass("volume-chapters")).SelectNodes("div"))
|
||||
{
|
||||
|
||||
string number = Regex.Match(chNode.SelectSingleNode("a").SelectSingleNode("span").InnerText,
|
||||
@"[Cc]apitolo ([0-9]+).*").Groups[1].Value;
|
||||
string number = chapterRex.Match(chNode.SelectSingleNode("a").SelectSingleNode("span").InnerText).Groups[1].Value;
|
||||
string url = chNode.SelectSingleNode("a").GetAttributeValue("href", "");
|
||||
ret.Add(new Chapter(manga, null, volume, number, url));
|
||||
string id = idRex.Match(chNode.SelectSingleNode("a").GetAttributeValue("href", "")).Groups[1].Value;
|
||||
try
|
||||
{
|
||||
ret.Add(new Chapter(manga, null, volume, number, url, id));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log($"Failed to load chapter {number}: {e.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -169,9 +178,17 @@ public class Mangaworld: MangaConnector
|
||||
{
|
||||
foreach (HtmlNode chNode in chaptersWrapper.SelectNodes("div").Where(node => node.HasClass("chapter")))
|
||||
{
|
||||
string number = chNode.SelectSingleNode("a").SelectSingleNode("span").InnerText.Split(" ")[^1];
|
||||
string number = chapterRex.Match(chNode.SelectSingleNode("a").SelectSingleNode("span").InnerText).Groups[1].Value;
|
||||
string url = chNode.SelectSingleNode("a").GetAttributeValue("href", "");
|
||||
ret.Add(new Chapter(manga, null, null, number, url));
|
||||
string id = idRex.Match(chNode.SelectSingleNode("a").GetAttributeValue("href", "")).Groups[1].Value;
|
||||
try
|
||||
{
|
||||
ret.Add(new Chapter(manga, null, null, number, url, id));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log($"Failed to load chapter {number}: {e.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -206,10 +223,7 @@ public class Mangaworld: MangaConnector
|
||||
|
||||
string[] imageUrls = ParseImageUrlsFromHtml(requestResult.htmlDocument);
|
||||
|
||||
string comicInfoPath = Path.GetTempFileName();
|
||||
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
|
||||
|
||||
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(), RequestType.MangaImage, comicInfoPath, "https://www.mangaworld.bz/", progressToken:progressToken);
|
||||
return DownloadChapterImages(imageUrls, chapter, RequestType.MangaImage,"https://www.mangaworld.bz/", progressToken:progressToken);
|
||||
}
|
||||
|
||||
private string[] ParseImageUrlsFromHtml(HtmlDocument document)
|
||||
|
@ -7,7 +7,7 @@ namespace Tranga.MangaConnectors;
|
||||
|
||||
public class ManhuaPlus : MangaConnector
|
||||
{
|
||||
public ManhuaPlus(GlobalBase clone) : base(clone, "ManhuaPlus")
|
||||
public ManhuaPlus(GlobalBase clone) : base(clone, "ManhuaPlus", ["en"])
|
||||
{
|
||||
this.downloadClient = new ChromiumDownloadClient(clone);
|
||||
}
|
||||
@ -82,21 +82,36 @@ public class ManhuaPlus : MangaConnector
|
||||
HtmlNode titleNode = document.DocumentNode.SelectSingleNode("//h1");
|
||||
string sortName = titleNode.InnerText.Replace("\n", "");
|
||||
|
||||
HtmlNode[] authorsNodes = document.DocumentNode
|
||||
.SelectNodes("//a[contains(@href, 'https://manhuaplus.org/authors/')]")
|
||||
.ToArray();
|
||||
List<string> authors = new();
|
||||
foreach (HtmlNode authorNode in authorsNodes)
|
||||
authors.Add(authorNode.InnerText);
|
||||
try
|
||||
{
|
||||
HtmlNode[] authorsNodes = document.DocumentNode
|
||||
.SelectNodes("//a[contains(@href, 'https://manhuaplus.org/authors/')]")
|
||||
.ToArray();
|
||||
foreach (HtmlNode authorNode in authorsNodes)
|
||||
authors.Add(authorNode.InnerText);
|
||||
}
|
||||
catch (ArgumentNullException e)
|
||||
{
|
||||
Log("No authors found.");
|
||||
}
|
||||
|
||||
HtmlNode[] genreNodes = document.DocumentNode
|
||||
.SelectNodes("//a[contains(@href, 'https://manhuaplus.org/genres/')]").ToArray();
|
||||
foreach (HtmlNode genreNode in genreNodes)
|
||||
tags.Add(genreNode.InnerText.Replace("\n", ""));
|
||||
try
|
||||
{
|
||||
HtmlNode[] genreNodes = document.DocumentNode
|
||||
.SelectNodes("//a[contains(@href, 'https://manhuaplus.org/genres/')]").ToArray();
|
||||
foreach (HtmlNode genreNode in genreNodes)
|
||||
tags.Add(genreNode.InnerText.Replace("\n", ""));
|
||||
}
|
||||
catch (ArgumentNullException e)
|
||||
{
|
||||
Log("No genres found");
|
||||
}
|
||||
|
||||
string yearNodeStr = document.DocumentNode
|
||||
.SelectSingleNode("//aside//i[contains(concat(' ',normalize-space(@class),' '),' fa-clock ')]/../span").InnerText.Replace("\n", "");
|
||||
int year = int.Parse(yearNodeStr.Split(' ')[0].Split('/')[^1]);
|
||||
Regex yearRex = new(@"(?:[0-9]{1,2}\/){2}([0-9]{2,4}) [0-9]{1,2}:[0-9]{1,2}");
|
||||
HtmlNode yearNode = document.DocumentNode.SelectSingleNode("//aside//i[contains(concat(' ',normalize-space(@class),' '),' fa-clock ')]/../span");
|
||||
Match match = yearRex.Match(yearNode.InnerText);
|
||||
int year = match.Success && match.Groups[1].Success ? int.Parse(match.Groups[1].Value) : 1960;
|
||||
|
||||
status = document.DocumentNode.SelectSingleNode("//aside//i[contains(concat(' ',normalize-space(@class),' '),' fa-rss ')]/../span").InnerText.Replace("\n", "");
|
||||
switch (status.ToLower())
|
||||
@ -140,7 +155,14 @@ public class ManhuaPlus : MangaConnector
|
||||
string volumeNumber = "1";
|
||||
string chapterNumber = rexMatch.Groups[1].Value;
|
||||
string fullUrl = url;
|
||||
chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, fullUrl));
|
||||
try
|
||||
{
|
||||
chapters.Add(new Chapter(manga, "", volumeNumber, chapterNumber, fullUrl));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log($"Failed to load chapter {chapterNumber}: {e.Message}");
|
||||
}
|
||||
}
|
||||
//Return Chapters ordered by Chapter-Number
|
||||
Log($"Got {chapters.Count} chapters. {manga}");
|
||||
@ -175,10 +197,7 @@ public class ManhuaPlus : MangaConnector
|
||||
|
||||
HtmlNode[] images = document.DocumentNode.SelectNodes("//a[contains(concat(' ',normalize-space(@class),' '),' readImg ')]/img").ToArray();
|
||||
List<string> urls = images.Select(node => node.GetAttributeValue("src", "")).ToList();
|
||||
|
||||
string comicInfoPath = Path.GetTempFileName();
|
||||
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());
|
||||
|
||||
return DownloadChapterImages(urls.ToArray(), chapter.GetArchiveFilePath(), RequestType.MangaImage, comicInfoPath, progressToken:progressToken);
|
||||
return DownloadChapterImages(urls.ToArray(), chapter, RequestType.MangaImage, progressToken:progressToken);
|
||||
}
|
||||
}
|
273
Tranga/MangaConnectors/Webtoons.cs
Normal file
273
Tranga/MangaConnectors/Webtoons.cs
Normal file
@ -0,0 +1,273 @@
|
||||
using System.Net;
|
||||
using System.Text.RegularExpressions;
|
||||
using HtmlAgilityPack;
|
||||
using Tranga.Jobs;
|
||||
|
||||
namespace Tranga.MangaConnectors;
|
||||
|
||||
public class Webtoons : MangaConnector
|
||||
{
|
||||
|
||||
public Webtoons(GlobalBase clone) : base(clone, "Webtoons", ["en"])
|
||||
{
|
||||
this.downloadClient = new HttpDownloadClient(clone);
|
||||
}
|
||||
|
||||
// Done
|
||||
public override Manga[] GetManga(string publicationTitle = "")
|
||||
{
|
||||
string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower();
|
||||
Log($"Searching Publications. Term=\"{publicationTitle}\"");
|
||||
string requestUrl = $"https://www.webtoons.com/en/search?keyword={sanitizedTitle}&searchType=WEBTOON";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) {
|
||||
Log($"Failed to retrieve site");
|
||||
return Array.Empty<Manga>();
|
||||
}
|
||||
|
||||
if (requestResult.htmlDocument is null)
|
||||
{
|
||||
Log($"Failed to retrieve site");
|
||||
return Array.Empty<Manga>();
|
||||
}
|
||||
|
||||
Manga[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument);
|
||||
Log($"Retrieved {publications.Length} publications. Term=\"{publicationTitle}\"");
|
||||
return publications;
|
||||
}
|
||||
|
||||
// Done
|
||||
public override Manga? GetMangaFromId(string publicationId)
|
||||
{
|
||||
PublicationManager pb = new PublicationManager(publicationId);
|
||||
return GetMangaFromUrl($"https://www.webtoons.com/en/{pb.Category}/{pb.Title}/list?title_no={pb.Id}");
|
||||
}
|
||||
|
||||
// Done
|
||||
public override Manga? GetMangaFromUrl(string url)
|
||||
{
|
||||
RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300) {
|
||||
return null;
|
||||
}
|
||||
if (requestResult.htmlDocument is null)
|
||||
{
|
||||
Log($"Failed to retrieve site");
|
||||
return null;
|
||||
}
|
||||
Regex regex = new Regex(@".*webtoons\.com\/en\/(?<category>[^\/]+)\/(?<title>[^\/]+)\/list\?title_no=(?<id>\d+).*");
|
||||
Match match = regex.Match(url);
|
||||
|
||||
if(match.Success) {
|
||||
PublicationManager pm = new PublicationManager(match.Groups["title"].Value, match.Groups["category"].Value, match.Groups["id"].Value);
|
||||
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, pm.getPublicationId(), url);
|
||||
}
|
||||
Log($"Failed match Regex ID");
|
||||
return null;
|
||||
}
|
||||
|
||||
// Done
|
||||
private Manga[] ParsePublicationsFromHtml(HtmlDocument document)
|
||||
{
|
||||
HtmlNode mangaList = document.DocumentNode.SelectSingleNode("//ul[contains(@class, 'card_lst')]");
|
||||
if (!mangaList.ChildNodes.Any(node => node.Name == "li")) {
|
||||
Log($"Failed to parse publication");
|
||||
return Array.Empty<Manga>();
|
||||
}
|
||||
|
||||
List<string> urls = document.DocumentNode
|
||||
.SelectNodes("//ul[contains(@class, 'card_lst')]/li/a")
|
||||
.Select(node => node.GetAttributeValue("href", "https://www.webtoons.com"))
|
||||
.ToList();
|
||||
|
||||
HashSet<Manga> ret = new();
|
||||
foreach (string url in urls)
|
||||
{
|
||||
Manga? manga = GetMangaFromUrl(url);
|
||||
if (manga is not null)
|
||||
ret.Add((Manga)manga);
|
||||
}
|
||||
|
||||
return ret.ToArray();
|
||||
}
|
||||
|
||||
private string capitalizeString(string str = "") {
|
||||
if(str.Length == 0) return "";
|
||||
if(str.Length == 1) return str.ToUpper();
|
||||
return char.ToUpper(str[0]) + str.Substring(1).ToLower();
|
||||
}
|
||||
|
||||
// Done
|
||||
private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl)
|
||||
{
|
||||
HtmlNode infoNode1 = document.DocumentNode.SelectSingleNode("//*[@id='content']/div[2]/div[1]/div[1]");
|
||||
HtmlNode infoNode2 = document.DocumentNode.SelectSingleNode("//*[@id='content']/div[2]/div[2]/div[2]");
|
||||
|
||||
string sortName = infoNode1.SelectSingleNode(".//h1[contains(@class, 'subj')]").InnerText;
|
||||
string description = infoNode2.SelectSingleNode(".//p[contains(@class, 'summary')]")
|
||||
.InnerText.Trim();
|
||||
|
||||
HtmlNode posterNode = document.DocumentNode.SelectSingleNode("//div[contains(@class, 'detail_body') and contains(@class, 'banner')]");
|
||||
|
||||
Regex regex = new Regex(@"url\('(?<url>.*?)'\)");
|
||||
Match match = regex.Match(posterNode.GetAttributeValue("style", ""));
|
||||
|
||||
string posterUrl = match.Groups["url"].Value;
|
||||
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover, websiteUrl);
|
||||
|
||||
string genre = infoNode1.SelectSingleNode(".//h2[contains(@class, 'genre')]")
|
||||
.InnerText.Trim();
|
||||
string[] tags = [ genre ];
|
||||
|
||||
List<HtmlNode> authorsNodes = infoNode1.SelectSingleNode(".//div[contains(@class, 'author_area')]").Descendants("a").ToList();
|
||||
List<string> authors = authorsNodes.Select(node => node.InnerText.Trim()).ToList();
|
||||
|
||||
string originalLanguage = "";
|
||||
|
||||
int year = DateTime.Now.Year;
|
||||
|
||||
string status1 = infoNode2.SelectSingleNode(".//p").InnerText;
|
||||
string status2 = infoNode2.SelectSingleNode(".//p/span").InnerText;
|
||||
Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased;
|
||||
if(status2.Length == 0 || status1.ToLower() == "completed") {
|
||||
releaseStatus = Manga.ReleaseStatusByte.Completed;
|
||||
} else if(status2.ToLower() == "up") {
|
||||
releaseStatus = Manga.ReleaseStatusByte.Continuing;
|
||||
}
|
||||
|
||||
Manga manga = new(sortName, authors, description, new Dictionary<string, string>(), tags, posterUrl, coverFileNameInCache, new Dictionary<string, string>(),
|
||||
year, originalLanguage, publicationId, releaseStatus, websiteUrl: websiteUrl);
|
||||
AddMangaToCache(manga);
|
||||
return manga;
|
||||
}
|
||||
|
||||
// Done
|
||||
public override Chapter[] GetChapters(Manga manga, string language = "en")
|
||||
{
|
||||
PublicationManager pm = new PublicationManager(manga.publicationId);
|
||||
string requestUrl = $"https://www.webtoons.com/en/{pm.Category}/{pm.Title}/list?title_no={pm.Id}";
|
||||
// Leaving this in for verification if the page exists
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return Array.Empty<Chapter>();
|
||||
|
||||
// Get number of pages
|
||||
int pages = requestResult.htmlDocument.DocumentNode
|
||||
.SelectNodes("//div[contains(@class, 'paginate')]/a")
|
||||
.ToList()
|
||||
.Count;
|
||||
List<Chapter> chapters = new List<Chapter>();
|
||||
|
||||
for(int page = 1; page <= pages; page++) {
|
||||
string pageRequestUrl = $"{requestUrl}&page={page}";
|
||||
chapters.AddRange(ParseChaptersFromHtml(manga, pageRequestUrl));
|
||||
}
|
||||
Log($"Got {chapters.Count} chapters. {manga}");
|
||||
return chapters.Order().ToArray();
|
||||
}
|
||||
|
||||
// Done
|
||||
private List<Chapter> ParseChaptersFromHtml(Manga manga, string mangaUrl)
|
||||
{
|
||||
RequestResult result = downloadClient.MakeRequest(mangaUrl, RequestType.Default);
|
||||
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null)
|
||||
{
|
||||
Log("Failed to load site");
|
||||
return new List<Chapter>();
|
||||
}
|
||||
|
||||
List<Chapter> ret = new();
|
||||
|
||||
foreach (HtmlNode chapterInfo in result.htmlDocument.DocumentNode.SelectNodes("//ul/li[contains(@class, '_episodeItem')]"))
|
||||
{
|
||||
HtmlNode infoNode = chapterInfo.SelectSingleNode(".//a");
|
||||
string url = infoNode.GetAttributeValue("href", "");
|
||||
|
||||
string id = chapterInfo.GetAttributeValue("id", "");
|
||||
if(id == "") continue;
|
||||
string? volumeNumber = null;
|
||||
string chapterNumber = chapterInfo.GetAttributeValue("data-episode-no", "");
|
||||
if(chapterNumber == "") continue;
|
||||
string chapterName = infoNode.SelectSingleNode(".//span[contains(@class, 'subj')]/span").InnerText.Trim();
|
||||
ret.Add(new Chapter(manga, chapterName, volumeNumber, chapterNumber, url));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
|
||||
{
|
||||
if (progressToken?.cancellationRequested ?? false)
|
||||
{
|
||||
progressToken.Cancel();
|
||||
return HttpStatusCode.RequestTimeout;
|
||||
}
|
||||
|
||||
Manga chapterParentManga = chapter.parentManga;
|
||||
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
|
||||
string requestUrl = chapter.url;
|
||||
// Leaving this in to check if the page exists
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
{
|
||||
progressToken?.Cancel();
|
||||
return requestResult.statusCode;
|
||||
}
|
||||
|
||||
string[] imageUrls = ParseImageUrlsFromHtml(requestUrl);
|
||||
return DownloadChapterImages(imageUrls, chapter, RequestType.MangaImage, progressToken:progressToken, referrer: requestUrl);
|
||||
}
|
||||
|
||||
private string[] ParseImageUrlsFromHtml(string mangaUrl)
|
||||
{
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(mangaUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
if (requestResult.htmlDocument is null)
|
||||
{
|
||||
Log($"Failed to retrieve site");
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
return requestResult.htmlDocument.DocumentNode
|
||||
.SelectNodes("//*[@id='_imageList']/img")
|
||||
.Select(node =>
|
||||
node.GetAttributeValue("data-url", ""))
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
internal class PublicationManager {
|
||||
public PublicationManager(string title = "", string category = "", string id = "") {
|
||||
this.Title = title;
|
||||
this.Category = category;
|
||||
this.Id = id;
|
||||
}
|
||||
|
||||
public PublicationManager(string publicationId) {
|
||||
string[] parts = publicationId.Split("|");
|
||||
if(parts.Length == 3) {
|
||||
this.Title = parts[0];
|
||||
this.Category = parts[1];
|
||||
this.Id = parts[2];
|
||||
} else {
|
||||
this.Title = "";
|
||||
this.Category = "";
|
||||
this.Id = "";
|
||||
}
|
||||
}
|
||||
|
||||
public string getPublicationId() {
|
||||
return $"{this.Title}|{this.Category}|{this.Id}";
|
||||
}
|
||||
|
||||
public string Title { get; set; }
|
||||
public string Category { get; set; }
|
||||
public string Id { get; set; }
|
||||
}
|
215
Tranga/MangaConnectors/WeebCentral.cs
Normal file
215
Tranga/MangaConnectors/WeebCentral.cs
Normal file
@ -0,0 +1,215 @@
|
||||
using System.Net;
|
||||
using System.Text.RegularExpressions;
|
||||
using HtmlAgilityPack;
|
||||
using Tranga.Jobs;
|
||||
|
||||
namespace Tranga.MangaConnectors;
|
||||
|
||||
public class Weebcentral : MangaConnector
|
||||
{
|
||||
private readonly string _baseUrl = "https://weebcentral.com";
|
||||
|
||||
private readonly string[] _filterWords =
|
||||
{ "a", "the", "of", "as", "to", "no", "for", "on", "with", "be", "and", "in", "wa", "at", "be", "ni" };
|
||||
|
||||
public Weebcentral(GlobalBase clone) : base(clone, "Weebcentral", ["en"])
|
||||
{
|
||||
downloadClient = new ChromiumDownloadClient(clone);
|
||||
}
|
||||
|
||||
public override Manga[] GetManga(string publicationTitle = "")
|
||||
{
|
||||
Log($"Searching Publications. Term=\"{publicationTitle}\"");
|
||||
const int limit = 32; //How many values we want returned at once
|
||||
int offset = 0; //"Page"
|
||||
string requestUrl =
|
||||
$"{_baseUrl}/search/data?limit={limit}&offset={offset}&text={publicationTitle}&sort=Best+Match&order=Ascending&official=Any&display_mode=Minimal%20Display";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300 ||
|
||||
requestResult.htmlDocument == null)
|
||||
{
|
||||
Log($"Failed to retrieve search: {requestResult.statusCode}");
|
||||
return [];
|
||||
}
|
||||
|
||||
Manga[] publications = ParsePublicationsFromHtml(requestResult.htmlDocument);
|
||||
Log($"Retrieved {publications.Length} publications. Term=\"{publicationTitle}\"");
|
||||
|
||||
return publications;
|
||||
}
|
||||
|
||||
private Manga[] ParsePublicationsFromHtml(HtmlDocument document)
|
||||
{
|
||||
if (document.DocumentNode.SelectNodes("//article") == null)
|
||||
return [];
|
||||
|
||||
List<string> urls = document.DocumentNode.SelectNodes("/html/body/article/a[@class='link link-hover tooltip tooltip-bottom']")
|
||||
.Select(elem => elem.GetAttributeValue("href", "")).ToList();
|
||||
|
||||
HashSet<Manga> ret = new();
|
||||
foreach (string url in urls)
|
||||
{
|
||||
Manga? manga = GetMangaFromUrl(url);
|
||||
if (manga is not null)
|
||||
ret.Add((Manga)manga);
|
||||
}
|
||||
|
||||
return ret.ToArray();
|
||||
}
|
||||
|
||||
public override Manga? GetMangaFromUrl(string url)
|
||||
{
|
||||
Regex publicationIdRex = new(@"https:\/\/weebcentral\.com\/series\/(\w*)\/(.*)");
|
||||
string publicationId = publicationIdRex.Match(url).Groups[1].Value;
|
||||
|
||||
RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
|
||||
if ((int)requestResult.statusCode < 300 && (int)requestResult.statusCode >= 200 &&
|
||||
requestResult.htmlDocument is not null)
|
||||
return ParseSinglePublicationFromHtml(requestResult.htmlDocument, publicationId, url);
|
||||
return null;
|
||||
}
|
||||
|
||||
private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publicationId, string websiteUrl)
|
||||
{
|
||||
HtmlNode? posterNode =
|
||||
document.DocumentNode.SelectSingleNode("//section[@class='flex items-center justify-center']/picture/img");
|
||||
string posterUrl = posterNode?.GetAttributeValue("src", "") ?? "";
|
||||
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, publicationId, RequestType.MangaCover);
|
||||
|
||||
HtmlNode? titleNode = document.DocumentNode.SelectSingleNode("//section/h1");
|
||||
string sortName = titleNode?.InnerText ?? "Undefined";
|
||||
|
||||
HtmlNode[] authorsNodes =
|
||||
document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Author(s): ']/span")?.ToArray() ?? [];
|
||||
List<string> authors = authorsNodes.Select(n => n.InnerText).ToList();
|
||||
|
||||
HtmlNode[] genreNodes =
|
||||
document.DocumentNode.SelectNodes("//ul/li[strong/text() = 'Tags(s): ']/span")?.ToArray() ?? [];
|
||||
HashSet<string> tags = genreNodes.Select(n => n.InnerText).ToHashSet();
|
||||
|
||||
HtmlNode? statusNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Status: ']/a");
|
||||
string status = statusNode?.InnerText ?? "";
|
||||
Log("unable to parse status");
|
||||
Manga.ReleaseStatusByte releaseStatus = Manga.ReleaseStatusByte.Unreleased;
|
||||
switch (status.ToLower())
|
||||
{
|
||||
case "cancelled": releaseStatus = Manga.ReleaseStatusByte.Cancelled; break;
|
||||
case "hiatus": releaseStatus = Manga.ReleaseStatusByte.OnHiatus; break;
|
||||
case "complete": releaseStatus = Manga.ReleaseStatusByte.Completed; break;
|
||||
case "ongoing": releaseStatus = Manga.ReleaseStatusByte.Continuing; break;
|
||||
}
|
||||
|
||||
HtmlNode? yearNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Released: ']/span");
|
||||
int year = Convert.ToInt32(yearNode?.InnerText ?? "0");
|
||||
|
||||
HtmlNode? descriptionNode = document.DocumentNode.SelectSingleNode("//ul/li[strong/text() = 'Description']/p");
|
||||
string description = descriptionNode?.InnerText ?? "Undefined";
|
||||
|
||||
HtmlNode[] altTitleNodes = document.DocumentNode
|
||||
.SelectNodes("//ul/li[strong/text() = 'Associated Name(s)']/ul/li")?.ToArray() ?? [];
|
||||
Dictionary<string, string> altTitles = new(), links = new();
|
||||
for (int i = 0; i < altTitleNodes.Length; i++)
|
||||
altTitles.Add(i.ToString(), altTitleNodes[i].InnerText);
|
||||
|
||||
string originalLanguage = "";
|
||||
|
||||
Manga manga = new(sortName, authors.ToList(), description, altTitles, tags.ToArray(), posterUrl,
|
||||
coverFileNameInCache, links,
|
||||
year, originalLanguage, publicationId, releaseStatus, websiteUrl);
|
||||
AddMangaToCache(manga);
|
||||
return manga;
|
||||
}
|
||||
|
||||
public override Manga? GetMangaFromId(string publicationId)
|
||||
{
|
||||
return GetMangaFromUrl($"https://weebcentral.com/series/{publicationId}");
|
||||
}
|
||||
|
||||
public override Chapter[] GetChapters(Manga manga, string language = "en")
|
||||
{
|
||||
Log($"Getting chapters {manga}");
|
||||
string requestUrl = $"{_baseUrl}/series/{manga.publicationId}/full-chapter-list";
|
||||
RequestResult requestResult =
|
||||
downloadClient.MakeRequest(requestUrl, RequestType.Default);
|
||||
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
|
||||
return [];
|
||||
|
||||
//Return Chapters ordered by Chapter-Number
|
||||
if (requestResult.htmlDocument is null)
|
||||
return [];
|
||||
List<Chapter> chapters = ParseChaptersFromHtml(manga, requestResult.htmlDocument);
|
||||
Log($"Got {chapters.Count} chapters. {manga}");
|
||||
return chapters.OrderByDescending(c => c.name).ThenBy(c => c.volumeNumber).ThenBy(c => c.chapterNumber).ToArray();
|
||||
}
|
||||
|
||||
private List<Chapter> ParseChaptersFromHtml(Manga manga, HtmlDocument document)
|
||||
{
|
||||
HtmlNode? chaptersWrapper = document.DocumentNode.SelectSingleNode("/html/body");
|
||||
|
||||
Regex chapterRex = new(@"(\d+(?:\.\d+)*)");
|
||||
Regex chapterNameRex = new(@"(\w* )+");
|
||||
Regex idRex = new(@"https:\/\/weebcentral\.com\/chapters\/(\w*)");
|
||||
|
||||
List<Chapter> ret = chaptersWrapper.Descendants("a").Select(elem =>
|
||||
{
|
||||
string url = elem.GetAttributeValue("href", "") ?? "Undefined";
|
||||
|
||||
if (!url.StartsWith("https://") && !url.StartsWith("http://"))
|
||||
return new Chapter(manga, null, null, "-1", "undefined");
|
||||
|
||||
Match idMatch = idRex.Match(url);
|
||||
string? id = idMatch.Success ? idMatch.Groups[1].Value : null;
|
||||
|
||||
string chapterNode = elem.SelectSingleNode("span[@class='grow flex items-center gap-2']/span")?.InnerText ??
|
||||
"Undefined";
|
||||
|
||||
MatchCollection chapterNumberMatch = chapterRex.Matches(chapterNode);
|
||||
string chapterNumber = chapterNumberMatch.Count > 0 ? chapterNumberMatch[^1].Groups[1].Value : "-1";
|
||||
MatchCollection chapterNameMatch = chapterNameRex.Matches(chapterNode);
|
||||
string chapterName = chapterNameMatch.Count > 0
|
||||
? string.Join(" - ",
|
||||
chapterNameMatch.Select(m => m.Groups[1].Value.Trim())
|
||||
.Where(name => name.Length > 0 && !name.Equals("Chapter", StringComparison.OrdinalIgnoreCase)).ToArray()).Trim()
|
||||
: "";
|
||||
|
||||
return new Chapter(manga, chapterName != "" ? chapterName : null, null, chapterNumber, url, id);
|
||||
}).Where(elem => elem.chapterNumber != -1 && elem.url != "undefined").ToList();
|
||||
|
||||
ret.Reverse();
|
||||
return ret;
|
||||
}
|
||||
|
||||
public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? progressToken = null)
|
||||
{
|
||||
if (progressToken?.cancellationRequested ?? false)
|
||||
{
|
||||
progressToken.Cancel();
|
||||
return HttpStatusCode.RequestTimeout;
|
||||
}
|
||||
|
||||
Manga chapterParentManga = chapter.parentManga;
|
||||
if (progressToken?.cancellationRequested ?? false)
|
||||
{
|
||||
progressToken.Cancel();
|
||||
return HttpStatusCode.RequestTimeout;
|
||||
}
|
||||
|
||||
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
|
||||
|
||||
RequestResult requestResult = downloadClient.MakeRequest(chapter.url, RequestType.Default);
|
||||
if (requestResult.htmlDocument is null)
|
||||
{
|
||||
progressToken?.Cancel();
|
||||
return HttpStatusCode.RequestTimeout;
|
||||
}
|
||||
|
||||
HtmlDocument? document = requestResult.htmlDocument;
|
||||
|
||||
HtmlNode[] imageNodes =
|
||||
document.DocumentNode.SelectNodes($"//section[@hx-get='{chapter.url}/images']/img")?.ToArray() ?? [];
|
||||
string[] urls = imageNodes.Select(imgNode => imgNode.GetAttributeValue("src", "")).ToArray();
|
||||
|
||||
return DownloadChapterImages(urls, chapter, RequestType.MangaImage, progressToken: progressToken, referrer: "https://weebcentral.com/");
|
||||
}
|
||||
}
|
@ -24,7 +24,7 @@ public class Gotify : NotificationConnector
|
||||
return $"Gotify {endpoint}";
|
||||
}
|
||||
|
||||
public override void SendNotification(string title, string notificationText)
|
||||
protected override void SendNotificationInternal(string title, string notificationText)
|
||||
{
|
||||
Log($"Sending notification: {title} - {notificationText}");
|
||||
MessageData message = new(title, notificationText);
|
||||
|
@ -20,7 +20,7 @@ public class LunaSea : NotificationConnector
|
||||
return $"LunaSea {id}";
|
||||
}
|
||||
|
||||
public override void SendNotification(string title, string notificationText)
|
||||
protected override void SendNotificationInternal(string title, string notificationText)
|
||||
{
|
||||
Log($"Sending notification: {title} - {notificationText}");
|
||||
MessageData message = new(title, notificationText);
|
||||
|
@ -3,14 +3,72 @@
|
||||
public abstract class NotificationConnector : GlobalBase
|
||||
{
|
||||
public readonly NotificationConnectorType notificationConnectorType;
|
||||
private DateTime? _notificationRequested = null;
|
||||
private readonly Thread? _notificationBufferThread = null;
|
||||
private const int NoChangeTimeout = 3, BiggestInterval = 30;
|
||||
private List<KeyValuePair<string, string>> _notifications = new();
|
||||
|
||||
protected NotificationConnector(GlobalBase clone, NotificationConnectorType notificationConnectorType) : base(clone)
|
||||
{
|
||||
Log($"Creating notificationConnector {Enum.GetName(notificationConnectorType)}");
|
||||
this.notificationConnectorType = notificationConnectorType;
|
||||
|
||||
|
||||
if (TrangaSettings.bufferLibraryUpdates)
|
||||
{
|
||||
_notificationBufferThread = new(CheckNotificationBuffer);
|
||||
_notificationBufferThread.Start();
|
||||
}
|
||||
}
|
||||
|
||||
private void CheckNotificationBuffer()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
if (_notificationRequested is not null && DateTime.Now.Subtract((DateTime)_notificationRequested) > TimeSpan.FromMinutes(NoChangeTimeout)) //If no updates have been requested for NoChangeTimeout minutes, update library
|
||||
{
|
||||
string[] uniqueTitles = _notifications.DistinctBy(n => n.Key).Select(n => n.Key).ToArray();
|
||||
Log($"Notification Buffer sending! Notifications: {string.Join(", ", uniqueTitles)}");
|
||||
foreach (string ut in uniqueTitles)
|
||||
{
|
||||
string[] texts = _notifications.Where(n => n.Key == ut).Select(n => n.Value).ToArray();
|
||||
SendNotificationInternal($"{ut} ({texts.Length})", string.Join('\n', texts));
|
||||
}
|
||||
_notificationRequested = null;
|
||||
_notifications.Clear();
|
||||
}
|
||||
Thread.Sleep(100);
|
||||
}
|
||||
}
|
||||
|
||||
public enum NotificationConnectorType : byte { Gotify = 0, LunaSea = 1, Ntfy = 2 }
|
||||
|
||||
public void SendNotification(string title, string notificationText, bool buffer = false)
|
||||
{
|
||||
_notificationRequested ??= DateTime.Now;
|
||||
if (!TrangaSettings.bufferNotifications || !buffer)
|
||||
{
|
||||
SendNotificationInternal(title, notificationText);
|
||||
return;
|
||||
}
|
||||
_notifications.Add(new(title, notificationText));
|
||||
if (_notificationRequested is not null &&
|
||||
DateTime.Now.Subtract((DateTime)_notificationRequested) > TimeSpan.FromMinutes(BiggestInterval)) //If the last update has been more than BiggestInterval minutes ago, update library
|
||||
{
|
||||
string[] uniqueTitles = _notifications.DistinctBy(n => n.Key).Select(n => n.Key).ToArray();
|
||||
foreach (string ut in uniqueTitles)
|
||||
{
|
||||
string[] texts = _notifications.Where(n => n.Key == ut).Select(n => n.Value).ToArray();
|
||||
SendNotificationInternal(ut, string.Join('\n', texts));
|
||||
}
|
||||
_notificationRequested = null;
|
||||
_notifications.Clear();
|
||||
}
|
||||
else if(_notificationRequested is not null)
|
||||
{
|
||||
Log($"Buffering Notifications (Updates in latest {((DateTime)_notificationRequested).Add(TimeSpan.FromMinutes(BiggestInterval)).Subtract(DateTime.Now)} or {((DateTime)_notificationRequested).Add(TimeSpan.FromMinutes(NoChangeTimeout)).Subtract(DateTime.Now)})");
|
||||
}
|
||||
}
|
||||
|
||||
public abstract void SendNotification(string title, string notificationText);
|
||||
protected abstract void SendNotificationInternal(string title, string notificationText);
|
||||
}
|
@ -54,7 +54,7 @@ public class Ntfy : NotificationConnector
|
||||
return $"Ntfy {endpoint} {topic}";
|
||||
}
|
||||
|
||||
public override void SendNotification(string title, string notificationText)
|
||||
protected override void SendNotificationInternal(string title, string notificationText)
|
||||
{
|
||||
Log($"Sending notification: {title} - {notificationText}");
|
||||
MessageData message = new(title, topic, notificationText);
|
||||
|
@ -63,10 +63,11 @@ public class Server : GlobalBase
|
||||
{
|
||||
HttpListenerRequest request = context.Request;
|
||||
HttpListenerResponse response = context.Response;
|
||||
if(request.HttpMethod == "OPTIONS")
|
||||
SendResponse(HttpStatusCode.OK, context.Response);
|
||||
if(request.Url!.LocalPath.Contains("favicon"))
|
||||
if (request.Url!.LocalPath.Contains("favicon"))
|
||||
{
|
||||
SendResponse(HttpStatusCode.NoContent, response);
|
||||
return;
|
||||
}
|
||||
|
||||
switch (request.HttpMethod)
|
||||
{
|
||||
@ -79,7 +80,10 @@ public class Server : GlobalBase
|
||||
case "DELETE":
|
||||
HandleDelete(request, response);
|
||||
break;
|
||||
default:
|
||||
case "OPTIONS":
|
||||
SendResponse(HttpStatusCode.OK, context.Response);
|
||||
break;
|
||||
default:
|
||||
SendResponse(HttpStatusCode.BadRequest, response);
|
||||
break;
|
||||
}
|
||||
@ -114,6 +118,15 @@ public class Server : GlobalBase
|
||||
case "Connectors":
|
||||
SendResponse(HttpStatusCode.OK, response, _parent.GetConnectors().Select(con => con.name).ToArray());
|
||||
break;
|
||||
case "Languages":
|
||||
if (!requestVariables.TryGetValue("connector", out connectorName) ||
|
||||
!_parent.TryGetConnector(connectorName, out connector))
|
||||
{
|
||||
SendResponse(HttpStatusCode.BadRequest, response);
|
||||
break;
|
||||
}
|
||||
SendResponse(HttpStatusCode.OK, response, connector);
|
||||
break;
|
||||
case "Manga/Cover":
|
||||
if (!requestVariables.TryGetValue("internalId", out internalId) ||
|
||||
!_parent.TryGetPublicationById(internalId, out manga))
|
||||
@ -707,14 +720,15 @@ public class Server : GlobalBase
|
||||
private void SendResponse(HttpStatusCode statusCode, HttpListenerResponse response, object? content = null)
|
||||
{
|
||||
//Log($"Response: {statusCode} {content}");
|
||||
|
||||
response.StatusCode = (int)statusCode;
|
||||
response.AddHeader("Access-Control-Allow-Headers", "Content-Type, Accept, X-Requested-With");
|
||||
response.AddHeader("Access-Control-Allow-Methods", "GET, POST, DELETE");
|
||||
response.AddHeader("Access-Control-Max-Age", "1728000");
|
||||
response.AppendHeader("Access-Control-Allow-Origin", "*");
|
||||
|
||||
try
|
||||
{
|
||||
|
||||
if (content is not Stream)
|
||||
{
|
||||
response.ContentType = "application/json";
|
||||
@ -750,7 +764,7 @@ public class Server : GlobalBase
|
||||
stream.Close();
|
||||
}
|
||||
}
|
||||
catch (HttpListenerException e)
|
||||
catch (Exception e)
|
||||
{
|
||||
Log(e.ToString());
|
||||
}
|
||||
|
@ -18,14 +18,15 @@ public partial class Tranga : GlobalBase
|
||||
_connectors = new HashSet<MangaConnector>()
|
||||
{
|
||||
new Manganato(this),
|
||||
new Mangasee(this),
|
||||
new MangaDex(this),
|
||||
new MangaKatana(this),
|
||||
new Mangaworld(this),
|
||||
new Bato(this),
|
||||
new MangaLife(this),
|
||||
new ManhuaPlus(this),
|
||||
new MangaHere(this),
|
||||
new AsuraToon(this),
|
||||
new Weebcentral(this),
|
||||
new Webtoons(this),
|
||||
};
|
||||
foreach(DirectoryInfo dir in new DirectoryInfo(Path.GetTempPath()).GetDirectories("trangatemp"))//Cleanup old temp folders
|
||||
dir.Delete();
|
||||
@ -34,6 +35,7 @@ public partial class Tranga : GlobalBase
|
||||
this._server = new Server(this);
|
||||
string[] emojis = { "(•‿•)", "(づ \u25d5‿\u25d5 )づ", "( \u02d8\u25bd\u02d8)っ\u2668", "=\uff3e\u25cf \u22cf \u25cf\uff3e=", "(ΦωΦ)", "(\u272a\u3268\u272a)", "( ノ・o・ )ノ", "(〜^\u2207^ )〜", "~(\u2267ω\u2266)~","૮ \u00b4• ﻌ \u00b4• ა", "(\u02c3ᆺ\u02c2)", "(=\ud83d\udf66 \u0f1d \ud83d\udf66=)"};
|
||||
SendNotifications("Tranga Started", emojis[Random.Shared.Next(0,emojis.Length-1)]);
|
||||
Log(TrangaSettings.AsJObject().ToString());
|
||||
}
|
||||
|
||||
public MangaConnector? GetConnector(string name)
|
||||
|
@ -1,17 +1,18 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<OutputType>Exe</OutputType>
|
||||
<LangVersion>12</LangVersion>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="GlaxArguments" Version="1.1.0" />
|
||||
<PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
|
||||
<PackageReference Include="HtmlAgilityPack" Version="1.11.72" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
||||
<PackageReference Include="PuppeteerSharp" Version="10.0.0" />
|
||||
<PackageReference Include="PuppeteerSharp" Version="20.1.0" />
|
||||
<PackageReference Include="Soenneker.Utils.String.NeedlemanWunsch" Version="2.1.301" />
|
||||
</ItemGroup>
|
||||
|
||||
|
@ -15,6 +15,8 @@ public static class TrangaSettings
|
||||
public static string workingDirectory { get; private set; } = Path.Join(RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? "/usr/share" : Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "tranga-api");
|
||||
public static int apiPortNumber { get; private set; } = 6531;
|
||||
public static string userAgent { get; private set; } = DefaultUserAgent;
|
||||
public static bool bufferLibraryUpdates { get; private set; } = false;
|
||||
public static bool bufferNotifications { get; private set; } = false;
|
||||
[JsonIgnore] public static string settingsFilePath => Path.Join(workingDirectory, "settings.json");
|
||||
[JsonIgnore] public static string libraryConnectorsFilePath => Path.Join(workingDirectory, "libraryConnectors.json");
|
||||
[JsonIgnore] public static string notificationConnectorsFilePath => Path.Join(workingDirectory, "notificationConnectors.json");
|
||||
@ -33,6 +35,8 @@ public static class TrangaSettings
|
||||
};
|
||||
|
||||
public static Dictionary<RequestType, int> requestLimits { get; set; } = DefaultRequestLimits;
|
||||
public static int ChromiumStartupTimeoutMs { get; set; } = 30000;
|
||||
public static int ChromiumPageTimeoutMs { get; set; } = 30000;
|
||||
|
||||
public static void LoadFromWorkingDirectory(string directory)
|
||||
{
|
||||
@ -46,15 +50,17 @@ public static class TrangaSettings
|
||||
ExportSettings();
|
||||
}
|
||||
|
||||
public static void CreateOrUpdate(string? downloadDirectory = null, string? pWorkingDirectory = null, int? pApiPortNumber = null, string? pUserAgent = null, bool? pAprilFoolsMode = null)
|
||||
public static void CreateOrUpdate(string? downloadDirectory = null, string? pWorkingDirectory = null, int? pApiPortNumber = null, string? pUserAgent = null, bool? pAprilFoolsMode = null, bool? pBufferLibraryUpdates = null, bool? pBufferNotifications = null)
|
||||
{
|
||||
if(pWorkingDirectory is null && File.Exists(settingsFilePath))
|
||||
LoadFromWorkingDirectory(workingDirectory);
|
||||
TrangaSettings.downloadLocation = downloadDirectory ?? TrangaSettings.downloadLocation;
|
||||
TrangaSettings.workingDirectory = pWorkingDirectory ?? TrangaSettings.workingDirectory;
|
||||
TrangaSettings.apiPortNumber = pApiPortNumber ?? TrangaSettings.apiPortNumber;
|
||||
TrangaSettings.userAgent = pUserAgent ?? TrangaSettings.userAgent;
|
||||
TrangaSettings.aprilFoolsMode = pAprilFoolsMode ?? TrangaSettings.aprilFoolsMode;
|
||||
downloadLocation = downloadDirectory ?? downloadLocation;
|
||||
workingDirectory = pWorkingDirectory ?? workingDirectory;
|
||||
apiPortNumber = pApiPortNumber ?? apiPortNumber;
|
||||
userAgent = pUserAgent ?? userAgent;
|
||||
aprilFoolsMode = pAprilFoolsMode ?? aprilFoolsMode;
|
||||
bufferLibraryUpdates = pBufferLibraryUpdates ?? bufferLibraryUpdates;
|
||||
bufferNotifications = pBufferNotifications ?? bufferNotifications;
|
||||
Directory.CreateDirectory(downloadLocation);
|
||||
Directory.CreateDirectory(workingDirectory);
|
||||
ExportSettings();
|
||||
@ -90,7 +96,7 @@ public static class TrangaSettings
|
||||
|
||||
public static void UpdateAprilFoolsMode(bool enabled)
|
||||
{
|
||||
TrangaSettings.aprilFoolsMode = enabled;
|
||||
aprilFoolsMode = enabled;
|
||||
ExportSettings();
|
||||
}
|
||||
|
||||
@ -102,10 +108,10 @@ public static class TrangaSettings
|
||||
else
|
||||
Directory.CreateDirectory(newPath);
|
||||
|
||||
if (moveFiles && Directory.Exists(TrangaSettings.downloadLocation))
|
||||
Directory.Move(TrangaSettings.downloadLocation, newPath);
|
||||
if (moveFiles && Directory.Exists(downloadLocation))
|
||||
Directory.Move(downloadLocation, newPath);
|
||||
|
||||
TrangaSettings.downloadLocation = newPath;
|
||||
downloadLocation = newPath;
|
||||
ExportSettings();
|
||||
}
|
||||
|
||||
@ -116,26 +122,26 @@ public static class TrangaSettings
|
||||
GroupRead | GroupWrite | None | OtherRead | OtherWrite | UserRead | UserWrite);
|
||||
else
|
||||
Directory.CreateDirectory(newPath);
|
||||
Directory.Move(TrangaSettings.workingDirectory, newPath);
|
||||
TrangaSettings.workingDirectory = newPath;
|
||||
Directory.Move(workingDirectory, newPath);
|
||||
workingDirectory = newPath;
|
||||
ExportSettings();
|
||||
}
|
||||
|
||||
public static void UpdateUserAgent(string? customUserAgent)
|
||||
{
|
||||
TrangaSettings.userAgent = customUserAgent ?? DefaultUserAgent;
|
||||
userAgent = customUserAgent ?? DefaultUserAgent;
|
||||
ExportSettings();
|
||||
}
|
||||
|
||||
public static void UpdateRateLimit(RequestType requestType, int newLimit)
|
||||
{
|
||||
TrangaSettings.requestLimits[requestType] = newLimit;
|
||||
requestLimits[requestType] = newLimit;
|
||||
ExportSettings();
|
||||
}
|
||||
|
||||
public static void ResetRateLimits()
|
||||
{
|
||||
TrangaSettings.requestLimits = DefaultRequestLimits;
|
||||
requestLimits = DefaultRequestLimits;
|
||||
ExportSettings();
|
||||
}
|
||||
|
||||
@ -154,13 +160,17 @@ public static class TrangaSettings
|
||||
public static JObject AsJObject()
|
||||
{
|
||||
JObject jobj = new JObject();
|
||||
jobj.Add("downloadLocation", JToken.FromObject(TrangaSettings.downloadLocation));
|
||||
jobj.Add("workingDirectory", JToken.FromObject(TrangaSettings.workingDirectory));
|
||||
jobj.Add("apiPortNumber", JToken.FromObject(TrangaSettings.apiPortNumber));
|
||||
jobj.Add("userAgent", JToken.FromObject(TrangaSettings.userAgent));
|
||||
jobj.Add("aprilFoolsMode", JToken.FromObject(TrangaSettings.aprilFoolsMode));
|
||||
jobj.Add("version", JToken.FromObject(TrangaSettings.version));
|
||||
jobj.Add("requestLimits", JToken.FromObject(TrangaSettings.requestLimits));
|
||||
jobj.Add("downloadLocation", JToken.FromObject(downloadLocation));
|
||||
jobj.Add("workingDirectory", JToken.FromObject(workingDirectory));
|
||||
jobj.Add("apiPortNumber", JToken.FromObject(apiPortNumber));
|
||||
jobj.Add("userAgent", JToken.FromObject(userAgent));
|
||||
jobj.Add("aprilFoolsMode", JToken.FromObject(aprilFoolsMode));
|
||||
jobj.Add("version", JToken.FromObject(version));
|
||||
jobj.Add("requestLimits", JToken.FromObject(requestLimits));
|
||||
jobj.Add("bufferLibraryUpdates", JToken.FromObject(bufferLibraryUpdates));
|
||||
jobj.Add("bufferNotifications", JToken.FromObject(bufferNotifications));
|
||||
jobj.Add("chromiumStartTimeout", JToken.FromObject(ChromiumStartupTimeoutMs));
|
||||
jobj.Add("chromiumPageTimeout", JToken.FromObject(ChromiumPageTimeoutMs));
|
||||
return jobj;
|
||||
}
|
||||
|
||||
@ -170,16 +180,24 @@ public static class TrangaSettings
|
||||
{
|
||||
JObject jobj = JObject.Parse(serialized);
|
||||
if (jobj.TryGetValue("downloadLocation", out JToken? dl))
|
||||
TrangaSettings.downloadLocation = dl.Value<string>()!;
|
||||
downloadLocation = dl.Value<string>()!;
|
||||
if (jobj.TryGetValue("workingDirectory", out JToken? wd))
|
||||
TrangaSettings.workingDirectory = wd.Value<string>()!;
|
||||
workingDirectory = wd.Value<string>()!;
|
||||
if (jobj.TryGetValue("apiPortNumber", out JToken? apn))
|
||||
TrangaSettings.apiPortNumber = apn.Value<int>();
|
||||
apiPortNumber = apn.Value<int>();
|
||||
if (jobj.TryGetValue("userAgent", out JToken? ua))
|
||||
TrangaSettings.userAgent = ua.Value<string>()!;
|
||||
userAgent = ua.Value<string>()!;
|
||||
if (jobj.TryGetValue("aprilFoolsMode", out JToken? afm))
|
||||
TrangaSettings.aprilFoolsMode = afm.Value<bool>()!;
|
||||
aprilFoolsMode = afm.Value<bool>()!;
|
||||
if (jobj.TryGetValue("requestLimits", out JToken? rl))
|
||||
TrangaSettings.requestLimits = rl.ToObject<Dictionary<RequestType, int>>()!;
|
||||
requestLimits = rl.ToObject<Dictionary<RequestType, int>>()!;
|
||||
if (jobj.TryGetValue("bufferLibraryUpdates", out JToken? blu))
|
||||
bufferLibraryUpdates = blu.Value<bool>()!;
|
||||
if (jobj.TryGetValue("bufferNotifications", out JToken? bn))
|
||||
bufferNotifications = bn.Value<bool>()!;
|
||||
if (jobj.TryGetValue("chromiumStartTimeout", out JToken? cst))
|
||||
ChromiumStartupTimeoutMs = cst.Value<int>();
|
||||
if (jobj.TryGetValue("chromiumPageTimeout", out JToken? cpt))
|
||||
ChromiumPageTimeoutMs = cpt.Value<int>();
|
||||
}
|
||||
}
|
21
docker-compose.local.yaml
Normal file
21
docker-compose.local.yaml
Normal file
@ -0,0 +1,21 @@
|
||||
version: '3'
|
||||
services:
|
||||
tranga-api:
|
||||
build:
|
||||
dockerfile: Dockerfile
|
||||
context: .
|
||||
container_name: tranga-api
|
||||
volumes:
|
||||
- ./Manga:/Manga
|
||||
- ./settings:/usr/share/tranga-api
|
||||
ports:
|
||||
- "6531:6531"
|
||||
restart: unless-stopped
|
||||
tranga-website:
|
||||
image: glax/tranga-website:latest
|
||||
container_name: tranga-website
|
||||
ports:
|
||||
- "9555:80"
|
||||
depends_on:
|
||||
- tranga-api
|
||||
restart: unless-stopped
|
Reference in New Issue
Block a user