Faster download for images-urls.

#224
This commit is contained in:
Glax 2024-08-20 20:47:13 +02:00
parent 34dd78810d
commit 21af75f410
2 changed files with 14 additions and 3 deletions

View File

@ -1,5 +1,6 @@
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using PuppeteerSharp;
using PuppeteerSharp.Input;
@ -11,10 +12,11 @@ internal class ChromiumDownloadClient : DownloadClient
private IBrowser browser { get; set; }
private const string ChromiumVersion = "1154303";
private const int StartTimeoutMs = 30000;
private HttpDownloadClient _httpDownloadClient;
private async Task<IBrowser> DownloadBrowser()
{
BrowserFetcher browserFetcher = new BrowserFetcher();
BrowserFetcher browserFetcher = new ();
foreach(string rev in browserFetcher.LocalRevisions().Where(rev => rev != ChromiumVersion))
browserFetcher.Remove(rev);
if (!browserFetcher.LocalRevisions().Contains(ChromiumVersion))
@ -58,9 +60,18 @@ internal class ChromiumDownloadClient : DownloadClient
public ChromiumDownloadClient(GlobalBase clone) : base(clone)
{
this.browser = DownloadBrowser().Result;
_httpDownloadClient = new(this);
}
protected override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null)
private readonly Regex _imageUrlRex = new(@"https?:\/\/.*\.(?:p?jpe?g|gif|a?png|bmp|avif|webp)(\?.*)?");
internal override RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null)
{
return _imageUrlRex.IsMatch(url)
? _httpDownloadClient.MakeRequestInternal(url, referrer)
: MakeRequestBrowser(url, referrer, clickButton);
}
private RequestResult MakeRequestBrowser(string url, string? referrer = null, string? clickButton = null)
{
IPage page = this.browser.NewPageAsync().Result;
page.DefaultTimeout = 10000;

View File

@ -40,6 +40,6 @@ internal abstract class DownloadClient : GlobalBase
return result;
}
protected abstract RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null);
internal abstract RequestResult MakeRequestInternal(string url, string? referrer = null, string? clickButton = null);
public abstract void Close();
}