Compare commits

..

10 commits

Author SHA1 Message Date
28ddd40286
various fixes to make it working 2024-09-28 02:14:21 +02:00
Thephaseless
6577ad2df8 disable invalid websites 2024-09-13 19:22:37 +00:00
Thephaseless
0d5dfb0eb4 quick check 2024-09-13 19:13:37 +00:00
Thephaseless
fda6f18657 change the discovery of shadow dom 2024-09-13 19:07:43 +00:00
Thephaseless
d1bccc3e4c tweaks 2024-09-13 19:00:20 +00:00
Thephaseless
518cf1de42 adjustments 2024-09-13 18:48:31 +00:00
Thephaseless
35fce85fcf use python 3.12 2024-09-13 18:40:34 +00:00
Thephaseless
4dbb63842b test in dockerfile 2024-09-13 18:30:48 +00:00
Thephaseless
88e7321daa build if tests pass 2024-09-13 18:30:14 +00:00
Thephaseless
baae2fddeb use fastapi to test 2024-09-13 18:28:36 +00:00
11 changed files with 403 additions and 115 deletions

View file

@ -22,30 +22,6 @@ env:
IMAGE_NAME: ${{ github.repository }} IMAGE_NAME: ${{ github.repository }}
jobs: jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: "3.12"
- name: Run image
uses: abatilo/actions-poetry@v2
- uses: actions/cache@v3
name: Define a cache for the virtual environment based on the dependencies lock file
with:
path: ./.venv
key: venv-${{ hashFiles('poetry.lock') }}
- name: Install the project dependencies
run: poetry install
- name: Run tests
run: poetry run pytest -v
build: build:
runs-on: ubuntu-latest runs-on: ubuntu-latest
permissions: permissions:

View file

@ -1,35 +1,78 @@
FROM python:3.12 FROM python:3.12-slim-bullseye AS builder
# Build dummy packages to skip installing them and their dependencies
RUN apt-get update \
&& apt-get install -y --no-install-recommends equivs \
&& equivs-control libgl1-mesa-dri \
&& printf 'Section: misc\nPriority: optional\nStandards-Version: 3.9.2\nPackage: libgl1-mesa-dri\nVersion: 99.0.0\nDescription: Dummy package for libgl1-mesa-dri\n' >> libgl1-mesa-dri \
&& equivs-build libgl1-mesa-dri \
&& mv libgl1-mesa-dri_*.deb /libgl1-mesa-dri.deb \
&& equivs-control adwaita-icon-theme \
&& printf 'Section: misc\nPriority: optional\nStandards-Version: 3.9.2\nPackage: adwaita-icon-theme\nVersion: 99.0.0\nDescription: Dummy package for adwaita-icon-theme\n' >> adwaita-icon-theme \
&& equivs-build adwaita-icon-theme \
&& mv adwaita-icon-theme_*.deb /adwaita-icon-theme.deb
FROM python:3.12-slim-bullseye
# Copy dummy packages
COPY --from=builder /*.deb /
# Install dependencies and create byparr user
# You can test Chromium running this command inside the container:
# xvfb-run -s "-screen 0 1600x1200x24" chromium --no-sandbox
# The error traces is like this: "*** stack smashing detected ***: terminated"
# To check the package versions available you can use this command:
# apt-cache madison chromium
WORKDIR /app WORKDIR /app
# Install dummy packages
RUN dpkg -i /libgl1-mesa-dri.deb \
&& dpkg -i /adwaita-icon-theme.deb \
# Install dependencies
&& apt-get update \
&& apt-get install -y --no-install-recommends chromium xvfb dumb-init \
procps xauth sudo \
# Remove temporary files and hardware decoding libraries
&& rm -rf /var/lib/apt/lists/* \
&& rm -f /usr/lib/x86_64-linux-gnu/libmfxhw* \
&& rm -f /usr/lib/x86_64-linux-gnu/mfx/* \
# Create byparr user
&& useradd --home-dir /app --shell /bin/sh byparr \
&& chown -R byparr:byparr . \
&& echo 'byparr ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
# Install Python dependencies
COPY requirements.txt fix_nodriver.py novnc.sh .
RUN pip install -r requirements.txt \
# Remove temporary files
&& rm -rf /root/.cache \
&& PYENV_PATH=/usr/local/lib/python3.12 python fix_nodriver.py
ENV INSTALL_NOVNC=false DISPLAY=:1.0
RUN ./novnc.sh
USER byparr
COPY src .
EXPOSE 8191 EXPOSE 8191
# python # dumb-init avoids zombie chromium processes
ENV \ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
DEBIAN_FRONTEND=noninteractive \
PYTHONUNBUFFERED=1 \
# prevents python creating .pyc files
PYTHONDONTWRITEBYTECODE=1 \
# do not ask any interactive question
POETRY_NO_INTERACTION=1 \
POETRY_VIRTUALENVS_IN_PROJECT=true
RUN apt update && apt upgrade -y CMD ["/bin/sh", "-c", "/usr/local/share/desktop-init.sh && python -u /app/main.py"]
RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
RUN apt install -y --no-install-recommends --no-install-suggests ./google-chrome-stable_current_amd64.deb && rm ./google-chrome-stable_current_amd64.deb
RUN apt install pipx -y # Local build
RUN pipx ensurepath # docker build -t ngosang/byparr:3.3.21 .
RUN pipx install poetry # docker run -p 8191:8191 ngosang/byparr:3.3.21
ENV PATH="/root/.local/bin:$PATH"
COPY pyproject.toml poetry.lock ./
RUN poetry install
ENV INSTALL_NOVNC=false # Multi-arch build
COPY novnc.sh . # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
RUN ./novnc.sh # docker buildx create --use
ENV DISPLAY=:1.0 # docker buildx build -t ngosang/byparr:3.3.21 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
# add --push to publish in DockerHub
COPY . . # Test multi-arch build
RUN . /app/.venv/bin/activate && python fix_nodriver.py # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
# docker buildx create --use
CMD /usr/local/share/desktop-init.sh && . /app/.venv/bin/activate && python main.py # docker buildx build -t ngosang/byparr:3.3.21 --platform linux/arm/v7 --load .
# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/byparr:3.3.21

View file

@ -6,10 +6,14 @@ import logging
import os import os
from pathlib import Path from pathlib import Path
env_path = os.getenv("VIRTUAL_ENV") pyenv_path = os.getenv("PYENV_PATH")
if env_path is None: if pyenv_path is None:
env_path = Path(os.__file__).parent.parent.parent.as_posix() env_path = os.getenv("VIRTUAL_ENV")
nodriver_path = Path(env_path + "/lib/python3.11/site-packages/nodriver/cdp/network.py") if env_path is None:
env_path = Path(os.__file__).parent.parent.parent.as_posix()
pyenv_path = Path(env_path + "/lib/python3.12")
nodriver_pkg_path = Path(pyenv_path) / "site-packages/nodriver"
nodriver_path = nodriver_pkg_path / "cdp/network.py"
if not nodriver_path.exists(): if not nodriver_path.exists():
msg = f"{nodriver_path} not found" msg = f"{nodriver_path} not found"
raise FileNotFoundError(msg) raise FileNotFoundError(msg)
@ -61,3 +65,19 @@ with nodriver_path.open("r+") as f:
with nodriver_path.open("w") as f: with nodriver_path.open("w") as f:
f.writelines(lines) f.writelines(lines)
browser_path = nodriver_pkg_path / "core/browser.py"
if not browser_path.exists():
msg = f"{browser_path} not found"
raise FileNotFoundError(msg)
with browser_path.open("r") as f:
browser_path_txt = f.read()
browser_path_txt2 = browser_path_txt.replace("for _ in range(5):", "for _ in range(20):")
if browser_path_txt2 != browser_path_txt:
logger.info("increasing browser open timeout")
with browser_path.open("w") as f:
f.write(browser_path_txt2)

168
poetry.lock generated
View file

@ -1,9 +1,10 @@
# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. # This file is automatically @generated by Poetry and should not be changed by hand.
[[package]] [[package]]
name = "annotated-types" name = "annotated-types"
version = "0.7.0" version = "0.7.0"
description = "Reusable constraint types to use with typing.Annotated" description = "Reusable constraint types to use with typing.Annotated"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -15,6 +16,7 @@ files = [
name = "anyio" name = "anyio"
version = "4.4.0" version = "4.4.0"
description = "High level compatibility layer for multiple asynchronous event loop implementations" description = "High level compatibility layer for multiple asynchronous event loop implementations"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -35,6 +37,7 @@ trio = ["trio (>=0.23)"]
name = "certifi" name = "certifi"
version = "2024.8.30" version = "2024.8.30"
description = "Python package for providing Mozilla's CA Bundle." description = "Python package for providing Mozilla's CA Bundle."
category = "main"
optional = false optional = false
python-versions = ">=3.6" python-versions = ">=3.6"
files = [ files = [
@ -46,6 +49,7 @@ files = [
name = "charset-normalizer" name = "charset-normalizer"
version = "3.3.2" version = "3.3.2"
description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
category = "main"
optional = false optional = false
python-versions = ">=3.7.0" python-versions = ">=3.7.0"
files = [ files = [
@ -145,6 +149,7 @@ files = [
name = "click" name = "click"
version = "8.1.7" version = "8.1.7"
description = "Composable command line interface toolkit" description = "Composable command line interface toolkit"
category = "main"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
files = [ files = [
@ -159,6 +164,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""}
name = "colorama" name = "colorama"
version = "0.4.6" version = "0.4.6"
description = "Cross-platform colored terminal text." description = "Cross-platform colored terminal text."
category = "main"
optional = false optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
files = [ files = [
@ -170,6 +176,7 @@ files = [
name = "deprecated" name = "deprecated"
version = "1.2.14" version = "1.2.14"
description = "Python @deprecated decorator to deprecate old python classes, functions or methods." description = "Python @deprecated decorator to deprecate old python classes, functions or methods."
category = "main"
optional = false optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
files = [ files = [
@ -187,6 +194,7 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"]
name = "dnspython" name = "dnspython"
version = "2.6.1" version = "2.6.1"
description = "DNS toolkit" description = "DNS toolkit"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -207,6 +215,7 @@ wmi = ["wmi (>=1.5.1)"]
name = "email-validator" name = "email-validator"
version = "2.2.0" version = "2.2.0"
description = "A robust email address syntax and deliverability validation library." description = "A robust email address syntax and deliverability validation library."
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -222,6 +231,7 @@ idna = ">=2.0.0"
name = "fastapi" name = "fastapi"
version = "0.111.1" version = "0.111.1"
description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -247,6 +257,7 @@ all = ["email_validator (>=2.0.0)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)"
name = "fastapi-cli" name = "fastapi-cli"
version = "0.0.5" version = "0.0.5"
description = "Run and manage FastAPI apps from the command line with FastAPI CLI. 🚀" description = "Run and manage FastAPI apps from the command line with FastAPI CLI. 🚀"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -265,6 +276,7 @@ standard = ["uvicorn[standard] (>=0.15.0)"]
name = "h11" name = "h11"
version = "0.14.0" version = "0.14.0"
description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
category = "main"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
files = [ files = [
@ -276,6 +288,7 @@ files = [
name = "httpcore" name = "httpcore"
version = "1.0.5" version = "1.0.5"
description = "A minimal low-level HTTP client." description = "A minimal low-level HTTP client."
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -290,13 +303,14 @@ h11 = ">=0.13,<0.15"
[package.extras] [package.extras]
asyncio = ["anyio (>=4.0,<5.0)"] asyncio = ["anyio (>=4.0,<5.0)"]
http2 = ["h2 (>=3,<5)"] http2 = ["h2 (>=3,<5)"]
socks = ["socksio (==1.*)"] socks = ["socksio (>=1.0.0,<2.0.0)"]
trio = ["trio (>=0.22.0,<0.26.0)"] trio = ["trio (>=0.22.0,<0.26.0)"]
[[package]] [[package]]
name = "httptools" name = "httptools"
version = "0.6.1" version = "0.6.1"
description = "A collection of framework independent HTTP protocol utils." description = "A collection of framework independent HTTP protocol utils."
category = "main"
optional = false optional = false
python-versions = ">=3.8.0" python-versions = ">=3.8.0"
files = [ files = [
@ -345,6 +359,7 @@ test = ["Cython (>=0.29.24,<0.30.0)"]
name = "httpx" name = "httpx"
version = "0.27.2" version = "0.27.2"
description = "The next generation HTTP client." description = "The next generation HTTP client."
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -355,21 +370,22 @@ files = [
[package.dependencies] [package.dependencies]
anyio = "*" anyio = "*"
certifi = "*" certifi = "*"
httpcore = "==1.*" httpcore = ">=1.0.0,<2.0.0"
idna = "*" idna = "*"
sniffio = "*" sniffio = "*"
[package.extras] [package.extras]
brotli = ["brotli", "brotlicffi"] brotli = ["brotli", "brotlicffi"]
cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<14)"]
http2 = ["h2 (>=3,<5)"] http2 = ["h2 (>=3,<5)"]
socks = ["socksio (==1.*)"] socks = ["socksio (>=1.0.0,<2.0.0)"]
zstd = ["zstandard (>=0.18.0)"] zstd = ["zstandard (>=0.18.0)"]
[[package]] [[package]]
name = "idna" name = "idna"
version = "3.8" version = "3.8"
description = "Internationalized Domain Names in Applications (IDNA)" description = "Internationalized Domain Names in Applications (IDNA)"
category = "main"
optional = false optional = false
python-versions = ">=3.6" python-versions = ">=3.6"
files = [ files = [
@ -381,6 +397,7 @@ files = [
name = "iniconfig" name = "iniconfig"
version = "2.0.0" version = "2.0.0"
description = "brain-dead simple config-ini parsing" description = "brain-dead simple config-ini parsing"
category = "main"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
files = [ files = [
@ -392,6 +409,7 @@ files = [
name = "jinja2" name = "jinja2"
version = "3.1.4" version = "3.1.4"
description = "A very fast and expressive template engine." description = "A very fast and expressive template engine."
category = "main"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
files = [ files = [
@ -409,6 +427,7 @@ i18n = ["Babel (>=2.7)"]
name = "markdown-it-py" name = "markdown-it-py"
version = "3.0.0" version = "3.0.0"
description = "Python port of markdown-it. Markdown parsing, done right!" description = "Python port of markdown-it. Markdown parsing, done right!"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -433,6 +452,7 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"]
name = "markupsafe" name = "markupsafe"
version = "2.1.5" version = "2.1.5"
description = "Safely add untrusted strings to HTML/XML markup." description = "Safely add untrusted strings to HTML/XML markup."
category = "main"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
files = [ files = [
@ -502,6 +522,7 @@ files = [
name = "mdurl" name = "mdurl"
version = "0.1.2" version = "0.1.2"
description = "Markdown URL utilities" description = "Markdown URL utilities"
category = "main"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
files = [ files = [
@ -513,6 +534,7 @@ files = [
name = "mss" name = "mss"
version = "9.0.2" version = "9.0.2"
description = "An ultra fast cross-platform multiple screenshots module in pure python using ctypes." description = "An ultra fast cross-platform multiple screenshots module in pure python using ctypes."
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -524,10 +546,26 @@ files = [
dev = ["build (==1.2.1)", "mypy (==1.11.2)", "ruff (==0.6.3)", "twine (==5.1.1)", "wheel (==0.44.0)"] dev = ["build (==1.2.1)", "mypy (==1.11.2)", "ruff (==0.6.3)", "twine (==5.1.1)", "wheel (==0.44.0)"]
test = ["numpy (==2.1.0)", "pillow (==10.4.0)", "pytest (==8.3.2)", "pytest-cov (==5.0.0)", "pytest-rerunfailures (==14.0.0)", "pyvirtualdisplay (==3.0)", "sphinx (==8.0.2)"] test = ["numpy (==2.1.0)", "pillow (==10.4.0)", "pytest (==8.3.2)", "pytest-cov (==5.0.0)", "pytest-rerunfailures (==14.0.0)", "pyvirtualdisplay (==3.0)", "sphinx (==8.0.2)"]
[[package]]
name = "nn-mouse"
version = "1.0.1"
description = "Neural network based humanized mouse movements"
category = "main"
optional = false
python-versions = ">=3"
files = [
{file = "nn_mouse-1.0.1-py3-none-any.whl", hash = "sha256:726d548ba3796914c454507b4b91d8e2555e870392fc03f7f136ee0cf1db0150"},
{file = "nn_mouse-1.0.1.tar.gz", hash = "sha256:d92566959a339b99f47d4b403069f3afdc81d47d019836368d6dfd43fe345f0a"},
]
[package.dependencies]
opencv-python = "*"
[[package]] [[package]]
name = "nodriver" name = "nodriver"
version = "0.34" version = "0.34"
description = "[Docs here](https://ultrafunkamsterdam.github.io/nodriver)" description = "[Docs here](https://ultrafunkamsterdam.github.io/nodriver)"
category = "main"
optional = false optional = false
python-versions = ">=3.9" python-versions = ">=3.9"
files = [ files = [
@ -543,10 +581,102 @@ websockets = ">=11"
[package.extras] [package.extras]
dev = ["black", "build", "furo", "pygments", "sphinx", "sphinx-autodoc-typehints", "sphinx-markdown-builder"] dev = ["black", "build", "furo", "pygments", "sphinx", "sphinx-autodoc-typehints", "sphinx-markdown-builder"]
[[package]]
name = "numpy"
version = "2.1.1"
description = "Fundamental package for array computing in Python"
category = "main"
optional = false
python-versions = ">=3.10"
files = [
{file = "numpy-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c8a0e34993b510fc19b9a2ce7f31cb8e94ecf6e924a40c0c9dd4f62d0aac47d9"},
{file = "numpy-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7dd86dfaf7c900c0bbdcb8b16e2f6ddf1eb1fe39c6c8cca6e94844ed3152a8fd"},
{file = "numpy-2.1.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:5889dd24f03ca5a5b1e8a90a33b5a0846d8977565e4ae003a63d22ecddf6782f"},
{file = "numpy-2.1.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:59ca673ad11d4b84ceb385290ed0ebe60266e356641428c845b39cd9df6713ab"},
{file = "numpy-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13ce49a34c44b6de5241f0b38b07e44c1b2dcacd9e36c30f9c2fcb1bb5135db7"},
{file = "numpy-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:913cc1d311060b1d409e609947fa1b9753701dac96e6581b58afc36b7ee35af6"},
{file = "numpy-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:caf5d284ddea7462c32b8d4a6b8af030b6c9fd5332afb70e7414d7fdded4bfd0"},
{file = "numpy-2.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:57eb525e7c2a8fdee02d731f647146ff54ea8c973364f3b850069ffb42799647"},
{file = "numpy-2.1.1-cp310-cp310-win32.whl", hash = "sha256:9a8e06c7a980869ea67bbf551283bbed2856915f0a792dc32dd0f9dd2fb56728"},
{file = "numpy-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:d10c39947a2d351d6d466b4ae83dad4c37cd6c3cdd6d5d0fa797da56f710a6ae"},
{file = "numpy-2.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0d07841fd284718feffe7dd17a63a2e6c78679b2d386d3e82f44f0108c905550"},
{file = "numpy-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b5613cfeb1adfe791e8e681128f5f49f22f3fcaa942255a6124d58ca59d9528f"},
{file = "numpy-2.1.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:0b8cc2715a84b7c3b161f9ebbd942740aaed913584cae9cdc7f8ad5ad41943d0"},
{file = "numpy-2.1.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:b49742cdb85f1f81e4dc1b39dcf328244f4d8d1ded95dea725b316bd2cf18c95"},
{file = "numpy-2.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8d5f8a8e3bc87334f025194c6193e408903d21ebaeb10952264943a985066ca"},
{file = "numpy-2.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d51fc141ddbe3f919e91a096ec739f49d686df8af254b2053ba21a910ae518bf"},
{file = "numpy-2.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:98ce7fb5b8063cfdd86596b9c762bf2b5e35a2cdd7e967494ab78a1fa7f8b86e"},
{file = "numpy-2.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:24c2ad697bd8593887b019817ddd9974a7f429c14a5469d7fad413f28340a6d2"},
{file = "numpy-2.1.1-cp311-cp311-win32.whl", hash = "sha256:397bc5ce62d3fb73f304bec332171535c187e0643e176a6e9421a6e3eacef06d"},
{file = "numpy-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:ae8ce252404cdd4de56dcfce8b11eac3c594a9c16c231d081fb705cf23bd4d9e"},
{file = "numpy-2.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7c803b7934a7f59563db459292e6aa078bb38b7ab1446ca38dd138646a38203e"},
{file = "numpy-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6435c48250c12f001920f0751fe50c0348f5f240852cfddc5e2f97e007544cbe"},
{file = "numpy-2.1.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3269c9eb8745e8d975980b3a7411a98976824e1fdef11f0aacf76147f662b15f"},
{file = "numpy-2.1.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:fac6e277a41163d27dfab5f4ec1f7a83fac94e170665a4a50191b545721c6521"},
{file = "numpy-2.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fcd8f556cdc8cfe35e70efb92463082b7f43dd7e547eb071ffc36abc0ca4699b"},
{file = "numpy-2.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b9cd92c8f8e7b313b80e93cedc12c0112088541dcedd9197b5dee3738c1201"},
{file = "numpy-2.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:afd9c680df4de71cd58582b51e88a61feed4abcc7530bcd3d48483f20fc76f2a"},
{file = "numpy-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8661c94e3aad18e1ea17a11f60f843a4933ccaf1a25a7c6a9182af70610b2313"},
{file = "numpy-2.1.1-cp312-cp312-win32.whl", hash = "sha256:950802d17a33c07cba7fd7c3dcfa7d64705509206be1606f196d179e539111ed"},
{file = "numpy-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:3fc5eabfc720db95d68e6646e88f8b399bfedd235994016351b1d9e062c4b270"},
{file = "numpy-2.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:046356b19d7ad1890c751b99acad5e82dc4a02232013bd9a9a712fddf8eb60f5"},
{file = "numpy-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6e5a9cb2be39350ae6c8f79410744e80154df658d5bea06e06e0ac5bb75480d5"},
{file = "numpy-2.1.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:d4c57b68c8ef5e1ebf47238e99bf27657511ec3f071c465f6b1bccbef12d4136"},
{file = "numpy-2.1.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:8ae0fd135e0b157365ac7cc31fff27f07a5572bdfc38f9c2d43b2aff416cc8b0"},
{file = "numpy-2.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:981707f6b31b59c0c24bcda52e5605f9701cb46da4b86c2e8023656ad3e833cb"},
{file = "numpy-2.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ca4b53e1e0b279142113b8c5eb7d7a877e967c306edc34f3b58e9be12fda8df"},
{file = "numpy-2.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e097507396c0be4e547ff15b13dc3866f45f3680f789c1a1301b07dadd3fbc78"},
{file = "numpy-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7506387e191fe8cdb267f912469a3cccc538ab108471291636a96a54e599556"},
{file = "numpy-2.1.1-cp313-cp313-win32.whl", hash = "sha256:251105b7c42abe40e3a689881e1793370cc9724ad50d64b30b358bbb3a97553b"},
{file = "numpy-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:f212d4f46b67ff604d11fff7cc62d36b3e8714edf68e44e9760e19be38c03eb0"},
{file = "numpy-2.1.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:920b0911bb2e4414c50e55bd658baeb78281a47feeb064ab40c2b66ecba85553"},
{file = "numpy-2.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bab7c09454460a487e631ffc0c42057e3d8f2a9ddccd1e60c7bb8ed774992480"},
{file = "numpy-2.1.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:cea427d1350f3fd0d2818ce7350095c1a2ee33e30961d2f0fef48576ddbbe90f"},
{file = "numpy-2.1.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:e30356d530528a42eeba51420ae8bf6c6c09559051887196599d96ee5f536468"},
{file = "numpy-2.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8dfa9e94fc127c40979c3eacbae1e61fda4fe71d84869cc129e2721973231ef"},
{file = "numpy-2.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:910b47a6d0635ec1bd53b88f86120a52bf56dcc27b51f18c7b4a2e2224c29f0f"},
{file = "numpy-2.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:13cc11c00000848702322af4de0147ced365c81d66053a67c2e962a485b3717c"},
{file = "numpy-2.1.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:53e27293b3a2b661c03f79aa51c3987492bd4641ef933e366e0f9f6c9bf257ec"},
{file = "numpy-2.1.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7be6a07520b88214ea85d8ac8b7d6d8a1839b0b5cb87412ac9f49fa934eb15d5"},
{file = "numpy-2.1.1-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:52ac2e48f5ad847cd43c4755520a2317f3380213493b9d8a4c5e37f3b87df504"},
{file = "numpy-2.1.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50a95ca3560a6058d6ea91d4629a83a897ee27c00630aed9d933dff191f170cd"},
{file = "numpy-2.1.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:99f4a9ee60eed1385a86e82288971a51e71df052ed0b2900ed30bc840c0f2e39"},
{file = "numpy-2.1.1.tar.gz", hash = "sha256:d0cf7d55b1051387807405b3898efafa862997b4cba8aa5dbe657be794afeafd"},
]
[[package]]
name = "opencv-python"
version = "4.10.0.84"
description = "Wrapper package for OpenCV python bindings."
category = "main"
optional = false
python-versions = ">=3.6"
files = [
{file = "opencv-python-4.10.0.84.tar.gz", hash = "sha256:72d234e4582e9658ffea8e9cae5b63d488ad06994ef12d81dc303b17472f3526"},
{file = "opencv_python-4.10.0.84-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fc182f8f4cda51b45f01c64e4cbedfc2f00aff799debebc305d8d0210c43f251"},
{file = "opencv_python-4.10.0.84-cp37-abi3-macosx_12_0_x86_64.whl", hash = "sha256:71e575744f1d23f79741450254660442785f45a0797212852ee5199ef12eed98"},
{file = "opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09a332b50488e2dda866a6c5573ee192fe3583239fb26ff2f7f9ceb0bc119ea6"},
{file = "opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ace140fc6d647fbe1c692bcb2abce768973491222c067c131d80957c595b71f"},
{file = "opencv_python-4.10.0.84-cp37-abi3-win32.whl", hash = "sha256:2db02bb7e50b703f0a2d50c50ced72e95c574e1e5a0bb35a8a86d0b35c98c236"},
{file = "opencv_python-4.10.0.84-cp37-abi3-win_amd64.whl", hash = "sha256:32dbbd94c26f611dc5cc6979e6b7aa1f55a64d6b463cc1dcd3c95505a63e48fe"},
]
[package.dependencies]
numpy = [
{version = ">=1.21.2", markers = "python_version >= \"3.10\""},
{version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""},
{version = ">=1.23.5", markers = "python_version >= \"3.11\""},
{version = ">=1.26.0", markers = "python_version >= \"3.12\""},
{version = ">=1.19.3", markers = "python_version >= \"3.6\" and platform_system == \"Linux\" and platform_machine == \"aarch64\" or python_version >= \"3.9\""},
{version = ">=1.17.0", markers = "python_version >= \"3.7\""},
{version = ">=1.17.3", markers = "python_version >= \"3.8\""},
]
[[package]] [[package]]
name = "packaging" name = "packaging"
version = "24.1" version = "24.1"
description = "Core utilities for Python packages" description = "Core utilities for Python packages"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -558,6 +688,7 @@ files = [
name = "pluggy" name = "pluggy"
version = "1.5.0" version = "1.5.0"
description = "plugin and hook calling mechanisms for python" description = "plugin and hook calling mechanisms for python"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -573,6 +704,7 @@ testing = ["pytest", "pytest-benchmark"]
name = "pydantic" name = "pydantic"
version = "2.9.1" version = "2.9.1"
description = "Data validation using Python type hints" description = "Data validation using Python type hints"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -596,6 +728,7 @@ timezone = ["tzdata"]
name = "pydantic-core" name = "pydantic-core"
version = "2.23.3" version = "2.23.3"
description = "Core functionality for Pydantic validation and serialization" description = "Core functionality for Pydantic validation and serialization"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -697,6 +830,7 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
name = "pygments" name = "pygments"
version = "2.18.0" version = "2.18.0"
description = "Pygments is a syntax highlighting package written in Python." description = "Pygments is a syntax highlighting package written in Python."
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -711,6 +845,7 @@ windows-terminal = ["colorama (>=0.4.6)"]
name = "pytest" name = "pytest"
version = "8.3.3" version = "8.3.3"
description = "pytest: simple powerful testing with Python" description = "pytest: simple powerful testing with Python"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -731,6 +866,7 @@ dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments
name = "pytest-asyncio" name = "pytest-asyncio"
version = "0.24.0" version = "0.24.0"
description = "Pytest support for asyncio" description = "Pytest support for asyncio"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -749,6 +885,7 @@ testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"]
name = "python-dotenv" name = "python-dotenv"
version = "1.0.1" version = "1.0.1"
description = "Read key-value pairs from a .env file and set them as environment variables" description = "Read key-value pairs from a .env file and set them as environment variables"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -763,6 +900,7 @@ cli = ["click (>=5.0)"]
name = "python-multipart" name = "python-multipart"
version = "0.0.9" version = "0.0.9"
description = "A streaming multipart parser for Python" description = "A streaming multipart parser for Python"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -777,6 +915,7 @@ dev = ["atomicwrites (==1.4.1)", "attrs (==23.2.0)", "coverage (==7.4.1)", "hatc
name = "pyyaml" name = "pyyaml"
version = "6.0.2" version = "6.0.2"
description = "YAML parser and emitter for Python" description = "YAML parser and emitter for Python"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -839,6 +978,7 @@ files = [
name = "requests" name = "requests"
version = "2.32.3" version = "2.32.3"
description = "Python HTTP for Humans." description = "Python HTTP for Humans."
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -860,6 +1000,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
name = "rich" name = "rich"
version = "13.8.1" version = "13.8.1"
description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
category = "main"
optional = false optional = false
python-versions = ">=3.7.0" python-versions = ">=3.7.0"
files = [ files = [
@ -878,6 +1019,7 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"]
name = "shellingham" name = "shellingham"
version = "1.5.4" version = "1.5.4"
description = "Tool to Detect Surrounding Shell" description = "Tool to Detect Surrounding Shell"
category = "main"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
files = [ files = [
@ -889,6 +1031,7 @@ files = [
name = "sniffio" name = "sniffio"
version = "1.3.1" version = "1.3.1"
description = "Sniff out which async library your code is running under" description = "Sniff out which async library your code is running under"
category = "main"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
files = [ files = [
@ -900,6 +1043,7 @@ files = [
name = "starlette" name = "starlette"
version = "0.37.2" version = "0.37.2"
description = "The little ASGI library that shines." description = "The little ASGI library that shines."
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -917,6 +1061,7 @@ full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.7
name = "typer" name = "typer"
version = "0.12.5" version = "0.12.5"
description = "Typer, build great CLIs. Easy to code. Based on Python type hints." description = "Typer, build great CLIs. Easy to code. Based on Python type hints."
category = "main"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
files = [ files = [
@ -934,6 +1079,7 @@ typing-extensions = ">=3.7.4.3"
name = "typing-extensions" name = "typing-extensions"
version = "4.12.2" version = "4.12.2"
description = "Backported and Experimental Type Hints for Python 3.8+" description = "Backported and Experimental Type Hints for Python 3.8+"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -945,6 +1091,7 @@ files = [
name = "urllib3" name = "urllib3"
version = "2.2.3" version = "2.2.3"
description = "HTTP library with thread-safe connection pooling, file post, and more." description = "HTTP library with thread-safe connection pooling, file post, and more."
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -962,6 +1109,7 @@ zstd = ["zstandard (>=0.18.0)"]
name = "uvicorn" name = "uvicorn"
version = "0.30.6" version = "0.30.6"
description = "The lightning-fast ASGI server." description = "The lightning-fast ASGI server."
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -976,7 +1124,7 @@ h11 = ">=0.8"
httptools = {version = ">=0.5.0", optional = true, markers = "extra == \"standard\""} httptools = {version = ">=0.5.0", optional = true, markers = "extra == \"standard\""}
python-dotenv = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} python-dotenv = {version = ">=0.13", optional = true, markers = "extra == \"standard\""}
pyyaml = {version = ">=5.1", optional = true, markers = "extra == \"standard\""} pyyaml = {version = ">=5.1", optional = true, markers = "extra == \"standard\""}
uvloop = {version = ">=0.14.0,<0.15.0 || >0.15.0,<0.15.1 || >0.15.1", optional = true, markers = "(sys_platform != \"win32\" and sys_platform != \"cygwin\") and platform_python_implementation != \"PyPy\" and extra == \"standard\""} uvloop = {version = ">=0.14.0,<0.15.0 || >0.15.0,<0.15.1 || >0.15.1", optional = true, markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" and extra == \"standard\""}
watchfiles = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} watchfiles = {version = ">=0.13", optional = true, markers = "extra == \"standard\""}
websockets = {version = ">=10.4", optional = true, markers = "extra == \"standard\""} websockets = {version = ">=10.4", optional = true, markers = "extra == \"standard\""}
@ -987,6 +1135,7 @@ standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)",
name = "uvloop" name = "uvloop"
version = "0.20.0" version = "0.20.0"
description = "Fast implementation of asyncio event loop on top of libuv" description = "Fast implementation of asyncio event loop on top of libuv"
category = "main"
optional = false optional = false
python-versions = ">=3.8.0" python-versions = ">=3.8.0"
files = [ files = [
@ -1031,6 +1180,7 @@ test = ["Cython (>=0.29.36,<0.30.0)", "aiohttp (==3.9.0b0)", "aiohttp (>=3.8.1)"
name = "watchfiles" name = "watchfiles"
version = "0.24.0" version = "0.24.0"
description = "Simple, modern and high performance file watching and code reload in python." description = "Simple, modern and high performance file watching and code reload in python."
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -1126,6 +1276,7 @@ anyio = ">=3.0.0"
name = "websockets" name = "websockets"
version = "13.0.1" version = "13.0.1"
description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)"
category = "main"
optional = false optional = false
python-versions = ">=3.8" python-versions = ">=3.8"
files = [ files = [
@ -1221,6 +1372,7 @@ files = [
name = "wrapt" name = "wrapt"
version = "1.16.0" version = "1.16.0"
description = "Module for decorators, wrappers and monkey patching." description = "Module for decorators, wrappers and monkey patching."
category = "main"
optional = false optional = false
python-versions = ">=3.6" python-versions = ">=3.6"
files = [ files = [
@ -1298,5 +1450,5 @@ files = [
[metadata] [metadata]
lock-version = "2.0" lock-version = "2.0"
python-versions = "^3.11" python-versions = "^3.12"
content-hash = "8e6c46b679a81d033850227859c9754ca5e58b3b10d1e13383bf0576bb34da5c" content-hash = "36519f0ca589faeaac9710ec3f1b055905d9560673ec53611abacbe259990b53"

View file

@ -2,18 +2,19 @@
name = "byparr" name = "byparr"
version = "0.1.0" version = "0.1.0"
description = "" description = ""
package-mode = false # package-mode = false
authors = ["Your Name <you@example.com>"] authors = ["Your Name <you@example.com>"]
readme = "README.md" readme = "README.md"
[tool.poetry.dependencies] [tool.poetry.dependencies]
python = "^3.11" python = "^3.12"
pytest = "^8.3.1" pytest = "^8.3.1"
fastapi = "^0.111.1" fastapi = "^0.111.1"
nodriver = "^0.34" nodriver = "^0.34"
requests = "^2.32.3" requests = "^2.32.3"
httpx = "^0.27.2" httpx = "^0.27.2"
pytest-asyncio = "^0.24.0" pytest-asyncio = "^0.24.0"
nn-mouse = "^1.0.1"
[build-system] [build-system]

5
requirements.txt Normal file
View file

@ -0,0 +1,5 @@
fastapi==0.111.1
nodriver==0.34
requests==2.32.3
httpx==0.27.2
nn-mouse==1.0.1

View file

@ -9,10 +9,10 @@ import uvicorn.config
from fastapi import FastAPI from fastapi import FastAPI
from fastapi.responses import RedirectResponse from fastapi.responses import RedirectResponse
from src.models.requests import LinkRequest, LinkResponse from models.requests import LinkRequest, LinkResponse
from src.utils import logger from utils import logger
from src.utils.browser import bypass_cloudflare, new_browser from utils.browser import bypass_cloudflare, new_browser
from src.utils.consts import LOG_LEVEL from utils.consts import LOG_LEVEL
app = FastAPI(debug=LOG_LEVEL == logging.DEBUG, log_level=LOG_LEVEL) app = FastAPI(debug=LOG_LEVEL == logging.DEBUG, log_level=LOG_LEVEL)
@ -31,25 +31,23 @@ async def read_item(request: LinkRequest):
logger.info(f"Request: {request}") logger.info(f"Request: {request}")
start_time = int(time.time() * 1000) start_time = int(time.time() * 1000)
browser = await new_browser() browser = await new_browser()
try: page = await browser.get(request.url)
page = await browser.get(request.url) await page.bring_to_front()
await page.bring_to_front() timeout = request.maxTimeout
if timeout == 0:
timeout = None
challenged = await asyncio.wait_for( challenged = await asyncio.wait_for(bypass_cloudflare(page), timeout=timeout)
bypass_cloudflare(page), timeout=request.maxTimeout
)
logger.info(f"Got webpage: {request.url}") logger.info(f"Got webpage: {request.url}")
response = await LinkResponse.create( response = await LinkResponse.create(
page=page, page=page,
start_timestamp=start_time, start_timestamp=start_time,
challenged=challenged, challenged=challenged,
) )
except asyncio.TimeoutError:
logger.fatal("Couldn't complete the request") browser.stop()
finally:
browser.stop()
return response return response

View file

@ -1,6 +1,6 @@
import logging import logging
from src.utils.consts import LOG_LEVEL from utils.consts import LOG_LEVEL
logger = logging.getLogger("uvicorn.error") logger = logging.getLogger("uvicorn.error")
logger.setLevel(LOG_LEVEL) logger.setLevel(LOG_LEVEL)

View file

@ -1,13 +1,13 @@
import asyncio import asyncio
import typing
import random
import nodriver as webdriver import nodriver as webdriver
from nodriver.core.element import Element from nodriver.core.element import Element, Position
from nn_mouse import get_path
from src.utils import logger from . import logger
from src.utils.consts import CHALLENGE_TITLES from .consts import CHALLENGE_TITLES
from src.utils.extentions import download_extentions
downloaded_extentions = download_extentions()
async def new_browser(): async def new_browser():
@ -25,7 +25,6 @@ async def new_browser():
""" """
config: webdriver.Config = webdriver.Config() config: webdriver.Config = webdriver.Config()
config.sandbox = False config.sandbox = False
config.add_argument(f"--load-extension={','.join(downloaded_extentions)}")
return await webdriver.start(config=config) return await webdriver.start(config=config)
@ -65,6 +64,12 @@ async def bypass_cloudflare(page: webdriver.Tab):
if not challenged: if not challenged:
logger.info("Found challenge") logger.info("Found challenge")
challenged = True challenged = True
elem = None
# get the size of the page
html_elem = await page.select("html")
html_pos = await html_elem.get_position()
try: try:
elem = await page.find( elem = await page.find(
"Verify you are human by completing the action below.", "Verify you are human by completing the action below.",
@ -74,31 +79,30 @@ async def bypass_cloudflare(page: webdriver.Tab):
except asyncio.TimeoutError: except asyncio.TimeoutError:
if page.target.title not in CHALLENGE_TITLES: if page.target.title not in CHALLENGE_TITLES:
return challenged return challenged
raise
if elem is None: if elem is None:
logger.debug("Couldn't find the title, trying again") logger.debug("Couldn't find the title, trying other method...")
continue continue
if not isinstance(elem, Element):
logger.fatal("Element is a string, please report this to Byparr dev")
raise InvalidElementError
elem = await page.find("input")
elem = elem.parent elem = elem.parent
# Get the element containing the shadow root # Get the element containing the shadow root
for _ in range(3):
if elem is not None:
elem = get_first_div(elem)
else:
raise InvalidElementError
if isinstance(elem, Element) and elem.shadow_roots: if isinstance(elem, Element) and elem.shadow_roots:
inner_elem = Element(elem.shadow_roots[0], page, elem.tree).children[0] inner_elem = Element(elem.shadow_roots[0], page, elem.tree).children[0]
if isinstance(inner_elem, Element): if isinstance(inner_elem, Element):
logger.debug("Clicking element") logger.debug("Clicking element")
await inner_elem.mouse_click() await mouse_click(inner_elem, html_pos=html_pos)
else: else:
logger.warn( logger.warning(
"Element is a string, please report this to Byparr dev" "Element is a string, please report this to Byparr dev"
) # I really hope this never happens ) # I really hope this never happens
else: else:
logger.warn("Coulnd't find checkbox, trying again...") logger.warning("Coulnd't find checkbox, trying again...")
def get_first_div(elem): def get_first_div(elem):
@ -120,5 +124,85 @@ def get_first_div(elem):
raise InvalidElementError raise InvalidElementError
def clamp(n: int, smallest: int, largest: int):
return max(smallest, min(n, largest))
async def mouse_click(
elm,
button: str = "left",
buttons: typing.Optional[int] = 1,
modifiers: typing.Optional[int] = 0,
hold: bool = False,
html_pos: typing.Optional[Position] = None,
_until_event: typing.Optional[type] = None,
):
"""native click (on element) . note: this likely does not work atm, use click() instead
:param button: str (default = "left")
:param buttons: which button (default 1 = left)
:param modifiers: *(Optional)* Bit field representing pressed modifier keys.
Alt=1, Ctrl=2, Meta/Command=4, Shift=8 (default: 0).
:param _until_event: internal. event to wait for before returning
:return:
"""
try:
pos = await elm.get_position()
except AttributeError:
return
if not pos:
logger.warning("could not calculate box model for %s", self)
return
oc = pos.center
center = (oc[0] + random.randint(-10, 10), oc[1] + random.randint(-10, 10))
html_height = 500
html_width = 500
if html_pos:
html_height = html_pos.height
html_width = html_pos.width
start_x = random.randint(0, html_width)
start_y = random.randint(0, html_height)
path = get_path(start_x, start_y, center[0], center[1], html_width, html_height)
for (x, y, t) in path:
x = clamp(int(x), 0, html_width)
y = clamp(int(y), 0, html_height)
await elm._tab.send(webdriver.cdp.input_.dispatch_mouse_event("mousemove", x, y))
await asyncio.sleep(float(t))
logger.debug("clicking on location %.2f, %.2f" % center)
await asyncio.sleep(random.uniform(0.05, 0.3))
await elm._tab.send(
webdriver.cdp.input_.dispatch_mouse_event(
"mousePressed",
x=center[0],
y=center[1],
modifiers=modifiers,
button=webdriver.cdp.input_.MouseButton(button),
buttons=buttons,
click_count=1,
)
)
await asyncio.sleep(random.uniform(0.02, 0.06))
await elm._tab.send(
webdriver.cdp.input_.dispatch_mouse_event(
"mouseReleased",
x=center[0],
y=center[1],
modifiers=modifiers,
button=webdriver.cdp.input_.MouseButton(button),
buttons=buttons,
click_count=1,
)
)
# Flash the checkbox for testing
try:
await elm.flash()
except: # noqa
pass
class InvalidElementError(Exception): class InvalidElementError(Exception):
pass pass

View file

@ -8,10 +8,10 @@ from zipfile import ZipFile
import httpx import httpx
import requests import requests
from src.models.github import GithubResponse from ..models.github import GithubResponse
from src.models.requests import NoChromeExtentionError from ..models.requests import NoChromeExtentionError
from src.utils import logger from . import logger
from src.utils.consts import EXTENTION_REPOSITIORIES, EXTENTIONS_PATH, GITHUB_WEBSITES from .consts import EXTENTION_REPOSITIORIES, EXTENTIONS_PATH, GITHUB_WEBSITES
def get_latest_github_chrome_release(url: str): def get_latest_github_chrome_release(url: str):

View file

@ -1,18 +1,27 @@
import pytest from http import HTTPStatus
from main import read_item import pytest
from starlette.testclient import TestClient
from main import app
from src.models.requests import LinkRequest from src.models.requests import LinkRequest
client = TestClient(app)
test_websites = [ test_websites = [
"https://ext.to/", "https://ext.to/",
"https://btmet.com/", "https://btmet.com/",
"https://extratorrent.st/", # "https://extratorrent.st/", # github is blocking these
"https://idope.se/", # "https://idope.se/", # github is blocking these
] ]
pytest_plugins = ("pytest_asyncio",)
@pytest.mark.parametrize("website", test_websites) @pytest.mark.parametrize("website", test_websites)
@pytest.mark.asyncio def test_bypass(website: str):
async def test_bypass(website: str): response = client.post(
await read_item(LinkRequest(url=website, maxTimeout=5, cmd="")) "/v1",
json=LinkRequest(
url=website, maxTimeout=60 * len(test_websites), cmd="request.get"
).model_dump(),
)
assert response.status_code == HTTPStatus.OK