Compare commits

...

10 commits

Author SHA1 Message Date
28ddd40286
various fixes to make it working 2024-09-28 02:14:21 +02:00
Thephaseless
6577ad2df8 disable invalid websites 2024-09-13 19:22:37 +00:00
Thephaseless
0d5dfb0eb4 quick check 2024-09-13 19:13:37 +00:00
Thephaseless
fda6f18657 change the discovery of shadow dom 2024-09-13 19:07:43 +00:00
Thephaseless
d1bccc3e4c tweaks 2024-09-13 19:00:20 +00:00
Thephaseless
518cf1de42 adjustments 2024-09-13 18:48:31 +00:00
Thephaseless
35fce85fcf use python 3.12 2024-09-13 18:40:34 +00:00
Thephaseless
4dbb63842b test in dockerfile 2024-09-13 18:30:48 +00:00
Thephaseless
88e7321daa build if tests pass 2024-09-13 18:30:14 +00:00
Thephaseless
baae2fddeb use fastapi to test 2024-09-13 18:28:36 +00:00
11 changed files with 403 additions and 115 deletions

View file

@ -22,30 +22,6 @@ env:
IMAGE_NAME: ${{ github.repository }}
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: "3.12"
- name: Run image
uses: abatilo/actions-poetry@v2
- uses: actions/cache@v3
name: Define a cache for the virtual environment based on the dependencies lock file
with:
path: ./.venv
key: venv-${{ hashFiles('poetry.lock') }}
- name: Install the project dependencies
run: poetry install
- name: Run tests
run: poetry run pytest -v
build:
runs-on: ubuntu-latest
permissions:

View file

@ -1,35 +1,78 @@
FROM python:3.12
FROM python:3.12-slim-bullseye AS builder
# Build dummy packages to skip installing them and their dependencies
RUN apt-get update \
&& apt-get install -y --no-install-recommends equivs \
&& equivs-control libgl1-mesa-dri \
&& printf 'Section: misc\nPriority: optional\nStandards-Version: 3.9.2\nPackage: libgl1-mesa-dri\nVersion: 99.0.0\nDescription: Dummy package for libgl1-mesa-dri\n' >> libgl1-mesa-dri \
&& equivs-build libgl1-mesa-dri \
&& mv libgl1-mesa-dri_*.deb /libgl1-mesa-dri.deb \
&& equivs-control adwaita-icon-theme \
&& printf 'Section: misc\nPriority: optional\nStandards-Version: 3.9.2\nPackage: adwaita-icon-theme\nVersion: 99.0.0\nDescription: Dummy package for adwaita-icon-theme\n' >> adwaita-icon-theme \
&& equivs-build adwaita-icon-theme \
&& mv adwaita-icon-theme_*.deb /adwaita-icon-theme.deb
FROM python:3.12-slim-bullseye
# Copy dummy packages
COPY --from=builder /*.deb /
# Install dependencies and create byparr user
# You can test Chromium running this command inside the container:
# xvfb-run -s "-screen 0 1600x1200x24" chromium --no-sandbox
# The error traces is like this: "*** stack smashing detected ***: terminated"
# To check the package versions available you can use this command:
# apt-cache madison chromium
WORKDIR /app
# Install dummy packages
RUN dpkg -i /libgl1-mesa-dri.deb \
&& dpkg -i /adwaita-icon-theme.deb \
# Install dependencies
&& apt-get update \
&& apt-get install -y --no-install-recommends chromium xvfb dumb-init \
procps xauth sudo \
# Remove temporary files and hardware decoding libraries
&& rm -rf /var/lib/apt/lists/* \
&& rm -f /usr/lib/x86_64-linux-gnu/libmfxhw* \
&& rm -f /usr/lib/x86_64-linux-gnu/mfx/* \
# Create byparr user
&& useradd --home-dir /app --shell /bin/sh byparr \
&& chown -R byparr:byparr . \
&& echo 'byparr ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
# Install Python dependencies
COPY requirements.txt fix_nodriver.py novnc.sh .
RUN pip install -r requirements.txt \
# Remove temporary files
&& rm -rf /root/.cache \
&& PYENV_PATH=/usr/local/lib/python3.12 python fix_nodriver.py
ENV INSTALL_NOVNC=false DISPLAY=:1.0
RUN ./novnc.sh
USER byparr
COPY src .
EXPOSE 8191
# python
ENV \
DEBIAN_FRONTEND=noninteractive \
PYTHONUNBUFFERED=1 \
# prevents python creating .pyc files
PYTHONDONTWRITEBYTECODE=1 \
# do not ask any interactive question
POETRY_NO_INTERACTION=1 \
POETRY_VIRTUALENVS_IN_PROJECT=true
# dumb-init avoids zombie chromium processes
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
RUN apt update && apt upgrade -y
RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
RUN apt install -y --no-install-recommends --no-install-suggests ./google-chrome-stable_current_amd64.deb && rm ./google-chrome-stable_current_amd64.deb
CMD ["/bin/sh", "-c", "/usr/local/share/desktop-init.sh && python -u /app/main.py"]
RUN apt install pipx -y
RUN pipx ensurepath
RUN pipx install poetry
ENV PATH="/root/.local/bin:$PATH"
COPY pyproject.toml poetry.lock ./
RUN poetry install
# Local build
# docker build -t ngosang/byparr:3.3.21 .
# docker run -p 8191:8191 ngosang/byparr:3.3.21
ENV INSTALL_NOVNC=false
COPY novnc.sh .
RUN ./novnc.sh
ENV DISPLAY=:1.0
# Multi-arch build
# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
# docker buildx create --use
# docker buildx build -t ngosang/byparr:3.3.21 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
# add --push to publish in DockerHub
COPY . .
RUN . /app/.venv/bin/activate && python fix_nodriver.py
CMD /usr/local/share/desktop-init.sh && . /app/.venv/bin/activate && python main.py
# Test multi-arch build
# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
# docker buildx create --use
# docker buildx build -t ngosang/byparr:3.3.21 --platform linux/arm/v7 --load .
# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/byparr:3.3.21

View file

@ -6,10 +6,14 @@ import logging
import os
from pathlib import Path
pyenv_path = os.getenv("PYENV_PATH")
if pyenv_path is None:
env_path = os.getenv("VIRTUAL_ENV")
if env_path is None:
env_path = Path(os.__file__).parent.parent.parent.as_posix()
nodriver_path = Path(env_path + "/lib/python3.11/site-packages/nodriver/cdp/network.py")
pyenv_path = Path(env_path + "/lib/python3.12")
nodriver_pkg_path = Path(pyenv_path) / "site-packages/nodriver"
nodriver_path = nodriver_pkg_path / "cdp/network.py"
if not nodriver_path.exists():
msg = f"{nodriver_path} not found"
raise FileNotFoundError(msg)
@ -61,3 +65,19 @@ with nodriver_path.open("r+") as f:
with nodriver_path.open("w") as f:
f.writelines(lines)
browser_path = nodriver_pkg_path / "core/browser.py"
if not browser_path.exists():
msg = f"{browser_path} not found"
raise FileNotFoundError(msg)
with browser_path.open("r") as f:
browser_path_txt = f.read()
browser_path_txt2 = browser_path_txt.replace("for _ in range(5):", "for _ in range(20):")
if browser_path_txt2 != browser_path_txt:
logger.info("increasing browser open timeout")
with browser_path.open("w") as f:
f.write(browser_path_txt2)

168
poetry.lock generated
View file

@ -1,9 +1,10 @@
# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
# This file is automatically @generated by Poetry and should not be changed by hand.
[[package]]
name = "annotated-types"
version = "0.7.0"
description = "Reusable constraint types to use with typing.Annotated"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -15,6 +16,7 @@ files = [
name = "anyio"
version = "4.4.0"
description = "High level compatibility layer for multiple asynchronous event loop implementations"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -35,6 +37,7 @@ trio = ["trio (>=0.23)"]
name = "certifi"
version = "2024.8.30"
description = "Python package for providing Mozilla's CA Bundle."
category = "main"
optional = false
python-versions = ">=3.6"
files = [
@ -46,6 +49,7 @@ files = [
name = "charset-normalizer"
version = "3.3.2"
description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
category = "main"
optional = false
python-versions = ">=3.7.0"
files = [
@ -145,6 +149,7 @@ files = [
name = "click"
version = "8.1.7"
description = "Composable command line interface toolkit"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -159,6 +164,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""}
name = "colorama"
version = "0.4.6"
description = "Cross-platform colored terminal text."
category = "main"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
files = [
@ -170,6 +176,7 @@ files = [
name = "deprecated"
version = "1.2.14"
description = "Python @deprecated decorator to deprecate old python classes, functions or methods."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
files = [
@ -187,6 +194,7 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"]
name = "dnspython"
version = "2.6.1"
description = "DNS toolkit"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -207,6 +215,7 @@ wmi = ["wmi (>=1.5.1)"]
name = "email-validator"
version = "2.2.0"
description = "A robust email address syntax and deliverability validation library."
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -222,6 +231,7 @@ idna = ">=2.0.0"
name = "fastapi"
version = "0.111.1"
description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -247,6 +257,7 @@ all = ["email_validator (>=2.0.0)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)"
name = "fastapi-cli"
version = "0.0.5"
description = "Run and manage FastAPI apps from the command line with FastAPI CLI. 🚀"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -265,6 +276,7 @@ standard = ["uvicorn[standard] (>=0.15.0)"]
name = "h11"
version = "0.14.0"
description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -276,6 +288,7 @@ files = [
name = "httpcore"
version = "1.0.5"
description = "A minimal low-level HTTP client."
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -290,13 +303,14 @@ h11 = ">=0.13,<0.15"
[package.extras]
asyncio = ["anyio (>=4.0,<5.0)"]
http2 = ["h2 (>=3,<5)"]
socks = ["socksio (==1.*)"]
socks = ["socksio (>=1.0.0,<2.0.0)"]
trio = ["trio (>=0.22.0,<0.26.0)"]
[[package]]
name = "httptools"
version = "0.6.1"
description = "A collection of framework independent HTTP protocol utils."
category = "main"
optional = false
python-versions = ">=3.8.0"
files = [
@ -345,6 +359,7 @@ test = ["Cython (>=0.29.24,<0.30.0)"]
name = "httpx"
version = "0.27.2"
description = "The next generation HTTP client."
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -355,21 +370,22 @@ files = [
[package.dependencies]
anyio = "*"
certifi = "*"
httpcore = "==1.*"
httpcore = ">=1.0.0,<2.0.0"
idna = "*"
sniffio = "*"
[package.extras]
brotli = ["brotli", "brotlicffi"]
cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<14)"]
http2 = ["h2 (>=3,<5)"]
socks = ["socksio (==1.*)"]
socks = ["socksio (>=1.0.0,<2.0.0)"]
zstd = ["zstandard (>=0.18.0)"]
[[package]]
name = "idna"
version = "3.8"
description = "Internationalized Domain Names in Applications (IDNA)"
category = "main"
optional = false
python-versions = ">=3.6"
files = [
@ -381,6 +397,7 @@ files = [
name = "iniconfig"
version = "2.0.0"
description = "brain-dead simple config-ini parsing"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -392,6 +409,7 @@ files = [
name = "jinja2"
version = "3.1.4"
description = "A very fast and expressive template engine."
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -409,6 +427,7 @@ i18n = ["Babel (>=2.7)"]
name = "markdown-it-py"
version = "3.0.0"
description = "Python port of markdown-it. Markdown parsing, done right!"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -433,6 +452,7 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"]
name = "markupsafe"
version = "2.1.5"
description = "Safely add untrusted strings to HTML/XML markup."
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -502,6 +522,7 @@ files = [
name = "mdurl"
version = "0.1.2"
description = "Markdown URL utilities"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -513,6 +534,7 @@ files = [
name = "mss"
version = "9.0.2"
description = "An ultra fast cross-platform multiple screenshots module in pure python using ctypes."
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -524,10 +546,26 @@ files = [
dev = ["build (==1.2.1)", "mypy (==1.11.2)", "ruff (==0.6.3)", "twine (==5.1.1)", "wheel (==0.44.0)"]
test = ["numpy (==2.1.0)", "pillow (==10.4.0)", "pytest (==8.3.2)", "pytest-cov (==5.0.0)", "pytest-rerunfailures (==14.0.0)", "pyvirtualdisplay (==3.0)", "sphinx (==8.0.2)"]
[[package]]
name = "nn-mouse"
version = "1.0.1"
description = "Neural network based humanized mouse movements"
category = "main"
optional = false
python-versions = ">=3"
files = [
{file = "nn_mouse-1.0.1-py3-none-any.whl", hash = "sha256:726d548ba3796914c454507b4b91d8e2555e870392fc03f7f136ee0cf1db0150"},
{file = "nn_mouse-1.0.1.tar.gz", hash = "sha256:d92566959a339b99f47d4b403069f3afdc81d47d019836368d6dfd43fe345f0a"},
]
[package.dependencies]
opencv-python = "*"
[[package]]
name = "nodriver"
version = "0.34"
description = "[Docs here](https://ultrafunkamsterdam.github.io/nodriver)"
category = "main"
optional = false
python-versions = ">=3.9"
files = [
@ -543,10 +581,102 @@ websockets = ">=11"
[package.extras]
dev = ["black", "build", "furo", "pygments", "sphinx", "sphinx-autodoc-typehints", "sphinx-markdown-builder"]
[[package]]
name = "numpy"
version = "2.1.1"
description = "Fundamental package for array computing in Python"
category = "main"
optional = false
python-versions = ">=3.10"
files = [
{file = "numpy-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c8a0e34993b510fc19b9a2ce7f31cb8e94ecf6e924a40c0c9dd4f62d0aac47d9"},
{file = "numpy-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7dd86dfaf7c900c0bbdcb8b16e2f6ddf1eb1fe39c6c8cca6e94844ed3152a8fd"},
{file = "numpy-2.1.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:5889dd24f03ca5a5b1e8a90a33b5a0846d8977565e4ae003a63d22ecddf6782f"},
{file = "numpy-2.1.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:59ca673ad11d4b84ceb385290ed0ebe60266e356641428c845b39cd9df6713ab"},
{file = "numpy-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13ce49a34c44b6de5241f0b38b07e44c1b2dcacd9e36c30f9c2fcb1bb5135db7"},
{file = "numpy-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:913cc1d311060b1d409e609947fa1b9753701dac96e6581b58afc36b7ee35af6"},
{file = "numpy-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:caf5d284ddea7462c32b8d4a6b8af030b6c9fd5332afb70e7414d7fdded4bfd0"},
{file = "numpy-2.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:57eb525e7c2a8fdee02d731f647146ff54ea8c973364f3b850069ffb42799647"},
{file = "numpy-2.1.1-cp310-cp310-win32.whl", hash = "sha256:9a8e06c7a980869ea67bbf551283bbed2856915f0a792dc32dd0f9dd2fb56728"},
{file = "numpy-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:d10c39947a2d351d6d466b4ae83dad4c37cd6c3cdd6d5d0fa797da56f710a6ae"},
{file = "numpy-2.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0d07841fd284718feffe7dd17a63a2e6c78679b2d386d3e82f44f0108c905550"},
{file = "numpy-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b5613cfeb1adfe791e8e681128f5f49f22f3fcaa942255a6124d58ca59d9528f"},
{file = "numpy-2.1.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:0b8cc2715a84b7c3b161f9ebbd942740aaed913584cae9cdc7f8ad5ad41943d0"},
{file = "numpy-2.1.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:b49742cdb85f1f81e4dc1b39dcf328244f4d8d1ded95dea725b316bd2cf18c95"},
{file = "numpy-2.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8d5f8a8e3bc87334f025194c6193e408903d21ebaeb10952264943a985066ca"},
{file = "numpy-2.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d51fc141ddbe3f919e91a096ec739f49d686df8af254b2053ba21a910ae518bf"},
{file = "numpy-2.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:98ce7fb5b8063cfdd86596b9c762bf2b5e35a2cdd7e967494ab78a1fa7f8b86e"},
{file = "numpy-2.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:24c2ad697bd8593887b019817ddd9974a7f429c14a5469d7fad413f28340a6d2"},
{file = "numpy-2.1.1-cp311-cp311-win32.whl", hash = "sha256:397bc5ce62d3fb73f304bec332171535c187e0643e176a6e9421a6e3eacef06d"},
{file = "numpy-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:ae8ce252404cdd4de56dcfce8b11eac3c594a9c16c231d081fb705cf23bd4d9e"},
{file = "numpy-2.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7c803b7934a7f59563db459292e6aa078bb38b7ab1446ca38dd138646a38203e"},
{file = "numpy-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6435c48250c12f001920f0751fe50c0348f5f240852cfddc5e2f97e007544cbe"},
{file = "numpy-2.1.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3269c9eb8745e8d975980b3a7411a98976824e1fdef11f0aacf76147f662b15f"},
{file = "numpy-2.1.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:fac6e277a41163d27dfab5f4ec1f7a83fac94e170665a4a50191b545721c6521"},
{file = "numpy-2.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fcd8f556cdc8cfe35e70efb92463082b7f43dd7e547eb071ffc36abc0ca4699b"},
{file = "numpy-2.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b9cd92c8f8e7b313b80e93cedc12c0112088541dcedd9197b5dee3738c1201"},
{file = "numpy-2.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:afd9c680df4de71cd58582b51e88a61feed4abcc7530bcd3d48483f20fc76f2a"},
{file = "numpy-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8661c94e3aad18e1ea17a11f60f843a4933ccaf1a25a7c6a9182af70610b2313"},
{file = "numpy-2.1.1-cp312-cp312-win32.whl", hash = "sha256:950802d17a33c07cba7fd7c3dcfa7d64705509206be1606f196d179e539111ed"},
{file = "numpy-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:3fc5eabfc720db95d68e6646e88f8b399bfedd235994016351b1d9e062c4b270"},
{file = "numpy-2.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:046356b19d7ad1890c751b99acad5e82dc4a02232013bd9a9a712fddf8eb60f5"},
{file = "numpy-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6e5a9cb2be39350ae6c8f79410744e80154df658d5bea06e06e0ac5bb75480d5"},
{file = "numpy-2.1.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:d4c57b68c8ef5e1ebf47238e99bf27657511ec3f071c465f6b1bccbef12d4136"},
{file = "numpy-2.1.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:8ae0fd135e0b157365ac7cc31fff27f07a5572bdfc38f9c2d43b2aff416cc8b0"},
{file = "numpy-2.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:981707f6b31b59c0c24bcda52e5605f9701cb46da4b86c2e8023656ad3e833cb"},
{file = "numpy-2.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ca4b53e1e0b279142113b8c5eb7d7a877e967c306edc34f3b58e9be12fda8df"},
{file = "numpy-2.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e097507396c0be4e547ff15b13dc3866f45f3680f789c1a1301b07dadd3fbc78"},
{file = "numpy-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7506387e191fe8cdb267f912469a3cccc538ab108471291636a96a54e599556"},
{file = "numpy-2.1.1-cp313-cp313-win32.whl", hash = "sha256:251105b7c42abe40e3a689881e1793370cc9724ad50d64b30b358bbb3a97553b"},
{file = "numpy-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:f212d4f46b67ff604d11fff7cc62d36b3e8714edf68e44e9760e19be38c03eb0"},
{file = "numpy-2.1.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:920b0911bb2e4414c50e55bd658baeb78281a47feeb064ab40c2b66ecba85553"},
{file = "numpy-2.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bab7c09454460a487e631ffc0c42057e3d8f2a9ddccd1e60c7bb8ed774992480"},
{file = "numpy-2.1.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:cea427d1350f3fd0d2818ce7350095c1a2ee33e30961d2f0fef48576ddbbe90f"},
{file = "numpy-2.1.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:e30356d530528a42eeba51420ae8bf6c6c09559051887196599d96ee5f536468"},
{file = "numpy-2.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8dfa9e94fc127c40979c3eacbae1e61fda4fe71d84869cc129e2721973231ef"},
{file = "numpy-2.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:910b47a6d0635ec1bd53b88f86120a52bf56dcc27b51f18c7b4a2e2224c29f0f"},
{file = "numpy-2.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:13cc11c00000848702322af4de0147ced365c81d66053a67c2e962a485b3717c"},
{file = "numpy-2.1.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:53e27293b3a2b661c03f79aa51c3987492bd4641ef933e366e0f9f6c9bf257ec"},
{file = "numpy-2.1.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7be6a07520b88214ea85d8ac8b7d6d8a1839b0b5cb87412ac9f49fa934eb15d5"},
{file = "numpy-2.1.1-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:52ac2e48f5ad847cd43c4755520a2317f3380213493b9d8a4c5e37f3b87df504"},
{file = "numpy-2.1.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50a95ca3560a6058d6ea91d4629a83a897ee27c00630aed9d933dff191f170cd"},
{file = "numpy-2.1.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:99f4a9ee60eed1385a86e82288971a51e71df052ed0b2900ed30bc840c0f2e39"},
{file = "numpy-2.1.1.tar.gz", hash = "sha256:d0cf7d55b1051387807405b3898efafa862997b4cba8aa5dbe657be794afeafd"},
]
[[package]]
name = "opencv-python"
version = "4.10.0.84"
description = "Wrapper package for OpenCV python bindings."
category = "main"
optional = false
python-versions = ">=3.6"
files = [
{file = "opencv-python-4.10.0.84.tar.gz", hash = "sha256:72d234e4582e9658ffea8e9cae5b63d488ad06994ef12d81dc303b17472f3526"},
{file = "opencv_python-4.10.0.84-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fc182f8f4cda51b45f01c64e4cbedfc2f00aff799debebc305d8d0210c43f251"},
{file = "opencv_python-4.10.0.84-cp37-abi3-macosx_12_0_x86_64.whl", hash = "sha256:71e575744f1d23f79741450254660442785f45a0797212852ee5199ef12eed98"},
{file = "opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09a332b50488e2dda866a6c5573ee192fe3583239fb26ff2f7f9ceb0bc119ea6"},
{file = "opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ace140fc6d647fbe1c692bcb2abce768973491222c067c131d80957c595b71f"},
{file = "opencv_python-4.10.0.84-cp37-abi3-win32.whl", hash = "sha256:2db02bb7e50b703f0a2d50c50ced72e95c574e1e5a0bb35a8a86d0b35c98c236"},
{file = "opencv_python-4.10.0.84-cp37-abi3-win_amd64.whl", hash = "sha256:32dbbd94c26f611dc5cc6979e6b7aa1f55a64d6b463cc1dcd3c95505a63e48fe"},
]
[package.dependencies]
numpy = [
{version = ">=1.21.2", markers = "python_version >= \"3.10\""},
{version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""},
{version = ">=1.23.5", markers = "python_version >= \"3.11\""},
{version = ">=1.26.0", markers = "python_version >= \"3.12\""},
{version = ">=1.19.3", markers = "python_version >= \"3.6\" and platform_system == \"Linux\" and platform_machine == \"aarch64\" or python_version >= \"3.9\""},
{version = ">=1.17.0", markers = "python_version >= \"3.7\""},
{version = ">=1.17.3", markers = "python_version >= \"3.8\""},
]
[[package]]
name = "packaging"
version = "24.1"
description = "Core utilities for Python packages"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -558,6 +688,7 @@ files = [
name = "pluggy"
version = "1.5.0"
description = "plugin and hook calling mechanisms for python"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -573,6 +704,7 @@ testing = ["pytest", "pytest-benchmark"]
name = "pydantic"
version = "2.9.1"
description = "Data validation using Python type hints"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -596,6 +728,7 @@ timezone = ["tzdata"]
name = "pydantic-core"
version = "2.23.3"
description = "Core functionality for Pydantic validation and serialization"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -697,6 +830,7 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
name = "pygments"
version = "2.18.0"
description = "Pygments is a syntax highlighting package written in Python."
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -711,6 +845,7 @@ windows-terminal = ["colorama (>=0.4.6)"]
name = "pytest"
version = "8.3.3"
description = "pytest: simple powerful testing with Python"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -731,6 +866,7 @@ dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments
name = "pytest-asyncio"
version = "0.24.0"
description = "Pytest support for asyncio"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -749,6 +885,7 @@ testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"]
name = "python-dotenv"
version = "1.0.1"
description = "Read key-value pairs from a .env file and set them as environment variables"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -763,6 +900,7 @@ cli = ["click (>=5.0)"]
name = "python-multipart"
version = "0.0.9"
description = "A streaming multipart parser for Python"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -777,6 +915,7 @@ dev = ["atomicwrites (==1.4.1)", "attrs (==23.2.0)", "coverage (==7.4.1)", "hatc
name = "pyyaml"
version = "6.0.2"
description = "YAML parser and emitter for Python"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -839,6 +978,7 @@ files = [
name = "requests"
version = "2.32.3"
description = "Python HTTP for Humans."
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -860,6 +1000,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
name = "rich"
version = "13.8.1"
description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
category = "main"
optional = false
python-versions = ">=3.7.0"
files = [
@ -878,6 +1019,7 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"]
name = "shellingham"
version = "1.5.4"
description = "Tool to Detect Surrounding Shell"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -889,6 +1031,7 @@ files = [
name = "sniffio"
version = "1.3.1"
description = "Sniff out which async library your code is running under"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -900,6 +1043,7 @@ files = [
name = "starlette"
version = "0.37.2"
description = "The little ASGI library that shines."
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -917,6 +1061,7 @@ full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.7
name = "typer"
version = "0.12.5"
description = "Typer, build great CLIs. Easy to code. Based on Python type hints."
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -934,6 +1079,7 @@ typing-extensions = ">=3.7.4.3"
name = "typing-extensions"
version = "4.12.2"
description = "Backported and Experimental Type Hints for Python 3.8+"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -945,6 +1091,7 @@ files = [
name = "urllib3"
version = "2.2.3"
description = "HTTP library with thread-safe connection pooling, file post, and more."
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -962,6 +1109,7 @@ zstd = ["zstandard (>=0.18.0)"]
name = "uvicorn"
version = "0.30.6"
description = "The lightning-fast ASGI server."
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -976,7 +1124,7 @@ h11 = ">=0.8"
httptools = {version = ">=0.5.0", optional = true, markers = "extra == \"standard\""}
python-dotenv = {version = ">=0.13", optional = true, markers = "extra == \"standard\""}
pyyaml = {version = ">=5.1", optional = true, markers = "extra == \"standard\""}
uvloop = {version = ">=0.14.0,<0.15.0 || >0.15.0,<0.15.1 || >0.15.1", optional = true, markers = "(sys_platform != \"win32\" and sys_platform != \"cygwin\") and platform_python_implementation != \"PyPy\" and extra == \"standard\""}
uvloop = {version = ">=0.14.0,<0.15.0 || >0.15.0,<0.15.1 || >0.15.1", optional = true, markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" and extra == \"standard\""}
watchfiles = {version = ">=0.13", optional = true, markers = "extra == \"standard\""}
websockets = {version = ">=10.4", optional = true, markers = "extra == \"standard\""}
@ -987,6 +1135,7 @@ standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)",
name = "uvloop"
version = "0.20.0"
description = "Fast implementation of asyncio event loop on top of libuv"
category = "main"
optional = false
python-versions = ">=3.8.0"
files = [
@ -1031,6 +1180,7 @@ test = ["Cython (>=0.29.36,<0.30.0)", "aiohttp (==3.9.0b0)", "aiohttp (>=3.8.1)"
name = "watchfiles"
version = "0.24.0"
description = "Simple, modern and high performance file watching and code reload in python."
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -1126,6 +1276,7 @@ anyio = ">=3.0.0"
name = "websockets"
version = "13.0.1"
description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -1221,6 +1372,7 @@ files = [
name = "wrapt"
version = "1.16.0"
description = "Module for decorators, wrappers and monkey patching."
category = "main"
optional = false
python-versions = ">=3.6"
files = [
@ -1298,5 +1450,5 @@ files = [
[metadata]
lock-version = "2.0"
python-versions = "^3.11"
content-hash = "8e6c46b679a81d033850227859c9754ca5e58b3b10d1e13383bf0576bb34da5c"
python-versions = "^3.12"
content-hash = "36519f0ca589faeaac9710ec3f1b055905d9560673ec53611abacbe259990b53"

View file

@ -2,18 +2,19 @@
name = "byparr"
version = "0.1.0"
description = ""
package-mode = false
# package-mode = false
authors = ["Your Name <you@example.com>"]
readme = "README.md"
[tool.poetry.dependencies]
python = "^3.11"
python = "^3.12"
pytest = "^8.3.1"
fastapi = "^0.111.1"
nodriver = "^0.34"
requests = "^2.32.3"
httpx = "^0.27.2"
pytest-asyncio = "^0.24.0"
nn-mouse = "^1.0.1"
[build-system]

5
requirements.txt Normal file
View file

@ -0,0 +1,5 @@
fastapi==0.111.1
nodriver==0.34
requests==2.32.3
httpx==0.27.2
nn-mouse==1.0.1

View file

@ -9,10 +9,10 @@ import uvicorn.config
from fastapi import FastAPI
from fastapi.responses import RedirectResponse
from src.models.requests import LinkRequest, LinkResponse
from src.utils import logger
from src.utils.browser import bypass_cloudflare, new_browser
from src.utils.consts import LOG_LEVEL
from models.requests import LinkRequest, LinkResponse
from utils import logger
from utils.browser import bypass_cloudflare, new_browser
from utils.consts import LOG_LEVEL
app = FastAPI(debug=LOG_LEVEL == logging.DEBUG, log_level=LOG_LEVEL)
@ -31,13 +31,13 @@ async def read_item(request: LinkRequest):
logger.info(f"Request: {request}")
start_time = int(time.time() * 1000)
browser = await new_browser()
try:
page = await browser.get(request.url)
await page.bring_to_front()
timeout = request.maxTimeout
if timeout == 0:
timeout = None
challenged = await asyncio.wait_for(
bypass_cloudflare(page), timeout=request.maxTimeout
)
challenged = await asyncio.wait_for(bypass_cloudflare(page), timeout=timeout)
logger.info(f"Got webpage: {request.url}")
@ -46,9 +46,7 @@ async def read_item(request: LinkRequest):
start_timestamp=start_time,
challenged=challenged,
)
except asyncio.TimeoutError:
logger.fatal("Couldn't complete the request")
finally:
browser.stop()
return response

View file

@ -1,6 +1,6 @@
import logging
from src.utils.consts import LOG_LEVEL
from utils.consts import LOG_LEVEL
logger = logging.getLogger("uvicorn.error")
logger.setLevel(LOG_LEVEL)

View file

@ -1,13 +1,13 @@
import asyncio
import typing
import random
import nodriver as webdriver
from nodriver.core.element import Element
from nodriver.core.element import Element, Position
from nn_mouse import get_path
from src.utils import logger
from src.utils.consts import CHALLENGE_TITLES
from src.utils.extentions import download_extentions
downloaded_extentions = download_extentions()
from . import logger
from .consts import CHALLENGE_TITLES
async def new_browser():
@ -25,7 +25,6 @@ async def new_browser():
"""
config: webdriver.Config = webdriver.Config()
config.sandbox = False
config.add_argument(f"--load-extension={','.join(downloaded_extentions)}")
return await webdriver.start(config=config)
@ -65,6 +64,12 @@ async def bypass_cloudflare(page: webdriver.Tab):
if not challenged:
logger.info("Found challenge")
challenged = True
elem = None
# get the size of the page
html_elem = await page.select("html")
html_pos = await html_elem.get_position()
try:
elem = await page.find(
"Verify you are human by completing the action below.",
@ -74,31 +79,30 @@ async def bypass_cloudflare(page: webdriver.Tab):
except asyncio.TimeoutError:
if page.target.title not in CHALLENGE_TITLES:
return challenged
raise
if elem is None:
logger.debug("Couldn't find the title, trying again")
logger.debug("Couldn't find the title, trying other method...")
continue
if not isinstance(elem, Element):
logger.fatal("Element is a string, please report this to Byparr dev")
raise InvalidElementError
elem = await page.find("input")
elem = elem.parent
# Get the element containing the shadow root
for _ in range(3):
if elem is not None:
elem = get_first_div(elem)
else:
raise InvalidElementError
if isinstance(elem, Element) and elem.shadow_roots:
inner_elem = Element(elem.shadow_roots[0], page, elem.tree).children[0]
if isinstance(inner_elem, Element):
logger.debug("Clicking element")
await inner_elem.mouse_click()
await mouse_click(inner_elem, html_pos=html_pos)
else:
logger.warn(
logger.warning(
"Element is a string, please report this to Byparr dev"
) # I really hope this never happens
else:
logger.warn("Coulnd't find checkbox, trying again...")
logger.warning("Coulnd't find checkbox, trying again...")
def get_first_div(elem):
@ -120,5 +124,85 @@ def get_first_div(elem):
raise InvalidElementError
def clamp(n: int, smallest: int, largest: int):
return max(smallest, min(n, largest))
async def mouse_click(
elm,
button: str = "left",
buttons: typing.Optional[int] = 1,
modifiers: typing.Optional[int] = 0,
hold: bool = False,
html_pos: typing.Optional[Position] = None,
_until_event: typing.Optional[type] = None,
):
"""native click (on element) . note: this likely does not work atm, use click() instead
:param button: str (default = "left")
:param buttons: which button (default 1 = left)
:param modifiers: *(Optional)* Bit field representing pressed modifier keys.
Alt=1, Ctrl=2, Meta/Command=4, Shift=8 (default: 0).
:param _until_event: internal. event to wait for before returning
:return:
"""
try:
pos = await elm.get_position()
except AttributeError:
return
if not pos:
logger.warning("could not calculate box model for %s", self)
return
oc = pos.center
center = (oc[0] + random.randint(-10, 10), oc[1] + random.randint(-10, 10))
html_height = 500
html_width = 500
if html_pos:
html_height = html_pos.height
html_width = html_pos.width
start_x = random.randint(0, html_width)
start_y = random.randint(0, html_height)
path = get_path(start_x, start_y, center[0], center[1], html_width, html_height)
for (x, y, t) in path:
x = clamp(int(x), 0, html_width)
y = clamp(int(y), 0, html_height)
await elm._tab.send(webdriver.cdp.input_.dispatch_mouse_event("mousemove", x, y))
await asyncio.sleep(float(t))
logger.debug("clicking on location %.2f, %.2f" % center)
await asyncio.sleep(random.uniform(0.05, 0.3))
await elm._tab.send(
webdriver.cdp.input_.dispatch_mouse_event(
"mousePressed",
x=center[0],
y=center[1],
modifiers=modifiers,
button=webdriver.cdp.input_.MouseButton(button),
buttons=buttons,
click_count=1,
)
)
await asyncio.sleep(random.uniform(0.02, 0.06))
await elm._tab.send(
webdriver.cdp.input_.dispatch_mouse_event(
"mouseReleased",
x=center[0],
y=center[1],
modifiers=modifiers,
button=webdriver.cdp.input_.MouseButton(button),
buttons=buttons,
click_count=1,
)
)
# Flash the checkbox for testing
try:
await elm.flash()
except: # noqa
pass
class InvalidElementError(Exception):
pass

View file

@ -8,10 +8,10 @@ from zipfile import ZipFile
import httpx
import requests
from src.models.github import GithubResponse
from src.models.requests import NoChromeExtentionError
from src.utils import logger
from src.utils.consts import EXTENTION_REPOSITIORIES, EXTENTIONS_PATH, GITHUB_WEBSITES
from ..models.github import GithubResponse
from ..models.requests import NoChromeExtentionError
from . import logger
from .consts import EXTENTION_REPOSITIORIES, EXTENTIONS_PATH, GITHUB_WEBSITES
def get_latest_github_chrome_release(url: str):

View file

@ -1,18 +1,27 @@
import pytest
from http import HTTPStatus
from main import read_item
import pytest
from starlette.testclient import TestClient
from main import app
from src.models.requests import LinkRequest
client = TestClient(app)
test_websites = [
"https://ext.to/",
"https://btmet.com/",
"https://extratorrent.st/",
"https://idope.se/",
# "https://extratorrent.st/", # github is blocking these
# "https://idope.se/", # github is blocking these
]
pytest_plugins = ("pytest_asyncio",)
@pytest.mark.parametrize("website", test_websites)
@pytest.mark.asyncio
async def test_bypass(website: str):
await read_item(LinkRequest(url=website, maxTimeout=5, cmd=""))
def test_bypass(website: str):
response = client.post(
"/v1",
json=LinkRequest(
url=website, maxTimeout=60 * len(test_websites), cmd="request.get"
).model_dump(),
)
assert response.status_code == HTTPStatus.OK