various fixes to make it working

.github/workflows/docker-publish.yml

@@ -22,30 +22,6 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  test:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.12"
-
-      - name: Run image
-        uses: abatilo/actions-poetry@v2
-
-      - uses: actions/cache@v3
-        name: Define a cache for the virtual environment based on the dependencies lock file
-        with:
-          path: ./.venv
-          key: venv-${{ hashFiles('poetry.lock') }}
-
-      - name: Install the project dependencies
-        run: poetry install
-
-      - name: Run tests
-        run: poetry run pytest -v
-
   build:
     runs-on: ubuntu-latest
     permissions:

Dockerfile

@@ -1,35 +1,78 @@
-FROM python:3.12
+FROM python:3.12-slim-bullseye AS builder
 
+# Build dummy packages to skip installing them and their dependencies
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends equivs \
+    && equivs-control libgl1-mesa-dri \
+    && printf 'Section: misc\nPriority: optional\nStandards-Version: 3.9.2\nPackage: libgl1-mesa-dri\nVersion: 99.0.0\nDescription: Dummy package for libgl1-mesa-dri\n' >> libgl1-mesa-dri \
+    && equivs-build libgl1-mesa-dri \
+    && mv libgl1-mesa-dri_*.deb /libgl1-mesa-dri.deb \
+    && equivs-control adwaita-icon-theme \
+    && printf 'Section: misc\nPriority: optional\nStandards-Version: 3.9.2\nPackage: adwaita-icon-theme\nVersion: 99.0.0\nDescription: Dummy package for adwaita-icon-theme\n' >> adwaita-icon-theme \
+    && equivs-build adwaita-icon-theme \
+    && mv adwaita-icon-theme_*.deb /adwaita-icon-theme.deb
+
+FROM python:3.12-slim-bullseye
+
+# Copy dummy packages
+COPY --from=builder /*.deb /
+
+# Install dependencies and create byparr user
+# You can test Chromium running this command inside the container:
+#    xvfb-run -s "-screen 0 1600x1200x24" chromium --no-sandbox
+# The error traces is like this: "*** stack smashing detected ***: terminated"
+# To check the package versions available you can use this command:
+#    apt-cache madison chromium
 WORKDIR /app
+    # Install dummy packages
+RUN dpkg -i /libgl1-mesa-dri.deb \
+    && dpkg -i /adwaita-icon-theme.deb \
+    # Install dependencies
+    && apt-get update \
+    && apt-get install -y --no-install-recommends chromium xvfb dumb-init \
+        procps xauth sudo \
+    # Remove temporary files and hardware decoding libraries
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -f /usr/lib/x86_64-linux-gnu/libmfxhw* \
+    && rm -f /usr/lib/x86_64-linux-gnu/mfx/* \
+    # Create byparr user
+    && useradd --home-dir /app --shell /bin/sh byparr \
+    && chown -R byparr:byparr . \
+    && echo 'byparr ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+# Install Python dependencies
+COPY requirements.txt fix_nodriver.py novnc.sh .
+RUN pip install -r requirements.txt \
+    # Remove temporary files
+    && rm -rf /root/.cache \
+    && PYENV_PATH=/usr/local/lib/python3.12 python fix_nodriver.py
+
+ENV INSTALL_NOVNC=false DISPLAY=:1.0
+RUN ./novnc.sh
+
+USER byparr
+
+COPY src .
+
 EXPOSE 8191
 
-# python
+# dumb-init avoids zombie chromium processes
-ENV \
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-    DEBIAN_FRONTEND=noninteractive \
-    PYTHONUNBUFFERED=1 \
-    # prevents python creating .pyc files
-    PYTHONDONTWRITEBYTECODE=1 \
-    # do not ask any interactive question
-    POETRY_NO_INTERACTION=1 \
-    POETRY_VIRTUALENVS_IN_PROJECT=true
 
-RUN apt update && apt upgrade -y
+CMD ["/bin/sh", "-c", "/usr/local/share/desktop-init.sh && python -u /app/main.py"]
-RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
-RUN apt install -y --no-install-recommends --no-install-suggests ./google-chrome-stable_current_amd64.deb && rm ./google-chrome-stable_current_amd64.deb
 
-RUN apt install pipx -y
+# Local build
-RUN pipx ensurepath
+# docker build -t ngosang/byparr:3.3.21 .
-RUN pipx install poetry
+# docker run -p 8191:8191 ngosang/byparr:3.3.21
-ENV PATH="/root/.local/bin:$PATH"
-COPY pyproject.toml poetry.lock ./
-RUN poetry install
 
-ENV INSTALL_NOVNC=false
+# Multi-arch build
-COPY novnc.sh .
+# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-RUN ./novnc.sh
+# docker buildx create --use
-ENV DISPLAY=:1.0
+# docker buildx build -t ngosang/byparr:3.3.21 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
+#   add --push to publish in DockerHub
 
-COPY . .
+# Test multi-arch build
-RUN . /app/.venv/bin/activate && python fix_nodriver.py
+# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-
+# docker buildx create --use
-CMD /usr/local/share/desktop-init.sh && . /app/.venv/bin/activate && python main.py
+# docker buildx build -t ngosang/byparr:3.3.21 --platform linux/arm/v7 --load .
+# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/byparr:3.3.21

fix_nodriver.py

@@ -6,10 +6,14 @@ import logging
 import os
 from pathlib import Path
 
-env_path = os.getenv("VIRTUAL_ENV")
+pyenv_path = os.getenv("PYENV_PATH")
-if env_path is None:
+if pyenv_path is None:
-    env_path = Path(os.__file__).parent.parent.parent.as_posix()
+    env_path = os.getenv("VIRTUAL_ENV")
-nodriver_path = Path(env_path + "/lib/python3.11/site-packages/nodriver/cdp/network.py")
+    if env_path is None:
+        env_path = Path(os.__file__).parent.parent.parent.as_posix()
+    pyenv_path = Path(env_path + "/lib/python3.12")
+nodriver_pkg_path = Path(pyenv_path) / "site-packages/nodriver"
+nodriver_path = nodriver_pkg_path / "cdp/network.py"
 if not nodriver_path.exists():
     msg = f"{nodriver_path} not found"
     raise FileNotFoundError(msg)
@@ -61,3 +65,19 @@ with nodriver_path.open("r+") as f:
 
 with nodriver_path.open("w") as f:
     f.writelines(lines)
+
+
+browser_path = nodriver_pkg_path / "core/browser.py"
+if not browser_path.exists():
+    msg = f"{browser_path} not found"
+    raise FileNotFoundError(msg)
+
+with browser_path.open("r") as f:
+    browser_path_txt = f.read()
+
+browser_path_txt2 = browser_path_txt.replace("for _ in range(5):", "for _ in range(20):")
+
+if browser_path_txt2 != browser_path_txt:
+    logger.info("increasing browser open timeout")
+    with browser_path.open("w") as f:
+        f.write(browser_path_txt2)

poetry.lock

@@ -1,9 +1,10 @@
-# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
+# This file is automatically @generated by Poetry and should not be changed by hand.
 
 [[package]]
 name = "annotated-types"
 version = "0.7.0"
 description = "Reusable constraint types to use with typing.Annotated"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -15,6 +16,7 @@ files = [
 name = "anyio"
 version = "4.4.0"
 description = "High level compatibility layer for multiple asynchronous event loop implementations"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -35,6 +37,7 @@ trio = ["trio (>=0.23)"]
 name = "certifi"
 version = "2024.8.30"
 description = "Python package for providing Mozilla's CA Bundle."
+category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -46,6 +49,7 @@ files = [
 name = "charset-normalizer"
 version = "3.3.2"
 description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
+category = "main"
 optional = false
 python-versions = ">=3.7.0"
 files = [
@@ -145,6 +149,7 @@ files = [
 name = "click"
 version = "8.1.7"
 description = "Composable command line interface toolkit"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -159,6 +164,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""}
 name = "colorama"
 version = "0.4.6"
 description = "Cross-platform colored terminal text."
+category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
 files = [
@@ -170,6 +176,7 @@ files = [
 name = "deprecated"
 version = "1.2.14"
 description = "Python @deprecated decorator to deprecate old python classes, functions or methods."
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -187,6 +194,7 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"]
 name = "dnspython"
 version = "2.6.1"
 description = "DNS toolkit"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -207,6 +215,7 @@ wmi = ["wmi (>=1.5.1)"]
 name = "email-validator"
 version = "2.2.0"
 description = "A robust email address syntax and deliverability validation library."
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -222,6 +231,7 @@ idna = ">=2.0.0"
 name = "fastapi"
 version = "0.111.1"
 description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -247,6 +257,7 @@ all = ["email_validator (>=2.0.0)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)"
 name = "fastapi-cli"
 version = "0.0.5"
 description = "Run and manage FastAPI apps from the command line with FastAPI CLI. 🚀"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -265,6 +276,7 @@ standard = ["uvicorn[standard] (>=0.15.0)"]
 name = "h11"
 version = "0.14.0"
 description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -276,6 +288,7 @@ files = [
 name = "httpcore"
 version = "1.0.5"
 description = "A minimal low-level HTTP client."
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -290,13 +303,14 @@ h11 = ">=0.13,<0.15"
 [package.extras]
 asyncio = ["anyio (>=4.0,<5.0)"]
 http2 = ["h2 (>=3,<5)"]
-socks = ["socksio (==1.*)"]
+socks = ["socksio (>=1.0.0,<2.0.0)"]
 trio = ["trio (>=0.22.0,<0.26.0)"]
 
 [[package]]
 name = "httptools"
 version = "0.6.1"
 description = "A collection of framework independent HTTP protocol utils."
+category = "main"
 optional = false
 python-versions = ">=3.8.0"
 files = [
@@ -345,6 +359,7 @@ test = ["Cython (>=0.29.24,<0.30.0)"]
 name = "httpx"
 version = "0.27.2"
 description = "The next generation HTTP client."
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -355,21 +370,22 @@ files = [
 [package.dependencies]
 anyio = "*"
 certifi = "*"
-httpcore = "==1.*"
+httpcore = ">=1.0.0,<2.0.0"
 idna = "*"
 sniffio = "*"
 
 [package.extras]
 brotli = ["brotli", "brotlicffi"]
-cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
+cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<14)"]
 http2 = ["h2 (>=3,<5)"]
-socks = ["socksio (==1.*)"]
+socks = ["socksio (>=1.0.0,<2.0.0)"]
 zstd = ["zstandard (>=0.18.0)"]
 
 [[package]]
 name = "idna"
 version = "3.8"
 description = "Internationalized Domain Names in Applications (IDNA)"
+category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -381,6 +397,7 @@ files = [
 name = "iniconfig"
 version = "2.0.0"
 description = "brain-dead simple config-ini parsing"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -392,6 +409,7 @@ files = [
 name = "jinja2"
 version = "3.1.4"
 description = "A very fast and expressive template engine."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -409,6 +427,7 @@ i18n = ["Babel (>=2.7)"]
 name = "markdown-it-py"
 version = "3.0.0"
 description = "Python port of markdown-it. Markdown parsing, done right!"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -433,6 +452,7 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"]
 name = "markupsafe"
 version = "2.1.5"
 description = "Safely add untrusted strings to HTML/XML markup."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -502,6 +522,7 @@ files = [
 name = "mdurl"
 version = "0.1.2"
 description = "Markdown URL utilities"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -513,6 +534,7 @@ files = [
 name = "mss"
 version = "9.0.2"
 description = "An ultra fast cross-platform multiple screenshots module in pure python using ctypes."
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -524,10 +546,26 @@ files = [
 dev = ["build (==1.2.1)", "mypy (==1.11.2)", "ruff (==0.6.3)", "twine (==5.1.1)", "wheel (==0.44.0)"]
 test = ["numpy (==2.1.0)", "pillow (==10.4.0)", "pytest (==8.3.2)", "pytest-cov (==5.0.0)", "pytest-rerunfailures (==14.0.0)", "pyvirtualdisplay (==3.0)", "sphinx (==8.0.2)"]
 
+[[package]]
+name = "nn-mouse"
+version = "1.0.1"
+description = "Neural network based humanized mouse movements"
+category = "main"
+optional = false
+python-versions = ">=3"
+files = [
+    {file = "nn_mouse-1.0.1-py3-none-any.whl", hash = "sha256:726d548ba3796914c454507b4b91d8e2555e870392fc03f7f136ee0cf1db0150"},
+    {file = "nn_mouse-1.0.1.tar.gz", hash = "sha256:d92566959a339b99f47d4b403069f3afdc81d47d019836368d6dfd43fe345f0a"},
+]
+
+[package.dependencies]
+opencv-python = "*"
+
 [[package]]
 name = "nodriver"
 version = "0.34"
 description = "[Docs here](https://ultrafunkamsterdam.github.io/nodriver)"
+category = "main"
 optional = false
 python-versions = ">=3.9"
 files = [
@@ -543,10 +581,102 @@ websockets = ">=11"
 [package.extras]
 dev = ["black", "build", "furo", "pygments", "sphinx", "sphinx-autodoc-typehints", "sphinx-markdown-builder"]
 
+[[package]]
+name = "numpy"
+version = "2.1.1"
+description = "Fundamental package for array computing in Python"
+category = "main"
+optional = false
+python-versions = ">=3.10"
+files = [
+    {file = "numpy-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c8a0e34993b510fc19b9a2ce7f31cb8e94ecf6e924a40c0c9dd4f62d0aac47d9"},
+    {file = "numpy-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7dd86dfaf7c900c0bbdcb8b16e2f6ddf1eb1fe39c6c8cca6e94844ed3152a8fd"},
+    {file = "numpy-2.1.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:5889dd24f03ca5a5b1e8a90a33b5a0846d8977565e4ae003a63d22ecddf6782f"},
+    {file = "numpy-2.1.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:59ca673ad11d4b84ceb385290ed0ebe60266e356641428c845b39cd9df6713ab"},
+    {file = "numpy-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13ce49a34c44b6de5241f0b38b07e44c1b2dcacd9e36c30f9c2fcb1bb5135db7"},
+    {file = "numpy-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:913cc1d311060b1d409e609947fa1b9753701dac96e6581b58afc36b7ee35af6"},
+    {file = "numpy-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:caf5d284ddea7462c32b8d4a6b8af030b6c9fd5332afb70e7414d7fdded4bfd0"},
+    {file = "numpy-2.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:57eb525e7c2a8fdee02d731f647146ff54ea8c973364f3b850069ffb42799647"},
+    {file = "numpy-2.1.1-cp310-cp310-win32.whl", hash = "sha256:9a8e06c7a980869ea67bbf551283bbed2856915f0a792dc32dd0f9dd2fb56728"},
+    {file = "numpy-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:d10c39947a2d351d6d466b4ae83dad4c37cd6c3cdd6d5d0fa797da56f710a6ae"},
+    {file = "numpy-2.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0d07841fd284718feffe7dd17a63a2e6c78679b2d386d3e82f44f0108c905550"},
+    {file = "numpy-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b5613cfeb1adfe791e8e681128f5f49f22f3fcaa942255a6124d58ca59d9528f"},
+    {file = "numpy-2.1.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:0b8cc2715a84b7c3b161f9ebbd942740aaed913584cae9cdc7f8ad5ad41943d0"},
+    {file = "numpy-2.1.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:b49742cdb85f1f81e4dc1b39dcf328244f4d8d1ded95dea725b316bd2cf18c95"},
+    {file = "numpy-2.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8d5f8a8e3bc87334f025194c6193e408903d21ebaeb10952264943a985066ca"},
+    {file = "numpy-2.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d51fc141ddbe3f919e91a096ec739f49d686df8af254b2053ba21a910ae518bf"},
+    {file = "numpy-2.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:98ce7fb5b8063cfdd86596b9c762bf2b5e35a2cdd7e967494ab78a1fa7f8b86e"},
+    {file = "numpy-2.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:24c2ad697bd8593887b019817ddd9974a7f429c14a5469d7fad413f28340a6d2"},
+    {file = "numpy-2.1.1-cp311-cp311-win32.whl", hash = "sha256:397bc5ce62d3fb73f304bec332171535c187e0643e176a6e9421a6e3eacef06d"},
+    {file = "numpy-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:ae8ce252404cdd4de56dcfce8b11eac3c594a9c16c231d081fb705cf23bd4d9e"},
+    {file = "numpy-2.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7c803b7934a7f59563db459292e6aa078bb38b7ab1446ca38dd138646a38203e"},
+    {file = "numpy-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6435c48250c12f001920f0751fe50c0348f5f240852cfddc5e2f97e007544cbe"},
+    {file = "numpy-2.1.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3269c9eb8745e8d975980b3a7411a98976824e1fdef11f0aacf76147f662b15f"},
+    {file = "numpy-2.1.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:fac6e277a41163d27dfab5f4ec1f7a83fac94e170665a4a50191b545721c6521"},
+    {file = "numpy-2.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fcd8f556cdc8cfe35e70efb92463082b7f43dd7e547eb071ffc36abc0ca4699b"},
+    {file = "numpy-2.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b9cd92c8f8e7b313b80e93cedc12c0112088541dcedd9197b5dee3738c1201"},
+    {file = "numpy-2.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:afd9c680df4de71cd58582b51e88a61feed4abcc7530bcd3d48483f20fc76f2a"},
+    {file = "numpy-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8661c94e3aad18e1ea17a11f60f843a4933ccaf1a25a7c6a9182af70610b2313"},
+    {file = "numpy-2.1.1-cp312-cp312-win32.whl", hash = "sha256:950802d17a33c07cba7fd7c3dcfa7d64705509206be1606f196d179e539111ed"},
+    {file = "numpy-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:3fc5eabfc720db95d68e6646e88f8b399bfedd235994016351b1d9e062c4b270"},
+    {file = "numpy-2.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:046356b19d7ad1890c751b99acad5e82dc4a02232013bd9a9a712fddf8eb60f5"},
+    {file = "numpy-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6e5a9cb2be39350ae6c8f79410744e80154df658d5bea06e06e0ac5bb75480d5"},
+    {file = "numpy-2.1.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:d4c57b68c8ef5e1ebf47238e99bf27657511ec3f071c465f6b1bccbef12d4136"},
+    {file = "numpy-2.1.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:8ae0fd135e0b157365ac7cc31fff27f07a5572bdfc38f9c2d43b2aff416cc8b0"},
+    {file = "numpy-2.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:981707f6b31b59c0c24bcda52e5605f9701cb46da4b86c2e8023656ad3e833cb"},
+    {file = "numpy-2.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ca4b53e1e0b279142113b8c5eb7d7a877e967c306edc34f3b58e9be12fda8df"},
+    {file = "numpy-2.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e097507396c0be4e547ff15b13dc3866f45f3680f789c1a1301b07dadd3fbc78"},
+    {file = "numpy-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7506387e191fe8cdb267f912469a3cccc538ab108471291636a96a54e599556"},
+    {file = "numpy-2.1.1-cp313-cp313-win32.whl", hash = "sha256:251105b7c42abe40e3a689881e1793370cc9724ad50d64b30b358bbb3a97553b"},
+    {file = "numpy-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:f212d4f46b67ff604d11fff7cc62d36b3e8714edf68e44e9760e19be38c03eb0"},
+    {file = "numpy-2.1.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:920b0911bb2e4414c50e55bd658baeb78281a47feeb064ab40c2b66ecba85553"},
+    {file = "numpy-2.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bab7c09454460a487e631ffc0c42057e3d8f2a9ddccd1e60c7bb8ed774992480"},
+    {file = "numpy-2.1.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:cea427d1350f3fd0d2818ce7350095c1a2ee33e30961d2f0fef48576ddbbe90f"},
+    {file = "numpy-2.1.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:e30356d530528a42eeba51420ae8bf6c6c09559051887196599d96ee5f536468"},
+    {file = "numpy-2.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8dfa9e94fc127c40979c3eacbae1e61fda4fe71d84869cc129e2721973231ef"},
+    {file = "numpy-2.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:910b47a6d0635ec1bd53b88f86120a52bf56dcc27b51f18c7b4a2e2224c29f0f"},
+    {file = "numpy-2.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:13cc11c00000848702322af4de0147ced365c81d66053a67c2e962a485b3717c"},
+    {file = "numpy-2.1.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:53e27293b3a2b661c03f79aa51c3987492bd4641ef933e366e0f9f6c9bf257ec"},
+    {file = "numpy-2.1.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7be6a07520b88214ea85d8ac8b7d6d8a1839b0b5cb87412ac9f49fa934eb15d5"},
+    {file = "numpy-2.1.1-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:52ac2e48f5ad847cd43c4755520a2317f3380213493b9d8a4c5e37f3b87df504"},
+    {file = "numpy-2.1.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50a95ca3560a6058d6ea91d4629a83a897ee27c00630aed9d933dff191f170cd"},
+    {file = "numpy-2.1.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:99f4a9ee60eed1385a86e82288971a51e71df052ed0b2900ed30bc840c0f2e39"},
+    {file = "numpy-2.1.1.tar.gz", hash = "sha256:d0cf7d55b1051387807405b3898efafa862997b4cba8aa5dbe657be794afeafd"},
+]
+
+[[package]]
+name = "opencv-python"
+version = "4.10.0.84"
+description = "Wrapper package for OpenCV python bindings."
+category = "main"
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "opencv-python-4.10.0.84.tar.gz", hash = "sha256:72d234e4582e9658ffea8e9cae5b63d488ad06994ef12d81dc303b17472f3526"},
+    {file = "opencv_python-4.10.0.84-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fc182f8f4cda51b45f01c64e4cbedfc2f00aff799debebc305d8d0210c43f251"},
+    {file = "opencv_python-4.10.0.84-cp37-abi3-macosx_12_0_x86_64.whl", hash = "sha256:71e575744f1d23f79741450254660442785f45a0797212852ee5199ef12eed98"},
+    {file = "opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09a332b50488e2dda866a6c5573ee192fe3583239fb26ff2f7f9ceb0bc119ea6"},
+    {file = "opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ace140fc6d647fbe1c692bcb2abce768973491222c067c131d80957c595b71f"},
+    {file = "opencv_python-4.10.0.84-cp37-abi3-win32.whl", hash = "sha256:2db02bb7e50b703f0a2d50c50ced72e95c574e1e5a0bb35a8a86d0b35c98c236"},
+    {file = "opencv_python-4.10.0.84-cp37-abi3-win_amd64.whl", hash = "sha256:32dbbd94c26f611dc5cc6979e6b7aa1f55a64d6b463cc1dcd3c95505a63e48fe"},
+]
+
+[package.dependencies]
+numpy = [
+    {version = ">=1.21.2", markers = "python_version >= \"3.10\""},
+    {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""},
+    {version = ">=1.23.5", markers = "python_version >= \"3.11\""},
+    {version = ">=1.26.0", markers = "python_version >= \"3.12\""},
+    {version = ">=1.19.3", markers = "python_version >= \"3.6\" and platform_system == \"Linux\" and platform_machine == \"aarch64\" or python_version >= \"3.9\""},
+    {version = ">=1.17.0", markers = "python_version >= \"3.7\""},
+    {version = ">=1.17.3", markers = "python_version >= \"3.8\""},
+]
+
 [[package]]
 name = "packaging"
 version = "24.1"
 description = "Core utilities for Python packages"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -558,6 +688,7 @@ files = [
 name = "pluggy"
 version = "1.5.0"
 description = "plugin and hook calling mechanisms for python"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -573,6 +704,7 @@ testing = ["pytest", "pytest-benchmark"]
 name = "pydantic"
 version = "2.9.1"
 description = "Data validation using Python type hints"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -596,6 +728,7 @@ timezone = ["tzdata"]
 name = "pydantic-core"
 version = "2.23.3"
 description = "Core functionality for Pydantic validation and serialization"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -697,6 +830,7 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
 name = "pygments"
 version = "2.18.0"
 description = "Pygments is a syntax highlighting package written in Python."
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -711,6 +845,7 @@ windows-terminal = ["colorama (>=0.4.6)"]
 name = "pytest"
 version = "8.3.3"
 description = "pytest: simple powerful testing with Python"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -731,6 +866,7 @@ dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments
 name = "pytest-asyncio"
 version = "0.24.0"
 description = "Pytest support for asyncio"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -749,6 +885,7 @@ testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"]
 name = "python-dotenv"
 version = "1.0.1"
 description = "Read key-value pairs from a .env file and set them as environment variables"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -763,6 +900,7 @@ cli = ["click (>=5.0)"]
 name = "python-multipart"
 version = "0.0.9"
 description = "A streaming multipart parser for Python"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -777,6 +915,7 @@ dev = ["atomicwrites (==1.4.1)", "attrs (==23.2.0)", "coverage (==7.4.1)", "hatc
 name = "pyyaml"
 version = "6.0.2"
 description = "YAML parser and emitter for Python"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -839,6 +978,7 @@ files = [
 name = "requests"
 version = "2.32.3"
 description = "Python HTTP for Humans."
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -860,6 +1000,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
 name = "rich"
 version = "13.8.1"
 description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
+category = "main"
 optional = false
 python-versions = ">=3.7.0"
 files = [
@@ -878,6 +1019,7 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"]
 name = "shellingham"
 version = "1.5.4"
 description = "Tool to Detect Surrounding Shell"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -889,6 +1031,7 @@ files = [
 name = "sniffio"
 version = "1.3.1"
 description = "Sniff out which async library your code is running under"
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -900,6 +1043,7 @@ files = [
 name = "starlette"
 version = "0.37.2"
 description = "The little ASGI library that shines."
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -917,6 +1061,7 @@ full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.7
 name = "typer"
 version = "0.12.5"
 description = "Typer, build great CLIs. Easy to code. Based on Python type hints."
+category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -934,6 +1079,7 @@ typing-extensions = ">=3.7.4.3"
 name = "typing-extensions"
 version = "4.12.2"
 description = "Backported and Experimental Type Hints for Python 3.8+"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -945,6 +1091,7 @@ files = [
 name = "urllib3"
 version = "2.2.3"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -962,6 +1109,7 @@ zstd = ["zstandard (>=0.18.0)"]
 name = "uvicorn"
 version = "0.30.6"
 description = "The lightning-fast ASGI server."
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -976,7 +1124,7 @@ h11 = ">=0.8"
 httptools = {version = ">=0.5.0", optional = true, markers = "extra == \"standard\""}
 python-dotenv = {version = ">=0.13", optional = true, markers = "extra == \"standard\""}
 pyyaml = {version = ">=5.1", optional = true, markers = "extra == \"standard\""}
-uvloop = {version = ">=0.14.0,<0.15.0 || >0.15.0,<0.15.1 || >0.15.1", optional = true, markers = "(sys_platform != \"win32\" and sys_platform != \"cygwin\") and platform_python_implementation != \"PyPy\" and extra == \"standard\""}
+uvloop = {version = ">=0.14.0,<0.15.0 || >0.15.0,<0.15.1 || >0.15.1", optional = true, markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" and extra == \"standard\""}
 watchfiles = {version = ">=0.13", optional = true, markers = "extra == \"standard\""}
 websockets = {version = ">=10.4", optional = true, markers = "extra == \"standard\""}
 
@@ -987,6 +1135,7 @@ standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)",
 name = "uvloop"
 version = "0.20.0"
 description = "Fast implementation of asyncio event loop on top of libuv"
+category = "main"
 optional = false
 python-versions = ">=3.8.0"
 files = [
@@ -1031,6 +1180,7 @@ test = ["Cython (>=0.29.36,<0.30.0)", "aiohttp (==3.9.0b0)", "aiohttp (>=3.8.1)"
 name = "watchfiles"
 version = "0.24.0"
 description = "Simple, modern and high performance file watching and code reload in python."
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1126,6 +1276,7 @@ anyio = ">=3.0.0"
 name = "websockets"
 version = "13.0.1"
 description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)"
+category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1221,6 +1372,7 @@ files = [
 name = "wrapt"
 version = "1.16.0"
 description = "Module for decorators, wrappers and monkey patching."
+category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1298,5 +1450,5 @@ files = [
 
 [metadata]
 lock-version = "2.0"
-python-versions = "^3.11"
+python-versions = "^3.12"
-content-hash = "8e6c46b679a81d033850227859c9754ca5e58b3b10d1e13383bf0576bb34da5c"
+content-hash = "36519f0ca589faeaac9710ec3f1b055905d9560673ec53611abacbe259990b53"

pyproject.toml

@@ -2,18 +2,19 @@
 name = "byparr"
 version = "0.1.0"
 description = ""
-package-mode = false
+# package-mode = false
 authors = ["Your Name <you@example.com>"]
 readme = "README.md"
 
 [tool.poetry.dependencies]
-python = "^3.11"
+python = "^3.12"
 pytest = "^8.3.1"
 fastapi = "^0.111.1"
 nodriver = "^0.34"
 requests = "^2.32.3"
 httpx = "^0.27.2"
 pytest-asyncio = "^0.24.0"
+nn-mouse = "^1.0.1"
 
 
 [build-system]

requirements.txt

@@ -0,0 +1,5 @@
+fastapi==0.111.1
+nodriver==0.34
+requests==2.32.3
+httpx==0.27.2
+nn-mouse==1.0.1

src/main.py

@@ -9,10 +9,10 @@ import uvicorn.config
 from fastapi import FastAPI
 from fastapi.responses import RedirectResponse
 
-from src.models.requests import LinkRequest, LinkResponse
+from models.requests import LinkRequest, LinkResponse
-from src.utils import logger
+from utils import logger
-from src.utils.browser import bypass_cloudflare, new_browser
+from utils.browser import bypass_cloudflare, new_browser
-from src.utils.consts import LOG_LEVEL
+from utils.consts import LOG_LEVEL
 
 app = FastAPI(debug=LOG_LEVEL == logging.DEBUG, log_level=LOG_LEVEL)
 
@@ -31,25 +31,23 @@ async def read_item(request: LinkRequest):
     logger.info(f"Request: {request}")
     start_time = int(time.time() * 1000)
     browser = await new_browser()
-    try:
+    page = await browser.get(request.url)
-        page = await browser.get(request.url)
+    await page.bring_to_front()
-        await page.bring_to_front()
+    timeout = request.maxTimeout
+    if timeout == 0:
+        timeout = None
 
-        challenged = await asyncio.wait_for(
+    challenged = await asyncio.wait_for(bypass_cloudflare(page), timeout=timeout)
-            bypass_cloudflare(page), timeout=request.maxTimeout
-        )
 
-        logger.info(f"Got webpage: {request.url}")
+    logger.info(f"Got webpage: {request.url}")
 
-        response = await LinkResponse.create(
+    response = await LinkResponse.create(
-            page=page,
+        page=page,
-            start_timestamp=start_time,
+        start_timestamp=start_time,
-            challenged=challenged,
+        challenged=challenged,
-        )
+    )
-    except asyncio.TimeoutError:
+
-        logger.fatal("Couldn't complete the request")
+    browser.stop()
-    finally:
-        browser.stop()
     return response
 
 

src/utils/__init__.py

@@ -1,6 +1,6 @@
 import logging
 
-from src.utils.consts import LOG_LEVEL
+from utils.consts import LOG_LEVEL
 
 logger = logging.getLogger("uvicorn.error")
 logger.setLevel(LOG_LEVEL)

src/utils/browser.py

@@ -1,13 +1,13 @@
 import asyncio
+import typing
+import random
 
 import nodriver as webdriver
-from nodriver.core.element import Element
+from nodriver.core.element import Element, Position
+from nn_mouse import get_path
 
-from src.utils import logger
+from . import logger
-from src.utils.consts import CHALLENGE_TITLES
+from .consts import CHALLENGE_TITLES
-from src.utils.extentions import download_extentions
-
-downloaded_extentions = download_extentions()
 
 
 async def new_browser():
@@ -25,7 +25,6 @@ async def new_browser():
     """
     config: webdriver.Config = webdriver.Config()
     config.sandbox = False
-    config.add_argument(f"--load-extension={','.join(downloaded_extentions)}")
 
     return await webdriver.start(config=config)
 
@@ -65,6 +64,12 @@ async def bypass_cloudflare(page: webdriver.Tab):
         if not challenged:
             logger.info("Found challenge")
             challenged = True
+        elem = None
+
+        # get the size of the page
+        html_elem = await page.select("html")
+        html_pos = await html_elem.get_position()
+
         try:
             elem = await page.find(
                 "Verify you are human by completing the action below.",
@@ -74,31 +79,30 @@ async def bypass_cloudflare(page: webdriver.Tab):
         except asyncio.TimeoutError:
             if page.target.title not in CHALLENGE_TITLES:
                 return challenged
-            raise
 
         if elem is None:
-            logger.debug("Couldn't find the title, trying again")
+            logger.debug("Couldn't find the title, trying other method...")
             continue
 
+        if not isinstance(elem, Element):
+            logger.fatal("Element is a string, please report this to Byparr dev")
+            raise InvalidElementError
+
+        elem = await page.find("input")
         elem = elem.parent
         # Get the element containing the shadow root
-        for _ in range(3):
-            if elem is not None:
-                elem = get_first_div(elem)
-            else:
-                raise InvalidElementError
 
         if isinstance(elem, Element) and elem.shadow_roots:
             inner_elem = Element(elem.shadow_roots[0], page, elem.tree).children[0]
             if isinstance(inner_elem, Element):
                 logger.debug("Clicking element")
-                await inner_elem.mouse_click()
+                await mouse_click(inner_elem, html_pos=html_pos)
             else:
-                logger.warn(
+                logger.warning(
                     "Element is a string, please report this to Byparr dev"
                 )  # I really hope this never happens
         else:
-            logger.warn("Coulnd't find checkbox, trying again...")
+            logger.warning("Coulnd't find checkbox, trying again...")
 
 
 def get_first_div(elem):
@@ -120,5 +124,85 @@ def get_first_div(elem):
     raise InvalidElementError
 
 
+def clamp(n: int, smallest: int, largest: int):
+    return max(smallest, min(n, largest))
+
+
+async def mouse_click(
+    elm,
+    button: str = "left",
+    buttons: typing.Optional[int] = 1,
+    modifiers: typing.Optional[int] = 0,
+    hold: bool = False,
+    html_pos: typing.Optional[Position] = None,
+    _until_event: typing.Optional[type] = None,
+):
+    """native click (on element) . note: this likely does not work atm, use click() instead
+    :param button: str (default = "left")
+    :param buttons: which button (default 1 = left)
+    :param modifiers: *(Optional)* Bit field representing pressed modifier keys.
+            Alt=1, Ctrl=2, Meta/Command=4, Shift=8 (default: 0).
+    :param _until_event: internal. event to wait for before returning
+    :return:
+    """
+    try:
+        pos = await elm.get_position()
+    except AttributeError:
+        return
+    if not pos:
+        logger.warning("could not calculate box model for %s", self)
+        return
+    oc = pos.center
+    center = (oc[0] + random.randint(-10, 10), oc[1] + random.randint(-10, 10))
+
+    html_height = 500
+    html_width = 500
+    if html_pos:
+        html_height = html_pos.height
+        html_width = html_pos.width
+
+    start_x = random.randint(0, html_width)
+    start_y = random.randint(0, html_height)
+    path = get_path(start_x, start_y, center[0], center[1], html_width, html_height)
+
+    for (x, y, t) in path:
+        x = clamp(int(x), 0, html_width)
+        y = clamp(int(y), 0, html_height)
+        await elm._tab.send(webdriver.cdp.input_.dispatch_mouse_event("mousemove", x, y))
+        await asyncio.sleep(float(t))
+
+    logger.debug("clicking on location %.2f, %.2f" % center)
+    await asyncio.sleep(random.uniform(0.05, 0.3))
+
+    await elm._tab.send(
+            webdriver.cdp.input_.dispatch_mouse_event(
+                "mousePressed",
+                x=center[0],
+                y=center[1],
+                modifiers=modifiers,
+                button=webdriver.cdp.input_.MouseButton(button),
+                buttons=buttons,
+                click_count=1,
+            )
+        )
+    await asyncio.sleep(random.uniform(0.02, 0.06))
+    await elm._tab.send(
+            webdriver.cdp.input_.dispatch_mouse_event(
+                "mouseReleased",
+                x=center[0],
+                y=center[1],
+                modifiers=modifiers,
+                button=webdriver.cdp.input_.MouseButton(button),
+                buttons=buttons,
+                click_count=1,
+            )
+        )
+    # Flash the checkbox for testing
+    try:
+        await elm.flash()
+    except:  # noqa
+        pass
+
+
 class InvalidElementError(Exception):
     pass

src/utils/extentions.py

@@ -8,10 +8,10 @@ from zipfile import ZipFile
 import httpx
 import requests
 
-from src.models.github import GithubResponse
+from ..models.github import GithubResponse
-from src.models.requests import NoChromeExtentionError
+from ..models.requests import NoChromeExtentionError
-from src.utils import logger
+from . import logger
-from src.utils.consts import EXTENTION_REPOSITIORIES, EXTENTIONS_PATH, GITHUB_WEBSITES
+from .consts import EXTENTION_REPOSITIORIES, EXTENTIONS_PATH, GITHUB_WEBSITES
 
 
 def get_latest_github_chrome_release(url: str):

tests/main_test.py

@@ -1,18 +1,27 @@
-import pytest
+from http import HTTPStatus
 
-from main import read_item
+import pytest
+from starlette.testclient import TestClient
+
+from main import app
 from src.models.requests import LinkRequest
 
+client = TestClient(app)
+
 test_websites = [
     "https://ext.to/",
     "https://btmet.com/",
-    "https://extratorrent.st/",
+    # "https://extratorrent.st/", # github is blocking these
-    "https://idope.se/",
+    # "https://idope.se/", # github is blocking these
 ]
-pytest_plugins = ("pytest_asyncio",)
 
 
 @pytest.mark.parametrize("website", test_websites)
-@pytest.mark.asyncio
+def test_bypass(website: str):
-async def test_bypass(website: str):
+    response = client.post(
-    await read_item(LinkRequest(url=website, maxTimeout=5, cmd=""))
+        "/v1",
+        json=LinkRequest(
+            url=website, maxTimeout=60 * len(test_websites), cmd="request.get"
+        ).model_dump(),
+    )
+    assert response.status_code == HTTPStatus.OK