diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 73161d3..c7ba739 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -22,6 +22,30 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: "3.12" + + - name: Run image + uses: abatilo/actions-poetry@v2 + + - uses: actions/cache@v3 + name: Define a cache for the virtual environment based on the dependencies lock file + with: + path: ./.venv + key: venv-${{ hashFiles('poetry.lock') }} + + - name: Install the project dependencies + run: poetry install + + - name: Run tests + run: poetry run pytest -v + build: runs-on: ubuntu-latest permissions: diff --git a/Dockerfile b/Dockerfile index f9718bb..7e43e77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,78 +1,35 @@ -FROM python:3.12-slim-bullseye AS builder +FROM python:3.12 -# Build dummy packages to skip installing them and their dependencies -RUN apt-get update \ - && apt-get install -y --no-install-recommends equivs \ - && equivs-control libgl1-mesa-dri \ - && printf 'Section: misc\nPriority: optional\nStandards-Version: 3.9.2\nPackage: libgl1-mesa-dri\nVersion: 99.0.0\nDescription: Dummy package for libgl1-mesa-dri\n' >> libgl1-mesa-dri \ - && equivs-build libgl1-mesa-dri \ - && mv libgl1-mesa-dri_*.deb /libgl1-mesa-dri.deb \ - && equivs-control adwaita-icon-theme \ - && printf 'Section: misc\nPriority: optional\nStandards-Version: 3.9.2\nPackage: adwaita-icon-theme\nVersion: 99.0.0\nDescription: Dummy package for adwaita-icon-theme\n' >> adwaita-icon-theme \ - && equivs-build adwaita-icon-theme \ - && mv adwaita-icon-theme_*.deb /adwaita-icon-theme.deb - -FROM python:3.12-slim-bullseye - -# Copy dummy packages -COPY --from=builder /*.deb / - -# Install dependencies and create byparr user -# You can test Chromium running this command inside the container: -# xvfb-run -s "-screen 0 1600x1200x24" chromium --no-sandbox -# The error traces is like this: "*** stack smashing detected ***: terminated" -# To check the package versions available you can use this command: -# apt-cache madison chromium WORKDIR /app - # Install dummy packages -RUN dpkg -i /libgl1-mesa-dri.deb \ - && dpkg -i /adwaita-icon-theme.deb \ - # Install dependencies - && apt-get update \ - && apt-get install -y --no-install-recommends chromium xvfb dumb-init \ - procps xauth sudo \ - # Remove temporary files and hardware decoding libraries - && rm -rf /var/lib/apt/lists/* \ - && rm -f /usr/lib/x86_64-linux-gnu/libmfxhw* \ - && rm -f /usr/lib/x86_64-linux-gnu/mfx/* \ - # Create byparr user - && useradd --home-dir /app --shell /bin/sh byparr \ - && chown -R byparr:byparr . \ - && echo 'byparr ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -# Install Python dependencies -COPY requirements.txt fix_nodriver.py novnc.sh . -RUN pip install -r requirements.txt \ - # Remove temporary files - && rm -rf /root/.cache \ - && PYENV_PATH=/usr/local/lib/python3.12 python fix_nodriver.py - -ENV INSTALL_NOVNC=false DISPLAY=:1.0 -RUN ./novnc.sh - -USER byparr - -COPY src . - EXPOSE 8191 -# dumb-init avoids zombie chromium processes -ENTRYPOINT ["/usr/bin/dumb-init", "--"] +# python +ENV \ + DEBIAN_FRONTEND=noninteractive \ + PYTHONUNBUFFERED=1 \ + # prevents python creating .pyc files + PYTHONDONTWRITEBYTECODE=1 \ + # do not ask any interactive question + POETRY_NO_INTERACTION=1 \ + POETRY_VIRTUALENVS_IN_PROJECT=true -CMD ["/bin/sh", "-c", "/usr/local/share/desktop-init.sh && python -u /app/main.py"] +RUN apt update && apt upgrade -y +RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb +RUN apt install -y --no-install-recommends --no-install-suggests ./google-chrome-stable_current_amd64.deb && rm ./google-chrome-stable_current_amd64.deb -# Local build -# docker build -t ngosang/byparr:3.3.21 . -# docker run -p 8191:8191 ngosang/byparr:3.3.21 +RUN apt install pipx -y +RUN pipx ensurepath +RUN pipx install poetry +ENV PATH="/root/.local/bin:$PATH" +COPY pyproject.toml poetry.lock ./ +RUN poetry install -# Multi-arch build -# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -# docker buildx create --use -# docker buildx build -t ngosang/byparr:3.3.21 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . -# add --push to publish in DockerHub +ENV INSTALL_NOVNC=false +COPY novnc.sh . +RUN ./novnc.sh +ENV DISPLAY=:1.0 -# Test multi-arch build -# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -# docker buildx create --use -# docker buildx build -t ngosang/byparr:3.3.21 --platform linux/arm/v7 --load . -# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/byparr:3.3.21 +COPY . . +RUN . /app/.venv/bin/activate && python fix_nodriver.py + +CMD /usr/local/share/desktop-init.sh && . /app/.venv/bin/activate && python main.py \ No newline at end of file diff --git a/fix_nodriver.py b/fix_nodriver.py index 8677921..1e7c0bb 100644 --- a/fix_nodriver.py +++ b/fix_nodriver.py @@ -6,14 +6,10 @@ import logging import os from pathlib import Path -pyenv_path = os.getenv("PYENV_PATH") -if pyenv_path is None: - env_path = os.getenv("VIRTUAL_ENV") - if env_path is None: - env_path = Path(os.__file__).parent.parent.parent.as_posix() - pyenv_path = Path(env_path + "/lib/python3.12") -nodriver_pkg_path = Path(pyenv_path) / "site-packages/nodriver" -nodriver_path = nodriver_pkg_path / "cdp/network.py" +env_path = os.getenv("VIRTUAL_ENV") +if env_path is None: + env_path = Path(os.__file__).parent.parent.parent.as_posix() +nodriver_path = Path(env_path + "/lib/python3.11/site-packages/nodriver/cdp/network.py") if not nodriver_path.exists(): msg = f"{nodriver_path} not found" raise FileNotFoundError(msg) @@ -65,19 +61,3 @@ with nodriver_path.open("r+") as f: with nodriver_path.open("w") as f: f.writelines(lines) - - -browser_path = nodriver_pkg_path / "core/browser.py" -if not browser_path.exists(): - msg = f"{browser_path} not found" - raise FileNotFoundError(msg) - -with browser_path.open("r") as f: - browser_path_txt = f.read() - -browser_path_txt2 = browser_path_txt.replace("for _ in range(5):", "for _ in range(20):") - -if browser_path_txt2 != browser_path_txt: - logger.info("increasing browser open timeout") - with browser_path.open("w") as f: - f.write(browser_path_txt2) diff --git a/src/main.py b/main.py similarity index 52% rename from src/main.py rename to main.py index 1f40b6c..847fc06 100644 --- a/src/main.py +++ b/main.py @@ -9,10 +9,10 @@ import uvicorn.config from fastapi import FastAPI from fastapi.responses import RedirectResponse -from models.requests import LinkRequest, LinkResponse -from utils import logger -from utils.browser import bypass_cloudflare, new_browser -from utils.consts import LOG_LEVEL +from src.models.requests import LinkRequest, LinkResponse +from src.utils import logger +from src.utils.browser import bypass_cloudflare, new_browser +from src.utils.consts import LOG_LEVEL app = FastAPI(debug=LOG_LEVEL == logging.DEBUG, log_level=LOG_LEVEL) @@ -31,23 +31,25 @@ async def read_item(request: LinkRequest): logger.info(f"Request: {request}") start_time = int(time.time() * 1000) browser = await new_browser() - page = await browser.get(request.url) - await page.bring_to_front() - timeout = request.maxTimeout - if timeout == 0: - timeout = None + try: + page = await browser.get(request.url) + await page.bring_to_front() - challenged = await asyncio.wait_for(bypass_cloudflare(page), timeout=timeout) + challenged = await asyncio.wait_for( + bypass_cloudflare(page), timeout=request.maxTimeout + ) - logger.info(f"Got webpage: {request.url}") + logger.info(f"Got webpage: {request.url}") - response = await LinkResponse.create( - page=page, - start_timestamp=start_time, - challenged=challenged, - ) - - browser.stop() + response = await LinkResponse.create( + page=page, + start_timestamp=start_time, + challenged=challenged, + ) + except asyncio.TimeoutError: + logger.fatal("Couldn't complete the request") + finally: + browser.stop() return response diff --git a/poetry.lock b/poetry.lock index f7add5b..7e311f7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "annotated-types" version = "0.7.0" description = "Reusable constraint types to use with typing.Annotated" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -16,7 +15,6 @@ files = [ name = "anyio" version = "4.4.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -37,7 +35,6 @@ trio = ["trio (>=0.23)"] name = "certifi" version = "2024.8.30" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -49,7 +46,6 @@ files = [ name = "charset-normalizer" version = "3.3.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -149,7 +145,6 @@ files = [ name = "click" version = "8.1.7" description = "Composable command line interface toolkit" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -164,7 +159,6 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -176,7 +170,6 @@ files = [ name = "deprecated" version = "1.2.14" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -194,7 +187,6 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] name = "dnspython" version = "2.6.1" description = "DNS toolkit" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -215,7 +207,6 @@ wmi = ["wmi (>=1.5.1)"] name = "email-validator" version = "2.2.0" description = "A robust email address syntax and deliverability validation library." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -231,7 +222,6 @@ idna = ">=2.0.0" name = "fastapi" version = "0.111.1" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -257,7 +247,6 @@ all = ["email_validator (>=2.0.0)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)" name = "fastapi-cli" version = "0.0.5" description = "Run and manage FastAPI apps from the command line with FastAPI CLI. 🚀" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -276,7 +265,6 @@ standard = ["uvicorn[standard] (>=0.15.0)"] name = "h11" version = "0.14.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -288,7 +276,6 @@ files = [ name = "httpcore" version = "1.0.5" description = "A minimal low-level HTTP client." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -303,14 +290,13 @@ h11 = ">=0.13,<0.15" [package.extras] asyncio = ["anyio (>=4.0,<5.0)"] http2 = ["h2 (>=3,<5)"] -socks = ["socksio (>=1.0.0,<2.0.0)"] +socks = ["socksio (==1.*)"] trio = ["trio (>=0.22.0,<0.26.0)"] [[package]] name = "httptools" version = "0.6.1" description = "A collection of framework independent HTTP protocol utils." -category = "main" optional = false python-versions = ">=3.8.0" files = [ @@ -359,7 +345,6 @@ test = ["Cython (>=0.29.24,<0.30.0)"] name = "httpx" version = "0.27.2" description = "The next generation HTTP client." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -370,22 +355,21 @@ files = [ [package.dependencies] anyio = "*" certifi = "*" -httpcore = ">=1.0.0,<2.0.0" +httpcore = "==1.*" idna = "*" sniffio = "*" [package.extras] brotli = ["brotli", "brotlicffi"] -cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<14)"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] -socks = ["socksio (>=1.0.0,<2.0.0)"] +socks = ["socksio (==1.*)"] zstd = ["zstandard (>=0.18.0)"] [[package]] name = "idna" version = "3.8" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -397,7 +381,6 @@ files = [ name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -409,7 +392,6 @@ files = [ name = "jinja2" version = "3.1.4" description = "A very fast and expressive template engine." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -427,7 +409,6 @@ i18n = ["Babel (>=2.7)"] name = "markdown-it-py" version = "3.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -452,7 +433,6 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "markupsafe" version = "2.1.5" description = "Safely add untrusted strings to HTML/XML markup." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -522,7 +502,6 @@ files = [ name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -534,7 +513,6 @@ files = [ name = "mss" version = "9.0.2" description = "An ultra fast cross-platform multiple screenshots module in pure python using ctypes." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -546,26 +524,10 @@ files = [ dev = ["build (==1.2.1)", "mypy (==1.11.2)", "ruff (==0.6.3)", "twine (==5.1.1)", "wheel (==0.44.0)"] test = ["numpy (==2.1.0)", "pillow (==10.4.0)", "pytest (==8.3.2)", "pytest-cov (==5.0.0)", "pytest-rerunfailures (==14.0.0)", "pyvirtualdisplay (==3.0)", "sphinx (==8.0.2)"] -[[package]] -name = "nn-mouse" -version = "1.0.1" -description = "Neural network based humanized mouse movements" -category = "main" -optional = false -python-versions = ">=3" -files = [ - {file = "nn_mouse-1.0.1-py3-none-any.whl", hash = "sha256:726d548ba3796914c454507b4b91d8e2555e870392fc03f7f136ee0cf1db0150"}, - {file = "nn_mouse-1.0.1.tar.gz", hash = "sha256:d92566959a339b99f47d4b403069f3afdc81d47d019836368d6dfd43fe345f0a"}, -] - -[package.dependencies] -opencv-python = "*" - [[package]] name = "nodriver" version = "0.34" description = "[Docs here](https://ultrafunkamsterdam.github.io/nodriver)" -category = "main" optional = false python-versions = ">=3.9" files = [ @@ -581,102 +543,10 @@ websockets = ">=11" [package.extras] dev = ["black", "build", "furo", "pygments", "sphinx", "sphinx-autodoc-typehints", "sphinx-markdown-builder"] -[[package]] -name = "numpy" -version = "2.1.1" -description = "Fundamental package for array computing in Python" -category = "main" -optional = false -python-versions = ">=3.10" -files = [ - {file = "numpy-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c8a0e34993b510fc19b9a2ce7f31cb8e94ecf6e924a40c0c9dd4f62d0aac47d9"}, - {file = "numpy-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7dd86dfaf7c900c0bbdcb8b16e2f6ddf1eb1fe39c6c8cca6e94844ed3152a8fd"}, - {file = "numpy-2.1.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:5889dd24f03ca5a5b1e8a90a33b5a0846d8977565e4ae003a63d22ecddf6782f"}, - {file = "numpy-2.1.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:59ca673ad11d4b84ceb385290ed0ebe60266e356641428c845b39cd9df6713ab"}, - {file = "numpy-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13ce49a34c44b6de5241f0b38b07e44c1b2dcacd9e36c30f9c2fcb1bb5135db7"}, - {file = "numpy-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:913cc1d311060b1d409e609947fa1b9753701dac96e6581b58afc36b7ee35af6"}, - {file = "numpy-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:caf5d284ddea7462c32b8d4a6b8af030b6c9fd5332afb70e7414d7fdded4bfd0"}, - {file = "numpy-2.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:57eb525e7c2a8fdee02d731f647146ff54ea8c973364f3b850069ffb42799647"}, - {file = "numpy-2.1.1-cp310-cp310-win32.whl", hash = "sha256:9a8e06c7a980869ea67bbf551283bbed2856915f0a792dc32dd0f9dd2fb56728"}, - {file = "numpy-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:d10c39947a2d351d6d466b4ae83dad4c37cd6c3cdd6d5d0fa797da56f710a6ae"}, - {file = "numpy-2.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0d07841fd284718feffe7dd17a63a2e6c78679b2d386d3e82f44f0108c905550"}, - {file = "numpy-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b5613cfeb1adfe791e8e681128f5f49f22f3fcaa942255a6124d58ca59d9528f"}, - {file = "numpy-2.1.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:0b8cc2715a84b7c3b161f9ebbd942740aaed913584cae9cdc7f8ad5ad41943d0"}, - {file = "numpy-2.1.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:b49742cdb85f1f81e4dc1b39dcf328244f4d8d1ded95dea725b316bd2cf18c95"}, - {file = "numpy-2.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8d5f8a8e3bc87334f025194c6193e408903d21ebaeb10952264943a985066ca"}, - {file = "numpy-2.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d51fc141ddbe3f919e91a096ec739f49d686df8af254b2053ba21a910ae518bf"}, - {file = "numpy-2.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:98ce7fb5b8063cfdd86596b9c762bf2b5e35a2cdd7e967494ab78a1fa7f8b86e"}, - {file = "numpy-2.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:24c2ad697bd8593887b019817ddd9974a7f429c14a5469d7fad413f28340a6d2"}, - {file = "numpy-2.1.1-cp311-cp311-win32.whl", hash = "sha256:397bc5ce62d3fb73f304bec332171535c187e0643e176a6e9421a6e3eacef06d"}, - {file = "numpy-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:ae8ce252404cdd4de56dcfce8b11eac3c594a9c16c231d081fb705cf23bd4d9e"}, - {file = "numpy-2.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7c803b7934a7f59563db459292e6aa078bb38b7ab1446ca38dd138646a38203e"}, - {file = "numpy-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6435c48250c12f001920f0751fe50c0348f5f240852cfddc5e2f97e007544cbe"}, - {file = "numpy-2.1.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3269c9eb8745e8d975980b3a7411a98976824e1fdef11f0aacf76147f662b15f"}, - {file = "numpy-2.1.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:fac6e277a41163d27dfab5f4ec1f7a83fac94e170665a4a50191b545721c6521"}, - {file = "numpy-2.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fcd8f556cdc8cfe35e70efb92463082b7f43dd7e547eb071ffc36abc0ca4699b"}, - {file = "numpy-2.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b9cd92c8f8e7b313b80e93cedc12c0112088541dcedd9197b5dee3738c1201"}, - {file = "numpy-2.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:afd9c680df4de71cd58582b51e88a61feed4abcc7530bcd3d48483f20fc76f2a"}, - {file = "numpy-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8661c94e3aad18e1ea17a11f60f843a4933ccaf1a25a7c6a9182af70610b2313"}, - {file = "numpy-2.1.1-cp312-cp312-win32.whl", hash = "sha256:950802d17a33c07cba7fd7c3dcfa7d64705509206be1606f196d179e539111ed"}, - {file = "numpy-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:3fc5eabfc720db95d68e6646e88f8b399bfedd235994016351b1d9e062c4b270"}, - {file = "numpy-2.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:046356b19d7ad1890c751b99acad5e82dc4a02232013bd9a9a712fddf8eb60f5"}, - {file = "numpy-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6e5a9cb2be39350ae6c8f79410744e80154df658d5bea06e06e0ac5bb75480d5"}, - {file = "numpy-2.1.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:d4c57b68c8ef5e1ebf47238e99bf27657511ec3f071c465f6b1bccbef12d4136"}, - {file = "numpy-2.1.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:8ae0fd135e0b157365ac7cc31fff27f07a5572bdfc38f9c2d43b2aff416cc8b0"}, - {file = "numpy-2.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:981707f6b31b59c0c24bcda52e5605f9701cb46da4b86c2e8023656ad3e833cb"}, - {file = "numpy-2.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ca4b53e1e0b279142113b8c5eb7d7a877e967c306edc34f3b58e9be12fda8df"}, - {file = "numpy-2.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e097507396c0be4e547ff15b13dc3866f45f3680f789c1a1301b07dadd3fbc78"}, - {file = "numpy-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7506387e191fe8cdb267f912469a3cccc538ab108471291636a96a54e599556"}, - {file = "numpy-2.1.1-cp313-cp313-win32.whl", hash = "sha256:251105b7c42abe40e3a689881e1793370cc9724ad50d64b30b358bbb3a97553b"}, - {file = "numpy-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:f212d4f46b67ff604d11fff7cc62d36b3e8714edf68e44e9760e19be38c03eb0"}, - {file = "numpy-2.1.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:920b0911bb2e4414c50e55bd658baeb78281a47feeb064ab40c2b66ecba85553"}, - {file = "numpy-2.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bab7c09454460a487e631ffc0c42057e3d8f2a9ddccd1e60c7bb8ed774992480"}, - {file = "numpy-2.1.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:cea427d1350f3fd0d2818ce7350095c1a2ee33e30961d2f0fef48576ddbbe90f"}, - {file = "numpy-2.1.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:e30356d530528a42eeba51420ae8bf6c6c09559051887196599d96ee5f536468"}, - {file = "numpy-2.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8dfa9e94fc127c40979c3eacbae1e61fda4fe71d84869cc129e2721973231ef"}, - {file = "numpy-2.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:910b47a6d0635ec1bd53b88f86120a52bf56dcc27b51f18c7b4a2e2224c29f0f"}, - {file = "numpy-2.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:13cc11c00000848702322af4de0147ced365c81d66053a67c2e962a485b3717c"}, - {file = "numpy-2.1.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:53e27293b3a2b661c03f79aa51c3987492bd4641ef933e366e0f9f6c9bf257ec"}, - {file = "numpy-2.1.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7be6a07520b88214ea85d8ac8b7d6d8a1839b0b5cb87412ac9f49fa934eb15d5"}, - {file = "numpy-2.1.1-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:52ac2e48f5ad847cd43c4755520a2317f3380213493b9d8a4c5e37f3b87df504"}, - {file = "numpy-2.1.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50a95ca3560a6058d6ea91d4629a83a897ee27c00630aed9d933dff191f170cd"}, - {file = "numpy-2.1.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:99f4a9ee60eed1385a86e82288971a51e71df052ed0b2900ed30bc840c0f2e39"}, - {file = "numpy-2.1.1.tar.gz", hash = "sha256:d0cf7d55b1051387807405b3898efafa862997b4cba8aa5dbe657be794afeafd"}, -] - -[[package]] -name = "opencv-python" -version = "4.10.0.84" -description = "Wrapper package for OpenCV python bindings." -category = "main" -optional = false -python-versions = ">=3.6" -files = [ - {file = "opencv-python-4.10.0.84.tar.gz", hash = "sha256:72d234e4582e9658ffea8e9cae5b63d488ad06994ef12d81dc303b17472f3526"}, - {file = "opencv_python-4.10.0.84-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fc182f8f4cda51b45f01c64e4cbedfc2f00aff799debebc305d8d0210c43f251"}, - {file = "opencv_python-4.10.0.84-cp37-abi3-macosx_12_0_x86_64.whl", hash = "sha256:71e575744f1d23f79741450254660442785f45a0797212852ee5199ef12eed98"}, - {file = "opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09a332b50488e2dda866a6c5573ee192fe3583239fb26ff2f7f9ceb0bc119ea6"}, - {file = "opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ace140fc6d647fbe1c692bcb2abce768973491222c067c131d80957c595b71f"}, - {file = "opencv_python-4.10.0.84-cp37-abi3-win32.whl", hash = "sha256:2db02bb7e50b703f0a2d50c50ced72e95c574e1e5a0bb35a8a86d0b35c98c236"}, - {file = "opencv_python-4.10.0.84-cp37-abi3-win_amd64.whl", hash = "sha256:32dbbd94c26f611dc5cc6979e6b7aa1f55a64d6b463cc1dcd3c95505a63e48fe"}, -] - -[package.dependencies] -numpy = [ - {version = ">=1.21.2", markers = "python_version >= \"3.10\""}, - {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""}, - {version = ">=1.23.5", markers = "python_version >= \"3.11\""}, - {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, - {version = ">=1.19.3", markers = "python_version >= \"3.6\" and platform_system == \"Linux\" and platform_machine == \"aarch64\" or python_version >= \"3.9\""}, - {version = ">=1.17.0", markers = "python_version >= \"3.7\""}, - {version = ">=1.17.3", markers = "python_version >= \"3.8\""}, -] - [[package]] name = "packaging" version = "24.1" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -688,7 +558,6 @@ files = [ name = "pluggy" version = "1.5.0" description = "plugin and hook calling mechanisms for python" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -704,7 +573,6 @@ testing = ["pytest", "pytest-benchmark"] name = "pydantic" version = "2.9.1" description = "Data validation using Python type hints" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -728,7 +596,6 @@ timezone = ["tzdata"] name = "pydantic-core" version = "2.23.3" description = "Core functionality for Pydantic validation and serialization" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -830,7 +697,6 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" name = "pygments" version = "2.18.0" description = "Pygments is a syntax highlighting package written in Python." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -845,7 +711,6 @@ windows-terminal = ["colorama (>=0.4.6)"] name = "pytest" version = "8.3.3" description = "pytest: simple powerful testing with Python" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -866,7 +731,6 @@ dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments name = "pytest-asyncio" version = "0.24.0" description = "Pytest support for asyncio" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -885,7 +749,6 @@ testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] name = "python-dotenv" version = "1.0.1" description = "Read key-value pairs from a .env file and set them as environment variables" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -900,7 +763,6 @@ cli = ["click (>=5.0)"] name = "python-multipart" version = "0.0.9" description = "A streaming multipart parser for Python" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -915,7 +777,6 @@ dev = ["atomicwrites (==1.4.1)", "attrs (==23.2.0)", "coverage (==7.4.1)", "hatc name = "pyyaml" version = "6.0.2" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -978,7 +839,6 @@ files = [ name = "requests" version = "2.32.3" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1000,7 +860,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "rich" version = "13.8.1" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -1019,7 +878,6 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] name = "shellingham" version = "1.5.4" description = "Tool to Detect Surrounding Shell" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1031,7 +889,6 @@ files = [ name = "sniffio" version = "1.3.1" description = "Sniff out which async library your code is running under" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1043,7 +900,6 @@ files = [ name = "starlette" version = "0.37.2" description = "The little ASGI library that shines." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1061,7 +917,6 @@ full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.7 name = "typer" version = "0.12.5" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1079,7 +934,6 @@ typing-extensions = ">=3.7.4.3" name = "typing-extensions" version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1091,7 +945,6 @@ files = [ name = "urllib3" version = "2.2.3" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1109,7 +962,6 @@ zstd = ["zstandard (>=0.18.0)"] name = "uvicorn" version = "0.30.6" description = "The lightning-fast ASGI server." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1124,7 +976,7 @@ h11 = ">=0.8" httptools = {version = ">=0.5.0", optional = true, markers = "extra == \"standard\""} python-dotenv = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} pyyaml = {version = ">=5.1", optional = true, markers = "extra == \"standard\""} -uvloop = {version = ">=0.14.0,<0.15.0 || >0.15.0,<0.15.1 || >0.15.1", optional = true, markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" and extra == \"standard\""} +uvloop = {version = ">=0.14.0,<0.15.0 || >0.15.0,<0.15.1 || >0.15.1", optional = true, markers = "(sys_platform != \"win32\" and sys_platform != \"cygwin\") and platform_python_implementation != \"PyPy\" and extra == \"standard\""} watchfiles = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} websockets = {version = ">=10.4", optional = true, markers = "extra == \"standard\""} @@ -1135,7 +987,6 @@ standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", name = "uvloop" version = "0.20.0" description = "Fast implementation of asyncio event loop on top of libuv" -category = "main" optional = false python-versions = ">=3.8.0" files = [ @@ -1180,7 +1031,6 @@ test = ["Cython (>=0.29.36,<0.30.0)", "aiohttp (==3.9.0b0)", "aiohttp (>=3.8.1)" name = "watchfiles" version = "0.24.0" description = "Simple, modern and high performance file watching and code reload in python." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1276,7 +1126,6 @@ anyio = ">=3.0.0" name = "websockets" version = "13.0.1" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1372,7 +1221,6 @@ files = [ name = "wrapt" version = "1.16.0" description = "Module for decorators, wrappers and monkey patching." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1450,5 +1298,5 @@ files = [ [metadata] lock-version = "2.0" -python-versions = "^3.12" -content-hash = "36519f0ca589faeaac9710ec3f1b055905d9560673ec53611abacbe259990b53" +python-versions = "^3.11" +content-hash = "8e6c46b679a81d033850227859c9754ca5e58b3b10d1e13383bf0576bb34da5c" diff --git a/pyproject.toml b/pyproject.toml index 9fed117..4bc5750 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,19 +2,18 @@ name = "byparr" version = "0.1.0" description = "" -# package-mode = false +package-mode = false authors = ["Your Name "] readme = "README.md" [tool.poetry.dependencies] -python = "^3.12" +python = "^3.11" pytest = "^8.3.1" fastapi = "^0.111.1" nodriver = "^0.34" requests = "^2.32.3" httpx = "^0.27.2" pytest-asyncio = "^0.24.0" -nn-mouse = "^1.0.1" [build-system] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c16e9f8..0000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -fastapi==0.111.1 -nodriver==0.34 -requests==2.32.3 -httpx==0.27.2 -nn-mouse==1.0.1 diff --git a/src/utils/__init__.py b/src/utils/__init__.py index 932f32e..e0dde19 100644 --- a/src/utils/__init__.py +++ b/src/utils/__init__.py @@ -1,6 +1,6 @@ import logging -from utils.consts import LOG_LEVEL +from src.utils.consts import LOG_LEVEL logger = logging.getLogger("uvicorn.error") logger.setLevel(LOG_LEVEL) diff --git a/src/utils/browser.py b/src/utils/browser.py index 49a1924..1f696b6 100644 --- a/src/utils/browser.py +++ b/src/utils/browser.py @@ -1,13 +1,13 @@ import asyncio -import typing -import random import nodriver as webdriver -from nodriver.core.element import Element, Position -from nn_mouse import get_path +from nodriver.core.element import Element -from . import logger -from .consts import CHALLENGE_TITLES +from src.utils import logger +from src.utils.consts import CHALLENGE_TITLES +from src.utils.extentions import download_extentions + +downloaded_extentions = download_extentions() async def new_browser(): @@ -25,6 +25,7 @@ async def new_browser(): """ config: webdriver.Config = webdriver.Config() config.sandbox = False + config.add_argument(f"--load-extension={','.join(downloaded_extentions)}") return await webdriver.start(config=config) @@ -64,12 +65,6 @@ async def bypass_cloudflare(page: webdriver.Tab): if not challenged: logger.info("Found challenge") challenged = True - elem = None - - # get the size of the page - html_elem = await page.select("html") - html_pos = await html_elem.get_position() - try: elem = await page.find( "Verify you are human by completing the action below.", @@ -79,30 +74,31 @@ async def bypass_cloudflare(page: webdriver.Tab): except asyncio.TimeoutError: if page.target.title not in CHALLENGE_TITLES: return challenged + raise if elem is None: - logger.debug("Couldn't find the title, trying other method...") + logger.debug("Couldn't find the title, trying again") continue - if not isinstance(elem, Element): - logger.fatal("Element is a string, please report this to Byparr dev") - raise InvalidElementError - - elem = await page.find("input") elem = elem.parent # Get the element containing the shadow root + for _ in range(3): + if elem is not None: + elem = get_first_div(elem) + else: + raise InvalidElementError if isinstance(elem, Element) and elem.shadow_roots: inner_elem = Element(elem.shadow_roots[0], page, elem.tree).children[0] if isinstance(inner_elem, Element): logger.debug("Clicking element") - await mouse_click(inner_elem, html_pos=html_pos) + await inner_elem.mouse_click() else: - logger.warning( + logger.warn( "Element is a string, please report this to Byparr dev" ) # I really hope this never happens else: - logger.warning("Coulnd't find checkbox, trying again...") + logger.warn("Coulnd't find checkbox, trying again...") def get_first_div(elem): @@ -124,85 +120,5 @@ def get_first_div(elem): raise InvalidElementError -def clamp(n: int, smallest: int, largest: int): - return max(smallest, min(n, largest)) - - -async def mouse_click( - elm, - button: str = "left", - buttons: typing.Optional[int] = 1, - modifiers: typing.Optional[int] = 0, - hold: bool = False, - html_pos: typing.Optional[Position] = None, - _until_event: typing.Optional[type] = None, -): - """native click (on element) . note: this likely does not work atm, use click() instead - :param button: str (default = "left") - :param buttons: which button (default 1 = left) - :param modifiers: *(Optional)* Bit field representing pressed modifier keys. - Alt=1, Ctrl=2, Meta/Command=4, Shift=8 (default: 0). - :param _until_event: internal. event to wait for before returning - :return: - """ - try: - pos = await elm.get_position() - except AttributeError: - return - if not pos: - logger.warning("could not calculate box model for %s", self) - return - oc = pos.center - center = (oc[0] + random.randint(-10, 10), oc[1] + random.randint(-10, 10)) - - html_height = 500 - html_width = 500 - if html_pos: - html_height = html_pos.height - html_width = html_pos.width - - start_x = random.randint(0, html_width) - start_y = random.randint(0, html_height) - path = get_path(start_x, start_y, center[0], center[1], html_width, html_height) - - for (x, y, t) in path: - x = clamp(int(x), 0, html_width) - y = clamp(int(y), 0, html_height) - await elm._tab.send(webdriver.cdp.input_.dispatch_mouse_event("mousemove", x, y)) - await asyncio.sleep(float(t)) - - logger.debug("clicking on location %.2f, %.2f" % center) - await asyncio.sleep(random.uniform(0.05, 0.3)) - - await elm._tab.send( - webdriver.cdp.input_.dispatch_mouse_event( - "mousePressed", - x=center[0], - y=center[1], - modifiers=modifiers, - button=webdriver.cdp.input_.MouseButton(button), - buttons=buttons, - click_count=1, - ) - ) - await asyncio.sleep(random.uniform(0.02, 0.06)) - await elm._tab.send( - webdriver.cdp.input_.dispatch_mouse_event( - "mouseReleased", - x=center[0], - y=center[1], - modifiers=modifiers, - button=webdriver.cdp.input_.MouseButton(button), - buttons=buttons, - click_count=1, - ) - ) - # Flash the checkbox for testing - try: - await elm.flash() - except: # noqa - pass - - class InvalidElementError(Exception): pass diff --git a/src/utils/extentions.py b/src/utils/extentions.py index 0047966..bdfdcdd 100644 --- a/src/utils/extentions.py +++ b/src/utils/extentions.py @@ -8,10 +8,10 @@ from zipfile import ZipFile import httpx import requests -from ..models.github import GithubResponse -from ..models.requests import NoChromeExtentionError -from . import logger -from .consts import EXTENTION_REPOSITIORIES, EXTENTIONS_PATH, GITHUB_WEBSITES +from src.models.github import GithubResponse +from src.models.requests import NoChromeExtentionError +from src.utils import logger +from src.utils.consts import EXTENTION_REPOSITIORIES, EXTENTIONS_PATH, GITHUB_WEBSITES def get_latest_github_chrome_release(url: str): diff --git a/tests/main_test.py b/tests/main_test.py index f2c62b4..421fb87 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -1,27 +1,18 @@ -from http import HTTPStatus - import pytest -from starlette.testclient import TestClient -from main import app +from main import read_item from src.models.requests import LinkRequest -client = TestClient(app) - test_websites = [ "https://ext.to/", "https://btmet.com/", - # "https://extratorrent.st/", # github is blocking these - # "https://idope.se/", # github is blocking these + "https://extratorrent.st/", + "https://idope.se/", ] +pytest_plugins = ("pytest_asyncio",) @pytest.mark.parametrize("website", test_websites) -def test_bypass(website: str): - response = client.post( - "/v1", - json=LinkRequest( - url=website, maxTimeout=60 * len(test_websites), cmd="request.get" - ).model_dump(), - ) - assert response.status_code == HTTPStatus.OK +@pytest.mark.asyncio +async def test_bypass(website: str): + await read_item(LinkRequest(url=website, maxTimeout=5, cmd=""))