From 0d34a96227a7dc354edff90ed02709afc2b9a263 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 27 Jun 2022 12:42:59 +0200 Subject: [PATCH 1/2] add downloads view fix yt-dlp cache not writable fix video count --- deploy/Dockerfile | 3 +- ucast/queue.py | 20 +++++- ucast/service/youtube.py | 14 +++- ucast/tasks/download.py | 10 +++ ucast/templates/bulma/base.html | 4 +- ucast/templates/ucast/download_errors.html | 50 ------------- ucast/templates/ucast/downloads.html | 81 ++++++++++++++++++++++ ucast/templates/ucast/downloads_items.html | 13 ++++ ucast/templates/ucast/videos.html | 2 +- ucast/tests/conftest.py | 8 +++ ucast/tests/tasks/test_download.py | 2 +- ucast/urls.py | 10 +-- ucast/views.py | 25 +++++-- 13 files changed, 175 insertions(+), 67 deletions(-) delete mode 100644 ucast/templates/ucast/download_errors.html create mode 100644 ucast/templates/ucast/downloads.html create mode 100644 ucast/templates/ucast/downloads_items.html diff --git a/deploy/Dockerfile b/deploy/Dockerfile index c997332..b22b23d 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -9,7 +9,8 @@ FROM python:3.10 ARG TARGETPLATFORM COPY --from=0 /build/dist /install -RUN pip install -- /install/*.whl gunicorn honcho +RUN pip install -- /install/*.whl gunicorn honcho && \ + rm -rf ~/.cache/pip # ffmpeg static source (https://johnvansickle.com/ffmpeg/) RUN set -e; \ diff --git a/ucast/queue.py b/ucast/queue.py index 5e56343..fbdf55e 100644 --- a/ucast/queue.py +++ b/ucast/queue.py @@ -4,6 +4,7 @@ import rq_scheduler from django.conf import settings from rq import registry +from ucast.models import Video from ucast.service import util @@ -33,8 +34,7 @@ def get_worker(**kwargs) -> rq.Worker: def enqueue(f, *args, **kwargs) -> rq.job.Job: queue = get_queue() - # return queue.enqueue(f, *args, **kwargs) - return queue.enqueue_call(f, args, kwargs) + return queue.enqueue(f, *args, **kwargs) def get_statistics() -> dict: @@ -90,3 +90,19 @@ def get_statistics() -> dict: def get_failed_job_registry(): queue = get_queue() return registry.FailedJobRegistry(queue.name, queue.connection) + + +def get_downloading_videos(): + queue = get_queue() + videos = {} + + for job in queue.jobs: + if ( + job.func_name == "ucast.tasks.download.download_video" + and job.args + and isinstance(job.args[0], Video) + ): + video = job.args[0] + videos[video.id] = video + + return list(videos.values()) diff --git a/ucast/service/youtube.py b/ucast/service/youtube.py index 1ae4479..670cc46 100644 --- a/ucast/service/youtube.py +++ b/ucast/service/youtube.py @@ -125,7 +125,19 @@ def download_thumbnail(vinfo: VideoDetails, download_path: Path): def get_video_details(video_id: str) -> VideoDetails: - with YoutubeDL() as ydl: + """ + Get the details of a YouTube video without downloading it. + + :param video_id: YouTube video ID + :return: VideoDetails + """ + cache = storage.Cache() + + ydl_params = { + "cachedir": str(cache.dir_ytdlp_cache), + } + + with YoutubeDL(ydl_params) as ydl: info = ydl.extract_info(video_id, download=False) return VideoDetails.from_vinfo(info) diff --git a/ucast/tasks/download.py b/ucast/tasks/download.py index a91ec4d..be14ddb 100644 --- a/ucast/tasks/download.py +++ b/ucast/tasks/download.py @@ -10,6 +10,13 @@ from ucast.service import controller, cover, storage, util, videoutil, youtube def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): + # Use Redis to ensure the same video is not processed multiple times + redis = queue.get_redis_connection() + lock_key = f"ucast:lock_load_video:{vid.id}" + + if not redis.set(lock_key, "1", 120, nx=True): + return + # Create video object if it does not exist try: video = Video.objects.get(video_id=vid.id) @@ -18,6 +25,7 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): # Dont load active livestreams if details.is_currently_live: + redis.delete(lock_key) return slug = Video.get_new_slug( @@ -44,6 +52,8 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): ): queue.enqueue(download_video, video) + redis.delete(lock_key) + def download_video(video: Video): """ diff --git a/ucast/templates/bulma/base.html b/ucast/templates/bulma/base.html index 431b8ce..c20531b 100644 --- a/ucast/templates/bulma/base.html +++ b/ucast/templates/bulma/base.html @@ -27,8 +27,8 @@
- - Errors + + Downloads {% url 'login' as login_url %} {% url 'logout' as logout_url %} diff --git a/ucast/templates/ucast/download_errors.html b/ucast/templates/ucast/download_errors.html deleted file mode 100644 index a337d62..0000000 --- a/ucast/templates/ucast/download_errors.html +++ /dev/null @@ -1,50 +0,0 @@ -{% extends 'base.html' %} - -{% block title %}ucast - Errors{% endblock %} - -{% block content %} -
-
- Download errors -
-
- - {% if jobs %} -
-
- {% csrf_token %} - -
-
- - - - - - - - - - - - {% for job in jobs %} - - - - - - - {% endfor %} - -
IDFunctionDetailsRequeue
{{ job.id }}{{ job.func_name }}Details -
- {% csrf_token %} - - -
-
- {% else %} -

No download errors

- {% endif %} - -{% endblock content %} diff --git a/ucast/templates/ucast/downloads.html b/ucast/templates/ucast/downloads.html new file mode 100644 index 0000000..73f24e5 --- /dev/null +++ b/ucast/templates/ucast/downloads.html @@ -0,0 +1,81 @@ +{% extends 'base.html' %} + +{% block title %}ucast - Errors{% endblock %} + +{% block content %} +
+
+ Downloading + {% if downloading_videos %}({{ downloading_videos.paginator.count }}){% endif %} + +
+
+ +
+ {% if downloading_videos %} + + + + + + + + + + + {% include "ucast/downloads_items.html" %} + {% else %} +

Not downloading any videos

+ {% endif %} + +
Video-IDTitleChannelPublished
+ +
+ +
+
+ Download errors +
+
+ +
+ {% if failed_jobs %} +
+
+ {% csrf_token %} + +
+
+ + + + + + + + + + + + {% for job in failed_jobs %} + + + + + + + {% endfor %} + +
IDFunctionDetailsRequeue
{{ job.id }}{{ job.func_name }}Details +
+ {% csrf_token %} + + +
+
+ {% else %} +

No download errors

+ {% endif %} +
+ +{% endblock content %} diff --git a/ucast/templates/ucast/downloads_items.html b/ucast/templates/ucast/downloads_items.html new file mode 100644 index 0000000..9dd1658 --- /dev/null +++ b/ucast/templates/ucast/downloads_items.html @@ -0,0 +1,13 @@ +{% for video in downloading_videos %} + + {{ video.video_id }} + {{ video.title }} + {{ video.channel.name }} + + {{ video.published|date:"SHORT_DATE_FORMAT" }} + +{% endfor %} diff --git a/ucast/templates/ucast/videos.html b/ucast/templates/ucast/videos.html index 05bcdf8..d43569c 100644 --- a/ucast/templates/ucast/videos.html +++ b/ucast/templates/ucast/videos.html @@ -13,7 +13,7 @@   {{ channel.subscribers }}   {{ videos|length }} + class="fas fa-video">  {{ videos.paginator.count }}   {{ channel.download_size|filesizeformat }} Path: yield tmpdir +@pytest.fixture +def mock_redis(mocker) -> FakeRedis: + redis = FakeRedis() + mocker.patch.object(queue, "get_redis_connection") + queue.get_redis_connection.return_value = redis + return redis + + @pytest.fixture def rq_queue(mocker) -> rq.Queue: test_queue = rq.Queue(is_async=False, connection=FakeRedis()) diff --git a/ucast/tests/tasks/test_download.py b/ucast/tests/tasks/test_download.py index 919156d..dc6fd6c 100644 --- a/ucast/tests/tasks/test_download.py +++ b/ucast/tests/tasks/test_download.py @@ -30,7 +30,7 @@ def test_download_video(download_dir, rq_queue): @pytest.mark.django_db def test_update_channel( - download_dir, rq_queue, mock_get_video_details, mock_download_audio + download_dir, rq_queue, mock_redis, mock_get_video_details, mock_download_audio ): # Remove 2 videos from the database so they can be imported Video.objects.get(video_id="ZPxEr4YdWt8").delete() diff --git a/ucast/urls.py b/ucast/urls.py index c495bca..f4f4f3d 100644 --- a/ucast/urls.py +++ b/ucast/urls.py @@ -17,16 +17,18 @@ urlpatterns = [ views.channel_download, name="channel_download", ), - path("errors", views.download_errors, name="download_errors"), + path("downloads", views.downloads, name="downloads"), path( - "errors/requeue", views.download_errors_requeue, name="download_errors_requeue" + "downloads/requeue", + views.download_errors_requeue, + name="download_errors_requeue", ), path( - "errors/requeue_all", + "downloads/requeue_all", views.download_errors_requeue_all, name="download_errors_requeue_all", ), - path("errors/", views.error_details, name="error_details"), + path("downloads/error/", views.error_details, name="error_details"), path("feed/", views.podcast_feed, name="feed"), path("files/audio//", views.audio), path("files/cover//", views.cover), diff --git a/ucast/views.py b/ucast/views.py index f6786c2..182ba2e 100644 --- a/ucast/views.py +++ b/ucast/views.py @@ -154,12 +154,27 @@ def channel_download(request: http.HttpRequest, channel: str): @login_required -def download_errors(request: http.HttpRequest): +def downloads(request: http.HttpRequest): freg = queue.get_failed_job_registry() ids = freg.get_job_ids(0, 50) - jobs = freg.job_class.fetch_many(ids, freg.connection, freg.serializer) + failed_jobs = freg.job_class.fetch_many(ids, freg.connection, freg.serializer) - return render(request, "ucast/download_errors.html", {"jobs": jobs}) + page_number = request.GET.get("page") + downloading_videos = queue.get_downloading_videos() + downloading_videos_p = Paginator(downloading_videos, 100) + + template_name = "ucast/downloads.html" + if request.htmx or request.GET.get("items"): + template_name = "ucast/downloads_items.html" + + return render( + request, + template_name, + { + "failed_jobs": failed_jobs, + "downloading_videos": downloading_videos_p.get_page(page_number), + }, + ) @login_required @@ -178,7 +193,7 @@ def download_errors_requeue(request: http.HttpRequest): freg = queue.get_failed_job_registry() freg.requeue(str(form.cleaned_data["id"])) - return http.HttpResponseRedirect(reverse(download_errors)) + return http.HttpResponseRedirect(reverse(downloads)) @login_required @@ -187,7 +202,7 @@ def download_errors_requeue_all(request: http.HttpRequest): for job_id in freg.get_job_ids(): freg.requeue(job_id) - return http.HttpResponseRedirect(reverse(download_errors)) + return http.HttpResponseRedirect(reverse(downloads)) def _channel_file(channel: str, get_file: Callable[[storage.ChannelFolder], Path]): From ede37ebe1780ed1cc361ace5ad1035a5d6bbdb0a Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 27 Jun 2022 12:43:06 +0200 Subject: [PATCH 2/2] =?UTF-8?q?Bump=20version:=200.1.1=20=E2=86=92=200.2.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- pyproject.toml | 2 +- ucast/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1da39e5..e01dcc4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.1 +current_version = 0.2.0 commit = True tag = True diff --git a/pyproject.toml b/pyproject.toml index de2bdfa..96a2355 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ucast" -version = "0.1.1" +version = "0.2.0" description = "YouTube to Podcast converter" authors = ["Theta-Dev "] packages = [ diff --git a/ucast/__init__.py b/ucast/__init__.py index 93b1944..7d8805f 100644 --- a/ucast/__init__.py +++ b/ucast/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.1.1" +__version__ = "0.2.0" def template_context(request):