diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e01dcc4..1da39e5 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.0 +current_version = 0.1.1 commit = True tag = True diff --git a/deploy/Dockerfile b/deploy/Dockerfile index b22b23d..c997332 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -9,8 +9,7 @@ FROM python:3.10 ARG TARGETPLATFORM COPY --from=0 /build/dist /install -RUN pip install -- /install/*.whl gunicorn honcho && \ - rm -rf ~/.cache/pip +RUN pip install -- /install/*.whl gunicorn honcho # ffmpeg static source (https://johnvansickle.com/ffmpeg/) RUN set -e; \ diff --git a/pyproject.toml b/pyproject.toml index 96a2355..de2bdfa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ucast" -version = "0.2.0" +version = "0.1.1" description = "YouTube to Podcast converter" authors = ["Theta-Dev "] packages = [ diff --git a/ucast/__init__.py b/ucast/__init__.py index 7d8805f..93b1944 100644 --- a/ucast/__init__.py +++ b/ucast/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.2.0" +__version__ = "0.1.1" def template_context(request): diff --git a/ucast/queue.py b/ucast/queue.py index fbdf55e..5e56343 100644 --- a/ucast/queue.py +++ b/ucast/queue.py @@ -4,7 +4,6 @@ import rq_scheduler from django.conf import settings from rq import registry -from ucast.models import Video from ucast.service import util @@ -34,7 +33,8 @@ def get_worker(**kwargs) -> rq.Worker: def enqueue(f, *args, **kwargs) -> rq.job.Job: queue = get_queue() - return queue.enqueue(f, *args, **kwargs) + # return queue.enqueue(f, *args, **kwargs) + return queue.enqueue_call(f, args, kwargs) def get_statistics() -> dict: @@ -90,19 +90,3 @@ def get_statistics() -> dict: def get_failed_job_registry(): queue = get_queue() return registry.FailedJobRegistry(queue.name, queue.connection) - - -def get_downloading_videos(): - queue = get_queue() - videos = {} - - for job in queue.jobs: - if ( - job.func_name == "ucast.tasks.download.download_video" - and job.args - and isinstance(job.args[0], Video) - ): - video = job.args[0] - videos[video.id] = video - - return list(videos.values()) diff --git a/ucast/service/youtube.py b/ucast/service/youtube.py index 670cc46..1ae4479 100644 --- a/ucast/service/youtube.py +++ b/ucast/service/youtube.py @@ -125,19 +125,7 @@ def download_thumbnail(vinfo: VideoDetails, download_path: Path): def get_video_details(video_id: str) -> VideoDetails: - """ - Get the details of a YouTube video without downloading it. - - :param video_id: YouTube video ID - :return: VideoDetails - """ - cache = storage.Cache() - - ydl_params = { - "cachedir": str(cache.dir_ytdlp_cache), - } - - with YoutubeDL(ydl_params) as ydl: + with YoutubeDL() as ydl: info = ydl.extract_info(video_id, download=False) return VideoDetails.from_vinfo(info) diff --git a/ucast/tasks/download.py b/ucast/tasks/download.py index be14ddb..a91ec4d 100644 --- a/ucast/tasks/download.py +++ b/ucast/tasks/download.py @@ -10,13 +10,6 @@ from ucast.service import controller, cover, storage, util, videoutil, youtube def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): - # Use Redis to ensure the same video is not processed multiple times - redis = queue.get_redis_connection() - lock_key = f"ucast:lock_load_video:{vid.id}" - - if not redis.set(lock_key, "1", 120, nx=True): - return - # Create video object if it does not exist try: video = Video.objects.get(video_id=vid.id) @@ -25,7 +18,6 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): # Dont load active livestreams if details.is_currently_live: - redis.delete(lock_key) return slug = Video.get_new_slug( @@ -52,8 +44,6 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): ): queue.enqueue(download_video, video) - redis.delete(lock_key) - def download_video(video: Video): """ diff --git a/ucast/templates/bulma/base.html b/ucast/templates/bulma/base.html index c20531b..431b8ce 100644 --- a/ucast/templates/bulma/base.html +++ b/ucast/templates/bulma/base.html @@ -27,8 +27,8 @@
- - Downloads + + Errors {% url 'login' as login_url %} {% url 'logout' as logout_url %} diff --git a/ucast/templates/ucast/download_errors.html b/ucast/templates/ucast/download_errors.html new file mode 100644 index 0000000..a337d62 --- /dev/null +++ b/ucast/templates/ucast/download_errors.html @@ -0,0 +1,50 @@ +{% extends 'base.html' %} + +{% block title %}ucast - Errors{% endblock %} + +{% block content %} +
+
+ Download errors +
+
+ + {% if jobs %} +
+
+ {% csrf_token %} + +
+
+ + + + + + + + + + + + {% for job in jobs %} + + + + + + + {% endfor %} + +
IDFunctionDetailsRequeue
{{ job.id }}{{ job.func_name }}Details +
+ {% csrf_token %} + + +
+
+ {% else %} +

No download errors

+ {% endif %} + +{% endblock content %} diff --git a/ucast/templates/ucast/downloads.html b/ucast/templates/ucast/downloads.html deleted file mode 100644 index 73f24e5..0000000 --- a/ucast/templates/ucast/downloads.html +++ /dev/null @@ -1,81 +0,0 @@ -{% extends 'base.html' %} - -{% block title %}ucast - Errors{% endblock %} - -{% block content %} -
-
- Downloading - {% if downloading_videos %}({{ downloading_videos.paginator.count }}){% endif %} - -
-
- -
- {% if downloading_videos %} - - - - - - - - - - - {% include "ucast/downloads_items.html" %} - {% else %} -

Not downloading any videos

- {% endif %} - -
Video-IDTitleChannelPublished
- -
- -
-
- Download errors -
-
- -
- {% if failed_jobs %} -
-
- {% csrf_token %} - -
-
- - - - - - - - - - - - {% for job in failed_jobs %} - - - - - - - {% endfor %} - -
IDFunctionDetailsRequeue
{{ job.id }}{{ job.func_name }}Details -
- {% csrf_token %} - - -
-
- {% else %} -

No download errors

- {% endif %} -
- -{% endblock content %} diff --git a/ucast/templates/ucast/downloads_items.html b/ucast/templates/ucast/downloads_items.html deleted file mode 100644 index 9dd1658..0000000 --- a/ucast/templates/ucast/downloads_items.html +++ /dev/null @@ -1,13 +0,0 @@ -{% for video in downloading_videos %} - - {{ video.video_id }} - {{ video.title }} - {{ video.channel.name }} - - {{ video.published|date:"SHORT_DATE_FORMAT" }} - -{% endfor %} diff --git a/ucast/templates/ucast/videos.html b/ucast/templates/ucast/videos.html index d43569c..05bcdf8 100644 --- a/ucast/templates/ucast/videos.html +++ b/ucast/templates/ucast/videos.html @@ -13,7 +13,7 @@   {{ channel.subscribers }}   {{ videos.paginator.count }} + class="fas fa-video">  {{ videos|length }}   {{ channel.download_size|filesizeformat }} Path: yield tmpdir -@pytest.fixture -def mock_redis(mocker) -> FakeRedis: - redis = FakeRedis() - mocker.patch.object(queue, "get_redis_connection") - queue.get_redis_connection.return_value = redis - return redis - - @pytest.fixture def rq_queue(mocker) -> rq.Queue: test_queue = rq.Queue(is_async=False, connection=FakeRedis()) diff --git a/ucast/tests/tasks/test_download.py b/ucast/tests/tasks/test_download.py index dc6fd6c..919156d 100644 --- a/ucast/tests/tasks/test_download.py +++ b/ucast/tests/tasks/test_download.py @@ -30,7 +30,7 @@ def test_download_video(download_dir, rq_queue): @pytest.mark.django_db def test_update_channel( - download_dir, rq_queue, mock_redis, mock_get_video_details, mock_download_audio + download_dir, rq_queue, mock_get_video_details, mock_download_audio ): # Remove 2 videos from the database so they can be imported Video.objects.get(video_id="ZPxEr4YdWt8").delete() diff --git a/ucast/urls.py b/ucast/urls.py index f4f4f3d..c495bca 100644 --- a/ucast/urls.py +++ b/ucast/urls.py @@ -17,18 +17,16 @@ urlpatterns = [ views.channel_download, name="channel_download", ), - path("downloads", views.downloads, name="downloads"), + path("errors", views.download_errors, name="download_errors"), path( - "downloads/requeue", - views.download_errors_requeue, - name="download_errors_requeue", + "errors/requeue", views.download_errors_requeue, name="download_errors_requeue" ), path( - "downloads/requeue_all", + "errors/requeue_all", views.download_errors_requeue_all, name="download_errors_requeue_all", ), - path("downloads/error/", views.error_details, name="error_details"), + path("errors/", views.error_details, name="error_details"), path("feed/", views.podcast_feed, name="feed"), path("files/audio//", views.audio), path("files/cover//", views.cover), diff --git a/ucast/views.py b/ucast/views.py index 182ba2e..f6786c2 100644 --- a/ucast/views.py +++ b/ucast/views.py @@ -154,27 +154,12 @@ def channel_download(request: http.HttpRequest, channel: str): @login_required -def downloads(request: http.HttpRequest): +def download_errors(request: http.HttpRequest): freg = queue.get_failed_job_registry() ids = freg.get_job_ids(0, 50) - failed_jobs = freg.job_class.fetch_many(ids, freg.connection, freg.serializer) + jobs = freg.job_class.fetch_many(ids, freg.connection, freg.serializer) - page_number = request.GET.get("page") - downloading_videos = queue.get_downloading_videos() - downloading_videos_p = Paginator(downloading_videos, 100) - - template_name = "ucast/downloads.html" - if request.htmx or request.GET.get("items"): - template_name = "ucast/downloads_items.html" - - return render( - request, - template_name, - { - "failed_jobs": failed_jobs, - "downloading_videos": downloading_videos_p.get_page(page_number), - }, - ) + return render(request, "ucast/download_errors.html", {"jobs": jobs}) @login_required @@ -193,7 +178,7 @@ def download_errors_requeue(request: http.HttpRequest): freg = queue.get_failed_job_registry() freg.requeue(str(form.cleaned_data["id"])) - return http.HttpResponseRedirect(reverse(downloads)) + return http.HttpResponseRedirect(reverse(download_errors)) @login_required @@ -202,7 +187,7 @@ def download_errors_requeue_all(request: http.HttpRequest): for job_id in freg.get_job_ids(): freg.requeue(job_id) - return http.HttpResponseRedirect(reverse(downloads)) + return http.HttpResponseRedirect(reverse(download_errors)) def _channel_file(channel: str, get_file: Callable[[storage.ChannelFolder], Path]):