From 0d34a96227a7dc354edff90ed02709afc2b9a263 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 27 Jun 2022 12:42:59 +0200 Subject: [PATCH 01/36] add downloads view fix yt-dlp cache not writable fix video count --- deploy/Dockerfile | 3 +- ucast/queue.py | 20 +++++- ucast/service/youtube.py | 14 +++- ucast/tasks/download.py | 10 +++ ucast/templates/bulma/base.html | 4 +- ucast/templates/ucast/download_errors.html | 50 ------------- ucast/templates/ucast/downloads.html | 81 ++++++++++++++++++++++ ucast/templates/ucast/downloads_items.html | 13 ++++ ucast/templates/ucast/videos.html | 2 +- ucast/tests/conftest.py | 8 +++ ucast/tests/tasks/test_download.py | 2 +- ucast/urls.py | 10 +-- ucast/views.py | 25 +++++-- 13 files changed, 175 insertions(+), 67 deletions(-) delete mode 100644 ucast/templates/ucast/download_errors.html create mode 100644 ucast/templates/ucast/downloads.html create mode 100644 ucast/templates/ucast/downloads_items.html diff --git a/deploy/Dockerfile b/deploy/Dockerfile index c997332..b22b23d 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -9,7 +9,8 @@ FROM python:3.10 ARG TARGETPLATFORM COPY --from=0 /build/dist /install -RUN pip install -- /install/*.whl gunicorn honcho +RUN pip install -- /install/*.whl gunicorn honcho && \ + rm -rf ~/.cache/pip # ffmpeg static source (https://johnvansickle.com/ffmpeg/) RUN set -e; \ diff --git a/ucast/queue.py b/ucast/queue.py index 5e56343..fbdf55e 100644 --- a/ucast/queue.py +++ b/ucast/queue.py @@ -4,6 +4,7 @@ import rq_scheduler from django.conf import settings from rq import registry +from ucast.models import Video from ucast.service import util @@ -33,8 +34,7 @@ def get_worker(**kwargs) -> rq.Worker: def enqueue(f, *args, **kwargs) -> rq.job.Job: queue = get_queue() - # return queue.enqueue(f, *args, **kwargs) - return queue.enqueue_call(f, args, kwargs) + return queue.enqueue(f, *args, **kwargs) def get_statistics() -> dict: @@ -90,3 +90,19 @@ def get_statistics() -> dict: def get_failed_job_registry(): queue = get_queue() return registry.FailedJobRegistry(queue.name, queue.connection) + + +def get_downloading_videos(): + queue = get_queue() + videos = {} + + for job in queue.jobs: + if ( + job.func_name == "ucast.tasks.download.download_video" + and job.args + and isinstance(job.args[0], Video) + ): + video = job.args[0] + videos[video.id] = video + + return list(videos.values()) diff --git a/ucast/service/youtube.py b/ucast/service/youtube.py index 1ae4479..670cc46 100644 --- a/ucast/service/youtube.py +++ b/ucast/service/youtube.py @@ -125,7 +125,19 @@ def download_thumbnail(vinfo: VideoDetails, download_path: Path): def get_video_details(video_id: str) -> VideoDetails: - with YoutubeDL() as ydl: + """ + Get the details of a YouTube video without downloading it. + + :param video_id: YouTube video ID + :return: VideoDetails + """ + cache = storage.Cache() + + ydl_params = { + "cachedir": str(cache.dir_ytdlp_cache), + } + + with YoutubeDL(ydl_params) as ydl: info = ydl.extract_info(video_id, download=False) return VideoDetails.from_vinfo(info) diff --git a/ucast/tasks/download.py b/ucast/tasks/download.py index a91ec4d..be14ddb 100644 --- a/ucast/tasks/download.py +++ b/ucast/tasks/download.py @@ -10,6 +10,13 @@ from ucast.service import controller, cover, storage, util, videoutil, youtube def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): + # Use Redis to ensure the same video is not processed multiple times + redis = queue.get_redis_connection() + lock_key = f"ucast:lock_load_video:{vid.id}" + + if not redis.set(lock_key, "1", 120, nx=True): + return + # Create video object if it does not exist try: video = Video.objects.get(video_id=vid.id) @@ -18,6 +25,7 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): # Dont load active livestreams if details.is_currently_live: + redis.delete(lock_key) return slug = Video.get_new_slug( @@ -44,6 +52,8 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): ): queue.enqueue(download_video, video) + redis.delete(lock_key) + def download_video(video: Video): """ diff --git a/ucast/templates/bulma/base.html b/ucast/templates/bulma/base.html index 431b8ce..c20531b 100644 --- a/ucast/templates/bulma/base.html +++ b/ucast/templates/bulma/base.html @@ -27,8 +27,8 @@ + {% endif %}
diff --git a/ucast/templates/ucast/downloads_items.html b/ucast/templates/ucast/downloads_items.html index 9dd1658..860ab82 100644 --- a/ucast/templates/ucast/downloads_items.html +++ b/ucast/templates/ucast/downloads_items.html @@ -1,13 +1,26 @@ -{% for video in downloading_videos %} - - {{ video.video_id }} - {{ video.title }} - {{ video.channel.name }} - - {{ video.published|date:"SHORT_DATE_FORMAT" }} - -{% endfor %} + +
+ + + + + + + + + + + {% for video in downloading_videos %} + + + + + + + {% endfor %} + +
Video-IDTitleChannelPublished
{{ video.video_id }}{{ video.title }}{{ video.channel.name }} + {{ video.published|date:"SHORT_DATE_FORMAT" }}
+
diff --git a/ucast/views.py b/ucast/views.py index 182ba2e..68253ac 100644 --- a/ucast/views.py +++ b/ucast/views.py @@ -159,12 +159,10 @@ def downloads(request: http.HttpRequest): ids = freg.get_job_ids(0, 50) failed_jobs = freg.job_class.fetch_many(ids, freg.connection, freg.serializer) - page_number = request.GET.get("page") - downloading_videos = queue.get_downloading_videos() - downloading_videos_p = Paginator(downloading_videos, 100) + downloading_videos = queue.get_downloading_videos(limit=100) template_name = "ucast/downloads.html" - if request.htmx or request.GET.get("items"): + if request.htmx: template_name = "ucast/downloads_items.html" return render( @@ -172,7 +170,8 @@ def downloads(request: http.HttpRequest): template_name, { "failed_jobs": failed_jobs, - "downloading_videos": downloading_videos_p.get_page(page_number), + "downloading_videos": downloading_videos, + "n_downloading_videos": 1, }, ) From f21387a23c5df9d052a9a7188acb8619e0a00637 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 27 Jun 2022 22:38:12 +0200 Subject: [PATCH 04/36] add opml download --- ucast/service/opml.py | 40 +++++++++++++++++++++++++++++ ucast/templates/ucast/channels.html | 4 +++ ucast/urls.py | 1 + ucast/views.py | 15 ++++++++++- 4 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 ucast/service/opml.py diff --git a/ucast/service/opml.py b/ucast/service/opml.py new file mode 100644 index 0000000..e0913e3 --- /dev/null +++ b/ucast/service/opml.py @@ -0,0 +1,40 @@ +from dataclasses import dataclass +from typing import Iterable + +from django.utils.xmlutils import SimplerXMLGenerator + +from ucast.models import Channel + + +@dataclass +class FeedElement: + url: str + title: str + + +def __add_feed_element(handler: SimplerXMLGenerator, element: FeedElement): + handler.addQuickElement( + "outline", attrs={"xmlUrl": element.url, "title": element.title} + ) + + +def write_opml(elements: Iterable[FeedElement], outfile): + handler = SimplerXMLGenerator(outfile, "utf-8", short_empty_elements=True) + handler.startDocument() + handler.startElement("opml", {}) + handler.addQuickElement("head") + handler.startElement("body", {"version": "1.0"}) + + for element in elements: + __add_feed_element(handler, element) + + handler.endElement("body") + handler.endElement("opml") + handler.endDocument() + + +def write_channels_opml(channels: Iterable[Channel], site_url: str, key: str, outfile): + elements = [ + FeedElement(f"{site_url}/feed/{c.slug}?key={key}", c.name) for c in channels + ] + write_opml(elements, outfile) diff --git a/ucast/templates/ucast/channels.html b/ucast/templates/ucast/channels.html index 88a3aa1..9b93a86 100644 --- a/ucast/templates/ucast/channels.html +++ b/ucast/templates/ucast/channels.html @@ -68,6 +68,10 @@
{% endfor %} + +
+ Download OPML +
{% endblock content %} {% block javascript %} diff --git a/ucast/urls.py b/ucast/urls.py index f4f4f3d..1cf1003 100644 --- a/ucast/urls.py +++ b/ucast/urls.py @@ -30,6 +30,7 @@ urlpatterns = [ ), path("downloads/error/", views.error_details, name="error_details"), path("feed/", views.podcast_feed, name="feed"), + path("opml", views.channels_opml, name="channels_opml"), path("files/audio//", views.audio), path("files/cover//", views.cover), path("files/thumbnail//", views.thumbnail), diff --git a/ucast/views.py b/ucast/views.py index 68253ac..4985408 100644 --- a/ucast/views.py +++ b/ucast/views.py @@ -16,7 +16,7 @@ from django.utils.decorators import decorator_from_middleware from ucast import feed, forms, queue from ucast.models import Channel, User, Video -from ucast.service import controller, storage +from ucast.service import controller, opml, storage from ucast.tasks import download @@ -204,6 +204,19 @@ def download_errors_requeue_all(request: http.HttpRequest): return http.HttpResponseRedirect(reverse(downloads)) +@login_required +def channels_opml(request: http.HttpRequest): + response = http.HttpResponse( + content_type="application/xml", + headers={"Content-Disposition": "attachment; filename=ucast_channels.opml"}, + ) + site_url = add_domain(get_current_site(request).domain, "", request.is_secure()) + opml.write_channels_opml( + Channel.objects.all(), site_url, request.user.get_feed_key(), response + ) + return response + + def _channel_file(channel: str, get_file: Callable[[storage.ChannelFolder], Path]): store = storage.Storage() From df90e427296fd734f8a407db85eab37a643cd801 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 27 Jun 2022 22:44:16 +0200 Subject: [PATCH 05/36] allow variable number of feed items --- ucast/feed.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ucast/feed.py b/ucast/feed.py index c7dc07e..3d3214f 100644 --- a/ucast/feed.py +++ b/ucast/feed.py @@ -147,6 +147,12 @@ class UcastFeed(Feed): return Channel.objects.get(slug=channel_slug) def get_feed(self, channel: Channel, request: http.HttpRequest): + max_items = settings.FEED_MAX_ITEMS + try: + max_items = int(request.GET.get("items")) + except TypeError or ValueError: + pass + feed = self.feed_type( title=channel.name, link=channel.get_absolute_url(), @@ -158,7 +164,7 @@ class UcastFeed(Feed): for video in channel.video_set.filter(downloaded__isnull=False).order_by( "-published" - )[: settings.FEED_MAX_ITEMS]: + )[:max_items]: feed.add_item( title=video.title, link=video.get_absolute_url(), From e5c1fbdfb4c192dd88266cd642b710428e677b90 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 27 Jun 2022 23:09:42 +0200 Subject: [PATCH 06/36] add search --- ucast/templates/bulma/base.html | 3 ++ ucast/templates/ucast/search.html | 61 +++++++++++++++++++++++++ ucast/templates/ucast/videos_items.html | 4 +- ucast/urls.py | 1 + ucast/views.py | 12 +++++ 5 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 ucast/templates/ucast/search.html diff --git a/ucast/templates/bulma/base.html b/ucast/templates/bulma/base.html index c20531b..fee155e 100644 --- a/ucast/templates/bulma/base.html +++ b/ucast/templates/bulma/base.html @@ -27,6 +27,9 @@