Bump version: 0.1.1 → 0.2.0

fix yt-dlp cache not writable
fix video count

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.1
+current_version = 0.2.0
 commit = True
 tag = True
 

deploy/Dockerfile

@@ -9,7 +9,8 @@ FROM python:3.10
 ARG TARGETPLATFORM
 
 COPY --from=0 /build/dist /install
-RUN pip install -- /install/*.whl gunicorn honcho
+RUN pip install -- /install/*.whl gunicorn honcho && \
+    rm -rf ~/.cache/pip
 
 # ffmpeg static source (https://johnvansickle.com/ffmpeg/)
 RUN set -e; \

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ucast"
-version = "0.1.1"
+version = "0.2.0"
 description = "YouTube to Podcast converter"
 authors = ["Theta-Dev <t.testboy@gmail.com>"]
 packages = [

ucast/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.1"
+__version__ = "0.2.0"
 
 
 def template_context(request):

ucast/queue.py

@@ -4,6 +4,7 @@ import rq_scheduler
 from django.conf import settings
 from rq import registry
 
+from ucast.models import Video
 from ucast.service import util
 
 
@@ -33,8 +34,7 @@ def get_worker(**kwargs) -> rq.Worker:
 
 def enqueue(f, *args, **kwargs) -> rq.job.Job:
     queue = get_queue()
-    # return queue.enqueue(f, *args, **kwargs)
-    return queue.enqueue_call(f, args, kwargs)
+    return queue.enqueue(f, *args, **kwargs)
 
 
 def get_statistics() -> dict:
@@ -90,3 +90,19 @@ def get_statistics() -> dict:
 def get_failed_job_registry():
     queue = get_queue()
     return registry.FailedJobRegistry(queue.name, queue.connection)
+
+
+def get_downloading_videos():
+    queue = get_queue()
+    videos = {}
+
+    for job in queue.jobs:
+        if (
+            job.func_name == "ucast.tasks.download.download_video"
+            and job.args
+            and isinstance(job.args[0], Video)
+        ):
+            video = job.args[0]
+            videos[video.id] = video
+
+    return list(videos.values())

ucast/service/youtube.py

@@ -125,7 +125,19 @@ def download_thumbnail(vinfo: VideoDetails, download_path: Path):
 
 
 def get_video_details(video_id: str) -> VideoDetails:
-    with YoutubeDL() as ydl:
+    """
+    Get the details of a YouTube video without downloading it.
+
+    :param video_id: YouTube video ID
+    :return: VideoDetails
+    """
+    cache = storage.Cache()
+
+    ydl_params = {
+        "cachedir": str(cache.dir_ytdlp_cache),
+    }
+
+    with YoutubeDL(ydl_params) as ydl:
         info = ydl.extract_info(video_id, download=False)
         return VideoDetails.from_vinfo(info)
 

ucast/tasks/download.py

@@ -10,6 +10,13 @@ from ucast.service import controller, cover, storage, util, videoutil, youtube
 
 
 def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel):
+    # Use Redis to ensure the same video is not processed multiple times
+    redis = queue.get_redis_connection()
+    lock_key = f"ucast:lock_load_video:{vid.id}"
+
+    if not redis.set(lock_key, "1", 120, nx=True):
+        return
+
     # Create video object if it does not exist
     try:
         video = Video.objects.get(video_id=vid.id)
@@ -18,6 +25,7 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel):
 
         # Dont load active livestreams
         if details.is_currently_live:
+            redis.delete(lock_key)
             return
 
         slug = Video.get_new_slug(
@@ -44,6 +52,8 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel):
     ):
         queue.enqueue(download_video, video)
 
+    redis.delete(lock_key)
+
 
 def download_video(video: Video):
     """

ucast/templates/bulma/base.html

@@ -27,8 +27,8 @@
       </div>
 
       <div class="navbar-end">
-        <a class="navbar-item" href="{% url 'download_errors' %}">
-          Errors
+        <a class="navbar-item" href="{% url 'downloads' %}">
+          Downloads
         </a>
         {% url 'login' as login_url %}
         {% url 'logout' as logout_url %}

ucast/templates/ucast/download_errors.html

@@ -1,50 +0,0 @@
-{% extends 'base.html' %}
-
-{% block title %}ucast - Errors{% endblock %}
-
-{% block content %}
-  <div class="mb-4">
-    <div>
-      <span class="title">Download errors</span>
-    </div>
-  </div>
-
-  {% if jobs %}
-    <div class="mb-4">
-      <form method="post" action="{% url 'download_errors_requeue_all' %}">
-        {% csrf_token %}
-        <button class="button is-primary">Requeue all</button>
-      </form>
-    </div>
-
-    <table class="table">
-      <thead>
-      <tr>
-        <th>ID</th>
-        <th>Function</th>
-        <th>Details</th>
-        <th>Requeue</th>
-      </tr>
-      </thead>
-      <tbody>
-      {% for job in jobs %}
-        <tr>
-          <td>{{ job.id }}</td>
-          <td>{{ job.func_name }}</td>
-          <td><a href="{% url 'error_details' job.id %}">Details</a></td>
-          <td>
-            <form method="post" action="{% url 'download_errors_requeue' %}">
-              {% csrf_token %}
-              <input type="hidden" name="id" value="{{ job.id }}">
-              <button class="button is-small">Requeue</button>
-            </form>
-          </td>
-        </tr>
-      {% endfor %}
-      </tbody>
-    </table>
-  {% else %}
-    <p>No download errors</p>
-  {% endif %}
-
-{% endblock content %}

ucast/templates/ucast/downloads.html

@@ -0,0 +1,81 @@
+{% extends 'base.html' %}
+
+{% block title %}ucast - Errors{% endblock %}
+
+{% block content %}
+  <div class="mb-4">
+    <div>
+      <span class="title">Downloading
+        {% if downloading_videos %}({{ downloading_videos.paginator.count }}){% endif %}
+      </span>
+    </div>
+  </div>
+
+  <div class="mb-4">
+    {% if downloading_videos %}
+      <table class="table">
+      <thead>
+      <tr>
+        <th>Video-ID</th>
+        <th>Title</th>
+        <th>Channel</th>
+        <th>Published</th>
+      </tr>
+      </thead>
+      <tbody hx-get="?items" hx-trigger="every 5s">
+      {% include "ucast/downloads_items.html" %}
+    {% else %}
+      <p>Not downloading any videos</p>
+    {% endif %}
+    </tbody>
+    </table>
+
+  </div>
+
+  <div class="mb-4">
+    <div>
+      <span class="title">Download errors</span>
+    </div>
+  </div>
+
+  <div class="mb-4">
+    {% if failed_jobs %}
+      <div class="mb-4">
+        <form method="post" action="{% url 'download_errors_requeue_all' %}">
+          {% csrf_token %}
+          <button class="button is-primary">Requeue all</button>
+        </form>
+      </div>
+
+      <table class="table">
+        <thead>
+        <tr>
+          <th>ID</th>
+          <th>Function</th>
+          <th>Details</th>
+          <th>Requeue</th>
+        </tr>
+        </thead>
+        <tbody>
+        {% for job in failed_jobs %}
+          <tr>
+            <td>{{ job.id }}</td>
+            <td>{{ job.func_name }}</td>
+            <td><a href="{% url 'error_details' job.id %}">Details</a></td>
+            <td>
+              <form method="post" action="{% url 'download_errors_requeue' %}">
+                {% csrf_token %}
+                <input type="hidden" name="id" value="{{ job.id }}">
+                <button class="button is-small">Requeue</button>
+              </form>
+            </td>
+          </tr>
+        {% endfor %}
+        </tbody>
+      </table>
+    {% else %}
+      <p>No download errors</p>
+    {% endif %}
+  </div>
+
+{% endblock content %}

ucast/templates/ucast/downloads_items.html

@@ -0,0 +1,13 @@
+{% for video in downloading_videos %}
+  <tr {% if forloop.last and downloading_videos.has_next %}
+    hx-get="?page={{ downloading_videos.next_page_number }}"
+    hx-trigger="revealed"
+    hx-swap="afterend"
+  {% endif %}>
+    <td><a href="{{ video.get_absolute_url }}">{{ video.video_id }}</a></td>
+    <td>{{ video.title }}</td>
+    <td><a href="{% url 'videos' video.channel.slug %}">{{ video.channel.name }}</a>
+    </td>
+    <td>{{ video.published|date:"SHORT_DATE_FORMAT" }}</td>
+  </tr>
+{% endfor %}

ucast/templates/ucast/videos.html

@@ -13,7 +13,7 @@
       <span class="tag"><i
         class="fas fa-user-group"></i>&nbsp; {{ channel.subscribers }}</span>
       <span class="tag"><i
-        class="fas fa-video"></i>&nbsp; {{ videos|length }}</span>
+        class="fas fa-video"></i>&nbsp; {{ videos.paginator.count }}</span>
       <span class="tag"><i
         class="fas fa-database"></i>&nbsp; {{ channel.download_size|filesizeformat }}</span>
       <a class="tag" href="{{ channel.get_absolute_url }}" target="_blank"><i

ucast/tests/conftest.py

@@ -116,6 +116,14 @@ def download_dir_content_mut() -> Path:
     yield tmpdir
 
 
+@pytest.fixture
+def mock_redis(mocker) -> FakeRedis:
+    redis = FakeRedis()
+    mocker.patch.object(queue, "get_redis_connection")
+    queue.get_redis_connection.return_value = redis
+    return redis
+
+
 @pytest.fixture
 def rq_queue(mocker) -> rq.Queue:
     test_queue = rq.Queue(is_async=False, connection=FakeRedis())

ucast/tests/tasks/test_download.py

@@ -30,7 +30,7 @@ def test_download_video(download_dir, rq_queue):
 
 @pytest.mark.django_db
 def test_update_channel(
-    download_dir, rq_queue, mock_get_video_details, mock_download_audio
+    download_dir, rq_queue, mock_redis, mock_get_video_details, mock_download_audio
 ):
     # Remove 2 videos from the database so they can be imported
     Video.objects.get(video_id="ZPxEr4YdWt8").delete()

ucast/urls.py

@@ -17,16 +17,18 @@ urlpatterns = [
         views.channel_download,
         name="channel_download",
     ),
-    path("errors", views.download_errors, name="download_errors"),
+    path("downloads", views.downloads, name="downloads"),
     path(
-        "errors/requeue", views.download_errors_requeue, name="download_errors_requeue"
+        "downloads/requeue",
+        views.download_errors_requeue,
+        name="download_errors_requeue",
     ),
     path(
-        "errors/requeue_all",
+        "downloads/requeue_all",
         views.download_errors_requeue_all,
         name="download_errors_requeue_all",
     ),
-    path("errors/<str:job_id>", views.error_details, name="error_details"),
+    path("downloads/error/<str:job_id>", views.error_details, name="error_details"),
     path("feed/<str:channel>", views.podcast_feed, name="feed"),
     path("files/audio/<str:channel>/<str:video>", views.audio),
     path("files/cover/<str:channel>/<str:video>", views.cover),

ucast/views.py

@@ -154,12 +154,27 @@ def channel_download(request: http.HttpRequest, channel: str):
 
 
 @login_required
-def download_errors(request: http.HttpRequest):
+def downloads(request: http.HttpRequest):
     freg = queue.get_failed_job_registry()
     ids = freg.get_job_ids(0, 50)
-    jobs = freg.job_class.fetch_many(ids, freg.connection, freg.serializer)
+    failed_jobs = freg.job_class.fetch_many(ids, freg.connection, freg.serializer)
 
-    return render(request, "ucast/download_errors.html", {"jobs": jobs})
+    page_number = request.GET.get("page")
+    downloading_videos = queue.get_downloading_videos()
+    downloading_videos_p = Paginator(downloading_videos, 100)
+
+    template_name = "ucast/downloads.html"
+    if request.htmx or request.GET.get("items"):
+        template_name = "ucast/downloads_items.html"
+
+    return render(
+        request,
+        template_name,
+        {
+            "failed_jobs": failed_jobs,
+            "downloading_videos": downloading_videos_p.get_page(page_number),
+        },
+    )
 
 
 @login_required
@@ -178,7 +193,7 @@ def download_errors_requeue(request: http.HttpRequest):
         freg = queue.get_failed_job_registry()
         freg.requeue(str(form.cleaned_data["id"]))
 
-    return http.HttpResponseRedirect(reverse(download_errors))
+    return http.HttpResponseRedirect(reverse(downloads))
 
 
 @login_required
@@ -187,7 +202,7 @@ def download_errors_requeue_all(request: http.HttpRequest):
     for job_id in freg.get_job_ids():
         freg.requeue(job_id)
 
-    return http.HttpResponseRedirect(reverse(download_errors))
+    return http.HttpResponseRedirect(reverse(downloads))
 
 
 def _channel_file(channel: str, get_file: Callable[[storage.ChannelFolder], Path]):