publish docker images with ci

handle unavailable videos

.drone.yml

@@ -36,6 +36,27 @@ steps:
     depends_on:
       - install dependencies
 
+  - name: build container
+    image: quay.io/buildah/stable
+    when:
+      event:
+        - tag
+    commands:
+      - buildah login -u $DOCKER_USER -p $DOCKER_PASS -- $DOCKER_REGISTRY
+      - buildah manifest create ucast
+      - buildah bud --tag code.thetadev.de/hsa/ucast:latest --manifest ucast --arch amd64 -f deploy/Dockerfile .
+      - buildah bud --tag code.thetadev.de/hsa/ucast:latest --manifest ucast --arch arm64 -f deploy/Dockerfile .
+      - buildah manifest push --all ucast docker://code.thetadev.de/hsa/ucast:latest
+    environment:
+      DOCKER_REGISTRY:
+        from_secret: docker_registry
+      DOCKER_USER:
+        from_secret: docker_username
+      DOCKER_PASS:
+        from_secret: docker_password
+    depends_on:
+      - test
+
 volumes:
   - name: cache
     temp: { }

ucast/tasks/download.py

@@ -3,6 +3,7 @@ import os
 
 from django.db.models import ObjectDoesNotExist
 from django.utils import timezone
+from yt_dlp.utils import DownloadError
 
 from ucast import queue
 from ucast.models import Channel, Video
@@ -21,7 +22,25 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel):
     try:
         video = Video.objects.get(video_id=vid.id)
     except ObjectDoesNotExist:
-        details = youtube.get_video_details(vid.id)
+        try:
+            details = youtube.get_video_details(vid.id)
+        except DownloadError as e:
+            if "available" in e.msg:
+                # Create dummy video to prevent further download attempts
+                # of unavailable videos
+                video = Video(
+                    video_id=vid.id,
+                    title="",
+                    slug="",
+                    channel=channel,
+                    published=timezone.datetime(2000, 1, 1, tzinfo=timezone.utc),
+                    description="",
+                    duration=0,
+                    is_deleted=True,
+                )
+                video.save()
+                return
+            raise e
 
         # Dont load active livestreams
         if details.is_currently_live:
@@ -50,20 +69,23 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel):
         and video.is_deleted is False
         and channel.should_download(video)
     ):
-        queue.enqueue(download_video, video)
+        queue.enqueue(download_video, video.id)
 
     redis.delete(lock_key)
 
 
-def download_video(video: Video):
+def download_video(v_id: int):
     """
     Download a video including its thumbnail, create a cover image
     and store everything in the channel folder.
 
-    :param video: Video object
+    :param v_id: Video ID
     """
     # Return if the video was already downloaded by a previous task
-    video.refresh_from_db()
+    try:
+        video = Video.objects.get(id=v_id)
+    except ObjectDoesNotExist:
+        return
     if video.downloaded:
         return
 
@@ -71,7 +93,14 @@ def download_video(video: Video):
     channel_folder = store.get_or_create_channel_folder(video.channel.slug)
 
     audio_file = channel_folder.get_audio(video.slug)
-    details = youtube.download_audio(video.video_id, audio_file)
+    try:
+        details = youtube.download_audio(video.video_id, audio_file)
+    except DownloadError as e:
+        if "available" in e.msg:
+            video.is_deleted = True
+            video.save()
+            return
+        raise e
 
     # Download/convert thumbnails
     tn_path = channel_folder.get_thumbnail(video.slug)
@@ -106,8 +135,12 @@ def download_video(video: Video):
     video.save()
 
 
-def update_channel(channel: Channel):
+def update_channel(c_id: int):
     """Update a single channel from its RSS feed"""
+    try:
+        channel = Channel.objects.get(id=c_id)
+    except ObjectDoesNotExist:
+        return
     videos = youtube.get_channel_videos_from_feed(channel.channel_id)
 
     for vid in videos:
@@ -123,18 +156,23 @@ def update_channels():
     This task is scheduled a regular intervals.
     """
     for channel in Channel.objects.filter(active=True):
-        queue.enqueue(update_channel, channel)
+        queue.enqueue(update_channel, channel.id)
 
 
-def download_channel(channel: Channel, limit: int):
+def download_channel(c_id: int, limit: int):
     """
     Download maximum number of videos from a channel.
 
-    :param channel: Channel object
+    :param c_id: Channel ID (Database)
     :param limit: Max number of videos
     """
     if limit < 1:
         return
 
+    try:
+        channel = Channel.objects.get(id=c_id)
+    except ObjectDoesNotExist:
+        return
+
     for vid in youtube.get_channel_videos_from_scraper(channel.channel_id, limit):
         _load_scraped_video(vid, channel)

ucast/tasks/library.py

@@ -1,5 +1,6 @@
 import os
 
+from django.db.models import ObjectDoesNotExist
 from django.utils import timezone
 
 from ucast import queue
@@ -7,7 +8,12 @@ from ucast.models import Channel, Video
 from ucast.service import cover, storage, util, videoutil, youtube
 
 
-def recreate_cover(video: Video):
+def recreate_cover(v_id: int):
+    try:
+        video = Video.objects.get(id=v_id)
+    except ObjectDoesNotExist:
+        return
+
     store = storage.Storage()
     cf = store.get_channel_folder(video.channel.slug)
 
@@ -42,7 +48,7 @@ def recreate_cover(video: Video):
 
 def recreate_covers():
     for video in Video.objects.filter(downloaded__isnull=False):
-        queue.enqueue(recreate_cover, video)
+        queue.enqueue(recreate_cover, video.id)
 
 
 def update_file_storage():
@@ -81,7 +87,12 @@ def update_file_storage():
         video.save()
 
 
-def update_channel_info(channel: Channel):
+def update_channel_info(ch_id: int):
+    try:
+        channel = Channel.objects.get(id=ch_id)
+    except ObjectDoesNotExist:
+        return
+
     channel_data = youtube.get_channel_metadata(
         youtube.channel_url_from_id(channel.channel_id)
     )
@@ -104,4 +115,4 @@ def update_channel_info(channel: Channel):
 
 def update_channel_infos():
     for channel in Channel.objects.filter(active=True):
-        queue.enqueue(update_channel_info, channel)
+        queue.enqueue(update_channel_info, channel.id)

ucast/templates/ucast/downloads.html

@@ -27,11 +27,15 @@
 
   <div class="mb-4">
     {% if failed_jobs %}
-      <div class="mb-4">
+      <div class="level mb-4">
         <form method="post" action="{% url 'download_errors_requeue_all' %}">
           {% csrf_token %}
           <button class="button is-primary">Requeue all</button>
         </form>
+        <form method="post" action="{% url 'download_errors_delete_all' %}">
+          {% csrf_token %}
+          <button class="button is-danger">Delete all</button>
+        </form>
       </div>
 
       <table class="table">
@@ -41,6 +45,7 @@
           <th>Function</th>
           <th>Details</th>
           <th>Requeue</th>
+          <th>Delete</th>
         </tr>
         </thead>
         <tbody>
@@ -56,6 +61,13 @@
                 <button class="button is-small">Requeue</button>
               </form>
             </td>
+            <td>
+              <form method="post" action="{% url 'download_errors_delete' %}">
+                {% csrf_token %}
+                <input type="hidden" name="id" value="{{ job.id }}">
+                <button class="button is-small is-danger">Delete</button>
+              </form>
+            </td>
           </tr>
         {% endfor %}
         </tbody>

ucast/templates/ucast/error_details.html

@@ -16,12 +16,17 @@
   {{ job.exc_info }}
   </pre>
 
-  <div>
+  <div class="level">
     <form method="post" action="{% url 'download_errors_requeue' %}">
       {% csrf_token %}
       <input type="hidden" name="id" value="{{ job.id }}">
       <button class="button is-primary">Requeue</button>
     </form>
+    <form method="post" action="{% url 'download_errors_delete' %}">
+      {% csrf_token %}
+      <input type="hidden" name="id" value="{{ job.id }}">
+      <button class="button is-danger">Delete</button>
+    </form>
   </div>
 
 {% endblock content %}

ucast/templates/ucast/videos.html

@@ -13,7 +13,11 @@
       <span class="tag"><i
         class="fas fa-user-group"></i>&nbsp; {{ channel.subscribers }}</span>
       <span class="tag"><i
-        class="fas fa-video"></i>&nbsp; {{ videos.paginator.count }}</span>
+        class="fas fa-video"></i>&nbsp; {{ videos.paginator.count }}
+        {% if n_pending %}
+          ({{ n_pending }})
+        {% endif %}
+      </span>
       <span class="tag"><i
         class="fas fa-database"></i>&nbsp; {{ channel.download_size|filesizeformat }}</span>
       <a class="tag" href="{{ channel.get_absolute_url }}" target="_blank"><i
@@ -44,13 +48,14 @@
             <i class="fas fa-edit"></i>
           </a>
         </div>
-      <div class="control">
+        <div class="control">
           <a class="button is-info" href="{% url 'channel_download' channel.slug %}">
             <i class="fas fa-download"></i>
           </a>
         </div>
         <div class="control">
-          <button type="submit" name="delete_channel" class="button is-danger dialog-confirm"
+          <button type="submit" name="delete_channel"
+                  class="button is-danger dialog-confirm"
                   confirm-msg="Do you want to delete the channel '{{ channel.name }}' including {{ videos|length }} videos?">
             <i class="fas fa-trash"></i>
           </button>
@@ -60,28 +65,38 @@
     </form>
   </div>
 
-  <div class="video-grid">
-    {% if not videos %}
-      <p>No videos</p>
+  {% if not videos %}
+    {% if n_pending %}
+      <p>There are {{ n_pending }} videos waiting to be downloaded.
+        Please wait a few minutes and refesh this page.
+        You can see the current status in the <i>Downloads</i> tab.
+      </p>
+    {% else %}
+      <p>No videos. If you have just added this channel,
+        you have to wait a minute for ucast to start looking for videos.</p>
     {% endif %}
-    {% include "ucast/videos_items.html" %}
-  </div>
+  {% else %}
+    <div class="video-grid">
+      {% include "ucast/videos_items.html" %}
+    </div>
+  {% endif %}
 
   {% if videos.has_previous or videos.has_next %}
-  <noscript>
-    <nav class="pagination is-centered mt-4" role="navigation" aria-label="pagination">
-      {% if videos.has_previous %}
-        <a class="pagination-previous" href="?page={{ videos.previous_page_number }}">Previous</a>
-      {% else %}
-        <a class="pagination-previous" disabled>Previous</a>
-      {% endif %}
-      {% if videos.has_next %}
-        <a class="pagination-next" href="?page={{ videos.next_page_number }}">Next
-          page</a>
-      {% else %}
-        <a class="pagination-previous" disabled>Previous</a>
-      {% endif %}
-    </nav>
-  </noscript>
+    <noscript>
+      <nav class="pagination is-centered mt-4" role="navigation"
+           aria-label="pagination">
+        {% if videos.has_previous %}
+          <a class="pagination-previous" href="?page={{ videos.previous_page_number }}">Previous</a>
+        {% else %}
+          <a class="pagination-previous" disabled>Previous</a>
+        {% endif %}
+        {% if videos.has_next %}
+          <a class="pagination-next" href="?page={{ videos.next_page_number }}">Next
+            page</a>
+        {% else %}
+          <a class="pagination-previous" disabled>Previous</a>
+        {% endif %}
+      </nav>
+    </noscript>
   {% endif %}
 {% endblock content %}

ucast/tests/tasks/test_download.py

@@ -1,21 +1,24 @@
 import os
 
 import pytest
+from django.utils import timezone
 
 from ucast import queue, tests
 from ucast.models import Channel, Video
 from ucast.service import storage
+from ucast.service.youtube import VideoScraped
 from ucast.tasks import download
 
 CHANNEL_ID_THETADEV = "UCGiJh0NZ52wRhYKYnuZI08Q"
 VIDEO_ID_INTRO = "I0RRENheeTo"
 VIDEO_SLUG_INTRO = "20211010_No_copyright_intro_free_fire_intro_no_text_free_copy_right_free_templates_free_download"
+VIDEO_ID_UNAVAILABLE = "K6CBuTy09CE"
 
 
 @pytest.mark.django_db
 def test_download_video(download_dir, rq_queue):
     video = Video.objects.get(video_id=VIDEO_ID_INTRO)
-    job = queue.enqueue(download.download_video, video)
+    job = queue.enqueue(download.download_video, video.id)
 
     store = storage.Storage()
     cf = store.get_or_create_channel_folder(video.channel.slug)
@@ -28,6 +31,35 @@ def test_download_video(download_dir, rq_queue):
     assert os.path.isfile(cf.get_thumbnail(VIDEO_SLUG_INTRO, True))
 
 
+@pytest.mark.django_db
+def test_load_unavailable_video(download_dir, rq_queue, mock_redis):
+    channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV)
+    download._load_scraped_video(VideoScraped(VIDEO_ID_UNAVAILABLE, None), channel)
+
+    video = Video.objects.get(video_id=VIDEO_ID_UNAVAILABLE)
+    assert video.is_deleted is True
+
+
+@pytest.mark.django_db
+def test_download_unavailable_video(download_dir, rq_queue):
+    channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV)
+    video = Video(
+        video_id=VIDEO_ID_UNAVAILABLE,
+        title="",
+        slug="",
+        channel=channel,
+        published=timezone.datetime(2000, 1, 1, tzinfo=timezone.utc),
+        description="",
+        duration=0,
+    )
+    video.save()
+    job = queue.enqueue(download.download_video, video.id)
+    video.refresh_from_db()
+
+    assert job.is_finished
+    assert video.is_deleted
+
+
 @pytest.mark.django_db
 def test_update_channel(
     download_dir, rq_queue, mock_redis, mock_get_video_details, mock_download_audio
@@ -37,7 +69,7 @@ def test_update_channel(
     Video.objects.get(video_id="_I5IFObm_-k").delete()
 
     channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV)
-    job = rq_queue.enqueue(download.update_channel, channel)
+    job = rq_queue.enqueue(download.update_channel, channel.id)
     assert job.is_finished
 
     mock_download_audio.assert_any_call(

ucast/tests/tasks/test_library.py

@@ -19,7 +19,7 @@ def test_recreate_cover(download_dir_content_mut, rq_queue, mocker):
     store = storage.Storage()
     cf = store.get_or_create_channel_folder(video.channel.slug)
 
-    job = rq_queue.enqueue(library.recreate_cover, video)
+    job = rq_queue.enqueue(library.recreate_cover, video.id)
     assert job.is_finished
 
     create_cover_mock.assert_called_once_with(
@@ -53,7 +53,7 @@ def test_update_channel_info(rq_queue, mock_get_channel_metadata):
     channel.avatar_url = "Old avatar url"
     channel.save()
 
-    job = rq_queue.enqueue(library.update_channel_info, channel)
+    job = rq_queue.enqueue(library.update_channel_info, channel.id)
     assert job.is_finished
 
     channel.refresh_from_db()

ucast/urls.py

@@ -28,6 +28,16 @@ urlpatterns = [
         views.download_errors_requeue_all,
         name="download_errors_requeue_all",
     ),
+    path(
+        "downloads/delete",
+        views.download_errors_delete,
+        name="download_errors_delete",
+    ),
+    path(
+        "downloads/delete_all",
+        views.download_errors_delete_all,
+        name="download_errors_delete_all",
+    ),
     path("downloads/error/<str:job_id>", views.error_details, name="error_details"),
     path("feed/<str:channel>", views.podcast_feed, name="feed"),
     path("opml", views.channels_opml, name="channels_opml"),

ucast/views.py

@@ -33,7 +33,7 @@ def home(request: http.HttpRequest):
             channel_str = form.cleaned_data["channel_str"]
             try:
                 channel = controller.create_channel(channel_str)
-                queue.enqueue(download.update_channel, channel)
+                queue.enqueue(download.update_channel, channel.id)
             except ValueError:
                 form.add_error("channel_str", "Channel URL invalid")
             except controller.ChannelAlreadyExistsException:
@@ -91,6 +91,10 @@ def videos(request: http.HttpRequest, channel: str):
     if request.htmx:
         template_name = "ucast/videos_items.html"
 
+    n_pending = Video.objects.filter(
+        channel=chan, downloaded__isnull=True, is_deleted=False
+    ).count()
+
     return render(
         request,
         template_name,
@@ -98,6 +102,7 @@ def videos(request: http.HttpRequest, channel: str):
             "videos": videos_p.get_page(page_number),
             "channel": chan,
             "site_url": site_url,
+            "n_pending": n_pending,
         },
     )
 
@@ -139,7 +144,7 @@ def channel_download(request: http.HttpRequest, channel: str):
         form = forms.DownloadChannelForm(request.POST)
         if form.is_valid():
             queue.enqueue(
-                download.download_channel, chan, form.cleaned_data["n_videos"]
+                download.download_channel, chan.id, form.cleaned_data["n_videos"]
             )
             return http.HttpResponseRedirect(reverse(videos, args=[channel]))
 
@@ -204,6 +209,26 @@ def download_errors_requeue_all(request: http.HttpRequest):
     return http.HttpResponseRedirect(reverse(downloads))
 
 
+@login_required
+def download_errors_delete(request: http.HttpRequest):
+    form = forms.RequeueForm(request.POST)
+
+    if form.is_valid():
+        freg = queue.get_failed_job_registry()
+        freg.remove(str(form.cleaned_data["id"]), delete_job=True)
+
+    return http.HttpResponseRedirect(reverse(downloads))
+
+
+@login_required
+def download_errors_delete_all(request: http.HttpRequest):
+    freg = queue.get_failed_job_registry()
+    for job_id in freg.get_job_ids():
+        freg.remove(job_id, delete_job=True)
+
+    return http.HttpResponseRedirect(reverse(downloads))
+
+
 @login_required
 def channels_opml(request: http.HttpRequest):
     response = http.HttpResponse(