.drone.yml

@@ -36,27 +36,6 @@ steps:
     depends_on:
       - install dependencies
 
-  - name: build container
-    image: quay.io/buildah/stable
-    when:
-      event:
-        - tag
-    commands:
-      - buildah login -u $DOCKER_USER -p $DOCKER_PASS -- $DOCKER_REGISTRY
-      - buildah manifest create ucast
-      - buildah bud --tag code.thetadev.de/hsa/ucast:latest --manifest ucast --arch amd64 -f deploy/Dockerfile .
-      - buildah bud --tag code.thetadev.de/hsa/ucast:latest --manifest ucast --arch arm64 -f deploy/Dockerfile .
-      - buildah manifest push --all ucast docker://code.thetadev.de/hsa/ucast:latest
-    environment:
-      DOCKER_REGISTRY:
-        from_secret: docker_registry
-      DOCKER_USER:
-        from_secret: docker_username
-      DOCKER_PASS:
-        from_secret: docker_password
-    depends_on:
-      - test
-
 volumes:
   - name: cache
     temp: { }

ucast/tasks/download.py

@@ -3,7 +3,6 @@ import os
 
 from django.db.models import ObjectDoesNotExist
 from django.utils import timezone
-from yt_dlp.utils import DownloadError
 
 from ucast import queue
 from ucast.models import Channel, Video
@@ -22,25 +21,7 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel):
     try:
         video = Video.objects.get(video_id=vid.id)
     except ObjectDoesNotExist:
-        try:
-            details = youtube.get_video_details(vid.id)
-        except DownloadError as e:
-            if "available" in e.msg:
-                # Create dummy video to prevent further download attempts
-                # of unavailable videos
-                video = Video(
-                    video_id=vid.id,
-                    title="",
-                    slug="",
-                    channel=channel,
-                    published=timezone.datetime(2000, 1, 1, tzinfo=timezone.utc),
-                    description="",
-                    duration=0,
-                    is_deleted=True,
-                )
-                video.save()
-                return
-            raise e
+        details = youtube.get_video_details(vid.id)
 
         # Dont load active livestreams
         if details.is_currently_live:
@@ -69,23 +50,20 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel):
         and video.is_deleted is False
         and channel.should_download(video)
     ):
-        queue.enqueue(download_video, video.id)
+        queue.enqueue(download_video, video)
 
     redis.delete(lock_key)
 
 
-def download_video(v_id: int):
+def download_video(video: Video):
     """
     Download a video including its thumbnail, create a cover image
     and store everything in the channel folder.
 
-    :param v_id: Video ID
+    :param video: Video object
     """
     # Return if the video was already downloaded by a previous task
-    try:
-        video = Video.objects.get(id=v_id)
-    except ObjectDoesNotExist:
-        return
+    video.refresh_from_db()
     if video.downloaded:
         return
 
@@ -93,14 +71,7 @@ def download_video(v_id: int):
     channel_folder = store.get_or_create_channel_folder(video.channel.slug)
 
     audio_file = channel_folder.get_audio(video.slug)
-    try:
-        details = youtube.download_audio(video.video_id, audio_file)
-    except DownloadError as e:
-        if "available" in e.msg:
-            video.is_deleted = True
-            video.save()
-            return
-        raise e
+    details = youtube.download_audio(video.video_id, audio_file)
 
     # Download/convert thumbnails
     tn_path = channel_folder.get_thumbnail(video.slug)
@@ -135,12 +106,8 @@ def download_video(v_id: int):
     video.save()
 
 
-def update_channel(c_id: int):
+def update_channel(channel: Channel):
     """Update a single channel from its RSS feed"""
-    try:
-        channel = Channel.objects.get(id=c_id)
-    except ObjectDoesNotExist:
-        return
     videos = youtube.get_channel_videos_from_feed(channel.channel_id)
 
     for vid in videos:
@@ -156,23 +123,18 @@ def update_channels():
     This task is scheduled a regular intervals.
     """
     for channel in Channel.objects.filter(active=True):
-        queue.enqueue(update_channel, channel.id)
+        queue.enqueue(update_channel, channel)
 
 
-def download_channel(c_id: int, limit: int):
+def download_channel(channel: Channel, limit: int):
     """
     Download maximum number of videos from a channel.
 
-    :param c_id: Channel ID (Database)
+    :param channel: Channel object
     :param limit: Max number of videos
     """
     if limit < 1:
         return
 
-    try:
-        channel = Channel.objects.get(id=c_id)
-    except ObjectDoesNotExist:
-        return
-
     for vid in youtube.get_channel_videos_from_scraper(channel.channel_id, limit):
         _load_scraped_video(vid, channel)

ucast/tasks/library.py

@@ -1,6 +1,5 @@
 import os
 
-from django.db.models import ObjectDoesNotExist
 from django.utils import timezone
 
 from ucast import queue
@@ -8,12 +7,7 @@ from ucast.models import Channel, Video
 from ucast.service import cover, storage, util, videoutil, youtube
 
 
-def recreate_cover(v_id: int):
-    try:
-        video = Video.objects.get(id=v_id)
-    except ObjectDoesNotExist:
-        return
-
+def recreate_cover(video: Video):
     store = storage.Storage()
     cf = store.get_channel_folder(video.channel.slug)
 
@@ -48,7 +42,7 @@ def recreate_cover(v_id: int):
 
 def recreate_covers():
     for video in Video.objects.filter(downloaded__isnull=False):
-        queue.enqueue(recreate_cover, video.id)
+        queue.enqueue(recreate_cover, video)
 
 
 def update_file_storage():
@@ -87,12 +81,7 @@ def update_file_storage():
         video.save()
 
 
-def update_channel_info(ch_id: int):
-    try:
-        channel = Channel.objects.get(id=ch_id)
-    except ObjectDoesNotExist:
-        return
-
+def update_channel_info(channel: Channel):
     channel_data = youtube.get_channel_metadata(
         youtube.channel_url_from_id(channel.channel_id)
     )
@@ -115,4 +104,4 @@ def update_channel_info(ch_id: int):
 
 def update_channel_infos():
     for channel in Channel.objects.filter(active=True):
-        queue.enqueue(update_channel_info, channel.id)
+        queue.enqueue(update_channel_info, channel)

ucast/templates/ucast/downloads.html

@@ -27,15 +27,11 @@
 
   <div class="mb-4">
     {% if failed_jobs %}
-      <div class="level mb-4">
+      <div class="mb-4">
         <form method="post" action="{% url 'download_errors_requeue_all' %}">
           {% csrf_token %}
           <button class="button is-primary">Requeue all</button>
         </form>
-        <form method="post" action="{% url 'download_errors_delete_all' %}">
-          {% csrf_token %}
-          <button class="button is-danger">Delete all</button>
-        </form>
       </div>
 
       <table class="table">
@@ -45,7 +41,6 @@
           <th>Function</th>
           <th>Details</th>
           <th>Requeue</th>
-          <th>Delete</th>
         </tr>
         </thead>
         <tbody>
@@ -61,13 +56,6 @@
                 <button class="button is-small">Requeue</button>
               </form>
             </td>
-            <td>
-              <form method="post" action="{% url 'download_errors_delete' %}">
-                {% csrf_token %}
-                <input type="hidden" name="id" value="{{ job.id }}">
-                <button class="button is-small is-danger">Delete</button>
-              </form>
-            </td>
           </tr>
         {% endfor %}
         </tbody>

ucast/templates/ucast/error_details.html

@@ -16,17 +16,12 @@
   {{ job.exc_info }}
   </pre>
 
-  <div class="level">
+  <div>
     <form method="post" action="{% url 'download_errors_requeue' %}">
       {% csrf_token %}
       <input type="hidden" name="id" value="{{ job.id }}">
       <button class="button is-primary">Requeue</button>
     </form>
-    <form method="post" action="{% url 'download_errors_delete' %}">
-      {% csrf_token %}
-      <input type="hidden" name="id" value="{{ job.id }}">
-      <button class="button is-danger">Delete</button>
-    </form>
   </div>
 
 {% endblock content %}

ucast/templates/ucast/videos.html

@@ -13,11 +13,7 @@
       <span class="tag"><i
         class="fas fa-user-group"></i>&nbsp; {{ channel.subscribers }}</span>
       <span class="tag"><i
-        class="fas fa-video"></i>&nbsp; {{ videos.paginator.count }}
-        {% if n_pending %}
-          ({{ n_pending }})
-        {% endif %}
-      </span>
+        class="fas fa-video"></i>&nbsp; {{ videos.paginator.count }}</span>
       <span class="tag"><i
         class="fas fa-database"></i>&nbsp; {{ channel.download_size|filesizeformat }}</span>
       <a class="tag" href="{{ channel.get_absolute_url }}" target="_blank"><i
@@ -48,14 +44,13 @@
             <i class="fas fa-edit"></i>
           </a>
         </div>
-        <div class="control">
+      <div class="control">
           <a class="button is-info" href="{% url 'channel_download' channel.slug %}">
             <i class="fas fa-download"></i>
           </a>
         </div>
         <div class="control">
-          <button type="submit" name="delete_channel"
-                  class="button is-danger dialog-confirm"
+          <button type="submit" name="delete_channel" class="button is-danger dialog-confirm"
                   confirm-msg="Do you want to delete the channel '{{ channel.name }}' including {{ videos|length }} videos?">
             <i class="fas fa-trash"></i>
           </button>
@@ -65,38 +60,28 @@
     </form>
   </div>
 
-  {% if not videos %}
-    {% if n_pending %}
-      <p>There are {{ n_pending }} videos waiting to be downloaded.
-        Please wait a few minutes and refesh this page.
-        You can see the current status in the <i>Downloads</i> tab.
-      </p>
-    {% else %}
-      <p>No videos. If you have just added this channel,
-        you have to wait a minute for ucast to start looking for videos.</p>
+  <div class="video-grid">
+    {% if not videos %}
+      <p>No videos</p>
     {% endif %}
-  {% else %}
-    <div class="video-grid">
-      {% include "ucast/videos_items.html" %}
-    </div>
-  {% endif %}
+    {% include "ucast/videos_items.html" %}
+  </div>
 
   {% if videos.has_previous or videos.has_next %}
-    <noscript>
-      <nav class="pagination is-centered mt-4" role="navigation"
-           aria-label="pagination">
-        {% if videos.has_previous %}
-          <a class="pagination-previous" href="?page={{ videos.previous_page_number }}">Previous</a>
-        {% else %}
-          <a class="pagination-previous" disabled>Previous</a>
-        {% endif %}
-        {% if videos.has_next %}
-          <a class="pagination-next" href="?page={{ videos.next_page_number }}">Next
-            page</a>
-        {% else %}
-          <a class="pagination-previous" disabled>Previous</a>
-        {% endif %}
-      </nav>
-    </noscript>
+  <noscript>
+    <nav class="pagination is-centered mt-4" role="navigation" aria-label="pagination">
+      {% if videos.has_previous %}
+        <a class="pagination-previous" href="?page={{ videos.previous_page_number }}">Previous</a>
+      {% else %}
+        <a class="pagination-previous" disabled>Previous</a>
+      {% endif %}
+      {% if videos.has_next %}
+        <a class="pagination-next" href="?page={{ videos.next_page_number }}">Next
+          page</a>
+      {% else %}
+        <a class="pagination-previous" disabled>Previous</a>
+      {% endif %}
+    </nav>
+  </noscript>
   {% endif %}
 {% endblock content %}

ucast/tests/tasks/test_download.py

@@ -1,24 +1,21 @@
 import os
 
 import pytest
-from django.utils import timezone
 
 from ucast import queue, tests
 from ucast.models import Channel, Video
 from ucast.service import storage
-from ucast.service.youtube import VideoScraped
 from ucast.tasks import download
 
 CHANNEL_ID_THETADEV = "UCGiJh0NZ52wRhYKYnuZI08Q"
 VIDEO_ID_INTRO = "I0RRENheeTo"
 VIDEO_SLUG_INTRO = "20211010_No_copyright_intro_free_fire_intro_no_text_free_copy_right_free_templates_free_download"
-VIDEO_ID_UNAVAILABLE = "K6CBuTy09CE"
 
 
 @pytest.mark.django_db
 def test_download_video(download_dir, rq_queue):
     video = Video.objects.get(video_id=VIDEO_ID_INTRO)
-    job = queue.enqueue(download.download_video, video.id)
+    job = queue.enqueue(download.download_video, video)
 
     store = storage.Storage()
     cf = store.get_or_create_channel_folder(video.channel.slug)
@@ -31,35 +28,6 @@ def test_download_video(download_dir, rq_queue):
     assert os.path.isfile(cf.get_thumbnail(VIDEO_SLUG_INTRO, True))
 
 
-@pytest.mark.django_db
-def test_load_unavailable_video(download_dir, rq_queue, mock_redis):
-    channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV)
-    download._load_scraped_video(VideoScraped(VIDEO_ID_UNAVAILABLE, None), channel)
-
-    video = Video.objects.get(video_id=VIDEO_ID_UNAVAILABLE)
-    assert video.is_deleted is True
-
-
-@pytest.mark.django_db
-def test_download_unavailable_video(download_dir, rq_queue):
-    channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV)
-    video = Video(
-        video_id=VIDEO_ID_UNAVAILABLE,
-        title="",
-        slug="",
-        channel=channel,
-        published=timezone.datetime(2000, 1, 1, tzinfo=timezone.utc),
-        description="",
-        duration=0,
-    )
-    video.save()
-    job = queue.enqueue(download.download_video, video.id)
-    video.refresh_from_db()
-
-    assert job.is_finished
-    assert video.is_deleted
-
-
 @pytest.mark.django_db
 def test_update_channel(
     download_dir, rq_queue, mock_redis, mock_get_video_details, mock_download_audio
@@ -69,7 +37,7 @@ def test_update_channel(
     Video.objects.get(video_id="_I5IFObm_-k").delete()
 
     channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV)
-    job = rq_queue.enqueue(download.update_channel, channel.id)
+    job = rq_queue.enqueue(download.update_channel, channel)
     assert job.is_finished
 
     mock_download_audio.assert_any_call(

ucast/tests/tasks/test_library.py

@@ -19,7 +19,7 @@ def test_recreate_cover(download_dir_content_mut, rq_queue, mocker):
     store = storage.Storage()
     cf = store.get_or_create_channel_folder(video.channel.slug)
 
-    job = rq_queue.enqueue(library.recreate_cover, video.id)
+    job = rq_queue.enqueue(library.recreate_cover, video)
     assert job.is_finished
 
     create_cover_mock.assert_called_once_with(
@@ -53,7 +53,7 @@ def test_update_channel_info(rq_queue, mock_get_channel_metadata):
     channel.avatar_url = "Old avatar url"
     channel.save()
 
-    job = rq_queue.enqueue(library.update_channel_info, channel.id)
+    job = rq_queue.enqueue(library.update_channel_info, channel)
     assert job.is_finished
 
     channel.refresh_from_db()

ucast/urls.py

@@ -28,16 +28,6 @@ urlpatterns = [
         views.download_errors_requeue_all,
         name="download_errors_requeue_all",
     ),
-    path(
-        "downloads/delete",
-        views.download_errors_delete,
-        name="download_errors_delete",
-    ),
-    path(
-        "downloads/delete_all",
-        views.download_errors_delete_all,
-        name="download_errors_delete_all",
-    ),
     path("downloads/error/<str:job_id>", views.error_details, name="error_details"),
     path("feed/<str:channel>", views.podcast_feed, name="feed"),
     path("opml", views.channels_opml, name="channels_opml"),

ucast/views.py

@@ -33,7 +33,7 @@ def home(request: http.HttpRequest):
             channel_str = form.cleaned_data["channel_str"]
             try:
                 channel = controller.create_channel(channel_str)
-                queue.enqueue(download.update_channel, channel.id)
+                queue.enqueue(download.update_channel, channel)
             except ValueError:
                 form.add_error("channel_str", "Channel URL invalid")
             except controller.ChannelAlreadyExistsException:
@@ -91,10 +91,6 @@ def videos(request: http.HttpRequest, channel: str):
     if request.htmx:
         template_name = "ucast/videos_items.html"
 
-    n_pending = Video.objects.filter(
-        channel=chan, downloaded__isnull=True, is_deleted=False
-    ).count()
-
     return render(
         request,
         template_name,
@@ -102,7 +98,6 @@ def videos(request: http.HttpRequest, channel: str):
             "videos": videos_p.get_page(page_number),
             "channel": chan,
             "site_url": site_url,
-            "n_pending": n_pending,
         },
     )
 
@@ -144,7 +139,7 @@ def channel_download(request: http.HttpRequest, channel: str):
         form = forms.DownloadChannelForm(request.POST)
         if form.is_valid():
             queue.enqueue(
-                download.download_channel, chan.id, form.cleaned_data["n_videos"]
+                download.download_channel, chan, form.cleaned_data["n_videos"]
             )
             return http.HttpResponseRedirect(reverse(videos, args=[channel]))
 
@@ -209,26 +204,6 @@ def download_errors_requeue_all(request: http.HttpRequest):
     return http.HttpResponseRedirect(reverse(downloads))
 
 
-@login_required
-def download_errors_delete(request: http.HttpRequest):
-    form = forms.RequeueForm(request.POST)
-
-    if form.is_valid():
-        freg = queue.get_failed_job_registry()
-        freg.remove(str(form.cleaned_data["id"]), delete_job=True)
-
-    return http.HttpResponseRedirect(reverse(downloads))
-
-
-@login_required
-def download_errors_delete_all(request: http.HttpRequest):
-    freg = queue.get_failed_job_registry()
-    for job_id in freg.get_job_ids():
-        freg.remove(job_id, delete_job=True)
-
-    return http.HttpResponseRedirect(reverse(downloads))
-
-
 @login_required
 def channels_opml(request: http.HttpRequest):
     response = http.HttpResponse(