diff --git a/.drone.yml b/.drone.yml index ebe4a65..297b7c6 100644 --- a/.drone.yml +++ b/.drone.yml @@ -36,6 +36,27 @@ steps: depends_on: - install dependencies + - name: build container + image: quay.io/buildah/stable + when: + event: + - tag + commands: + - buildah login -u $DOCKER_USER -p $DOCKER_PASS -- $DOCKER_REGISTRY + - buildah manifest create ucast + - buildah bud --tag code.thetadev.de/hsa/ucast:latest --manifest ucast --arch amd64 -f deploy/Dockerfile . + - buildah bud --tag code.thetadev.de/hsa/ucast:latest --manifest ucast --arch arm64 -f deploy/Dockerfile . + - buildah manifest push --all ucast docker://code.thetadev.de/hsa/ucast:latest + environment: + DOCKER_REGISTRY: + from_secret: docker_registry + DOCKER_USER: + from_secret: docker_username + DOCKER_PASS: + from_secret: docker_password + depends_on: + - test + volumes: - name: cache temp: { } diff --git a/ucast/tasks/download.py b/ucast/tasks/download.py index be14ddb..d3bbe81 100644 --- a/ucast/tasks/download.py +++ b/ucast/tasks/download.py @@ -3,6 +3,7 @@ import os from django.db.models import ObjectDoesNotExist from django.utils import timezone +from yt_dlp.utils import DownloadError from ucast import queue from ucast.models import Channel, Video @@ -21,7 +22,25 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): try: video = Video.objects.get(video_id=vid.id) except ObjectDoesNotExist: - details = youtube.get_video_details(vid.id) + try: + details = youtube.get_video_details(vid.id) + except DownloadError as e: + if "available" in e.msg: + # Create dummy video to prevent further download attempts + # of unavailable videos + video = Video( + video_id=vid.id, + title="", + slug="", + channel=channel, + published=timezone.datetime(2000, 1, 1, tzinfo=timezone.utc), + description="", + duration=0, + is_deleted=True, + ) + video.save() + return + raise e # Dont load active livestreams if details.is_currently_live: @@ -50,20 +69,23 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): and video.is_deleted is False and channel.should_download(video) ): - queue.enqueue(download_video, video) + queue.enqueue(download_video, video.id) redis.delete(lock_key) -def download_video(video: Video): +def download_video(v_id: int): """ Download a video including its thumbnail, create a cover image and store everything in the channel folder. - :param video: Video object + :param v_id: Video ID """ # Return if the video was already downloaded by a previous task - video.refresh_from_db() + try: + video = Video.objects.get(id=v_id) + except ObjectDoesNotExist: + return if video.downloaded: return @@ -71,7 +93,14 @@ def download_video(video: Video): channel_folder = store.get_or_create_channel_folder(video.channel.slug) audio_file = channel_folder.get_audio(video.slug) - details = youtube.download_audio(video.video_id, audio_file) + try: + details = youtube.download_audio(video.video_id, audio_file) + except DownloadError as e: + if "available" in e.msg: + video.is_deleted = True + video.save() + return + raise e # Download/convert thumbnails tn_path = channel_folder.get_thumbnail(video.slug) @@ -106,8 +135,12 @@ def download_video(video: Video): video.save() -def update_channel(channel: Channel): +def update_channel(c_id: int): """Update a single channel from its RSS feed""" + try: + channel = Channel.objects.get(id=c_id) + except ObjectDoesNotExist: + return videos = youtube.get_channel_videos_from_feed(channel.channel_id) for vid in videos: @@ -123,18 +156,23 @@ def update_channels(): This task is scheduled a regular intervals. """ for channel in Channel.objects.filter(active=True): - queue.enqueue(update_channel, channel) + queue.enqueue(update_channel, channel.id) -def download_channel(channel: Channel, limit: int): +def download_channel(c_id: int, limit: int): """ Download maximum number of videos from a channel. - :param channel: Channel object + :param c_id: Channel ID (Database) :param limit: Max number of videos """ if limit < 1: return + try: + channel = Channel.objects.get(id=c_id) + except ObjectDoesNotExist: + return + for vid in youtube.get_channel_videos_from_scraper(channel.channel_id, limit): _load_scraped_video(vid, channel) diff --git a/ucast/tasks/library.py b/ucast/tasks/library.py index b6b3481..797ea49 100644 --- a/ucast/tasks/library.py +++ b/ucast/tasks/library.py @@ -1,5 +1,6 @@ import os +from django.db.models import ObjectDoesNotExist from django.utils import timezone from ucast import queue @@ -7,7 +8,12 @@ from ucast.models import Channel, Video from ucast.service import cover, storage, util, videoutil, youtube -def recreate_cover(video: Video): +def recreate_cover(v_id: int): + try: + video = Video.objects.get(id=v_id) + except ObjectDoesNotExist: + return + store = storage.Storage() cf = store.get_channel_folder(video.channel.slug) @@ -42,7 +48,7 @@ def recreate_cover(video: Video): def recreate_covers(): for video in Video.objects.filter(downloaded__isnull=False): - queue.enqueue(recreate_cover, video) + queue.enqueue(recreate_cover, video.id) def update_file_storage(): @@ -81,7 +87,12 @@ def update_file_storage(): video.save() -def update_channel_info(channel: Channel): +def update_channel_info(ch_id: int): + try: + channel = Channel.objects.get(id=ch_id) + except ObjectDoesNotExist: + return + channel_data = youtube.get_channel_metadata( youtube.channel_url_from_id(channel.channel_id) ) @@ -104,4 +115,4 @@ def update_channel_info(channel: Channel): def update_channel_infos(): for channel in Channel.objects.filter(active=True): - queue.enqueue(update_channel_info, channel) + queue.enqueue(update_channel_info, channel.id) diff --git a/ucast/templates/ucast/downloads.html b/ucast/templates/ucast/downloads.html index 9cbe830..9f51790 100644 --- a/ucast/templates/ucast/downloads.html +++ b/ucast/templates/ucast/downloads.html @@ -27,11 +27,15 @@
{% if failed_jobs %} -
+
{% csrf_token %}
+
+ {% csrf_token %} + +
@@ -41,6 +45,7 @@ + @@ -56,6 +61,13 @@ + {% endfor %} diff --git a/ucast/templates/ucast/error_details.html b/ucast/templates/ucast/error_details.html index 04f89fa..4928008 100644 --- a/ucast/templates/ucast/error_details.html +++ b/ucast/templates/ucast/error_details.html @@ -16,12 +16,17 @@ {{ job.exc_info }} -
+
{% csrf_token %} +
+ {% csrf_token %} + + +
{% endblock content %} diff --git a/ucast/templates/ucast/videos.html b/ucast/templates/ucast/videos.html index d43569c..a491a8b 100644 --- a/ucast/templates/ucast/videos.html +++ b/ucast/templates/ucast/videos.html @@ -13,7 +13,11 @@   {{ channel.subscribers }}   {{ videos.paginator.count }} + class="fas fa-video">  {{ videos.paginator.count }} + {% if n_pending %} + ({{ n_pending }}) + {% endif %} +   {{ channel.download_size|filesizeformat }}
-
+
- @@ -60,28 +65,38 @@
-
- {% if not videos %} -

No videos

+ {% if not videos %} + {% if n_pending %} +

There are {{ n_pending }} videos waiting to be downloaded. + Please wait a few minutes and refesh this page. + You can see the current status in the Downloads tab. +

+ {% else %} +

No videos. If you have just added this channel, + you have to wait a minute for ucast to start looking for videos.

{% endif %} - {% include "ucast/videos_items.html" %} -
+ {% else %} +
+ {% include "ucast/videos_items.html" %} +
+ {% endif %} {% if videos.has_previous or videos.has_next %} - + {% endif %} {% endblock content %} diff --git a/ucast/tests/tasks/test_download.py b/ucast/tests/tasks/test_download.py index dc6fd6c..8d12f27 100644 --- a/ucast/tests/tasks/test_download.py +++ b/ucast/tests/tasks/test_download.py @@ -1,21 +1,24 @@ import os import pytest +from django.utils import timezone from ucast import queue, tests from ucast.models import Channel, Video from ucast.service import storage +from ucast.service.youtube import VideoScraped from ucast.tasks import download CHANNEL_ID_THETADEV = "UCGiJh0NZ52wRhYKYnuZI08Q" VIDEO_ID_INTRO = "I0RRENheeTo" VIDEO_SLUG_INTRO = "20211010_No_copyright_intro_free_fire_intro_no_text_free_copy_right_free_templates_free_download" +VIDEO_ID_UNAVAILABLE = "K6CBuTy09CE" @pytest.mark.django_db def test_download_video(download_dir, rq_queue): video = Video.objects.get(video_id=VIDEO_ID_INTRO) - job = queue.enqueue(download.download_video, video) + job = queue.enqueue(download.download_video, video.id) store = storage.Storage() cf = store.get_or_create_channel_folder(video.channel.slug) @@ -28,6 +31,35 @@ def test_download_video(download_dir, rq_queue): assert os.path.isfile(cf.get_thumbnail(VIDEO_SLUG_INTRO, True)) +@pytest.mark.django_db +def test_load_unavailable_video(download_dir, rq_queue, mock_redis): + channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV) + download._load_scraped_video(VideoScraped(VIDEO_ID_UNAVAILABLE, None), channel) + + video = Video.objects.get(video_id=VIDEO_ID_UNAVAILABLE) + assert video.is_deleted is True + + +@pytest.mark.django_db +def test_download_unavailable_video(download_dir, rq_queue): + channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV) + video = Video( + video_id=VIDEO_ID_UNAVAILABLE, + title="", + slug="", + channel=channel, + published=timezone.datetime(2000, 1, 1, tzinfo=timezone.utc), + description="", + duration=0, + ) + video.save() + job = queue.enqueue(download.download_video, video.id) + video.refresh_from_db() + + assert job.is_finished + assert video.is_deleted + + @pytest.mark.django_db def test_update_channel( download_dir, rq_queue, mock_redis, mock_get_video_details, mock_download_audio @@ -37,7 +69,7 @@ def test_update_channel( Video.objects.get(video_id="_I5IFObm_-k").delete() channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV) - job = rq_queue.enqueue(download.update_channel, channel) + job = rq_queue.enqueue(download.update_channel, channel.id) assert job.is_finished mock_download_audio.assert_any_call( diff --git a/ucast/tests/tasks/test_library.py b/ucast/tests/tasks/test_library.py index ddbc605..19b213b 100644 --- a/ucast/tests/tasks/test_library.py +++ b/ucast/tests/tasks/test_library.py @@ -19,7 +19,7 @@ def test_recreate_cover(download_dir_content_mut, rq_queue, mocker): store = storage.Storage() cf = store.get_or_create_channel_folder(video.channel.slug) - job = rq_queue.enqueue(library.recreate_cover, video) + job = rq_queue.enqueue(library.recreate_cover, video.id) assert job.is_finished create_cover_mock.assert_called_once_with( @@ -53,7 +53,7 @@ def test_update_channel_info(rq_queue, mock_get_channel_metadata): channel.avatar_url = "Old avatar url" channel.save() - job = rq_queue.enqueue(library.update_channel_info, channel) + job = rq_queue.enqueue(library.update_channel_info, channel.id) assert job.is_finished channel.refresh_from_db() diff --git a/ucast/urls.py b/ucast/urls.py index 327f61d..35fbc2d 100644 --- a/ucast/urls.py +++ b/ucast/urls.py @@ -28,6 +28,16 @@ urlpatterns = [ views.download_errors_requeue_all, name="download_errors_requeue_all", ), + path( + "downloads/delete", + views.download_errors_delete, + name="download_errors_delete", + ), + path( + "downloads/delete_all", + views.download_errors_delete_all, + name="download_errors_delete_all", + ), path("downloads/error/", views.error_details, name="error_details"), path("feed/", views.podcast_feed, name="feed"), path("opml", views.channels_opml, name="channels_opml"), diff --git a/ucast/views.py b/ucast/views.py index 415dd9f..0850a17 100644 --- a/ucast/views.py +++ b/ucast/views.py @@ -33,7 +33,7 @@ def home(request: http.HttpRequest): channel_str = form.cleaned_data["channel_str"] try: channel = controller.create_channel(channel_str) - queue.enqueue(download.update_channel, channel) + queue.enqueue(download.update_channel, channel.id) except ValueError: form.add_error("channel_str", "Channel URL invalid") except controller.ChannelAlreadyExistsException: @@ -91,6 +91,10 @@ def videos(request: http.HttpRequest, channel: str): if request.htmx: template_name = "ucast/videos_items.html" + n_pending = Video.objects.filter( + channel=chan, downloaded__isnull=True, is_deleted=False + ).count() + return render( request, template_name, @@ -98,6 +102,7 @@ def videos(request: http.HttpRequest, channel: str): "videos": videos_p.get_page(page_number), "channel": chan, "site_url": site_url, + "n_pending": n_pending, }, ) @@ -139,7 +144,7 @@ def channel_download(request: http.HttpRequest, channel: str): form = forms.DownloadChannelForm(request.POST) if form.is_valid(): queue.enqueue( - download.download_channel, chan, form.cleaned_data["n_videos"] + download.download_channel, chan.id, form.cleaned_data["n_videos"] ) return http.HttpResponseRedirect(reverse(videos, args=[channel])) @@ -204,6 +209,26 @@ def download_errors_requeue_all(request: http.HttpRequest): return http.HttpResponseRedirect(reverse(downloads)) +@login_required +def download_errors_delete(request: http.HttpRequest): + form = forms.RequeueForm(request.POST) + + if form.is_valid(): + freg = queue.get_failed_job_registry() + freg.remove(str(form.cleaned_data["id"]), delete_job=True) + + return http.HttpResponseRedirect(reverse(downloads)) + + +@login_required +def download_errors_delete_all(request: http.HttpRequest): + freg = queue.get_failed_job_registry() + for job_id in freg.get_job_ids(): + freg.remove(job_id, delete_job=True) + + return http.HttpResponseRedirect(reverse(downloads)) + + @login_required def channels_opml(request: http.HttpRequest): response = http.HttpResponse(
Function Details RequeueDelete
+
+ {% csrf_token %} + + +
+