diff --git a/.drone.yml b/.drone.yml index 297b7c6..ebe4a65 100644 --- a/.drone.yml +++ b/.drone.yml @@ -36,27 +36,6 @@ steps: depends_on: - install dependencies - - name: build container - image: quay.io/buildah/stable - when: - event: - - tag - commands: - - buildah login -u $DOCKER_USER -p $DOCKER_PASS -- $DOCKER_REGISTRY - - buildah manifest create ucast - - buildah bud --tag code.thetadev.de/hsa/ucast:latest --manifest ucast --arch amd64 -f deploy/Dockerfile . - - buildah bud --tag code.thetadev.de/hsa/ucast:latest --manifest ucast --arch arm64 -f deploy/Dockerfile . - - buildah manifest push --all ucast docker://code.thetadev.de/hsa/ucast:latest - environment: - DOCKER_REGISTRY: - from_secret: docker_registry - DOCKER_USER: - from_secret: docker_username - DOCKER_PASS: - from_secret: docker_password - depends_on: - - test - volumes: - name: cache temp: { } diff --git a/ucast/tasks/download.py b/ucast/tasks/download.py index d3bbe81..be14ddb 100644 --- a/ucast/tasks/download.py +++ b/ucast/tasks/download.py @@ -3,7 +3,6 @@ import os from django.db.models import ObjectDoesNotExist from django.utils import timezone -from yt_dlp.utils import DownloadError from ucast import queue from ucast.models import Channel, Video @@ -22,25 +21,7 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): try: video = Video.objects.get(video_id=vid.id) except ObjectDoesNotExist: - try: - details = youtube.get_video_details(vid.id) - except DownloadError as e: - if "available" in e.msg: - # Create dummy video to prevent further download attempts - # of unavailable videos - video = Video( - video_id=vid.id, - title="", - slug="", - channel=channel, - published=timezone.datetime(2000, 1, 1, tzinfo=timezone.utc), - description="", - duration=0, - is_deleted=True, - ) - video.save() - return - raise e + details = youtube.get_video_details(vid.id) # Dont load active livestreams if details.is_currently_live: @@ -69,23 +50,20 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): and video.is_deleted is False and channel.should_download(video) ): - queue.enqueue(download_video, video.id) + queue.enqueue(download_video, video) redis.delete(lock_key) -def download_video(v_id: int): +def download_video(video: Video): """ Download a video including its thumbnail, create a cover image and store everything in the channel folder. - :param v_id: Video ID + :param video: Video object """ # Return if the video was already downloaded by a previous task - try: - video = Video.objects.get(id=v_id) - except ObjectDoesNotExist: - return + video.refresh_from_db() if video.downloaded: return @@ -93,14 +71,7 @@ def download_video(v_id: int): channel_folder = store.get_or_create_channel_folder(video.channel.slug) audio_file = channel_folder.get_audio(video.slug) - try: - details = youtube.download_audio(video.video_id, audio_file) - except DownloadError as e: - if "available" in e.msg: - video.is_deleted = True - video.save() - return - raise e + details = youtube.download_audio(video.video_id, audio_file) # Download/convert thumbnails tn_path = channel_folder.get_thumbnail(video.slug) @@ -135,12 +106,8 @@ def download_video(v_id: int): video.save() -def update_channel(c_id: int): +def update_channel(channel: Channel): """Update a single channel from its RSS feed""" - try: - channel = Channel.objects.get(id=c_id) - except ObjectDoesNotExist: - return videos = youtube.get_channel_videos_from_feed(channel.channel_id) for vid in videos: @@ -156,23 +123,18 @@ def update_channels(): This task is scheduled a regular intervals. """ for channel in Channel.objects.filter(active=True): - queue.enqueue(update_channel, channel.id) + queue.enqueue(update_channel, channel) -def download_channel(c_id: int, limit: int): +def download_channel(channel: Channel, limit: int): """ Download maximum number of videos from a channel. - :param c_id: Channel ID (Database) + :param channel: Channel object :param limit: Max number of videos """ if limit < 1: return - try: - channel = Channel.objects.get(id=c_id) - except ObjectDoesNotExist: - return - for vid in youtube.get_channel_videos_from_scraper(channel.channel_id, limit): _load_scraped_video(vid, channel) diff --git a/ucast/tasks/library.py b/ucast/tasks/library.py index 797ea49..b6b3481 100644 --- a/ucast/tasks/library.py +++ b/ucast/tasks/library.py @@ -1,6 +1,5 @@ import os -from django.db.models import ObjectDoesNotExist from django.utils import timezone from ucast import queue @@ -8,12 +7,7 @@ from ucast.models import Channel, Video from ucast.service import cover, storage, util, videoutil, youtube -def recreate_cover(v_id: int): - try: - video = Video.objects.get(id=v_id) - except ObjectDoesNotExist: - return - +def recreate_cover(video: Video): store = storage.Storage() cf = store.get_channel_folder(video.channel.slug) @@ -48,7 +42,7 @@ def recreate_cover(v_id: int): def recreate_covers(): for video in Video.objects.filter(downloaded__isnull=False): - queue.enqueue(recreate_cover, video.id) + queue.enqueue(recreate_cover, video) def update_file_storage(): @@ -87,12 +81,7 @@ def update_file_storage(): video.save() -def update_channel_info(ch_id: int): - try: - channel = Channel.objects.get(id=ch_id) - except ObjectDoesNotExist: - return - +def update_channel_info(channel: Channel): channel_data = youtube.get_channel_metadata( youtube.channel_url_from_id(channel.channel_id) ) @@ -115,4 +104,4 @@ def update_channel_info(ch_id: int): def update_channel_infos(): for channel in Channel.objects.filter(active=True): - queue.enqueue(update_channel_info, channel.id) + queue.enqueue(update_channel_info, channel) diff --git a/ucast/templates/ucast/downloads.html b/ucast/templates/ucast/downloads.html index 9f51790..9cbe830 100644 --- a/ucast/templates/ucast/downloads.html +++ b/ucast/templates/ucast/downloads.html @@ -27,15 +27,11 @@
{% if failed_jobs %} -
+
{% csrf_token %}
-
- {% csrf_token %} - -
@@ -45,7 +41,6 @@ - @@ -61,13 +56,6 @@ - {% endfor %} diff --git a/ucast/templates/ucast/error_details.html b/ucast/templates/ucast/error_details.html index 4928008..04f89fa 100644 --- a/ucast/templates/ucast/error_details.html +++ b/ucast/templates/ucast/error_details.html @@ -16,17 +16,12 @@ {{ job.exc_info }} -
+
{% csrf_token %} -
- {% csrf_token %} - - -
{% endblock content %} diff --git a/ucast/templates/ucast/videos.html b/ucast/templates/ucast/videos.html index a491a8b..d43569c 100644 --- a/ucast/templates/ucast/videos.html +++ b/ucast/templates/ucast/videos.html @@ -13,11 +13,7 @@   {{ channel.subscribers }}   {{ videos.paginator.count }} - {% if n_pending %} - ({{ n_pending }}) - {% endif %} - + class="fas fa-video">  {{ videos.paginator.count }}   {{ channel.download_size|filesizeformat }}
-
+
- @@ -65,38 +60,28 @@
- {% if not videos %} - {% if n_pending %} -

There are {{ n_pending }} videos waiting to be downloaded. - Please wait a few minutes and refesh this page. - You can see the current status in the Downloads tab. -

- {% else %} -

No videos. If you have just added this channel, - you have to wait a minute for ucast to start looking for videos.

+
+ {% if not videos %} +

No videos

{% endif %} - {% else %} -
- {% include "ucast/videos_items.html" %} -
- {% endif %} + {% include "ucast/videos_items.html" %} +
{% if videos.has_previous or videos.has_next %} - + {% endif %} {% endblock content %} diff --git a/ucast/tests/tasks/test_download.py b/ucast/tests/tasks/test_download.py index 8d12f27..dc6fd6c 100644 --- a/ucast/tests/tasks/test_download.py +++ b/ucast/tests/tasks/test_download.py @@ -1,24 +1,21 @@ import os import pytest -from django.utils import timezone from ucast import queue, tests from ucast.models import Channel, Video from ucast.service import storage -from ucast.service.youtube import VideoScraped from ucast.tasks import download CHANNEL_ID_THETADEV = "UCGiJh0NZ52wRhYKYnuZI08Q" VIDEO_ID_INTRO = "I0RRENheeTo" VIDEO_SLUG_INTRO = "20211010_No_copyright_intro_free_fire_intro_no_text_free_copy_right_free_templates_free_download" -VIDEO_ID_UNAVAILABLE = "K6CBuTy09CE" @pytest.mark.django_db def test_download_video(download_dir, rq_queue): video = Video.objects.get(video_id=VIDEO_ID_INTRO) - job = queue.enqueue(download.download_video, video.id) + job = queue.enqueue(download.download_video, video) store = storage.Storage() cf = store.get_or_create_channel_folder(video.channel.slug) @@ -31,35 +28,6 @@ def test_download_video(download_dir, rq_queue): assert os.path.isfile(cf.get_thumbnail(VIDEO_SLUG_INTRO, True)) -@pytest.mark.django_db -def test_load_unavailable_video(download_dir, rq_queue, mock_redis): - channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV) - download._load_scraped_video(VideoScraped(VIDEO_ID_UNAVAILABLE, None), channel) - - video = Video.objects.get(video_id=VIDEO_ID_UNAVAILABLE) - assert video.is_deleted is True - - -@pytest.mark.django_db -def test_download_unavailable_video(download_dir, rq_queue): - channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV) - video = Video( - video_id=VIDEO_ID_UNAVAILABLE, - title="", - slug="", - channel=channel, - published=timezone.datetime(2000, 1, 1, tzinfo=timezone.utc), - description="", - duration=0, - ) - video.save() - job = queue.enqueue(download.download_video, video.id) - video.refresh_from_db() - - assert job.is_finished - assert video.is_deleted - - @pytest.mark.django_db def test_update_channel( download_dir, rq_queue, mock_redis, mock_get_video_details, mock_download_audio @@ -69,7 +37,7 @@ def test_update_channel( Video.objects.get(video_id="_I5IFObm_-k").delete() channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV) - job = rq_queue.enqueue(download.update_channel, channel.id) + job = rq_queue.enqueue(download.update_channel, channel) assert job.is_finished mock_download_audio.assert_any_call( diff --git a/ucast/tests/tasks/test_library.py b/ucast/tests/tasks/test_library.py index 19b213b..ddbc605 100644 --- a/ucast/tests/tasks/test_library.py +++ b/ucast/tests/tasks/test_library.py @@ -19,7 +19,7 @@ def test_recreate_cover(download_dir_content_mut, rq_queue, mocker): store = storage.Storage() cf = store.get_or_create_channel_folder(video.channel.slug) - job = rq_queue.enqueue(library.recreate_cover, video.id) + job = rq_queue.enqueue(library.recreate_cover, video) assert job.is_finished create_cover_mock.assert_called_once_with( @@ -53,7 +53,7 @@ def test_update_channel_info(rq_queue, mock_get_channel_metadata): channel.avatar_url = "Old avatar url" channel.save() - job = rq_queue.enqueue(library.update_channel_info, channel.id) + job = rq_queue.enqueue(library.update_channel_info, channel) assert job.is_finished channel.refresh_from_db() diff --git a/ucast/urls.py b/ucast/urls.py index 35fbc2d..327f61d 100644 --- a/ucast/urls.py +++ b/ucast/urls.py @@ -28,16 +28,6 @@ urlpatterns = [ views.download_errors_requeue_all, name="download_errors_requeue_all", ), - path( - "downloads/delete", - views.download_errors_delete, - name="download_errors_delete", - ), - path( - "downloads/delete_all", - views.download_errors_delete_all, - name="download_errors_delete_all", - ), path("downloads/error/", views.error_details, name="error_details"), path("feed/", views.podcast_feed, name="feed"), path("opml", views.channels_opml, name="channels_opml"), diff --git a/ucast/views.py b/ucast/views.py index 0850a17..415dd9f 100644 --- a/ucast/views.py +++ b/ucast/views.py @@ -33,7 +33,7 @@ def home(request: http.HttpRequest): channel_str = form.cleaned_data["channel_str"] try: channel = controller.create_channel(channel_str) - queue.enqueue(download.update_channel, channel.id) + queue.enqueue(download.update_channel, channel) except ValueError: form.add_error("channel_str", "Channel URL invalid") except controller.ChannelAlreadyExistsException: @@ -91,10 +91,6 @@ def videos(request: http.HttpRequest, channel: str): if request.htmx: template_name = "ucast/videos_items.html" - n_pending = Video.objects.filter( - channel=chan, downloaded__isnull=True, is_deleted=False - ).count() - return render( request, template_name, @@ -102,7 +98,6 @@ def videos(request: http.HttpRequest, channel: str): "videos": videos_p.get_page(page_number), "channel": chan, "site_url": site_url, - "n_pending": n_pending, }, ) @@ -144,7 +139,7 @@ def channel_download(request: http.HttpRequest, channel: str): form = forms.DownloadChannelForm(request.POST) if form.is_valid(): queue.enqueue( - download.download_channel, chan.id, form.cleaned_data["n_videos"] + download.download_channel, chan, form.cleaned_data["n_videos"] ) return http.HttpResponseRedirect(reverse(videos, args=[channel])) @@ -209,26 +204,6 @@ def download_errors_requeue_all(request: http.HttpRequest): return http.HttpResponseRedirect(reverse(downloads)) -@login_required -def download_errors_delete(request: http.HttpRequest): - form = forms.RequeueForm(request.POST) - - if form.is_valid(): - freg = queue.get_failed_job_registry() - freg.remove(str(form.cleaned_data["id"]), delete_job=True) - - return http.HttpResponseRedirect(reverse(downloads)) - - -@login_required -def download_errors_delete_all(request: http.HttpRequest): - freg = queue.get_failed_job_registry() - for job_id in freg.get_job_ids(): - freg.remove(job_id, delete_job=True) - - return http.HttpResponseRedirect(reverse(downloads)) - - @login_required def channels_opml(request: http.HttpRequest): response = http.HttpResponse(
Function Details RequeueDelete
-
- {% csrf_token %} - - -
-