Compare commits

..

3 commits

Author SHA1 Message Date
3479365c52 publish docker images with ci
All checks were successful
continuous-integration/drone/push Build is passing
2022-07-05 13:56:07 +02:00
9d53d79f95 show number of pending videos 2022-07-05 13:33:20 +02:00
83e1d9a406 add deletion of jobs
handle unavailable videos
2022-07-05 13:17:10 +02:00
10 changed files with 214 additions and 45 deletions

View file

@ -36,6 +36,27 @@ steps:
depends_on:
- install dependencies
- name: build container
image: quay.io/buildah/stable
when:
event:
- tag
commands:
- buildah login -u $DOCKER_USER -p $DOCKER_PASS -- $DOCKER_REGISTRY
- buildah manifest create ucast
- buildah bud --tag code.thetadev.de/hsa/ucast:latest --manifest ucast --arch amd64 -f deploy/Dockerfile .
- buildah bud --tag code.thetadev.de/hsa/ucast:latest --manifest ucast --arch arm64 -f deploy/Dockerfile .
- buildah manifest push --all ucast docker://code.thetadev.de/hsa/ucast:latest
environment:
DOCKER_REGISTRY:
from_secret: docker_registry
DOCKER_USER:
from_secret: docker_username
DOCKER_PASS:
from_secret: docker_password
depends_on:
- test
volumes:
- name: cache
temp: { }

View file

@ -3,6 +3,7 @@ import os
from django.db.models import ObjectDoesNotExist
from django.utils import timezone
from yt_dlp.utils import DownloadError
from ucast import queue
from ucast.models import Channel, Video
@ -21,7 +22,25 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel):
try:
video = Video.objects.get(video_id=vid.id)
except ObjectDoesNotExist:
details = youtube.get_video_details(vid.id)
try:
details = youtube.get_video_details(vid.id)
except DownloadError as e:
if "available" in e.msg:
# Create dummy video to prevent further download attempts
# of unavailable videos
video = Video(
video_id=vid.id,
title="",
slug="",
channel=channel,
published=timezone.datetime(2000, 1, 1, tzinfo=timezone.utc),
description="",
duration=0,
is_deleted=True,
)
video.save()
return
raise e
# Dont load active livestreams
if details.is_currently_live:
@ -50,20 +69,23 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel):
and video.is_deleted is False
and channel.should_download(video)
):
queue.enqueue(download_video, video)
queue.enqueue(download_video, video.id)
redis.delete(lock_key)
def download_video(video: Video):
def download_video(v_id: int):
"""
Download a video including its thumbnail, create a cover image
and store everything in the channel folder.
:param video: Video object
:param v_id: Video ID
"""
# Return if the video was already downloaded by a previous task
video.refresh_from_db()
try:
video = Video.objects.get(id=v_id)
except ObjectDoesNotExist:
return
if video.downloaded:
return
@ -71,7 +93,14 @@ def download_video(video: Video):
channel_folder = store.get_or_create_channel_folder(video.channel.slug)
audio_file = channel_folder.get_audio(video.slug)
details = youtube.download_audio(video.video_id, audio_file)
try:
details = youtube.download_audio(video.video_id, audio_file)
except DownloadError as e:
if "available" in e.msg:
video.is_deleted = True
video.save()
return
raise e
# Download/convert thumbnails
tn_path = channel_folder.get_thumbnail(video.slug)
@ -106,8 +135,12 @@ def download_video(video: Video):
video.save()
def update_channel(channel: Channel):
def update_channel(c_id: int):
"""Update a single channel from its RSS feed"""
try:
channel = Channel.objects.get(id=c_id)
except ObjectDoesNotExist:
return
videos = youtube.get_channel_videos_from_feed(channel.channel_id)
for vid in videos:
@ -123,18 +156,23 @@ def update_channels():
This task is scheduled a regular intervals.
"""
for channel in Channel.objects.filter(active=True):
queue.enqueue(update_channel, channel)
queue.enqueue(update_channel, channel.id)
def download_channel(channel: Channel, limit: int):
def download_channel(c_id: int, limit: int):
"""
Download maximum number of videos from a channel.
:param channel: Channel object
:param c_id: Channel ID (Database)
:param limit: Max number of videos
"""
if limit < 1:
return
try:
channel = Channel.objects.get(id=c_id)
except ObjectDoesNotExist:
return
for vid in youtube.get_channel_videos_from_scraper(channel.channel_id, limit):
_load_scraped_video(vid, channel)

View file

@ -1,5 +1,6 @@
import os
from django.db.models import ObjectDoesNotExist
from django.utils import timezone
from ucast import queue
@ -7,7 +8,12 @@ from ucast.models import Channel, Video
from ucast.service import cover, storage, util, videoutil, youtube
def recreate_cover(video: Video):
def recreate_cover(v_id: int):
try:
video = Video.objects.get(id=v_id)
except ObjectDoesNotExist:
return
store = storage.Storage()
cf = store.get_channel_folder(video.channel.slug)
@ -42,7 +48,7 @@ def recreate_cover(video: Video):
def recreate_covers():
for video in Video.objects.filter(downloaded__isnull=False):
queue.enqueue(recreate_cover, video)
queue.enqueue(recreate_cover, video.id)
def update_file_storage():
@ -81,7 +87,12 @@ def update_file_storage():
video.save()
def update_channel_info(channel: Channel):
def update_channel_info(ch_id: int):
try:
channel = Channel.objects.get(id=ch_id)
except ObjectDoesNotExist:
return
channel_data = youtube.get_channel_metadata(
youtube.channel_url_from_id(channel.channel_id)
)
@ -104,4 +115,4 @@ def update_channel_info(channel: Channel):
def update_channel_infos():
for channel in Channel.objects.filter(active=True):
queue.enqueue(update_channel_info, channel)
queue.enqueue(update_channel_info, channel.id)

View file

@ -27,11 +27,15 @@
<div class="mb-4">
{% if failed_jobs %}
<div class="mb-4">
<div class="level mb-4">
<form method="post" action="{% url 'download_errors_requeue_all' %}">
{% csrf_token %}
<button class="button is-primary">Requeue all</button>
</form>
<form method="post" action="{% url 'download_errors_delete_all' %}">
{% csrf_token %}
<button class="button is-danger">Delete all</button>
</form>
</div>
<table class="table">
@ -41,6 +45,7 @@
<th>Function</th>
<th>Details</th>
<th>Requeue</th>
<th>Delete</th>
</tr>
</thead>
<tbody>
@ -56,6 +61,13 @@
<button class="button is-small">Requeue</button>
</form>
</td>
<td>
<form method="post" action="{% url 'download_errors_delete' %}">
{% csrf_token %}
<input type="hidden" name="id" value="{{ job.id }}">
<button class="button is-small is-danger">Delete</button>
</form>
</td>
</tr>
{% endfor %}
</tbody>

View file

@ -16,12 +16,17 @@
{{ job.exc_info }}
</pre>
<div>
<div class="level">
<form method="post" action="{% url 'download_errors_requeue' %}">
{% csrf_token %}
<input type="hidden" name="id" value="{{ job.id }}">
<button class="button is-primary">Requeue</button>
</form>
<form method="post" action="{% url 'download_errors_delete' %}">
{% csrf_token %}
<input type="hidden" name="id" value="{{ job.id }}">
<button class="button is-danger">Delete</button>
</form>
</div>
{% endblock content %}

View file

@ -13,7 +13,11 @@
<span class="tag"><i
class="fas fa-user-group"></i>&nbsp; {{ channel.subscribers }}</span>
<span class="tag"><i
class="fas fa-video"></i>&nbsp; {{ videos.paginator.count }}</span>
class="fas fa-video"></i>&nbsp; {{ videos.paginator.count }}
{% if n_pending %}
({{ n_pending }})
{% endif %}
</span>
<span class="tag"><i
class="fas fa-database"></i>&nbsp; {{ channel.download_size|filesizeformat }}</span>
<a class="tag" href="{{ channel.get_absolute_url }}" target="_blank"><i
@ -44,13 +48,14 @@
<i class="fas fa-edit"></i>
</a>
</div>
<div class="control">
<div class="control">
<a class="button is-info" href="{% url 'channel_download' channel.slug %}">
<i class="fas fa-download"></i>
</a>
</div>
<div class="control">
<button type="submit" name="delete_channel" class="button is-danger dialog-confirm"
<button type="submit" name="delete_channel"
class="button is-danger dialog-confirm"
confirm-msg="Do you want to delete the channel '{{ channel.name }}' including {{ videos|length }} videos?">
<i class="fas fa-trash"></i>
</button>
@ -60,28 +65,38 @@
</form>
</div>
<div class="video-grid">
{% if not videos %}
<p>No videos</p>
{% if not videos %}
{% if n_pending %}
<p>There are {{ n_pending }} videos waiting to be downloaded.
Please wait a few minutes and refesh this page.
You can see the current status in the <i>Downloads</i> tab.
</p>
{% else %}
<p>No videos. If you have just added this channel,
you have to wait a minute for ucast to start looking for videos.</p>
{% endif %}
{% include "ucast/videos_items.html" %}
</div>
{% else %}
<div class="video-grid">
{% include "ucast/videos_items.html" %}
</div>
{% endif %}
{% if videos.has_previous or videos.has_next %}
<noscript>
<nav class="pagination is-centered mt-4" role="navigation" aria-label="pagination">
{% if videos.has_previous %}
<a class="pagination-previous" href="?page={{ videos.previous_page_number }}">Previous</a>
{% else %}
<a class="pagination-previous" disabled>Previous</a>
{% endif %}
{% if videos.has_next %}
<a class="pagination-next" href="?page={{ videos.next_page_number }}">Next
page</a>
{% else %}
<a class="pagination-previous" disabled>Previous</a>
{% endif %}
</nav>
</noscript>
<noscript>
<nav class="pagination is-centered mt-4" role="navigation"
aria-label="pagination">
{% if videos.has_previous %}
<a class="pagination-previous" href="?page={{ videos.previous_page_number }}">Previous</a>
{% else %}
<a class="pagination-previous" disabled>Previous</a>
{% endif %}
{% if videos.has_next %}
<a class="pagination-next" href="?page={{ videos.next_page_number }}">Next
page</a>
{% else %}
<a class="pagination-previous" disabled>Previous</a>
{% endif %}
</nav>
</noscript>
{% endif %}
{% endblock content %}

View file

@ -1,21 +1,24 @@
import os
import pytest
from django.utils import timezone
from ucast import queue, tests
from ucast.models import Channel, Video
from ucast.service import storage
from ucast.service.youtube import VideoScraped
from ucast.tasks import download
CHANNEL_ID_THETADEV = "UCGiJh0NZ52wRhYKYnuZI08Q"
VIDEO_ID_INTRO = "I0RRENheeTo"
VIDEO_SLUG_INTRO = "20211010_No_copyright_intro_free_fire_intro_no_text_free_copy_right_free_templates_free_download"
VIDEO_ID_UNAVAILABLE = "K6CBuTy09CE"
@pytest.mark.django_db
def test_download_video(download_dir, rq_queue):
video = Video.objects.get(video_id=VIDEO_ID_INTRO)
job = queue.enqueue(download.download_video, video)
job = queue.enqueue(download.download_video, video.id)
store = storage.Storage()
cf = store.get_or_create_channel_folder(video.channel.slug)
@ -28,6 +31,35 @@ def test_download_video(download_dir, rq_queue):
assert os.path.isfile(cf.get_thumbnail(VIDEO_SLUG_INTRO, True))
@pytest.mark.django_db
def test_load_unavailable_video(download_dir, rq_queue, mock_redis):
channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV)
download._load_scraped_video(VideoScraped(VIDEO_ID_UNAVAILABLE, None), channel)
video = Video.objects.get(video_id=VIDEO_ID_UNAVAILABLE)
assert video.is_deleted is True
@pytest.mark.django_db
def test_download_unavailable_video(download_dir, rq_queue):
channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV)
video = Video(
video_id=VIDEO_ID_UNAVAILABLE,
title="",
slug="",
channel=channel,
published=timezone.datetime(2000, 1, 1, tzinfo=timezone.utc),
description="",
duration=0,
)
video.save()
job = queue.enqueue(download.download_video, video.id)
video.refresh_from_db()
assert job.is_finished
assert video.is_deleted
@pytest.mark.django_db
def test_update_channel(
download_dir, rq_queue, mock_redis, mock_get_video_details, mock_download_audio
@ -37,7 +69,7 @@ def test_update_channel(
Video.objects.get(video_id="_I5IFObm_-k").delete()
channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV)
job = rq_queue.enqueue(download.update_channel, channel)
job = rq_queue.enqueue(download.update_channel, channel.id)
assert job.is_finished
mock_download_audio.assert_any_call(

View file

@ -19,7 +19,7 @@ def test_recreate_cover(download_dir_content_mut, rq_queue, mocker):
store = storage.Storage()
cf = store.get_or_create_channel_folder(video.channel.slug)
job = rq_queue.enqueue(library.recreate_cover, video)
job = rq_queue.enqueue(library.recreate_cover, video.id)
assert job.is_finished
create_cover_mock.assert_called_once_with(
@ -53,7 +53,7 @@ def test_update_channel_info(rq_queue, mock_get_channel_metadata):
channel.avatar_url = "Old avatar url"
channel.save()
job = rq_queue.enqueue(library.update_channel_info, channel)
job = rq_queue.enqueue(library.update_channel_info, channel.id)
assert job.is_finished
channel.refresh_from_db()

View file

@ -28,6 +28,16 @@ urlpatterns = [
views.download_errors_requeue_all,
name="download_errors_requeue_all",
),
path(
"downloads/delete",
views.download_errors_delete,
name="download_errors_delete",
),
path(
"downloads/delete_all",
views.download_errors_delete_all,
name="download_errors_delete_all",
),
path("downloads/error/<str:job_id>", views.error_details, name="error_details"),
path("feed/<str:channel>", views.podcast_feed, name="feed"),
path("opml", views.channels_opml, name="channels_opml"),

View file

@ -33,7 +33,7 @@ def home(request: http.HttpRequest):
channel_str = form.cleaned_data["channel_str"]
try:
channel = controller.create_channel(channel_str)
queue.enqueue(download.update_channel, channel)
queue.enqueue(download.update_channel, channel.id)
except ValueError:
form.add_error("channel_str", "Channel URL invalid")
except controller.ChannelAlreadyExistsException:
@ -91,6 +91,10 @@ def videos(request: http.HttpRequest, channel: str):
if request.htmx:
template_name = "ucast/videos_items.html"
n_pending = Video.objects.filter(
channel=chan, downloaded__isnull=True, is_deleted=False
).count()
return render(
request,
template_name,
@ -98,6 +102,7 @@ def videos(request: http.HttpRequest, channel: str):
"videos": videos_p.get_page(page_number),
"channel": chan,
"site_url": site_url,
"n_pending": n_pending,
},
)
@ -139,7 +144,7 @@ def channel_download(request: http.HttpRequest, channel: str):
form = forms.DownloadChannelForm(request.POST)
if form.is_valid():
queue.enqueue(
download.download_channel, chan, form.cleaned_data["n_videos"]
download.download_channel, chan.id, form.cleaned_data["n_videos"]
)
return http.HttpResponseRedirect(reverse(videos, args=[channel]))
@ -204,6 +209,26 @@ def download_errors_requeue_all(request: http.HttpRequest):
return http.HttpResponseRedirect(reverse(downloads))
@login_required
def download_errors_delete(request: http.HttpRequest):
form = forms.RequeueForm(request.POST)
if form.is_valid():
freg = queue.get_failed_job_registry()
freg.remove(str(form.cleaned_data["id"]), delete_job=True)
return http.HttpResponseRedirect(reverse(downloads))
@login_required
def download_errors_delete_all(request: http.HttpRequest):
freg = queue.get_failed_job_registry()
for job_id in freg.get_job_ids():
freg.remove(job_id, delete_job=True)
return http.HttpResponseRedirect(reverse(downloads))
@login_required
def channels_opml(request: http.HttpRequest):
response = http.HttpResponse(