Compare commits

..

2 commits

Author SHA1 Message Date
ede37ebe17 Bump version: 0.1.1 → 0.2.0
All checks were successful
continuous-integration/drone/push Build is passing
2022-06-27 12:43:06 +02:00
0d34a96227 add downloads view
fix yt-dlp cache not writable
fix video count
2022-06-27 12:42:59 +02:00
16 changed files with 178 additions and 70 deletions

View file

@ -1,5 +1,5 @@
[bumpversion]
current_version = 0.1.1
current_version = 0.2.0
commit = True
tag = True

View file

@ -9,7 +9,8 @@ FROM python:3.10
ARG TARGETPLATFORM
COPY --from=0 /build/dist /install
RUN pip install -- /install/*.whl gunicorn honcho
RUN pip install -- /install/*.whl gunicorn honcho && \
rm -rf ~/.cache/pip
# ffmpeg static source (https://johnvansickle.com/ffmpeg/)
RUN set -e; \

View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "ucast"
version = "0.1.1"
version = "0.2.0"
description = "YouTube to Podcast converter"
authors = ["Theta-Dev <t.testboy@gmail.com>"]
packages = [

View file

@ -1,4 +1,4 @@
__version__ = "0.1.1"
__version__ = "0.2.0"
def template_context(request):

View file

@ -4,6 +4,7 @@ import rq_scheduler
from django.conf import settings
from rq import registry
from ucast.models import Video
from ucast.service import util
@ -33,8 +34,7 @@ def get_worker(**kwargs) -> rq.Worker:
def enqueue(f, *args, **kwargs) -> rq.job.Job:
queue = get_queue()
# return queue.enqueue(f, *args, **kwargs)
return queue.enqueue_call(f, args, kwargs)
return queue.enqueue(f, *args, **kwargs)
def get_statistics() -> dict:
@ -90,3 +90,19 @@ def get_statistics() -> dict:
def get_failed_job_registry():
queue = get_queue()
return registry.FailedJobRegistry(queue.name, queue.connection)
def get_downloading_videos():
queue = get_queue()
videos = {}
for job in queue.jobs:
if (
job.func_name == "ucast.tasks.download.download_video"
and job.args
and isinstance(job.args[0], Video)
):
video = job.args[0]
videos[video.id] = video
return list(videos.values())

View file

@ -125,7 +125,19 @@ def download_thumbnail(vinfo: VideoDetails, download_path: Path):
def get_video_details(video_id: str) -> VideoDetails:
with YoutubeDL() as ydl:
"""
Get the details of a YouTube video without downloading it.
:param video_id: YouTube video ID
:return: VideoDetails
"""
cache = storage.Cache()
ydl_params = {
"cachedir": str(cache.dir_ytdlp_cache),
}
with YoutubeDL(ydl_params) as ydl:
info = ydl.extract_info(video_id, download=False)
return VideoDetails.from_vinfo(info)

View file

@ -10,6 +10,13 @@ from ucast.service import controller, cover, storage, util, videoutil, youtube
def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel):
# Use Redis to ensure the same video is not processed multiple times
redis = queue.get_redis_connection()
lock_key = f"ucast:lock_load_video:{vid.id}"
if not redis.set(lock_key, "1", 120, nx=True):
return
# Create video object if it does not exist
try:
video = Video.objects.get(video_id=vid.id)
@ -18,6 +25,7 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel):
# Dont load active livestreams
if details.is_currently_live:
redis.delete(lock_key)
return
slug = Video.get_new_slug(
@ -44,6 +52,8 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel):
):
queue.enqueue(download_video, video)
redis.delete(lock_key)
def download_video(video: Video):
"""

View file

@ -27,8 +27,8 @@
</div>
<div class="navbar-end">
<a class="navbar-item" href="{% url 'download_errors' %}">
Errors
<a class="navbar-item" href="{% url 'downloads' %}">
Downloads
</a>
{% url 'login' as login_url %}
{% url 'logout' as logout_url %}

View file

@ -1,50 +0,0 @@
{% extends 'base.html' %}
{% block title %}ucast - Errors{% endblock %}
{% block content %}
<div class="mb-4">
<div>
<span class="title">Download errors</span>
</div>
</div>
{% if jobs %}
<div class="mb-4">
<form method="post" action="{% url 'download_errors_requeue_all' %}">
{% csrf_token %}
<button class="button is-primary">Requeue all</button>
</form>
</div>
<table class="table">
<thead>
<tr>
<th>ID</th>
<th>Function</th>
<th>Details</th>
<th>Requeue</th>
</tr>
</thead>
<tbody>
{% for job in jobs %}
<tr>
<td>{{ job.id }}</td>
<td>{{ job.func_name }}</td>
<td><a href="{% url 'error_details' job.id %}">Details</a></td>
<td>
<form method="post" action="{% url 'download_errors_requeue' %}">
{% csrf_token %}
<input type="hidden" name="id" value="{{ job.id }}">
<button class="button is-small">Requeue</button>
</form>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p>No download errors</p>
{% endif %}
{% endblock content %}

View file

@ -0,0 +1,81 @@
{% extends 'base.html' %}
{% block title %}ucast - Errors{% endblock %}
{% block content %}
<div class="mb-4">
<div>
<span class="title">Downloading
{% if downloading_videos %}({{ downloading_videos.paginator.count }}){% endif %}
</span>
</div>
</div>
<div class="mb-4">
{% if downloading_videos %}
<table class="table">
<thead>
<tr>
<th>Video-ID</th>
<th>Title</th>
<th>Channel</th>
<th>Published</th>
</tr>
</thead>
<tbody hx-get="?items" hx-trigger="every 5s">
{% include "ucast/downloads_items.html" %}
{% else %}
<p>Not downloading any videos</p>
{% endif %}
</tbody>
</table>
</div>
<div class="mb-4">
<div>
<span class="title">Download errors</span>
</div>
</div>
<div class="mb-4">
{% if failed_jobs %}
<div class="mb-4">
<form method="post" action="{% url 'download_errors_requeue_all' %}">
{% csrf_token %}
<button class="button is-primary">Requeue all</button>
</form>
</div>
<table class="table">
<thead>
<tr>
<th>ID</th>
<th>Function</th>
<th>Details</th>
<th>Requeue</th>
</tr>
</thead>
<tbody>
{% for job in failed_jobs %}
<tr>
<td>{{ job.id }}</td>
<td>{{ job.func_name }}</td>
<td><a href="{% url 'error_details' job.id %}">Details</a></td>
<td>
<form method="post" action="{% url 'download_errors_requeue' %}">
{% csrf_token %}
<input type="hidden" name="id" value="{{ job.id }}">
<button class="button is-small">Requeue</button>
</form>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p>No download errors</p>
{% endif %}
</div>
{% endblock content %}

View file

@ -0,0 +1,13 @@
{% for video in downloading_videos %}
<tr {% if forloop.last and downloading_videos.has_next %}
hx-get="?page={{ downloading_videos.next_page_number }}"
hx-trigger="revealed"
hx-swap="afterend"
{% endif %}>
<td><a href="{{ video.get_absolute_url }}">{{ video.video_id }}</a></td>
<td>{{ video.title }}</td>
<td><a href="{% url 'videos' video.channel.slug %}">{{ video.channel.name }}</a>
</td>
<td>{{ video.published|date:"SHORT_DATE_FORMAT" }}</td>
</tr>
{% endfor %}

View file

@ -13,7 +13,7 @@
<span class="tag"><i
class="fas fa-user-group"></i>&nbsp; {{ channel.subscribers }}</span>
<span class="tag"><i
class="fas fa-video"></i>&nbsp; {{ videos|length }}</span>
class="fas fa-video"></i>&nbsp; {{ videos.paginator.count }}</span>
<span class="tag"><i
class="fas fa-database"></i>&nbsp; {{ channel.download_size|filesizeformat }}</span>
<a class="tag" href="{{ channel.get_absolute_url }}" target="_blank"><i

View file

@ -116,6 +116,14 @@ def download_dir_content_mut() -> Path:
yield tmpdir
@pytest.fixture
def mock_redis(mocker) -> FakeRedis:
redis = FakeRedis()
mocker.patch.object(queue, "get_redis_connection")
queue.get_redis_connection.return_value = redis
return redis
@pytest.fixture
def rq_queue(mocker) -> rq.Queue:
test_queue = rq.Queue(is_async=False, connection=FakeRedis())

View file

@ -30,7 +30,7 @@ def test_download_video(download_dir, rq_queue):
@pytest.mark.django_db
def test_update_channel(
download_dir, rq_queue, mock_get_video_details, mock_download_audio
download_dir, rq_queue, mock_redis, mock_get_video_details, mock_download_audio
):
# Remove 2 videos from the database so they can be imported
Video.objects.get(video_id="ZPxEr4YdWt8").delete()

View file

@ -17,16 +17,18 @@ urlpatterns = [
views.channel_download,
name="channel_download",
),
path("errors", views.download_errors, name="download_errors"),
path("downloads", views.downloads, name="downloads"),
path(
"errors/requeue", views.download_errors_requeue, name="download_errors_requeue"
"downloads/requeue",
views.download_errors_requeue,
name="download_errors_requeue",
),
path(
"errors/requeue_all",
"downloads/requeue_all",
views.download_errors_requeue_all,
name="download_errors_requeue_all",
),
path("errors/<str:job_id>", views.error_details, name="error_details"),
path("downloads/error/<str:job_id>", views.error_details, name="error_details"),
path("feed/<str:channel>", views.podcast_feed, name="feed"),
path("files/audio/<str:channel>/<str:video>", views.audio),
path("files/cover/<str:channel>/<str:video>", views.cover),

View file

@ -154,12 +154,27 @@ def channel_download(request: http.HttpRequest, channel: str):
@login_required
def download_errors(request: http.HttpRequest):
def downloads(request: http.HttpRequest):
freg = queue.get_failed_job_registry()
ids = freg.get_job_ids(0, 50)
jobs = freg.job_class.fetch_many(ids, freg.connection, freg.serializer)
failed_jobs = freg.job_class.fetch_many(ids, freg.connection, freg.serializer)
return render(request, "ucast/download_errors.html", {"jobs": jobs})
page_number = request.GET.get("page")
downloading_videos = queue.get_downloading_videos()
downloading_videos_p = Paginator(downloading_videos, 100)
template_name = "ucast/downloads.html"
if request.htmx or request.GET.get("items"):
template_name = "ucast/downloads_items.html"
return render(
request,
template_name,
{
"failed_jobs": failed_jobs,
"downloading_videos": downloading_videos_p.get_page(page_number),
},
)
@login_required
@ -178,7 +193,7 @@ def download_errors_requeue(request: http.HttpRequest):
freg = queue.get_failed_job_registry()
freg.requeue(str(form.cleaned_data["id"]))
return http.HttpResponseRedirect(reverse(download_errors))
return http.HttpResponseRedirect(reverse(downloads))
@login_required
@ -187,7 +202,7 @@ def download_errors_requeue_all(request: http.HttpRequest):
for job_id in freg.get_job_ids():
freg.requeue(job_id)
return http.HttpResponseRedirect(reverse(download_errors))
return http.HttpResponseRedirect(reverse(downloads))
def _channel_file(channel: str, get_file: Callable[[storage.ChannelFolder], Path]):