From 2bb670a5c6681b72433d5dada2f94c1ce6b2a058 Mon Sep 17 00:00:00 2001 From: Theta-Dev Date: Thu, 26 May 2022 00:49:26 +0200 Subject: [PATCH 1/5] add podcast feed --- .env | 2 - .env.example | 3 + .gitignore | 1 + deploy/docker-compose_develop.yml | 8 ++ deploy/nginx/ucast.conf | 26 ++++ notes/Feed.md | 34 +++++ poetry.lock | 47 ++----- pyproject.toml | 1 - ucast/feed.py | 190 ++++++++++++++++++++++++++++ ucast/models.py | 6 + ucast/service/storage.py | 6 + ucast/tasks/download.py | 4 +- ucast/tasks/library.py | 15 ++- ucast/tests/conftest.py | 4 +- ucast/tests/service/test_storage.py | 4 +- ucast/tests/tasks/test_download.py | 2 +- ucast/tests/tasks/test_library.py | 2 +- ucast/urls.py | 9 +- ucast/views.py | 74 +++++++++++ ucast_project/settings.py | 11 +- 20 files changed, 395 insertions(+), 54 deletions(-) delete mode 100644 .env create mode 100644 .env.example create mode 100644 deploy/nginx/ucast.conf create mode 100644 notes/Feed.md create mode 100644 ucast/feed.py diff --git a/.env b/.env deleted file mode 100644 index f600be1..0000000 --- a/.env +++ /dev/null @@ -1,2 +0,0 @@ -UCAST_DEBUG=True -UCAST_WORKDIR=_run diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..02ed387 --- /dev/null +++ b/.env.example @@ -0,0 +1,3 @@ +UCAST_DEBUG=True +UCAST_WORKDIR=_run +UCAST_ALLOWED_HOSTS=localhost,127.0.0.1 diff --git a/.gitignore b/.gitignore index 261201c..e83646a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ node_modules .ipynb_checkpoints # Application data +/.env /_run* *.sqlite3 diff --git a/deploy/docker-compose_develop.yml b/deploy/docker-compose_develop.yml index 6d05622..c422bdb 100644 --- a/deploy/docker-compose_develop.yml +++ b/deploy/docker-compose_develop.yml @@ -12,3 +12,11 @@ services: - "127.0.0.1:9181:9181" environment: RQ_DASHBOARD_REDIS_URL: "redis://redis:6379" + + nginx: + image: nginx:1 + network_mode: "host" + volumes: + - "./nginx:/etc/nginx/conf.d:ro" + - "../_run/static:/static:ro" + - "../_run/data:/files:ro" diff --git a/deploy/nginx/ucast.conf b/deploy/nginx/ucast.conf new file mode 100644 index 0000000..a1d9b56 --- /dev/null +++ b/deploy/nginx/ucast.conf @@ -0,0 +1,26 @@ +server { + listen 80; + server_name localhost; + + client_max_body_size 1M; + + # serve media files + location /static/ { + alias /static/; + } + + location /internal_files/ { + internal; + alias /files/; + } + + location / { + proxy_set_header Host $http_host; + proxy_pass http://127.0.0.1:8000; + } + + # location /errors/ { + # alias /etc/nginx/conf.d/errorpages/; + # internal; + # } +} diff --git a/notes/Feed.md b/notes/Feed.md new file mode 100644 index 0000000..6c5e79b --- /dev/null +++ b/notes/Feed.md @@ -0,0 +1,34 @@ +Django-Klasse: `django.utils.feedgenerator.Rss201rev2Feed` + +### Channel-Attribute + +| Tag | Beschreibung | Django-Attribut | +|--------------------------------------|-----------------------------------------|----------------------| +| `\` | Feed-URL | `feed_url` | +| `\` | Kanalname | `title` | +| `\<language>` | Sprache | `language` | +| `\<lastBuildDate>` | Datum der letzten Veränderung des Feeds | `latest_post_date()` | +| `\<description>` | Kanalbeschreibung | `description` | +| `\<link>` | Link zum Kanal | `link` | +| `\<copyright>` | Autor | `feed_copyright` | +| `\<image><url><title><link></image>` | Cover-URL / Kanalname / Link | - | +| `\<itunes:image href="">` | Cover-URL | - | +| `\<itunes:author>` | Autor | - | +| `\<itunes:summary>` | Kanalbeschreibung | - | + + +### Item-Attribute + +| Tag | Beschreibung | Django-Attribut | +|--------------------------------------------------|------------------------|-----------------| +| `\<title>` | Titel | `title` | +| `\<itunes:title>` | Titel | - | +| `\<description>` | Beschreibung | `description` | +| `\<pubDate>` | Veröffentlichungsdatum | `pubdate` | +| `\<link>` | Link | `link` | +| `\<guid>` | Eindeutige ID/ | `unique_id` | +| `\<itunes:summary>` | Bechreibung | - | +| `\<itunes:author>` | Autor | - | +| `\<enclosure url="" type="audio/mpeg" length=1>` | Audiodatei | `enclosures ` | +| `\<itunes:duration>00:40:35</itunes:duration>` | Dauer | - | +| `\<itunes:image href="">` | Cover-URL | - | diff --git a/poetry.lock b/poetry.lock index 2138b01..2252a0d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -203,21 +203,20 @@ django = ">=2.2" [[package]] name = "fakeredis" -version = "1.7.5" +version = "1.7.6.1" description = "Fake implementation of redis API for testing purposes." category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8,<4.0" [package.dependencies] -packaging = "*" redis = "<=4.3.1" -six = ">=1.12" -sortedcontainers = "*" +six = ">=1.16.0,<2.0.0" +sortedcontainers = ">=2.4.0,<3.0.0" [package.extras] -aioredis = ["aioredis"] -lua = ["lupa"] +aioredis = ["aioredis (>=2.0.1,<3.0.0)"] +lua = ["lupa (>=1.13,<2.0)"] [[package]] name = "feedparser" @@ -307,19 +306,6 @@ category = "dev" optional = false python-versions = "*" -[[package]] -name = "mock" -version = "4.0.3" -description = "Rolling backport of unittest.mock for all Pythons" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.extras] -build = ["twine", "wheel", "blurb"] -docs = ["sphinx"] -test = ["pytest (<5.4)", "pytest-cov"] - [[package]] name = "mutagen" version = "1.45.1" @@ -594,14 +580,6 @@ urllib3 = ">=1.21.1,<1.27" socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] -[[package]] -name = "rfeed" -version = "1.1.1" -description = "Python RSS 2.0 Generator" -category = "main" -optional = false -python-versions = "*" - [[package]] name = "rq" version = "1.10.1" @@ -765,7 +743,7 @@ websockets = "*" [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "ad3a5ecd6fc1152dfdfda51ed1e401ec11a048661a04f42985c15bc28e8eda9f" +content-hash = "d0101b3a682960de22ddde7f1affe6fc0b78d45f351749469f9f63047c725b27" [metadata.files] asgiref = [ @@ -1020,8 +998,8 @@ django-bulma = [ {file = "django_bulma-0.8.3-py3-none-any.whl", hash = "sha256:0ef6e5c171c2a32010e724a8be61ba6cd0e55ebbd242cf6780560518483c4d00"}, ] fakeredis = [ - {file = "fakeredis-1.7.5-py3-none-any.whl", hash = "sha256:c4ca2be686e7e7637756ccc7dcad8472a5e4866b065431107d7a4b7a250d4e6f"}, - {file = "fakeredis-1.7.5.tar.gz", hash = "sha256:49375c630981dd4045d9a92e2709fcd4476c91f927e0228493eefa625e705133"}, + {file = "fakeredis-1.7.6.1-py3-none-any.whl", hash = "sha256:2bc92cece6535961a465991d01841888e0fe2b742ca49aa97ce247b04c9a0ecc"}, + {file = "fakeredis-1.7.6.1.tar.gz", hash = "sha256:e71ca849167052f42f5469a764def9ef35ccffd04773d30b017a7adcc12940c1"}, ] feedparser = [ {file = "feedparser-6.0.10-py3-none-any.whl", hash = "sha256:79c257d526d13b944e965f6095700587f27388e50ea16fd245babe4dfae7024f"}, @@ -1061,10 +1039,6 @@ invoke = [ {file = "invoke-1.7.1-py3-none-any.whl", hash = "sha256:2dc975b4f92be0c0a174ad2d063010c8a1fdb5e9389d69871001118b4fcac4fb"}, {file = "invoke-1.7.1.tar.gz", hash = "sha256:7b6deaf585eee0a848205d0b8c0014b9bf6f287a8eb798818a642dff1df14b19"}, ] -mock = [ - {file = "mock-4.0.3-py3-none-any.whl", hash = "sha256:122fcb64ee37cfad5b3f48d7a7d51875d7031aaf3d8be7c42e2bee25044eee62"}, - {file = "mock-4.0.3.tar.gz", hash = "sha256:7d3fbbde18228f4ff2f1f119a45cdffa458b4c0dee32eb4d2bb2f82554bac7bc"}, -] mutagen = [ {file = "mutagen-1.45.1-py3-none-any.whl", hash = "sha256:9c9f243fcec7f410f138cb12c21c84c64fde4195481a30c9bfb05b5f003adfed"}, {file = "mutagen-1.45.1.tar.gz", hash = "sha256:6397602efb3c2d7baebd2166ed85731ae1c1d475abca22090b7141ff5034b3e1"}, @@ -1261,9 +1235,6 @@ requests = [ {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, ] -rfeed = [ - {file = "rfeed-1.1.1.tar.gz", hash = "sha256:aa9506f2866b74f5a322d394a14a63c19a6825c2d94755ff19d46dd1e2434819"}, -] rq = [ {file = "rq-1.10.1-py2.py3-none-any.whl", hash = "sha256:92f4cf38b2364c1697b541e77c0fe62b7e5242fa864324f262be126ee2a07e3a"}, {file = "rq-1.10.1.tar.gz", hash = "sha256:62d06b44c3acfa5d1933c5a4ec3fbc2484144a8af60e318d0b8447c5236271e2"}, diff --git a/pyproject.toml b/pyproject.toml index a978110..63dfa34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,6 @@ python = "^3.10" Django = "^4.0.4" yt-dlp = "^2022.3.8" requests = "^2.27.1" -rfeed = "^1.1.1" feedparser = "^6.0.8" Pillow = "^9.1.0" colorthief = "^0.2.1" diff --git a/ucast/feed.py b/ucast/feed.py new file mode 100644 index 0000000..8a5fe20 --- /dev/null +++ b/ucast/feed.py @@ -0,0 +1,190 @@ +import re +from xml.sax import saxutils + +from django import http +from django.contrib.sites.shortcuts import get_current_site +from django.contrib.syndication.views import Feed, add_domain +from django.utils import feedgenerator +from django.utils.feedgenerator import Rss201rev2Feed, rfc2822_date +from django.utils.xmlutils import SimplerXMLGenerator + +from ucast.models import Channel, Video + +URL_REGEX = r"""http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+""" + + +class PodcastFeedType(Rss201rev2Feed): + content_type = "application/xml; charset=utf-8" + + def rss_attributes(self): + attrs = super().rss_attributes() + attrs["xmlns:itunes"] = "http://www.itunes.com/dtds/podcast-1.0.dtd" + return attrs + + @staticmethod + def _xml_escape(text: str) -> str: + text = saxutils.escape(text) + text = re.sub(URL_REGEX, lambda m: f'<a href="{m[0]}">{m[0]}</a>', text) + text = text.replace("\n", "<br>") + return text + + @staticmethod + def _add_text_element(handler: SimplerXMLGenerator, text: str): + handler.startElement("description", {}) + handler.ignorableWhitespace(f"<![CDATA[{PodcastFeedType._xml_escape(text)}]]>") + handler.endElement("description") + + @staticmethod + def _format_secs(secs: int) -> str: + mm, ss = divmod(secs, 60) + hh, mm = divmod(mm, 60) + s = "%02d:%02d:%02d" % (hh, mm, ss) + return s + + def add_root_elements(self, handler: SimplerXMLGenerator): + handler.addQuickElement("title", self.feed["title"]) + handler.addQuickElement("link", self.feed["link"]) + self._add_text_element(handler, self.feed["description"]) + if self.feed["feed_url"] is not None: + handler.addQuickElement( + "atom:link", None, {"rel": "self", "href": self.feed["feed_url"]} + ) + if self.feed["language"] is not None: + handler.addQuickElement("language", self.feed["language"]) + for cat in self.feed["categories"]: + handler.addQuickElement("category", cat) + if self.feed["feed_copyright"] is not None: + handler.addQuickElement("copyright", self.feed["feed_copyright"]) + handler.addQuickElement("lastBuildDate", rfc2822_date(self.latest_post_date())) + if self.feed["ttl"] is not None: + handler.addQuickElement("ttl", self.feed["ttl"]) + + if self.feed.get("image_url") is not None: + handler.startElement("image", {}) + handler.addQuickElement("url", self.feed["image_url"]) + handler.addQuickElement("title", self.feed["title"]) + handler.addQuickElement("link", self.feed["link"]) + handler.endElement("image") + + handler.addQuickElement( + "itunes:image", None, {"href": self.feed["image_url"]} + ) + + def add_item_elements(self, handler: SimplerXMLGenerator, item): + handler.addQuickElement("title", item["title"]) + handler.addQuickElement("link", item["link"]) + + if item["description"] is not None: + self._add_text_element(handler, item["description"]) + + # Author information. + if item["author_name"] and item["author_email"]: + handler.addQuickElement( + "author", "%s (%s)" % (item["author_email"], item["author_name"]) + ) + elif item["author_email"]: + handler.addQuickElement("author", item["author_email"]) + elif item["author_name"]: + handler.addQuickElement( + "dc:creator", + item["author_name"], + {"xmlns:dc": "http://purl.org/dc/elements/1.1/"}, + ) + + if item["pubdate"] is not None: + handler.addQuickElement("pubDate", rfc2822_date(item["pubdate"])) + if item["comments"] is not None: + handler.addQuickElement("comments", item["comments"]) + if item["unique_id"] is not None: + guid_attrs = {} + if isinstance(item.get("unique_id_is_permalink"), bool): + guid_attrs["isPermaLink"] = str(item["unique_id_is_permalink"]).lower() + handler.addQuickElement("guid", item["unique_id"], guid_attrs) + if item["ttl"] is not None: + handler.addQuickElement("ttl", item["ttl"]) + + # Enclosure. + if item["enclosures"]: + enclosures = list(item["enclosures"]) + if len(enclosures) > 1: + raise ValueError( + "RSS feed items may only have one enclosure, see " + "http://www.rssboard.org/rss-profile#element-channel-item-enclosure" + ) + enclosure = enclosures[0] + handler.addQuickElement( + "enclosure", + "", + { + "url": enclosure.url, + "length": enclosure.length, + "type": enclosure.mime_type, + }, + ) + + # Categories. + for cat in item["categories"]: + handler.addQuickElement("category", cat) + + # Cover image + if item.get("image_url"): + handler.addQuickElement("itunes:image", None, {"href": item["image_url"]}) + + # Duration + if item.get("duration"): + handler.addQuickElement( + "itunes:duration", self._format_secs(item["duration"]) + ) + + +class UcastFeed(Feed): + feed_type = PodcastFeedType + + def get_object(self, request, *args, **kwargs): + channel_slug = kwargs["channel"] + return Channel.objects.get(slug=channel_slug) + + def get_feed(self, channel: Channel, request: http.HttpRequest): + feed = self.feed_type( + title=channel.name, + link=channel.get_absolute_url(), + description=channel.description, + language=self.language, + feed_url=self.full_link_url(request, f"/feed/{channel.slug}"), + image_url=self.full_link_url(request, f"/files/avatar/{channel.slug}.jpg"), + ) + + for video in channel.video_set.order_by("-published"): + feed.add_item( + title=video.title, + link=video.get_absolute_url(), + description=video.description, + unique_id=video.get_absolute_url(), + unique_id_is_permalink=True, + enclosures=self.item_enclosures_domain(video, request), + pubdate=video.published, + updateddate=video.downloaded, + image_url=self.full_link_url( + request, f"/files/cover/{channel.slug}/{video.slug}.png" + ), + duration=video.duration, + ) + return feed + + @staticmethod + def full_link_url(request: http.HttpRequest, page_url: str) -> str: + return add_domain( + get_current_site(request).domain, + page_url, + request.is_secure(), + ) + + def item_enclosures_domain(self, item: Video, request: http.HttpRequest): + enc = feedgenerator.Enclosure( + url=self.full_link_url( + request, f"/files/audio/{item.channel.slug}/{item.slug}.mp3" + ), + length=str(item.download_size), + mime_type="audio/mpg", + ) + return [enc] diff --git a/ucast/models.py b/ucast/models.py index 1079281..d79177e 100644 --- a/ucast/models.py +++ b/ucast/models.py @@ -48,6 +48,9 @@ class Channel(models.Model): desc = f"{self.description}\n\n{desc}" return desc + def get_absolute_url(self) -> str: + return "https://www.youtube.com/channel/" + self.id + def __str__(self): return self.name @@ -79,5 +82,8 @@ class Video(models.Model): desc = f"{self.description}\n\n{desc}" return desc + def get_absolute_url(self) -> str: + return f"https://www.youtube.com/watch?v={self.id}" + def __str__(self): return self.title diff --git a/ucast/service/storage.py b/ucast/service/storage.py index fbf5206..dc46f36 100644 --- a/ucast/service/storage.py +++ b/ucast/service/storage.py @@ -52,6 +52,12 @@ class Storage: self.dir_data = settings.DOWNLOAD_ROOT def get_channel_folder(self, channel_slug: str) -> ChannelFolder: + cf = ChannelFolder(self.dir_data / channel_slug) + if not cf.does_exist(): + raise FileNotFoundError + return cf + + def get_or_create_channel_folder(self, channel_slug: str) -> ChannelFolder: cf = ChannelFolder(self.dir_data / channel_slug) if not cf.does_exist(): cf.create() diff --git a/ucast/tasks/download.py b/ucast/tasks/download.py index 5e4df48..4a05fb0 100644 --- a/ucast/tasks/download.py +++ b/ucast/tasks/download.py @@ -16,7 +16,7 @@ def _get_or_create_channel(channel_id: str) -> Channel: ) channel_slug = Channel.get_new_slug(channel_data.name) store = storage.Storage() - channel_folder = store.get_channel_folder(channel_slug) + channel_folder = store.get_or_create_channel_folder(channel_slug) util.download_image_file(channel_data.avatar_url, channel_folder.file_avatar) util.resize_avatar(channel_folder.file_avatar, channel_folder.file_avatar_sm) @@ -71,7 +71,7 @@ def download_video(video: Video): :param video: Video object """ store = storage.Storage() - channel_folder = store.get_channel_folder(video.channel.slug) + channel_folder = store.get_or_create_channel_folder(video.channel.slug) audio_file = channel_folder.get_audio(video.slug) details = youtube.download_audio(video.id, audio_file) diff --git a/ucast/tasks/library.py b/ucast/tasks/library.py index f7681b8..a9aa82d 100644 --- a/ucast/tasks/library.py +++ b/ucast/tasks/library.py @@ -4,7 +4,7 @@ from django.utils import timezone from ucast import queue from ucast.models import Channel, Video -from ucast.service import cover, storage, util, youtube +from ucast.service import cover, storage, util, videoutil, youtube def recreate_cover(video: Video): @@ -13,6 +13,7 @@ def recreate_cover(video: Video): thumbnail_file = cf.get_thumbnail(video.slug) cover_file = cf.get_cover(video.slug) + audio_file = cf.get_audio(video.slug) if not os.path.isfile(cf.file_avatar): raise FileNotFoundError(f"could not find avatar for channel {video.channel_id}") @@ -29,6 +30,8 @@ def recreate_cover(video: Video): cover_file, ) + videoutil.tag_audio(audio_file, video, cover_file) + def recreate_covers(): for video in Video.objects.filter(downloaded__isnull=False): @@ -39,7 +42,13 @@ def update_file_storage(): store = storage.Storage() for video in Video.objects.all(): - cf = store.get_channel_folder(video.channel.slug) + try: + cf = store.get_channel_folder(video.channel.slug) + except FileNotFoundError: + video.downloaded = None + video.download_size = None + video.save() + return audio_file = cf.get_audio(video.slug) cover_file = cf.get_cover(video.slug) @@ -70,7 +79,7 @@ def update_channel_info(channel: Channel): if channel_data.avatar_url != channel.avatar_url: store = storage.Storage() - channel_folder = store.get_channel_folder(channel.slug) + channel_folder = store.get_or_create_channel_folder(channel.slug) util.download_image_file(channel_data.avatar_url, channel_folder.file_avatar) util.resize_avatar(channel_folder.file_avatar, channel_folder.file_avatar_sm) diff --git a/ucast/tests/conftest.py b/ucast/tests/conftest.py index b61ae27..6c92ab3 100644 --- a/ucast/tests/conftest.py +++ b/ucast/tests/conftest.py @@ -35,7 +35,7 @@ def download_dir() -> Path: ("media_ccc_de", "a3"), ("Creative_Commons", "a4"), ): - cf = store.get_channel_folder(slug) + cf = store.get_or_create_channel_folder(slug) shutil.copyfile( tests.DIR_TESTFILES / "avatar" / f"{avatar}.jpg", cf.file_avatar ) @@ -50,7 +50,7 @@ def download_dir_content(download_dir) -> Path: store = storage.Storage() for video in Video.objects.filter(downloaded__isnull=False): - cf = store.get_channel_folder(video.channel.slug) + cf = store.get_or_create_channel_folder(video.channel.slug) file_audio = cf.get_audio(video.slug) file_tn = cf.get_thumbnail(video.slug) file_cover = cf.get_cover(video.slug) diff --git a/ucast/tests/service/test_storage.py b/ucast/tests/service/test_storage.py index 3c5a29c..083e3e5 100644 --- a/ucast/tests/service/test_storage.py +++ b/ucast/tests/service/test_storage.py @@ -11,8 +11,8 @@ def test_create_channel_folders(settings): settings.DOWNLOAD_ROOT = tmpdir store = storage.Storage() - cf1 = store.get_channel_folder("ThetaDev") - cf2 = store.get_channel_folder("Jeff_Geerling") + cf1 = store.get_or_create_channel_folder("ThetaDev") + cf2 = store.get_or_create_channel_folder("Jeff_Geerling") cf1b = store.get_channel_folder("ThetaDev") cf1_path = tmpdir / "ThetaDev" diff --git a/ucast/tests/tasks/test_download.py b/ucast/tests/tasks/test_download.py index c5818c2..d4ec883 100644 --- a/ucast/tests/tasks/test_download.py +++ b/ucast/tests/tasks/test_download.py @@ -18,7 +18,7 @@ def test_download_video(download_dir, rq_queue): job = queue.enqueue(download.download_video, video) store = storage.Storage() - cf = store.get_channel_folder(video.channel.slug) + cf = store.get_or_create_channel_folder(video.channel.slug) assert job.is_finished diff --git a/ucast/tests/tasks/test_library.py b/ucast/tests/tasks/test_library.py index 9722316..96d8c06 100644 --- a/ucast/tests/tasks/test_library.py +++ b/ucast/tests/tasks/test_library.py @@ -17,7 +17,7 @@ def test_recreate_cover(download_dir_content, rq_queue, mocker): video = Video.objects.get(id="ZPxEr4YdWt8") store = storage.Storage() - cf = store.get_channel_folder(video.channel.slug) + cf = store.get_or_create_channel_folder(video.channel.slug) job = rq_queue.enqueue(library.recreate_cover, video) assert job.is_finished diff --git a/ucast/urls.py b/ucast/urls.py index 08537b9..6d904c0 100644 --- a/ucast/urls.py +++ b/ucast/urls.py @@ -2,4 +2,11 @@ from django.urls import path from ucast import views -urlpatterns = [path("", views.home)] +urlpatterns = [ + path("", views.home), + path("feed/<str:channel>", views.podcast_feed), + path("files/audio/<str:channel>/<str:video>", views.audio), + path("files/cover/<str:channel>/<str:video>", views.cover), + path("files/thumbnail/<str:channel>/<str:video>", views.thumbnail), + path("files/avatar/<str:channel>", views.avatar), +] diff --git a/ucast/views.py b/ucast/views.py index 2e907b9..d8881fc 100644 --- a/ucast/views.py +++ b/ucast/views.py @@ -1,6 +1,80 @@ +import os +from pathlib import Path +from typing import Callable + from django import http +from django.conf import settings +from django.middleware.http import ConditionalGetMiddleware from django.shortcuts import render +from django.utils.decorators import decorator_from_middleware + +from ucast import feed +from ucast.service import storage def home(request: http.HttpRequest): return render(request, "ucast/main.html") + + +def channel_file(channel: str, get_file: Callable[[storage.ChannelFolder], Path]): + store = storage.Storage() + + try: + cf = store.get_channel_folder(channel) + except FileNotFoundError: + raise http.Http404 + + file_path = get_file(cf) + if not os.path.isfile(file_path): + raise http.Http404 + + if not settings.INTERNAL_REDIRECT_HEADER: + return http.FileResponse(open(file_path, "rb")) + + file_path_rel = os.path.relpath(file_path, store.dir_data) + file_path_internal = os.path.join("/internal_files", file_path_rel) + + response = http.HttpResponse() + response.headers[settings.INTERNAL_REDIRECT_HEADER] = file_path_internal + # Content type is set to text/html by default and has to be unset + del response.headers["Content-Type"] + return response + + +def audio(request: http.HttpRequest, channel: str, video: str): + # Trim off file extension + video_slug = video.rsplit(".")[0] + + return channel_file(channel, lambda cf: cf.get_audio(video_slug)) + + +def cover(request: http.HttpRequest, channel: str, video: str): + # Trim off file extension + video_slug = video.rsplit(".")[0] + + return channel_file(channel, lambda cf: cf.get_cover(video_slug)) + + +def thumbnail(request: http.HttpRequest, channel: str, video: str): + # Trim off file extension + video_slug = video.rsplit(".")[0] + + is_sm = "sm" in request.GET + + return channel_file(channel, lambda cf: cf.get_thumbnail(video_slug, is_sm)) + + +def avatar(request: http.HttpRequest, channel: str): + # Trim off file extension + channel_slug = channel.rsplit(".")[0] + + is_sm = "sm" in request.GET + + if is_sm: + return channel_file(channel_slug, lambda cf: cf.file_avatar_sm) + return channel_file(channel_slug, lambda cf: cf.file_avatar) + + +@decorator_from_middleware(ConditionalGetMiddleware) +def podcast_feed(request: http.HttpRequest, *args, **kwargs): + return feed.UcastFeed()(request, *args, **kwargs) diff --git a/ucast_project/settings.py b/ucast_project/settings.py index 639db56..9b4cae2 100644 --- a/ucast_project/settings.py +++ b/ucast_project/settings.py @@ -36,6 +36,13 @@ def get_env_path(name, default=None): return Path(raw_env).absolute() +def get_env_list(name): + raw_env = get_env(name) + if not raw_env: + return [] + return [i.strip() for i in raw_env.split(",")] + + def _load_dotenv() -> Path: """ Look for a .env file in the current working directory or @@ -78,7 +85,7 @@ SECRET_KEY = get_env( # SECURITY WARNING: don't run with debug turned on in production! DEBUG = get_env("DEBUG", False) -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = get_env_list("ALLOWED_HOSTS") # Application definition @@ -206,3 +213,5 @@ REDIS_QUEUE_TIMEOUT = get_env("REDIS_QUEUE_TIMEOUT", 600) REDIS_QUEUE_RESULT_TTL = 600 YT_UPDATE_INTERVAL = get_env("YT_UPDATE_INTERVAL", 900) + +INTERNAL_REDIRECT_HEADER = get_env("INTERNAL_REDIRECT_HEADER", "X-Accel-Redirect") From 630b29951a12bb65d519f154fee96d38e6c20e60 Mon Sep 17 00:00:00 2001 From: Theta-Dev <t.testboy@gmail.com> Date: Thu, 26 May 2022 20:45:18 +0200 Subject: [PATCH 2/5] add unique id field to channels and videos --- ucast/migrations/0001_initial.py | 34 +++++++++----- ucast/models.py | 16 +++---- ucast/service/youtube.py | 12 ++++- ucast/tasks/download.py | 17 ++++--- ucast/tasks/library.py | 7 ++- .../tests/_testfiles/fixture/channelmeta.json | 9 ++-- ucast/tests/_testfiles/fixture/videos.json | 45 +++++++++++-------- ucast/tests/conftest.py | 4 +- ucast/tests/service/test_videoutil.py | 2 +- ucast/tests/tasks/test_download.py | 12 ++--- ucast/tests/tasks/test_library.py | 13 +++++- 11 files changed, 111 insertions(+), 60 deletions(-) diff --git a/ucast/migrations/0001_initial.py b/ucast/migrations/0001_initial.py index 12d2a90..7494990 100644 --- a/ucast/migrations/0001_initial.py +++ b/ucast/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.0.4 on 2022-05-05 00:02 +# Generated by Django 4.0.4 on 2022-05-25 22:57 import django.db.models.deletion from django.db import migrations, models @@ -16,15 +16,21 @@ class Migration(migrations.Migration): fields=[ ( "id", - models.CharField(max_length=30, primary_key=True, serialize=False), + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), ), + ("channel_id", models.CharField(max_length=30)), ("name", models.CharField(max_length=100)), ("slug", models.CharField(max_length=100)), ("description", models.TextField()), + ("subscribers", models.CharField(max_length=20, null=True)), ("active", models.BooleanField(default=True)), ("skip_livestreams", models.BooleanField(default=True)), ("skip_shorts", models.BooleanField(default=True)), - ("keep_videos", models.IntegerField(default=None, null=True)), ("avatar_url", models.CharField(max_length=250, null=True)), ], ), @@ -33,16 +39,16 @@ class Migration(migrations.Migration): fields=[ ( "id", - models.CharField(max_length=30, primary_key=True, serialize=False), - ), - ("title", models.CharField(max_length=200)), - ("slug", models.CharField(max_length=209)), - ( - "channel", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to="ucast.channel" + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", ), ), + ("video_id", models.CharField(max_length=30)), + ("title", models.CharField(max_length=200)), + ("slug", models.CharField(max_length=209)), ("published", models.DateTimeField()), ("downloaded", models.DateTimeField(null=True)), ("description", models.TextField()), @@ -50,6 +56,12 @@ class Migration(migrations.Migration): ("is_livestream", models.BooleanField(default=False)), ("is_short", models.BooleanField(default=False)), ("download_size", models.IntegerField(null=True)), + ( + "channel", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="ucast.channel" + ), + ), ], ), ] diff --git a/ucast/models.py b/ucast/models.py index d79177e..f6e7538 100644 --- a/ucast/models.py +++ b/ucast/models.py @@ -28,14 +28,14 @@ def _get_unique_slug( class Channel(models.Model): - id = models.CharField(max_length=30, primary_key=True) + channel_id = models.CharField(max_length=30) name = models.CharField(max_length=100) slug = models.CharField(max_length=100) description = models.TextField() + subscribers = models.CharField(max_length=20, null=True) active = models.BooleanField(default=True) skip_livestreams = models.BooleanField(default=True) skip_shorts = models.BooleanField(default=True) - keep_videos = models.IntegerField(null=True, default=None) avatar_url = models.CharField(max_length=250, null=True) @classmethod @@ -43,20 +43,20 @@ class Channel(models.Model): return _get_unique_slug(name, cls.objects, "channel") def get_full_description(self) -> str: - desc = f"https://www.youtube.com/channel/{self.id}" + desc = f"https://www.youtube.com/channel/{self.channel_id}" if self.description: desc = f"{self.description}\n\n{desc}" return desc def get_absolute_url(self) -> str: - return "https://www.youtube.com/channel/" + self.id + return "https://www.youtube.com/channel/" + self.channel_id def __str__(self): return self.name class Video(models.Model): - id = models.CharField(max_length=30, primary_key=True) + video_id = models.CharField(max_length=30) title = models.CharField(max_length=200) slug = models.CharField(max_length=209) channel = models.ForeignKey(Channel, on_delete=models.CASCADE) @@ -73,17 +73,17 @@ class Video(models.Model): title_w_date = f"{date.strftime('%Y%m%d')}_{title}" return _get_unique_slug( - title_w_date, cls.objects.filter(channel_id=channel_id), "video" + title_w_date, cls.objects.filter(channel__channel_id=channel_id), "video" ) def get_full_description(self) -> str: - desc = f"https://youtu.be/{self.id}" + desc = f"https://youtu.be/{self.video_id}" if self.description: desc = f"{self.description}\n\n{desc}" return desc def get_absolute_url(self) -> str: - return f"https://www.youtube.com/watch?v={self.id}" + return f"https://www.youtube.com/watch?v={self.video_id}" def __str__(self): return self.title diff --git a/ucast/service/youtube.py b/ucast/service/youtube.py index 7830727..2ce93c8 100644 --- a/ucast/service/youtube.py +++ b/ucast/service/youtube.py @@ -96,6 +96,7 @@ class ChannelMetadata: name: str description: str avatar_url: str + subscribers: Optional[str] def download_thumbnail(vinfo: VideoDetails, download_path: Path): @@ -215,6 +216,15 @@ def get_channel_metadata(channel_url: str) -> ChannelMetadata: name = metadata["title"] description = metadata["description"].strip() avatar = metadata["avatar"]["thumbnails"][0]["url"] + subscribers = None + # The subscriber count is not always visible + try: + raw_subscribers = data["header"]["c4TabbedHeaderRenderer"][ + "subscriberCountText" + ]["simpleText"] + subscribers = raw_subscribers.split(" ", 1)[0] + except KeyError: + pass if not CHANID_REGEX.match(channel_id): raise InvalidMetadataError(f"got invalid channel id {repr(channel_id)}") @@ -227,7 +237,7 @@ def get_channel_metadata(channel_url: str) -> ChannelMetadata: f"got invalid avatar url for channel {channel_id}: {avatar}" ) - return ChannelMetadata(channel_id, name, description, avatar) + return ChannelMetadata(channel_id, name, description, avatar, subscribers) def get_channel_videos_from_feed(channel_id: str) -> List[VideoScraped]: diff --git a/ucast/tasks/download.py b/ucast/tasks/download.py index 4a05fb0..f24e238 100644 --- a/ucast/tasks/download.py +++ b/ucast/tasks/download.py @@ -9,7 +9,7 @@ from ucast.service import cover, storage, util, videoutil, youtube def _get_or_create_channel(channel_id: str) -> Channel: try: - return Channel.objects.get(id=channel_id) + return Channel.objects.get(channel_id=channel_id) except Channel.DoesNotExist: channel_data = youtube.get_channel_metadata( youtube.channel_url_from_id(channel_id) @@ -22,17 +22,18 @@ def _get_or_create_channel(channel_id: str) -> Channel: util.resize_avatar(channel_folder.file_avatar, channel_folder.file_avatar_sm) channel = Channel( - id=channel_id, + channel_id=channel_id, name=channel_data.name, slug=channel_slug, description=channel_data.description, + subscribers=channel_data.subscribers, ) channel.save() return channel def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): - if Video.objects.filter(id=vid.id).exists(): + if Video.objects.filter(video_id=vid.id).exists(): return details = youtube.get_video_details(vid.id) @@ -45,10 +46,12 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): ): return - slug = Video.get_new_slug(details.title, details.published.date(), channel.id) + slug = Video.get_new_slug( + details.title, details.published.date(), channel.channel_id + ) video = Video( - id=details.id, + video_id=details.id, title=details.title, slug=slug, channel=channel, @@ -74,7 +77,7 @@ def download_video(video: Video): channel_folder = store.get_or_create_channel_folder(video.channel.slug) audio_file = channel_folder.get_audio(video.slug) - details = youtube.download_audio(video.id, audio_file) + details = youtube.download_audio(video.video_id, audio_file) # Download/convert thumbnails tn_path = channel_folder.get_thumbnail(video.slug) @@ -117,7 +120,7 @@ def import_channel(channel_id: str, limit: int = None): def update_channel(channel: Channel): """Update a single channel from its RSS feed""" - videos = youtube.get_channel_videos_from_feed(channel.id) + videos = youtube.get_channel_videos_from_feed(channel.channel_id) for vid in videos: _load_scraped_video(vid, channel) diff --git a/ucast/tasks/library.py b/ucast/tasks/library.py index a9aa82d..3c28c5e 100644 --- a/ucast/tasks/library.py +++ b/ucast/tasks/library.py @@ -19,7 +19,7 @@ def recreate_cover(video: Video): raise FileNotFoundError(f"could not find avatar for channel {video.channel_id}") if not os.path.isfile(thumbnail_file): - raise FileNotFoundError(f"could not find thumbnail for video {video.id}") + raise FileNotFoundError(f"could not find thumbnail for video {video.video_id}") cover.create_cover_file( thumbnail_file, @@ -75,7 +75,9 @@ def update_file_storage(): def update_channel_info(channel: Channel): - channel_data = youtube.get_channel_metadata(youtube.channel_url_from_id(channel.id)) + channel_data = youtube.get_channel_metadata( + youtube.channel_url_from_id(channel.channel_id) + ) if channel_data.avatar_url != channel.avatar_url: store = storage.Storage() @@ -88,6 +90,7 @@ def update_channel_info(channel: Channel): channel.name = channel_data.name channel.description = channel_data.description + channel.subscribers = channel_data.subscribers channel.save() diff --git a/ucast/tests/_testfiles/fixture/channelmeta.json b/ucast/tests/_testfiles/fixture/channelmeta.json index 8147023..5fd41ef 100644 --- a/ucast/tests/_testfiles/fixture/channelmeta.json +++ b/ucast/tests/_testfiles/fixture/channelmeta.json @@ -3,18 +3,21 @@ "id": "UCGiJh0NZ52wRhYKYnuZI08Q", "name": "ThetaDev", "description": "I'm ThetaDev. I love creating cool projects using electronics, 3D printers and other awesome tech-based stuff.", - "avatar_url": "https://yt3.ggpht.com/ytc/AKedOLSnFfmpibLLoqyaYdsF6bJ-zaLPzomII__FrJve1w=s900-c-k-c0x00ffffff-no-rj" + "avatar_url": "https://yt3.ggpht.com/ytc/AKedOLSnFfmpibLLoqyaYdsF6bJ-zaLPzomII__FrJve1w=s900-c-k-c0x00ffffff-no-rj", + "subscribers": "37" }, "https://www.youtube.com/channel/UC2TXq_t06Hjdr2g_KdKpHQg": { "id": "UC2TXq_t06Hjdr2g_KdKpHQg", "name": "media.ccc.de", "description": "The real official channel of the chaos computer club, operated by the CCC VOC (https://c3voc.de)", - "avatar_url": "https://yt3.ggpht.com/c1jcNSbPuOMDUieixkWIlXc82kMNJ8pCDmq5KtL8hjt74rAXLobsT9Y078-w5DK7ymKyDaqr=s900-c-k-c0x00ffffff-no-rj" + "avatar_url": "https://yt3.ggpht.com/c1jcNSbPuOMDUieixkWIlXc82kMNJ8pCDmq5KtL8hjt74rAXLobsT9Y078-w5DK7ymKyDaqr=s900-c-k-c0x00ffffff-no-rj", + "subscribers": "166K" }, "https://www.youtube.com/channel/UCmLTTbctUZobNQrr8RtX8uQ": { "id": "UCmLTTbctUZobNQrr8RtX8uQ", "name": "Creative Commons", "description": "Hello friends,\nWelcome to my channel CREATIVE COMMONS.\nOn this channel you will get all the videos absolutely free copyright and no matter how many videos you download there is no copyright claim you can download them and upload them to your channel and all the music is young Is on the channel they can also download and use in their videos on this channel you will find different videos in which OUTRO Videos, INTRO Videos, FREE MUSIC, FREE SOUND EFFECTS, LOWER THIRDS, and more.", - "avatar_url": "https://yt3.ggpht.com/-ybcsEHc8YCmKUZMr2bf4DZoDv7SKrutgKIh8kSxXugj296QkqtBZQXVzpuZ1Izs8kNUz35B=s900-c-k-c0x00ffffff-no-rj" + "avatar_url": "https://yt3.ggpht.com/-ybcsEHc8YCmKUZMr2bf4DZoDv7SKrutgKIh8kSxXugj296QkqtBZQXVzpuZ1Izs8kNUz35B=s900-c-k-c0x00ffffff-no-rj", + "subscribers": null } } diff --git a/ucast/tests/_testfiles/fixture/videos.json b/ucast/tests/_testfiles/fixture/videos.json index f2a9b65..4ac568f 100644 --- a/ucast/tests/_testfiles/fixture/videos.json +++ b/ucast/tests/_testfiles/fixture/videos.json @@ -1,54 +1,58 @@ [ { "model": "ucast.channel", - "pk": "UCGiJh0NZ52wRhYKYnuZI08Q", + "pk": 1, "fields": { + "channel_id": "UCGiJh0NZ52wRhYKYnuZI08Q", "name": "ThetaDev", "slug": "ThetaDev", "description": "I'm ThetaDev. I love creating cool projects using electronics, 3D printers and other awesome tech-based stuff.", + "subscribers": "37", "active": true, "skip_livestreams": true, "skip_shorts": true, - "keep_videos": null, "avatar_url": "https://yt3.ggpht.com/ytc/AKedOLSnFfmpibLLoqyaYdsF6bJ-zaLPzomII__FrJve1w=s900-c-k-c0x00ffffff-no-rj" } }, { "model": "ucast.channel", - "pk": "UC2TXq_t06Hjdr2g_KdKpHQg", + "pk": 2, "fields": { + "channel_id": "UC2TXq_t06Hjdr2g_KdKpHQg", "name": "media.ccc.de", "slug": "media_ccc_de", "description": "The real official channel of the chaos computer club, operated by the CCC VOC (https://c3voc.de)", + "subscribers": "166K", "active": true, "skip_livestreams": true, "skip_shorts": true, - "keep_videos": null, "avatar_url": "https://yt3.ggpht.com/c1jcNSbPuOMDUieixkWIlXc82kMNJ8pCDmq5KtL8hjt74rAXLobsT9Y078-w5DK7ymKyDaqr=s900-c-k-c0x00ffffff-no-rj" } }, { "model": "ucast.channel", - "pk": "UCmLTTbctUZobNQrr8RtX8uQ", + "pk": 3, "fields": { + "channel_id": "UCmLTTbctUZobNQrr8RtX8uQ", "name": "Creative Commons", "slug": "Creative_Commons", "description": "Hello friends,\nWelcome to my channel CREATIVE COMMONS.\nOn this channel you will get all the videos absolutely free copyright and no matter how many videos you download there is no copyright claim you can download them and upload them to your channel and all the music is young Is on the channel they can also download and use in their videos on this channel you will find different videos in which OUTRO Videos, INTRO Videos, FREE MUSIC, FREE SOUND EFFECTS, LOWER THIRDS, and more.", + "subscribers": null, "active": true, "skip_livestreams": true, "skip_shorts": true, - "keep_videos": null, "avatar_url": "https://yt3.ggpht.com/-ybcsEHc8YCmKUZMr2bf4DZoDv7SKrutgKIh8kSxXugj296QkqtBZQXVzpuZ1Izs8kNUz35B=s900-c-k-c0x00ffffff-no-rj" } }, { "model": "ucast.video", - "pk": "ZPxEr4YdWt8", + "pk": 1, "fields": { + "video_id": "ZPxEr4YdWt8", "title": "ThetaDev @ Embedded World 2019", "slug": "20190602_ThetaDev_Embedded_World_2019", - "channel": "UCGiJh0NZ52wRhYKYnuZI08Q", + "channel": 1, "published": "2019-06-02T00:00:00Z", "downloaded": "2022-05-15T22:16:03.096Z", "description": "This february I spent one day at the Embedded World in Nuremberg. They showed tons of interesting electronics stuff, so I had to take some pictures and videos for you to see ;-)\n\nSorry for the late upload, I just didn't have time to edit my footage.\n\nEmbedded World: https://www.embedded-world.de/\n\nMy website: https://thdev.org\nTwitter: https://twitter.com/Theta_Dev", @@ -60,11 +64,12 @@ }, { "model": "ucast.video", - "pk": "_I5IFObm_-k", + "pk": 2, "fields": { + "video_id": "_I5IFObm_-k", "title": "Easter special: 3D printed Bunny", "slug": "20180331_Easter_special_3D_printed_Bunny", - "channel": "UCGiJh0NZ52wRhYKYnuZI08Q", + "channel": 1, "published": "2018-03-31T00:00:00Z", "downloaded": "2022-05-15T22:16:12.514Z", "description": "Happy Easter 2018!\nThis is just a special video where I print a little bunny as an Easter gift for friends or relatives. I hope you like the model, too.\n\nSadly my camera doesn't support timelapses, so I had to record the whole 4h printing process in real time, resulting in 30GB of footage. But I think it was worth it ;-)\n\n__PROJECT_LINKS___________________________\nBunny: https://www.thingiverse.com/thing:287884\n\n__COMPONENT_SUPPLIERS__________________\n3D printer: https://www.prusa3d.com/\n3D printing filament: https://www.dasfilament.de/\n______________________________________________\nMy website: https://thdev.org\nTwitter: https://twitter.com/Theta_Dev", @@ -76,11 +81,12 @@ }, { "model": "ucast.video", - "pk": "mmEDPbbSnaY", + "pk": 3, "fields": { + "video_id": "mmEDPbbSnaY", "title": "ThetaDevlog#2 - MySensors singleLED", "slug": "20180326_ThetaDevlog_2_MySensors_singleLED", - "channel": "UCGiJh0NZ52wRhYKYnuZI08Q", + "channel": 1, "published": "2018-03-26T00:00:00Z", "downloaded": "2022-05-15T22:16:20.280Z", "description": "The PCBs and components for the MySensors smart home devices arrived!\nIn this video I'll show you how to build the singleLED controller to switch/dim your 12V led lights. Detailed building instructions can be found on OpenHardware or GitHub.\n\n__PROJECT_LINKS___________________________\nOpenHardware: https://www.openhardware.io/view/563\nGitHub: https://github.com/Theta-Dev/MySensors-singleLED\n\nProgramming adapter: https://thdev.org/?Projects___misc___micro_JST\nBoard definitions: http://files.thdev.org/arduino/atmega.zip\n\n__COMPONENT_SUPPLIERS__________________\nElectronic components: https://www.aliexpress.com/\nPCBs: http://www.allpcb.com/\n3D printing filament: https://www.dasfilament.de/\n______________________________________________\nMy website: https://thdev.org\nTwitter: https://twitter.com/Theta_Dev\n______________________________________________\nMusic by Bartlebeats: https://bartlebeats.bandcamp.com", @@ -92,11 +98,12 @@ }, { "model": "ucast.video", - "pk": "Cda4zS-1j-k", + "pk": 4, "fields": { + "video_id": "Cda4zS-1j-k", "title": "ThetaDevlog#1 - MySensors Smart Home!", "slug": "20180217_ThetaDevlog_1_MySensors_Smart_Home", - "channel": "UCGiJh0NZ52wRhYKYnuZI08Q", + "channel": 1, "published": "2018-02-17T00:00:00Z", "downloaded": "2022-05-15T22:16:25.237Z", "description": "Smart Home devices have been around for some time and can really make your life easier. But most of them are quite pricey and not always worth the money.\n\nHow about a sytem that costs only 5€ per device and has all the benefits of the expensive solutions? The open source project MySensors claims to do that. In this series I'll try this and find out whether it works!\n\n______________________________________________\nMy website: https://thdev.org\nTwitter: https://twitter.com/Theta_Dev", @@ -108,11 +115,12 @@ }, { "model": "ucast.video", - "pk": "2xfXsqyd8YA", + "pk": 5, "fields": { + "video_id": "2xfXsqyd8YA", "title": "cy: Log4Shell - Bug oder Feature", "slug": "20220521_cy_Log4Shell_Bug_oder_Feature", - "channel": "UC2TXq_t06Hjdr2g_KdKpHQg", + "channel": 2, "published": "2022-05-21T00:00:00Z", "downloaded": null, "description": "https://media.ccc.de/v/gpn20-60-log4shell-bug-oder-feature\n\n\n\nUm den Jahreswechsel ging ein Aufschrei durch die IT-Abteilungen der Welt, der es bis in die Mainstream-Medien geschafft hat. Noch Wochen später zeigen sich Folgeprobleme in weit verbreiteter Software.\n \nIn Log4j, einer weit verbreiteten Java-Bibliothek wurde eine massive Sicherheitslücke gefunden, die die Ausführung von Schadcode auf einem entfernten System erlaubt.\nIn diesem Vortrag soll rekapitulierend erklärt werden, warum und wann es zu dem Problem kam und welche Auswirkungen bisher erkennbar sind. Ausserdem werden die technischen Details der Schwachstelle erklärt und in einer Live-Demo gezeigt, wie die Schwachstelle ausgenutzt werden kann.\n\n\n\ncy\n\nhttps://cfp.gulas.ch/gpn20/talk/77BCXN/\n\n#gpn20 #Security", @@ -124,11 +132,12 @@ }, { "model": "ucast.video", - "pk": "I0RRENheeTo", + "pk": 6, "fields": { + "video_id": "I0RRENheeTo", "title": "No copyright intro free fire intro | no text | free copy right | free templates | free download", "slug": "20211010_No_copyright_intro_free_fire_intro_no_text_free_copy_right_free_templates_free_download", - "channel": "UCmLTTbctUZobNQrr8RtX8uQ", + "channel": 3, "published": "2021-10-10T00:00:00Z", "downloaded": null, "description": "Like Video▬▬▬▬▬❤\uD83D\uDC4D❤\n▬▬\uD83D\uDC47SUBSCRIBE OUR CHANNEL FOR LATEST UPDATES\uD83D\uDC46▬▬\nThis Channel: https://www.youtube.com/channel/UCmLTTbctUZobNQrr8RtX8uQ?sub_confirmation=1\nOther Channel: https://www.youtube.com/channel/UCKtfYFXi5A4KLIUdjgvfmHg?sub_confirmation=1\n▬▬▬▬▬▬▬▬/Subscription Free\\▬▬▬▬▬▬▬▬▬\n▬▬▬▬▬\uD83C\uDF81...Share Video To Friends...\uD83C\uDF81▬▬▬▬▬▬▬\n▬▬▬▬\uD83E\uDD14...Comment Any Questions....\uD83E\uDD14▬▬▬▬▬▬\nHello friends, \n Shahzaib Hassan and you are watching Creative Commons YouTube channel. On this channel, you will find all the videos absolutely free copyright which you can download and use in any project.\n It is copyright free so you won't have any problem using end screen for YouTube. if you use it or download and reupload it to your channel. By doing this you can use it for YouTube its use is absolutely free.\n ►I hope you'll like the video.◄\n ►Thanks For Watching◄ \nIf you really like this video then please don't forget to...\n\n\n▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬\n▬▬▬▬▬▬▬▬▬▬Tags\uD83D\uDC47▬▬▬▬▬▬▬▬▬▬\n#Creativecommons #commoncreative #free #freecopyright #nocopyright #nowatermark #freetouse #intro #notext #fireefire #channelintro", diff --git a/ucast/tests/conftest.py b/ucast/tests/conftest.py index 6c92ab3..c89e72f 100644 --- a/ucast/tests/conftest.py +++ b/ucast/tests/conftest.py @@ -56,7 +56,9 @@ def download_dir_content(download_dir) -> Path: file_cover = cf.get_cover(video.slug) shutil.copyfile(tests.DIR_TESTFILES / "audio" / "audio1.mp3", file_audio) - shutil.copyfile(tests.DIR_TESTFILES / "thumbnail" / f"{video.id}.webp", file_tn) + shutil.copyfile( + tests.DIR_TESTFILES / "thumbnail" / f"{video.video_id}.webp", file_tn + ) util.resize_thumbnail(file_tn, cf.get_thumbnail(video.slug, True)) cover.create_cover_file( file_tn, diff --git a/ucast/tests/service/test_videoutil.py b/ucast/tests/service/test_videoutil.py index 3b709b9..bd66da6 100644 --- a/ucast/tests/service/test_videoutil.py +++ b/ucast/tests/service/test_videoutil.py @@ -14,7 +14,7 @@ from ucast.service import videoutil @pytest.mark.django_db def test_tag_audio(): - video = Video.objects.get(id="ZPxEr4YdWt8") + video = Video.objects.get(video_id="ZPxEr4YdWt8") tmpdir_o = tempfile.TemporaryDirectory() tmpdir = Path(tmpdir_o.name) diff --git a/ucast/tests/tasks/test_download.py b/ucast/tests/tasks/test_download.py index d4ec883..c9b8b8f 100644 --- a/ucast/tests/tasks/test_download.py +++ b/ucast/tests/tasks/test_download.py @@ -14,7 +14,7 @@ VIDEO_SLUG_INTRO = "20211010_No_copyright_intro_free_fire_intro_no_text_free_cop @pytest.mark.django_db def test_download_video(download_dir, rq_queue): - video = Video.objects.get(id=VIDEO_ID_INTRO) + video = Video.objects.get(video_id=VIDEO_ID_INTRO) job = queue.enqueue(download.download_video, video) store = storage.Storage() @@ -33,8 +33,8 @@ def test_import_channel( download_dir, rq_queue, mock_get_video_details, mock_download_audio ): # Remove 2 videos from the database so they can be imported - Video.objects.get(id="ZPxEr4YdWt8").delete() - Video.objects.get(id="_I5IFObm_-k").delete() + Video.objects.get(video_id="ZPxEr4YdWt8").delete() + Video.objects.get(video_id="_I5IFObm_-k").delete() job = rq_queue.enqueue(download.import_channel, CHANNEL_ID_THETADEV) assert job.is_finished @@ -54,10 +54,10 @@ def test_update_channel( download_dir, rq_queue, mock_get_video_details, mock_download_audio ): # Remove 2 videos from the database so they can be imported - Video.objects.get(id="ZPxEr4YdWt8").delete() - Video.objects.get(id="_I5IFObm_-k").delete() + Video.objects.get(video_id="ZPxEr4YdWt8").delete() + Video.objects.get(video_id="_I5IFObm_-k").delete() - channel = Channel.objects.get(id=CHANNEL_ID_THETADEV) + channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV) job = rq_queue.enqueue(download.update_channel, channel) assert job.is_finished diff --git a/ucast/tests/tasks/test_library.py b/ucast/tests/tasks/test_library.py index 96d8c06..e36c0d4 100644 --- a/ucast/tests/tasks/test_library.py +++ b/ucast/tests/tasks/test_library.py @@ -14,7 +14,7 @@ CHANNEL_ID_THETADEV = "UCGiJh0NZ52wRhYKYnuZI08Q" def test_recreate_cover(download_dir_content, rq_queue, mocker): create_cover_mock: mock.Mock = mocker.patch.object(cover, "create_cover_file") - video = Video.objects.get(id="ZPxEr4YdWt8") + video = Video.objects.get(video_id="ZPxEr4YdWt8") store = storage.Storage() cf = store.get_or_create_channel_folder(video.channel.slug) @@ -45,20 +45,29 @@ def test_recreate_covers(rq_queue, mocker): @pytest.mark.django_db def test_update_channel_info(rq_queue, mock_get_channel_metadata): - channel = Channel.objects.get(id=CHANNEL_ID_THETADEV) + channel = Channel.objects.get(channel_id=CHANNEL_ID_THETADEV) + channel.name = "Old name" channel.description = "Old description" + channel.subscribers = "Old" + channel.avatar_url = "Old avatar url" channel.save() job = rq_queue.enqueue(library.update_channel_info, channel) assert job.is_finished channel.refresh_from_db() + assert channel.name == "ThetaDev" assert ( channel.description == "I'm ThetaDev. I love creating cool projects \ using electronics, 3D printers and other awesome tech-based stuff." ) + assert channel.subscribers == "37" + assert ( + channel.avatar_url + == "https://yt3.ggpht.com/ytc/AKedOLSnFfmpibLLoqyaYdsF6bJ-zaLPzomII__FrJve1w=s900-c-k-c0x00ffffff-no-rj" + ) @pytest.mark.django_db From 55d7d5f3b2260f8c62171d9ba37c1b3e56c38c33 Mon Sep 17 00:00:00 2001 From: Theta-Dev <t.testboy@gmail.com> Date: Sat, 28 May 2022 22:16:25 +0200 Subject: [PATCH 3/5] add tests for views and feed --- ucast/service/videoutil.py | 22 +++-- ucast/tasks/download.py | 9 +- ucast/tasks/library.py | 9 +- .../fixture/{videos.json => models.json} | 0 ucast/tests/conftest.py | 72 +++++++++++---- ucast/tests/service/test_videoutil.py | 9 +- ucast/tests/test_feed.py | 82 +++++++++++++++++ ucast/tests/test_views.py | 92 +++++++++++++++++++ ucast/views.py | 18 ++-- ucast_project/settings.py | 1 + 10 files changed, 274 insertions(+), 40 deletions(-) rename ucast/tests/_testfiles/fixture/{videos.json => models.json} (100%) create mode 100644 ucast/tests/test_feed.py create mode 100644 ucast/tests/test_views.py diff --git a/ucast/service/videoutil.py b/ucast/service/videoutil.py index 601b000..45e7af5 100644 --- a/ucast/service/videoutil.py +++ b/ucast/service/videoutil.py @@ -1,19 +1,25 @@ +from datetime import date from pathlib import Path from mutagen import id3 -from ucast.models import Video - -def tag_audio(audio_path: Path, video: Video, cover_path: Path): - title_text = f"{video.published.date().isoformat()} {video.title}" +def tag_audio( + audio_path: Path, + title: str, + channel: str, + published: date, + description: str, + cover_path: Path, +): + title_text = f"{published.isoformat()} {title}" tag = id3.ID3(audio_path) - tag["TPE1"] = id3.TPE1(encoding=3, text=video.channel.name) # Artist - tag["TALB"] = id3.TALB(encoding=3, text=video.channel.name) # Album + tag["TPE1"] = id3.TPE1(encoding=3, text=channel) # Artist + tag["TALB"] = id3.TALB(encoding=3, text=channel) # Album tag["TIT2"] = id3.TIT2(encoding=3, text=title_text) # Title - tag["TDRC"] = id3.TDRC(encoding=3, text=video.published.date().isoformat()) # Date - tag["COMM"] = id3.COMM(encoding=3, text=video.get_full_description()) # Comment + tag["TDRC"] = id3.TDRC(encoding=3, text=published.isoformat()) # Date + tag["COMM"] = id3.COMM(encoding=3, text=description) # Comment with open(cover_path, "rb") as albumart: tag["APIC"] = id3.APIC( diff --git a/ucast/tasks/download.py b/ucast/tasks/download.py index f24e238..a26a0c2 100644 --- a/ucast/tasks/download.py +++ b/ucast/tasks/download.py @@ -93,7 +93,14 @@ def download_video(video: Video): cover_file, ) - videoutil.tag_audio(audio_file, video, cover_file) + videoutil.tag_audio( + audio_file, + video.title, + video.channel.name, + video.published.date(), + video.get_full_description(), + cover_file, + ) video.downloaded = timezone.now() video.download_size = os.path.getsize(audio_file) diff --git a/ucast/tasks/library.py b/ucast/tasks/library.py index 3c28c5e..b6b3481 100644 --- a/ucast/tasks/library.py +++ b/ucast/tasks/library.py @@ -30,7 +30,14 @@ def recreate_cover(video: Video): cover_file, ) - videoutil.tag_audio(audio_file, video, cover_file) + videoutil.tag_audio( + audio_file, + video.title, + video.channel.name, + video.published.date(), + video.get_full_description(), + cover_file, + ) def recreate_covers(): diff --git a/ucast/tests/_testfiles/fixture/videos.json b/ucast/tests/_testfiles/fixture/models.json similarity index 100% rename from ucast/tests/_testfiles/fixture/videos.json rename to ucast/tests/_testfiles/fixture/models.json diff --git a/ucast/tests/conftest.py b/ucast/tests/conftest.py index c89e72f..04e7fb0 100644 --- a/ucast/tests/conftest.py +++ b/ucast/tests/conftest.py @@ -1,6 +1,10 @@ +import json import shutil import tempfile +from datetime import datetime from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Tuple from unittest import mock import pytest @@ -10,19 +14,23 @@ from django.core.management import call_command from fakeredis import FakeRedis from ucast import queue, tests -from ucast.models import Video from ucast.service import cover, storage, util, videoutil, youtube +@pytest.fixture(scope="session", autouse=True) +def default_config(): + settings.DOWNLOAD_ROOT = Path("does/not/exist") + settings.REDIS_URL = "no redis" + + @pytest.fixture(scope="session") def django_db_setup(django_db_setup, django_db_blocker): with django_db_blocker.unblock(): - fixture_path = tests.DIR_TESTFILES / "fixture" / "videos.json" + fixture_path = tests.DIR_TESTFILES / "fixture" / "models.json" call_command("loaddata", fixture_path) -@pytest.fixture -def download_dir() -> Path: +def _create_download_dir() -> Tuple[Path, TemporaryDirectory]: tmpdir_o = tempfile.TemporaryDirectory() tmpdir = Path(tmpdir_o.name) settings.DOWNLOAD_ROOT = tmpdir @@ -41,36 +49,60 @@ def download_dir() -> Path: ) util.resize_avatar(cf.file_avatar, cf.file_avatar_sm) - yield tmpdir + return tmpdir, tmpdir_o @pytest.fixture -@pytest.mark.django_db -def download_dir_content(download_dir) -> Path: +def download_dir() -> Path: + tmpdir, tmpdir_o = _create_download_dir() + yield tmpdir + + +@pytest.fixture(scope="session") +def download_dir_content() -> Path: + tmpdir, tmpdir_o = _create_download_dir() + settings.DOWNLOAD_ROOT = tmpdir + store = storage.Storage() - for video in Video.objects.filter(downloaded__isnull=False): - cf = store.get_or_create_channel_folder(video.channel.slug) - file_audio = cf.get_audio(video.slug) - file_tn = cf.get_thumbnail(video.slug) - file_cover = cf.get_cover(video.slug) + with open(tests.DIR_TESTFILES / "fixture" / "videodetails.json") as f: + videodetails = json.load(f) + + for vid in ("ZPxEr4YdWt8", "_I5IFObm_-k", "mmEDPbbSnaY", "Cda4zS-1j-k"): + video_detail = videodetails[vid] + channel_name = video_detail["channel_name"] + channel_slug = util.get_slug(channel_name) + published = datetime.fromisoformat(video_detail["published"]) + title = video_detail["title"] + video_slug = util.get_slug(f"{published.strftime('%Y%m%d')}_{title}") + description = video_detail["description"] + + cf = store.get_or_create_channel_folder(channel_slug) + file_audio = cf.get_audio(video_slug) + file_tn = cf.get_thumbnail(video_slug) + file_cover = cf.get_cover(video_slug) shutil.copyfile(tests.DIR_TESTFILES / "audio" / "audio1.mp3", file_audio) - shutil.copyfile( - tests.DIR_TESTFILES / "thumbnail" / f"{video.video_id}.webp", file_tn - ) - util.resize_thumbnail(file_tn, cf.get_thumbnail(video.slug, True)) + shutil.copyfile(tests.DIR_TESTFILES / "thumbnail" / f"{vid}.webp", file_tn) + util.resize_thumbnail(file_tn, cf.get_thumbnail(video_slug, True)) cover.create_cover_file( file_tn, cf.file_avatar, - video.title, - video.channel.name, + title, + channel_name, cover.COVER_STYLE_BLUR, file_cover, ) - videoutil.tag_audio(file_audio, video, file_cover) + videoutil.tag_audio( + file_audio, + title, + channel_name, + published.date(), + f"{description}\n\nhttps://youtu.be/{vid}", + file_cover, + ) - yield download_dir + yield tmpdir @pytest.fixture diff --git a/ucast/tests/service/test_videoutil.py b/ucast/tests/service/test_videoutil.py index bd66da6..c7de7ae 100644 --- a/ucast/tests/service/test_videoutil.py +++ b/ucast/tests/service/test_videoutil.py @@ -22,7 +22,14 @@ def test_tag_audio(): cover_file = tests.DIR_TESTFILES / "cover" / "c1_blur.png" shutil.copyfile(tests.DIR_TESTFILES / "audio" / "audio1.mp3", audio_file) - videoutil.tag_audio(audio_file, video, cover_file) + videoutil.tag_audio( + audio_file, + video.title, + video.channel.name, + video.published.date(), + video.get_full_description(), + cover_file, + ) tag = id3.ID3(audio_file) assert tag["TPE1"].text[0] == "ThetaDev" diff --git a/ucast/tests/test_feed.py b/ucast/tests/test_feed.py new file mode 100644 index 0000000..0be7895 --- /dev/null +++ b/ucast/tests/test_feed.py @@ -0,0 +1,82 @@ +import feedparser +import pytest +from django.test.client import Client +from django.utils.feedgenerator import rfc2822_date + +from ucast import feed +from ucast.models import Video + + +@pytest.mark.django_db +def test_feed(): + client = Client() + response = client.get("/feed/ThetaDev") + parsed_feed = feedparser.parse(response.getvalue()) + + assert parsed_feed["bozo"] is False + + assert parsed_feed["namespaces"] == { + "": "http://www.w3.org/2005/Atom", + "itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd", + } + + meta = parsed_feed["feed"] + assert meta["title"] == "ThetaDev" + assert meta["link"] == "https://www.youtube.com/channel/UCGiJh0NZ52wRhYKYnuZI08Q" + assert ( + meta["subtitle"] + == "I'm ThetaDev. I love creating cool projects using electronics, 3D printers and other awesome tech-based stuff." + ) + assert meta["updated"] == "Sun, 15 May 2022 22:16:25 +0000" + assert meta["image"]["href"] == "http://testserver/files/avatar/ThetaDev.jpg" + + assert len(parsed_feed["entries"]) == 4 + + video_ids = ["ZPxEr4YdWt8", "_I5IFObm_-k", "mmEDPbbSnaY", "Cda4zS-1j-k"] + + for i, entry in enumerate(parsed_feed["entries"]): + video = Video.objects.get(video_id=video_ids[i]) + assert entry["title"] == video.title + assert entry["link"] == video.get_absolute_url() + assert entry["summary"] == feed.PodcastFeedType._xml_escape( + video.description + ).replace("<br>", "<br />") + assert entry["published"] == rfc2822_date(video.published) + assert entry["id"] == video.get_absolute_url() + assert ( + entry["image"]["href"] + == f"http://testserver/files/cover/ThetaDev/{video.slug}.png" + ) + assert entry["itunes_duration"] == feed.PodcastFeedType._format_secs( + video.duration + ) + + +@pytest.mark.parametrize( + "text,expect", + [ + ("Hello <World>", "Hello <World>"), + ( + "go to https://example.org/test", + 'go to <a href="https://example.org/test">https://example.org/test</a>', + ), + ("line1\nline2\nline3", "line1<br>line2<br>line3"), + ], +) +def test_xml_escape(text: str, expect: str): + escaped = feed.PodcastFeedType._xml_escape(text) + assert escaped == expect + + +@pytest.mark.parametrize( + "secs,expect", + [ + (0, "00:00:00"), + (16, "00:00:16"), + (100, "00:01:40"), + (3800, "01:03:20"), + ], +) +def test_format_secs(secs: int, expect: str): + time_str = feed.PodcastFeedType._format_secs(secs) + assert time_str == expect diff --git a/ucast/tests/test_views.py b/ucast/tests/test_views.py new file mode 100644 index 0000000..a9b462d --- /dev/null +++ b/ucast/tests/test_views.py @@ -0,0 +1,92 @@ +import pytest +from django.conf import settings +from django.test.client import Client + + +@pytest.mark.parametrize( + "url_path,internal_path", + [ + ( + "audio/ThetaDev/20190602_ThetaDev_Embedded_World_2019.mp3", + "/internal_files/ThetaDev/20190602_ThetaDev_Embedded_World_2019.mp3", + ), + ( + "cover/ThetaDev/20190602_ThetaDev_Embedded_World_2019.png", + "/internal_files/ThetaDev/_ucast/covers/20190602_ThetaDev_Embedded_World_2019.png", + ), + ( + "thumbnail/ThetaDev/20190602_ThetaDev_Embedded_World_2019.webp", + "/internal_files/ThetaDev/_ucast/thumbnails/20190602_ThetaDev_Embedded_World_2019.webp", + ), + ( + "thumbnail/ThetaDev/20190602_ThetaDev_Embedded_World_2019.webp?sm", + "/internal_files/ThetaDev/_ucast/thumbnails/20190602_ThetaDev_Embedded_World_2019_sm.webp", + ), + ( + "avatar/ThetaDev.jpg", + "/internal_files/ThetaDev/_ucast/avatar.jpg", + ), + ( + "avatar/ThetaDev.webp?sm", + "/internal_files/ThetaDev/_ucast/avatar_sm.webp", + ), + ], +) +def test_files_internal_redirect( + url_path: str, internal_path: str, download_dir_content +): + settings.INTERNAL_REDIRECT_HEADER = "X-Accel-Redirect" + + client = Client() + response = client.get("/files/" + url_path) + assert response.getvalue() == b"" + assert response.headers.get("X-Accel-Redirect") == internal_path + assert "Content-Type" not in response.headers + + +@pytest.mark.parametrize( + "url_path,file_path", + [ + ( + "audio/ThetaDev/20190602_ThetaDev_Embedded_World_2019.mp3", + "ThetaDev/20190602_ThetaDev_Embedded_World_2019.mp3", + ), + ( + "cover/ThetaDev/20190602_ThetaDev_Embedded_World_2019.png", + "ThetaDev/_ucast/covers/20190602_ThetaDev_Embedded_World_2019.png", + ), + ( + "thumbnail/ThetaDev/20190602_ThetaDev_Embedded_World_2019.webp", + "ThetaDev/_ucast/thumbnails/20190602_ThetaDev_Embedded_World_2019.webp", + ), + ( + "thumbnail/ThetaDev/20190602_ThetaDev_Embedded_World_2019.webp?sm", + "ThetaDev/_ucast/thumbnails/20190602_ThetaDev_Embedded_World_2019_sm.webp", + ), + ( + "avatar/ThetaDev.jpg", + "ThetaDev/_ucast/avatar.jpg", + ), + ( + "avatar/ThetaDev.webp?sm", + "ThetaDev/_ucast/avatar_sm.webp", + ), + ], +) +def test_files_response(url_path: str, file_path: str, download_dir_content): + # store = storage.Storage() + # cf = store.get_channel_folder("ThetaDev") + # audio_file = cf.get_audio("20190602_ThetaDev_Embedded_World_2019") + + settings.INTERNAL_REDIRECT_HEADER = "" + + response_file = settings.DOWNLOAD_ROOT / file_path + + client = Client() + response = client.get("/files/" + url_path) + response_bts = response.getvalue() + + with open(response_file, "rb") as f: + file_bts = f.read() + + assert response_bts == file_bts diff --git a/ucast/views.py b/ucast/views.py index d8881fc..126f6b7 100644 --- a/ucast/views.py +++ b/ucast/views.py @@ -16,7 +16,7 @@ def home(request: http.HttpRequest): return render(request, "ucast/main.html") -def channel_file(channel: str, get_file: Callable[[storage.ChannelFolder], Path]): +def _channel_file(channel: str, get_file: Callable[[storage.ChannelFolder], Path]): store = storage.Storage() try: @@ -31,11 +31,11 @@ def channel_file(channel: str, get_file: Callable[[storage.ChannelFolder], Path] if not settings.INTERNAL_REDIRECT_HEADER: return http.FileResponse(open(file_path, "rb")) - file_path_rel = os.path.relpath(file_path, store.dir_data) - file_path_internal = os.path.join("/internal_files", file_path_rel) + file_path_rel = file_path.relative_to(store.dir_data) + url_path_internal = f"/{settings.INTERNAL_FILES_ROOT}/{file_path_rel.as_posix()}" response = http.HttpResponse() - response.headers[settings.INTERNAL_REDIRECT_HEADER] = file_path_internal + response.headers[settings.INTERNAL_REDIRECT_HEADER] = url_path_internal # Content type is set to text/html by default and has to be unset del response.headers["Content-Type"] return response @@ -45,14 +45,14 @@ def audio(request: http.HttpRequest, channel: str, video: str): # Trim off file extension video_slug = video.rsplit(".")[0] - return channel_file(channel, lambda cf: cf.get_audio(video_slug)) + return _channel_file(channel, lambda cf: cf.get_audio(video_slug)) def cover(request: http.HttpRequest, channel: str, video: str): # Trim off file extension video_slug = video.rsplit(".")[0] - return channel_file(channel, lambda cf: cf.get_cover(video_slug)) + return _channel_file(channel, lambda cf: cf.get_cover(video_slug)) def thumbnail(request: http.HttpRequest, channel: str, video: str): @@ -61,7 +61,7 @@ def thumbnail(request: http.HttpRequest, channel: str, video: str): is_sm = "sm" in request.GET - return channel_file(channel, lambda cf: cf.get_thumbnail(video_slug, is_sm)) + return _channel_file(channel, lambda cf: cf.get_thumbnail(video_slug, is_sm)) def avatar(request: http.HttpRequest, channel: str): @@ -71,8 +71,8 @@ def avatar(request: http.HttpRequest, channel: str): is_sm = "sm" in request.GET if is_sm: - return channel_file(channel_slug, lambda cf: cf.file_avatar_sm) - return channel_file(channel_slug, lambda cf: cf.file_avatar) + return _channel_file(channel_slug, lambda cf: cf.file_avatar_sm) + return _channel_file(channel_slug, lambda cf: cf.file_avatar) @decorator_from_middleware(ConditionalGetMiddleware) diff --git a/ucast_project/settings.py b/ucast_project/settings.py index 9b4cae2..d3b20e6 100644 --- a/ucast_project/settings.py +++ b/ucast_project/settings.py @@ -214,4 +214,5 @@ REDIS_QUEUE_RESULT_TTL = 600 YT_UPDATE_INTERVAL = get_env("YT_UPDATE_INTERVAL", 900) +INTERNAL_FILES_ROOT = get_env("INTERNAL_FILES_ROOT", "internal_files") INTERNAL_REDIRECT_HEADER = get_env("INTERNAL_REDIRECT_HEADER", "X-Accel-Redirect") From 48cecbb621a73fbe40b40a97a4fbe5174f016464 Mon Sep 17 00:00:00 2001 From: Theta-Dev <t.testboy@gmail.com> Date: Thu, 2 Jun 2022 01:41:48 +0200 Subject: [PATCH 4/5] add main page, protect feed --- .editorconfig | 2 +- assets/sass/style.sass | 11 +- poetry.lock | 10 +- pyproject.toml | 1 + tasks.py | 16 +++ ucast/admin.py | 3 +- ucast/feed.py | 4 +- ucast/forms.py | 5 + ucast/migrations/0001_initial.py | 124 +++++++++++++++++- ucast/models.py | 37 ++++++ ucast/service/util.py | 55 ++++++++ ucast/service/youtube.py | 12 +- ucast/static/ucast/logo.svg | 2 + ucast/static/ucast/logo_dark.svg | 2 + ucast/tasks/download.py | 67 +++++----- ucast/tasks/schedule.py | 9 +- ucast/templates/bulma/base.html | 77 +++++++++++ ucast/templates/ucast/channels.html | 48 +++++++ ucast/templates/ucast/main.html | 11 -- ucast/templates/ucast/videos.html | 94 +++++++++++++ ucast/tests/__init__.py | 4 +- ucast/tests/_testfiles/fixture/models.json | 20 ++- .../{fixture => object}/channelmeta.json | 0 .../{fixture => object}/videodetails.json | 0 ucast/tests/conftest.py | 40 ++++-- ucast/tests/service/test_youtube.py | 2 +- ucast/tests/tasks/test_library.py | 2 +- ucast/tests/test_feed.py | 11 +- ucast/tests/test_views.py | 12 +- ucast/urls.py | 1 + ucast/views.py | 69 +++++++++- ucast_project/settings.py | 5 + ucast_project/urls.py | 1 + 33 files changed, 662 insertions(+), 95 deletions(-) create mode 100644 ucast/forms.py create mode 100644 ucast/static/ucast/logo.svg create mode 100644 ucast/static/ucast/logo_dark.svg create mode 100644 ucast/templates/bulma/base.html create mode 100644 ucast/templates/ucast/channels.html delete mode 100644 ucast/templates/ucast/main.html create mode 100644 ucast/templates/ucast/videos.html rename ucast/tests/_testfiles/{fixture => object}/channelmeta.json (100%) rename ucast/tests/_testfiles/{fixture => object}/videodetails.json (100%) diff --git a/.editorconfig b/.editorconfig index 71f453a..914c96c 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,5 +10,5 @@ max_line_length = 88 [{Makefile,*.go}] indent_style = tab -[*.{json,md,rst,ini,yml,yaml}] +[*.{json,md,rst,ini,yml,yaml,html,js,jsx,ts,tsx,vue}] indent_size = 2 diff --git a/assets/sass/style.sass b/assets/sass/style.sass index a74478f..27219f7 100644 --- a/assets/sass/style.sass +++ b/assets/sass/style.sass @@ -1,9 +1,8 @@ -// 1. Import the initial variables @import "../../node_modules/bulma/sass/utilities/initial-variables" - -// 2. Set your own initial variables - -// 3. Import the rest of Bulma @import "../../node_modules/bulma/bulma" -// 4. Import your stuff here \ No newline at end of file +.channel-icon + max-height: 64px + +.video-thumbnail + max-height: 128px diff --git a/poetry.lock b/poetry.lock index 2252a0d..a453813 100644 --- a/poetry.lock +++ b/poetry.lock @@ -203,11 +203,11 @@ django = ">=2.2" [[package]] name = "fakeredis" -version = "1.7.6.1" +version = "1.8" description = "Fake implementation of redis API for testing purposes." category = "dev" optional = false -python-versions = ">=3.8,<4.0" +python-versions = ">=3.7,<4.0" [package.dependencies] redis = "<=4.3.1" @@ -743,7 +743,7 @@ websockets = "*" [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "d0101b3a682960de22ddde7f1affe6fc0b78d45f351749469f9f63047c725b27" +content-hash = "1d1799636eadf391bd9545eb3d8750a1e882cdb9b84d47395284d90a3cfeb609" [metadata.files] asgiref = [ @@ -998,8 +998,8 @@ django-bulma = [ {file = "django_bulma-0.8.3-py3-none-any.whl", hash = "sha256:0ef6e5c171c2a32010e724a8be61ba6cd0e55ebbd242cf6780560518483c4d00"}, ] fakeredis = [ - {file = "fakeredis-1.7.6.1-py3-none-any.whl", hash = "sha256:2bc92cece6535961a465991d01841888e0fe2b742ca49aa97ce247b04c9a0ecc"}, - {file = "fakeredis-1.7.6.1.tar.gz", hash = "sha256:e71ca849167052f42f5469a764def9ef35ccffd04773d30b017a7adcc12940c1"}, + {file = "fakeredis-1.8-py3-none-any.whl", hash = "sha256:65dcd78c0cd29d17daccce9f58698f6ab61ad7a404eab373fcad2b76fe8db03d"}, + {file = "fakeredis-1.8.tar.gz", hash = "sha256:cbf8d74ae06672d40b2fa88b9ee4f1d6efd56b06b2e7f0be2c639647f00643f1"}, ] feedparser = [ {file = "feedparser-6.0.10-py3-none-any.whl", hash = "sha256:79c257d526d13b944e965f6095700587f27388e50ea16fd245babe4dfae7024f"}, diff --git a/pyproject.toml b/pyproject.toml index 63dfa34..12a3f8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ python-slugify = "^6.1.2" mutagen = "^1.45.1" rq = "^1.10.1" rq-scheduler = "^0.11.0" +pycryptodomex = "^3.14.1" [tool.poetry.dev-dependencies] pytest = "^7.1.1" diff --git a/tasks.py b/tasks.py index 62c614f..7798bf3 100644 --- a/tasks.py +++ b/tasks.py @@ -130,3 +130,19 @@ def worker(c, n=2): m.loop() sys.exit(m.returncode) + + +@task +def optimize_svg(c): + out_dir = Path("ucast/static/ucast") + + for icon in (Path("assets/icons/logo.svg"), Path("assets/icons/logo_dark.svg")): + c.run( + f"scour --indent=none --no-line-breaks --enable-comment-stripping {icon} {out_dir / icon.name}" + ) + + +@task +def build_sass(c): + c.run("npm run build") + collectstatic(c) diff --git a/ucast/admin.py b/ucast/admin.py index 580c34b..e50cc42 100644 --- a/ucast/admin.py +++ b/ucast/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin -from ucast.models import Channel, Video +from ucast.models import Channel, User, Video class ChannelAdmin(admin.ModelAdmin): @@ -14,3 +14,4 @@ class VideoAdmin(admin.ModelAdmin): admin.site.register(Channel, ChannelAdmin) admin.site.register(Video, VideoAdmin) +admin.site.register(User) diff --git a/ucast/feed.py b/ucast/feed.py index 8a5fe20..68c8cb9 100644 --- a/ucast/feed.py +++ b/ucast/feed.py @@ -9,6 +9,7 @@ from django.utils.feedgenerator import Rss201rev2Feed, rfc2822_date from django.utils.xmlutils import SimplerXMLGenerator from ucast.models import Channel, Video +from ucast.service import util URL_REGEX = r"""http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+""" @@ -173,11 +174,12 @@ class UcastFeed(Feed): @staticmethod def full_link_url(request: http.HttpRequest, page_url: str) -> str: - return add_domain( + anon_url = add_domain( get_current_site(request).domain, page_url, request.is_secure(), ) + return util.add_key_to_url(anon_url, request.user.get_feed_key()) def item_enclosures_domain(self, item: Video, request: http.HttpRequest): enc = feedgenerator.Enclosure( diff --git a/ucast/forms.py b/ucast/forms.py new file mode 100644 index 0000000..c06cb99 --- /dev/null +++ b/ucast/forms.py @@ -0,0 +1,5 @@ +from django import forms + + +class AddChannelForm(forms.Form): + channel_str = forms.CharField(label="Channel-ID / URL") diff --git a/ucast/migrations/0001_initial.py b/ucast/migrations/0001_initial.py index 7494990..268fe5d 100644 --- a/ucast/migrations/0001_initial.py +++ b/ucast/migrations/0001_initial.py @@ -1,6 +1,9 @@ -# Generated by Django 4.0.4 on 2022-05-25 22:57 +# Generated by Django 4.0.4 on 2022-05-29 21:01 +import django.contrib.auth.models +import django.contrib.auth.validators import django.db.models.deletion +import django.utils.timezone from django.db import migrations, models @@ -8,7 +11,9 @@ class Migration(migrations.Migration): initial = True - dependencies = [] + dependencies = [ + ("auth", "0012_alter_user_first_name_max_length"), + ] operations = [ migrations.CreateModel( @@ -64,4 +69,119 @@ class Migration(migrations.Migration): ), ], ), + migrations.CreateModel( + name="User", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("password", models.CharField(max_length=128, verbose_name="password")), + ( + "last_login", + models.DateTimeField( + blank=True, null=True, verbose_name="last login" + ), + ), + ( + "is_superuser", + models.BooleanField( + default=False, + help_text="Designates that this user has all permissions without explicitly assigning them.", + verbose_name="superuser status", + ), + ), + ( + "username", + models.CharField( + error_messages={ + "unique": "A user with that username already exists." + }, + help_text="Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.", + max_length=150, + unique=True, + validators=[ + django.contrib.auth.validators.UnicodeUsernameValidator() + ], + verbose_name="username", + ), + ), + ( + "first_name", + models.CharField( + blank=True, max_length=150, verbose_name="first name" + ), + ), + ( + "last_name", + models.CharField( + blank=True, max_length=150, verbose_name="last name" + ), + ), + ( + "email", + models.EmailField( + blank=True, max_length=254, verbose_name="email address" + ), + ), + ( + "is_staff", + models.BooleanField( + default=False, + help_text="Designates whether the user can log into this admin site.", + verbose_name="staff status", + ), + ), + ( + "is_active", + models.BooleanField( + default=True, + help_text="Designates whether this user should be treated as active. Unselect this instead of deleting accounts.", + verbose_name="active", + ), + ), + ( + "date_joined", + models.DateTimeField( + default=django.utils.timezone.now, verbose_name="date joined" + ), + ), + ("feed_key", models.CharField(default=None, max_length=50, null=True)), + ( + "groups", + models.ManyToManyField( + blank=True, + help_text="The groups this user belongs to. A user will get all permissions granted to each of their groups.", + related_name="user_set", + related_query_name="user", + to="auth.group", + verbose_name="groups", + ), + ), + ( + "user_permissions", + models.ManyToManyField( + blank=True, + help_text="Specific permissions for this user.", + related_name="user_set", + related_query_name="user", + to="auth.permission", + verbose_name="user permissions", + ), + ), + ], + options={ + "verbose_name": "user", + "verbose_name_plural": "users", + "abstract": False, + }, + managers=[ + ("objects", django.contrib.auth.models.UserManager()), + ], + ), ] diff --git a/ucast/models.py b/ucast/models.py index f6e7538..75eaa15 100644 --- a/ucast/models.py +++ b/ucast/models.py @@ -1,5 +1,8 @@ +import base64 import datetime +from Cryptodome import Random +from django.contrib.auth.models import AbstractUser from django.db import models from ucast.service import util @@ -51,6 +54,20 @@ class Channel(models.Model): def get_absolute_url(self) -> str: return "https://www.youtube.com/channel/" + self.channel_id + def should_download(self, video: "Video") -> bool: + if self.skip_livestreams and video.is_livestream: + return False + + if self.skip_shorts and video.is_short: + return False + + return True + + def download_size(self) -> int: + return self.video_set.aggregate(models.Sum("download_size")).get( + "download_size__sum" + ) + def __str__(self): return self.name @@ -87,3 +104,23 @@ class Video(models.Model): def __str__(self): return self.title + + +class User(AbstractUser): + feed_key = models.CharField(max_length=50, null=True, default=None) + + def generate_feed_key(self): + for _ in range(0, User.objects.count()): + key = base64.urlsafe_b64encode(Random.get_random_bytes(18)).decode() + + if not User.objects.filter(feed_key=key).exists(): + self.feed_key = key + self.save() + return + + raise Exception("unique feed key could not be found") + + def get_feed_key(self) -> str: + if self.feed_key is None: + self.generate_feed_key() + return self.feed_key diff --git a/ucast/service/util.py b/ucast/service/util.py index c01cb3f..21fa73a 100644 --- a/ucast/service/util.py +++ b/ucast/service/util.py @@ -3,6 +3,7 @@ import io import json from pathlib import Path from typing import Any, Union +from urllib import parse import requests import slugify @@ -108,3 +109,57 @@ def to_json(o, pretty=False) -> str: return json.dumps( o, default=serializer, indent=2 if pretty else None, ensure_ascii=False ) + + +def _urlencode(query, safe="", encoding=None, errors=None, quote_via=parse.quote_plus): + """ + Same as the urllib.parse.urlencode function, but does not add an + equals sign to no-value flags. + """ + + if hasattr(query, "items"): + query = query.items() + else: + # It's a bother at times that strings and string-like objects are + # sequences. + try: + # non-sequence items should not work with len() + # non-empty strings will fail this + if len(query) and not isinstance(query[0], tuple): + raise TypeError + # Zero-length sequences of all types will get here and succeed, + # but that's a minor nit. Since the original implementation + # allowed empty dicts that type of behavior probably should be + # preserved for consistency + except TypeError: + raise TypeError("not a valid non-string sequence " "or mapping object") + + lst = [] + + for k, v in query: + if isinstance(k, bytes): + k = quote_via(k, safe) + else: + k = quote_via(str(k), safe, encoding, errors) + + if isinstance(v, bytes): + v = quote_via(v, safe) + else: + v = quote_via(str(v), safe, encoding, errors) + + if v: + lst.append(k + "=" + v) + else: + lst.append(k) + + return "&".join(lst) + + +def add_key_to_url(url: str, key: str): + if not key: + return url + url_parts = list(parse.urlparse(url)) + query = dict(parse.parse_qsl(url_parts[4], keep_blank_values=True)) + query["key"] = key + url_parts[4] = _urlencode(query) + return parse.urlunparse(url_parts) diff --git a/ucast/service/youtube.py b/ucast/service/youtube.py index 2ce93c8..737220b 100644 --- a/ucast/service/youtube.py +++ b/ucast/service/youtube.py @@ -5,7 +5,7 @@ import shutil from dataclasses import dataclass from operator import itemgetter from pathlib import Path -from typing import List, Optional +from typing import Generator, List, Optional import feedparser import requests @@ -272,16 +272,14 @@ def get_channel_videos_from_feed(channel_id: str) -> List[VideoScraped]: def get_channel_videos_from_scraper( channel_id: str, limit: int = None -) -> List[VideoScraped]: +) -> Generator[VideoScraped, None, None]: """ Return all videos of a channel by scraping the YouTube website. - May take a while depending on the number of videos. :param channel_id: YouTube channel id :param limit: Limit number of scraped videos - :return: Videos: video_id -> VideoScraped + :return: Generator of Videos """ - videos = [] for item in scrapetube.get_channel(channel_url_from_id(channel_id), limit): video_id = item.get("videoId") @@ -291,6 +289,4 @@ def get_channel_videos_from_scraper( ) continue - videos.append(VideoScraped(video_id, None)) - - return videos + yield VideoScraped(video_id, None) diff --git a/ucast/static/ucast/logo.svg b/ucast/static/ucast/logo.svg new file mode 100644 index 0000000..fa59e11 --- /dev/null +++ b/ucast/static/ucast/logo.svg @@ -0,0 +1,2 @@ +<?xml version="1.0" encoding="UTF-8"?> +<svg id="svg5" width="68.5mm" height="15.79mm" version="1.1" viewBox="0 0 68.5 15.79" xmlns="http://www.w3.org/2000/svg"><g id="layer1" transform="translate(-1.4688 -18.46)" fill="none" stroke-linecap="square"><path id="path3041" d="m67.469 21.167h-10.583" stroke="#282828"/><path id="path3043" d="m62.177 21.445v10.305" stroke="#282828" stroke-width=".98677"/><path id="path3572" d="m3.9688 21.167v6.6146l3.9687 3.9688h2.6458l3.9688-3.9688v-6.6146" stroke="#e00"/><path id="path3687" d="m27.781 21.167h-6.6146l-3.9688 3.9688v2.6458l3.9688 3.9688h6.6146" stroke="#282828"/><path id="path3802" d="m30.427 31.75v-5.2917l5.2917-5.2917 5.2917 5.2917v5.2917" stroke="#282828"/><path id="path3954" d="m54.24 21.167h-7.9375l-2.6458 2.6458 2.6458 2.6458h5.2917l2.6458 2.6458-2.6458 2.6458h-7.9375" stroke="#282828"/></g></svg> diff --git a/ucast/static/ucast/logo_dark.svg b/ucast/static/ucast/logo_dark.svg new file mode 100644 index 0000000..25c7b5d --- /dev/null +++ b/ucast/static/ucast/logo_dark.svg @@ -0,0 +1,2 @@ +<?xml version="1.0" encoding="UTF-8"?> +<svg id="svg5" width="68.5mm" height="15.79mm" version="1.1" viewBox="0 0 68.5 15.79" xmlns="http://www.w3.org/2000/svg"><g id="layer1" transform="translate(-1.4688 -18.46)" fill="none" stroke-linecap="square"><path id="path3041" d="m67.469 21.167h-10.583" stroke="#fff"/><path id="path3043" d="m62.177 21.445v10.305" stroke="#fff" stroke-width=".98677"/><path id="path3572" d="m3.9688 21.167v6.6146l3.9687 3.9688h2.6458l3.9688-3.9688v-6.6146" stroke="#e00"/><path id="path3687" d="m27.781 21.167h-6.6146l-3.9688 3.9688v2.6458l3.9688 3.9688h6.6146" stroke="#fff"/><path id="path3802" d="m30.427 31.75v-5.2917l5.2917-5.2917 5.2917 5.2917v5.2917" stroke="#fff"/><path id="path3954" d="m54.24 21.167h-7.9375l-2.6458 2.6458 2.6458 2.6458h5.2917l2.6458 2.6458-2.6458 2.6458h-7.9375" stroke="#fff"/></g></svg> diff --git a/ucast/tasks/download.py b/ucast/tasks/download.py index a26a0c2..6b06bd5 100644 --- a/ucast/tasks/download.py +++ b/ucast/tasks/download.py @@ -7,29 +7,37 @@ from ucast.models import Channel, Video from ucast.service import cover, storage, util, videoutil, youtube -def _get_or_create_channel(channel_id: str) -> Channel: +def _get_or_create_channel(channel_str: str) -> Channel: + if youtube.CHANID_REGEX.match(channel_str): + try: + return Channel.objects.get(channel_id=channel_str) + except Channel.DoesNotExist: + pass + + channel_url = youtube.channel_url_from_str(channel_str) + channel_data = youtube.get_channel_metadata(channel_url) + try: - return Channel.objects.get(channel_id=channel_id) + return Channel.objects.get(channel_id=channel_data.id) except Channel.DoesNotExist: - channel_data = youtube.get_channel_metadata( - youtube.channel_url_from_id(channel_id) - ) - channel_slug = Channel.get_new_slug(channel_data.name) - store = storage.Storage() - channel_folder = store.get_or_create_channel_folder(channel_slug) + pass - util.download_image_file(channel_data.avatar_url, channel_folder.file_avatar) - util.resize_avatar(channel_folder.file_avatar, channel_folder.file_avatar_sm) + channel_slug = Channel.get_new_slug(channel_data.name) + store = storage.Storage() + channel_folder = store.get_or_create_channel_folder(channel_slug) - channel = Channel( - channel_id=channel_id, - name=channel_data.name, - slug=channel_slug, - description=channel_data.description, - subscribers=channel_data.subscribers, - ) - channel.save() - return channel + util.download_image_file(channel_data.avatar_url, channel_folder.file_avatar) + util.resize_avatar(channel_folder.file_avatar, channel_folder.file_avatar_sm) + + channel = Channel( + channel_id=channel_data.id, + name=channel_data.name, + slug=channel_slug, + description=channel_data.description, + subscribers=channel_data.subscribers, + ) + channel.save() + return channel def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): @@ -38,12 +46,8 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): details = youtube.get_video_details(vid.id) - # Check filter - if ( - details.is_currently_live - or (details.is_short and channel.skip_shorts) - or (details.is_livestream and channel.skip_livestreams) - ): + # Dont load active livestreams + if details.is_currently_live: return slug = Video.get_new_slug( @@ -63,7 +67,8 @@ def _load_scraped_video(vid: youtube.VideoScraped, channel: Channel): ) video.save() - queue.enqueue(download_video, video) + if channel.should_download(video): + queue.enqueue(download_video, video) def download_video(video: Video): @@ -107,21 +112,19 @@ def download_video(video: Video): video.save() -def import_channel(channel_id: str, limit: int = None): +def import_channel(channel_str: str, limit: int = None): """ Add a new channel to ucast and download all existing videos. - :param channel_id: YT-Channel-ID + :param channel_str: YT-Channel-ID / URL :param limit: Maximum number of videos to download """ - channel = _get_or_create_channel(channel_id) + channel = _get_or_create_channel(channel_str) if limit == 0: return - videos = youtube.get_channel_videos_from_scraper(channel_id, limit) - - for vid in videos[:limit]: + for vid in youtube.get_channel_videos_from_scraper(channel.channel_id, limit): _load_scraped_video(vid, channel) diff --git a/ucast/tasks/schedule.py b/ucast/tasks/schedule.py index 35d41d5..f56b86b 100644 --- a/ucast/tasks/schedule.py +++ b/ucast/tasks/schedule.py @@ -4,7 +4,7 @@ from datetime import datetime from django.conf import settings from ucast import queue -from ucast.tasks import download +from ucast.tasks import download, library log = logging.getLogger(__name__) @@ -26,3 +26,10 @@ def register_scheduled_jobs(): id="schedule_update_channels", interval=settings.YT_UPDATE_INTERVAL, ) + + scheduler.schedule( + datetime.utcnow(), + library.update_channel_infos, + id="schedule_update_channel_infos", + interval=24 * 3600, + ) diff --git a/ucast/templates/bulma/base.html b/ucast/templates/bulma/base.html new file mode 100644 index 0000000..e43b0dd --- /dev/null +++ b/ucast/templates/bulma/base.html @@ -0,0 +1,77 @@ +{% load static bulma_tags %} +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>{% block title %}{% endblock title %} + {% block css %} + + + {% block extra_css %}{% endblock extra_css %} + {% endblock css %} + + + +{% block header %} +
+ +
+{% endblock header %} + +{% block hero %}{% endblock hero %} + +
+
+ {% block messages %} + {% if messages %} +
+
+ {% for message in messages %} +
+
{{ message }}
+
+ {% endfor %} +
+
+ {% endif %} + {% endblock messages %} + + {% block content_area %} + {% block content_title %}{% endblock content_title %} + {% block content %}{% endblock content %} + {% endblock content_area %} +
+
+ +{% block modal %}{% endblock modal %} + +{% block footer %} +{% endblock footer %} + +{% block javascript %} + {% block extra_javascript %}{% endblock extra_javascript %} +{% endblock javascript %} + + diff --git a/ucast/templates/ucast/channels.html b/ucast/templates/ucast/channels.html new file mode 100644 index 0000000..5431794 --- /dev/null +++ b/ucast/templates/ucast/channels.html @@ -0,0 +1,48 @@ +{% extends 'base.html' %} + +{% block title %}ucast - Channels{% endblock %} + +{% block content %} +

Channels

+ +
+
+ {% csrf_token %} +
+
+ +
+
+ +
+
+
+
+ + {% for channel in channels %} +
+
+ + + +
+
+ {{ channel.name }} +
+
+ +
+
+ +
+
+
+
+ {% endfor %} +{% endblock content %} diff --git a/ucast/templates/ucast/main.html b/ucast/templates/ucast/main.html deleted file mode 100644 index aa7cb3c..0000000 --- a/ucast/templates/ucast/main.html +++ /dev/null @@ -1,11 +0,0 @@ - - - - - Ucast - - -

Ucast

- Hello World! - - \ No newline at end of file diff --git a/ucast/templates/ucast/videos.html b/ucast/templates/ucast/videos.html new file mode 100644 index 0000000..537d25b --- /dev/null +++ b/ucast/templates/ucast/videos.html @@ -0,0 +1,94 @@ +{% extends 'base.html' %} + +{% block title %}ucast - Videos{% endblock %} + +{% block content %} +
+

{{ channel.name }}

+ +
+   {{ channel.subscribers }} +   {{ channel.video_set.count }} +   {{ channel.download_size|filesizeformat }} +   {{ channel.channel_id }} +
+ +
+
+ + + +
+
+ +
+
+ +
+
+ +
+
+
+ +
+
+ + {% for video in videos %} +
+
+ + + +
+
+ {{ video.title }} + +
+   {{ video.published|date }} +   {{ video.download_size|filesizeformat }} +   {{ video.video_id }} +
+ +
+
+ {% if video.downloaded %} +
+ + + +
+
+ +
+ {% else %} +
+ +
+ {% endif %} +
+
+ {% endfor %} +{% endblock content %} diff --git a/ucast/tests/__init__.py b/ucast/tests/__init__.py index 62fd2f6..bb7d9a4 100644 --- a/ucast/tests/__init__.py +++ b/ucast/tests/__init__.py @@ -11,7 +11,7 @@ DIR_TESTFILES = resources.path("ucast.tests", "_testfiles") def get_video_details(video_id: str): - with open(DIR_TESTFILES / "fixture" / "videodetails.json") as f: + with open(DIR_TESTFILES / "object" / "videodetails.json") as f: videodetails = json.load(f) vd_raw = videodetails[video_id] @@ -21,7 +21,7 @@ def get_video_details(video_id: str): def get_channel_metadata(channel_url: str): - with open(DIR_TESTFILES / "fixture" / "channelmeta.json") as f: + with open(DIR_TESTFILES / "object" / "channelmeta.json") as f: channelmeta = json.load(f) return youtube.ChannelMetadata(**channelmeta[channel_url]) diff --git a/ucast/tests/_testfiles/fixture/models.json b/ucast/tests/_testfiles/fixture/models.json index 4ac568f..d437539 100644 --- a/ucast/tests/_testfiles/fixture/models.json +++ b/ucast/tests/_testfiles/fixture/models.json @@ -44,7 +44,6 @@ "avatar_url": "https://yt3.ggpht.com/-ybcsEHc8YCmKUZMr2bf4DZoDv7SKrutgKIh8kSxXugj296QkqtBZQXVzpuZ1Izs8kNUz35B=s900-c-k-c0x00ffffff-no-rj" } }, - { "model": "ucast.video", "pk": 1, @@ -146,5 +145,24 @@ "is_short": false, "download_size": null } + }, + { + "model": "ucast.user", + "pk": 1, + "fields": { + "password": "pbkdf2_sha256$320000$2XXzT2OZlzSOnB1n7NCgAB$9zBJXdGbJv9YnS+kP5RUMkGxeIuqAbDRBBzXlmPJizw=", + "last_login": "2022-05-29T21:08:21.383Z", + "is_superuser": true, + "username": "admin", + "first_name": "", + "last_name": "", + "email": "admin@example.com", + "is_staff": true, + "is_active": true, + "date_joined": "2022-05-29T21:05:24.014Z", + "feed_key": null, + "groups": [], + "user_permissions": [] + } } ] diff --git a/ucast/tests/_testfiles/fixture/channelmeta.json b/ucast/tests/_testfiles/object/channelmeta.json similarity index 100% rename from ucast/tests/_testfiles/fixture/channelmeta.json rename to ucast/tests/_testfiles/object/channelmeta.json diff --git a/ucast/tests/_testfiles/fixture/videodetails.json b/ucast/tests/_testfiles/object/videodetails.json similarity index 100% rename from ucast/tests/_testfiles/fixture/videodetails.json rename to ucast/tests/_testfiles/object/videodetails.json diff --git a/ucast/tests/conftest.py b/ucast/tests/conftest.py index 04e7fb0..90c2154 100644 --- a/ucast/tests/conftest.py +++ b/ucast/tests/conftest.py @@ -14,6 +14,7 @@ from django.core.management import call_command from fakeredis import FakeRedis from ucast import queue, tests +from ucast.models import User from ucast.service import cover, storage, util, videoutil, youtube @@ -52,20 +53,10 @@ def _create_download_dir() -> Tuple[Path, TemporaryDirectory]: return tmpdir, tmpdir_o -@pytest.fixture -def download_dir() -> Path: - tmpdir, tmpdir_o = _create_download_dir() - yield tmpdir - - -@pytest.fixture(scope="session") -def download_dir_content() -> Path: - tmpdir, tmpdir_o = _create_download_dir() - settings.DOWNLOAD_ROOT = tmpdir - +def _add_download_dir_content(): store = storage.Storage() - with open(tests.DIR_TESTFILES / "fixture" / "videodetails.json") as f: + with open(tests.DIR_TESTFILES / "object" / "videodetails.json") as f: videodetails = json.load(f) for vid in ("ZPxEr4YdWt8", "_I5IFObm_-k", "mmEDPbbSnaY", "Cda4zS-1j-k"): @@ -102,6 +93,26 @@ def download_dir_content() -> Path: file_cover, ) + +@pytest.fixture +def download_dir() -> Path: + tmpdir, tmpdir_o = _create_download_dir() + yield tmpdir + + +@pytest.fixture(scope="session") +def download_dir_content() -> Path: + tmpdir, tmpdir_o = _create_download_dir() + settings.DOWNLOAD_ROOT = tmpdir + _add_download_dir_content() + yield tmpdir + + +@pytest.fixture +def download_dir_content_mut() -> Path: + tmpdir, tmpdir_o = _create_download_dir() + settings.DOWNLOAD_ROOT = tmpdir + _add_download_dir_content() yield tmpdir @@ -138,3 +149,8 @@ def mock_get_channel_metadata(mocker) -> mock.Mock: channel_meta_mock: mock.Mock = mocker.patch.object(youtube, "get_channel_metadata") channel_meta_mock.side_effect = tests.get_channel_metadata return channel_meta_mock + + +@pytest.fixture +def feed_key(db) -> str: + return User.objects.get(id=1).get_feed_key() diff --git a/ucast/tests/service/test_youtube.py b/ucast/tests/service/test_youtube.py index c93a476..c57c2c7 100644 --- a/ucast/tests/service/test_youtube.py +++ b/ucast/tests/service/test_youtube.py @@ -169,6 +169,6 @@ def test_get_channel_videos_from_scraper(): videos = youtube.get_channel_videos_from_scraper(CHANNEL_ID_THETADEV) assert videos - v1 = videos[0] + v1 = videos.__next__() assert len(v1.id) == 11 assert v1.published is None diff --git a/ucast/tests/tasks/test_library.py b/ucast/tests/tasks/test_library.py index e36c0d4..ddbc605 100644 --- a/ucast/tests/tasks/test_library.py +++ b/ucast/tests/tasks/test_library.py @@ -11,7 +11,7 @@ CHANNEL_ID_THETADEV = "UCGiJh0NZ52wRhYKYnuZI08Q" @pytest.mark.django_db -def test_recreate_cover(download_dir_content, rq_queue, mocker): +def test_recreate_cover(download_dir_content_mut, rq_queue, mocker): create_cover_mock: mock.Mock = mocker.patch.object(cover, "create_cover_file") video = Video.objects.get(video_id="ZPxEr4YdWt8") diff --git a/ucast/tests/test_feed.py b/ucast/tests/test_feed.py index 0be7895..22bfcf1 100644 --- a/ucast/tests/test_feed.py +++ b/ucast/tests/test_feed.py @@ -8,9 +8,9 @@ from ucast.models import Video @pytest.mark.django_db -def test_feed(): +def test_feed(feed_key): client = Client() - response = client.get("/feed/ThetaDev") + response = client.get(f"/feed/ThetaDev?key={feed_key}") parsed_feed = feedparser.parse(response.getvalue()) assert parsed_feed["bozo"] is False @@ -28,7 +28,10 @@ def test_feed(): == "I'm ThetaDev. I love creating cool projects using electronics, 3D printers and other awesome tech-based stuff." ) assert meta["updated"] == "Sun, 15 May 2022 22:16:25 +0000" - assert meta["image"]["href"] == "http://testserver/files/avatar/ThetaDev.jpg" + assert ( + meta["image"]["href"] + == f"http://testserver/files/avatar/ThetaDev.jpg?key={feed_key}" + ) assert len(parsed_feed["entries"]) == 4 @@ -45,7 +48,7 @@ def test_feed(): assert entry["id"] == video.get_absolute_url() assert ( entry["image"]["href"] - == f"http://testserver/files/cover/ThetaDev/{video.slug}.png" + == f"http://testserver/files/cover/ThetaDev/{video.slug}.png?key={feed_key}" ) assert entry["itunes_duration"] == feed.PodcastFeedType._format_secs( video.duration diff --git a/ucast/tests/test_views.py b/ucast/tests/test_views.py index a9b462d..04d9335 100644 --- a/ucast/tests/test_views.py +++ b/ucast/tests/test_views.py @@ -2,6 +2,8 @@ import pytest from django.conf import settings from django.test.client import Client +from ucast.service import util + @pytest.mark.parametrize( "url_path,internal_path", @@ -33,12 +35,13 @@ from django.test.client import Client ], ) def test_files_internal_redirect( - url_path: str, internal_path: str, download_dir_content + url_path: str, internal_path: str, download_dir_content, feed_key ): settings.INTERNAL_REDIRECT_HEADER = "X-Accel-Redirect" client = Client() - response = client.get("/files/" + url_path) + url = util.add_key_to_url("/files/" + url_path, feed_key) + response = client.get(url) assert response.getvalue() == b"" assert response.headers.get("X-Accel-Redirect") == internal_path assert "Content-Type" not in response.headers @@ -73,7 +76,7 @@ def test_files_internal_redirect( ), ], ) -def test_files_response(url_path: str, file_path: str, download_dir_content): +def test_files_response(url_path: str, file_path: str, download_dir_content, feed_key): # store = storage.Storage() # cf = store.get_channel_folder("ThetaDev") # audio_file = cf.get_audio("20190602_ThetaDev_Embedded_World_2019") @@ -83,7 +86,8 @@ def test_files_response(url_path: str, file_path: str, download_dir_content): response_file = settings.DOWNLOAD_ROOT / file_path client = Client() - response = client.get("/files/" + url_path) + url = util.add_key_to_url("/files/" + url_path, feed_key) + response = client.get(url) response_bts = response.getvalue() with open(response_file, "rb") as f: diff --git a/ucast/urls.py b/ucast/urls.py index 6d904c0..bd11551 100644 --- a/ucast/urls.py +++ b/ucast/urls.py @@ -4,6 +4,7 @@ from ucast import views urlpatterns = [ path("", views.home), + path("channel/", views.videos), path("feed/", views.podcast_feed), path("files/audio//", views.audio), path("files/cover//", views.cover), diff --git a/ucast/views.py b/ucast/views.py index 126f6b7..07051e5 100644 --- a/ucast/views.py +++ b/ucast/views.py @@ -1,19 +1,58 @@ import os +from functools import wraps from pathlib import Path from typing import Callable from django import http from django.conf import settings +from django.contrib.auth.decorators import login_required +from django.contrib.sites.shortcuts import get_current_site +from django.contrib.syndication.views import add_domain from django.middleware.http import ConditionalGetMiddleware from django.shortcuts import render from django.utils.decorators import decorator_from_middleware -from ucast import feed +from ucast import feed, forms, queue +from ucast.models import Channel, User, Video from ucast.service import storage +from ucast.tasks import download +@login_required def home(request: http.HttpRequest): - return render(request, "ucast/main.html") + channels = Channel.objects.all() + + site_url = add_domain(get_current_site(request).domain, "", request.is_secure()) + + if request.method == "POST": + form = forms.AddChannelForm(request.POST) + if form.is_valid(): + channel_str = form.cleaned_data["channel_str"] + queue.enqueue(download.import_channel, channel_str) + + return render( + request, + "ucast/channels.html", + { + "channels": channels, + "site_url": site_url, + "add_channel_form": forms.AddChannelForm, + }, + ) + + +@login_required +def videos(request: http.HttpRequest, channel: str): + chan = Channel.objects.get(slug=channel) + vids = Video.objects.filter(channel=chan).order_by("-published") + + site_url = add_domain(get_current_site(request).domain, "", request.is_secure()) + + return render( + request, + "ucast/videos.html", + {"videos": vids, "channel": chan, "site_url": site_url}, + ) def _channel_file(channel: str, get_file: Callable[[storage.ChannelFolder], Path]): @@ -41,6 +80,28 @@ def _channel_file(channel: str, get_file: Callable[[storage.ChannelFolder], Path return response +def login_or_key_required(function): + def decorator(view_func): + @wraps(view_func) + def _wrapped_view(request, *args, **kwargs): + key = request.GET.get("key") + if key: + try: + request.user = User.objects.get(feed_key=key) + except User.DoesNotExist: + pass + + if request.user.is_authenticated: + return view_func(request, *args, **kwargs) + + return http.HttpResponse("401 Unauthorized", status=401) + + return _wrapped_view + + return decorator(function) + + +@login_or_key_required def audio(request: http.HttpRequest, channel: str, video: str): # Trim off file extension video_slug = video.rsplit(".")[0] @@ -48,6 +109,7 @@ def audio(request: http.HttpRequest, channel: str, video: str): return _channel_file(channel, lambda cf: cf.get_audio(video_slug)) +@login_or_key_required def cover(request: http.HttpRequest, channel: str, video: str): # Trim off file extension video_slug = video.rsplit(".")[0] @@ -55,6 +117,7 @@ def cover(request: http.HttpRequest, channel: str, video: str): return _channel_file(channel, lambda cf: cf.get_cover(video_slug)) +@login_or_key_required def thumbnail(request: http.HttpRequest, channel: str, video: str): # Trim off file extension video_slug = video.rsplit(".")[0] @@ -64,6 +127,7 @@ def thumbnail(request: http.HttpRequest, channel: str, video: str): return _channel_file(channel, lambda cf: cf.get_thumbnail(video_slug, is_sm)) +@login_or_key_required def avatar(request: http.HttpRequest, channel: str): # Trim off file extension channel_slug = channel.rsplit(".")[0] @@ -75,6 +139,7 @@ def avatar(request: http.HttpRequest, channel: str): return _channel_file(channel_slug, lambda cf: cf.file_avatar) +@login_or_key_required @decorator_from_middleware(ConditionalGetMiddleware) def podcast_feed(request: http.HttpRequest, *args, **kwargs): return feed.UcastFeed()(request, *args, **kwargs) diff --git a/ucast_project/settings.py b/ucast_project/settings.py index d3b20e6..04feb01 100644 --- a/ucast_project/settings.py +++ b/ucast_project/settings.py @@ -183,6 +183,11 @@ AUTH_PASSWORD_VALIDATORS = [ }, ] +AUTH_USER_MODEL = "ucast.user" + +LOGIN_REDIRECT_URL = "/" +LOGOUT_REDIRECT_URL = "/" + # Internationalization # https://docs.djangoproject.com/en/4.0/topics/i18n/ diff --git a/ucast_project/urls.py b/ucast_project/urls.py index 52ea2a4..42cb61d 100644 --- a/ucast_project/urls.py +++ b/ucast_project/urls.py @@ -18,5 +18,6 @@ from django.urls import include, path urlpatterns = [ path("admin/", admin.site.urls), + path("accounts/", include("django.contrib.auth.urls")), path("", include("ucast.urls")), ] From 16a509ee2023606ec62b2a76cd0e66e74b8ba515 Mon Sep 17 00:00:00 2001 From: Theta-Dev Date: Thu, 2 Jun 2022 01:58:28 +0200 Subject: [PATCH 5/5] add view tests if logged in --- ucast/tests/conftest.py | 9 ++++++-- ucast/tests/service/test_util.py | 20 ++++++++++++++++++ ucast/tests/test_views.py | 36 ++++++++++++++++++++------------ 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/ucast/tests/conftest.py b/ucast/tests/conftest.py index 90c2154..841ae6d 100644 --- a/ucast/tests/conftest.py +++ b/ucast/tests/conftest.py @@ -152,5 +152,10 @@ def mock_get_channel_metadata(mocker) -> mock.Mock: @pytest.fixture -def feed_key(db) -> str: - return User.objects.get(id=1).get_feed_key() +def user_admin(db): + return User.objects.get(id=1) + + +@pytest.fixture +def feed_key(user_admin) -> str: + return user_admin.get_feed_key() diff --git a/ucast/tests/service/test_util.py b/ucast/tests/service/test_util.py index f095804..8d98db7 100644 --- a/ucast/tests/service/test_util.py +++ b/ucast/tests/service/test_util.py @@ -90,3 +90,23 @@ def test_resize_thumbnail(): def test_slug(text: str, expected_slug: str): slug = util.get_slug(text) assert slug == expected_slug + + +@pytest.mark.parametrize( + "url,expect", + [ + ("/files/audio/My_Video.mp3", "/files/audio/My_Video.mp3?key=my-key"), + ( + "https://example.com/files/audio/My_Video.mp3", + "https://example.com/files/audio/My_Video.mp3?key=my-key", + ), + ("/files/avatar/ThetaDev.webp?sm", "/files/avatar/ThetaDev.webp?sm&key=my-key"), + ( + "https://example.com/files/avatar/ThetaDev.webp?sm", + "https://example.com/files/avatar/ThetaDev.webp?sm&key=my-key", + ), + ], +) +def test_add_key_to_url(url: str, expect: str): + url_w_key = util.add_key_to_url(url, "my-key") + assert url_w_key == expect diff --git a/ucast/tests/test_views.py b/ucast/tests/test_views.py index 04d9335..7c6b27d 100644 --- a/ucast/tests/test_views.py +++ b/ucast/tests/test_views.py @@ -35,16 +35,24 @@ from ucast.service import util ], ) def test_files_internal_redirect( - url_path: str, internal_path: str, download_dir_content, feed_key + url_path: str, internal_path: str, download_dir_content, feed_key, admin_client ): settings.INTERNAL_REDIRECT_HEADER = "X-Accel-Redirect" + def check_response(resp): + assert resp.getvalue() == b"" + assert resp.headers.get("X-Accel-Redirect") == internal_path + assert "Content-Type" not in resp.headers + + # Access with key client = Client() url = util.add_key_to_url("/files/" + url_path, feed_key) response = client.get(url) - assert response.getvalue() == b"" - assert response.headers.get("X-Accel-Redirect") == internal_path - assert "Content-Type" not in response.headers + check_response(response) + + # Access with login + response = admin_client.get("/files/" + url_path) + check_response(response) @pytest.mark.parametrize( @@ -76,21 +84,23 @@ def test_files_internal_redirect( ), ], ) -def test_files_response(url_path: str, file_path: str, download_dir_content, feed_key): - # store = storage.Storage() - # cf = store.get_channel_folder("ThetaDev") - # audio_file = cf.get_audio("20190602_ThetaDev_Embedded_World_2019") - +def test_files_response( + url_path: str, file_path: str, download_dir_content, feed_key, admin_client +): settings.INTERNAL_REDIRECT_HEADER = "" response_file = settings.DOWNLOAD_ROOT / file_path + with open(response_file, "rb") as f: + file_bts = f.read() + # Access with key client = Client() url = util.add_key_to_url("/files/" + url_path, feed_key) response = client.get(url) response_bts = response.getvalue() - - with open(response_file, "rb") as f: - file_bts = f.read() - + assert response_bts == file_bts + + # Access with login + response = admin_client.get("/files/" + url_path) + response_bts = response.getvalue() assert response_bts == file_bts