diff --git a/poetry.lock b/poetry.lock index 1aeba43..9d928f5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -60,11 +60,11 @@ cffi = ">=1.0.0" [[package]] name = "certifi" -version = "2022.5.18" +version = "2022.5.18.1" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" [[package]] name = "cffi" @@ -220,7 +220,7 @@ testing = ["mock (>=2.0.0)"] [[package]] name = "feedparser" -version = "6.0.8" +version = "6.0.9" description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds" category = "main" optional = false @@ -273,7 +273,7 @@ export = ["jinja2 (>=2.7,<3)"] [[package]] name = "identify" -version = "2.5.0" +version = "2.5.1" description = "File identification library for Python" category = "dev" optional = false @@ -845,8 +845,8 @@ brotlicffi = [ {file = "brotlicffi-1.0.9.2.tar.gz", hash = "sha256:0c248a68129d8fc6a217767406c731e498c3e19a7be05ea0a90c3c86637b7d96"}, ] certifi = [ - {file = "certifi-2022.5.18-py3-none-any.whl", hash = "sha256:8d15a5a7fde18536a249c49e07e8e462b8fc13de21b3c80e8a68315dfa227c99"}, - {file = "certifi-2022.5.18.tar.gz", hash = "sha256:6ae10321df3e464305a46e997da41ea56c1d311fb9ff1dd4e04d6f14653ec63a"}, + {file = "certifi-2022.5.18.1-py3-none-any.whl", hash = "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a"}, + {file = "certifi-2022.5.18.1.tar.gz", hash = "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7"}, ] cffi = [ {file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"}, @@ -988,8 +988,8 @@ django-rq = [ {file = "django_rq-2.5.1-py2.py3-none-any.whl", hash = "sha256:7be1e10e7091555f9f36edf100b0dbb205ea2b98683d74443d2bdf3c6649a03f"}, ] feedparser = [ - {file = "feedparser-6.0.8-py3-none-any.whl", hash = "sha256:1b7f57841d9cf85074deb316ed2c795091a238adb79846bc46dccdaf80f9c59a"}, - {file = "feedparser-6.0.8.tar.gz", hash = "sha256:5ce0410a05ab248c8c7cfca3a0ea2203968ee9ff4486067379af4827a59f9661"}, + {file = "feedparser-6.0.9-py3-none-any.whl", hash = "sha256:a522b2b81f3914a74ae44161a341940f74811bd29be5b4c2a689e6e6be51cd39"}, + {file = "feedparser-6.0.9.tar.gz", hash = "sha256:dad42e7beaec55f99c08b2b0cf7288bc7cfd24b6f72c8ef85478bcb55648cd42"}, ] filelock = [ {file = "filelock-3.7.0-py3-none-any.whl", hash = "sha256:c7b5fdb219b398a5b28c8e4c1893ef5f98ece6a38c6ab2c22e26ec161556fed6"}, @@ -1010,8 +1010,8 @@ honcho = [ {file = "honcho-1.1.0.tar.gz", hash = "sha256:c5eca0bded4bef6697a23aec0422fd4f6508ea3581979a3485fc4b89357eb2a9"}, ] identify = [ - {file = "identify-2.5.0-py2.py3-none-any.whl", hash = "sha256:3acfe15a96e4272b4ec5662ee3e231ceba976ef63fd9980ed2ce9cc415df393f"}, - {file = "identify-2.5.0.tar.gz", hash = "sha256:c83af514ea50bf2be2c4a3f2fb349442b59dc87284558ae9ff54191bff3541d2"}, + {file = "identify-2.5.1-py2.py3-none-any.whl", hash = "sha256:0dca2ea3e4381c435ef9c33ba100a78a9b40c0bab11189c7cf121f75815efeaa"}, + {file = "identify-2.5.1.tar.gz", hash = "sha256:3d11b16f3fe19f52039fb7e39c9c884b21cb1b586988114fbe42671f03de3e82"}, ] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, diff --git a/ucast/__init__.py b/ucast/__init__.py index e69de29..024a442 100644 --- a/ucast/__init__.py +++ b/ucast/__init__.py @@ -0,0 +1,3 @@ +__version__ = "0.0.1" + +default_app_config = "ucast.apps.UcastConfig" diff --git a/ucast/apps.py b/ucast/apps.py index ad4b110..41af053 100644 --- a/ucast/apps.py +++ b/ucast/apps.py @@ -4,3 +4,8 @@ from django.apps import AppConfig class UcastConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "ucast" + + def ready(self): + from ucast.tasks import download + + download.schedule_update_channels() diff --git a/ucast/service/cover.py b/ucast/service/cover.py index 670268c..a50ef90 100644 --- a/ucast/service/cover.py +++ b/ucast/service/cover.py @@ -1,4 +1,5 @@ import math +import random from importlib import resources from pathlib import Path from typing import List, Literal, Optional, Tuple @@ -6,7 +7,7 @@ from typing import List, Literal, Optional, Tuple import wcag_contrast_ratio from colorthief import ColorThief from fonts.ttf import SourceSansPro -from PIL import Image, ImageDraw, ImageFilter, ImageFont +from PIL import Image, ImageDraw, ImageEnhance, ImageFilter, ImageFont from ucast.service import typ @@ -16,6 +17,7 @@ CoverStyle = Literal["blur", "gradient"] CHAR_ELLIPSIS = "…" COVER_WIDTH = 500 +MIN_CONTRAST = 4.5 def _split_text( @@ -30,7 +32,7 @@ def _split_text( :param text: Input text :param font: Pillow ImageFont :param line_spacing: Line spacing [px] - :return: + :return: List of lines """ if height < font.size: return [] @@ -99,7 +101,6 @@ def _draw_text_box( :param color: Text color :param line_spacing: Line spacing [px] :param vertical_center: Center text vertically in the box - :return: """ x_tl, y_tl, x_br, y_br = box height = y_br - y_tl @@ -144,7 +145,11 @@ def _interpolate_color(color_from: typ.Color, color_to: typ.Color, steps: int): yield [round(f + det * i) for f, det in zip(color_from, det_co)] -def _get_text_color(bg_color) -> typ.Color: +def _color_to_float(color: typ.Color) -> tuple[float, ...]: + return tuple(c / 255 for c in color) + + +def _get_text_color(bg_color: typ.Color) -> typ.Color: """ Return the text color (black or white) with the largest contrast to a given background color. @@ -152,26 +157,19 @@ def _get_text_color(bg_color) -> typ.Color: :param bg_color: Background color :return: Text color """ - color_decimal = tuple([c / 255 for c in bg_color]) - c_blk = wcag_contrast_ratio.rgb((0, 0, 0), color_decimal) - c_wht = wcag_contrast_ratio.rgb((1, 1, 1), color_decimal) + color_float = _color_to_float(bg_color) + c_blk = wcag_contrast_ratio.rgb((0, 0, 0), color_float) + c_wht = wcag_contrast_ratio.rgb((1, 1, 1), color_float) if c_wht > c_blk: return 255, 255, 255 return 0, 0, 0 -def _get_baseimage( - thumbnail: Image.Image, - top_color: typ.Color, - bottom_color: typ.Color, - style: CoverStyle, -): +def _get_baseimage(thumbnail: Image.Image, style: CoverStyle): """ Return the background image for the cover. :param thumbnail: Thumbnail image object - :param top_color: Top color of the thumbnail image - :param bottom_color: Bottom color of the thumbnail image :param style: Style of the cover image :return: Base image """ @@ -179,6 +177,15 @@ def _get_baseimage( if style == COVER_STYLE_GRADIENT: # Thumbnail with color gradient background + + # Get dominant colors from the top and bottom 20% of the thumbnail image + top_part = thumbnail.crop((0, 0, COVER_WIDTH, int(thumbnail.height * 0.2))) + bottom_part = thumbnail.crop( + (0, int(thumbnail.height * 0.8), COVER_WIDTH, thumbnail.height) + ) + top_color = _get_dominant_color(top_part) + bottom_color = _get_dominant_color(bottom_part) + cover_draw = ImageDraw.Draw(cover) for i, color in enumerate( @@ -217,27 +224,139 @@ def _resize_thumbnail(thumbnail: Image.Image) -> Image.Image: ) +def _prepare_text_background( + base_img: Image.Image, bboxes: List[Tuple[int, int, int, int]] +) -> Tuple[Image.Image, typ.Color]: + """ + Return the preferred text color (black or white) and darken + the image if necessary + + :param base_img: Image object + :param bboxes: Text boxes + :return: Updated image, text color + """ + rng = random.Random() + rng.seed(0x9B38D30461B7F0E6) + + min_contrast_bk = 22 + min_contrast_wt = 22 + worst_color_wt = None + + def corr_x(x: int) -> int: + return min(max(0, x), base_img.width) + + def corr_y(y: int) -> int: + return min(max(0, y), base_img.height) + + for bbox in bboxes: + x_tl, y_tl, x_br, y_br = bbox + x_tl = corr_x(x_tl) + y_tl = corr_y(y_tl) + x_br = corr_x(x_br) + y_br = corr_y(y_br) + + height = y_br - y_tl + width = x_br - x_tl + + for _ in range(math.ceil(width * height * 0.01)): + target_pos = (rng.randint(x_tl, x_br - 1), rng.randint(y_tl, y_br - 1)) + img_color = base_img.getpixel(target_pos) + img_color_float = _color_to_float(img_color) + + ct_bk = wcag_contrast_ratio.rgb((0, 0, 0), img_color_float) + ct_wt = wcag_contrast_ratio.rgb((1, 1, 1), img_color_float) + + if ct_bk < min_contrast_bk: + min_contrast_bk = ct_bk + + if ct_wt < min_contrast_wt: + worst_color_wt = img_color + min_contrast_wt = ct_wt + + if min_contrast_bk >= MIN_CONTRAST: + return base_img, (0, 0, 0) + if min_contrast_wt >= MIN_CONTRAST: + return base_img, (255, 255, 255) + + pixel = Image.new("RGB", (1, 1), worst_color_wt) + + for i in range(1, 100): + brightness_f = 1 - i / 100 + contrast_f = 1 - i / 1000 + + pixel_c = ImageEnhance.Brightness(pixel).enhance(brightness_f) + pixel_c = ImageEnhance.Contrast(pixel_c).enhance(contrast_f) + new_color = pixel_c.getpixel((0, 0)) + + if ( + wcag_contrast_ratio.rgb((1, 1, 1), _color_to_float(new_color)) + >= MIN_CONTRAST + ): + new_img = ImageEnhance.Brightness(base_img).enhance(brightness_f) + new_img = ImageEnhance.Contrast(new_img).enhance(contrast_f) + return new_img, (255, 255, 255) + + return base_img, (255, 255, 255) + + def _draw_text_avatar( cover: Image.Image, avatar: Optional[Image.Image], title: str, channel: str, - top_color: typ.Color, - bottom_color: typ.Color, -): - cover_draw = ImageDraw.Draw(cover) - +) -> Image.Image: # Add channel avatar avt_margin = 0 avt_size = 0 - tn_16_9_height = int(COVER_WIDTH / 16 * 9) - tn_16_9_margin = int((COVER_WIDTH - tn_16_9_height) / 2) + tn_16_9_height = int(COVER_WIDTH / 16 * 9) # typical: 281 + tn_16_9_margin = int((COVER_WIDTH - tn_16_9_height) / 2) # typical: 110 if avatar: - avt_margin = int(tn_16_9_margin * 0.05) - avt_size = tn_16_9_margin - 2 * avt_margin + avt_margin = int(tn_16_9_margin * 0.05) # typical: 14 + avt_size = tn_16_9_margin - 2 * avt_margin # typical: 82 + # Add text + text_margin_x = 16 + text_margin_topleft = avt_margin + avt_size + text_margin_x # typical: 112 + text_vertical_offset = -17 + text_line_space = -4 + + fnt = ImageFont.truetype(SourceSansPro, 50) + top_text_box = ( # typical: (112, -17, 484, 110) + text_margin_topleft, + text_vertical_offset, + COVER_WIDTH - text_margin_x, + tn_16_9_margin, + ) + bottom_text_box = ( # typical: (16, 373, 484, 500) + text_margin_x, + COVER_WIDTH - tn_16_9_margin + text_vertical_offset, + COVER_WIDTH - text_margin_x, + COVER_WIDTH, + ) + + cover, text_color = _prepare_text_background(cover, [top_text_box, bottom_text_box]) + cover_draw = ImageDraw.Draw(cover) + + _draw_text_box( + cover_draw, + top_text_box, + channel, + fnt, + text_color, + text_line_space, + ) + _draw_text_box( + cover_draw, + bottom_text_box, + title, + fnt, + text_color, + text_line_space, + ) + + if avatar: avt = avatar.resize((avt_size, avt_size), Image.LANCZOS) circle_mask = Image.new("L", (avt_size, avt_size)) @@ -246,42 +365,7 @@ def _draw_text_avatar( cover.paste(avt, (avt_margin, avt_margin), circle_mask) - # Add text - text_margin_x = 16 - text_margin_topleft = avt_margin + avt_size + text_margin_x - text_vertical_offset = -17 - text_line_space = -4 - - fnt = ImageFont.truetype(SourceSansPro, 50) - top_text_color = _get_text_color(top_color) - bottom_text_color = _get_text_color(bottom_color) - - _draw_text_box( - cover_draw, - ( - text_margin_topleft, - text_vertical_offset, - COVER_WIDTH - text_margin_x, - tn_16_9_margin, - ), - channel, - fnt, - top_text_color, - text_line_space, - ) - _draw_text_box( - cover_draw, - ( - text_margin_x, - COVER_WIDTH - tn_16_9_margin + text_vertical_offset, - COVER_WIDTH - text_margin_x, - COVER_WIDTH, - ), - title, - fnt, - bottom_text_color, - text_line_space, - ) + return cover def _create_cover_image( @@ -303,20 +387,14 @@ def _create_cover_image( """ tn = _resize_thumbnail(thumbnail) - # Get dominant colors from the top and bottom 20% of the thumbnail image - top_part = tn.crop((0, 0, COVER_WIDTH, int(tn.height * 0.2))) - bottom_part = tn.crop((0, int(tn.height * 0.8), COVER_WIDTH, tn.height)) - top_color = _get_dominant_color(top_part) - bottom_color = _get_dominant_color(bottom_part) + cover = _get_baseimage(tn, style) - cover = _get_baseimage(tn, top_color, bottom_color, style) + cover = _draw_text_avatar(cover, avatar, title, channel) # Insert thumbnail image in the middle tn_margin = int((COVER_WIDTH - tn.height) / 2) cover.paste(tn, (0, tn_margin)) - _draw_text_avatar(cover, avatar, title, channel, top_color, bottom_color) - return cover @@ -332,7 +410,7 @@ def _create_blank_cover_image( yt_icon_y_top = int((COVER_WIDTH - yt_icon.height) / 2) cover.paste(yt_icon, (yt_icon_x_left, yt_icon_y_top)) - _draw_text_avatar(cover, avatar, title, channel, bg_color, bg_color) + _draw_text_avatar(cover, avatar, title, channel) return cover diff --git a/ucast/service/youtube.py b/ucast/service/youtube.py index 04df14c..00b1c69 100644 --- a/ucast/service/youtube.py +++ b/ucast/service/youtube.py @@ -9,7 +9,6 @@ from typing import List, Optional import feedparser import requests -from django.conf import settings from mutagen import id3 from yt_dlp import YoutubeDL @@ -291,9 +290,7 @@ def get_channel_videos_from_scraper( """ videos = [] - for item in scrapetube.get_channel( - channel_url_from_id(channel_id), limit, settings.YOUTUBE_SCRAPE_DELAY - ): + for item in scrapetube.get_channel(channel_url_from_id(channel_id), limit): video_id = item.get("videoId") if not video_id: logging.warning( diff --git a/ucast/tasks/download.py b/ucast/tasks/download.py index c997a59..924947c 100644 --- a/ucast/tasks/download.py +++ b/ucast/tasks/download.py @@ -1,6 +1,8 @@ import os +from datetime import datetime import django_rq +from django.conf import settings from django.utils import timezone from ucast.models import Channel, Video @@ -118,6 +120,15 @@ def import_channel(channel_id: str, limit: int = None): _load_scraped_video(vid, channel) +def schedule_update_channels(): + django_rq.get_scheduler().schedule( + datetime.utcnow(), + update_channels, + id="schedule_update_channels", + interval=settings.YT_UPDATE_INTERVAL, + ) + + def update_channels(): """ Update all channels from their RSS feeds and download new videos. diff --git a/ucast/tasks/library.py b/ucast/tasks/library.py new file mode 100644 index 0000000..a7f7bb5 --- /dev/null +++ b/ucast/tasks/library.py @@ -0,0 +1,34 @@ +import os + +from ucast.models import Video +from ucast.service import cover, storage + + +def recreate_cover(video: Video): + store = storage.Storage() + cf = store.get_channel_folder(video.channel.slug) + + thumbnail_file = cf.get_thumbnail(video.slug) + cover_file = cf.get_cover(video.slug) + + if not os.path.isfile(cf.file_avatar): + print(f"could not find avatar for channel {video.channel_id}") + return + + if not os.path.isfile(thumbnail_file): + print(f"could not find thumbnail for video {video.id}") + return + + cover.create_cover_file( + thumbnail_file, + cf.file_avatar, + video.title, + video.channel.name, + cover.COVER_STYLE_BLUR, + cover_file, + ) + + +def recreate_covers(): + for video in Video.objects.all(): + recreate_cover(video) diff --git a/ucast/tests/test_youtube.py b/ucast/tests/test_youtube.py index 4b3afce..0cedf7c 100644 --- a/ucast/tests/test_youtube.py +++ b/ucast/tests/test_youtube.py @@ -10,7 +10,7 @@ from PIL import Image, ImageChops from ucast import tests from ucast.service import youtube -VIDEO_ID_SINTEL = "eRsGyueVLvQ" +VIDEO_ID_THETADEV = "ZPxEr4YdWt8" VIDEO_ID_SHORT = "lcQZ6YwQHiw" VIDEO_ID_PERSUASION = "DWjFW7Yq1fA" @@ -21,14 +21,14 @@ CHANNEL_URL_BLENDER = "https://www.youtube.com/c/BlenderFoundation" @pytest.fixture(scope="module") def video_info() -> youtube.VideoDetails: - return youtube.get_video_details(VIDEO_ID_SINTEL) + return youtube.get_video_details(VIDEO_ID_THETADEV) def test_download_thumbnail(video_info): tmpdir_o = tempfile.TemporaryDirectory() tmpdir = Path(tmpdir_o.name) tn_file = tmpdir / "thumbnail" - expected_tn_file = tests.DIR_TESTFILES / "thumbnail" / "t2.webp" + expected_tn_file = tests.DIR_TESTFILES / "thumbnail" / "t1.webp" tn_file = youtube.download_thumbnail(video_info, tn_file) assert tn_file.suffix == ".webp" @@ -41,32 +41,26 @@ def test_download_thumbnail(video_info): def test_get_video_info(video_info): - assert video_info.id == VIDEO_ID_SINTEL - assert video_info.title == "Sintel - Open Movie by Blender Foundation" - assert video_info.channel_id == "UCSMOQeBJ2RAnuFungnQOxLg" + assert video_info.id == VIDEO_ID_THETADEV + assert video_info.title == "ThetaDev @ Embedded World 2019" + assert video_info.channel_id == "UCGiJh0NZ52wRhYKYnuZI08Q" assert ( video_info.description - == """Help us making Free/Open Movies: https://cloud.blender.org/join + == """This february I spent one day at the Embedded World in Nuremberg. They showed tons of interesting electronics stuff, so I had to take some pictures and videos for you to see ;-) -"Sintel" is an independently produced short film, initiated by the Blender Foundation \ -as a means to further improve and validate the free/open source 3D creation suite \ -Blender. With initial funding provided by 1000s of donations via the internet \ -community, it has \ -again proven to be a viable development model for both open 3D technology as for \ -independent animation film. -This 15 minute film has been realized in the studio of the Amsterdam Blender \ -Institute, by an international team of artists and developers. In addition to \ -that, several crucial technical and creative targets have been realized online, \ -by developers and artists and teams all over the world. +Sorry for the late upload, I just didn't have time to edit my footage. -www.sintel.org""" +Embedded World: https://www.embedded-world.de/ + +My website: https://thdev.org +Twitter: https://twitter.com/Theta_Dev""" ) - assert video_info.duration == 888 + assert video_info.duration == 267 assert not video_info.is_currently_live assert not video_info.is_livestream assert not video_info.is_short assert video_info.published == datetime.datetime( - 2010, 9, 30, tzinfo=datetime.timezone.utc + 2019, 6, 2, tzinfo=datetime.timezone.utc ) diff --git a/ucast/tests/testfiles/cover/c1_blur.png b/ucast/tests/testfiles/cover/c1_blur.png index 8b0bd15..12a71c8 100644 Binary files a/ucast/tests/testfiles/cover/c1_blur.png and b/ucast/tests/testfiles/cover/c1_blur.png differ diff --git a/ucast_project/settings.py b/ucast_project/settings.py index 3cfd083..c7790e7 100644 --- a/ucast_project/settings.py +++ b/ucast_project/settings.py @@ -84,7 +84,7 @@ ALLOWED_HOSTS = [] # Application definition INSTALLED_APPS = [ - "ucast.apps.UcastConfig", + "ucast", "django.contrib.admin", "django.contrib.auth", "django.contrib.contenttypes", @@ -203,13 +203,12 @@ STATICFILES_DIRS = [resources.path("ucast", "static")] DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" -# Delay between YouTube API calls -YOUTUBE_SCRAPE_DELAY = 1 - REDIS_HOST = get_env("REDIS_HOST", "localhost") REDIS_PORT = get_env("REDIS_PORT", 6379) REDIS_PASSWORD = get_env("REDIS_PASSWORD", "") REDIS_DB = get_env("REDIS_DB", 0) +REDIS_QUEUE_TIMEOUT = get_env("REDIS_QUEUE_TIMEOUT", 600) +REDIS_QUEUE_RESULT_TTL = 600 RQ_QUEUES = { "default": { @@ -217,8 +216,11 @@ RQ_QUEUES = { "PORT": REDIS_PORT, "DB": REDIS_DB, "PASSWORD": REDIS_PASSWORD, - "DEFAULT_TIMEOUT": get_env("REDIS_QUEUE_TIMEOUT", 360), + "DEFAULT_TIMEOUT": REDIS_QUEUE_TIMEOUT, + "DEFAULT_RESULT_TTL": REDIS_QUEUE_RESULT_TTL, } } RQ_SHOW_ADMIN_LINK = True + +YT_UPDATE_INTERVAL = get_env("YT_UPDATE_INTERVAL", 900)