From 1f7140131fce6c8c1da8a41595f23f8eaf491cd4 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 29 Jun 2022 22:46:42 +0200 Subject: [PATCH 1/3] move db folder --- ucast_project/settings.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/ucast_project/settings.py b/ucast_project/settings.py index 3e16b0a..b98373b 100644 --- a/ucast_project/settings.py +++ b/ucast_project/settings.py @@ -32,8 +32,11 @@ def get_env(name, default=None): def get_env_path(name, default=None): raw_env = get_env(name) if not raw_env: - return default - return Path(raw_env).absolute() + folder = default + else: + folder = Path(raw_env).absolute() + os.makedirs(folder, exist_ok=True) + return folder def get_env_list(name): @@ -142,7 +145,7 @@ def _get_db_config() -> dict: if db_engine == "sqlite": return { "ENGINE": "django.db.backends.sqlite3", - "NAME": BASE_DIR / f"{db_name}.sqlite", + "NAME": DB_DIR / f"{db_name}.sqlite", } db_port = get_env("DB_PORT") @@ -162,6 +165,18 @@ def _get_db_config() -> dict: } +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.0/howto/static-files/ + +STATIC_URL = "static/" +STATIC_ROOT = get_env_path("STATIC_ROOT", BASE_DIR / "static") +DOWNLOAD_ROOT = get_env_path("DOWNLOAD_ROOT", BASE_DIR / "data") +CACHE_ROOT = get_env_path("CACHE_ROOT", BASE_DIR / "cache") +DB_DIR = get_env_path("DB_DIR", BASE_DIR / "db") + +STATICFILES_DIRS = [resources.path("ucast", "static")] + + # Database # https://docs.djangoproject.com/en/4.0/ref/settings/#databases DATABASES = { @@ -202,16 +217,6 @@ USE_I18N = True USE_TZ = True -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/4.0/howto/static-files/ - -STATIC_URL = "static/" -STATIC_ROOT = get_env_path("STATIC_ROOT", BASE_DIR / "static") -DOWNLOAD_ROOT = get_env_path("DOWNLOAD_ROOT", BASE_DIR / "data") -CACHE_ROOT = get_env_path("CACHE_ROOT", BASE_DIR / "cache") - -STATICFILES_DIRS = [resources.path("ucast", "static")] - # Default primary key field type # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field From ca5062cced97ecd8d03ade6f5fab576904a5bdd6 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 29 Jun 2022 23:25:43 +0200 Subject: [PATCH 2/3] add daily cache cleanup --- ucast/service/storage.py | 17 +++++++++++++++++ ucast/tasks/library.py | 5 +++++ ucast/tasks/schedule.py | 11 +++++++++-- ucast/tests/service/test_storage.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/ucast/service/storage.py b/ucast/service/storage.py index e0a87ba..e1b6af0 100644 --- a/ucast/service/storage.py +++ b/ucast/service/storage.py @@ -1,5 +1,7 @@ import os +import shutil import tempfile +from datetime import datetime, timedelta from pathlib import Path from django.conf import settings @@ -73,3 +75,18 @@ class Cache: def create_tmpdir(self, prefix="dld") -> tempfile.TemporaryDirectory: return tempfile.TemporaryDirectory(prefix=prefix + "_", dir=self.dir_cache) + + def cleanup(self): + """ + Delete temporary directories that are older than 24h and are most likely left + over after unexpected shutdowns. + """ + for dirname in os.listdir(self.dir_cache): + if dirname == "yt_dlp": + continue + + ctime = os.path.getctime(dirname) + age = datetime.now() - datetime.fromtimestamp(ctime) + + if age > timedelta(days=1): + shutil.rmtree(self.dir_cache / dirname) diff --git a/ucast/tasks/library.py b/ucast/tasks/library.py index b6b3481..2a19bb6 100644 --- a/ucast/tasks/library.py +++ b/ucast/tasks/library.py @@ -105,3 +105,8 @@ def update_channel_info(channel: Channel): def update_channel_infos(): for channel in Channel.objects.filter(active=True): queue.enqueue(update_channel_info, channel) + + +def clean_cache(): + cache = storage.Cache() + cache.cleanup() diff --git a/ucast/tasks/schedule.py b/ucast/tasks/schedule.py index f56b86b..9786e93 100644 --- a/ucast/tasks/schedule.py +++ b/ucast/tasks/schedule.py @@ -1,5 +1,5 @@ import logging -from datetime import datetime +from datetime import datetime, timedelta from django.conf import settings @@ -28,8 +28,15 @@ def register_scheduled_jobs(): ) scheduler.schedule( - datetime.utcnow(), + datetime.utcnow() + timedelta(days=1), library.update_channel_infos, id="schedule_update_channel_infos", interval=24 * 3600, ) + + scheduler.schedule( + datetime.utcnow() + timedelta(days=1), + library.clean_cache, + id="schedule_clean_cache", + interval=24 * 3600, + ) diff --git a/ucast/tests/service/test_storage.py b/ucast/tests/service/test_storage.py index 083e3e5..fa99f2a 100644 --- a/ucast/tests/service/test_storage.py +++ b/ucast/tests/service/test_storage.py @@ -1,5 +1,6 @@ import os import tempfile +from datetime import datetime, timedelta from pathlib import Path from ucast.service import storage @@ -54,3 +55,30 @@ def test_channel_folder(): == ucast_dir / "thumbnails" / "my_video_title_sm.webp" ) assert cf.get_audio("my_video_title") == tmpdir / "my_video_title.mp3" + + +def test_clean_cache(settings, mocker): + tmpdir_o = tempfile.TemporaryDirectory() + tmpdir = Path(tmpdir_o.name) + + os.mkdir(tmpdir / "yt_dlp") + os.mkdir(tmpdir / "dld_old") + os.mkdir(tmpdir / "dld_new") + + def mock_ctime(path): + if path == "dld_new": + return datetime.now().timestamp() + if path == "dld_old": + return (datetime.now() - timedelta(days=1, minutes=1)).timestamp() + raise Exception("invalid path") + + mocker.patch.object(os.path, "getctime", mock_ctime) + + settings.CACHE_ROOT = tmpdir + cache = storage.Cache() + + cache.cleanup() + + assert os.path.isdir(tmpdir / "yt_dlp") + assert os.path.isdir(tmpdir / "dld_new") + assert not os.path.exists(tmpdir / "dld_old") From 991fe3df305c5fd26df51b677da39ce2e4623479 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 29 Jun 2022 23:26:04 +0200 Subject: [PATCH 3/3] =?UTF-8?q?Bump=20version:=200.3.1=20=E2=86=92=200.3.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- pyproject.toml | 2 +- ucast/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 02482c5..d332eb4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.1 +current_version = 0.3.2 commit = True tag = True diff --git a/pyproject.toml b/pyproject.toml index 96ab12a..d2ba66e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ucast" -version = "0.3.1" +version = "0.3.2" description = "YouTube to Podcast converter" authors = ["Theta-Dev "] packages = [ diff --git a/ucast/__init__.py b/ucast/__init__.py index 521568f..0b6e0b2 100644 --- a/ucast/__init__.py +++ b/ucast/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.3.1" +__version__ = "0.3.2" def template_context(request):