From 5b0b909d26838c904c7b300bf6b4c148c5977c55 Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Sat, 14 Sep 2024 08:28:25 +1000 Subject: [PATCH] download_media: Add function for checking if a URL has been downloaded before --- common/common/media.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/common/common/media.py b/common/common/media.py index 57872ee..9d13a56 100644 --- a/common/common/media.py +++ b/common/common/media.py @@ -14,7 +14,7 @@ import prometheus_client as prom import urllib3.connection from ipaddress import ip_address -from . import atomic_write, ensure_directory, jitter +from . import atomic_write, ensure_directory, jitter, listdir from .stats import timed @@ -52,6 +52,12 @@ class WrongContent(Rejected): """Response was not a video or image""" +def check_for_media(output_dir, url): + """Returns True if we have at least one version of content for the given url already.""" + url_dir = get_url_dir(output_dir, url) + return any(filename.endswith(".metadata.json") for filename in listdir(url_dir)) + + @timed() def download_media( url,