From 6d790a1b3654461ea0c3505d0cf9c053a96dac18 Mon Sep 17 00:00:00 2001 From: HubbeKing Date: Wed, 7 Jul 2021 16:12:04 +0300 Subject: [PATCH] Do a first naive pass for py3 compatibility Check that open() calls for reading and writing use binary modes Use alpine version with py3-pip package Use python3 in Dockerfile CMD Remove sys.setdefaultencoding() "hack" Simplify ensure_directory() in common.common package --- api_ping/Dockerfile | 6 +++--- backfiller/Dockerfile | 6 +++--- backfiller/backfiller/main.py | 2 +- common/common/__init__.py | 19 +------------------ common/common/segments.py | 4 ++-- cutter/Dockerfile | 6 +++--- cutter/cutter/upload_backends.py | 4 ++-- downloader/Dockerfile | 6 +++--- downloader/downloader/main.py | 2 +- playlist_manager/Dockerfile | 6 +++--- restreamer/Dockerfile | 6 +++--- restreamer/restreamer/main.py | 8 +++++--- segment_coverage/Dockerfile | 6 +++--- segment_coverage/segment_coverage/main.py | 2 +- sheetsync/Dockerfile | 6 +++--- thrimshim/Dockerfile | 6 +++--- 16 files changed, 40 insertions(+), 55 deletions(-) diff --git a/api_ping/Dockerfile b/api_ping/Dockerfile index fe406ff..52a6c0c 100644 --- a/api_ping/Dockerfile +++ b/api_ping/Dockerfile @@ -1,7 +1,7 @@ -FROM alpine:3.7 +FROM alpine:3.14 # dependencies needed for compiling c extensions # also busybox-extras for telnet for easier use of backdoor -RUN apk --update add py2-pip gcc python-dev musl-dev file make busybox-extras +RUN apk --update add py3-pip gcc python-dev musl-dev file make busybox-extras # Install gevent so that we don't need to re-install it when common changes RUN pip install gevent @@ -14,4 +14,4 @@ RUN pip install /tmp/common && rm -r /tmp/common COPY api_ping /tmp/api_ping RUN pip install /tmp/api_ping && rm -r /tmp/api_ping -ENTRYPOINT ["python2", "-m", "api_ping"] +ENTRYPOINT ["python3", "-m", "api_ping"] diff --git a/backfiller/Dockerfile b/backfiller/Dockerfile index 4b3fe20..6d4cd76 100644 --- a/backfiller/Dockerfile +++ b/backfiller/Dockerfile @@ -1,7 +1,7 @@ -FROM alpine:3.7 +FROM alpine:3.14 # dependencies needed for compiling c extensions # also busybox-extras for telnet for easier use of backdoor -RUN apk --update add py2-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras +RUN apk --update add py3-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras # Install gevent so that we don't need to re-install it when common changes RUN pip install gevent @@ -15,4 +15,4 @@ RUN apk add postgresql-dev postgresql-libs COPY backfiller /tmp/backfiller RUN pip install /tmp/backfiller && rm -r /tmp/backfiller -ENTRYPOINT ["python2", "-m", "backfiller", "--base-dir", "/mnt"] +ENTRYPOINT ["python3", "-m", "backfiller", "--base-dir", "/mnt"] diff --git a/backfiller/backfiller/main.py b/backfiller/backfiller/main.py index a603c9a..fd011a5 100644 --- a/backfiller/backfiller/main.py +++ b/backfiller/backfiller/main.py @@ -147,7 +147,7 @@ def get_remote_segment(base_dir, node, channel, quality, hour, missing_segment, resp.raise_for_status() - with open(temp_path, 'w') as f: + with open(temp_path, 'wb') as f: for chunk in resp.iter_content(8192): f.write(chunk) hash.update(chunk) diff --git a/common/common/__init__.py b/common/common/__init__.py index c0cb4f3..a1a4299 100644 --- a/common/common/__init__.py +++ b/common/common/__init__.py @@ -1,14 +1,5 @@ """A place for common utilities between wubloader components""" - - -# HACK: This sets the default encoding for the entire process. -# It is possible this may break (badly-written) third party libs. -import sys -reload(sys) -sys.setdefaultencoding('utf-8') - - import datetime import errno import os @@ -104,15 +95,7 @@ def ensure_directory(path): """Create directory that contains path, as well as any parent directories, if they don't already exist.""" dir_path = os.path.dirname(path) - if os.path.exists(dir_path): - return - ensure_directory(dir_path) - try: - os.mkdir(dir_path) - except OSError as e: - # Ignore if EEXISTS. This is needed to avoid a race if two getters run at once. - if e.errno != errno.EEXIST: - raise + os.mkdirs(dir_path, exist_ok=True) def jitter(interval): diff --git a/common/common/segments.py b/common/common/segments.py index 341b5f1..c222b23 100644 --- a/common/common/segments.py +++ b/common/common/segments.py @@ -364,7 +364,7 @@ def rough_cut_segments(segments, start, end): This method works by simply concatenating all the segments, without any re-encoding. """ for segment in segments: - with open(segment.path) as f: + with open(segment.path, 'rb') as f: for chunk in read_chunks(f): yield chunk @@ -437,7 +437,7 @@ def fast_cut_segments(segments, start, end): ) else: # no cutting needed, just serve the file - with open(segment.path) as f: + with open(segment.path, 'rb') as f: for chunk in read_chunks(f): yield chunk diff --git a/cutter/Dockerfile b/cutter/Dockerfile index 9d911f7..6443a6f 100644 --- a/cutter/Dockerfile +++ b/cutter/Dockerfile @@ -1,7 +1,7 @@ -FROM alpine:3.7 +FROM alpine:3.14 # dependencies needed for compiling c extensions # also busybox-extras for telnet for easier use of backdoor -RUN apk --update add py2-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras +RUN apk --update add py3-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras # Install gevent so that we don't need to re-install it when common changes RUN pip install gevent @@ -15,4 +15,4 @@ RUN apk add postgresql-dev postgresql-client ffmpeg COPY cutter /tmp/cutter RUN pip install /tmp/cutter && rm -r /tmp/cutter -ENTRYPOINT ["python2", "-m", "cutter", "--base-dir", "/mnt"] +ENTRYPOINT ["python3", "-m", "cutter", "--base-dir", "/mnt"] diff --git a/cutter/cutter/upload_backends.py b/cutter/cutter/upload_backends.py index 49fd528..96b558a 100644 --- a/cutter/cutter/upload_backends.py +++ b/cutter/cutter/upload_backends.py @@ -241,13 +241,13 @@ class Local(UploadBackend): filepath = os.path.join(self.path, filename) try: if self.write_info: - with open(os.path.join(self.path, '{}-{}.json'.format(safe_title, video_id)), 'w') as f: + with open(os.path.join(self.path, '{}-{}.json'.format(safe_title, video_id)), 'wb') as f: f.write(json.dumps({ 'title': title, 'description': description, 'tags': tags, }) + '\n') - with open(filepath, 'w') as f: + with open(filepath, 'wb') as f: for chunk in data: f.write(chunk) except (OSError, IOError) as e: diff --git a/downloader/Dockerfile b/downloader/Dockerfile index 3c0ff11..f1266ce 100644 --- a/downloader/Dockerfile +++ b/downloader/Dockerfile @@ -1,7 +1,7 @@ -FROM alpine:3.7 +FROM alpine:3.14 # dependencies needed for compiling c extensions # also busybox-extras for telnet for easier use of backdoor -RUN apk --update add py2-pip gcc python-dev musl-dev file make busybox-extras +RUN apk --update add py3-pip gcc python-dev musl-dev file make busybox-extras # Install gevent so that we don't need to re-install it when common changes RUN pip install gevent @@ -14,4 +14,4 @@ RUN pip install /tmp/common && rm -r /tmp/common COPY downloader /tmp/downloader RUN pip install /tmp/downloader && rm -r /tmp/downloader -ENTRYPOINT ["python2", "-m", "downloader", "--base-dir", "/mnt"] +ENTRYPOINT ["python3", "-m", "downloader", "--base-dir", "/mnt"] diff --git a/downloader/downloader/main.py b/downloader/downloader/main.py index e505e5b..b7fa10a 100644 --- a/downloader/downloader/main.py +++ b/downloader/downloader/main.py @@ -555,7 +555,7 @@ class SegmentGetter(object): return resp.raise_for_status() common.ensure_directory(temp_path) - with open(temp_path, 'w') as f: + with open(temp_path, 'wb') as f: file_created = True # We read chunk-wise in 8KiB chunks. Note that if the connection cuts halfway, # we may lose part of the last chunk even though we did receive it. diff --git a/playlist_manager/Dockerfile b/playlist_manager/Dockerfile index 2e4bb8d..c68c098 100644 --- a/playlist_manager/Dockerfile +++ b/playlist_manager/Dockerfile @@ -1,7 +1,7 @@ -FROM alpine:3.7 +FROM alpine:3.14 # dependencies needed for compiling c extensions # also busybox-extras for telnet for easier use of backdoor -RUN apk --update add py2-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras +RUN apk --update add py3-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras # Install gevent so that we don't need to re-install it when common changes RUN pip install gevent @@ -14,4 +14,4 @@ RUN pip install /tmp/common && rm -r /tmp/common COPY playlist_manager /tmp/playlist_manager RUN pip install /tmp/playlist_manager && rm -r /tmp/playlist_manager -ENTRYPOINT ["python2", "-m", "playlist_manager"] +ENTRYPOINT ["python3", "-m", "playlist_manager"] diff --git a/restreamer/Dockerfile b/restreamer/Dockerfile index 0522150..5f68c1e 100644 --- a/restreamer/Dockerfile +++ b/restreamer/Dockerfile @@ -1,7 +1,7 @@ -FROM alpine:3.7 +FROM alpine:3.14 # dependencies needed for compiling c extensions # also busybox-extras for telnet for easier use of backdoor -RUN apk --update add py2-pip gcc python-dev musl-dev file make busybox-extras +RUN apk --update add py3-pip gcc python-dev musl-dev file make busybox-extras # Install gevent so that we don't need to re-install it when common changes RUN pip install gevent @@ -15,4 +15,4 @@ RUN apk add ffmpeg COPY restreamer /tmp/restreamer RUN pip install /tmp/restreamer && rm -r /tmp/restreamer -ENTRYPOINT ["python2", "-m", "restreamer", "--base-dir", "/mnt"] +ENTRYPOINT ["python3", "-m", "restreamer", "--base-dir", "/mnt"] diff --git a/restreamer/restreamer/main.py b/restreamer/restreamer/main.py index f62176e..098a353 100644 --- a/restreamer/restreamer/main.py +++ b/restreamer/restreamer/main.py @@ -306,9 +306,7 @@ def cut(channel, quality): @request_stats @has_path_args def generate_videos(channel, quality): - """Generate one video for each contiguous range of segments (ie. split at holes), - and save them as CHANNEL_QUALITY_N.ts in the segments directory. - + """ Takes a JSON body {name: [start, end]} where start and end are timestamps. Creates files CHANNEL_QUALITY_NAME_N.mkv for each contiguous range of segments in that hour range (ie. split at holes) and saves them in the segments directory. @@ -319,6 +317,10 @@ def generate_videos(channel, quality): start = dateutil.parse_utc_only(start) end = dateutil.parse_utc_only(end) + # protect against directory traversal + if "/" in name: + return "Name cannot contain /", 400 + if end <= start: return "End must be after start", 400 diff --git a/segment_coverage/Dockerfile b/segment_coverage/Dockerfile index 9a99796..f1e7c1f 100644 --- a/segment_coverage/Dockerfile +++ b/segment_coverage/Dockerfile @@ -1,7 +1,7 @@ -FROM alpine:3.7 +FROM alpine:3.14 # dependencies needed for compiling c extensions # also busybox-extras for telnet for easier use of backdoor -RUN apk --update add py2-pip gcc python-dev musl-dev file make busybox-extras +RUN apk --update add py3-pip gcc python-dev musl-dev file make busybox-extras # Install gevent so that we don't need to re-install it when common changes RUN pip install gevent @@ -21,4 +21,4 @@ RUN ln -s /usr/include/locale.h /usr/include/xlocale.h \ COPY segment_coverage /tmp/segment_coverage RUN pip install /tmp/segment_coverage && rm -r /tmp/segment_coverage -ENTRYPOINT ["python2", "-m", "segment_coverage"] +ENTRYPOINT ["python3", "-m", "segment_coverage"] diff --git a/segment_coverage/segment_coverage/main.py b/segment_coverage/segment_coverage/main.py index d64968e..dea070e 100644 --- a/segment_coverage/segment_coverage/main.py +++ b/segment_coverage/segment_coverage/main.py @@ -231,7 +231,7 @@ class CoverageChecker(object): temp_path = '{}_{}.html'.format(path_prefix, uuid.uuid4()) final_path = '{}_coverage.html'.format(path_prefix) common.ensure_directory(temp_path) - with open(temp_path, 'w') as f: + with open(temp_path, 'wb') as f: f.write(html) os.rename(temp_path, final_path) self.logger.info('Coverage page for {} created'.format(quality)) diff --git a/sheetsync/Dockerfile b/sheetsync/Dockerfile index a524eab..3dc92ca 100644 --- a/sheetsync/Dockerfile +++ b/sheetsync/Dockerfile @@ -1,7 +1,7 @@ -FROM alpine:3.7 +FROM alpine:3.14 # dependencies needed for compiling c extensions # also busybox-extras for telnet for easier use of backdoor -RUN apk --update add py2-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras +RUN apk --update add py3-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras # Install gevent so that we don't need to re-install it when common changes RUN pip install gevent @@ -15,4 +15,4 @@ RUN apk add postgresql-dev postgresql-client COPY sheetsync /tmp/sheetsync RUN pip install /tmp/sheetsync && rm -r /tmp/sheetsync -ENTRYPOINT ["python2", "-m", "sheetsync"] +ENTRYPOINT ["python3", "-m", "sheetsync"] diff --git a/thrimshim/Dockerfile b/thrimshim/Dockerfile index 2a2ff92..1ee0060 100644 --- a/thrimshim/Dockerfile +++ b/thrimshim/Dockerfile @@ -1,7 +1,7 @@ -FROM alpine:3.7 +FROM alpine:3.14 # dependencies needed for compiling c extensions # also busybox-extras for telnet for easier use of backdoor -RUN apk --update add py2-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras +RUN apk --update add py3-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras # Install gevent so that we don't need to re-install it when common changes RUN pip install gevent @@ -15,4 +15,4 @@ RUN apk add postgresql-dev postgresql-libs COPY thrimshim /tmp/thrimshim RUN pip install /tmp/thrimshim && rm -r /tmp/thrimshim -ENTRYPOINT ["python2", "-m", "thrimshim"] +ENTRYPOINT ["python3", "-m", "thrimshim"]