Do a first naive pass for py3 compatibility

Check that open() calls for reading and writing use binary modes
Use alpine version with py3-pip package
Use python3 in Dockerfile CMD
Remove sys.setdefaultencoding() "hack"
Simplify ensure_directory() in common.common package

api_ping/Dockerfile

@@ -1,7 +1,7 @@
-FROM alpine:3.7
+FROM alpine:3.14
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-RUN apk --update add py2-pip gcc python-dev musl-dev file make busybox-extras
+RUN apk --update add py3-pip gcc python-dev musl-dev file make busybox-extras
 
 # Install gevent so that we don't need to re-install it when common changes
 RUN pip install gevent
@@ -14,4 +14,4 @@ RUN pip install /tmp/common && rm -r /tmp/common
 COPY api_ping /tmp/api_ping
 RUN pip install /tmp/api_ping && rm -r /tmp/api_ping
 
-ENTRYPOINT ["python2", "-m", "api_ping"]
+ENTRYPOINT ["python3", "-m", "api_ping"]

backfiller/Dockerfile

@@ -1,7 +1,7 @@
-FROM alpine:3.7
+FROM alpine:3.14
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-RUN apk --update add py2-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras
+RUN apk --update add py3-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras
 
 # Install gevent so that we don't need to re-install it when common changes
 RUN pip install gevent
@@ -15,4 +15,4 @@ RUN apk add postgresql-dev postgresql-libs
 COPY backfiller /tmp/backfiller
 RUN pip install /tmp/backfiller && rm -r /tmp/backfiller
 
-ENTRYPOINT ["python2", "-m", "backfiller", "--base-dir", "/mnt"]
+ENTRYPOINT ["python3", "-m", "backfiller", "--base-dir", "/mnt"]

backfiller/backfiller/main.py

@@ -147,7 +147,7 @@ def get_remote_segment(base_dir, node, channel, quality, hour, missing_segment,
 
 		resp.raise_for_status()
 
-		with open(temp_path, 'w') as f:
+		with open(temp_path, 'wb') as f:
 			for chunk in resp.iter_content(8192):
 				f.write(chunk)
 				hash.update(chunk)

common/common/__init__.py

@@ -1,14 +1,5 @@
 
 """A place for common utilities between wubloader components"""
-
-
-# HACK: This sets the default encoding for the entire process.
-# It is possible this may break (badly-written) third party libs.
-import sys
-reload(sys)
-sys.setdefaultencoding('utf-8')
-
-
 import datetime
 import errno
 import os
@@ -104,15 +95,7 @@ def ensure_directory(path):
 	"""Create directory that contains path, as well as any parent directories,
 	if they don't already exist."""
 	dir_path = os.path.dirname(path)
-	if os.path.exists(dir_path):
+	os.mkdirs(dir_path, exist_ok=True)
-		return
-	ensure_directory(dir_path)
-	try:
-		os.mkdir(dir_path)
-	except OSError as e:
-		# Ignore if EEXISTS. This is needed to avoid a race if two getters run at once.
-		if e.errno != errno.EEXIST:
-			raise
 
 
 def jitter(interval):

common/common/segments.py

@@ -364,7 +364,7 @@ def rough_cut_segments(segments, start, end):
 	This method works by simply concatenating all the segments, without any re-encoding.
 	"""
 	for segment in segments:
-		with open(segment.path) as f:
+		with open(segment.path, 'rb') as f:
 			for chunk in read_chunks(f):
 				yield chunk
 
@@ -437,7 +437,7 @@ def fast_cut_segments(segments, start, end):
 					)
 		else:
 			# no cutting needed, just serve the file
-			with open(segment.path) as f:
+			with open(segment.path, 'rb') as f:
 				for chunk in read_chunks(f):
 					yield chunk
 

cutter/Dockerfile

@@ -1,7 +1,7 @@
-FROM alpine:3.7
+FROM alpine:3.14
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-RUN apk --update add py2-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras
+RUN apk --update add py3-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras
 
 # Install gevent so that we don't need to re-install it when common changes
 RUN pip install gevent
@@ -15,4 +15,4 @@ RUN apk add postgresql-dev postgresql-client ffmpeg
 COPY cutter /tmp/cutter
 RUN pip install /tmp/cutter && rm -r /tmp/cutter
 
-ENTRYPOINT ["python2", "-m", "cutter", "--base-dir", "/mnt"]
+ENTRYPOINT ["python3", "-m", "cutter", "--base-dir", "/mnt"]

cutter/cutter/upload_backends.py

@@ -241,13 +241,13 @@ class Local(UploadBackend):
 		filepath = os.path.join(self.path, filename)
 		try:
 			if self.write_info:
-				with open(os.path.join(self.path, '{}-{}.json'.format(safe_title, video_id)), 'w') as f:
+				with open(os.path.join(self.path, '{}-{}.json'.format(safe_title, video_id)), 'wb') as f:
 					f.write(json.dumps({
 						'title': title,
 						'description': description,
 						'tags': tags,
 					}) + '\n')
-			with open(filepath, 'w') as f:
+			with open(filepath, 'wb') as f:
 				for chunk in data:
 					f.write(chunk)
 		except (OSError, IOError) as e:

downloader/Dockerfile

@@ -1,7 +1,7 @@
-FROM alpine:3.7
+FROM alpine:3.14
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-RUN apk --update add py2-pip gcc python-dev musl-dev file make busybox-extras
+RUN apk --update add py3-pip gcc python-dev musl-dev file make busybox-extras
 
 # Install gevent so that we don't need to re-install it when common changes
 RUN pip install gevent
@@ -14,4 +14,4 @@ RUN pip install /tmp/common && rm -r /tmp/common
 COPY downloader /tmp/downloader
 RUN pip install /tmp/downloader && rm -r /tmp/downloader
 
-ENTRYPOINT ["python2", "-m", "downloader", "--base-dir", "/mnt"]
+ENTRYPOINT ["python3", "-m", "downloader", "--base-dir", "/mnt"]

downloader/downloader/main.py

@@ -555,7 +555,7 @@ class SegmentGetter(object):
 					return
 				resp.raise_for_status()
 				common.ensure_directory(temp_path)
-				with open(temp_path, 'w') as f:
+				with open(temp_path, 'wb') as f:
 					file_created = True
 					# We read chunk-wise in 8KiB chunks. Note that if the connection cuts halfway,
 					# we may lose part of the last chunk even though we did receive it.

playlist_manager/Dockerfile

@@ -1,7 +1,7 @@
-FROM alpine:3.7
+FROM alpine:3.14
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-RUN apk --update add py2-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras
+RUN apk --update add py3-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras
 
 # Install gevent so that we don't need to re-install it when common changes
 RUN pip install gevent
@@ -14,4 +14,4 @@ RUN pip install /tmp/common && rm -r /tmp/common
 COPY playlist_manager /tmp/playlist_manager
 RUN pip install /tmp/playlist_manager && rm -r /tmp/playlist_manager
 
-ENTRYPOINT ["python2", "-m", "playlist_manager"]
+ENTRYPOINT ["python3", "-m", "playlist_manager"]

restreamer/Dockerfile

@@ -1,7 +1,7 @@
-FROM alpine:3.7
+FROM alpine:3.14
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-RUN apk --update add py2-pip gcc python-dev musl-dev file make busybox-extras
+RUN apk --update add py3-pip gcc python-dev musl-dev file make busybox-extras
 
 # Install gevent so that we don't need to re-install it when common changes
 RUN pip install gevent
@@ -15,4 +15,4 @@ RUN apk add ffmpeg
 COPY restreamer /tmp/restreamer
 RUN pip install /tmp/restreamer && rm -r /tmp/restreamer
 
-ENTRYPOINT ["python2", "-m", "restreamer", "--base-dir", "/mnt"]
+ENTRYPOINT ["python3", "-m", "restreamer", "--base-dir", "/mnt"]

restreamer/restreamer/main.py

@@ -306,9 +306,7 @@ def cut(channel, quality):
 @request_stats
 @has_path_args
 def generate_videos(channel, quality):
-	"""Generate one video for each contiguous range of segments (ie. split at holes),
+	"""
-	and save them as CHANNEL_QUALITY_N.ts in the segments directory.
-
 	Takes a JSON body {name: [start, end]} where start and end are timestamps.
 	Creates files CHANNEL_QUALITY_NAME_N.mkv for each contiguous range of segments
 	in that hour range (ie. split at holes) and saves them in the segments directory.
@@ -319,6 +317,10 @@ def generate_videos(channel, quality):
 		start = dateutil.parse_utc_only(start)
 		end = dateutil.parse_utc_only(end)
 
+		# protect against directory traversal
+		if "/" in name:
+			return "Name cannot contain /", 400
+
 		if end <= start:
 			return "End must be after start", 400
 

segment_coverage/Dockerfile

@@ -1,7 +1,7 @@
-FROM alpine:3.7
+FROM alpine:3.14
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-RUN apk --update add py2-pip gcc python-dev musl-dev file make busybox-extras
+RUN apk --update add py3-pip gcc python-dev musl-dev file make busybox-extras
 
 # Install gevent so that we don't need to re-install it when common changes
 RUN pip install gevent
@@ -21,4 +21,4 @@ RUN ln -s /usr/include/locale.h /usr/include/xlocale.h \
 COPY segment_coverage /tmp/segment_coverage
 RUN pip install /tmp/segment_coverage && rm -r /tmp/segment_coverage
 
-ENTRYPOINT ["python2", "-m", "segment_coverage"]
+ENTRYPOINT ["python3", "-m", "segment_coverage"]

segment_coverage/segment_coverage/main.py

@@ -231,7 +231,7 @@ class CoverageChecker(object):
 		temp_path = '{}_{}.html'.format(path_prefix, uuid.uuid4())
 		final_path = '{}_coverage.html'.format(path_prefix)
 		common.ensure_directory(temp_path)
-		with open(temp_path, 'w') as f:
+		with open(temp_path, 'wb') as f:
 			f.write(html)
 		os.rename(temp_path, final_path)
 		self.logger.info('Coverage page for {} created'.format(quality))

sheetsync/Dockerfile

@@ -1,7 +1,7 @@
-FROM alpine:3.7
+FROM alpine:3.14
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-RUN apk --update add py2-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras
+RUN apk --update add py3-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras
 
 # Install gevent so that we don't need to re-install it when common changes
 RUN pip install gevent
@@ -15,4 +15,4 @@ RUN apk add postgresql-dev postgresql-client
 COPY sheetsync /tmp/sheetsync
 RUN pip install /tmp/sheetsync && rm -r /tmp/sheetsync
 
-ENTRYPOINT ["python2", "-m", "sheetsync"]
+ENTRYPOINT ["python3", "-m", "sheetsync"]

thrimshim/Dockerfile

@@ -1,7 +1,7 @@
-FROM alpine:3.7
+FROM alpine:3.14
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-RUN apk --update add py2-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras
+RUN apk --update add py3-pip gcc python-dev musl-dev postgresql-dev file make busybox-extras
 
 # Install gevent so that we don't need to re-install it when common changes
 RUN pip install gevent
@@ -15,4 +15,4 @@ RUN apk add postgresql-dev postgresql-libs
 COPY thrimshim /tmp/thrimshim
 RUN pip install /tmp/thrimshim && rm -r /tmp/thrimshim
 
-ENTRYPOINT ["python2", "-m", "thrimshim"]
+ENTRYPOINT ["python3", "-m", "thrimshim"]