Refactor dockerfiles for more shared layers

By carefully ensuring most of our dockerfiles are identical in their first few layers,
we only need to build those layers once instead of every time.

In particular, we move installing gevent to before installing common,
so that even when common changes gevent doesn't need to be reinstalled.

This is important because gevent takes ages to install.

Also fixes segment_coverage, which wasn't being installed.

backfiller/Dockerfile

@@ -1,13 +1,17 @@
 FROM alpine:3.7
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras postgresql-dev postgresql-libs
+RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras
+
+# Install gevent so that we don't need to re-install it when common changes
+RUN pip install gevent
 
 # Install common lib first as it changes less
 COPY common /tmp/common
 RUN pip install /tmp/common && rm -r /tmp/common
 
 # Install actual application
+RUN apk add postgresql-dev postgresql-libs
 COPY backfiller /tmp/backfiller
 RUN pip install /tmp/backfiller && rm -r /tmp/backfiller
 

build

@@ -8,7 +8,7 @@ set -eu
 # Pass PUSH=true to also push the resulting images, or PUSH=latest to push them as :latest tag
 
 # The different images we can build
-COMPONENTS=(downloader restreamer backfiller thrimshim cutter sheetsync nginx postgres)
+COMPONENTS=(downloader restreamer backfiller thrimshim cutter sheetsync nginx postgres segment_coverage)
 
 # Define push if not already defined
 PUSH=${PUSH:-}

cutter/Dockerfile

@@ -1,15 +1,17 @@
 FROM alpine:3.7
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-# and postgresql-dev as a dependency of psycopg2.
+RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras
-# Add postgresql-client for easier debugging of DB issues.
+
-RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras postgresql-dev postgresql-client ffmpeg
+# Install gevent so that we don't need to re-install it when common changes
+RUN pip install gevent
 
 # Install common lib first as it changes less
 COPY common /tmp/common
 RUN pip install /tmp/common && rm -r /tmp/common
 
 # Install actual application
+RUN apk add postgresql-dev postgresql-client ffmpeg
 COPY cutter /tmp/cutter
 RUN pip install /tmp/cutter && rm -r /tmp/cutter
 

downloader/Dockerfile

@@ -3,6 +3,9 @@ FROM alpine:3.7
 # also busybox-extras for telnet for easier use of backdoor
 RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras
 
+# Install gevent so that we don't need to re-install it when common changes
+RUN pip install gevent
+
 # Install common lib first as it changes less
 COPY common /tmp/common
 RUN pip install /tmp/common && rm -r /tmp/common

restreamer/Dockerfile

@@ -1,13 +1,17 @@
 FROM alpine:3.7
-# dependencies needed for compiling c extensions, plus ffmpeg for cutting
+# dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-RUN apk --update add py2-pip gcc python-dev musl-dev ffmpeg busybox-extras
+RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras
+
+# Install gevent so that we don't need to re-install it when common changes
+RUN pip install gevent
 
 # Install common lib first as it changes less
 COPY common /tmp/common
 RUN pip install /tmp/common && rm -r /tmp/common
 
 # Install actual application
+RUN apk add ffmpeg
 COPY restreamer /tmp/restreamer
 RUN pip install /tmp/restreamer && rm -r /tmp/restreamer
 

segment_coverage/Dockerfile

@@ -1,18 +1,23 @@
 FROM alpine:3.7
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-# freetype-dev and libpng-dev are required for matplotlib
+RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras
-RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras freetype-dev libpng-dev build-base libstdc++
+
-#need to install these manually 
+# Install gevent so that we don't need to re-install it when common changes
-RUN ln -s /usr/include/locale.h /usr/include/xlocale.h \
+RUN pip install gevent
-	&& pip install numpy \
-	&& pip install matplotlib
 
 # Install common lib first as it changes less
 COPY common /tmp/common
 RUN pip install /tmp/common && rm -r /tmp/common
 
 # Install actual application
+
+# freetype-dev and libpng-dev are required for matplotlib
+RUN apk add freetype-dev libpng-dev build-base libstdc++
+# need to install these manually
+RUN ln -s /usr/include/locale.h /usr/include/xlocale.h \
+	&& pip install numpy \
+	&& pip install matplotlib
 COPY segment_coverage /tmp/segment_coverage
 RUN pip install /tmp/segment_coverage && rm -r /tmp/segment_coverage
 

sheetsync/Dockerfile

@@ -1,15 +1,17 @@
 FROM alpine:3.7
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-# and postgresql-dev as a dependency of psycopg2.
+RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras
-# Add postgresql-client for easier debugging of DB issues.
+
-RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras postgresql-dev postgresql-client
+# Install gevent so that we don't need to re-install it when common changes
+RUN pip install gevent
 
 # Install common lib first as it changes less
 COPY common /tmp/common
 RUN pip install /tmp/common && rm -r /tmp/common
 
 # Install actual application
+RUN apk add postgresql-dev postgresql-client
 COPY sheetsync /tmp/sheetsync
 RUN pip install /tmp/sheetsync && rm -r /tmp/sheetsync
 

thrimshim/Dockerfile

@@ -1,14 +1,17 @@
 FROM alpine:3.7
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-# plus postgres dependenices
+RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras
-RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras postgresql-dev postgresql-libs
+
+# Install gevent so that we don't need to re-install it when common changes
+RUN pip install gevent
 
 # Install common lib first as it changes less
 COPY common /tmp/common
 RUN pip install /tmp/common && rm -r /tmp/common
 
 # Install actual application
+RUN apk add postgresql-dev postgresql-libs
 COPY thrimshim /tmp/thrimshim
 RUN pip install /tmp/thrimshim && rm -r /tmp/thrimshim