Merge pull request #140 from ekimekim/mike/build-improvements

Refactor dockerfiles for more shared layers

backfiller/Dockerfile

@@ -1,13 +1,17 @@
 FROM alpine:3.7
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras postgresql-dev postgresql-libs
+RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras
+
+# Install gevent so that we don't need to re-install it when common changes
+RUN pip install gevent
 
 # Install common lib first as it changes less
 COPY common /tmp/common
 RUN pip install /tmp/common && rm -r /tmp/common
 
 # Install actual application
+RUN apk add postgresql-dev postgresql-libs
 COPY backfiller /tmp/backfiller
 RUN pip install /tmp/backfiller && rm -r /tmp/backfiller
 

build

@@ -2,13 +2,16 @@
 
 set -eu
 
+# cd to location of script
+cd "$(dirname "$(readlink -f "$0")")"
+
 # Builds the docker images.
 # Usage: ./build {COMPONENTS}, or just ./build to build all.
 # The resulting images are named wubloader-COMPONENT.
 # Pass PUSH=true to also push the resulting images, or PUSH=latest to push them as :latest tag
 
 # The different images we can build
-COMPONENTS=(downloader restreamer backfiller thrimshim cutter sheetsync nginx postgres)
+COMPONENTS=(downloader restreamer backfiller thrimshim cutter sheetsync nginx postgres segment_coverage)
 
 # Define push if not already defined
 PUSH=${PUSH:-}
@@ -17,10 +20,7 @@ PUSH=${PUSH:-}
 BASE="quay.io/ekimekim"
 
 # The docker image tag, derived from the git commit + whether working tree is clean
-TAG=$(git rev-parse --short HEAD)
+TAG=$(./get-build-tag)
-if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
-	TAG="$TAG-wip"
-fi
 
 if [ "$#" -gt 0 ]; then
 	COMPONENTS=("$@")

cutter/Dockerfile

@@ -1,15 +1,17 @@
 FROM alpine:3.7
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-# and postgresql-dev as a dependency of psycopg2.
+RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras
-# Add postgresql-client for easier debugging of DB issues.
+
-RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras postgresql-dev postgresql-client ffmpeg
+# Install gevent so that we don't need to re-install it when common changes
+RUN pip install gevent
 
 # Install common lib first as it changes less
 COPY common /tmp/common
 RUN pip install /tmp/common && rm -r /tmp/common
 
 # Install actual application
+RUN apk add postgresql-dev postgresql-client ffmpeg
 COPY cutter /tmp/cutter
 RUN pip install /tmp/cutter && rm -r /tmp/cutter
 

docker-compose.jsonnet

@@ -7,9 +7,11 @@
   // Change these to configure the services.
 
   // Image tag (application version) to use.
+  // By default, will use the current commit, ie. the same thing that ./build would tag
+  // things it builds with.
   // Note: "latest" is not recommended in production, as you can't be sure what version
   // you're actually running, and must manually re-pull to get an updated copy.
-  image_tag:: "latest",
+  image_tag:: std.extVar("tag"),
 
   // For each service, whether to deploy that service.
   enabled:: {

downloader/Dockerfile

@@ -3,6 +3,9 @@ FROM alpine:3.7
 # also busybox-extras for telnet for easier use of backdoor
 RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras
 
+# Install gevent so that we don't need to re-install it when common changes
+RUN pip install gevent
+
 # Install common lib first as it changes less
 COPY common /tmp/common
 RUN pip install /tmp/common && rm -r /tmp/common

generate-docker-compose

@@ -2,9 +2,12 @@
 
 set -eu
 
+# cd to location of script
+cd "$(dirname "$(readlink -f "$0")")"
+
 # We generate first, and capture the output, to avoid overwriting the file on error.
 # To avoid jsonnet needing to exist locally, we run it in a container.
-output=$(docker run --rm -i sparkprime/jsonnet - < docker-compose.jsonnet)
+output=$(docker run --rm -i sparkprime/jsonnet -V tag="$(./get-build-tag)" - < docker-compose.jsonnet)
 
 {
 	echo "# DO NOT EDIT THIS FILE!"

get-build-tag

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# cd to location of script
+cd "$(dirname "$(readlink -f "$0")")"
+
+TAG=$(git rev-parse --short HEAD)
+if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
+	TAG="$TAG-wip"
+fi
+
+echo "$TAG"

restreamer/Dockerfile

@@ -1,13 +1,17 @@
 FROM alpine:3.7
-# dependencies needed for compiling c extensions, plus ffmpeg for cutting
+# dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-RUN apk --update add py2-pip gcc python-dev musl-dev ffmpeg busybox-extras
+RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras
+
+# Install gevent so that we don't need to re-install it when common changes
+RUN pip install gevent
 
 # Install common lib first as it changes less
 COPY common /tmp/common
 RUN pip install /tmp/common && rm -r /tmp/common
 
 # Install actual application
+RUN apk add ffmpeg
 COPY restreamer /tmp/restreamer
 RUN pip install /tmp/restreamer && rm -r /tmp/restreamer
 

segment_coverage/Dockerfile

@@ -1,18 +1,23 @@
 FROM alpine:3.7
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-# freetype-dev and libpng-dev are required for matplotlib
+RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras
-RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras freetype-dev libpng-dev build-base libstdc++
+
-#need to install these manually 
+# Install gevent so that we don't need to re-install it when common changes
-RUN ln -s /usr/include/locale.h /usr/include/xlocale.h \
+RUN pip install gevent
-	&& pip install numpy \
-	&& pip install matplotlib
 
 # Install common lib first as it changes less
 COPY common /tmp/common
 RUN pip install /tmp/common && rm -r /tmp/common
 
 # Install actual application
+
+# freetype-dev and libpng-dev are required for matplotlib
+RUN apk add freetype-dev libpng-dev build-base libstdc++
+# need to install these manually
+RUN ln -s /usr/include/locale.h /usr/include/xlocale.h \
+	&& pip install numpy \
+	&& pip install matplotlib
 COPY segment_coverage /tmp/segment_coverage
 RUN pip install /tmp/segment_coverage && rm -r /tmp/segment_coverage
 

sheetsync/Dockerfile

@@ -1,15 +1,17 @@
 FROM alpine:3.7
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-# and postgresql-dev as a dependency of psycopg2.
+RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras
-# Add postgresql-client for easier debugging of DB issues.
+
-RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras postgresql-dev postgresql-client
+# Install gevent so that we don't need to re-install it when common changes
+RUN pip install gevent
 
 # Install common lib first as it changes less
 COPY common /tmp/common
 RUN pip install /tmp/common && rm -r /tmp/common
 
 # Install actual application
+RUN apk add postgresql-dev postgresql-client
 COPY sheetsync /tmp/sheetsync
 RUN pip install /tmp/sheetsync && rm -r /tmp/sheetsync
 

thrimshim/Dockerfile

@@ -1,14 +1,17 @@
 FROM alpine:3.7
 # dependencies needed for compiling c extensions
 # also busybox-extras for telnet for easier use of backdoor
-# plus postgres dependenices
+RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras
-RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras postgresql-dev postgresql-libs
+
+# Install gevent so that we don't need to re-install it when common changes
+RUN pip install gevent
 
 # Install common lib first as it changes less
 COPY common /tmp/common
 RUN pip install /tmp/common && rm -r /tmp/common
 
 # Install actual application
+RUN apk add postgresql-dev postgresql-libs
 COPY thrimshim /tmp/thrimshim
 RUN pip install /tmp/thrimshim && rm -r /tmp/thrimshim