From 731ef9e2d049daa43a507587ce319149d5f84b21 Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Sat, 2 Nov 2019 23:08:55 -0700 Subject: [PATCH 1/2] Refactor dockerfiles for more shared layers By carefully ensuring most of our dockerfiles are identical in their first few layers, we only need to build those layers once instead of every time. In particular, we move installing gevent to before installing common, so that even when common changes gevent doesn't need to be reinstalled. This is important because gevent takes ages to install. Also fixes segment_coverage, which wasn't being installed. --- backfiller/Dockerfile | 6 +++++- build | 2 +- cutter/Dockerfile | 8 +++++--- downloader/Dockerfile | 3 +++ restreamer/Dockerfile | 8 ++++++-- segment_coverage/Dockerfile | 17 +++++++++++------ sheetsync/Dockerfile | 8 +++++--- thrimshim/Dockerfile | 7 +++++-- 8 files changed, 41 insertions(+), 18 deletions(-) diff --git a/backfiller/Dockerfile b/backfiller/Dockerfile index de300f7..5ac8da6 100644 --- a/backfiller/Dockerfile +++ b/backfiller/Dockerfile @@ -1,13 +1,17 @@ FROM alpine:3.7 # dependencies needed for compiling c extensions # also busybox-extras for telnet for easier use of backdoor -RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras postgresql-dev postgresql-libs +RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras + +# Install gevent so that we don't need to re-install it when common changes +RUN pip install gevent # Install common lib first as it changes less COPY common /tmp/common RUN pip install /tmp/common && rm -r /tmp/common # Install actual application +RUN apk add postgresql-dev postgresql-libs COPY backfiller /tmp/backfiller RUN pip install /tmp/backfiller && rm -r /tmp/backfiller diff --git a/build b/build index 42b5497..b4ed219 100755 --- a/build +++ b/build @@ -8,7 +8,7 @@ set -eu # Pass PUSH=true to also push the resulting images, or PUSH=latest to push them as :latest tag # The different images we can build -COMPONENTS=(downloader restreamer backfiller thrimshim cutter sheetsync nginx postgres) +COMPONENTS=(downloader restreamer backfiller thrimshim cutter sheetsync nginx postgres segment_coverage) # Define push if not already defined PUSH=${PUSH:-} diff --git a/cutter/Dockerfile b/cutter/Dockerfile index 5956b8a..29f540b 100644 --- a/cutter/Dockerfile +++ b/cutter/Dockerfile @@ -1,15 +1,17 @@ FROM alpine:3.7 # dependencies needed for compiling c extensions # also busybox-extras for telnet for easier use of backdoor -# and postgresql-dev as a dependency of psycopg2. -# Add postgresql-client for easier debugging of DB issues. -RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras postgresql-dev postgresql-client ffmpeg +RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras + +# Install gevent so that we don't need to re-install it when common changes +RUN pip install gevent # Install common lib first as it changes less COPY common /tmp/common RUN pip install /tmp/common && rm -r /tmp/common # Install actual application +RUN apk add postgresql-dev postgresql-client ffmpeg COPY cutter /tmp/cutter RUN pip install /tmp/cutter && rm -r /tmp/cutter diff --git a/downloader/Dockerfile b/downloader/Dockerfile index c147f80..d6b57ca 100644 --- a/downloader/Dockerfile +++ b/downloader/Dockerfile @@ -3,6 +3,9 @@ FROM alpine:3.7 # also busybox-extras for telnet for easier use of backdoor RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras +# Install gevent so that we don't need to re-install it when common changes +RUN pip install gevent + # Install common lib first as it changes less COPY common /tmp/common RUN pip install /tmp/common && rm -r /tmp/common diff --git a/restreamer/Dockerfile b/restreamer/Dockerfile index 443ff2f..6d3d798 100644 --- a/restreamer/Dockerfile +++ b/restreamer/Dockerfile @@ -1,13 +1,17 @@ FROM alpine:3.7 -# dependencies needed for compiling c extensions, plus ffmpeg for cutting +# dependencies needed for compiling c extensions # also busybox-extras for telnet for easier use of backdoor -RUN apk --update add py2-pip gcc python-dev musl-dev ffmpeg busybox-extras +RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras + +# Install gevent so that we don't need to re-install it when common changes +RUN pip install gevent # Install common lib first as it changes less COPY common /tmp/common RUN pip install /tmp/common && rm -r /tmp/common # Install actual application +RUN apk add ffmpeg COPY restreamer /tmp/restreamer RUN pip install /tmp/restreamer && rm -r /tmp/restreamer diff --git a/segment_coverage/Dockerfile b/segment_coverage/Dockerfile index d7924af..25f07d2 100644 --- a/segment_coverage/Dockerfile +++ b/segment_coverage/Dockerfile @@ -1,18 +1,23 @@ FROM alpine:3.7 # dependencies needed for compiling c extensions # also busybox-extras for telnet for easier use of backdoor -# freetype-dev and libpng-dev are required for matplotlib -RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras freetype-dev libpng-dev build-base libstdc++ -#need to install these manually -RUN ln -s /usr/include/locale.h /usr/include/xlocale.h \ - && pip install numpy \ - && pip install matplotlib +RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras + +# Install gevent so that we don't need to re-install it when common changes +RUN pip install gevent # Install common lib first as it changes less COPY common /tmp/common RUN pip install /tmp/common && rm -r /tmp/common # Install actual application + +# freetype-dev and libpng-dev are required for matplotlib +RUN apk add freetype-dev libpng-dev build-base libstdc++ +# need to install these manually +RUN ln -s /usr/include/locale.h /usr/include/xlocale.h \ + && pip install numpy \ + && pip install matplotlib COPY segment_coverage /tmp/segment_coverage RUN pip install /tmp/segment_coverage && rm -r /tmp/segment_coverage diff --git a/sheetsync/Dockerfile b/sheetsync/Dockerfile index 12e06f5..0190a50 100644 --- a/sheetsync/Dockerfile +++ b/sheetsync/Dockerfile @@ -1,15 +1,17 @@ FROM alpine:3.7 # dependencies needed for compiling c extensions # also busybox-extras for telnet for easier use of backdoor -# and postgresql-dev as a dependency of psycopg2. -# Add postgresql-client for easier debugging of DB issues. -RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras postgresql-dev postgresql-client +RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras + +# Install gevent so that we don't need to re-install it when common changes +RUN pip install gevent # Install common lib first as it changes less COPY common /tmp/common RUN pip install /tmp/common && rm -r /tmp/common # Install actual application +RUN apk add postgresql-dev postgresql-client COPY sheetsync /tmp/sheetsync RUN pip install /tmp/sheetsync && rm -r /tmp/sheetsync diff --git a/thrimshim/Dockerfile b/thrimshim/Dockerfile index 7526c3e..6e3ccb8 100644 --- a/thrimshim/Dockerfile +++ b/thrimshim/Dockerfile @@ -1,14 +1,17 @@ FROM alpine:3.7 # dependencies needed for compiling c extensions # also busybox-extras for telnet for easier use of backdoor -# plus postgres dependenices -RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras postgresql-dev postgresql-libs +RUN apk --update add py2-pip gcc python-dev musl-dev busybox-extras + +# Install gevent so that we don't need to re-install it when common changes +RUN pip install gevent # Install common lib first as it changes less COPY common /tmp/common RUN pip install /tmp/common && rm -r /tmp/common # Install actual application +RUN apk add postgresql-dev postgresql-libs COPY thrimshim /tmp/thrimshim RUN pip install /tmp/thrimshim && rm -r /tmp/thrimshim From 751472217b421a8b57f2cd666e1a7764358dcecb Mon Sep 17 00:00:00 2001 From: Mike Lang Date: Sat, 2 Nov 2019 23:29:23 -0700 Subject: [PATCH 2/2] Default to using the current commit, instead of latest This is generally what you want, and will behave better than using latest. --- build | 8 ++++---- docker-compose.jsonnet | 4 +++- generate-docker-compose | 5 ++++- get-build-tag | 11 +++++++++++ 4 files changed, 22 insertions(+), 6 deletions(-) create mode 100755 get-build-tag diff --git a/build b/build index b4ed219..24e80f0 100755 --- a/build +++ b/build @@ -2,6 +2,9 @@ set -eu +# cd to location of script +cd "$(dirname "$(readlink -f "$0")")" + # Builds the docker images. # Usage: ./build {COMPONENTS}, or just ./build to build all. # The resulting images are named wubloader-COMPONENT. @@ -17,10 +20,7 @@ PUSH=${PUSH:-} BASE="quay.io/ekimekim" # The docker image tag, derived from the git commit + whether working tree is clean -TAG=$(git rev-parse --short HEAD) -if [ -n "$(git status --porcelain --untracked-files=no)" ]; then - TAG="$TAG-wip" -fi +TAG=$(./get-build-tag) if [ "$#" -gt 0 ]; then COMPONENTS=("$@") diff --git a/docker-compose.jsonnet b/docker-compose.jsonnet index 16d3f05..297e5cb 100644 --- a/docker-compose.jsonnet +++ b/docker-compose.jsonnet @@ -7,9 +7,11 @@ // Change these to configure the services. // Image tag (application version) to use. + // By default, will use the current commit, ie. the same thing that ./build would tag + // things it builds with. // Note: "latest" is not recommended in production, as you can't be sure what version // you're actually running, and must manually re-pull to get an updated copy. - image_tag:: "latest", + image_tag:: std.extVar("tag"), // For each service, whether to deploy that service. enabled:: { diff --git a/generate-docker-compose b/generate-docker-compose index 108d190..b30a62a 100755 --- a/generate-docker-compose +++ b/generate-docker-compose @@ -2,9 +2,12 @@ set -eu +# cd to location of script +cd "$(dirname "$(readlink -f "$0")")" + # We generate first, and capture the output, to avoid overwriting the file on error. # To avoid jsonnet needing to exist locally, we run it in a container. -output=$(docker run --rm -i sparkprime/jsonnet - < docker-compose.jsonnet) +output=$(docker run --rm -i sparkprime/jsonnet -V tag="$(./get-build-tag)" - < docker-compose.jsonnet) { echo "# DO NOT EDIT THIS FILE!" diff --git a/get-build-tag b/get-build-tag new file mode 100755 index 0000000..95cc564 --- /dev/null +++ b/get-build-tag @@ -0,0 +1,11 @@ +#!/bin/bash + +# cd to location of script +cd "$(dirname "$(readlink -f "$0")")" + +TAG=$(git rev-parse --short HEAD) +if [ -n "$(git status --porcelain --untracked-files=no)" ]; then + TAG="$TAG-wip" +fi + +echo "$TAG"