docker: use multi-stages dockerfiles

The image we currently produce is over 4GB large. In order to reduce
this, we perform the compilation in a build container, and just copy
the coordinator and aggregator binaries to the final image.

This commit also removes the dependency vendoring. The main reason for
having it was to cache the dependencies. But that doesn't cache the
builds. Instead we use `cargo fetch` and `cargo build` on an empty
project that contains all our dependencies to force cargo to cache
them.

Finally, we split the images:

- an image to upload to docker hub, that contains a release build
- an image for development that contains a debug build as well as some
  dev tools (Valgrind)

docker/Dockerfile

@@ -1,53 +1,34 @@
-FROM python:3.7.6-slim-buster
-
-ARG RELEASE_BUILD=false
+FROM python:3.7.6-slim-buster AS builder
 
-# Install rust nightly
+# Install Rust nightly
 ENV RUSTUP_HOME=/usr/local/rustup \
     CARGO_HOME=/usr/local/cargo \
     PATH=/usr/local/cargo/bin:$PATH
-
-RUN set -eux; \
-    apt-get update; \
-    apt-get install -y --no-install-recommends \
-        ca-certificates \
-        gcc \
-        libc6-dev \
-        wget \
-        libssl-dev \
-        pkg-config \
-        ; \
-    \
-    url="https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init"; \
-    wget "$url"; \
-    chmod +x rustup-init; \
-    ./rustup-init -y --no-modify-path --default-toolchain nightly; \
-    rm rustup-init; \
-    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
-    rustup --version; \
-    cargo --version; \
-    rustc --version; \
-    \
-    apt-get remove -y --auto-remove \
-        wget \
-        ; \
-    rm -rf /var/lib/apt/lists/*;
+COPY docker/install_rust.sh .
+RUN bash ./install_rust.sh
 
 # Install dev tools
-RUN apt-get update; \
-    apt-get install -y valgrind \
-    ;
+RUN apt-get update && apt-get install -y valgrind
 
-WORKDIR /usr/src/coordinator
-# https://benjamincongdon.me/blog/2019/12/04/Fast-Rust-Docker-Builds-with-cargo-vendor/
+# First, fetch and build all the dependencies by compiling an empty
+# crate that depends on everything
 COPY rust/Cargo.lock .
 COPY rust/Cargo.toml .
-RUN mkdir .cargo
-RUN cargo vendor > .cargo/config
+RUN cargo fetch
+RUN mkdir -p ./src && \
+    touch src/lib.rs && \
+    cargo build --lib --all-features
 
+# Now copy the actual source code
 COPY rust/src src
-COPY ./docker/compile.sh .
-RUN bash compile.sh
+# Fix timestamp. cargo incremental build as issues caused by the above
+# trick to compile the deps separately
+RUN touch src/lib.rs
+RUN cargo build --all-features
+
+FROM python:3.7.6-slim-buster
 
+COPY --from=builder /target/debug/aggregator /bin/aggregator
+COPY --from=builder /target/debug/coordinator /bin/coordinator
 COPY python/aggregators aggregators/
 RUN pip install aggregators/

docker/Dockerfile.release

@@ -0,0 +1,21 @@
+FROM python:3.7.6-slim-buster AS builder
+
+# Install Rust nightly
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH
+COPY docker/install_rust.sh .
+RUN bash ./install_rust.sh
+
+COPY rust/Cargo.lock .
+COPY rust/Cargo.toml .
+COPY rust/src src
+RUN cargo build --release --features=telemetry,influx_metrics
+
+
+FROM python:3.7.6-slim-buster
+
+COPY --from=builder /target/release/aggregator /bin/aggregator
+COPY --from=builder /target/release/coordinator /bin/coordinator
+COPY python/aggregators aggregators/
+RUN pip install aggregators/

docker/docker-compose-release.yml

@@ -5,9 +5,7 @@ services:
     command: coordinator -c /bin/config.toml
     build:
       context: ..
-      dockerfile: docker/Dockerfile
-      args:
-          RELEASE_BUILD: "true"
+      dockerfile: docker/Dockerfile.release
     image: docker_coordinator:release
     depends_on:
       - influxdb
@@ -23,7 +21,7 @@ services:
     command: aggregator -c /bin/config.toml
     build:
       context: ..
-      dockerfile: docker/Dockerfile
+      dockerfile: docker/Dockerfile.release
       args:
           RELEASE_BUILD: "true"
     image: docker_aggregator:release

docker/install_rust.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -eux
+
+apt-get update
+apt-get install -y --no-install-recommends \
+        ca-certificates \
+        gcc \
+        libc6-dev \
+        wget \
+        libssl-dev \
+        pkg-config \
+
+wget "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init"
+
+chmod +x rustup-init
+
+./rustup-init -y --no-modify-path --default-toolchain nightly
+chmod -R a+w $RUSTUP_HOME $CARGO_HOME
+
+rustup --version
+cargo --version
+rustc --version