build all arch in Azure

.azure-pipelines/templates/stages/deploy-stage.yml

@@ -81,7 +81,7 @@ stages:
               # Certbot organization on Docker Hub.
               containerRegistry: docker-hub
             displayName: Login to Docker Hub
-          - bash: tools/docker/build.sh  ${{ parameters.dockerTag }}
+          - bash: tools/docker/build.sh  ${{ parameters.dockerTag }} all
             displayName: Build the Docker images
           - bash: tools/docker/deploy.sh ${{ parameters.dockerTag }}
             displayName: Deploy the Docker images

tools/docker/README.md

@@ -20,11 +20,12 @@ DNS plugin Docker images to Docker Hub.
 High-level behavior
 -------------------
 
-Running `./build.sh <TAG> && ./deploy.sh <TAG>` causes the Docker images to be
-built and deployed to Docker Hub where `<TAG>` is the base of the tag that
-should be given to the given images. The tag should either be `nightly` or a
-git version tag like `v0.34.0`. The given tag is only the base of the tag
-because the CPU architecture is also added to the tag.
+Running `./build.sh <TAG> all && ./deploy.sh <TAG>` causes the Docker images to
+be built and deployed to Docker Hub for all supported architectures where
+`<TAG>` is the base of the tag that should be given to the given images. The
+tag should either be `nightly` or a git version tag like `v0.34.0`. The given
+tag is only the base of the tag because the CPU architecture is also added to
+the tag.
 
 Configuration
 -------------

tools/docker/build.sh

@@ -3,14 +3,14 @@ set -euxo pipefail
 IFS=$'\n\t'
 
 # This script builds certbot docker and certbot dns plugins docker using the
-# local Certbot files.  The build is currently done following the environment used by
-# Dockerhub since this code previously used Docker Hub's automated build feature.
+# local Certbot files.
 
-# Usage: ./build.sh [TAG]
-#   with [TAG] corresponding the base of the tag to give the Docker images.
-#   Values should be something like `v0.34.0` or `nightly`. The given value is
-#   only the base of the tag because the things like the CPU architecture are
-#   also added to the full tag.
+# Usage: ./build.sh [TAG] [all|amd64|arm32v6|arm64v8]
+#   with the [TAG] value corresponding the base of the tag to give the Docker
+#   images and the 2nd value being the architecture to build snaps for.
+#   Values for the tag should be something like `v0.34.0` or `nightly`. The
+#   given value is only the base of the tag because the things like the CPU
+#   architecture are also added to the full tag.
 
 # As of writing this, runs of this script consistently fail in Azure
 # Pipelines, but they are fixed by using Docker BuildKit. A log of the failures
@@ -38,26 +38,100 @@ Cleanup() {
     done
 }
 
-Build() {
-    DOCKER_REPO="$1"
-    TAG_BASE="$2"
-    CONTEXT_PATH="$3"
-    DOCKERFILE_DIR="$4"
-    DOCKERFILE_PATH="$DOCKERFILE_DIR/Dockerfile"
-    pushd "$CONTEXT_PATH"
-        DOCKER_TAG="$TAG_BASE" DOCKER_REPO="$DOCKER_REPO" DOCKERFILE_PATH="$DOCKERFILE_PATH" bash "$DOCKERFILE_DIR/hooks/pre_build"
-        DOCKER_TAG="$TAG_BASE" DOCKER_REPO="$DOCKER_REPO" DOCKERFILE_PATH="$DOCKERFILE_PATH" bash "$DOCKERFILE_DIR/hooks/build"
-    popd
+# Returns the translation from Docker to QEMU architecture
+# Usage: GetQemuArch [amd64|arm32v6|arm64v8]
+GetQemuArch() {
+    ARCH=$1
+
+    case "$ARCH" in
+        "amd64")
+            echo "x86_64"
+            ;;
+        "arm32v6")
+            echo "arm"
+            ;;
+        "arm64v8")
+            echo "aarch64"
+            ;;
+        "*")
+            echo "Not supported build architecture '$1'." >&2
+            exit 1
+    esac
+}
+
+# Downloads QEMU static binary file for architecture
+# Usage: DownloadQemuStatic [x86_64|arm|aarch64]
+DownloadQemuStatic() {
+    ARCH=$1
+
+    QEMU_ARCH=$(GetQemuArch "$ARCH")
+    if [ ! -f "qemu-${QEMU_ARCH}-static" ]; then
+        QEMU_DOWNLOAD_URL="https://github.com/multiarch/qemu-user-static/releases/download"
+        QEMU_LATEST_TAG=$(curl -s https://api.github.com/repos/multiarch/qemu-user-static/tags \
+            | grep 'name.*v[0-9]' \
+            | head -n 1 \
+            | cut -d '"' -f 4)
+        curl -SL "${QEMU_DOWNLOAD_URL}/${QEMU_LATEST_TAG}/x86_64_qemu-$QEMU_ARCH-static.tar.gz" \
+            | tar xzv
+    fi
+}
+
+# Parses the requested architecture string and sets ALL_REQUESTED_ARCH to
+# result.
+# Usage: ParseRequestedArch [all|amd64|arm32v6|arm64v8]
+ParseRequestedArch() {
+    REQUESTED_ARCH="${1}"
+    if [[ "${REQUESTED_ARCH}" == "all" ]]; then
+        ALL_REQUESTED_ARCH=("${ALL_TARGET_ARCH[@]}")
+        return 0
+    fi
+    for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do
+        if [[ "${TARGET_ARCH}" == "${REQUESTED_ARCH}" ]]; then
+            ALL_REQUESTED_ARCH=("${REQUESTED_ARCH}")
+            return 0
+        fi
+    done
+    # If we didn't return above, REQUESTED_ARCH has an unexpected value.
+    echo "Unexpected target architecture \"${REQUESTED_ARCH}\"". >&2
+    exit 1
 }
 
 TAG_BASE="$1"
+ParseRequestedArch "${2}"
+
+# Register QEMU handlers
+docker run --rm --privileged multiarch/qemu-user-static:register --reset
 
 # Step 1: Certbot core Docker
-Build "$DOCKER_HUB_ORG/certbot" "$TAG_BASE" "$REPO_ROOT" "$WORK_DIR/core"
+DOCKER_REPO="${DOCKER_HUB_ORG}/certbot"
+for TARGET_ARCH in "${ALL_REQUESTED_ARCH[@]}"; do
+    pushd "${REPO_ROOT}"
+    DownloadQemuStatic "${TARGET_ARCH}"
+    QEMU_ARCH=$(GetQemuArch "${TARGET_ARCH}")
+    docker build \
+        --build-arg TARGET_ARCH="${TARGET_ARCH}" \
+        --build-arg QEMU_ARCH="${QEMU_ARCH}" \
+        -f "${WORK_DIR}/core/Dockerfile" \
+        -t "${DOCKER_REPO}:${TARGET_ARCH}-${TAG_BASE}" \
+        .
+    popd
+done
 
 # Step 2: Certbot DNS plugins Docker images
 for plugin in "${CERTBOT_PLUGINS[@]}"; do
-    Build "$DOCKER_HUB_ORG/$plugin" "$TAG_BASE" "$REPO_ROOT/certbot-$plugin" "$WORK_DIR/plugin"
+    DOCKER_REPO="${DOCKER_HUB_ORG}/${plugin}"
+    pushd "${REPO_ROOT}/certbot-${plugin}"
+    # Copy QEMU static binaries downloaded when building the core Certbot image
+    cp ../qemu-*-static .
+    for TARGET_ARCH in "${ALL_REQUESTED_ARCH[@]}"; do
+        QEMU_ARCH=$(GetQemuArch "${TARGET_ARCH}")
+        BASE_IMAGE="${DOCKER_HUB_ORG}/certbot:${TARGET_ARCH}-${TAG_BASE}"
+        docker build \
+            --build-arg BASE_IMAGE="${BASE_IMAGE}" \
+            --build-arg QEMU_ARCH="${QEMU_ARCH}" \
+            -f "${WORK_DIR}/plugin/Dockerfile" \
+            -t "${DOCKER_REPO}:${TARGET_ARCH}-${TAG_BASE}" \
+            .
+    done
+    popd
 done
-
-Cleanup

tools/docker/core/hooks/build

@@ -1,9 +0,0 @@
-#!/bin/bash
-set -ex
-
-WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-source "$WORK_DIR/../../lib/common"
-
-for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do
-    BuildDockerCoreImage "${TARGET_ARCH}"
-done

tools/docker/core/hooks/post_push

@@ -1,10 +0,0 @@
-#!/bin/bash
-set -ex
-
-WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-source "$WORK_DIR/../../lib/common"
-
-for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do
-    TagDockerImageAliases "${TARGET_ARCH}"
-    PushDockerImageAliases "${TARGET_ARCH}"
-done

tools/docker/core/hooks/pre_build

@@ -1,10 +0,0 @@
-#!/bin/bash
-set -ex
-
-WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-source "$WORK_DIR/../../lib/common"
-
-RegisterQemuHandlers
-for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do
-    DownloadQemuStatic "${TARGET_ARCH}"
-done

tools/docker/core/hooks/push

@@ -1,9 +0,0 @@
-#!/bin/bash
-set -ex
-
-WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-source "$WORK_DIR/../../lib/common"
-
-for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do
-    PushDockerImage "${TARGET_ARCH}"
-done

tools/docker/deploy.sh

@@ -3,8 +3,6 @@ set -euxo pipefail
 IFS=$'\n\t'
 
 # This script deploys new versions of Certbot and Certbot plugin docker images.
-# This process is currently done using a similar approach to Docker Hub since
-# this code previously used Docker Hub's automated build feature.
 
 # Usage: ./deploy.sh [TAG]
 #   with [TAG] corresponding the base of the tag to give the Docker images.
@@ -14,22 +12,53 @@ IFS=$'\n\t'
 
 WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
-Deploy() {
-    DOCKER_REPO="$1"
-    TAG_BASE="$2"
-    DOCKERFILE_DIR="$3"
-    DOCKERFILE_PATH="$DOCKERFILE_DIR/Dockerfile"
-    DOCKER_TAG="$TAG_BASE" DOCKER_REPO="$DOCKER_REPO" DOCKERFILE_PATH="$DOCKERFILE_PATH" bash "$DOCKERFILE_DIR/hooks/push"
-    DOCKER_TAG="$TAG_BASE" DOCKER_REPO="$DOCKER_REPO" DOCKERFILE_PATH="$DOCKERFILE_PATH" bash "$DOCKERFILE_DIR/hooks/post_push"
-}
-
 TAG_BASE="$1"  # Eg. v0.35.0 or nightly
 source "$WORK_DIR/lib/common"
 
+# Creates and pushes all Docker images aliases for all architectures.
+# If the value of the global variable TAG_BASE is a version tag such as
+# v0.35.0, the "latest" tag is also updated. Tags without the architecture part
+# are also created for the default architecture.
+# As an example, for amd64 (the default architecture) and the tag v0.35.0, the
+# following tags would be created:
+#  - certbot/certbot:v0.35.0
+#  - certbot/certbot:latest
+#  - certbot/certbot:amd64-latest
+# For the architecture arm32v6 and the tag v0.35.0, only the following tag
+# would be created:
+#  - certbot/certbot:arm32v6-latest
+# For other tags such as "nightly", aliases are only created for the default
+# architecture where the tag "nightly" would be used without an architecture
+# part.
+# Usage: TagAndPush [IMAGE NAME]
+#   where [IMAGE NAME] is the name of the Docker image in the Docker repository
+#   such as "certbot" or "dns-cloudflare".
+# Read globals:
+# * TAG_BASE
+TagAndPushForAllArch() {
+    DOCKER_REPO="${DOCKER_HUB_ORG}/${1}"
+    for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do
+        docker push "${DOCKER_REPO}:${TARGET_ARCH}-${TAG_BASE}"
+
+        if [[ "${TAG_BASE}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            docker tag "${DOCKER_REPO}:${TARGET_ARCH}-${TAG_BASE}" "${DOCKER_REPO}:${TARGET_ARCH}-latest"
+            docker push "${DOCKER_REPO}:${TARGET_ARCH}-latest"
+            if [ "${TARGET_ARCH}" == "${DEFAULT_ARCH}" ]; then
+                docker tag "${DOCKER_REPO}:${TARGET_ARCH}-${TAG_BASE}" "${DOCKER_REPO}:latest"
+                docker push "${DOCKER_REPO}:latest"
+            fi
+        fi
+        if [ "${TARGET_ARCH}" == "${DEFAULT_ARCH}" ]; then
+            docker tag "${DOCKER_REPO}:${TARGET_ARCH}-${TAG_BASE}" "${DOCKER_REPO}:${TAG_BASE}"
+            docker push "${DOCKER_REPO}:${TAG_BASE}"
+        fi
+    done
+}
+
 # Step 1: Certbot core Docker
-Deploy "$DOCKER_HUB_ORG/certbot" "$TAG_BASE" "$WORK_DIR/core"
+TagAndPushForAllArch "certbot"
 
 # Step 2: Certbot DNS plugins Docker images
 for plugin in "${CERTBOT_PLUGINS[@]}"; do
-    Deploy "$DOCKER_HUB_ORG/$plugin" "$TAG_BASE" "$WORK_DIR/plugin"
+    TagAndPushForAllArch "${plugin}"
 done

tools/docker/lib/common

@@ -31,130 +31,3 @@ export CERTBOT_PLUGINS=(
     "dns-linode"
     "dns-sakuracloud"
 )
-
-# Returns the translation from Docker to QEMU architecture
-# Usage: GetQemuArch [amd64|arm32v6|arm64v8]
-GetQemuArch() {
-    ARCH=$1
-
-    case "$ARCH" in
-        "amd64")
-            echo "x86_64"
-            ;;
-        "arm32v6")
-            echo "arm"
-            ;;
-        "arm64v8")
-            echo "aarch64"
-            ;;
-        "*")
-            echo "Not supported build architecture '$1'." >&2
-            exit -1
-    esac
-}
-
-# Downloads QEMU static binary file for architecture
-# Usage: DownloadQemuStatic [x86_64|arm|aarch64]
-DownloadQemuStatic() {
-    ARCH=$1
-
-    QEMU_ARCH=$(GetQemuArch "$ARCH")
-    if [ ! -f "qemu-${QEMU_ARCH}-static" ]; then
-        QEMU_DOWNLOAD_URL="https://github.com/multiarch/qemu-user-static/releases/download"
-        QEMU_LATEST_TAG=$(curl -s https://api.github.com/repos/multiarch/qemu-user-static/tags \
-            | grep 'name.*v[0-9]' \
-            | head -n 1 \
-            | cut -d '"' -f 4)
-        curl -SL "${QEMU_DOWNLOAD_URL}/${QEMU_LATEST_TAG}/x86_64_qemu-$QEMU_ARCH-static.tar.gz" \
-            | tar xzv
-    fi
-}
-
-# Executes the QEMU register script
-# Usage: RegisterQemuHandlers
-RegisterQemuHandlers() {
-    docker run --rm --privileged multiarch/qemu-user-static:register --reset
-}
-
-# Builds docker certbot core image for a specific architecture.
-# Usage: BuildDockerCoreImage [amd64|arm32v6|arm64v8]
-BuildDockerCoreImage() {
-    ARCH=$1
-
-    QEMU=$(GetQemuArch "$ARCH")
-    docker build \
-        --build-arg TARGET_ARCH="${ARCH}" \
-        --build-arg QEMU_ARCH="${QEMU}" \
-        -f "${DOCKERFILE_PATH}" \
-        -t "${DOCKER_REPO}:${ARCH}-${DOCKER_TAG}" \
-        .
-}
-
-# Builds docker certbot plugin image for a specific architecture and tag.
-# Usage: BuildDockerPluginImage [amd64|arm32v6|arm64v8]
-BuildDockerPluginImage() {
-    ARCH=$1
-
-    QEMU=$(GetQemuArch "$ARCH")
-    BASE_IMAGE="${DOCKER_HUB_ORG}/certbot:${TARGET_ARCH}-${DOCKER_TAG}"
-    docker build \
-        --build-arg BASE_IMAGE="${BASE_IMAGE}" \
-        --build-arg QEMU_ARCH="${QEMU}" \
-        -f "${DOCKERFILE_PATH}" \
-        -t "${DOCKER_REPO}:${ARCH}-${DOCKER_TAG}" \
-        .
-}
-
-# Pushes docker image for a specific architecture.
-# Usage: BuildDockerCoreImage [amd64|arm32v6|arm64v8]
-PushDockerImage() {
-    ARCH=$1
-
-    docker push "${DOCKER_REPO}:${ARCH}-${DOCKER_TAG}"
-}
-
-# Creates any docker images aliases for a given architecture.
-# If DOCKER_TAG tag is a version tag such as v0.35.0, the "latest" tag is also
-# updated. In the case of the default architecture, tags without the
-# architecture part are also created.
-# As an example, for amd64 (the default architecture) and the tag v0.35.0, the
-# following tags would be created:
-#  - certbot/certbot:v0.35.0
-#  - certbot/certbot:latest
-#  - certbot/certbot:amd64-latest
-# For the architecture arm32v6 and the tag v0.35.0, only the following tag
-# would be created:
-#  - certbot/certbot:arm32v6-latest
-# For other tags such as "nightly", aliases are only created for the default
-# architecture where the tag "nightly" would be used without an architecture
-# part.
-# Usage: TagDockerImageAliases [amd64|arm32v6|arm64v8]
-TagDockerImageAliases() {
-    ARCH=$1
-
-    if [[ "${DOCKER_TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-        docker tag "${DOCKER_REPO}:${ARCH}-${DOCKER_TAG}" "${DOCKER_REPO}:${ARCH}-latest"
-        if [ "${ARCH}" == "${DEFAULT_ARCH}" ]; then
-            docker tag "${DOCKER_REPO}:${ARCH}-${DOCKER_TAG}" "${DOCKER_REPO}:latest"
-        fi
-    fi
-    if [ "${ARCH}" == "${DEFAULT_ARCH}" ]; then
-        docker tag "${DOCKER_REPO}:${ARCH}-${DOCKER_TAG}" "${DOCKER_REPO}:${DOCKER_TAG}"
-    fi
-}
-
-# Pushes docker images aliases created by the TagDockerImageAliases function.
-# Usage: PushDockerImageAliases [amd64|arm32v6|arm64v8]
-PushDockerImageAliases() {
-    ARCH=$1
-
-    if [[ "${DOCKER_TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-        docker push "${DOCKER_REPO}:${ARCH}-latest"
-        if [ "${ARCH}" == "${DEFAULT_ARCH}" ]; then
-            docker push "${DOCKER_REPO}:latest"
-        fi
-    fi
-    if [ "${ARCH}" == "${DEFAULT_ARCH}" ]; then
-        docker push "${DOCKER_REPO}:${DOCKER_TAG}"
-    fi
-}

tools/docker/plugin/hooks/build

@@ -1,9 +0,0 @@
-#!/bin/bash
-set -ex
-
-WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-source "$WORK_DIR/../../lib/common"
-
-for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do
-    BuildDockerPluginImage "${TARGET_ARCH}"
-done

tools/docker/plugin/hooks/post_push

@@ -1,10 +0,0 @@
-#!/bin/bash
-set -ex
-
-WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-source "$WORK_DIR/../../lib/common"
-
-for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do
-    TagDockerImageAliases "${TARGET_ARCH}"
-    PushDockerImageAliases "${TARGET_ARCH}"
-done

tools/docker/plugin/hooks/pre_build

@@ -1,10 +0,0 @@
-#!/bin/bash
-set -ex
-
-WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-source "$WORK_DIR/../../lib/common"
-
-RegisterQemuHandlers
-for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do
-    DownloadQemuStatic "${TARGET_ARCH}"
-done

tools/docker/plugin/hooks/push

@@ -1,9 +0,0 @@
-#!/bin/bash
-set -ex
-
-WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-source "$WORK_DIR/../../lib/common"
-
-for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do
-    PushDockerImage "${TARGET_ARCH}"
-done