test docker only

Co-authored-by: Brad Warren <bmw@users.noreply.github.com>

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -4,12 +4,12 @@ jobs:
       vmImage: ubuntu-22.04
     strategy:
       matrix:
-        amd64:
-          DOCKER_ARCH: amd64
         arm32v6:
           DOCKER_ARCH: arm32v6
         arm64v8:
           DOCKER_ARCH: arm64v8
+        amd64:
+          DOCKER_ARCH: amd64
     # The default timeout of 60 minutes is a little low for compiling
     # cryptography on ARM architectures.
     timeoutInMinutes: 180
@@ -32,183 +32,26 @@ jobs:
           path: $(Build.ArtifactStagingDirectory)
           artifact: docker_$(DOCKER_ARCH)
         displayName: Store Docker artifact
-  - job: docker_run
+  - job: docker_test
     dependsOn: docker_build
     pool:
       vmImage: ubuntu-22.04
+    strategy:
+      matrix:
+        arm32v6:
+          DOCKER_ARCH: arm32v6
+        arm64v8:
+          DOCKER_ARCH: arm64v8
+        amd64:
+          DOCKER_ARCH: amd64
     steps:
       - task: DownloadPipelineArtifact@2
         inputs:
-          artifact: docker_amd64
+          artifact: docker_$(DOCKER_ARCH)
           path: $(Build.SourcesDirectory)
         displayName: Retrieve Docker images
       - bash: set -e && docker load --input $(Build.SourcesDirectory)/images.tar
         displayName: Load Docker images
       - bash: |
-          set -ex
-          DOCKER_IMAGES=$(docker images --filter reference='*/certbot' --filter reference='*/dns-*' --format '{{.Repository}}:{{.Tag}}')
-          for DOCKER_IMAGE in ${DOCKER_IMAGES}
-            do docker run --rm "${DOCKER_IMAGE}" plugins --prepare
-          done
+          set -e && tools/docker/test.sh $(dockerTag) $DOCKER_ARCH
         displayName: Run integration tests for Docker images
-  - job: installer_build
-    pool:
-      vmImage: windows-2019
-    steps:
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: 3.9
-          architecture: x64
-          addToPath: true
-      - script: |
-          python -m venv venv
-          venv\Scripts\python tools\pip_install.py -e windows-installer
-        displayName: Prepare Windows installer build environment
-      - script: |
-          venv\Scripts\construct-windows-installer
-        displayName: Build Certbot installer
-      - task: CopyFiles@2
-        inputs:
-          sourceFolder: $(System.DefaultWorkingDirectory)/windows-installer/build/nsis
-          contents: '*.exe'
-          targetFolder: $(Build.ArtifactStagingDirectory)
-      - task: PublishPipelineArtifact@1
-        inputs:
-          path: $(Build.ArtifactStagingDirectory)
-          # If we change the artifact's name, it should also be changed in tools/create_github_release.py
-          artifact: windows-installer
-        displayName: Publish Windows installer
-  - job: installer_run
-    dependsOn: installer_build
-    strategy:
-      matrix:
-        win2019:
-          imageName: windows-2019
-    pool:
-      vmImage: $(imageName)
-    steps:
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: 3.9
-          addToPath: true
-      - task: DownloadPipelineArtifact@2
-        inputs:
-          artifact: windows-installer
-          path: $(Build.SourcesDirectory)/bin
-        displayName: Retrieve Windows installer
-      - script: |
-          python -m venv venv
-          venv\Scripts\python tools\pip_install.py -e certbot-ci
-        env:
-          PIP_NO_BUILD_ISOLATION: no
-        displayName: Prepare Certbot-CI
-      - script: |
-          set PATH=%ProgramFiles%\Certbot\bin;%PATH%
-          venv\Scripts\python -m pytest certbot-ci\windows_installer_integration_tests --allow-persistent-changes --installer-path $(Build.SourcesDirectory)\bin\certbot-beta-installer-win_amd64.exe
-        displayName: Run windows installer integration tests
-      - script: |
-          set PATH=%ProgramFiles%\Certbot\bin;%PATH%
-          venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
-        displayName: Run certbot integration tests
-  - job: snaps_build
-    pool:
-      vmImage: ubuntu-22.04
-    strategy:
-      matrix:
-        amd64:
-          SNAP_ARCH: amd64
-        armhf:
-          SNAP_ARCH: armhf
-        arm64:
-          SNAP_ARCH: arm64
-    timeoutInMinutes: 0
-    steps:
-      - script: |
-          set -e
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends snapd
-          sudo snap install --classic snapcraft
-        displayName: Install dependencies
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: 3.8
-          addToPath: true
-      - task: DownloadSecureFile@1
-        name: credentials
-        inputs:
-          secureFile: launchpad-credentials
-      - script: |
-          set -e
-          git config --global user.email "$(Build.RequestedForEmail)"
-          git config --global user.name "$(Build.RequestedFor)"
-          mkdir -p ~/.local/share/snapcraft/provider/launchpad
-          cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials
-          python3 tools/snap/build_remote.py ALL --archs ${SNAP_ARCH} --timeout $(snapBuildTimeout)
-        displayName: Build snaps
-      - script: |
-          set -e
-          mv *.snap $(Build.ArtifactStagingDirectory)
-          mv certbot-dns-*/*.snap $(Build.ArtifactStagingDirectory)
-        displayName: Prepare artifacts
-      - task: PublishPipelineArtifact@1
-        inputs:
-          path: $(Build.ArtifactStagingDirectory)
-          artifact: snaps_$(SNAP_ARCH)
-        displayName: Store snaps artifacts
-  - job: snap_run
-    dependsOn: snaps_build
-    pool:
-      vmImage: ubuntu-22.04
-    steps:
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: 3.8
-          addToPath: true
-      - script: |
-          set -e
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends nginx-light snapd
-          python3 -m venv venv
-          venv/bin/python tools/pip_install.py -U tox
-        displayName: Install dependencies
-      - task: DownloadPipelineArtifact@2
-        inputs:
-          artifact: snaps_amd64
-          path: $(Build.SourcesDirectory)/snap
-        displayName: Retrieve Certbot snaps
-      - script: |
-          set -e
-          sudo snap install --dangerous --classic snap/certbot_*.snap
-        displayName: Install Certbot snap
-      - script: |
-          set -e
-          venv/bin/python -m tox -e integration-external,apacheconftest-external-with-pebble
-        displayName: Run tox
-  - job: snap_dns_run
-    dependsOn: snaps_build
-    pool:
-      vmImage: ubuntu-22.04
-    steps:
-      - script: |
-          set -e
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends snapd
-        displayName: Install dependencies
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: 3.8
-          addToPath: true
-      - task: DownloadPipelineArtifact@2
-        inputs:
-          artifact: snaps_amd64
-          path: $(Build.SourcesDirectory)/snap
-        displayName: Retrieve Certbot snaps
-      - script: |
-          set -e
-          python3 -m venv venv
-          venv/bin/python tools/pip_install.py -e certbot-ci
-        displayName: Prepare Certbot-CI
-      - script: |
-          set -e
-          sudo -E venv/bin/pytest certbot-ci/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap --snap-arch amd64
-        displayName: Test DNS plugins snaps

.azure-pipelines/templates/stages/deploy-stage.yml

@@ -32,12 +32,12 @@ stages:
           vmImage: ubuntu-22.04
         strategy:
           matrix:
-            amd64:
-              DOCKER_ARCH: amd64
             arm32v6:
               DOCKER_ARCH: arm32v6
             arm64v8:
               DOCKER_ARCH: arm64v8
+            amd64:
+              DOCKER_ARCH: amd64
         steps:
           - task: DownloadPipelineArtifact@2
             inputs:
@@ -51,7 +51,7 @@ stages:
               command: login
               containerRegistry: docker-hub
             displayName: Login to Docker Hub
-          - bash: set -e && tools/docker/deploy_by_arch.sh $(dockerTag) $DOCKER_ARCH
+          - bash: set -e && tools/docker/deploy_images.sh $(dockerTag) $DOCKER_ARCH
             displayName: Deploy the Docker images by architecture
       - job: publish_docker_multiarch
         dependsOn: publish_docker_by_arch
@@ -63,5 +63,5 @@ stages:
               command: login
               containerRegistry: docker-hub
             displayName: Login to Docker Hub
-          - bash: set -e && tools/docker/deploy_multiarch.sh $(dockerTag)
+          - bash: set -e && tools/docker/deploy_manifests.sh $(dockerTag) all
             displayName: Deploy the Docker multiarch manifests

.azure-pipelines/templates/stages/test-and-package-stage.yml

@@ -1,6 +1,4 @@
 stages:
   - stage: TestAndPackage
     jobs:
-      - template: ../jobs/standard-tests-jobs.yml
-      - template: ../jobs/extended-tests-jobs.yml
       - template: ../jobs/packaging-jobs.yml

tools/docker/Dockerfile

@@ -1,11 +1,5 @@
-# Docker Arch (amd64, arm32v6, ...)
-ARG TARGET_ARCH
-FROM ${TARGET_ARCH}/python:3.10-alpine3.16
-
-# Qemu Arch (x86_64, arm, ...)
-ARG QEMU_ARCH
-ENV QEMU_ARCH=${QEMU_ARCH}
-COPY qemu-${QEMU_ARCH}-static /usr/bin/
+#base image 
+FROM python:3.10-alpine3.16 as certbot
 
 ENTRYPOINT [ "certbot" ]
 EXPOSE 80 443
@@ -45,3 +39,11 @@ RUN apk add --no-cache --virtual .build-deps \
             --editable src/certbot \
     && apk del .build-deps \
     && rm -rf ${HOME}/.cargo
+
+#static definition for making a plugin, but beware that
+#using this layer definition will cause collisions if you make
+#extensive use of the cache.
+FROM certbot as certbot-plugin
+COPY --from=plugin-src . /opt/certbot/src/plugin
+RUN python tools/pip_install.py --no-cache-dir --editable /opt/certbot/src/plugin
+

tools/docker/README.md

@@ -21,22 +21,29 @@ High-level behavior
 -------------------
 
 Running `./build.sh <TAG> all` causes the Docker images to be built for all 
-supported architectures, where `<TAG>` is the base of the tag that should be 
-given to the generated images. The tag should either be `nightly` or a git 
-version tag like `v2.2.0`. The given tag is only the base of the tag because 
-the CPU architecture is also added to the tag. For version tags above `v2.0.0`,
-Additional tags for `latest` are also generated. The generated images are stored 
-in the local docker image cache.
+supported architectures. The generated images are stored in the local docker image cache.
 
-Running `./deploy_by_arch.sh <TAG> all && ./deploy_multiarch.sh <TAG>` will 
-push the previously generated images to Docker Hub and then generate multi-arch
-manifests for easy access to the underlying images appropriate for a given 
-architecture.
+Running `./test.sh <TAG> all` loads images from the docker image cache
+and runs a test command to validate the image contents.
+
+Running `./deploy_images.sh <TAG> all` will push the previously generated images 
+to Docker Hub.  The <TAG> argument is an identifier applied to all docker 
+images and manifests. It may be something like `nightly` or `v2.3.2`. If 
+the tag is a version stamp greater than `v2.0.0`, then a `latest` tag will 
+also be generated and pushed to the docker hub repo. 
+
+Running `./deploy_manifests.sh <TAG> all` will add multiarch manifests to 
+Docker Hub. This command assumes that `./deploy_images.sh <TAG> all` has
+been previously run with the same tag.
 
 Configuration
 -------------
 
 To run these scripts you need:
 
-1. An x86_64 machine with Docker installed and the Docker daemon running. You probably don't want to use the docker snap as these scripts have failed when using that in the past.
-2. To be logged into Docker Hub with an account able to push to the Certbot and Certbot DNS Docker images on Docker Hub. Altering the value of `DOCKER_HUB_ORG` in `lib/common` will allow you to push to your own account for testing.
+1. A computer with Docker installed and the Docker daemon running. You probably 
+don't want to use the docker snap as these scripts have failed when using that 
+in the past.
+2. To be logged into Docker Hub with an account able to push to the Certbot and 
+Certbot DNS Docker images on Docker Hub. Altering the value of `DOCKER_HUB_ORG` 
+in `lib/common` will allow you to push to your own account for testing.

tools/docker/build.sh

@@ -1,108 +1,58 @@
 #!/bin/bash
 set -euxo pipefail
-IFS=$'\n\t'
 
-# This script builds certbot docker and certbot dns plugins docker using the
-# local Certbot files.
+# This script builds docker images for certbot and each dns plugin from the
+# local Certbot source files. Results are stored in the docker image cache
 
-# Usage: ./build.sh [TAG] [all|amd64|arm32v6|arm64v8]
-#   with the [TAG] value corresponding the base of the tag to give the Docker
-#   images and the 2nd value being the architecture to build snaps for.
-#   Values for the tag should be something like `v0.34.0` or `nightly`. The
-#   given value is only the base of the tag because the things like the CPU
-#   architecture are also added to the full tag.
+# Usage: 
+#       ./build.sh <tag> all 
+#       ./build.sh <tag> <architectures> 
+#   The <tag> argument is used to identify the code version (e.g v2.3.1) or type of build
+#   (e.g. nightly). This will be used when saving images to the docker image cache.
+#   The argument "all" will build all known architectures. Alternatively, the
+#   user may provide a comma separated list of architectures drawn from the
+#   known architectures. Known architectures include amd64, arm32v6, and arm64v8.
 
-WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-REPO_ROOT="$(dirname "$(dirname "${WORK_DIR}")")"
-source "$WORK_DIR/lib/common"
+source "$(realpath "$(dirname "${BASH_SOURCE[0]}")")/lib/common"
 
+ParseArgs "$@"
+
+#jump to root, matching popd handed by Cleanup on EXIT via trap
+pushd "${REPO_ROOT}"
+
+# Set trap here, as the popd won't work as expected if invoked prior to pushd
 trap Cleanup EXIT
+# Create the builder
+CreateBuilder
+InstallMultiarchSupport
 
-Cleanup() {
-    rm -rf "$REPO_ROOT"/qemu-*-static || true
-    for plugin in "${CERTBOT_PLUGINS[@]}"; do
-        rm -rf "$REPO_ROOT/certbot-$plugin"/qemu-*-static || true
-    done
-}
 
-# Returns the translation from Docker to QEMU architecture
-# Usage: GetQemuArch [amd64|arm32v6|arm64v8]
-GetQemuArch() {
-    ARCH=$1
-
-    case "$ARCH" in
-        "amd64")
-            echo "x86_64"
-            ;;
-        "arm32v6")
-            echo "arm"
-            ;;
-        "arm64v8")
-            echo "aarch64"
-            ;;
-        "*")
-            echo "Not supported build architecture '$1'." >&2
-            exit 1
-    esac
-}
-
-# Downloads QEMU static binary file for architecture
-# Usage: DownloadQemuStatic [x86_64|arm|aarch64]
-DownloadQemuStatic() {
-    ARCH=$1
-
-    QEMU_ARCH=$(GetQemuArch "$ARCH")
-    if [ ! -f "qemu-${QEMU_ARCH}-static" ]; then
-        QEMU_DOWNLOAD_URL="https://github.com/multiarch/qemu-user-static/releases/download"
-        QEMU_LATEST_TAG=$(curl -s https://api.github.com/repos/multiarch/qemu-user-static/tags \
-            | grep 'name.*v[0-9]' \
-            | head -n 1 \
-            | cut -d '"' -f 4)
-        curl -SL "${QEMU_DOWNLOAD_URL}/${QEMU_LATEST_TAG}/x86_64_qemu-$QEMU_ARCH-static.tar.gz" \
-            | tar xzv
-    fi
-}
-
-TAG_BASE="$1"
-if [ -z "$TAG_BASE" ]; then
-    echo "We cannot tag Docker images with an empty string!" >&2
-    exit 1
-fi
-ParseRequestedArch "${2}"
-
-# Register QEMU handlers
-docker run --rm --privileged multiarch/qemu-user-static:register --reset
-
-# Step 1: Certbot core Docker
-DOCKER_REPO="${DOCKER_HUB_ORG}/certbot"
-for TARGET_ARCH in "${ALL_REQUESTED_ARCH[@]}"; do
-    pushd "${REPO_ROOT}"
-    DownloadQemuStatic "${TARGET_ARCH}"
-    QEMU_ARCH=$(GetQemuArch "${TARGET_ARCH}")
-    DOCKER_BUILDKIT=0 docker build \
-        --build-arg TARGET_ARCH="${TARGET_ARCH}" \
-        --build-arg QEMU_ARCH="${QEMU_ARCH}" \
-        -f "${WORK_DIR}/core/Dockerfile" \
-        -t "${DOCKER_REPO}:${TARGET_ARCH}-${TAG_BASE}" \
+BuildAndLoadByArch() {
+    TAG_ARCH=$1
+    docker buildx build --target certbot --builder certbot_builder \
+        --platform "$(arch2platform "$TAG_ARCH")" \
+        -f "${WORK_DIR}/Dockerfile" \
+        -t "${DOCKER_HUB_ORG}/certbot:${TAG_ARCH}-${TAG_VER}" \
+        --load \
         .
-    popd
-done
-
-# Step 2: Certbot DNS plugins Docker images
-for plugin in "${CERTBOT_PLUGINS[@]}"; do
-    DOCKER_REPO="${DOCKER_HUB_ORG}/${plugin}"
-    pushd "${REPO_ROOT}/certbot-${plugin}"
-    # Copy QEMU static binaries downloaded when building the core Certbot image
-    cp ../qemu-*-static .
-    for TARGET_ARCH in "${ALL_REQUESTED_ARCH[@]}"; do
-        QEMU_ARCH=$(GetQemuArch "${TARGET_ARCH}")
-        BASE_IMAGE="${DOCKER_HUB_ORG}/certbot:${TARGET_ARCH}-${TAG_BASE}"
-        DOCKER_BUILDKIT=0 docker build \
-            --build-arg BASE_IMAGE="${BASE_IMAGE}" \
-            --build-arg QEMU_ARCH="${QEMU_ARCH}" \
-            -f "${WORK_DIR}/plugin/Dockerfile" \
-            -t "${DOCKER_REPO}:${TARGET_ARCH}-${TAG_BASE}" \
+    for plugin in "${CERTBOT_PLUGINS[@]}"; do
+        docker buildx build --target certbot-plugin --builder certbot_builder \
+            --platform "$(arch2platform "$TAG_ARCH")" \
+            --build-context plugin-src="${REPO_ROOT}/certbot-${plugin}" \
+            -f "${WORK_DIR}/Dockerfile" \
+            -t "${DOCKER_HUB_ORG}/${plugin}:${TAG_ARCH}-${TAG_VER}" \
+            --load \
             .
     done
-    popd
-done
+}
+
+# In principle, there is a better way to do with by using `docker buildx bake`
+# instead of a for-loop. However, issues have been found in the results
+# of such a build. See the branch buildx-bake and
+# https://github.com/certbot/certbot/issues/9587.
+
+for ARCH in "${REQUESTED_ARCH_ARRAY[@]}"; do
+    BuildAndLoadByArch "$ARCH"
+done    
+
+

tools/docker/deploy_by_arch.sh

@@ -1,64 +0,0 @@
-#!/bin/bash
-set -euxo pipefail
-IFS=$'\n\t'
-
-# This script deploys new versions of Certbot and Certbot plugin docker images.
-
-# Usage: ./deploy_by_arch.sh [TAG] [all|amd64|arm32v6|arm64v8]
-#   with the [TAG] value corresponding the base of the tag to give the Docker
-#   images and the 2nd value being the architecture to build snaps for.
-#   Values should be something like `v0.34.0` or `nightly`. The given value is
-#   only the base of the tag because the things like the CPU architecture are
-#   also added to the full tag.
-
-WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-
-TAG_BASE="$1"  # Eg. v0.35.0 or nightly
-if [ -z "$TAG_BASE" ]; then
-    echo "We cannot tag Docker images with an empty string!" >&2
-    exit 1
-fi
-source "$WORK_DIR/lib/common"
-ParseRequestedArch "${2}"
-
-# Creates and pushes all Docker images aliases for the requested architectures
-# set in the environment variable ALL_REQUESTED_ARCH.  If the value of the
-# global variable TAG_BASE is a 2.0.0 or greater version tag such as v2.1.0,
-# tags for "latest" are also created. Tags such as "nightly" do not recieve 
-# "latest" tags.
-# As an example, for the tag v2.2.0 and the default set of all target 
-# architectures as of writing this, the following tags would be created:
-#  - certbot/certbot:amd64-v2.2.0       <- image
-#  - certbot/certbot:arm32v6-v2.2.0     <- image
-#  - certbot/certbot:arm64v8-v2.2.0     <- image
-#  - certbot/certbot:amd64-latest       <- image
-#  - certbot/certbot:arm32v6-latest     <- image
-#  - certbot/certbot:arm64v8-latest     <- image
-# Usage: TagAndPushForAllRequestedArch [IMAGE NAME]
-#   where [IMAGE NAME] is the name of the Docker image in the Docker repository
-#   such as "certbot" or "dns-cloudflare".
-# Read globals:
-# * TAG_BASE
-# * ALL_REQUESTED_ARCH
-TagAndPushForAllRequestedArch() {
-    DOCKER_REPO="${DOCKER_HUB_ORG}/${1}"
-    for TARGET_ARCH in "${ALL_REQUESTED_ARCH[@]}"; do
-        # NOTE: In early 2022, we were experiencing regular "docker push"
-        # timeouts, so we added these "--debug" flags to learn more. Since we
-        # added them, we haven't had another timeout, so until we experience
-        # another timeout & can get the deubg logs, we're leaving them in.
-        docker --debug push "${DOCKER_REPO}:${TARGET_ARCH}-${TAG_BASE}"
-        if [[ "${TAG_BASE}" =~ ^v([2-9]|[1-9][0-9]+)\.[0-9]+\.[0-9]+$ ]]; then
-            docker tag "${DOCKER_REPO}:${TARGET_ARCH}-${TAG_BASE}" "${DOCKER_REPO}:${TARGET_ARCH}-latest"
-            docker --debug push "${DOCKER_REPO}:${TARGET_ARCH}-latest"
-        fi
-    done
-}
-
-# Step 1: Certbot core Docker
-TagAndPushForAllRequestedArch "certbot"
-
-# Step 2: Certbot DNS plugins Docker images
-for plugin in "${CERTBOT_PLUGINS[@]}"; do
-    TagAndPushForAllRequestedArch "${plugin}"
-done

tools/docker/deploy_images.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+set -euxo pipefail
+
+# This script takes docker images in the local docker cache and pushes them to
+# Docker Hub.
+
+# Usage: 
+#       ./deploy_images.sh <TAG> all
+#       ./deploy_images.sh <TAG> <architectures>
+#   The <TAG> argument is an identifier applied to all docker images and manifests.
+#   It may be something like `nightly` or `v2.3.2`. If the tag is a version
+#   stamp greater than v2.0.0, then a `latest` tag will also be generated and
+#   pushed to the docker hub repo.
+#   The argument "all" will push all known architectures. Alternatively, the
+#   user may provide a comma separated list of architectures drawn from the
+#   known architectures. Known architectures include amd64, arm32v6, and arm64v8.
+
+source "$(realpath $(dirname "${BASH_SOURCE[0]}"))/lib/common"
+
+ParseArgs "$@"
+
+#jump to root, matching popd handed by Cleanup on EXIT via trap
+pushd "${REPO_ROOT}"
+
+# Set trap here, as the popd won't work as expected if invoked prior to pushd
+trap popd EXIT
+
+REGISTRY_SPEC="${DOCKER_HUB_ORG}/"
+
+DeployImage() {
+    IMAGE_NAME=$1
+    TAG_ARCH=$2
+    docker push "${REGISTRY_SPEC}${IMAGE_NAME}:${TAG_ARCH}-${TAG_VER}"
+    if [[ "${TAG_VER}" =~ ^v([2-9]|[1-9][0-9]+)\.[0-9]+\.[0-9]+$ ]]; then
+        docker tag "${REGISTRY_SPEC}${IMAGE_NAME}:${TAG_ARCH}-${TAG_VER}" "${REGISTRY_SPEC}${IMAGE_NAME}:${TAG_ARCH}-latest"
+        docker push "${REGISTRY_SPEC}${IMAGE_NAME}:${TAG_ARCH}-latest"
+    fi
+}
+
+
+for TAG_ARCH in "${REQUESTED_ARCH_ARRAY[@]}"; do
+    DeployImage certbot "$TAG_ARCH"
+    for PLUGIN in "${CERTBOT_PLUGINS[@]}"; do
+        DeployImage "$PLUGIN" "$TAG_ARCH"
+    done
+done
+
+

tools/docker/deploy_manifests.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -euxo pipefail
+
+# This script generates multi-arch manifests for images previously pushed to
+# Docker Hub via deploy_images.sh
+
+# Usage: 
+#       ./deploy_manifest.sh <TAG> all
+#       ./deploy_manifest.sh <TAG> <architectures>
+#   The <TAG> argument is an identifier applied to all docker images and manifests.
+#   It may be something like `nightly` or `v2.3.2`. If the tag is a version
+#   stamp greater than v2.0.0, then a `latest` tag will also be generated and
+#   pushed to the docker hub repo.
+#   The argument "all" will push all know architectures. Alternatively, the
+#   user may provide a comma separated list of architectures drawn from the
+#   known architectures. Know architectures include amd64, arm32v6, and arm64v8.
+
+
+source "$(realpath $(dirname ${BASH_SOURCE[0]}))/lib/common"
+
+ParseArgs "$@"
+
+#jump to root, matching popd handed by Cleanup on EXIT via trap
+pushd "${REPO_ROOT}"
+
+# Set trap here, as the popd won't work as expected if invoked prior to pushd
+trap popd EXIT
+
+REGISTRY_SPEC="${DOCKER_HUB_ORG}/"
+
+DeployManifest() {
+    IMAGE_NAME=$1
+    
+    SRC_IMAGES=""
+    for TAG_ARCH in "${REQUESTED_ARCH_ARRAY[@]}"; do
+        SRC_IMAGES+="${REGISTRY_SPEC}${IMAGE_NAME}:${TAG_ARCH}-${TAG_VER} "
+    done
+    docker buildx imagetools create -t "${REGISTRY_SPEC}${IMAGE_NAME}:${TAG_VER}" "$SRC_IMAGES"
+
+    if [[ "${TAG_VER}" =~ ^v([2-9]|[1-9][0-9]+)\.[0-9]+\.[0-9]+$ ]]; then
+        docker buildx imagetools create -t "${REGISTRY_SPEC}${IMAGE_NAME}:latest" "$SRC_IMAGES"
+    fi
+}
+
+DeployManifest certbot
+for PLUGIN in "${CERTBOT_PLUGINS[@]}"; do
+    DeployManifest "$PLUGIN"
+done
+
+
+
+

tools/docker/deploy_multiarch.sh

@@ -1,36 +0,0 @@
-#!/bin/bash
-set -euxo pipefail
-IFS=$'\n\t'
-
-WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-
-TAG_BASE="$1"  # Eg. v0.35.0 or nightly
-if [ -z "$TAG_BASE" ]; then
-    echo "We cannot tag Docker images with an empty string!" >&2
-    exit 1
-fi
-source "$WORK_DIR/lib/common"
-
-# Creates multiarch manifests for TAG_BASE, and 'latest' if TAG_BASE > 2.0.0
-#  - certbot/certbot:v2.2.0             <- multiarch manifest 
-#  - certbot/certbot:latest             <- multiarch manifest
-MakeMultiarchManifestForAllTargetArch() {
-    DOCKER_REPO="${DOCKER_HUB_ORG}/${1}"
-    SRC_IMAGES=()
-    for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do
-        SRC_IMAGES+=("${DOCKER_REPO}:${TARGET_ARCH}-${TAG_BASE}")
-    done
-    docker buildx imagetools create -t ${DOCKER_REPO}:${TAG_BASE} "${SRC_IMAGES[@]}"
-    if [[ "${TAG_BASE}" =~ ^v([2-9]|[1-9][0-9]+)\.[0-9]+\.[0-9]+$ ]]; then
-        docker buildx imagetools create -t ${DOCKER_REPO}:latest "${SRC_IMAGES[@]}"
-    fi
-}
-
-
-# Step 1: Certbot core Docker
-MakeMultiarchManifestForAllTargetArch "certbot"
-
-# Step 2: Certbot DNS plugins Docker images
-for plugin in "${CERTBOT_PLUGINS[@]}"; do
-    MakeMultiarchManifestForAllTargetArch "${plugin}"
-done

tools/docker/lib/common

@@ -1,12 +1,9 @@
 #!/bin/bash
-set -ex
+set -euxo pipefail
 
 # Current supported architectures
 export ALL_TARGET_ARCH=(amd64 arm32v6 arm64v8)
 
-# Architecture used in tags with no architecture specified (certbot/certbot:latest, certbot/certbot:v0.35.0, ...)
-export DEFAULT_ARCH=amd64
-
 # Name of the Certbot Docker organizaation on GitHub. After creating
 # repositories with the same names (e.g. "certbot", "dns-dnsmadeeasy", etc.)
 # using a different account on Docker Hub, you can change this value to have
@@ -31,22 +28,85 @@ export CERTBOT_PLUGINS=(
     "dns-sakuracloud"
 )
 
-# Parses the requested architecture string and sets ALL_REQUESTED_ARCH to
-# result.
-# Usage: ParseRequestedArch [all|amd64|arm32v6|arm64v8]
-ParseRequestedArch() {
+# WORK_DIR is two levels above this file
+export WORK_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")/..")"
+# REPO_ROOT is two levels above that
+export REPO_ROOT="$(realpath "${WORK_DIR}/../..")"
+
+# Converts input architecture identifier to the platform specification
+# understood by `docker build buildx --platform <specification>`.
+# Usage: arch2platform [arm64|arm32v6|arm64v8]
+# If the input is not recognized, an error is returned
+arch2platform() {
     REQUESTED_ARCH="${1}"
-    if [[ "${REQUESTED_ARCH}" == "all" ]]; then
-        ALL_REQUESTED_ARCH=("${ALL_TARGET_ARCH[@]}")
-        return 0
+    case $REQUESTED_ARCH in
+        amd64)
+            echo "linux/amd64"
+            ;;
+        arm32v6)
+            echo "linux/arm/v6"
+            ;;
+        arm64v8)
+            echo "linux/arm64"
+            ;;
+        *)
+            return 1
+            ;;
+    esac
+}
+
+
+ParseArgs() {
+
+    export TAG_VER="$1"
+    if [ -z "$TAG_VER" ]; then
+        echo "We cannot tag Docker images with an empty string!" >&2
+        exit 1
     fi
-    for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do
-        if [[ "${TARGET_ARCH}" == "${REQUESTED_ARCH}" ]]; then
-            ALL_REQUESTED_ARCH=("${REQUESTED_ARCH}")
-            return 0
+    ARCH_LIST="$2"
+    if [ -z "$ARCH_LIST" ]; then
+        echo "Architectures must be specified!" >&2
+        exit 1
+    fi
+
+    local IFS=","
+    # Handle the special value "all"
+    if [[ "${ARCH_LIST}" == "all" ]]; then
+        # Replace with comma separated 
+        ARCH_LIST="${ALL_TARGET_ARCH[*]}"
+    fi
+
+    # Turn arch list into an array
+    read -ra REQUESTED_ARCH_ARRAY <<< "$ARCH_LIST"
+    # And make sure all individual elements are in the list of all known architectures
+    for REQUESTED_ARCH in "${REQUESTED_ARCH_ARRAY[@]}"; do
+        local IFS=" "
+        if [[ ! " ${ALL_TARGET_ARCH[*]} " =~ " ${REQUESTED_ARCH} " ]]; then
+            echo "unknown architecture identifier: ${REQUESTED_ARCH}" >&2
+            exit 1
         fi
     done
-    # If we didn't return above, REQUESTED_ARCH has an unexpected value.
-    echo "Unexpected target architecture \"${REQUESTED_ARCH}\"". >&2
-    exit 1
+    export REQUESTED_ARCH_ARRAY
+}
+
+
+# Function for use with trap in the primary scripts to remove the 
+# docker builder and restore the original directory
+Cleanup() {
+    docker buildx rm certbot_builder || true
+    popd
+}
+
+# add binfmt tools to the docker environment, with integration into the new builder instance
+InstallMultiarchSupport() {
+    docker run --privileged --rm tonistiigi/binfmt --install all
+}
+
+# Function to create a docker builder using the buildkit docker-container
+# driver
+CreateBuilder() {
+    # just incase the env is not perfectly clean, remove any old instance of the builder
+    docker buildx rm certbot_builder || true
+    # create the builder instance
+    docker buildx create --name certbot_builder --driver docker-container --driver-opt=network=host --bootstrap
 }

tools/docker/plugin/Dockerfile

@@ -1,14 +0,0 @@
-# Certbot image to build on (e.g. certbot/certbot:amd64-v0.35.0)
-ARG BASE_IMAGE
-FROM ${BASE_IMAGE}
-
-# Qemu Arch (x86_64, arm, ...)
-ARG QEMU_ARCH
-ENV QEMU_ARCH=${QEMU_ARCH}
-COPY qemu-${QEMU_ARCH}-static /usr/bin/
-
-# Copy Certbot DNS plugin code
-COPY . /opt/certbot/src/plugin
-
-# Install the DNS plugin
-RUN python tools/pip_install.py --no-cache-dir --editable /opt/certbot/src/plugin

tools/docker/test.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+set -euxo pipefail
+
+# This script tests certbot docker and certbot dns plugin images.
+
+# Usage: 
+#       ./test.sh <tag> all
+#       ./test.sh <tag> <architectures>
+#   The <tag> argument is used to identify the code version (e.g v2.3.1) or type of build
+#   (e.g. nightly). This will be used when saving images to the docker image cache.
+#   The argument "all" will build all know architectures. Alternatively, the
+#   user may provide a comma separated list of architectures drawn from the
+#   known architectures. Known architectures include amd64, arm32v6, and arm64v8.
+
+source "$(realpath $(dirname "${BASH_SOURCE[0]}"))/lib/common"
+
+ParseArgs "$@"
+
+#jump to root, matching popd handed by Cleanup on EXIT via trap
+pushd "${REPO_ROOT}"
+
+# Set trap here, as the popd won't work as expected if invoked prior to pushd
+trap popd EXIT
+
+InstallMultiarchSupport
+
+
+REGISTRY_SPEC="${DOCKER_HUB_ORG}/"
+
+TestImage() {
+    IMAGE_NAME=$1
+    TAG_ARCH=$2
+    TAG_VER=$3
+    docker run --rm "${REGISTRY_SPEC}${IMAGE_NAME}:${TAG_ARCH}-${TAG_VER}" plugins --prepare
+}
+
+
+for TAG_ARCH in "${REQUESTED_ARCH_ARRAY[@]}"; do
+    TestImage certbot "$TAG_ARCH" "$TAG_VER"
+    for PLUGIN in "${CERTBOT_PLUGINS[@]}"; do
+        TestImage "$PLUGIN" "$TAG_ARCH" "$TAG_VER"
+    done
+done