test full deploy

Co-authored-by: Brad Warren <bmw@users.noreply.github.com>

.azure-pipelines/advanced-test.yml

@@ -8,6 +8,7 @@ pr: none
 
 stages:
   - template: templates/stages/test-and-package-stage.yml
+  - template: templates/stages/deploy-stage.yml
   # Notify failures only for release branches.
   - ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
     - template: templates/stages/notify-failure-stage.yml

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -58,54 +58,44 @@ jobs:
           set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
           venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
         displayName: Run certbot integration tests
-  - job: snap_build
-    strategy:
-      matrix:
-        amd64:
-          ARCH: amd64
-        # Do not run the QEMU jobs for test branches
-        ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
-          arm64:
-            ARCH: arm64
-          armhf:
-            ARCH: armhf
+  - job: snaps_build
     pool:
       vmImage: ubuntu-18.04
+    timeoutInMinutes: 0
+    variables:
+        ARCHS: amd64 arm64 armhf
     steps:
       - script: |
-          tools/snap/build.sh ${ARCH}
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends snapd
+          sudo snap install --classic snapcraft
+        displayName: Install dependencies
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: 3.8
+          addToPath: true
+      - task: DownloadSecureFile@1
+        name: credentials
+        inputs:
+          secureFile: launchpad-credentials
+      - script: |
+          git config --global user.email "$(Build.RequestedForEmail)"
+          git config --global user.name "$(Build.RequestedFor)"
+          mkdir -p ~/.local/share/snapcraft/provider/launchpad
+          cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials
+          python3 tools/snap/build_remote.py ALL --archs ${ARCHS}
+        displayName: Build snaps
+      - script: |
           mv *.snap $(Build.ArtifactStagingDirectory)
-        displayName: Build Certbot snap
-      - task: PublishPipelineArtifact@1
-        inputs:
-          path: $(Build.ArtifactStagingDirectory)
-          artifact: snap-$(arch)
-        displayName: Store snap artifact
-  - job: snap_dns_build
-    strategy:
-      matrix:
-        amd64:
-          ARCH: amd64
-        # Do not run the QEMU jobs for test branches
-        ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
-          arm64:
-            ARCH: arm64
-          armhf:
-            ARCH: armhf
-    pool:
-      vmImage: ubuntu-18.04
-    steps:
-      - script: |
-          tools/snap/build_dns.sh ${ARCH} ALL
           mv certbot-dns-*/*.snap $(Build.ArtifactStagingDirectory)
-        displayName: Build Certbot DNS snaps
+        displayName: Prepare artifacts
       - task: PublishPipelineArtifact@1
         inputs:
           path: $(Build.ArtifactStagingDirectory)
-          artifact: dns-snap-$(arch)
+          artifact: snaps
         displayName: Store snaps artifacts
   - job: snap_run
-    dependsOn: snap_build
+    dependsOn: snaps_build
     pool:
       vmImage: ubuntu-18.04
     steps:
@@ -116,19 +106,17 @@ jobs:
         displayName: Install dependencies
       - task: DownloadPipelineArtifact@2
         inputs:
-          artifact: snap-amd64
+          artifact: snaps
           path: $(Build.SourcesDirectory)/snap
-        displayName: Retrieve Certbot snap
+        displayName: Retrieve Certbot snaps
       - script: |
-          sudo snap install --dangerous --classic snap/*.snap
+          sudo snap install --dangerous --classic snap/certbot_*_amd64.snap
         displayName: Install Certbot snap
       - script: |
           python -m tox -e integration-external,apacheconftest-external-with-pebble
         displayName: Run tox
   - job: snap_dns_run
-    dependsOn:
-      - snap_build
-      - snap_dns_build
+    dependsOn: snaps_build
     pool:
       vmImage: ubuntu-18.04
     steps:
@@ -142,18 +130,13 @@ jobs:
           addToPath: true
       - task: DownloadPipelineArtifact@2
         inputs:
-          artifact: snap-amd64
+          artifact: snaps
           path: $(Build.SourcesDirectory)/snap
-        displayName: Retrieve Certbot snap
-      - task: DownloadPipelineArtifact@2
-        inputs:
-          artifact: dns-snap-amd64
-          path: $(Build.SourcesDirectory)/snap
-        displayName: Retrieve Certbot DNS plugins snaps
+        displayName: Retrieve Certbot snaps
       - script: |
           python3 -m venv venv
           venv/bin/python tools/pip_install.py -e certbot-ci
         displayName: Prepare Certbot-CI
       - script: |
-          sudo -E venv/bin/pytest certbot-ci/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap
+          sudo -E venv/bin/pytest certbot-ci/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap --snap-arch amd64
         displayName: Test DNS plugins snaps

.azure-pipelines/templates/stages/deploy-stage.yml

@@ -9,14 +9,6 @@ stages:
       # prevent automated deploys from breaking. Remembering to do this is
       # also tracked by https://github.com/certbot/certbot/issues/7931.
       - job: publish_snap
-        strategy:
-          matrix:
-            amd64:
-              ARCH: amd64
-            arm64:
-              ARCH: arm64
-            armhf:
-              ARCH: armhf
         pool:
           vmImage: ubuntu-18.04
         variables:
@@ -29,14 +21,9 @@ stages:
             displayName: Install dependencies
           - task: DownloadPipelineArtifact@2
             inputs:
-              artifact: snap-$(arch)
+              artifact: snaps
               path: $(Build.SourcesDirectory)/snap
-            displayName: Retrieve Certbot snap
-          - task: DownloadPipelineArtifact@2
-            inputs:
-              artifact: dns-snap-$(arch)
-              path: $(Build.SourcesDirectory)/snap
-            displayName: Retrieve DNS plugins snaps
+            displayName: Retrieve Certbot snaps
           - task: DownloadSecureFile@1
             name: snapcraftCfg
             inputs:

certbot-ci/snap_integration_tests/conftest.py

@@ -17,6 +17,8 @@ def pytest_addoption(parser):
     """
     parser.addoption('--snap-folder', required=True,
                      help='set the folder path where snaps to test are located')
+    parser.addoption('--snap-arch', default='amd64',
+                    help='set the architecture do test (default: amd64)')
     parser.addoption('--allow-persistent-changes', action='store_true',
                      help='needs to be set, and confirm that the test will make persistent changes on this machine')
 
@@ -36,5 +38,8 @@ def pytest_generate_tests(metafunc):
     Generate (multiple) parametrized calls to a test function.
     """
     if "dns_snap_path" in metafunc.fixturenames:
-        snap_dns_path_list = glob.glob(os.path.join(metafunc.config.getoption('snap_folder'), 'certbot-dns-*_*.snap'))
+        snap_arch = metafunc.config.getoption('snap_arch')
+        snap_folder = metafunc.config.getoption('snap_folder')
+        snap_dns_path_list = glob.glob(os.path.join(snap_folder, 
+                                                    'certbot-dns-*_{0}.snap'.format(snap_arch)))
         metafunc.parametrize("dns_snap_path", snap_dns_path_list)

certbot-ci/snap_integration_tests/dns_tests/test_main.py

@@ -11,8 +11,9 @@ def install_certbot_snap(request):
     with pytest.raises(Exception):
         subprocess.check_call(['certbot', '--version'])
     try:
-        snap_path = glob.glob(os.path.join(request.config.getoption("snap_folder"),
-                                           'certbot_*.snap'))[0]
+        snap_folder = request.config.getoption("snap_folder")
+        snap_arch = request.config.getoption("snap_arch")
+        snap_path = glob.glob(os.path.join(snap_folder, 'certbot_*_{0}.snap'.format(snap_arch)))[0]
         subprocess.check_call(['snap', 'install', '--classic', '--dangerous', snap_path])
         subprocess.check_call(['certbot', '--version'])
         yield

certbot-dns-cloudflare/snap/snapcraft.yaml

@@ -16,6 +16,8 @@ parts:
         snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
     build-environment:
       - EXCLUDE_CERTBOT_DEPS: "True"
+    # To build cryptography and cffi if needed
+    build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
 
 slots:
   certbot:

certbot-dns-cloudxns/snap/snapcraft.yaml

@@ -16,6 +16,8 @@ parts:
         snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
     build-environment:
       - EXCLUDE_CERTBOT_DEPS: "True"
+    # To build cryptography and cffi if needed
+    build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
 
 slots:
   certbot:

certbot-dns-digitalocean/snap/snapcraft.yaml

@@ -16,6 +16,8 @@ parts:
         snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
     build-environment:
       - EXCLUDE_CERTBOT_DEPS: "True"
+    # To build cryptography and cffi if needed
+    build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
 
 slots:
   certbot:

certbot-dns-dnsimple/snap/snapcraft.yaml

@@ -16,6 +16,8 @@ parts:
         snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
     build-environment:
       - EXCLUDE_CERTBOT_DEPS: "True"
+    # To build cryptography and cffi if needed
+    build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
 
 slots:
   certbot:

certbot-dns-dnsmadeeasy/snap/snapcraft.yaml

@@ -16,6 +16,8 @@ parts:
         snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
     build-environment:
       - EXCLUDE_CERTBOT_DEPS: "True"
+    # To build cryptography and cffi if needed
+    build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
 
 slots:
   certbot:

certbot-dns-gehirn/snap/snapcraft.yaml

@@ -16,6 +16,8 @@ parts:
         snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
     build-environment:
       - EXCLUDE_CERTBOT_DEPS: "True"
+    # To build cryptography and cffi if needed
+    build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
 
 slots:
   certbot:

certbot-dns-google/snap/snapcraft.yaml

@@ -16,6 +16,8 @@ parts:
         snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
     build-environment:
       - EXCLUDE_CERTBOT_DEPS: "True"
+    # To build cryptography and cffi if needed
+    build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
 
 slots:
   certbot:

certbot-dns-linode/snap/snapcraft.yaml

@@ -16,6 +16,8 @@ parts:
         snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
     build-environment:
       - EXCLUDE_CERTBOT_DEPS: "True"
+    # To build cryptography and cffi if needed
+    build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
 
 slots:
   certbot:

certbot-dns-luadns/snap/snapcraft.yaml

@@ -16,6 +16,8 @@ parts:
         snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
     build-environment:
       - EXCLUDE_CERTBOT_DEPS: "True"
+    # To build cryptography and cffi if needed
+    build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
 
 slots:
   certbot:

certbot-dns-nsone/snap/snapcraft.yaml

@@ -16,6 +16,8 @@ parts:
         snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
     build-environment:
       - EXCLUDE_CERTBOT_DEPS: "True"
+    # To build cryptography and cffi if needed
+    build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
 
 slots:
   certbot:

certbot-dns-ovh/snap/snapcraft.yaml

@@ -16,6 +16,8 @@ parts:
         snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
     build-environment:
       - EXCLUDE_CERTBOT_DEPS: "True"
+    # To build cryptography and cffi if needed
+    build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
 
 slots:
   certbot:

certbot-dns-rfc2136/snap/snapcraft.yaml

@@ -16,6 +16,8 @@ parts:
         snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
     build-environment:
       - EXCLUDE_CERTBOT_DEPS: "True"
+    # To build cryptography and cffi if needed
+    build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
 
 slots:
   certbot:

certbot-dns-route53/snap/snapcraft.yaml

@@ -16,6 +16,8 @@ parts:
         snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
     build-environment:
       - EXCLUDE_CERTBOT_DEPS: "True"
+    # To build cryptography and cffi if needed
+    build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
 
 slots:
   certbot:

certbot-dns-sakuracloud/snap/snapcraft.yaml

@@ -16,6 +16,8 @@ parts:
         snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
     build-environment:
       - EXCLUDE_CERTBOT_DEPS: "True"
+    # To build cryptography and cffi if needed
+    build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
 
 slots:
   certbot:

snap/snapcraft.yaml

@@ -60,6 +60,7 @@ parts:
       # effect so we now stage the file to keep the auto-generated cffi file.
     stage-packages:
       - libaugeas0
+      - libpython3.8-dev
       # added to stage python:
       - libpython3-stdlib
       - libpython3.8-stdlib
@@ -73,7 +74,9 @@ parts:
       - python3-pkg-resources
       - python3.8-minimal
     # To build cryptography and cffi if needed
-    build-packages: [libffi-dev, libssl-dev, git, libaugeas-dev, python3-dev]
+    build-packages: [gcc, libffi-dev, libssl-dev, git, libaugeas-dev, python3-dev]
+    build-environment:
+      - SNAPCRAFT_PYTHON_VENV_ARGS: --system-site-packages
     override-pull: |
         snapcraftctl pull
         cd $SNAPCRAFT_PART_SRC

tools/snap/build_remote.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+import argparse
+import glob
+import datetime
+from multiprocessing import Pool, Process, Manager, Event
+import re
+import subprocess
+import sys
+from os.path import join, realpath, dirname, basename
+
+
+CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__))))
+PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))]
+
+
+def _execute_build(target, archs, status, workspace):
+    process = subprocess.Popen([
+        'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover', '--build-on', ','.join(archs)
+    ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, cwd=workspace)
+
+    for line in process.stdout:
+        _extract_state(target, line, status)
+
+    return process.wait()
+
+
+def _build_snap(target, archs, status):
+    status[target] = {arch: '...' for arch in archs}
+
+    if target == 'certbot':
+        workspace = CERTBOT_DIR
+    else:
+        workspace = join(CERTBOT_DIR, target)
+        subprocess.check_output(
+            ('"{0}" tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt '
+             '| grep -v python-augeas > "{1}/snap-constraints.txt"').format(sys.executable, workspace),
+            shell=True, cwd=CERTBOT_DIR)
+
+    retry = 3
+    while retry:
+        exit_code = _execute_build(target, archs, status, workspace)
+
+        print(f'Build {target} for {",".join(archs)} (attempt {4-retry}/3) ended with exit code {exit_code}.')
+        sys.stdout.flush()
+
+        # Retry if the snapcraft remote-build command has been interrupted.
+        if exit_code == 0 and 'Failed to build' not in status[target].values():
+            break
+
+        retry = retry - 1
+
+    return {target: workspace}
+
+
+def _extract_state(project, output, status):
+    match = re.match(r'^.*arch=(\w+)\s+state=([\w ]+).*$', output)
+    if match:
+        arch = match.group(1)
+        state = status[project]
+        state[arch] = match.group(2)
+
+        # You need to reassign the value of status[project] here (rather than doing
+        # something like status[project][arch] = match.group(2)) for the state change
+        # to propagate to other processes. See
+        # https://docs.python.org/3.8/library/multiprocessing.html#proxy-objects for
+        # more info.
+        status[project] = state
+
+
+def _dump_status_helper(archs, status):
+    headers = ['project', *archs]
+    print(''.join(f'| {item:<25}' for item in headers))
+    print(f'|{"-" * 26}' * len(headers))
+    for project, states in sorted(status.items()):
+        print(''.join(f'| {item:<25}' for item in [project, *[states[arch] for arch in archs]]))
+    print(f'|{"-" * 26}' * len(headers))
+    print()
+
+    sys.stdout.flush()
+
+
+def _dump_status(archs, status, stop_event):
+    while not stop_event.wait(10):
+        print('Remote build status at {0}'.format(datetime.datetime.now()))
+        _dump_status_helper(archs, status)
+
+
+def _dump_status_final(archs, status):
+    print('Results for remote build finished at {0}'.format(datetime.datetime.now()))
+    _dump_status_helper(archs, status)
+
+
+def _dump_results(targets, archs, status, workspaces):
+    failures = False
+    for target in targets:
+        for arch in archs:
+            result = status[target][arch]
+
+            if result != 'Successfully built':
+                failures = True
+
+                with open(join(workspaces[target], '{0}_{1}.txt'.format(target, arch))) as file_h:
+                    build_output = file_h.read()
+
+                print('Output for failed build target={0} arch={1}'.format(target, arch))
+                print('-------------------------------------------')
+                print(build_output)
+                print('-------------------------------------------')
+                print()
+
+    if not failures:
+        print('All builds succeeded.')
+    else:
+        print('Some builds failed.')
+
+    return failures
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('targets', nargs='+', choices=['ALL', 'DNS_PLUGINS', 'certbot', *PLUGINS],
+                        help='the list of snaps to build')
+    parser.add_argument('--archs', nargs='+', choices=['amd64', 'arm64', 'armhf'], default='amd64',
+                        help='the architectures for which snaps are built')
+    args = parser.parse_args()
+
+    archs = set(args.archs)
+    targets = set(args.targets)
+
+    if 'ALL' in targets:
+        targets.remove('ALL')
+        targets.update(['certbot', 'DNS_PLUGINS'])
+
+    if 'DNS_PLUGINS' in targets:
+        targets.remove('DNS_PLUGINS')
+        targets.update(PLUGINS)
+
+    print('Start remote snap builds...')
+    print(f' - archs: {", ".join(archs)}')
+    print(f' - projects: {", ".join(sorted(targets))}')
+    print()
+
+    with Manager() as manager, Pool(processes=len(targets)) as pool:
+        status = manager.dict()
+
+        stop_event = Event()
+        state_process = Process(target=_dump_status, args=(archs, status, stop_event))
+        state_process.start()
+
+        async_results = [pool.apply_async(_build_snap, (target, archs, status)) for target in targets]
+
+        workspaces = {}
+        for async_result in async_results:
+            workspaces.update(async_result.get())
+
+        stop_event.set()
+        state_process.join()
+
+        failures = _dump_results(targets, archs, status, workspaces)
+        _dump_status_final(archs, status)
+
+        return 1 if failures else 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

tools/snap/generate_dnsplugins_snapcraft.sh

@@ -3,7 +3,7 @@
 set -e
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-CERTBOT_DIR="$(dirname "${DIR}")"
+CERTBOT_DIR="$(dirname "$(dirname "${DIR}")")"
 
 for PLUGIN_PATH in "${CERTBOT_DIR}"/certbot-dns-*; do
   PLUGIN=$(basename "${PLUGIN_PATH}")
@@ -28,6 +28,8 @@ parts:
         snapcraftctl set-version \`grep ^version \$SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"\`
     build-environment:
       - EXCLUDE_CERTBOT_DEPS: "True"
+    # To build cryptography and cffi if needed
+    build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
 
 slots:
   certbot: