test snaps only

.azure-pipelines/advanced-test.yml

@@ -12,3 +12,4 @@ variables:
 
 stages:
   - template: templates/stages/test-and-package-stage.yml
+  - template: templates/stages/deploy-stage.yml

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -1,102 +1,4 @@
 jobs:
-  - job: docker_build
-    pool:
-      vmImage: ubuntu-18.04
-    strategy:
-      matrix:
-        amd64:
-          DOCKER_ARCH: amd64
-        # Do not run the heavy non-amd64 builds for test branches
-        ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
-          arm32v6:
-            DOCKER_ARCH: arm32v6
-          arm64v8:
-            DOCKER_ARCH: arm64v8
-    steps:
-      - bash: tools/docker/build.sh $(dockerTag) $DOCKER_ARCH
-        displayName: Build the Docker images
-      # We don't filter for the Docker Hub organization to continue to allow
-      # easy testing of these scripts on forks.
-      - bash: |
-          DOCKER_IMAGES=$(docker images --filter reference='*/certbot' --filter reference='*/dns-*' --format '{{.Repository}}')
-          docker save --output images.tar $DOCKER_IMAGES
-        displayName: Save the Docker images
-      # If the name of the tar file or artifact changes, the deploy stage will
-      # also need to be updated.
-      - bash: mv images.tar $(Build.ArtifactStagingDirectory)
-        displayName: Prepare Docker artifact
-      - task: PublishPipelineArtifact@1
-        inputs:
-          path: $(Build.ArtifactStagingDirectory)
-          artifact: docker_$(DOCKER_ARCH)
-        displayName: Store Docker artifact
-  - job: installer_build
-    pool:
-      vmImage: vs2017-win2016
-    steps:
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: 3.7
-          architecture: x86
-          addToPath: true
-      - script: python windows-installer/construct.py
-        displayName: Build Certbot installer
-      - task: CopyFiles@2
-        inputs:
-          sourceFolder: $(System.DefaultWorkingDirectory)/windows-installer/build/nsis
-          contents: '*.exe'
-          targetFolder: $(Build.ArtifactStagingDirectory)
-      - task: PublishPipelineArtifact@1
-        inputs:
-          path: $(Build.ArtifactStagingDirectory)
-          # If we change the artifact's name, it should also be changed in tools/create_github_release.py
-          artifact: windows-installer
-        displayName: Publish Windows installer
-  - job: installer_run
-    dependsOn: installer_build
-    strategy:
-      matrix:
-        win2019:
-          imageName: windows-2019
-        win2016:
-          imageName: vs2017-win2016
-    pool:
-      vmImage: $(imageName)
-    steps:
-      - powershell: |
-          if ($PSVersionTable.PSVersion.Major -ne 5) {
-              throw "Powershell version is not 5.x"
-          }
-        condition: eq(variables['imageName'], 'vs2017-win2016')
-        displayName: Check Powershell 5.x is used in vs2017-win2016
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: 3.8
-          addToPath: true
-      - task: DownloadPipelineArtifact@2
-        inputs:
-          artifact: windows-installer
-          path: $(Build.SourcesDirectory)/bin
-        displayName: Retrieve Windows installer
-      # pip 9.0 provided by pipstrap is not able to resolve properly the pywin32 dependency
-      # required by certbot-ci: as a temporary workaround until pipstrap is updated, we install
-      # a recent version of pip, but we also to disable the isolated feature as described in
-      # https://github.com/certbot/certbot/issues/8256
-      - script: |
-          py -3 -m venv venv
-          venv\Scripts\python -m pip install pip==20.2.3 setuptools==50.3.0 wheel==0.35.1
-          venv\Scripts\python tools\pip_install.py -e certbot-ci
-        env:
-          PIP_NO_BUILD_ISOLATION: no
-        displayName: Prepare Certbot-CI
-      - script: |
-          set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
-          venv\Scripts\python -m pytest certbot-ci\windows_installer_integration_tests --allow-persistent-changes --installer-path $(Build.SourcesDirectory)\bin\certbot-beta-installer-win32.exe
-        displayName: Run windows installer integration tests
-      - script: |
-          set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
-          venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
-        displayName: Run certbot integration tests
   - job: snaps_build
     pool:
       vmImage: ubuntu-18.04
@@ -109,6 +11,7 @@ jobs:
         ARCHS: amd64
     steps:
       - script: |
+          set -e
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends snapd
           sudo snap install --classic snapcraft
@@ -122,6 +25,7 @@ jobs:
         inputs:
           secureFile: launchpad-credentials
       - script: |
+          set -e
           git config --global user.email "$(Build.RequestedForEmail)"
           git config --global user.name "$(Build.RequestedFor)"
           mkdir -p ~/.local/share/snapcraft/provider/launchpad
@@ -129,6 +33,7 @@ jobs:
           python3 tools/snap/build_remote.py ALL --archs ${ARCHS}
         displayName: Build snaps
       - script: |
+          set -e
           mv *.snap $(Build.ArtifactStagingDirectory)
           mv certbot-dns-*/*.snap $(Build.ArtifactStagingDirectory)
         displayName: Prepare artifacts
@@ -147,6 +52,7 @@ jobs:
           versionSpec: 3.8
           addToPath: true
       - script: |
+          set -e
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends nginx-light snapd
           python3 -m venv venv
@@ -159,9 +65,11 @@ jobs:
           path: $(Build.SourcesDirectory)/snap
         displayName: Retrieve Certbot snaps
       - script: |
+          set -e
           sudo snap install --dangerous --classic snap/certbot_*_amd64.snap
         displayName: Install Certbot snap
       - script: |
+          set -e
           venv/bin/python -m tox -e integration-external,apacheconftest-external-with-pebble
         displayName: Run tox
   - job: snap_dns_run
@@ -170,6 +78,7 @@ jobs:
       vmImage: ubuntu-18.04
     steps:
       - script: |
+          set -e
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends snapd
         displayName: Install dependencies
@@ -183,10 +92,12 @@ jobs:
           path: $(Build.SourcesDirectory)/snap
         displayName: Retrieve Certbot snaps
       - script: |
+          set -e
           python3 -m venv venv
           venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py
           venv/bin/python tools/pip_install.py -e certbot-ci
         displayName: Prepare Certbot-CI
       - script: |
+          set -e
           sudo -E venv/bin/pytest certbot-ci/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap --snap-arch amd64
         displayName: Test DNS plugins snaps

.azure-pipelines/templates/stages/changelog-stage.yml

@@ -7,6 +7,7 @@ stages:
         steps:
           # If we change the output filename from `release_notes.md`, it should also be changed in tools/create_github_release.py
           - bash: |
+              set -e
               CERTBOT_VERSION="$(cd certbot && python -c "import certbot; print(certbot.__version__)" && cd ~-)"
               "${BUILD_REPOSITORY_LOCALPATH}\tools\extract_changelog.py" "${CERTBOT_VERSION}" >> "${BUILD_ARTIFACTSTAGINGDIRECTORY}/release_notes.md"
             displayName: Prepare changelog

.azure-pipelines/templates/stages/deploy-stage.yml

@@ -39,6 +39,7 @@ stages:
           - group: certbot-common
         steps:
           - bash: |
+              set -e
               sudo apt-get update
               sudo apt-get install -y --no-install-recommends snapd
               sudo snap install --classic snapcraft
@@ -53,45 +54,25 @@ stages:
             inputs:
               secureFile: snapcraft.cfg
           - bash: |
+              set -e
+              retry_command() {
+                # based on travis_retry.bash https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_retry.bash
+                local result=0
+                local count=1
+                while [[ "${count}" -le 3 ]]; do
+                  # run the command in a way that doesn't disable setting `errexit`
+                  result=0
+                  "${@}" || result="${?}"
+                  if [[ $result -eq 0 ]]; then break; fi
+                  count="$((count + 1))"
+                  sleep 1
+                done
+
+                return "${result}"
+              }
               mkdir -p .snapcraft
               ln -s $(snapcraftCfg.secureFilePath) .snapcraft/snapcraft.cfg
               for SNAP_FILE in snap/*.snap; do
-                snapcraft upload --release=${{ parameters.snapReleaseChannel }} "${SNAP_FILE}"
+                retry_command eval snapcraft upload --release=${{ parameters.snapReleaseChannel }} "${SNAP_FILE}"
               done
             displayName: Publish to Snap store
-      - job: publish_docker
-        pool:
-          vmImage: ubuntu-18.04
-        strategy:
-          matrix:
-            amd64:
-              DOCKER_ARCH: amd64
-            arm32v6:
-              DOCKER_ARCH: arm32v6
-            arm64v8:
-              DOCKER_ARCH: arm64v8
-        steps:
-          - task: DownloadPipelineArtifact@2
-            inputs:
-              artifact: docker_$(DOCKER_ARCH)
-              path: $(Build.SourcesDirectory)
-            displayName: Retrieve Docker images
-          - bash: docker load --input $(Build.SourcesDirectory)/images.tar
-            displayName: Load Docker images
-          - task: Docker@2
-            inputs:
-              command: login
-              # The credentials used here are for the shared certbotbot account
-              # on Docker Hub. The credentials are stored in a service account
-              # which was created by following the instructions at
-              # https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#sep-docreg.
-              # The name given to this service account must match the value
-              # given to containerRegistry below. "Grant access to all
-              # pipelines" should also be checked.  To revoke these
-              # credentials, we can change the password on the certbotbot
-              # Docker Hub account or remove the account from the
-              # Certbot organization on Docker Hub.
-              containerRegistry: docker-hub
-            displayName: Login to Docker Hub
-          - bash: tools/docker/deploy.sh $(dockerTag) $DOCKER_ARCH
-            displayName: Deploy the Docker images

.azure-pipelines/templates/stages/notify-failure-stage.yml

@@ -8,6 +8,7 @@ stages:
           vmImage: ubuntu-latest
         steps:
           - bash: |
+              set -e
               MESSAGE="\
               ---\n\
               ##### Azure Pipeline

.azure-pipelines/templates/stages/test-and-package-stage.yml

@@ -1,6 +1,4 @@
 stages:
   - stage: TestAndPackage
     jobs:
-      - template: ../jobs/standard-tests-jobs.yml
-      - template: ../jobs/extended-tests-jobs.yml
       - template: ../jobs/packaging-jobs.yml

.azure-pipelines/templates/steps/tox-steps.yml

@@ -1,9 +1,11 @@
 steps:
   - bash: |
+      set -e
       brew install augeas
     condition: startswith(variables['IMAGE_NAME'], 'macOS')
     displayName: Install MacOS dependencies
   - bash: |
+      set -e
       sudo apt-get update
       sudo apt-get install -y --no-install-recommends \
         python-dev \
@@ -29,6 +31,7 @@ steps:
     # set, pip updates dependencies it thinks are already satisfied to avoid some
     # problems with its lack of real dependency resolution.
   - bash: |
+      set -e
       python letsencrypt-auto-source/pieces/pipstrap.py
       python tools/pip_install.py -I tox virtualenv
     displayName: Install runtime dependencies
@@ -38,6 +41,7 @@ steps:
       secureFile: azure-test-farm.pem
     condition: contains(variables['TOXENV'], 'test-farm')
   - bash: |
+      set -e
       export TARGET_BRANCH="`echo "${BUILD_SOURCEBRANCH}" | sed -E 's!refs/(heads|tags)/!!g'`"
       [ -z "${SYSTEM_PULLREQUEST_TARGETBRANCH}" ] || export TARGET_BRANCH="${SYSTEM_PULLREQUEST_TARGETBRANCH}"
       env