Setup CI system for Python3.9

Fixes #8294.

* Deprecate certbot-auto on Debian systems.

* Add changelog entry.

* Remove le_auto_xenial test.

* Update certbot-auto test farm tests.

* Add comments explaining expected behavior.

.azure-pipelines/templates/jobs/extended-tests-jobs.yml

@@ -3,6 +3,8 @@ jobs:
     variables:
       - name: IMAGE_NAME
         value: ubuntu-18.04
+      - name: PYTHON_VERSION
+        value: 3.9
       - group: certbot-common
     strategy:
       matrix:
@@ -12,9 +14,12 @@ jobs:
         linux-py37:
           PYTHON_VERSION: 3.7
           TOXENV: py37
-        linux-py37-nopin:
-          PYTHON_VERSION: 3.7
-          TOXENV: py37
+        linux-py38:
+          PYTHON_VERSION: 3.8
+          TOXENV: py38
+        linux-py39-nopin:
+          PYTHON_VERSION: 3.9
+          TOXENV: py39
           CERTBOT_NO_PIN: 1
         linux-boulder-v1-integration-certbot-oldest:
           TOXENV: integration-certbot-oldest
@@ -60,10 +65,16 @@ jobs:
           PYTHON_VERSION: 3.8
           TOXENV: integration
           ACME_SERVER: boulder-v2
+        linux-boulder-v1-py39-integration:
+          PYTHON_VERSION: 3.9
+          TOXENV: integration
+          ACME_SERVER: boulder-v1
+        linux-boulder-v2-py39-integration:
+          PYTHON_VERSION: 3.9
+          TOXENV: integration
+          ACME_SERVER: boulder-v2
         nginx-compat:
           TOXENV: nginx_compat
-        le-auto-centos6:
-          TOXENV: le_auto_centos6
         le-auto-oraclelinux6:
           TOXENV: le_auto_oraclelinux6
         docker-dev:
@@ -72,16 +83,16 @@ jobs:
           # We run one of these test farm tests on macOS to help ensure the
           # tests continue to work on the platform.
           IMAGE_NAME: macOS-10.15
-          PYTHON_VERSION: 3.8
+          PYTHON_VERSION: 3.9
           TOXENV: test-farm-apache2
         farmtest-leauto-upgrades:
-          PYTHON_VERSION: 3.7
+          PYTHON_VERSION: 3.9
           TOXENV: test-farm-leauto-upgrades
         farmtest-certonly-standalone:
-          PYTHON_VERSION: 3.7
+          PYTHON_VERSION: 3.9
           TOXENV: test-farm-certonly-standalone
         farmtest-sdists:
-          PYTHON_VERSION: 3.7
+          PYTHON_VERSION: 3.9
           TOXENV: test-farm-sdists
     pool:
       vmImage: $(IMAGE_NAME)

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -13,17 +13,18 @@ jobs:
           arm64v8:
             DOCKER_ARCH: arm64v8
     steps:
-      - bash: tools/docker/build.sh $(dockerTag) $DOCKER_ARCH
+      - bash: set -e && tools/docker/build.sh $(dockerTag) $DOCKER_ARCH
         displayName: Build the Docker images
       # We don't filter for the Docker Hub organization to continue to allow
       # easy testing of these scripts on forks.
       - bash: |
+          set -e
           DOCKER_IMAGES=$(docker images --filter reference='*/certbot' --filter reference='*/dns-*' --format '{{.Repository}}')
           docker save --output images.tar $DOCKER_IMAGES
         displayName: Save the Docker images
       # If the name of the tar file or artifact changes, the deploy stage will
       # also need to be updated.
-      - bash: mv images.tar $(Build.ArtifactStagingDirectory)
+      - bash: set -e && mv images.tar $(Build.ArtifactStagingDirectory)
         displayName: Prepare Docker artifact
       - task: PublishPipelineArtifact@1
         inputs:
@@ -71,16 +72,23 @@ jobs:
         displayName: Check Powershell 5.x is used in vs2017-win2016
       - task: UsePythonVersion@0
         inputs:
-          versionSpec: 3.8
+          versionSpec: 3.9
           addToPath: true
       - task: DownloadPipelineArtifact@2
         inputs:
           artifact: windows-installer
           path: $(Build.SourcesDirectory)/bin
         displayName: Retrieve Windows installer
+      # pip 9.0 provided by pipstrap is not able to resolve properly the pywin32 dependency
+      # required by certbot-ci: as a temporary workaround until pipstrap is updated, we install
+      # a recent version of pip, but we also to disable the isolated feature as described in
+      # https://github.com/certbot/certbot/issues/8256
       - script: |
           py -3 -m venv venv
+          venv\Scripts\python -m pip install pip==20.2.3 setuptools==50.3.0 wheel==0.35.1
           venv\Scripts\python tools\pip_install.py -e certbot-ci
+        env:
+          PIP_NO_BUILD_ISOLATION: no
         displayName: Prepare Certbot-CI
       - script: |
           set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
@@ -102,19 +110,21 @@ jobs:
         ARCHS: amd64
     steps:
       - script: |
+          set -e
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends snapd
           sudo snap install --classic snapcraft
         displayName: Install dependencies
       - task: UsePythonVersion@0
         inputs:
-          versionSpec: 3.8
+          versionSpec: 3.9
           addToPath: true
       - task: DownloadSecureFile@1
         name: credentials
         inputs:
           secureFile: launchpad-credentials
       - script: |
+          set -e
           git config --global user.email "$(Build.RequestedForEmail)"
           git config --global user.name "$(Build.RequestedFor)"
           mkdir -p ~/.local/share/snapcraft/provider/launchpad
@@ -122,6 +132,7 @@ jobs:
           python3 tools/snap/build_remote.py ALL --archs ${ARCHS}
         displayName: Build snaps
       - script: |
+          set -e
           mv *.snap $(Build.ArtifactStagingDirectory)
           mv certbot-dns-*/*.snap $(Build.ArtifactStagingDirectory)
         displayName: Prepare artifacts
@@ -135,10 +146,17 @@ jobs:
     pool:
       vmImage: ubuntu-18.04
     steps:
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: 3.9
+          addToPath: true
       - script: |
+          set -e
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends nginx-light snapd
-          python tools/pip_install.py -U tox
+          python3 -m venv venv
+          venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py
+          venv/bin/python tools/pip_install.py -U tox
         displayName: Install dependencies
       - task: DownloadPipelineArtifact@2
         inputs:
@@ -146,10 +164,12 @@ jobs:
           path: $(Build.SourcesDirectory)/snap
         displayName: Retrieve Certbot snaps
       - script: |
+          set -e
           sudo snap install --dangerous --classic snap/certbot_*_amd64.snap
         displayName: Install Certbot snap
       - script: |
-          python -m tox -e integration-external,apacheconftest-external-with-pebble
+          set -e
+          venv/bin/python -m tox -e integration-external,apacheconftest-external-with-pebble
         displayName: Run tox
   - job: snap_dns_run
     dependsOn: snaps_build
@@ -157,12 +177,13 @@ jobs:
       vmImage: ubuntu-18.04
     steps:
       - script: |
+          set -e
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends snapd
         displayName: Install dependencies
       - task: UsePythonVersion@0
         inputs:
-          versionSpec: 3.8
+          versionSpec: 3.9
           addToPath: true
       - task: DownloadPipelineArtifact@2
         inputs:
@@ -170,9 +191,12 @@ jobs:
           path: $(Build.SourcesDirectory)/snap
         displayName: Retrieve Certbot snaps
       - script: |
+          set -e
           python3 -m venv venv
+          venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py
           venv/bin/python tools/pip_install.py -e certbot-ci
         displayName: Prepare Certbot-CI
       - script: |
+          set -e
           sudo -E venv/bin/pytest certbot-ci/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap --snap-arch amd64
         displayName: Test DNS plugins snaps

.azure-pipelines/templates/jobs/standard-tests-jobs.yml

@@ -1,15 +1,17 @@
 jobs:
   - job: test
+    variables:
+      PYTHON_VERSION: 3.9
     strategy:
       matrix:
         macos-py27:
           IMAGE_NAME: macOS-10.15
           PYTHON_VERSION: 2.7
           TOXENV: py27
-        macos-py38:
+        macos-py39:
           IMAGE_NAME: macOS-10.15
-          PYTHON_VERSION: 3.8
-          TOXENV: py38
+          PYTHON_VERSION: 3.9
+          TOXENV: py39
         windows-py36:
           IMAGE_NAME: vs2017-win2016
           PYTHON_VERSION: 3.6
@@ -36,13 +38,13 @@ jobs:
           IMAGE_NAME: ubuntu-18.04
           PYTHON_VERSION: 3.6
           TOXENV: py36
-        linux-py38-cover:
+        linux-py39-cover:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.8
+          PYTHON_VERSION: 3.9
           TOXENV: py38-cover
-        linux-py37-lint:
+        linux-py39-lint:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.7
+          PYTHON_VERSION: 3.9
           TOXENV: lint
         linux-py36-mypy:
           IMAGE_NAME: ubuntu-18.04
@@ -50,22 +52,19 @@ jobs:
           TOXENV: mypy
         linux-integration:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 2.7
           TOXENV: integration
           ACME_SERVER: pebble
         apache-compat:
           IMAGE_NAME: ubuntu-18.04
           TOXENV: apache_compat
-        le-auto-xenial:
+        le-auto-centos6:
           IMAGE_NAME: ubuntu-18.04
-          TOXENV: le_auto_xenial
+          TOXENV: le_auto_centos6
         apacheconftest:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 2.7
           TOXENV: apacheconftest-with-pebble
         nginxroundtrip:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 2.7
           TOXENV: nginxroundtrip
     pool:
       vmImage: $(IMAGE_NAME)

.azure-pipelines/templates/stages/changelog-stage.yml

@@ -7,6 +7,7 @@ stages:
         steps:
           # If we change the output filename from `release_notes.md`, it should also be changed in tools/create_github_release.py
           - bash: |
+              set -e
               CERTBOT_VERSION="$(cd certbot && python -c "import certbot; print(certbot.__version__)" && cd ~-)"
               "${BUILD_REPOSITORY_LOCALPATH}\tools\extract_changelog.py" "${CERTBOT_VERSION}" >> "${BUILD_ARTIFACTSTAGINGDIRECTORY}/release_notes.md"
             displayName: Prepare changelog

.azure-pipelines/templates/stages/deploy-stage.yml

@@ -39,6 +39,7 @@ stages:
           - group: certbot-common
         steps:
           - bash: |
+              set -e
               sudo apt-get update
               sudo apt-get install -y --no-install-recommends snapd
               sudo snap install --classic snapcraft
@@ -53,10 +54,11 @@ stages:
             inputs:
               secureFile: snapcraft.cfg
           - bash: |
+              set -e
               mkdir -p .snapcraft
               ln -s $(snapcraftCfg.secureFilePath) .snapcraft/snapcraft.cfg
               for SNAP_FILE in snap/*.snap; do
-                snapcraft upload --release=${{ parameters.snapReleaseChannel }} "${SNAP_FILE}"
+                tools/retry.sh eval snapcraft upload --release=${{ parameters.snapReleaseChannel }} "${SNAP_FILE}"
               done
             displayName: Publish to Snap store
       - job: publish_docker
@@ -76,7 +78,7 @@ stages:
               artifact: docker_$(DOCKER_ARCH)
               path: $(Build.SourcesDirectory)
             displayName: Retrieve Docker images
-          - bash: docker load --input $(Build.SourcesDirectory)/images.tar
+          - bash: set -e && docker load --input $(Build.SourcesDirectory)/images.tar
             displayName: Load Docker images
           - task: Docker@2
             inputs:
@@ -93,5 +95,5 @@ stages:
               # Certbot organization on Docker Hub.
               containerRegistry: docker-hub
             displayName: Login to Docker Hub
-          - bash: tools/docker/deploy.sh $(dockerTag) $DOCKER_ARCH
+          - bash: set -e && tools/docker/deploy.sh $(dockerTag) $DOCKER_ARCH
             displayName: Deploy the Docker images

.azure-pipelines/templates/stages/notify-failure-stage.yml

@@ -8,6 +8,7 @@ stages:
           vmImage: ubuntu-latest
         steps:
           - bash: |
+              set -e
               MESSAGE="\
               ---\n\
               ##### Azure Pipeline

.azure-pipelines/templates/steps/tox-steps.yml

@@ -1,9 +1,11 @@
 steps:
   - bash: |
+      set -e
       brew install augeas
     condition: startswith(variables['IMAGE_NAME'], 'macOS')
     displayName: Install MacOS dependencies
   - bash: |
+      set -e
       sudo apt-get update
       sudo apt-get install -y --no-install-recommends \
         python-dev \
@@ -21,7 +23,6 @@ steps:
     inputs:
       versionSpec: $(PYTHON_VERSION)
       addToPath: true
-    condition: ne(variables['PYTHON_VERSION'], '')
     # tools/pip_install.py is used to pin packages to a known working version
     # except in tests where the environment variable CERTBOT_NO_PIN is set.
     # virtualenv is listed here explicitly to make sure it is upgraded when
@@ -30,6 +31,8 @@ steps:
     # set, pip updates dependencies it thinks are already satisfied to avoid some
     # problems with its lack of real dependency resolution.
   - bash: |
+      set -e
+      python letsencrypt-auto-source/pieces/pipstrap.py
       python tools/pip_install.py -I tox virtualenv
     displayName: Install runtime dependencies
   - task: DownloadSecureFile@1
@@ -38,6 +41,7 @@ steps:
       secureFile: azure-test-farm.pem
     condition: contains(variables['TOXENV'], 'test-farm')
   - bash: |
+      set -e
       export TARGET_BRANCH="`echo "${BUILD_SOURCEBRANCH}" | sed -E 's!refs/(heads|tags)/!!g'`"
       [ -z "${SYSTEM_PULLREQUEST_TARGETBRANCH}" ] || export TARGET_BRANCH="${SYSTEM_PULLREQUEST_TARGETBRANCH}"
       env

.gitignore

@@ -60,3 +60,7 @@ stage
 *.snap
 snap-constraints.txt
 qemu-*
+certbot-dns*/certbot-dns*_amd64*.txt
+certbot-dns*/certbot-dns*_arm*.txt
+/certbot_amd64*.txt
+/certbot_arm*.txt

acme/acme/client.py

@@ -448,7 +448,7 @@ class Client(ClientBase):
         heapq.heapify(waiting)
         # mapping between original Authorization Resource and the most
         # recently updated one
-        updated = dict((authzr, authzr) for authzr in authzrs)
+        updated = {authzr: authzr for authzr in authzrs}
 
         while waiting:
             # find the smallest Retry-After, and sleep if necessary

acme/acme/messages.py

@@ -206,7 +206,7 @@ class Directory(jose.JSONDeSerializable):
         external_account_required = jose.Field('externalAccountRequired', omitempty=True)
 
         def __init__(self, **kwargs):
-            kwargs = dict((self._internal_name(k), v) for k, v in kwargs.items())
+            kwargs = {self._internal_name(k): v for k, v in kwargs.items()}
             super(Directory.Meta, self).__init__(**kwargs)
 
         @property
@@ -465,7 +465,7 @@ class ChallengeBody(ResourceBody):
                        omitempty=True, default=None)
 
     def __init__(self, **kwargs):
-        kwargs = dict((self._internal_name(k), v) for k, v in kwargs.items())
+        kwargs = {self._internal_name(k): v for k, v in kwargs.items()}
         super(ChallengeBody, self).__init__(**kwargs)
 
     def encode(self, name):

acme/acme/util.py

@@ -4,4 +4,4 @@ import six
 
 def map_keys(dikt, func):
     """Map dictionary keys."""
-    return dict((func(key), value) for key, value in six.iteritems(dikt))
+    return {func(key): value for key, value in six.iteritems(dikt)}

acme/setup.py

@@ -5,7 +5,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
@@ -66,6 +66,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
     ],

certbot-apache/setup.py

@@ -5,7 +5,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -53,6 +53,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="1.8.0"
+LE_AUTO_VERSION="1.9.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -806,10 +806,7 @@ if [ -f /etc/debian_version ]; then
   BOOTSTRAP_VERSION="BootstrapDebCommon $BOOTSTRAP_DEB_COMMON_VERSION"
 elif [ -f /etc/mageia-release ]; then
   # Mageia has both /etc/mageia-release and /etc/redhat-release
-  Bootstrap() {
-    ExperimentalBootstrap "Mageia" BootstrapMageiaCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/redhat-release ]; then
   # Run DeterminePythonVersion to decide on the basis of available Python versions
   # whether to use 2.x or 3.x on RedHat-like systems.
@@ -884,31 +881,11 @@ elif [ -f /etc/redhat-release ]; then
 
   LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
-  Bootstrap() {
-    BootstrapMessage "openSUSE-based OSes"
-    BootstrapSuseCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapSuseCommon $BOOTSTRAP_SUSE_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/arch-release ]; then
-  Bootstrap() {
-    if [ "$DEBUG" = 1 ]; then
-      BootstrapMessage "Archlinux"
-      BootstrapArchCommon
-    else
-      error "Please use pacman to install letsencrypt packages:"
-      error "# pacman -S certbot certbot-apache"
-      error
-      error "If you would like to use the virtualenv way, please run the script again with the"
-      error "--debug flag."
-      exit 1
-    fi
-  }
-  BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/manjaro-release ]; then
-  Bootstrap() {
-    ExperimentalBootstrap "Manjaro Linux" BootstrapArchCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/gentoo-release ]; then
   DEPRECATED_OS=1
 elif uname | grep -iq FreeBSD ; then
@@ -921,19 +898,9 @@ elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then
   }
   BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
 elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then
-  Bootstrap() {
-    ExperimentalBootstrap "Joyent SmartOS Zone" BootstrapSmartOS
-  }
-  BOOTSTRAP_VERSION="BootstrapSmartOS $BOOTSTRAP_SMARTOS_VERSION"
+  DEPRECATED_OS=1
 else
-  Bootstrap() {
-    error "Sorry, I don't know how to bootstrap Certbot on your operating system!"
-    error
-    error "You will need to install OS dependencies, configure virtualenv, and run pip install manually."
-    error "Please see https://certbot.eff.org/docs/contributing.html#prerequisites"
-    error "for more info."
-    exit 1
-  }
+  DEPRECATED_OS=1
 fi
 
 # We handle this case after determining the normal bootstrap version to allow
@@ -1530,18 +1497,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==1.8.0 \
-    --hash=sha256:4bde86c53e30dc5bc0e78a0862045b053971703af727ac20c6a7da06596c7549 \
-    --hash=sha256:4837c516af6543ccd10d70f1498a2113bbdf9ef9a05d3a18b1558b291a2953e4
-acme==1.8.0 \
-    --hash=sha256:465033830a75f98042236f50f751f6e316735473ccb4edec0c718263f6c9ba8b \
-    --hash=sha256:ad8d067d14258d73ad2643439d9365913362308c04e66cc3010e39c868c5002d
-certbot-apache==1.8.0 \
-    --hash=sha256:8c9d981803e1156725fcfcf228afcb754b245c9d506e5b9f4fca948d6ae89aef \
-    --hash=sha256:a93c3a7ad929fe0ba5e0868e29ee2d0fe10aea2d4c638a902c4613a5c12c59b6
-certbot-nginx==1.8.0 \
-    --hash=sha256:e98e883b5ea7b29dd2e6a8ff286c7550a2d7af2fc859f47067303e510ad4fb52 \
-    --hash=sha256:fdb96c74fe42d90bbaf11a00314444ac5544ba87292a1b8b1d707f7561a3eacc
+certbot==1.9.0 \
+    --hash=sha256:d5a804d32e471050921f7b39ed9859e2e9de02824176ed78f57266222036b53a \
+    --hash=sha256:2ff9bf7d9af381c7efee22dec2dd6938d9d8fddcc9e11682b86e734164a30b57
+acme==1.9.0 \
+    --hash=sha256:d8061b396a22b21782c9b23ff9a945b23e50fca2573909a42f845e11d5658ac5 \
+    --hash=sha256:38a1630c98e144136c62eec4d2c545a1bdb1a3cd4eca82214be6b83a1f5a161f
+certbot-apache==1.9.0 \
+    --hash=sha256:09528a820d57e54984d490100644cd8a6603db97bf5776f86e95795ecfacf23d \
+    --hash=sha256:f47fb3f4a9bd927f4812121a0beefe56b163475a28f4db34c64dc838688d9e9e
+certbot-nginx==1.9.0 \
+    --hash=sha256:bb2e3f7fe17f071f350a3efa48571b8ef40a8e4b6db9c6da72539206a20b70be \
+    --hash=sha256:ab26a4f49d53b0e8bf0f903e58e2a840cda233fe1cbbc54c36ff17f973e57d65
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
@@ -1615,6 +1582,11 @@ maybe_argparse = (
     if sys.version_info < (2, 7, 0) else [])
 
 
+# Be careful when updating the pinned versions here, in particular for pip.
+# Indeed starting from 10.0, pip will build dependencies in isolation if the
+# related projects are compliant with PEP 517. This is not something we want
+# as of now, so the isolation build will need to be disabled wherever
+# pipstrap is used (see https://github.com/certbot/certbot/issues/8256).
 PACKAGES = maybe_argparse + [
     # Pip has no dependencies, as it vendors everything:
     ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/'

certbot-ci/setup.py

@@ -52,6 +52,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
     ],

certbot-compatibility-test/setup.py

@@ -5,7 +5,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 install_requires = [
     'certbot',
@@ -50,6 +50,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
     ],

certbot-dns-cloudflare/setup.py

@@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -63,6 +63,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-cloudflare/snap/snapcraft.yaml

@@ -3,7 +3,7 @@ name: certbot-dns-cloudflare
 summary: Cloudflare DNS Authenticator plugin for Certbot
 description: Cloudflare DNS Authenticator plugin for Certbot
 confinement: strict
-grade: devel
+grade: stable
 base: core20
 adopt-info: certbot-dns-cloudflare
 

certbot-dns-cloudxns/setup.py

@@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -63,6 +63,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-cloudxns/snap/snapcraft.yaml

@@ -3,7 +3,7 @@ name: certbot-dns-cloudxns
 summary: CloudXNS DNS Authenticator plugin for Certbot
 description: CloudXNS DNS Authenticator plugin for Certbot
 confinement: strict
-grade: devel
+grade: stable
 base: core20
 adopt-info: certbot-dns-cloudxns
 

certbot-dns-digitalocean/setup.py

@@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -64,6 +64,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-digitalocean/snap/snapcraft.yaml

@@ -3,7 +3,7 @@ name: certbot-dns-digitalocean
 summary: DigitalOcean DNS Authenticator plugin for Certbot
 description: DigitalOcean DNS Authenticator plugin for Certbot
 confinement: strict
-grade: devel
+grade: stable
 base: core20
 adopt-info: certbot-dns-digitalocean
 

certbot-dns-dnsimple/setup.py

@@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -74,6 +74,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-dnsimple/snap/snapcraft.yaml

@@ -3,7 +3,7 @@ name: certbot-dns-dnsimple
 summary: DNSimple DNS Authenticator plugin for Certbot
 description: DNSimple DNS Authenticator plugin for Certbot
 confinement: strict
-grade: devel
+grade: stable
 base: core20
 adopt-info: certbot-dns-dnsimple
 

certbot-dns-dnsmadeeasy/setup.py

@@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -63,6 +63,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-dnsmadeeasy/snap/snapcraft.yaml

@@ -3,7 +3,7 @@ name: certbot-dns-dnsmadeeasy
 summary: DNS Made Easy DNS Authenticator plugin for Certbot
 description: DNS Made Easy DNS Authenticator plugin for Certbot
 confinement: strict
-grade: devel
+grade: stable
 base: core20
 adopt-info: certbot-dns-dnsmadeeasy
 

certbot-dns-gehirn/setup.py

@@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
@@ -62,6 +62,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-gehirn/snap/snapcraft.yaml

@@ -3,7 +3,7 @@ name: certbot-dns-gehirn
 summary: Gehirn Infrastructure Service DNS Authenticator plugin for Certbot
 description: Gehirn Infrastructure Service DNS Authenticator plugin for Certbot
 confinement: strict
-grade: devel
+grade: stable
 base: core20
 adopt-info: certbot-dns-gehirn
 

certbot-dns-google/setup.py

@@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -66,6 +66,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-google/snap/snapcraft.yaml

@@ -3,7 +3,7 @@ name: certbot-dns-google
 summary: Google Cloud DNS Authenticator plugin for Certbot
 description: Google Cloud DNS Authenticator plugin for Certbot
 confinement: strict
-grade: devel
+grade: stable
 base: core20
 adopt-info: certbot-dns-google
 

certbot-dns-linode/setup.py

@@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
@@ -62,6 +62,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-linode/snap/snapcraft.yaml

@@ -3,7 +3,7 @@ name: certbot-dns-linode
 summary: Linode DNS Authenticator plugin for Certbot
 description: Linode DNS Authenticator plugin for Certbot
 confinement: strict
-grade: devel
+grade: stable
 base: core20
 adopt-info: certbot-dns-linode
 

certbot-dns-luadns/setup.py

@@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -63,6 +63,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-luadns/snap/snapcraft.yaml

@@ -3,7 +3,7 @@ name: certbot-dns-luadns
 summary: LuaDNS Authenticator plugin for Certbot
 description: LuaDNS Authenticator plugin for Certbot
 confinement: strict
-grade: devel
+grade: stable
 base: core20
 adopt-info: certbot-dns-luadns
 

certbot-dns-nsone/setup.py

@@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -63,6 +63,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-nsone/snap/snapcraft.yaml

@@ -3,7 +3,7 @@ name: certbot-dns-nsone
 summary: NS1 DNS Authenticator plugin for Certbot
 description: NS1 DNS Authenticator plugin for Certbot
 confinement: strict
-grade: devel
+grade: stable
 base: core20
 adopt-info: certbot-dns-nsone
 

certbot-dns-ovh/setup.py

@@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -63,6 +63,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-ovh/snap/snapcraft.yaml

@@ -3,7 +3,7 @@ name: certbot-dns-ovh
 summary: OVH DNS Authenticator plugin for Certbot
 description: OVH DNS Authenticator plugin for Certbot
 confinement: strict
-grade: devel
+grade: stable
 base: core20
 adopt-info: certbot-dns-ovh
 

certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py

@@ -18,6 +18,7 @@ from certbot.plugins import dns_common
 
 logger = logging.getLogger(__name__)
 
+DEFAULT_NETWORK_TIMEOUT = 45
 
 @zope.interface.implementer(interfaces.IAuthenticator)
 @zope.interface.provider(interfaces.IPluginFactory)
@@ -91,13 +92,15 @@ class _RFC2136Client(object):
     """
     Encapsulates all communication with the target DNS server.
     """
-    def __init__(self, server, port, key_name, key_secret, key_algorithm):
+    def __init__(self, server, port, key_name, key_secret, key_algorithm,
+        timeout=DEFAULT_NETWORK_TIMEOUT):
         self.server = server
         self.port = port
         self.keyring = dns.tsigkeyring.from_text({
             key_name: key_secret
         })
         self.algorithm = key_algorithm
+        self._default_timeout = timeout
 
     def add_txt_record(self, record_name, record_content, record_ttl):
         """
@@ -122,7 +125,7 @@ class _RFC2136Client(object):
         update.add(rel, record_ttl, dns.rdatatype.TXT, record_content)
 
         try:
-            response = dns.query.tcp(update, self.server, port=self.port)
+            response = dns.query.tcp(update, self.server, self._default_timeout, self.port)
         except Exception as e:
             raise errors.PluginError('Encountered error adding TXT record: {0}'
                                      .format(e))
@@ -157,7 +160,7 @@ class _RFC2136Client(object):
         update.delete(rel, dns.rdatatype.TXT, record_content)
 
         try:
-            response = dns.query.tcp(update, self.server, port=self.port)
+            response = dns.query.tcp(update, self.server, self._default_timeout, self.port)
         except Exception as e:
             raise errors.PluginError('Encountered error deleting TXT record: {0}'
                                      .format(e))
@@ -207,10 +210,10 @@ class _RFC2136Client(object):
 
         try:
             try:
-                response = dns.query.tcp(request, self.server, port=self.port)
-            except OSError as e:
+                response = dns.query.tcp(request, self.server, self._default_timeout, self.port)
+            except (OSError, dns.exception.Timeout) as e:
                 logger.debug('TCP query failed, fallback to UDP: %s', e)
-                response = dns.query.udp(request, self.server, port=self.port)
+                response = dns.query.udp(request, self.server, self._default_timeout, self.port)
             rcode = response.rcode()
 
             # Authoritative Answer bit should be set

certbot-dns-rfc2136/setup.py

@@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -63,6 +63,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-rfc2136/snap/snapcraft.yaml

@@ -3,7 +3,7 @@ name: certbot-dns-rfc2136
 summary: RFC 2136 DNS Authenticator plugin for Certbot
 description: RFC 2136 DNS Authenticator plugin for Certbot
 confinement: strict
-grade: devel
+grade: stable
 base: core20
 adopt-info: certbot-dns-rfc2136
 

certbot-dns-rfc2136/tests/dns_rfc2136_test.py

@@ -21,7 +21,7 @@ PORT = 53
 NAME = 'a-tsig-key.'
 SECRET = 'SSB3b25kZXIgd2hvIHdpbGwgYm90aGVyIHRvIGRlY29kZSB0aGlzIHRleHQK'
 VALID_CONFIG = {"rfc2136_server": SERVER, "rfc2136_name": NAME, "rfc2136_secret": SECRET}
-
+TIMEOUT = 45
 
 class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthenticatorTest):
 
@@ -78,7 +78,8 @@ class RFC2136ClientTest(unittest.TestCase):
     def setUp(self):
         from certbot_dns_rfc2136._internal.dns_rfc2136 import _RFC2136Client
 
-        self.rfc2136_client = _RFC2136Client(SERVER, PORT, NAME, SECRET, dns.tsig.HMAC_MD5)
+        self.rfc2136_client = _RFC2136Client(SERVER, PORT, NAME, SECRET, dns.tsig.HMAC_MD5,
+        TIMEOUT)
 
     @mock.patch("dns.query.tcp")
     def test_add_txt_record(self, query_mock):
@@ -88,7 +89,7 @@ class RFC2136ClientTest(unittest.TestCase):
 
         self.rfc2136_client.add_txt_record("bar", "baz", 42)
 
-        query_mock.assert_called_with(mock.ANY, SERVER, port=PORT)
+        query_mock.assert_called_with(mock.ANY, SERVER, TIMEOUT, PORT)
         self.assertTrue("bar. 42 IN TXT \"baz\"" in str(query_mock.call_args[0][0]))
 
     @mock.patch("dns.query.tcp")
@@ -121,7 +122,7 @@ class RFC2136ClientTest(unittest.TestCase):
 
         self.rfc2136_client.del_txt_record("bar", "baz")
 
-        query_mock.assert_called_with(mock.ANY, SERVER, port=PORT)
+        query_mock.assert_called_with(mock.ANY, SERVER, TIMEOUT, PORT)
         self.assertTrue("bar. 0 NONE TXT \"baz\"" in str(query_mock.call_args[0][0]))
 
     @mock.patch("dns.query.tcp")
@@ -173,7 +174,7 @@ class RFC2136ClientTest(unittest.TestCase):
         # _query_soa | pylint: disable=protected-access
         result = self.rfc2136_client._query_soa(DOMAIN)
 
-        query_mock.assert_called_with(mock.ANY, SERVER, port=PORT)
+        query_mock.assert_called_with(mock.ANY, SERVER, TIMEOUT, PORT)
         self.assertTrue(result)
 
     @mock.patch("dns.query.tcp")
@@ -183,7 +184,7 @@ class RFC2136ClientTest(unittest.TestCase):
         # _query_soa | pylint: disable=protected-access
         result = self.rfc2136_client._query_soa(DOMAIN)
 
-        query_mock.assert_called_with(mock.ANY, SERVER, port=PORT)
+        query_mock.assert_called_with(mock.ANY, SERVER, TIMEOUT, PORT)
         self.assertFalse(result)
 
     @mock.patch("dns.query.tcp")
@@ -206,8 +207,8 @@ class RFC2136ClientTest(unittest.TestCase):
         # _query_soa | pylint: disable=protected-access
         result = self.rfc2136_client._query_soa(DOMAIN)
 
-        tcp_mock.assert_called_with(mock.ANY, SERVER, port=PORT)
-        udp_mock.assert_called_with(mock.ANY, SERVER, port=PORT)
+        tcp_mock.assert_called_with(mock.ANY, SERVER, TIMEOUT, PORT)
+        udp_mock.assert_called_with(mock.ANY, SERVER, TIMEOUT, PORT)
         self.assertTrue(result)
 
 

certbot-dns-route53/setup.py

@@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -58,6 +58,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-route53/snap/snapcraft.yaml

@@ -3,7 +3,7 @@ name: certbot-dns-route53
 summary: Route53 DNS Authenticator plugin for Certbot
 description: Route53 DNS Authenticator plugin for Certbot
 confinement: strict
-grade: devel
+grade: stable
 base: core20
 adopt-info: certbot-dns-route53
 

certbot-dns-sakuracloud/setup.py

@@ -6,7 +6,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
@@ -62,6 +62,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-sakuracloud/snap/snapcraft.yaml

@@ -3,7 +3,7 @@ name: certbot-dns-sakuracloud
 summary: Sakura Cloud DNS Authenticator plugin for Certbot
 description: Sakura Cloud DNS Authenticator plugin for Certbot
 confinement: strict
-grade: devel
+grade: stable
 base: core20
 adopt-info: certbot-dns-sakuracloud
 

certbot-nginx/setup.py

@@ -5,7 +5,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.9.0.dev0'
+version = '1.10.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -49,6 +49,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot.wrapper

@@ -1,38 +0,0 @@
-#!/bin/sh
-#
-# TODO: We may want to consider rewriting this script in Python. See
-# https://github.com/certbot/certbot/issues/8251 for more info.
-set -e
-
-# This code is based on snapcraft's own patch to work around this problem at
-# https://github.com/snapcore/snapcraft/blob/a97fb5c7ea553a1bd20f4887a7c3393e75761890/patches/ctypes_init.diff.
-# We may not build the Certbot snap for all of these architectures (and as of
-# writing this we do not), but we keep the code for them to avoid having to
-# solve this problem again in the future if we add support for new
-# architectures.
-case "${SNAP_ARCH}" in
-    'arm64')
-        ARCH_TRIPLET='aarch64-linux-gnu';;
-    'armhf')
-        ARCH_TRIPLET='arm-linux-gnueabihf';;
-    'i386')
-        ARCH_TRIPLET='i386-linux-gnu';;
-    'ppc64el')
-        ARCH_TRIPLET='powerpc64le-linux-gnu';;
-    'powerpc')
-        ARCH_TRIPLET='powerpc-linux-gnu';;
-    'amd64')
-        ARCH_TRIPLET='x86_64-linux-gnu';;
-    's390x')
-        ARCH_TRIPLET='s390x-linux-gnu';;
-    *)
-        echo "Unrecongized value of SNAP_ARCH: ${SNAP_ARCH}" >&2
-        exit 1
-esac
-
-export CERTBOT_AUGEAS_PATH="${SNAP}/usr/lib/${ARCH_TRIPLET}/libaugeas.so.0"
-
-CERTBOT_PLUGIN_PATH="$(curl -s --unix-socket /run/snapd.socket "http://localhost/v2/connections?snap=certbot&interface=content" | jq -r  '.result.established | map(select(.plug.plug == "plugin" and ."plug-attrs".content == "certbot-1") | "/snap/"+.slot.snap+"/current/lib/python3.8/site-packages/" ) | join(":")')"
-export CERTBOT_PLUGIN_PATH
-
-exec certbot "$@" --preconfigured-renewal

certbot/CHANGELOG.md

@@ -2,7 +2,25 @@
 
 Certbot adheres to [Semantic Versioning](https://semver.org/).
 
-## 1.9.0 - master
+## 1.10.0 - master
+
+### Added
+
+* Added timeout to DNS query function calls for dns-rfc2136 plugin.
+* Confirmation when deleting certificates
+*
+
+### Changed
+
+* certbot-auto was deprecated on Debian based systems.
+
+### Fixed
+
+*
+
+More details about these changes can be found on our GitHub repo.
+
+## 1.9.0 - 2020-10-06
 
 ### Added
 
@@ -11,6 +29,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/).
 
 ### Changed
 
+* certbot-auto was deprecated on all systems except for those based on Debian or RHEL.
 * Update the packaging instructions to promote usage of `python -m pytest` to test Certbot
   instead of the deprecated `python setup.py test` setuptools approach.
 * Reduced CLI logging when reloading nginx, if it is not running.
@@ -23,6 +42,9 @@ Certbot adheres to [Semantic Versioning](https://semver.org/).
   In the previous release, Certbot said it required `acme>=1.6.0` when it
   actually required `acme>=1.8.0` to properly support removing contact
   information from an ACME account.
+* Upgraded the version of httplib2 used in our snaps and Docker images to add
+  support for proxy environment variables and fix the plugin for Google Cloud
+  DNS.
 
 More details about these changes can be found on our GitHub repo.
 

certbot/certbot/__init__.py

@@ -1,4 +1,4 @@
 """Certbot client."""
 
 # version number like 1.2.3a0, must have at least 2 parts, like 1.2
-__version__ = '1.9.0.dev0'
+__version__ = '1.10.0.dev0'

certbot/certbot/_internal/cert_manager.py

@@ -90,9 +90,16 @@ def certificates(config):
 def delete(config):
     """Delete Certbot files associated with a certificate lineage."""
     certnames = get_certnames(config, "delete", allow_multiple=True)
+    disp = zope.component.getUtility(interfaces.IDisplay)
+    msg = ["The following certificate(s) are selected for deletion:\n"]
+    for certname in certnames:
+        msg.append("  * " + certname)
+    msg.append("\nAre you sure you want to delete the above certificate(s)?")
+    if not disp.yesno("\n".join(msg), default=True):
+        logger.info("Deletion of certificate(s) canceled.")
+        return
     for certname in certnames:
         storage.delete_files(config, certname)
-        disp = zope.component.getUtility(interfaces.IDisplay)
         disp.notification("Deleted all files relating to certificate {0}."
             .format(certname), pause=False)
 

certbot/certbot/_internal/cli/helpful.py

@@ -192,8 +192,8 @@ class HelpfulArgumentParser(object):
         if self.detect_defaults:
             return parsed_args
 
-        self.defaults = dict((key, copy.deepcopy(self.parser.get_default(key)))
-                             for key in vars(parsed_args))
+        self.defaults = {key: copy.deepcopy(self.parser.get_default(key))
+                             for key in vars(parsed_args)}
 
         # Do any post-parsing homework here
 

certbot/certbot/_internal/main.py

@@ -28,6 +28,7 @@ from certbot._internal import hooks
 from certbot._internal import log
 from certbot._internal import renewal
 from certbot._internal import reporter
+from certbot._internal import snap_config
 from certbot._internal import storage
 from certbot._internal import updater
 from certbot._internal.plugins import disco as plugins_disco
@@ -1325,6 +1326,9 @@ def main(cli_args=None):
 
     log.pre_arg_parse_setup()
 
+    if os.environ.get('CERTBOT_SNAPPED') == 'True':
+        cli_args = snap_config.prepare_env(cli_args)
+
     plugins = plugins_disco.PluginsRegistry.find_all()
     logger.debug("certbot version: %s", certbot.__version__)
     # do not log `config`, as it contains sensitive data (e.g. revoke --key)!

certbot/certbot/_internal/plugins/disco.py

@@ -277,8 +277,8 @@ class PluginsRegistry(Mapping):
 
     def filter(self, pred):
         """Filter plugins based on predicate."""
-        return type(self)(dict((name, plugin_ep) for name, plugin_ep
-                               in six.iteritems(self._plugins) if pred(plugin_ep)))
+        return type(self)({name: plugin_ep for name, plugin_ep
+                               in six.iteritems(self._plugins) if pred(plugin_ep)})
 
     def visible(self):
         """Filter plugins based on visibility."""

certbot/certbot/_internal/renewal.py

@@ -9,16 +9,21 @@ import sys
 import time
 import traceback
 
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.primitives.serialization import load_pem_private_key
 import OpenSSL
 import six
 import zope.component
 
 from acme.magic_typing import List
+from acme.magic_typing import Optional  # pylint: disable=unused-import
 from certbot import crypto_util
 from certbot import errors
 from certbot import interfaces
 from certbot import util
 from certbot._internal import cli
+from certbot._internal import client  # pylint: disable=unused-import
 from certbot._internal import constants
 from certbot._internal import hooks
 from certbot._internal import storage
@@ -308,7 +313,8 @@ def _avoid_invalidating_lineage(config, lineage, original_server):
 
 
 def renew_cert(config, domains, le_client, lineage):
-    "Renew a certificate lineage."
+    # type: (interfaces.IConfig, Optional[List[str]], client.Client, storage.RenewableCert) -> None
+    """Renew a certificate lineage."""
     renewal_params = lineage.configuration["renewalparams"]
     original_server = renewal_params.get("server", cli.flag_default("server"))
     _avoid_invalidating_lineage(config, lineage, original_server)
@@ -316,11 +322,14 @@ def renew_cert(config, domains, le_client, lineage):
         domains = lineage.names()
     # The private key is the existing lineage private key if reuse_key is set.
     # Otherwise, generate a fresh private key by passing None.
-    new_key = os.path.normpath(lineage.privkey) if config.reuse_key else None
+    if config.reuse_key:
+        new_key = os.path.normpath(lineage.privkey)
+        _update_renewal_params_from_key(new_key, config)
+    else:
+        new_key = None
     new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains, new_key)
     if config.dry_run:
-        logger.debug("Dry run: skipping updating lineage at %s",
-                    os.path.dirname(lineage.cert))
+        logger.debug("Dry run: skipping updating lineage at %s", os.path.dirname(lineage.cert))
     else:
         prior_version = lineage.latest_common_version()
         # TODO: Check return value of save_successor
@@ -335,6 +344,7 @@ def report(msgs, category):
     lines = ("%s (%s)" % (m, category) for m in msgs)
     return "  " + "\n  ".join(lines)
 
+
 def _renew_describe_results(config, renew_successes, renew_failures,
                             renew_skipped, parse_failures):
 
@@ -489,3 +499,13 @@ def handle_renewal_request(config):
     # Windows installer integration tests rely on handle_renewal_request behavior here.
     # If the text below changes, these tests will need to be updated accordingly.
     logger.debug("no renewal failures")
+
+
+def _update_renewal_params_from_key(key_path, config):
+    # type: (str, interfaces.IConfig) -> None
+    with open(key_path, 'rb') as file_h:
+        key = load_pem_private_key(file_h.read(), password=None, backend=default_backend())
+    if isinstance(key, rsa.RSAPrivateKey):
+        config.rsa_key_size = key.key_size
+    else:
+        raise errors.Error('Key at {0} is of an unsupported type: {1}.'.format(key_path, type(key)))

certbot/certbot/_internal/snap_config.py

@@ -0,0 +1,102 @@
+"""Module configuring Certbot in a snap environment"""
+import logging
+import socket
+
+from requests import Session
+from requests.adapters import HTTPAdapter
+from requests.exceptions import HTTPError
+from requests.exceptions import RequestException
+
+from acme.magic_typing import List
+from certbot.compat import os
+from certbot.errors import Error
+
+try:
+    from urllib3.connection import HTTPConnection
+    from urllib3.connectionpool import HTTPConnectionPool
+except ImportError:
+    # Stub imports for oldest requirements, that will never be used in snaps.
+    HTTPConnection = object
+    HTTPConnectionPool = object
+
+
+_ARCH_TRIPLET_MAP = {
+    'arm64': 'aarch64-linux-gnu',
+    'armhf': 'arm-linux-gnueabihf',
+    'i386': 'i386-linux-gnu',
+    'ppc64el': 'powerpc64le-linux-gnu',
+    'powerpc': 'powerpc-linux-gnu',
+    'amd64': 'x86_64-linux-gnu',
+    's390x': 's390x-linux-gnu',
+}
+
+LOGGER = logging.getLogger(__name__)
+
+
+def prepare_env(cli_args):
+    # type: (List[str]) -> List[str]
+    """
+    Prepare runtime environment for a certbot execution in snap.
+    :param list cli_args: List of command line arguments
+    :return: Update list of command line arguments
+    :rtype: list
+    """
+    snap_arch = os.environ.get('SNAP_ARCH')
+
+    if snap_arch not in _ARCH_TRIPLET_MAP:
+        raise Error('Unrecognized value of SNAP_ARCH: {0}'.format(snap_arch))
+
+    os.environ['CERTBOT_AUGEAS_PATH'] = '{0}/usr/lib/{1}/libaugeas.so.0'.format(
+        os.environ.get('SNAP'), _ARCH_TRIPLET_MAP[snap_arch])
+
+    session = Session()
+    session.mount('http://snapd/', _SnapdAdapter())
+
+    try:
+        response = session.get('http://snapd/v2/connections?snap=certbot&interface=content')
+        response.raise_for_status()
+    except RequestException as e:
+        if isinstance(e, HTTPError) and e.response.status_code == 404:
+            LOGGER.error('An error occurred while fetching Certbot snap plugins: '
+                         'your version of snapd is outdated.')
+            LOGGER.error('Please run "sudo snap install core; sudo snap refresh core" '
+                         'in your terminal and try again.')
+        else:
+            LOGGER.error('An error occurred while fetching Certbot snap plugins: '
+                         'make sure the snapd service is running.')
+        raise e
+
+    data = response.json()
+    connections = ['/snap/{0}/current/lib/python3.8/site-packages/'.format(item['slot']['snap'])
+                   for item in data.get('result', {}).get('established', [])
+                   if item.get('plug', {}).get('plug') == 'plugin'
+                   and item.get('plug-attrs', {}).get('content') == 'certbot-1']
+
+    os.environ['CERTBOT_PLUGIN_PATH'] = ':'.join(connections)
+
+    cli_args.append('--preconfigured-renewal')
+
+    return cli_args
+
+
+class _SnapdConnection(HTTPConnection):
+    def __init__(self):
+        super(_SnapdConnection, self).__init__("localhost")
+        self.sock = None
+
+    def connect(self):
+        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        self.sock.connect("/run/snapd.socket")
+
+
+class _SnapdConnectionPool(HTTPConnectionPool):
+    def __init__(self):
+        super(_SnapdConnectionPool, self).__init__("localhost")
+
+    def _new_conn(self):
+        return _SnapdConnection()
+
+
+class _SnapdAdapter(HTTPAdapter):
+    def get_connection(self, url, proxies=None):
+        return _SnapdConnectionPool()

certbot/certbot/_internal/storage.py

@@ -1126,7 +1126,7 @@ class RenewableCert(interfaces.RenewableCert):
             logger.debug("Writing full chain to %s.", target["fullchain"])
             f.write(new_cert + new_chain)
 
-        symlinks = dict((kind, self.configuration[kind]) for kind in ALL_FOUR)
+        symlinks = {kind: self.configuration[kind] for kind in ALL_FOUR}
         # Update renewal config file
         self.configfile = update_configuration(
             self.lineagename, self.archive_dir, symlinks, cli_config)

certbot/certbot/util.py

@@ -401,14 +401,14 @@ def get_python_os_info(pretty=False):
         try:
             proc = subprocess.Popen(
                 ["/usr/bin/sw_vers", "-productVersion"],
-                stdout=subprocess.PIPE,
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                 universal_newlines=True,
                 env=env_no_snap_for_external_calls(),
             )
         except OSError:
             proc = subprocess.Popen(
                 ["sw_vers", "-productVersion"],
-                stdout=subprocess.PIPE,
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                 universal_newlines=True,
                 env=env_no_snap_for_external_calls(),
             )

certbot/docs/cli-help.txt

@@ -118,7 +118,7 @@ optional arguments:
                         case, and to know when to deprecate support for past
                         Python versions and flags. If you wish to hide this
                         information from the Let's Encrypt server, set this to
-                        "". (default: CertbotACMEClient/1.8.0 (certbot(-auto);
+                        "". (default: CertbotACMEClient/1.9.0 (certbot(-auto);
                         OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY
                         (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel).
                         The flags encoded in the user agent are: --duplicate,

certbot/docs/contributing.rst

@@ -303,7 +303,7 @@ configuration checkpoints and rollback.
 .. _dev-plugin:
 
 Writing your own plugin
-~~~~~~~~~~~~~~~~~~~~~~~
+-----------------------
 
 .. note:: The Certbot team is not currently accepting any new DNS plugins
     because we want to rethink our approach to the challenge and resolve some
@@ -338,17 +338,69 @@ it was not installed properly.
 Once you've finished your plugin and published it, you can have your
 users install it system-wide with `pip install`. Note that this will
 only work for users who have Certbot installed from OS packages or via
-pip. Users who run `certbot-auto` are currently unable to use third-party
-plugins. It's technically possible to install third-party plugins into
-the virtualenv used by `certbot-auto`, but they will be wiped away when
-`certbot-auto` upgrades. If you'd like your plugin to be used alongside
-the Certbot snap, you will also have to publish your plugin as a snap.
-Certbot's DNS plugins and the README file in ``tools/snap/`` provide a
-starting reference for how to do this.
+pip.
 
 .. _`setuptools entry points`:
     https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points
 
+Writing your own plugin snap
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you'd like your plugin to be used alongside the Certbot snap, you
+will also have to publish your plugin as a snap. Plugin snaps are
+regular confined snaps, but normally do not provide any "apps"
+themselves. Plugin snaps export loadable Python modules to the Certbot
+snap.
+
+When the Certbot snap runs, it will use its version of Python and prefer
+Python modules contained in its own snap over modules contained in
+external snaps. This means that your snap doesn't have to contain things
+like an extra copy of Python, Certbot, or their dependencies, but also
+that if you need a different version of a dependency than is already
+installed in the Certbot snap, the Certbot snap will have to be updated.
+
+Certbot plugin snaps expose their Python modules to the Certbot snap via a
+`snap content interface`_ where ``certbot-1`` is the value for the ``content``
+attribute. The Certbot snap only uses this to find the names of connected
+plugin snaps and it expects to find the Python modules to be loaded under
+``lib/python3.8/site-packages/`` in the plugin snap. This location is the
+default when using the ``core20`` `base snap`_ and the `python snapcraft
+plugin`_.
+
+The Certbot snap also provides a separate content interface which
+you can use to get metadata about the Certbot snap using the ``content``
+identifier ``metadata-1``.
+
+The script used to generate the snapcraft.yaml files for our own externally
+snapped plugins can be found at
+https://github.com/certbot/certbot/blob/master/tools/snap/generate_dnsplugins_snapcraft.sh.
+
+Once you have created your own snap, if you have the snap file locally,
+it can be installed for use with Certbot by running:
+
+.. code-block:: shell
+
+    snap install --classic certbot
+    snap set certbot trust-plugin-with-root=ok
+    snap install --dangerous your-snap-filename.snap
+    sudo snap connect certbot:plugin your-snap-name
+    sudo /snap/bin/certbot plugins
+
+If everything worked, the last command should list your plugin in the
+list of plugins found by Certbot. Once your snap is published to the
+snap store, it will be installable through the name of the snap on the
+snap store without the ``--dangerous`` flag. If you are also using
+Certbot's metadata interface, you can run ``sudo snap connect
+your-snap-name:your-plug-name-for-metadata certbot:certbot-metadata`` to
+connect your snap to it.
+
+.. _`snap content interface`:
+    https://snapcraft.io/docs/content-interface
+.. _`base snap`:
+    https://snapcraft.io/docs/base-snaps
+.. _`python snapcraft plugin`:
+    https://snapcraft.io/docs/python-plugin
+
 .. _coding-style:
 
 Coding style

certbot/docs/install.rst

@@ -248,18 +248,7 @@ replacing ``certbot`` with the name of the desired package.
 
 **Ubuntu**
 
-If you run Ubuntu Trusty, Xenial, or Bionic, certbot is available through the official PPA,
-that can be installed as followed:
-
-.. code-block:: shell
-
-   sudo apt-get update
-   sudo apt-get install software-properties-common
-   sudo add-apt-repository universe
-   sudo add-apt-repository ppa:certbot/certbot
-   sudo apt-get update
-
-Then, certbot can be installed using:
+If you run Ubuntu, certbot can be installed using:
 
 .. code-block:: shell
 

certbot/docs/using.rst

@@ -281,7 +281,8 @@ proxmox_           N    Y    Install certificates in Proxmox Virtualization serv
 dns-standalone_    Y    N    Obtain certificates via an integrated DNS server
 dns-ispconfig_     Y    N    DNS Authentication using ISPConfig as DNS server
 dns-clouddns_      Y    N    DNS Authentication using CloudDNS API
-dns-inwx           Y    Y    DNS Authentication for INWX through the XML API
+dns-lightsail_     Y    N    DNS Authentication using Amazon Lightsail DNS API
+dns-inwx_          Y    Y    DNS Authentication for INWX through the XML API
 ================== ==== ==== ===============================================================
 
 .. _haproxy: https://github.com/greenhost/certbot-haproxy
@@ -294,6 +295,7 @@ dns-inwx           Y    Y    DNS Authentication for INWX through the XML API
 .. _dns-standalone: https://github.com/siilike/certbot-dns-standalone
 .. _dns-ispconfig: https://github.com/m42e/certbot-dns-ispconfig
 .. _dns-clouddns: https://github.com/vshosting/certbot-dns-clouddns
+.. _dns-lightsail: https://github.com/noi/certbot-dns-lightsail
 .. _dns-inwx: https://github.com/oGGy990/certbot-dns-inwx/
 
 If you're interested, you can also :ref:`write your own plugin <dev-plugin>`.

certbot/setup.py

@@ -131,6 +131,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot/tests/cert_manager_test.py

@@ -35,8 +35,8 @@ class BaseCertManagerTest(test_util.ConfigTestCase):
             "example.org": None,
             "other.com": os.path.join(self.config.config_dir, "specialarchive")
         }
-        self.config_files = dict((domain, self._set_up_config(domain, self.domains[domain]))
-            for domain in self.domains)
+        self.config_files = {domain: self._set_up_config(domain, self.domains[domain])
+            for domain in self.domains}
 
         # We also create a file that isn't a renewal config in the same
         # location to test that logic that reads in all-and-only renewal
@@ -80,8 +80,8 @@ class UpdateLiveSymlinksTest(BaseCertManagerTest):
                 archive_dir_path = custom_archive
             else:
                 archive_dir_path = os.path.join(self.config.default_archive_dir, domain)
-            archive_paths[domain] = dict((kind,
-                os.path.join(archive_dir_path, kind + "1.pem")) for kind in ALL_FOUR)
+            archive_paths[domain] = {kind:
+                os.path.join(archive_dir_path, kind + "1.pem") for kind in ALL_FOUR}
             for kind in ALL_FOUR:
                 live_path = self.config_files[domain][kind]
                 archive_path = archive_paths[domain][kind]
@@ -116,10 +116,11 @@ class DeleteTest(storage_test.BaseRenewableCertTest):
     @test_util.patch_get_utility()
     @mock.patch('certbot._internal.cert_manager.lineage_for_certname')
     @mock.patch('certbot._internal.storage.delete_files')
-    def test_delete_from_config(self, mock_delete_files, mock_lineage_for_certname,
-        unused_get_utility):
+    def test_delete_from_config_yes(self, mock_delete_files, mock_lineage_for_certname,
+        mock_util):
         """Test delete"""
         mock_lineage_for_certname.return_value = self.test_rc
+        mock_util().yesno.return_value = True
         self.config.certname = "example.org"
         self._call()
         mock_delete_files.assert_called_once_with(self.config, "example.org")
@@ -127,27 +128,65 @@ class DeleteTest(storage_test.BaseRenewableCertTest):
     @test_util.patch_get_utility()
     @mock.patch('certbot._internal.cert_manager.lineage_for_certname')
     @mock.patch('certbot._internal.storage.delete_files')
-    def test_delete_interactive_single(self, mock_delete_files, mock_lineage_for_certname,
+    def test_delete_from_config_no(self, mock_delete_files, mock_lineage_for_certname,
+        mock_util):
+        """Test delete"""
+        mock_lineage_for_certname.return_value = self.test_rc
+        mock_util().yesno.return_value = False
+        self.config.certname = "example.org"
+        self._call()
+        self.assertEqual(mock_delete_files.call_count, 0)
+
+    @test_util.patch_get_utility()
+    @mock.patch('certbot._internal.cert_manager.lineage_for_certname')
+    @mock.patch('certbot._internal.storage.delete_files')
+    def test_delete_interactive_single_yes(self, mock_delete_files, mock_lineage_for_certname,
         mock_util):
         """Test delete"""
         mock_lineage_for_certname.return_value = self.test_rc
         mock_util().checklist.return_value = (display_util.OK, ["example.org"])
+        mock_util().yesno.return_value = True
         self._call()
         mock_delete_files.assert_called_once_with(self.config, "example.org")
 
     @test_util.patch_get_utility()
     @mock.patch('certbot._internal.cert_manager.lineage_for_certname')
     @mock.patch('certbot._internal.storage.delete_files')
-    def test_delete_interactive_multiple(self, mock_delete_files, mock_lineage_for_certname,
+    def test_delete_interactive_single_no(self, mock_delete_files, mock_lineage_for_certname,
+        mock_util):
+        """Test delete"""
+        mock_lineage_for_certname.return_value = self.test_rc
+        mock_util().checklist.return_value = (display_util.OK, ["example.org"])
+        mock_util().yesno.return_value = False
+        self._call()
+        self.assertEqual(mock_delete_files.call_count, 0)
+
+    @test_util.patch_get_utility()
+    @mock.patch('certbot._internal.cert_manager.lineage_for_certname')
+    @mock.patch('certbot._internal.storage.delete_files')
+    def test_delete_interactive_multiple_yes(self, mock_delete_files, mock_lineage_for_certname,
         mock_util):
         """Test delete"""
         mock_lineage_for_certname.return_value = self.test_rc
         mock_util().checklist.return_value = (display_util.OK, ["example.org", "other.org"])
+        mock_util().yesno.return_value = True
         self._call()
         mock_delete_files.assert_any_call(self.config, "example.org")
         mock_delete_files.assert_any_call(self.config, "other.org")
         self.assertEqual(mock_delete_files.call_count, 2)
 
+    @test_util.patch_get_utility()
+    @mock.patch('certbot._internal.cert_manager.lineage_for_certname')
+    @mock.patch('certbot._internal.storage.delete_files')
+    def test_delete_interactive_multiple_no(self, mock_delete_files, mock_lineage_for_certname,
+        mock_util):
+        """Test delete"""
+        mock_lineage_for_certname.return_value = self.test_rc
+        mock_util().checklist.return_value = (display_util.OK, ["example.org", "other.org"])
+        mock_util().yesno.return_value = False
+        self._call()
+        self.assertEqual(mock_delete_files.call_count, 0)
+
 
 class CertificatesTest(BaseCertManagerTest):
     """Tests for certbot._internal.cert_manager.certificates

certbot/tests/error_handler_test.py

@@ -14,7 +14,7 @@ from certbot.compat import os
 
 def get_signals(signums):
     """Get the handlers for an iterable of signums."""
-    return dict((s, signal.getsignal(s)) for s in signums)
+    return {s: signal.getsignal(s) for s in signums}
 
 
 def set_signals(sig_handler_dict):
@@ -28,7 +28,7 @@ def signal_receiver(signums):
     """Context manager to catch signals"""
     signals = []
     prev_handlers = get_signals(signums)  # type: Dict[int, Union[int, None, Callable]]
-    set_signals(dict((s, lambda s, _: signals.append(s)) for s in signums))
+    set_signals({s: lambda s, _: signals.append(s) for s in signums})
     yield signals
     set_signals(prev_handlers)
 

certbot/tests/main_test.py

@@ -1492,7 +1492,7 @@ class UnregisterTest(unittest.TestCase):
             'account': mock.patch('certbot._internal.main.account'),
             'client': mock.patch('certbot._internal.main.client'),
             'get_utility': test_util.patch_get_utility()}
-        self.mocks = dict((k, v.start()) for k, v in self.patchers.items())
+        self.mocks = {k: v.start() for k, v in self.patchers.items()}
 
     def tearDown(self):
         for patch in self.patchers.values():

certbot/tests/renewal_test.py

@@ -3,7 +3,7 @@ import unittest
 
 try:
     import mock
-except ImportError: # pragma: no cover
+except ImportError:  # pragma: no cover
     from unittest import mock
 
 from acme import challenges
@@ -54,6 +54,26 @@ class RenewalTest(test_util.ConfigTestCase):
         self.assertEqual(self.config.webroot_map, {})
         self.assertEqual(self.config.webroot_path, ['/var/www/test'])
 
+    def test_reuse_key_renewal_params(self):
+        self.config.rsa_key_size = 'INVALID_VALUE'
+        self.config.reuse_key = True
+        self.config.dry_run = True
+        config = configuration.NamespaceConfig(self.config)
+
+        rc_path = test_util.make_lineage(
+            self.config.config_dir, 'sample-renewal.conf')
+        lineage = storage.RenewableCert(rc_path, config)
+
+        le_client = mock.MagicMock()
+        le_client.obtain_certificate.return_value = (None, None, None, None)
+
+        from certbot._internal import renewal
+
+        with mock.patch('certbot._internal.renewal.hooks.renew_hook'):
+            renewal.renew_cert(self.config, None, le_client, lineage)
+
+        assert self.config.rsa_key_size == 2048
+
 
 class RestoreRequiredConfigElementsTest(test_util.ConfigTestCase):
     """Tests for certbot._internal.renewal.restore_required_config_elements."""

letsencrypt-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="1.8.0"
+LE_AUTO_VERSION="1.9.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -806,10 +806,7 @@ if [ -f /etc/debian_version ]; then
   BOOTSTRAP_VERSION="BootstrapDebCommon $BOOTSTRAP_DEB_COMMON_VERSION"
 elif [ -f /etc/mageia-release ]; then
   # Mageia has both /etc/mageia-release and /etc/redhat-release
-  Bootstrap() {
-    ExperimentalBootstrap "Mageia" BootstrapMageiaCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/redhat-release ]; then
   # Run DeterminePythonVersion to decide on the basis of available Python versions
   # whether to use 2.x or 3.x on RedHat-like systems.
@@ -884,31 +881,11 @@ elif [ -f /etc/redhat-release ]; then
 
   LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
-  Bootstrap() {
-    BootstrapMessage "openSUSE-based OSes"
-    BootstrapSuseCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapSuseCommon $BOOTSTRAP_SUSE_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/arch-release ]; then
-  Bootstrap() {
-    if [ "$DEBUG" = 1 ]; then
-      BootstrapMessage "Archlinux"
-      BootstrapArchCommon
-    else
-      error "Please use pacman to install letsencrypt packages:"
-      error "# pacman -S certbot certbot-apache"
-      error
-      error "If you would like to use the virtualenv way, please run the script again with the"
-      error "--debug flag."
-      exit 1
-    fi
-  }
-  BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/manjaro-release ]; then
-  Bootstrap() {
-    ExperimentalBootstrap "Manjaro Linux" BootstrapArchCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/gentoo-release ]; then
   DEPRECATED_OS=1
 elif uname | grep -iq FreeBSD ; then
@@ -921,19 +898,9 @@ elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then
   }
   BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
 elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then
-  Bootstrap() {
-    ExperimentalBootstrap "Joyent SmartOS Zone" BootstrapSmartOS
-  }
-  BOOTSTRAP_VERSION="BootstrapSmartOS $BOOTSTRAP_SMARTOS_VERSION"
+  DEPRECATED_OS=1
 else
-  Bootstrap() {
-    error "Sorry, I don't know how to bootstrap Certbot on your operating system!"
-    error
-    error "You will need to install OS dependencies, configure virtualenv, and run pip install manually."
-    error "Please see https://certbot.eff.org/docs/contributing.html#prerequisites"
-    error "for more info."
-    exit 1
-  }
+  DEPRECATED_OS=1
 fi
 
 # We handle this case after determining the normal bootstrap version to allow
@@ -1530,18 +1497,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==1.8.0 \
-    --hash=sha256:4bde86c53e30dc5bc0e78a0862045b053971703af727ac20c6a7da06596c7549 \
-    --hash=sha256:4837c516af6543ccd10d70f1498a2113bbdf9ef9a05d3a18b1558b291a2953e4
-acme==1.8.0 \
-    --hash=sha256:465033830a75f98042236f50f751f6e316735473ccb4edec0c718263f6c9ba8b \
-    --hash=sha256:ad8d067d14258d73ad2643439d9365913362308c04e66cc3010e39c868c5002d
-certbot-apache==1.8.0 \
-    --hash=sha256:8c9d981803e1156725fcfcf228afcb754b245c9d506e5b9f4fca948d6ae89aef \
-    --hash=sha256:a93c3a7ad929fe0ba5e0868e29ee2d0fe10aea2d4c638a902c4613a5c12c59b6
-certbot-nginx==1.8.0 \
-    --hash=sha256:e98e883b5ea7b29dd2e6a8ff286c7550a2d7af2fc859f47067303e510ad4fb52 \
-    --hash=sha256:fdb96c74fe42d90bbaf11a00314444ac5544ba87292a1b8b1d707f7561a3eacc
+certbot==1.9.0 \
+    --hash=sha256:d5a804d32e471050921f7b39ed9859e2e9de02824176ed78f57266222036b53a \
+    --hash=sha256:2ff9bf7d9af381c7efee22dec2dd6938d9d8fddcc9e11682b86e734164a30b57
+acme==1.9.0 \
+    --hash=sha256:d8061b396a22b21782c9b23ff9a945b23e50fca2573909a42f845e11d5658ac5 \
+    --hash=sha256:38a1630c98e144136c62eec4d2c545a1bdb1a3cd4eca82214be6b83a1f5a161f
+certbot-apache==1.9.0 \
+    --hash=sha256:09528a820d57e54984d490100644cd8a6603db97bf5776f86e95795ecfacf23d \
+    --hash=sha256:f47fb3f4a9bd927f4812121a0beefe56b163475a28f4db34c64dc838688d9e9e
+certbot-nginx==1.9.0 \
+    --hash=sha256:bb2e3f7fe17f071f350a3efa48571b8ef40a8e4b6db9c6da72539206a20b70be \
+    --hash=sha256:ab26a4f49d53b0e8bf0f903e58e2a840cda233fe1cbbc54c36ff17f973e57d65
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
@@ -1615,6 +1582,11 @@ maybe_argparse = (
     if sys.version_info < (2, 7, 0) else [])
 
 
+# Be careful when updating the pinned versions here, in particular for pip.
+# Indeed starting from 10.0, pip will build dependencies in isolation if the
+# related projects are compliant with PEP 517. This is not something we want
+# as of now, so the isolation build will need to be disabled wherever
+# pipstrap is used (see https://github.com/certbot/certbot/issues/8256).
 PACKAGES = maybe_argparse + [
     # Pip has no dependencies, as it vendors everything:
     ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/'

letsencrypt-auto-source/Dockerfile.xenial

@@ -1,36 +0,0 @@
-# For running tests, build a docker image with a passwordless sudo and a trust
-# store we can manipulate.
-
-FROM ubuntu:xenial
-
-# Add an unprivileged user:
-RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea
-
-# Install pip, sudo, and openssl:
-RUN apt-get update && \
-    apt-get -q -y install python-pip sudo openssl && \
-    apt-get clean
-
-# Use pipstrap to update to a stable and tested version of pip
-COPY ./pieces/pipstrap.py /opt
-RUN /opt/pipstrap.py
-# Pin pytest version for increased stability
-RUN pip install pytest==3.2.5 six==1.10.0
-
-# Let that user sudo:
-RUN sed -i.bkp -e \
-      's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' \
-      /etc/sudoers
-
-RUN mkdir -p /home/lea/certbot
-
-# Install fake testing CA:
-COPY ./tests/certs/ca/my-root-ca.crt.pem /usr/local/share/ca-certificates/
-
-# Copy code:
-COPY . /home/lea/certbot/letsencrypt-auto-source
-
-USER lea
-WORKDIR /home/lea
-
-CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]

letsencrypt-auto-source/certbot-auto.asc

@@ -1,11 +1,11 @@
 -----BEGIN PGP SIGNATURE-----
 
-iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl9XuHIACgkQTRfJlc2X
-dfIU8wgAkwXao63sZxfiRfeQfzyM01oYEaqjp17gX/f0QhxvmrBUIdBKsF3TBZ9H
-7c3NYlBxJ31/a5PVfzElQJAzqMl4yEdlZK1mxKEepQycmW+vHOq8DOfpvOU957ro
-cRBpDcu5BK+/tKPqVTHpLRZX7SFjzpunwKmmdCz1JzxLuf0Wgrqmq678Yyh6rLdT
-96j7bDhHCDg0R2RC3hL1yk9HyMdh/nSKUYNnQdqAi/YSybclHXBU2NJURupMrei1
-6LLoE6I8wo4LXptCaM48kQEHBKGwdMWeimVkos0YbmIzcPbmmetmu+MvrL/T/Dz8
-6OEBdYbAkMdT2gzseq76CYEUeWhL0g==
-=1K2n
+iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl98wk8ACgkQTRfJlc2X
+dfIctgf/TO83xXJJ8haqxke0ehHCwcmipX7ijPhwvaUTSqciMa56KnGJLNp1lAVz
+vv8sfHUf7NSvGlRg+5M0szWY25+JzveJDNzse3rOzFmxA1GNKUycE3/zE/IdBRwN
+fmxJHaUBrBL2erBZPHe8gFGTvlzopBoGSmQpWGY3hIufPWKBJohCbTscKbaa9hyz
+njmMvwRdeqzvLWVZ4jNDDsil9kKl2Emue3guzA/cvVxHe17DZyLDfqni7ysZIcTn
+wPAQzpLBKHyiqVRoVk+BJ6Z6wamW4NAxKbjXy9GrHy4txlfW8tGd3jXha8yWqJeH
+xEFK02Zp+T17+C5uqEW4o0cIofMjCw==
+=9UGf
 -----END PGP SIGNATURE-----

letsencrypt-auto-source/letsencrypt-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="1.9.0.dev0"
+LE_AUTO_VERSION="1.10.0.dev0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -799,17 +799,10 @@ BootstrapMageiaCommon() {
 # that function. If Bootstrap is set to a function that doesn't install any
 # packages BOOTSTRAP_VERSION is not set.
 if [ -f /etc/debian_version ]; then
-  Bootstrap() {
-    BootstrapMessage "Debian-based OSes"
-    BootstrapDebCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapDebCommon $BOOTSTRAP_DEB_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/mageia-release ]; then
   # Mageia has both /etc/mageia-release and /etc/redhat-release
-  Bootstrap() {
-    ExperimentalBootstrap "Mageia" BootstrapMageiaCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/redhat-release ]; then
   # Run DeterminePythonVersion to decide on the basis of available Python versions
   # whether to use 2.x or 3.x on RedHat-like systems.
@@ -884,31 +877,11 @@ elif [ -f /etc/redhat-release ]; then
 
   LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
-  Bootstrap() {
-    BootstrapMessage "openSUSE-based OSes"
-    BootstrapSuseCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapSuseCommon $BOOTSTRAP_SUSE_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/arch-release ]; then
-  Bootstrap() {
-    if [ "$DEBUG" = 1 ]; then
-      BootstrapMessage "Archlinux"
-      BootstrapArchCommon
-    else
-      error "Please use pacman to install letsencrypt packages:"
-      error "# pacman -S certbot certbot-apache"
-      error
-      error "If you would like to use the virtualenv way, please run the script again with the"
-      error "--debug flag."
-      exit 1
-    fi
-  }
-  BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/manjaro-release ]; then
-  Bootstrap() {
-    ExperimentalBootstrap "Manjaro Linux" BootstrapArchCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/gentoo-release ]; then
   DEPRECATED_OS=1
 elif uname | grep -iq FreeBSD ; then
@@ -921,19 +894,9 @@ elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then
   }
   BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
 elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then
-  Bootstrap() {
-    ExperimentalBootstrap "Joyent SmartOS Zone" BootstrapSmartOS
-  }
-  BOOTSTRAP_VERSION="BootstrapSmartOS $BOOTSTRAP_SMARTOS_VERSION"
+  DEPRECATED_OS=1
 else
-  Bootstrap() {
-    error "Sorry, I don't know how to bootstrap Certbot on your operating system!"
-    error
-    error "You will need to install OS dependencies, configure virtualenv, and run pip install manually."
-    error "Please see https://certbot.eff.org/docs/contributing.html#prerequisites"
-    error "for more info."
-    exit 1
-  }
+  DEPRECATED_OS=1
 fi
 
 # We handle this case after determining the normal bootstrap version to allow
@@ -1530,18 +1493,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==1.8.0 \
-    --hash=sha256:4bde86c53e30dc5bc0e78a0862045b053971703af727ac20c6a7da06596c7549 \
-    --hash=sha256:4837c516af6543ccd10d70f1498a2113bbdf9ef9a05d3a18b1558b291a2953e4
-acme==1.8.0 \
-    --hash=sha256:465033830a75f98042236f50f751f6e316735473ccb4edec0c718263f6c9ba8b \
-    --hash=sha256:ad8d067d14258d73ad2643439d9365913362308c04e66cc3010e39c868c5002d
-certbot-apache==1.8.0 \
-    --hash=sha256:8c9d981803e1156725fcfcf228afcb754b245c9d506e5b9f4fca948d6ae89aef \
-    --hash=sha256:a93c3a7ad929fe0ba5e0868e29ee2d0fe10aea2d4c638a902c4613a5c12c59b6
-certbot-nginx==1.8.0 \
-    --hash=sha256:e98e883b5ea7b29dd2e6a8ff286c7550a2d7af2fc859f47067303e510ad4fb52 \
-    --hash=sha256:fdb96c74fe42d90bbaf11a00314444ac5544ba87292a1b8b1d707f7561a3eacc
+certbot==1.9.0 \
+    --hash=sha256:d5a804d32e471050921f7b39ed9859e2e9de02824176ed78f57266222036b53a \
+    --hash=sha256:2ff9bf7d9af381c7efee22dec2dd6938d9d8fddcc9e11682b86e734164a30b57
+acme==1.9.0 \
+    --hash=sha256:d8061b396a22b21782c9b23ff9a945b23e50fca2573909a42f845e11d5658ac5 \
+    --hash=sha256:38a1630c98e144136c62eec4d2c545a1bdb1a3cd4eca82214be6b83a1f5a161f
+certbot-apache==1.9.0 \
+    --hash=sha256:09528a820d57e54984d490100644cd8a6603db97bf5776f86e95795ecfacf23d \
+    --hash=sha256:f47fb3f4a9bd927f4812121a0beefe56b163475a28f4db34c64dc838688d9e9e
+certbot-nginx==1.9.0 \
+    --hash=sha256:bb2e3f7fe17f071f350a3efa48571b8ef40a8e4b6db9c6da72539206a20b70be \
+    --hash=sha256:ab26a4f49d53b0e8bf0f903e58e2a840cda233fe1cbbc54c36ff17f973e57d65
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
@@ -1615,6 +1578,11 @@ maybe_argparse = (
     if sys.version_info < (2, 7, 0) else [])
 
 
+# Be careful when updating the pinned versions here, in particular for pip.
+# Indeed starting from 10.0, pip will build dependencies in isolation if the
+# related projects are compliant with PEP 517. This is not something we want
+# as of now, so the isolation build will need to be disabled wherever
+# pipstrap is used (see https://github.com/certbot/certbot/issues/8256).
 PACKAGES = maybe_argparse + [
     # Pip has no dependencies, as it vendors everything:
     ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/'

letsencrypt-auto-source/letsencrypt-auto.template

@@ -321,17 +321,10 @@ DeterminePythonVersion() {
 # that function. If Bootstrap is set to a function that doesn't install any
 # packages BOOTSTRAP_VERSION is not set.
 if [ -f /etc/debian_version ]; then
-  Bootstrap() {
-    BootstrapMessage "Debian-based OSes"
-    BootstrapDebCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapDebCommon $BOOTSTRAP_DEB_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/mageia-release ]; then
   # Mageia has both /etc/mageia-release and /etc/redhat-release
-  Bootstrap() {
-    ExperimentalBootstrap "Mageia" BootstrapMageiaCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/redhat-release ]; then
   # Run DeterminePythonVersion to decide on the basis of available Python versions
   # whether to use 2.x or 3.x on RedHat-like systems.
@@ -406,31 +399,11 @@ elif [ -f /etc/redhat-release ]; then
 
   LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
-  Bootstrap() {
-    BootstrapMessage "openSUSE-based OSes"
-    BootstrapSuseCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapSuseCommon $BOOTSTRAP_SUSE_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/arch-release ]; then
-  Bootstrap() {
-    if [ "$DEBUG" = 1 ]; then
-      BootstrapMessage "Archlinux"
-      BootstrapArchCommon
-    else
-      error "Please use pacman to install letsencrypt packages:"
-      error "# pacman -S certbot certbot-apache"
-      error
-      error "If you would like to use the virtualenv way, please run the script again with the"
-      error "--debug flag."
-      exit 1
-    fi
-  }
-  BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/manjaro-release ]; then
-  Bootstrap() {
-    ExperimentalBootstrap "Manjaro Linux" BootstrapArchCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/gentoo-release ]; then
   DEPRECATED_OS=1
 elif uname | grep -iq FreeBSD ; then
@@ -443,19 +416,9 @@ elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then
   }
   BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
 elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then
-  Bootstrap() {
-    ExperimentalBootstrap "Joyent SmartOS Zone" BootstrapSmartOS
-  }
-  BOOTSTRAP_VERSION="BootstrapSmartOS $BOOTSTRAP_SMARTOS_VERSION"
+  DEPRECATED_OS=1
 else
-  Bootstrap() {
-    error "Sorry, I don't know how to bootstrap Certbot on your operating system!"
-    error
-    error "You will need to install OS dependencies, configure virtualenv, and run pip install manually."
-    error "Please see https://certbot.eff.org/docs/contributing.html#prerequisites"
-    error "for more info."
-    exit 1
-  }
+  DEPRECATED_OS=1
 fi
 
 # We handle this case after determining the normal bootstrap version to allow

letsencrypt-auto-source/pieces/certbot-requirements.txt

@@ -1,12 +1,12 @@
-certbot==1.8.0 \
-    --hash=sha256:4bde86c53e30dc5bc0e78a0862045b053971703af727ac20c6a7da06596c7549 \
-    --hash=sha256:4837c516af6543ccd10d70f1498a2113bbdf9ef9a05d3a18b1558b291a2953e4
-acme==1.8.0 \
-    --hash=sha256:465033830a75f98042236f50f751f6e316735473ccb4edec0c718263f6c9ba8b \
-    --hash=sha256:ad8d067d14258d73ad2643439d9365913362308c04e66cc3010e39c868c5002d
-certbot-apache==1.8.0 \
-    --hash=sha256:8c9d981803e1156725fcfcf228afcb754b245c9d506e5b9f4fca948d6ae89aef \
-    --hash=sha256:a93c3a7ad929fe0ba5e0868e29ee2d0fe10aea2d4c638a902c4613a5c12c59b6
-certbot-nginx==1.8.0 \
-    --hash=sha256:e98e883b5ea7b29dd2e6a8ff286c7550a2d7af2fc859f47067303e510ad4fb52 \
-    --hash=sha256:fdb96c74fe42d90bbaf11a00314444ac5544ba87292a1b8b1d707f7561a3eacc
+certbot==1.9.0 \
+    --hash=sha256:d5a804d32e471050921f7b39ed9859e2e9de02824176ed78f57266222036b53a \
+    --hash=sha256:2ff9bf7d9af381c7efee22dec2dd6938d9d8fddcc9e11682b86e734164a30b57
+acme==1.9.0 \
+    --hash=sha256:d8061b396a22b21782c9b23ff9a945b23e50fca2573909a42f845e11d5658ac5 \
+    --hash=sha256:38a1630c98e144136c62eec4d2c545a1bdb1a3cd4eca82214be6b83a1f5a161f
+certbot-apache==1.9.0 \
+    --hash=sha256:09528a820d57e54984d490100644cd8a6603db97bf5776f86e95795ecfacf23d \
+    --hash=sha256:f47fb3f4a9bd927f4812121a0beefe56b163475a28f4db34c64dc838688d9e9e
+certbot-nginx==1.9.0 \
+    --hash=sha256:bb2e3f7fe17f071f350a3efa48571b8ef40a8e4b6db9c6da72539206a20b70be \
+    --hash=sha256:ab26a4f49d53b0e8bf0f903e58e2a840cda233fe1cbbc54c36ff17f973e57d65

letsencrypt-auto-source/pieces/pipstrap.py

@@ -67,6 +67,11 @@ maybe_argparse = (
     if sys.version_info < (2, 7, 0) else [])
 
 
+# Be careful when updating the pinned versions here, in particular for pip.
+# Indeed starting from 10.0, pip will build dependencies in isolation if the
+# related projects are compliant with PEP 517. This is not something we want
+# as of now, so the isolation build will need to be disabled wherever
+# pipstrap is used (see https://github.com/certbot/certbot/issues/8256).
 PACKAGES = maybe_argparse + [
     # Pip has no dependencies, as it vendors everything:
     ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/'

snap/hooks/prepare-plug-plugin

@@ -5,7 +5,8 @@ if [ "$(snapctl get trust-plugin-with-root)" = "ok" ]; then
     snapctl unset trust-plugin-with-root
     exit 0
 else
-    echo "Only connect this interface if you trust the plugin author to have root on the system"
-    echo "Run \`snap set $SNAP_NAME trust-plugin-with-root=ok\` to acknowledge this and then run this command again to perform the connection"
+    echo "Only connect this interface if you trust the plugin author to have root on the system."
+    echo "Run \`snap set $SNAP_NAME trust-plugin-with-root=ok\` to acknowledge this and then run this command again to perform the connection."
+    echo "If that doesn't work, you may need to remove all certbot-dns-* plugins from the system, then try installing the certbot snap again."
     exit 1
 fi

snap/snapcraft.yaml

@@ -20,13 +20,13 @@ adopt-info: certbot
 
 apps:
   certbot:
-    command: certbot.wrapper
+    command: bin/python3 $SNAP/bin/certbot
     environment:
       PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
       AUGEAS_LENS_LIB: "$SNAP/usr/share/augeas/lenses/dist"
       CERTBOT_SNAPPED: "True"
   renew:
-    command: certbot.wrapper -q renew
+    command: bin/python3 $SNAP/bin/certbot -q renew
     daemon: oneshot
     environment:
       PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
@@ -71,9 +71,6 @@ parts:
       - python3-distutils
       - python3-pkg-resources
       - python3.8-minimal
-      # added for certbot.wrapper script:
-      - curl
-      - jq
     # To build cryptography and cffi if needed
     build-packages: [gcc, libffi-dev, libssl-dev, git, libaugeas-dev, python3-dev]
     build-environment:
@@ -83,11 +80,7 @@ parts:
         snapcraftctl pull
         cd $SNAPCRAFT_PART_SRC
         python3 tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt | grep -v python-augeas > snap-constraints.txt
-        snapcraftctl set-version `git describe|sed s/^v//`
-  wrappers:
-    plugin: dump
-    source: .
-    stage: [certbot.wrapper]
+        snapcraftctl set-version `grep -oP "__version__ = '\K.*(?=')" $SNAPCRAFT_PART_SRC/certbot/certbot/__init__.py`
   shared-metadata:
     plugin: dump
     source: .

tests/letstest/scripts/test_leauto_upgrades.sh

@@ -105,9 +105,18 @@ if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python
     exit 1
 fi
 
-EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2)
+# On systems like Debian where certbot-auto is deprecated, we expect it to
+# leave existing Certbot installations unmodified so we check for the same
+# version that was initially installed below.  Once certbot-auto is deprecated
+# on RHEL systems, we can unconditionally check for INITIAL_VERSION.
+if [ -f /etc/debian_version ]; then
+    EXPECTED_VERSION="$INITIAL_VERSION"
+else
+    EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2)
+fi
+
 if ! /opt/eff.org/certbot/venv/bin/letsencrypt --version 2>&1 | tail -n1 | grep "^certbot $EXPECTED_VERSION$" ; then
-    echo upgrade appeared to fail
+    echo unexpected certbot version found
     exit 1
 fi
 

tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh

@@ -16,6 +16,24 @@ sudo chown root "$LE_AUTO_PATH"
 sudo chmod 0755 "$LE_AUTO_PATH"
 export PATH="$LE_AUTO_DIR:$PATH"
 
+# On systems like Debian where certbot-auto is deprecated, we expect
+# certbot-auto to error and refuse to install Certbot.  Once certbot-auto is
+# deprecated on RHEL systems, we can unconditionally run this code.
+if [ -f /etc/debian_version ]; then
+    set +o pipefail
+    if ! letsencrypt-auto --debug --version | grep "Certbot cannot be installed."; then
+        echo "letsencrypt-auto didn't report being uninstallable."
+        exit 1
+    fi
+    if [ ${PIPESTATUS[0]} != 1 ]; then
+        echo "letsencrypt-auto didn't exit with status 1 as expected"
+        exit 1
+    fi
+    # letsencrypt-auto is deprecated and cannot be installed on this system so
+    # we cannot run the rest of this test.
+    exit 0
+fi
+
 letsencrypt-auto --os-packages-only --debug --version
 
 # This script sets the environment variables PYTHON_NAME, VENV_PATH, and

tools/create_github_release.py

@@ -1,88 +0,0 @@
-#!/usr/bin/env python
-"""
-Post-release script to download artifacts from azure pipelines and use them to create
-a GitHub release.
-
-Setup:
- - Create a github personal access token
-   - https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token#creating-a-token
-   - You'll need repo scope
-   - Save the token to somewhere like ~/.ssh/githubpat.txt
-
-Run:
-
-python tools/create_github_release.py ~/.ssh/githubpat.txt
-"""
-
-import requests
-import sys
-import tempfile
-from zipfile import ZipFile
-
-from azure.devops.connection import Connection
-from github import Github
-
-def download_azure_artifacts(tempdir):
-    """Download and unzip build artifacts from Azure pipelines.
-
-    :param str path: path to a temporary directory to save the files
-
-    :returns: released certbot version number as a prefix-free string
-    :rtype: str
-
-    """
-    # Create a connection to the azure org
-    organization_url = 'https://dev.azure.com/certbot'
-    connection = Connection(base_url=organization_url)
-
-    # Find the build artifacts
-    build_client = connection.clients.get_build_client()
-    get_builds_response = build_client.get_builds('certbot', definitions='3')
-    build_id = get_builds_response.value[0].id
-    artifacts = build_client.get_artifacts('certbot', build_id)
-
-    # Save and unzip files
-    for filename in ('windows-installer', 'changelog'):
-        url = build_client.get_artifact('certbot', build_id, filename).resource.download_url
-        r = requests.get(url)
-        r.raise_for_status()
-        with open(tempdir + '/' + filename + '.zip', 'wb') as f:
-            f.write(r.content)
-        with ZipFile(tempdir + '/' + filename + '.zip', 'r') as zipObj:
-           zipObj.extractall(tempdir)
-
-    version = build_client.get_build('certbot', build_id).source_branch.split('v')[1]
-    return version
-
-def create_github_release(github_access_token, tempdir, version):
-    """Use build artifacts to create a github release, including uploading additional assets
-
-    :param str github_access_token: string containing github access token
-    :param str path: path to a temporary directory where azure artifacts are located
-    :param str version: Certbot version number, e.g. 1.7.0
-
-    """
-    # Create release
-    g = Github(github_access_token)
-    repo = g.get_user('certbot').get_repo('certbot')
-    release_notes = open(tempdir + '/changelog/release_notes.md', 'r').read()
-    release= repo.create_git_release('v{0}'.format(version),
-                                     'Certbot {0}'.format(version),
-                                     release_notes,
-                                     draft=True)
-
-    # Upload windows installer to release
-    release.upload_asset(tempdir + '/windows-installer/certbot-beta-installer-win32.exe')
-    release.update_release(release.title, release.body, draft=False)
-
-def main(args):
-    github_access_token_file = args[0]
-
-    github_access_token = open(github_access_token_file, 'r').read().rstrip()
-
-    with tempfile.TemporaryDirectory() as tempdir:
-        version = download_azure_artifacts(tempdir)
-        create_github_release(github_access_token, tempdir, version)
-
-if __name__ == "__main__":
-    main(sys.argv[1:])

tools/dev_constraints.txt

@@ -39,7 +39,7 @@ future==0.16.0
 futures==3.3.0
 filelock==3.0.12
 google-api-python-client==1.5.5
-httplib2==0.10.3
+httplib2==0.18.1
 imagesize==0.7.1
 importlib-metadata==0.23
 ipdb==0.12.3

tools/finish_release.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python
+"""
+Post-release script to publish artifacts created from Azure Pipelines.
+
+This currently includes:
+
+* Moving snaps from the beta channel to the stable channel
+* Publishing the Windows installer in a GitHub release
+
+Setup:
+ - Create a github personal access token
+   - https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token#creating-a-token
+   - You'll need repo scope
+   - Save the token to somewhere like ~/.ssh/githubpat.txt
+ - Install the snapcraft command line tool and log in to a privileged account.
+   - https://snapcraft.io/docs/installing-snapcraft
+   - Use the command `snapcraft login` to log in.
+
+Run:
+
+python tools/finish_release.py ~/.ssh/githubpat.txt
+"""
+
+import glob
+import os.path
+import re
+import subprocess
+import sys
+import tempfile
+from zipfile import ZipFile
+
+from azure.devops.connection import Connection
+from github import Github
+import requests
+
+# Path to the root directory of the Certbot repository containing this script
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+# This list contains the names of all Certbot DNS plugins
+DNS_PLUGINS = [os.path.basename(path) for path in glob.glob(os.path.join(REPO_ROOT, 'certbot-dns-*'))]
+# This list contains the name of all Certbot snaps that should be published to
+# the stable channel.
+SNAPS = ['certbot'] + DNS_PLUGINS
+# This is the count of the architectures currently supported by our snaps used
+# for sanity checking.
+SNAP_ARCH_COUNT = 3
+
+def download_azure_artifacts(tempdir):
+    """Download and unzip build artifacts from Azure pipelines.
+
+    :param str path: path to a temporary directory to save the files
+
+    :returns: released certbot version number as a prefix-free string
+    :rtype: str
+
+    """
+    # Create a connection to the azure org
+    organization_url = 'https://dev.azure.com/certbot'
+    connection = Connection(base_url=organization_url)
+
+    # Find the build artifacts
+    build_client = connection.clients.get_build_client()
+    get_builds_response = build_client.get_builds('certbot', definitions='3')
+    build_id = get_builds_response.value[0].id
+    artifacts = build_client.get_artifacts('certbot', build_id)
+
+    # Save and unzip files
+    for filename in ('windows-installer', 'changelog'):
+        print("Downloading artifact %s" % filename)
+        url = build_client.get_artifact('certbot', build_id, filename).resource.download_url
+        r = requests.get(url)
+        r.raise_for_status()
+        with open(tempdir + '/' + filename + '.zip', 'wb') as f:
+            f.write(r.content)
+        print("Extracting %s" % filename)
+        with ZipFile(tempdir + '/' + filename + '.zip', 'r') as zipObj:
+           zipObj.extractall(tempdir)
+
+    version = build_client.get_build('certbot', build_id).source_branch.split('v')[1]
+    return version
+
+def create_github_release(github_access_token, tempdir, version):
+    """Use build artifacts to create a github release, including uploading additional assets
+
+    :param str github_access_token: string containing github access token
+    :param str path: path to a temporary directory where azure artifacts are located
+    :param str version: Certbot version number, e.g. 1.7.0
+
+    """
+    # Create release
+    g = Github(github_access_token)
+    repo = g.get_user('certbot').get_repo('certbot')
+    release_notes = open(tempdir + '/changelog/release_notes.md', 'r').read()
+    print("Creating git release")
+    release= repo.create_git_release('v{0}'.format(version),
+                                     'Certbot {0}'.format(version),
+                                     release_notes,
+                                     draft=True)
+
+    # Upload windows installer to release
+    print("Uploading windows installer")
+    release.upload_asset(tempdir + '/windows-installer/certbot-beta-installer-win32.exe')
+    release.update_release(release.title, release.body, draft=False)
+
+
+def assert_logged_into_snapcraft():
+    """Confirms that snapcraft is logged in to an account.
+
+    :raises SystemExit: if the command snapcraft is unavailable or it
+        isn't logged into an account
+
+    """
+    cmd = 'snapcraft whoami'.split()
+    try:
+        subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL,
+                       stderr=subprocess.DEVNULL, universal_newlines=True)
+    except (subprocess.CalledProcessError, OSError):
+        print("Please make sure that the command line tool snapcraft is")
+        print("installed and that you have logged in to an account by running")
+        print("'snapcraft login'.")
+        sys.exit(1)
+
+
+def get_snap_revisions(snap, version):
+    """Finds the revisions for the snap and version in the beta channel.
+
+    If you call this function without being logged in with snapcraft, it
+    will hang with no output.
+
+    :param str snap: the name of the snap on the snap store
+    :param str version: snap version number, e.g. 1.7.0
+
+    :returns: list of revision numbers
+    :rtype: `list` of `str`
+
+    :raises subprocess.CalledProcessError: if the snapcraft command
+        fails
+
+    :raises AssertionError: if the expected snaps are not found
+
+    """
+    print('Getting revision numbers for', snap, version)
+    cmd = ['snapcraft', 'status', snap]
+    process = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, universal_newlines=True)
+    pattern = f'^\s+beta\s+{version}\s+(\d+)\s*$'
+    revisions = re.findall(pattern, process.stdout, re.MULTILINE)
+    assert len(revisions) == SNAP_ARCH_COUNT, f'Unexpected number of snaps found for {snap} {version}'
+    return revisions
+
+
+def promote_snaps(version):
+    """Promotes all Certbot snaps from the beta to stable channel.
+
+    If the snaps have already been released to the stable channel, this
+    function will try to release them again which has no effect.
+
+    :param str version: the version number that should be found in the
+        beta channel, e.g. 1.7.0
+
+    :raises SystemExit: if the command snapcraft is unavailable or it
+        isn't logged into an account
+
+    :raises subprocess.CalledProcessError: if a snapcraft command fails
+        for another reason
+
+    """
+    assert_logged_into_snapcraft()
+    for snap in SNAPS:
+        revisions = get_snap_revisions(snap, version)
+        # The loop below is kind of slow, so let's print some output about what
+        # it is doing.
+        print('Releasing', snap, 'snaps to the stable channel')
+        for revision in revisions:
+            cmd = ['snapcraft', 'release', snap, revision, 'stable']
+            try:
+                subprocess.run(cmd, check=True, stdout=subprocess.PIPE, universal_newlines=True)
+            except subprocess.CalledProcessError as e:
+                print("The command", f"'{' '.join(cmd)}'", "failed.")
+                print("The output printed to stdout was:")
+                print(e.stdout)
+                raise
+
+
+def main(args):
+    github_access_token_file = args[0]
+
+    github_access_token = open(github_access_token_file, 'r').read().rstrip()
+
+    with tempfile.TemporaryDirectory() as tempdir:
+        version = download_azure_artifacts(tempdir)
+        # Once the GitHub release has been published, trying to publish it
+        # again fails. Publishing the snaps can be done multiple times though
+        # so we do that first to make it easier to run the script again later
+        # if something goes wrong.
+        promote_snaps(version)
+        create_github_release(github_access_token, tempdir, version)
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

tools/retry.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Retries a command if it fails.
+#
+# This is based on travis_retry.bash
+# https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_retry.bash.
+set -e
+result=0
+count=1
+while [[ "${count}" -le 3 ]]; do
+  result=0
+  "${@}" || result="${?}"
+  if [[ $result -eq 0 ]]; then break; fi
+  count="$((count + 1))"
+  sleep 1
+done
+
+exit "${result}"

tools/snap/README.md

@@ -1,58 +1,6 @@
-# Certbot Plugin Snaps
+# Certbot Snaps
 
-This is a proof of concept of how a Certbot snap might support plugin snaps
-that add functionality to Certbot using its existing plugin API.
-
-## Architecture
-
-This is a description of how Certbot plugin functionality is exposed via snaps.
-For information on Certbot's plugin architecture itself, see the [Certbot
-documentation on
-plugins](https://certbot.eff.org/docs/contributing.html#plugin-architecture).
-
-The Certbot snap itself is a classic snap. Plugin snaps are regular confined
-snaps, but normally do not provide any "apps" themselves. Plugin snaps export
-loadable Python modules to the Certbot snap via a snap content interface.
-
-Certbot itself accepts a `CERTBOT_PLUGIN_PATH` environment variable. This
-support is currently patched but this is intended to be upstreamed. The
-variable, if set, should contain a `:`-separated list of paths to add to
-Certbot's plugin search path.
-
-The Certbot snap runs Certbot via a wrapper which examines its list of
-connected interfaces, sets `CERTBOT_PLUGIN_PATH` accordingly, and then `exec`s
-Certbot itself.
-
-## Use (Production)
-
-_Note: this production use example assumes that these snaps are available in
-stable channels in the Snap Store, which they aren't yet. See below for
-development instructions._
-
-To use a Certbot plugin snap, install both the plugin snap and the Certbot snap
-as usual. Plugin snaps are confined as normal; the Certbot snap is a classic
-snap and thus needs `--classic` during installation. For example:
-
-    snap install --classic certbot
-    snap set certbot trust-plugin-with-root=ok
-    snap install certbot-dns-dnsimple
-
-Then connect the plugin snap to the main certbot snap as follows. Note that
-this connection allows the plugin snap code to run inside the certbot process,
-which has access to your host system. Only perform this step if you trust the
-plugin author to have "root" on your system.
-
-    sudo snap connect certbot:plugin certbot-dns-dnsimple
-
-Now certbot will automatically load and use the plugin when it is run. To check
-that this has worked, `certbot plugins` should list the plugin.
-
-You can now operate the plugin as normal.
-
-## Use (Testing and Development)
-
-To try this out, you'll need to build the snaps (a patched Certbot snap and a
-plugin snap) manually.
+## Local Testing and Development
 
 ### Initial VM Set Up
 
@@ -95,25 +43,3 @@ The instructions below clean up the build environment so it can reliably be used
 5. `rm certbot-dns-dnsimple_*_amd64.snap`
 6. `snapcraft clean --use-lxd`
 7. `cd ..`
-
-## Publishing Permissions
-
-There are security implications to permitting anyone to publish, without
-review, a plugin into the Snap Store which will then run in Certbot's classic
-snap context, with full access to the host system.
-
-At a minimum, it is clear that this should happen only with the user's explicit
-opt-in action.
-
-As implemented, Certbot will only load plugins connected via the snap interface
-mechanism, so permission is effectively delegated to what interface connections
-the snap infrastucture will permit.
-
-We have approval from the snap team to use this design as long as we make it
-explicit what a user is agreeing to when they connect a plugin to the
-Certbot snap. That work was completed in
-https://github.com/certbot/certbot/issues/8013.
-
-## Outstanding issues
-
-[Outstanding items relating to plugin support in Certbot snaps are tracked on GitHub](https://github.com/certbot/certbot/issues?q=is%3Aopen+is%3Aissue+label%3A%22area%3A+snaps%22).

tools/snap/build_remote.py

@@ -52,27 +52,25 @@ def _build_snap(target, archs, status, lock):
         sys.stdout.flush()
 
         with lock:
+            dump_output = exit_code != 0
             failed_archs = [arch for arch in archs if status[target][arch] == 'Failed to build']
             if exit_code == 0 and not failed_archs:
                 # We expect to have all target snaps available, or something bad happened.
                 snaps_list = glob.glob(join(workspace, '*.snap'))
                 if not len(snaps_list) == len(archs):
                     print(f'Some of the expected snaps for a successful build are missing (current list: {snaps_list}).')
-                    print('Dumping snapcraft remote-build output build:')
-                    print('\n'.join(process_output))
+                    dump_output = True
                 else:
                     break
-
             if failed_archs:
                 # We expect each failed build to have a log file, or something bad happened.
-                missing_outputs = False
                 for arch in failed_archs:
                     if not exists(join(workspace, f'{target}_{arch}.txt')):
-                        missing_outputs = True
+                        dump_output = True
                         print(f'Missing output on a failed build {target} for {arch}.')
-                if missing_outputs:
-                    print('Dumping snapcraft remote-build output build:')
-                    print('\n'.join(process_output))
+            if dump_output:
+                print(f'Dumping snapcraft remote-build output build for {target}:')
+                print('\n'.join(process_output))
 
         # Retry the remote build if it has been interrupted (non zero status code) or if some builds have failed.
         retry = retry - 1

tools/snap/generate_dnsplugins_snapcraft.sh

@@ -15,7 +15,7 @@ name: ${PLUGIN}
 summary: ${DESCRIPTION}
 description: ${DESCRIPTION}
 confinement: strict
-grade: devel
+grade: stable
 base: core20
 adopt-info: ${PLUGIN}
 

tox.ini

@@ -62,6 +62,7 @@ source_paths =
 [testenv]
 passenv =
     CERTBOT_NO_PIN
+commands_pre = python {toxinidir}/letsencrypt-auto-source/pieces/pipstrap.py
 commands =
     !cover: {[base]install_and_test} {[base]all_packages}
     !cover: python tests/lock_test.py
@@ -187,11 +188,12 @@ whitelist_externals =
 passenv =
     DOCKER_*
 
-[testenv:le_auto_xenial]
-# At the moment, this tests under Python 2.7 only.
+[testenv:le_auto_centos6]
+# At the moment, this tests under Python 2.6 only, as only that version is
+# readily available on the CentOS 6 Docker image.
 commands =
     python {toxinidir}/tests/modification-check.py
-    docker build -f letsencrypt-auto-source/Dockerfile.xenial -t lea letsencrypt-auto-source
+    docker build -f letsencrypt-auto-source/Dockerfile.redhat6 --build-arg REDHAT_DIST_FLAVOR=centos -t lea letsencrypt-auto-source
     docker run --rm -t lea
 whitelist_externals =
     docker
@@ -199,16 +201,6 @@ passenv =
     DOCKER_*
     TARGET_BRANCH
 
-[testenv:le_auto_centos6]
-# At the moment, this tests under Python 2.6 only, as only that version is
-# readily available on the CentOS 6 Docker image.
-commands =
-    docker build -f letsencrypt-auto-source/Dockerfile.redhat6 --build-arg REDHAT_DIST_FLAVOR=centos -t lea letsencrypt-auto-source
-    docker run --rm -t lea
-whitelist_externals =
-    docker
-passenv = DOCKER_*
-
 [testenv:le_auto_oraclelinux6]
 # At the moment, this tests under Python 2.6 only, as only that version is
 # readily available on the Oracle Linux 6 Docker image.
@@ -283,28 +275,28 @@ setenv = AWS_DEFAULT_REGION=us-east-1
 
 [testenv:test-farm-apache2]
 changedir = {[testenv:test-farm-tests-base]changedir}
-commands = python multitester.py apache2_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir}
+commands = {toxinidir}/tools/retry.sh python multitester.py apache2_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir}
 deps = {[testenv:test-farm-tests-base]deps}
 passenv = {[testenv:test-farm-tests-base]passenv}
 setenv = {[testenv:test-farm-tests-base]setenv}
 
 [testenv:test-farm-leauto-upgrades]
 changedir = {[testenv:test-farm-tests-base]changedir}
-commands = python multitester.py auto_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_leauto_upgrades.sh --repo {toxinidir}
+commands = {toxinidir}/tools/retry.sh python multitester.py auto_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_leauto_upgrades.sh --repo {toxinidir}
 deps = {[testenv:test-farm-tests-base]deps}
 passenv = {[testenv:test-farm-tests-base]passenv}
 setenv = {[testenv:test-farm-tests-base]setenv}
 
 [testenv:test-farm-certonly-standalone]
 changedir = {[testenv:test-farm-tests-base]changedir}
-commands = python multitester.py auto_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_letsencrypt_auto_certonly_standalone.sh --repo {toxinidir}
+commands = {toxinidir}/tools/retry.sh python multitester.py auto_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_letsencrypt_auto_certonly_standalone.sh --repo {toxinidir}
 deps = {[testenv:test-farm-tests-base]deps}
 passenv = {[testenv:test-farm-tests-base]passenv}
 setenv = {[testenv:test-farm-tests-base]setenv}
 
 [testenv:test-farm-sdists]
 changedir = {[testenv:test-farm-tests-base]changedir}
-commands = python multitester.py targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_sdists.sh --repo {toxinidir}
+commands = {toxinidir}/tools/retry.sh python multitester.py targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_sdists.sh --repo {toxinidir}
 deps = {[testenv:test-farm-tests-base]deps}
 passenv = {[testenv:test-farm-tests-base]passenv}
 setenv = {[testenv:test-farm-tests-base]setenv}

windows-installer/construct.py

@@ -177,7 +177,7 @@ def _prepare_environment():
 
 
 if __name__ == '__main__':
-    if not os.name == 'nt':
+    if os.name != 'nt':
         raise RuntimeError('This script must be run under Windows.')
 
     if ctypes.windll.shell32.IsUserAnAdmin() == 0: