Focus on docker jobs

.azure-pipelines/main.yml

@@ -5,4 +5,3 @@ pr:
 
 jobs:
   - template: templates/jobs/standard-tests-jobs.yml
-

.azure-pipelines/templates/jobs/extended-tests-jobs.yml

@@ -4,7 +4,7 @@ jobs:
       - name: IMAGE_NAME
         value: ubuntu-18.04
       - name: PYTHON_VERSION
-        value: 3.9
+        value: 3.8
       - group: certbot-common
     strategy:
       matrix:
@@ -14,31 +14,30 @@ jobs:
         linux-py37:
           PYTHON_VERSION: 3.7
           TOXENV: py37
-        linux-py38:
-          PYTHON_VERSION: 3.8
-          TOXENV: py38
         linux-py37-nopin:
           PYTHON_VERSION: 3.7
           TOXENV: py37
           CERTBOT_NO_PIN: 1
-        linux-external-mock:
-          TOXENV: external-mock
         linux-boulder-v1-integration-certbot-oldest:
-          PYTHON_VERSION: 3.6
           TOXENV: integration-certbot-oldest
           ACME_SERVER: boulder-v1
         linux-boulder-v2-integration-certbot-oldest:
-          PYTHON_VERSION: 3.6
           TOXENV: integration-certbot-oldest
           ACME_SERVER: boulder-v2
         linux-boulder-v1-integration-nginx-oldest:
-          PYTHON_VERSION: 3.6
           TOXENV: integration-nginx-oldest
           ACME_SERVER: boulder-v1
         linux-boulder-v2-integration-nginx-oldest:
-          PYTHON_VERSION: 3.6
           TOXENV: integration-nginx-oldest
           ACME_SERVER: boulder-v2
+        linux-boulder-v1-py27-integration:
+          PYTHON_VERSION: 2.7
+          TOXENV: integration
+          ACME_SERVER: boulder-v1
+        linux-boulder-v2-py27-integration:
+          PYTHON_VERSION: 2.7
+          TOXENV: integration
+          ACME_SERVER: boulder-v2
         linux-boulder-v1-py36-integration:
           PYTHON_VERSION: 3.6
           TOXENV: integration
@@ -63,20 +62,10 @@ jobs:
           PYTHON_VERSION: 3.8
           TOXENV: integration
           ACME_SERVER: boulder-v2
-        linux-boulder-v1-py39-integration:
-          PYTHON_VERSION: 3.9
-          TOXENV: integration
-          ACME_SERVER: boulder-v1
-        linux-boulder-v2-py39-integration:
-          PYTHON_VERSION: 3.9
-          TOXENV: integration
-          ACME_SERVER: boulder-v2
         nginx-compat:
           TOXENV: nginx_compat
-        linux-integration-rfc2136:
-          IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.8
-          TOXENV: integration-dns-rfc2136
+        le-auto-oraclelinux6:
+          TOXENV: le_auto_oraclelinux6
         docker-dev:
           TOXENV: docker_dev
         macos-farmtest-apache2:

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -1,70 +1,220 @@
 jobs:
-  - job: installer_build
+  - job: docker_build
     pool:
-      vmImage: vs2017-win2016
+      vmImage: ubuntu-18.04
+    strategy:
+      matrix:
+        amd64:
+          DOCKER_ARCH: amd64
+        # Do not run the heavy non-amd64 builds for test branches
+        ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
+          arm32v6:
+            DOCKER_ARCH: arm32v6
+          arm64v8:
+            DOCKER_ARCH: arm64v8
     steps:
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: 3.8
-          architecture: x86
-          addToPath: true
-      - script: |
-          python -m venv venv
-          venv\Scripts\python tools\pipstrap.py
-          venv\Scripts\python tools\pip_install.py -e windows-installer
-        displayName: Prepare Windows installer build environment
-      - script: |
-          venv\Scripts\construct-windows-installer
-        displayName: Build Certbot installer
-      - task: CopyFiles@2
-        inputs:
-          sourceFolder: $(System.DefaultWorkingDirectory)/windows-installer/build/nsis
-          contents: '*.exe'
-          targetFolder: $(Build.ArtifactStagingDirectory)
+      - bash: set -e && tools/docker/build.sh $(dockerTag) $DOCKER_ARCH
+        displayName: Build the Docker images
+      # We don't filter for the Docker Hub organization to continue to allow
+      # easy testing of these scripts on forks.
+      - bash: |
+          set -e
+          DOCKER_IMAGES=$(docker images --filter reference='*/certbot' --filter reference='*/dns-*' --format '{{.Repository}}')
+          docker save --output images.tar $DOCKER_IMAGES
+        displayName: Save the Docker images
+      # If the name of the tar file or artifact changes, the deploy stage will
+      # also need to be updated.
+      - bash: set -e && mv images.tar $(Build.ArtifactStagingDirectory)
+        displayName: Prepare Docker artifact
       - task: PublishPipelineArtifact@1
         inputs:
           path: $(Build.ArtifactStagingDirectory)
-          # If we change the artifact's name, it should also be changed in tools/create_github_release.py
-          artifact: windows-installer
-        displayName: Publish Windows installer
-  - job: installer_run
-    dependsOn: installer_build
-    strategy:
-      matrix:
-        win2019:
-          imageName: windows-2019
-        win2016:
-          imageName: vs2017-win2016
+          artifact: docker_$(DOCKER_ARCH)
+        displayName: Store Docker artifact
+  - job: docker_run
+    dependsOn: docker_build
     pool:
-      vmImage: $(imageName)
+      vmImage: ubuntu-18.04
     steps:
-      - powershell: |
-          if ($PSVersionTable.PSVersion.Major -ne 5) {
-              throw "Powershell version is not 5.x"
-          }
-        condition: eq(variables['imageName'], 'vs2017-win2016')
-        displayName: Check Powershell 5.x is used in vs2017-win2016
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: 3.8
-          addToPath: true
       - task: DownloadPipelineArtifact@2
         inputs:
-          artifact: windows-installer
-          path: $(Build.SourcesDirectory)/bin
-        displayName: Retrieve Windows installer
-      - script: |
-          python -m venv venv
-          venv\Scripts\python tools\pipstrap.py
-          venv\Scripts\python tools\pip_install.py -e certbot-ci
-        env:
-          PIP_NO_BUILD_ISOLATION: no
-        displayName: Prepare Certbot-CI
-      - script: |
-          set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
-          venv\Scripts\python -m pytest certbot-ci\windows_installer_integration_tests --allow-persistent-changes --installer-path $(Build.SourcesDirectory)\bin\certbot-beta-installer-win32.exe
-        displayName: Run windows installer integration tests
-      - script: |
-          set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
-          venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
-        displayName: Run certbot integration tests
+          artifact: docker_amd64
+          path: $(Build.SourcesDirectory)
+        displayName: Retrieve Docker images
+      - bash: set -e && docker load --input $(Build.SourcesDirectory)/images.tar
+        displayName: Load Docker images
+      - bash: |
+          set -ex
+          DOCKER_IMAGES=$(docker images --filter reference='*/certbot' --filter reference='*/dns-*' --format '{{.Repository}}:{{.Tag}}')
+          for DOCKER_IMAGE in ${DOCKER_IMAGES}
+            do docker run --rm "${DOCKER_IMAGE}" plugins --prepare
+          done
+#  - job: installer_build
+#    pool:
+#      vmImage: vs2017-win2016
+#    steps:
+#      - task: UsePythonVersion@0
+#        inputs:
+#          versionSpec: 3.7
+#          architecture: x86
+#          addToPath: true
+#      - script: python windows-installer/construct.py
+#        displayName: Build Certbot installer
+#      - task: CopyFiles@2
+#        inputs:
+#          sourceFolder: $(System.DefaultWorkingDirectory)/windows-installer/build/nsis
+#          contents: '*.exe'
+#          targetFolder: $(Build.ArtifactStagingDirectory)
+#      - task: PublishPipelineArtifact@1
+#        inputs:
+#          path: $(Build.ArtifactStagingDirectory)
+#          # If we change the artifact's name, it should also be changed in tools/create_github_release.py
+#          artifact: windows-installer
+#        displayName: Publish Windows installer
+#  - job: installer_run
+#    dependsOn: installer_build
+#    strategy:
+#      matrix:
+#        win2019:
+#          imageName: windows-2019
+#        win2016:
+#          imageName: vs2017-win2016
+#    pool:
+#      vmImage: $(imageName)
+#    steps:
+#      - powershell: |
+#          if ($PSVersionTable.PSVersion.Major -ne 5) {
+#              throw "Powershell version is not 5.x"
+#          }
+#        condition: eq(variables['imageName'], 'vs2017-win2016')
+#        displayName: Check Powershell 5.x is used in vs2017-win2016
+#      - task: UsePythonVersion@0
+#        inputs:
+#          versionSpec: 3.8
+#          addToPath: true
+#      - task: DownloadPipelineArtifact@2
+#        inputs:
+#          artifact: windows-installer
+#          path: $(Build.SourcesDirectory)/bin
+#        displayName: Retrieve Windows installer
+#      # pip 9.0 provided by pipstrap is not able to resolve properly the pywin32 dependency
+#      # required by certbot-ci: as a temporary workaround until pipstrap is updated, we install
+#      # a recent version of pip, but we also to disable the isolated feature as described in
+#      # https://github.com/certbot/certbot/issues/8256
+#      - script: |
+#          py -3 -m venv venv
+#          venv\Scripts\python -m pip install pip==20.2.3 setuptools==50.3.0 wheel==0.35.1
+#          venv\Scripts\python tools\pip_install.py -e certbot-ci
+#        env:
+#          PIP_NO_BUILD_ISOLATION: no
+#        displayName: Prepare Certbot-CI
+#      - script: |
+#          set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
+#          venv\Scripts\python -m pytest certbot-ci\windows_installer_integration_tests --allow-persistent-changes --installer-path $(Build.SourcesDirectory)\bin\certbot-beta-installer-win32.exe
+#        displayName: Run windows installer integration tests
+#      - script: |
+#          set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
+#          venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
+#        displayName: Run certbot integration tests
+#  - job: snaps_build
+#    pool:
+#      vmImage: ubuntu-18.04
+#    timeoutInMinutes: 0
+#    variables:
+#      # Do not run the heavy non-amd64 builds for test branches
+#      ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
+#        ARCHS: amd64 arm64 armhf
+#      ${{ if startsWith(variables['Build.SourceBranchName'], 'test-') }}:
+#        ARCHS: amd64
+#    steps:
+#      - script: |
+#          set -e
+#          sudo apt-get update
+#          sudo apt-get install -y --no-install-recommends snapd
+#          sudo snap install --classic snapcraft
+#        displayName: Install dependencies
+#      - task: UsePythonVersion@0
+#        inputs:
+#          versionSpec: 3.8
+#          addToPath: true
+#      - task: DownloadSecureFile@1
+#        name: credentials
+#        inputs:
+#          secureFile: launchpad-credentials
+#      - script: |
+#          set -e
+#          git config --global user.email "$(Build.RequestedForEmail)"
+#          git config --global user.name "$(Build.RequestedFor)"
+#          mkdir -p ~/.local/share/snapcraft/provider/launchpad
+#          cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials
+#          python3 tools/snap/build_remote.py ALL --archs ${ARCHS}
+#        displayName: Build snaps
+#      - script: |
+#          set -e
+#          mv *.snap $(Build.ArtifactStagingDirectory)
+#          mv certbot-dns-*/*.snap $(Build.ArtifactStagingDirectory)
+#        displayName: Prepare artifacts
+#      - task: PublishPipelineArtifact@1
+#        inputs:
+#          path: $(Build.ArtifactStagingDirectory)
+#          artifact: snaps
+#        displayName: Store snaps artifacts
+#  - job: snap_run
+#    dependsOn: snaps_build
+#    pool:
+#      vmImage: ubuntu-18.04
+#    steps:
+#      - task: UsePythonVersion@0
+#        inputs:
+#          versionSpec: 3.8
+#          addToPath: true
+#      - script: |
+#          set -e
+#          sudo apt-get update
+#          sudo apt-get install -y --no-install-recommends nginx-light snapd
+#          python3 -m venv venv
+#          venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py
+#          venv/bin/python tools/pip_install.py -U tox
+#        displayName: Install dependencies
+#      - task: DownloadPipelineArtifact@2
+#        inputs:
+#          artifact: snaps
+#          path: $(Build.SourcesDirectory)/snap
+#        displayName: Retrieve Certbot snaps
+#      - script: |
+#          set -e
+#          sudo snap install --dangerous --classic snap/certbot_*_amd64.snap
+#        displayName: Install Certbot snap
+#      - script: |
+#          set -e
+#          venv/bin/python -m tox -e integration-external,apacheconftest-external-with-pebble
+#        displayName: Run tox
+#  - job: snap_dns_run
+#    dependsOn: snaps_build
+#    pool:
+#      vmImage: ubuntu-18.04
+#    steps:
+#      - script: |
+#          set -e
+#          sudo apt-get update
+#          sudo apt-get install -y --no-install-recommends snapd
+#        displayName: Install dependencies
+#      - task: UsePythonVersion@0
+#        inputs:
+#          versionSpec: 3.8
+#          addToPath: true
+#      - task: DownloadPipelineArtifact@2
+#        inputs:
+#          artifact: snaps
+#          path: $(Build.SourcesDirectory)/snap
+#        displayName: Retrieve Certbot snaps
+#      - script: |
+#          set -e
+#          python3 -m venv venv
+#          venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py
+#          venv/bin/python tools/pip_install.py -e certbot-ci
+#        displayName: Prepare Certbot-CI
+#      - script: |
+#          set -e
+#          sudo -E venv/bin/pytest certbot-ci/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap --snap-arch amd64
+#        displayName: Test DNS plugins snaps

.azure-pipelines/templates/jobs/standard-tests-jobs.yml

@@ -1,78 +1,75 @@
 jobs:
   - job: test
     variables:
-      PYTHON_VERSION: 3.9
+      PYTHON_VERSION: 3.8
     strategy:
       matrix:
-        macos-py36:
+        macos-py27:
           IMAGE_NAME: macOS-10.15
-          PYTHON_VERSION: 3.6
-          TOXENV: py36
-        macos-py39:
+          PYTHON_VERSION: 2.7
+          TOXENV: py27
+        macos-py38:
           IMAGE_NAME: macOS-10.15
-          PYTHON_VERSION: 3.9
-          TOXENV: py39
+          PYTHON_VERSION: 3.8
+          TOXENV: py38
         windows-py36:
           IMAGE_NAME: vs2017-win2016
           PYTHON_VERSION: 3.6
           TOXENV: py36
-        windows-py38-cover:
+        windows-py37-cover:
           IMAGE_NAME: vs2017-win2016
-          PYTHON_VERSION: 3.8
-          TOXENV: py38-cover
+          PYTHON_VERSION: 3.7
+          TOXENV: py37-cover
         windows-integration-certbot:
           IMAGE_NAME: vs2017-win2016
-          PYTHON_VERSION: 3.8
+          PYTHON_VERSION: 3.7
           TOXENV: integration-certbot
         linux-oldest-tests-1:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.6
-          TOXENV: '{acme,apache,apache-v2,certbot}-oldest'
+          TOXENV: py27-{acme,apache,apache-v2,certbot}-oldest
         linux-oldest-tests-2:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.6
-          TOXENV: '{dns,nginx}-oldest'
+          TOXENV: py27-{dns,nginx}-oldest
+        linux-py27:
+          IMAGE_NAME: ubuntu-18.04
+          PYTHON_VERSION: 2.7
+          TOXENV: py27
         linux-py36:
           IMAGE_NAME: ubuntu-18.04
           PYTHON_VERSION: 3.6
           TOXENV: py36
-        linux-py39-cover:
+        linux-py38-cover:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.9
-          TOXENV: py39-cover
-        linux-py39-lint:
+          PYTHON_VERSION: 3.8
+          TOXENV: py38-cover
+        linux-py37-lint:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.9
+          PYTHON_VERSION: 3.7
           TOXENV: lint
-        linux-py39-mypy:
+        linux-py36-mypy:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.9
+          PYTHON_VERSION: 3.6
           TOXENV: mypy
         linux-integration:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.8
+          PYTHON_VERSION: 2.7
           TOXENV: integration
           ACME_SERVER: pebble
         apache-compat:
           IMAGE_NAME: ubuntu-18.04
           TOXENV: apache_compat
-        le-modification:
+        le-auto-centos6:
           IMAGE_NAME: ubuntu-18.04
-          TOXENV: modification
+          TOXENV: le_auto_centos6
         apacheconftest:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.6
+          PYTHON_VERSION: 2.7
           TOXENV: apacheconftest-with-pebble
         nginxroundtrip:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.6
+          PYTHON_VERSION: 2.7
           TOXENV: nginxroundtrip
     pool:
       vmImage: $(IMAGE_NAME)
     steps:
       - template: ../steps/tox-steps.yml
-  - job: test_sphinx_builds
-    pool:
-      vmImage: ubuntu-20.04
-    steps:
-      - template: ../steps/sphinx-steps.yml

.azure-pipelines/templates/stages/deploy-stage.yml

@@ -37,14 +37,6 @@ stages:
           vmImage: ubuntu-18.04
         variables:
           - group: certbot-common
-        strategy:
-          matrix:
-            amd64:
-              SNAP_ARCH: amd64
-            arm32v6:
-              SNAP_ARCH: armhf
-            arm64v8:
-              SNAP_ARCH: arm64
         steps:
           - bash: |
               set -e
@@ -54,7 +46,7 @@ stages:
             displayName: Install dependencies
           - task: DownloadPipelineArtifact@2
             inputs:
-              artifact: snaps_$(SNAP_ARCH)
+              artifact: snaps
               path: $(Build.SourcesDirectory)/snap
             displayName: Retrieve Certbot snaps
           - task: DownloadSecureFile@1
@@ -63,7 +55,8 @@ stages:
               secureFile: snapcraft.cfg
           - bash: |
               set -e
-              snapcraft login --with $(snapcraftCfg.secureFilePath)
+              mkdir -p .snapcraft
+              ln -s $(snapcraftCfg.secureFilePath) .snapcraft/snapcraft.cfg
               for SNAP_FILE in snap/*.snap; do
                 tools/retry.sh eval snapcraft upload --release=${{ parameters.snapReleaseChannel }} "${SNAP_FILE}"
               done

.azure-pipelines/templates/stages/notify-failure-stage.yml

@@ -5,7 +5,7 @@ stages:
         variables:
           - group: certbot-common
         pool:
-          vmImage: ubuntu-20.04
+          vmImage: ubuntu-latest
         steps:
           - bash: |
               set -e

.azure-pipelines/templates/stages/test-and-package-stage.yml

@@ -1,4 +1,6 @@
 stages:
   - stage: TestAndPackage
     jobs:
+#      - template: ../jobs/standard-tests-jobs.yml
+#      - template: ../jobs/extended-tests-jobs.yml
       - template: ../jobs/packaging-jobs.yml

.azure-pipelines/templates/steps/sphinx-steps.yml

@@ -1,23 +0,0 @@
-steps:
-  - bash: |
-      FINAL_STATUS=0
-      declare -a FAILED_BUILDS
-      python3 -m venv .venv
-      source .venv/bin/activate
-      python tools/pipstrap.py
-      for doc_path in */docs
-      do
-        echo ""
-        echo "##[group]Building $doc_path"
-        pip install -q -e $doc_path/..[docs]
-        if ! sphinx-build -W --keep-going -b html $doc_path $doc_path/_build/html; then
-          FINAL_STATUS=1
-          FAILED_BUILDS[${#FAILED_BUILDS[@]}]="${doc_path%/docs}"
-        fi
-        echo "##[endgroup]"
-      done
-      if [[ $FINAL_STATUS -ne 0 ]]; then
-        echo "##[error]The following builds failed: ${FAILED_BUILDS[*]}"
-        exit 1
-      fi
-    displayName: Build Sphinx Documentation

.azure-pipelines/templates/steps/tox-steps.yml

@@ -32,7 +32,7 @@ steps:
     # problems with its lack of real dependency resolution.
   - bash: |
       set -e
-      python tools/pipstrap.py
+      python letsencrypt-auto-source/pieces/pipstrap.py
       python tools/pip_install.py -I tox virtualenv
     displayName: Install runtime dependencies
   - task: DownloadSecureFile@1
@@ -45,7 +45,11 @@ steps:
       export TARGET_BRANCH="`echo "${BUILD_SOURCEBRANCH}" | sed -E 's!refs/(heads|tags)/!!g'`"
       [ -z "${SYSTEM_PULLREQUEST_TARGETBRANCH}" ] || export TARGET_BRANCH="${SYSTEM_PULLREQUEST_TARGETBRANCH}"
       env
-      python -m tox
+      if [[ "${TOXENV}" == *"oldest"* ]]; then
+        tools/run_oldest_tests.sh
+      else
+        python -m tox
+      fi
     env:
       AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
       AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)

.dockerignore

@@ -8,4 +8,5 @@
 .git
 .tox
 venv
+venv3
 docs

.envrc

@@ -1,12 +0,0 @@
-# This file is just a nicety for developers who use direnv. When you cd under
-# the Certbot repo, Certbot's virtual environment will be automatically
-# activated and then deactivated when you cd elsewhere. Developers have to have
-# direnv set up and run `direnv allow` to allow this file to execute on their
-# system. You can find more information at https://direnv.net/.
-. venv/bin/activate
-# direnv doesn't support modifying PS1 so we unset it to squelch the error
-# it'll otherwise print about this being done in the activate script. See
-# https://github.com/direnv/direnv/wiki/PS1. If you would like your shell
-# prompt to change like it normally does, see
-# https://github.com/direnv/direnv/wiki/Python#restoring-the-ps1.
-unset PS1

.gitignore

@@ -11,7 +11,6 @@ dist*/
 letsencrypt.log
 certbot.log
 letsencrypt-auto-source/letsencrypt-auto.sig.lzma.base64
-poetry.lock
 
 # coverage
 .coverage
@@ -65,4 +64,3 @@ certbot-dns*/certbot-dns*_amd64*.txt
 certbot-dns*/certbot-dns*_arm*.txt
 /certbot_amd64*.txt
 /certbot_arm*.txt
-certbot-dns*/snap

.pylintrc

@@ -8,10 +8,7 @@ jobs=0
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-# CERTBOT COMMENT
-# This is needed for pylint to import linter_plugin.py since
-# https://github.com/PyCQA/pylint/pull/3396.
-init-hook="import pylint.config, os, sys; sys.path.append(os.path.dirname(pylint.config.PYLINTRC))"
+#init-hook=
 
 # Profiled execution.
 profile=no
@@ -257,7 +254,7 @@ ignore-mixin-members=yes
 # List of module names for which member attributes should not be checked
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis
-ignored-modules=pkg_resources,confargparse,argparse
+ignored-modules=pkg_resources,confargparse,argparse,six.moves,six.moves.urllib
 # import errors ignored only in 1.4.4
 # https://bitbucket.org/logilab/pylint/commits/cd000904c9e2
 

AUTHORS.md

@@ -1,7 +1,6 @@
 Authors
 =======
 
-* [Aaron Gable](https://github.com/aarongable)
 * [Aaron Zirbes](https://github.com/aaronzirbes)
 * Aaron Zuehlke
 * Ada Lovelace
@@ -61,7 +60,6 @@ Authors
 * [DanCld](https://github.com/DanCld)
 * [Daniel Albers](https://github.com/AID)
 * [Daniel Aleksandersen](https://github.com/da2x)
-* [Daniel Almasi](https://github.com/almasen)
 * [Daniel Convissor](https://github.com/convissor)
 * [Daniel "Drex" Drexler](https://github.com/aeturnum)
 * [Daniel Huang](https://github.com/dhuang)
@@ -151,13 +149,11 @@ Authors
 * [Lior Sabag](https://github.com/liorsbg)
 * [Lipis](https://github.com/lipis)
 * [lord63](https://github.com/lord63)
-* [Lorenzo Fundaró](https://github.com/lfundaro)
 * [Luca Beltrame](https://github.com/lbeltrame)
 * [Luca Ebach](https://github.com/lucebac)
 * [Luca Olivetti](https://github.com/olivluca)
 * [Luke Rogers](https://github.com/lukeroge)
 * [Maarten](https://github.com/mrtndwrd)
-* [Mads Jensen](https://github.com/atombrella)
 * [Maikel Martens](https://github.com/krukas)
 * [Malte Janduda](https://github.com/MalteJ)
 * [Mantas Mikulėnas](https://github.com/grawity)
@@ -217,7 +213,6 @@ Authors
 * [Richard Barnes](https://github.com/r-barnes)
 * [Richard Panek](https://github.com/kernelpanek)
 * [Robert Buchholz](https://github.com/rbu)
-* [Robert Dailey](https://github.com/pahrohfit)
 * [Robert Habermann](https://github.com/frennkie)
 * [Robert Xiao](https://github.com/nneonneo)
 * [Roland Shoemaker](https://github.com/rolandshoemaker)

Dockerfile-dev

@@ -15,6 +15,6 @@ RUN apt-get update && \
            /tmp/* \
            /var/tmp/*
 
-RUN VENV_NAME="../venv" python3 tools/venv.py
+RUN VENV_NAME="../venv3" python3 tools/venv3.py
 
-ENV PATH /opt/certbot/venv/bin:$PATH
+ENV PATH /opt/certbot/venv3/bin:$PATH

ISSUE_TEMPLATE.md

@@ -7,7 +7,7 @@ questions.
 ## My operating system is (include version):
 
 
-## I installed Certbot with (snap, OS package manager, pip, certbot-auto, etc):
+## I installed Certbot with (certbot-auto, OS package manager, pip, etc):
 
 
 ## I ran this command and it produced this output:

acme/acme/__init__.py

@@ -6,6 +6,7 @@ This module is an implementation of the `ACME protocol`_.
 
 """
 import sys
+import warnings
 
 # This code exists to keep backwards compatibility with people using acme.jose
 # before it became the standalone josepy package.

acme/acme/challenges.py

@@ -5,19 +5,18 @@ import functools
 import hashlib
 import logging
 import socket
-from typing import Type
 
 from cryptography.hazmat.primitives import hashes  # type: ignore
 import josepy as jose
-from OpenSSL import crypto
-from OpenSSL import SSL  # type: ignore # https://github.com/python/typeshed/issues/2052
 import requests
+import six
+from OpenSSL import SSL  # type: ignore # https://github.com/python/typeshed/issues/2052
+from OpenSSL import crypto
 
 from acme import crypto_util
 from acme import errors
 from acme import fields
-from acme.mixins import ResourceMixin
-from acme.mixins import TypeMixin
+from acme.mixins import ResourceMixin, TypeMixin
 
 logger = logging.getLogger(__name__)
 
@@ -25,7 +24,7 @@ logger = logging.getLogger(__name__)
 class Challenge(jose.TypedJSONObjectWithFields):
     # _fields_to_partial_json
     """ACME challenge."""
-    TYPES: dict = {}
+    TYPES = {}  # type: dict
 
     @classmethod
     def from_json(cls, jobj):
@@ -39,7 +38,7 @@ class Challenge(jose.TypedJSONObjectWithFields):
 class ChallengeResponse(ResourceMixin, TypeMixin, jose.TypedJSONObjectWithFields):
     # _fields_to_partial_json
     """ACME challenge response."""
-    TYPES: dict = {}
+    TYPES = {}  # type: dict
     resource_type = 'challenge'
     resource = fields.Resource(resource_type)
 
@@ -146,15 +145,16 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse):
         return jobj
 
 
-class KeyAuthorizationChallenge(_TokenChallenge, metaclass=abc.ABCMeta):
+@six.add_metaclass(abc.ABCMeta)
+class KeyAuthorizationChallenge(_TokenChallenge):
     """Challenge based on Key Authorization.
 
     :param response_cls: Subclass of `KeyAuthorizationChallengeResponse`
-        that will be used to generate ``response``.
+        that will be used to generate `response`.
     :param str typ: type of the challenge
     """
-    typ: str = NotImplemented
-    response_cls: Type[KeyAuthorizationChallengeResponse] = NotImplemented
+    typ = NotImplemented
+    response_cls = NotImplemented
     thumbprint_hash_function = (
         KeyAuthorizationChallengeResponse.thumbprint_hash_function)
 

acme/acme/client.py

@@ -4,16 +4,10 @@ import collections
 import datetime
 from email.utils import parsedate_tz
 import heapq
-import http.client as http_client
 import logging
 import re
+import sys
 import time
-from typing import cast
-from typing import Dict
-from typing import List
-from typing import Set
-from typing import Text
-from typing import Union
 
 import josepy as jose
 import OpenSSL
@@ -21,21 +15,38 @@ import requests
 from requests.adapters import HTTPAdapter
 from requests.utils import parse_header_links
 from requests_toolbelt.adapters.source import SourceAddressAdapter
+import six
+from six.moves import http_client
 
 from acme import crypto_util
 from acme import errors
 from acme import jws
 from acme import messages
+from acme.magic_typing import Dict
+from acme.magic_typing import List
+from acme.magic_typing import Set
+from acme.magic_typing import Text
 from acme.mixins import VersionedLEACMEMixin
 
 logger = logging.getLogger(__name__)
 
+# Prior to Python 2.7.9 the stdlib SSL module did not allow a user to configure
+# many important security related options. On these platforms we use PyOpenSSL
+# for SSL, which does allow these options to be configured.
+# https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning
+if sys.version_info < (2, 7, 9):  # pragma: no cover
+    try:
+        requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3()  # type: ignore
+    except AttributeError:
+        import urllib3.contrib.pyopenssl
+        urllib3.contrib.pyopenssl.inject_into_urllib3()
+
 DEFAULT_NETWORK_TIMEOUT = 45
 
 DER_CONTENT_TYPE = 'application/pkix-cert'
 
 
-class ClientBase:
+class ClientBase(object):
     """ACME client base object.
 
     :ivar messages.Directory directory:
@@ -114,9 +125,8 @@ class ClientBase:
         """
         return self.update_registration(regr, update={'status': 'deactivated'})
 
-    def deactivate_authorization(self,
-                                 authzr: messages.AuthorizationResource
-                                 ) -> messages.AuthorizationResource:
+    def deactivate_authorization(self, authzr):
+        # type: (messages.AuthorizationResource) -> messages.AuthorizationResource
         """Deactivate authorization.
 
         :param messages.AuthorizationResource authzr: The Authorization resource
@@ -191,7 +201,7 @@ class ClientBase:
             when = parsedate_tz(retry_after)
             if when is not None:
                 try:
-                    tz_secs = datetime.timedelta(when[-1] if when[-1] is not None else 0)
+                    tz_secs = datetime.timedelta(when[-1] if when[-1] else 0)
                     return datetime.datetime(*when[:7]) - tz_secs
                 except (ValueError, OverflowError):
                     pass
@@ -250,7 +260,7 @@ class Client(ClientBase):
         if net is None:
             net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl)
 
-        if isinstance(directory, str):
+        if isinstance(directory, six.string_types):
             directory = messages.Directory.from_json(
                 net.get(directory).json())
         super(Client, self).__init__(directory=directory,
@@ -426,7 +436,7 @@ class Client(ClientBase):
 
         """
         assert max_attempts > 0
-        attempts: Dict[messages.AuthorizationResource, int] = collections.defaultdict(int)
+        attempts = collections.defaultdict(int) # type: Dict[messages.AuthorizationResource, int]
         exhausted = set()
 
         # priority queue with datetime.datetime (based on Retry-After) as key,
@@ -465,7 +475,7 @@ class Client(ClientBase):
                     exhausted.add(authzr)
 
         if exhausted or any(authzr.body.status == messages.STATUS_INVALID
-                            for authzr in updated.values()):
+                            for authzr in six.itervalues(updated)):
             raise errors.PollError(exhausted, updated)
 
         updated_authzrs = tuple(updated[authzr] for authzr in authzrs)
@@ -539,7 +549,7 @@ class Client(ClientBase):
         :rtype: `list` of `OpenSSL.crypto.X509` wrapped in `.ComparableX509`
 
         """
-        chain: List[jose.ComparableX509] = []
+        chain = [] # type: List[jose.ComparableX509]
         uri = certr.cert_chain_uri
         while uri is not None and len(chain) < max_length:
             response, cert = self._get_cert(uri)
@@ -798,7 +808,7 @@ class ClientV2(ClientBase):
                 if 'rel' in l and 'url' in l and l['rel'] == relation_type]
 
 
-class BackwardsCompatibleClientV2:
+class BackwardsCompatibleClientV2(object):
     """ACME client wrapper that tends towards V2-style calls, but
     supports V1 servers.
 
@@ -820,7 +830,6 @@ class BackwardsCompatibleClientV2:
     def __init__(self, net, key, server):
         directory = messages.Directory.from_json(net.get(server).json())
         self.acme_version = self._acme_version_from_directory(directory)
-        self.client: Union[Client, ClientV2]
         if self.acme_version == 1:
             self.client = Client(directory, key=key, net=net)
         else:
@@ -840,18 +849,16 @@ class BackwardsCompatibleClientV2:
             if check_tos_cb is not None:
                 check_tos_cb(tos)
         if self.acme_version == 1:
-            client_v1 = cast(Client, self.client)
-            regr = client_v1.register(regr)
+            regr = self.client.register(regr)
             if regr.terms_of_service is not None:
                 _assess_tos(regr.terms_of_service)
-                return client_v1.agree_to_tos(regr)
+                return self.client.agree_to_tos(regr)
             return regr
         else:
-            client_v2 = cast(ClientV2, self.client)
-            if "terms_of_service" in client_v2.directory.meta:
-                _assess_tos(client_v2.directory.meta.terms_of_service)
+            if "terms_of_service" in self.client.directory.meta:
+                _assess_tos(self.client.directory.meta.terms_of_service)
                 regr = regr.update(terms_of_service_agreed=True)
-            return client_v2.new_account(regr)
+            return self.client.new_account(regr)
 
     def new_order(self, csr_pem):
         """Request a new Order object from the server.
@@ -869,15 +876,14 @@ class BackwardsCompatibleClientV2:
 
         """
         if self.acme_version == 1:
-            client_v1 = cast(Client, self.client)
             csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)
             # pylint: disable=protected-access
             dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr)
             authorizations = []
             for domain in dnsNames:
-                authorizations.append(client_v1.request_domain_challenges(domain))
+                authorizations.append(self.client.request_domain_challenges(domain))
             return messages.OrderResource(authorizations=authorizations, csr_pem=csr_pem)
-        return cast(ClientV2, self.client).new_order(csr_pem)
+        return self.client.new_order(csr_pem)
 
     def finalize_order(self, orderr, deadline, fetch_alternative_chains=False):
         """Finalize an order and obtain a certificate.
@@ -892,9 +898,8 @@ class BackwardsCompatibleClientV2:
 
         """
         if self.acme_version == 1:
-            client_v1 = cast(Client, self.client)
             csr_pem = orderr.csr_pem
-            certr = client_v1.request_issuance(
+            certr = self.client.request_issuance(
                 jose.ComparableX509(
                     OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)),
                     orderr.authorizations)
@@ -902,7 +907,7 @@ class BackwardsCompatibleClientV2:
             chain = None
             while datetime.datetime.now() < deadline:
                 try:
-                    chain = client_v1.fetch_chain(certr)
+                    chain = self.client.fetch_chain(certr)
                     break
                 except errors.Error:
                     time.sleep(1)
@@ -917,8 +922,7 @@ class BackwardsCompatibleClientV2:
             chain = crypto_util.dump_pyopenssl_chain(chain).decode()
 
             return orderr.update(fullchain_pem=(cert + chain))
-        return cast(ClientV2, self.client).finalize_order(
-            orderr, deadline, fetch_alternative_chains)
+        return self.client.finalize_order(orderr, deadline, fetch_alternative_chains)
 
     def revoke(self, cert, rsn):
         """Revoke certificate.
@@ -944,10 +948,10 @@ class BackwardsCompatibleClientV2:
         Always return False for ACMEv1 servers, as it doesn't use External Account Binding."""
         if self.acme_version == 1:
             return False
-        return cast(ClientV2, self.client).external_account_required()
+        return self.client.external_account_required()
 
 
-class ClientNetwork:
+class ClientNetwork(object):
     """Wrapper around requests that signs POSTs for authentication.
 
     Also adds user agent, and handles Content-Type.
@@ -977,7 +981,7 @@ class ClientNetwork:
         self.account = account
         self.alg = alg
         self.verify_ssl = verify_ssl
-        self._nonces: Set[Text] = set()
+        self._nonces = set() # type: Set[Text]
         self.user_agent = user_agent
         self.session = requests.Session()
         self._default_timeout = timeout
@@ -1137,7 +1141,6 @@ class ClientNetwork:
 
         # If content is DER, log the base64 of it instead of raw bytes, to keep
         # binary data out of the logs.
-        debug_content: Union[bytes, str]
         if response.headers.get("Content-Type") == DER_CONTENT_TYPE:
             debug_content = base64.b64encode(response.content)
         else:

acme/acme/crypto_util.py

@@ -5,15 +5,15 @@ import logging
 import os
 import re
 import socket
-from typing import Callable
-from typing import Tuple
-from typing import Union
 
 import josepy as jose
 from OpenSSL import crypto
 from OpenSSL import SSL  # type: ignore # https://github.com/python/typeshed/issues/2052
 
 from acme import errors
+from acme.magic_typing import Callable
+from acme.magic_typing import Tuple
+from acme.magic_typing import Union
 
 logger = logging.getLogger(__name__)
 
@@ -27,7 +27,7 @@ logger = logging.getLogger(__name__)
 _DEFAULT_SSL_METHOD = SSL.SSLv23_METHOD  # type: ignore
 
 
-class _DefaultCertSelection:
+class _DefaultCertSelection(object):
     def __init__(self, certs):
         self.certs = certs
 
@@ -36,7 +36,7 @@ class _DefaultCertSelection:
         return self.certs.get(server_name, None)
 
 
-class SSLSocket:  # pylint: disable=too-few-public-methods
+class SSLSocket(object):  # pylint: disable=too-few-public-methods
     """SSL wrapper for sockets.
 
     :ivar socket sock: Original wrapped socket.
@@ -93,7 +93,7 @@ class SSLSocket:  # pylint: disable=too-few-public-methods
             new_context.set_alpn_select_callback(self.alpn_selection)
         connection.set_context(new_context)
 
-    class FakeConnection:
+    class FakeConnection(object):
         """Fake OpenSSL.SSL.Connection."""
 
         # pylint: disable=missing-function-docstring
@@ -166,9 +166,9 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu
             " from {0}:{1}".format(
                 source_address[0],
                 source_address[1]
-            ) if any(source_address) else ""
+            ) if socket_kwargs else ""
         )
-        socket_tuple: Tuple[str, int] = (host, port)
+        socket_tuple = (host, port)  # type: Tuple[str, int]
         sock = socket.create_connection(socket_tuple, **socket_kwargs)  # type: ignore
     except socket.error as error:
         raise errors.Error(error)
@@ -186,7 +186,6 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu
             raise errors.Error(error)
     return client_ssl.get_peer_certificate()
 
-
 def make_csr(private_key_pem, domains, must_staple=False):
     """Generate a CSR containing a list of domains as subjectAltNames.
 
@@ -218,7 +217,6 @@ def make_csr(private_key_pem, domains, must_staple=False):
     return crypto.dump_certificate_request(
         crypto.FILETYPE_PEM, csr)
 
-
 def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req):
     common_name = loaded_cert_or_req.get_subject().CN
     sans = _pyopenssl_cert_or_req_san(loaded_cert_or_req)
@@ -227,7 +225,6 @@ def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req):
         return sans
     return [common_name] + [d for d in sans if d != common_name]
 
-
 def _pyopenssl_cert_or_req_san(cert_or_req):
     """Get Subject Alternative Names from certificate or CSR using pyOpenSSL.
 
@@ -256,7 +253,7 @@ def _pyopenssl_cert_or_req_san(cert_or_req):
 
     if isinstance(cert_or_req, crypto.X509):
         # pylint: disable=line-too-long
-        func: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]] = crypto.dump_certificate
+        func = crypto.dump_certificate # type: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]]
     else:
         func = crypto.dump_certificate_request
     text = func(crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8")
@@ -320,7 +317,6 @@ def gen_ss_cert(key, domains, not_before=None,
     cert.sign(key, "sha256")
     return cert
 
-
 def dump_pyopenssl_chain(chain, filetype=crypto.FILETYPE_PEM):
     """Dump certificate chain into a bundle.
 

acme/acme/errors.py

@@ -28,8 +28,13 @@ class NonceError(ClientError):
 
 class BadNonce(NonceError):
     """Bad nonce error."""
-    def __init__(self, nonce, error, *args):
-        super(BadNonce, self).__init__(*args)
+    def __init__(self, nonce, error, *args, **kwargs):
+        # MyPy complains here that there is too many arguments for BaseException constructor.
+        # This is an error fixed in typeshed, see https://github.com/python/mypy/issues/4183
+        # The fix is included in MyPy>=0.740, but upgrading it would bring dozen of errors due to
+        #   new types definitions. So we ignore the error until the code base is fixed to match
+        #   with MyPy>=0.740 referential.
+        super(BadNonce, self).__init__(*args, **kwargs)  # type: ignore
         self.nonce = nonce
         self.error = error
 
@@ -44,11 +49,12 @@ class MissingNonce(NonceError):
     Replay-Nonce header field in each successful response to a POST it
     provides to a client (...)".
 
-    :ivar requests.Response ~.response: HTTP Response
+    :ivar requests.Response response: HTTP Response
 
     """
-    def __init__(self, response, *args):
-        super(MissingNonce, self).__init__(*args)
+    def __init__(self, response, *args, **kwargs):
+        # See comment in BadNonce constructor above for an explanation of type: ignore here.
+        super(MissingNonce, self).__init__(*args, **kwargs)  # type: ignore
         self.response = response
 
     def __str__(self):

acme/acme/jws.py

@@ -14,9 +14,7 @@ class Header(jose.Header):
     kid = jose.Field('kid', omitempty=True)
     url = jose.Field('url', omitempty=True)
 
-    # Mypy does not understand the josepy magic happening here, and falsely claims
-    # that nonce is redefined. Let's ignore the type check here.
-    @nonce.decoder  # type: ignore
+    @nonce.decoder
     def nonce(value):  # pylint: disable=no-self-argument,missing-function-docstring
         try:
             return jose.decode_b64jose(value)

acme/acme/magic_typing.py

@@ -1,17 +1,15 @@
-"""Simple shim around the typing module.
+"""Shim class to not have to depend on typing module in prod."""
+import sys
 
-This was useful when this code supported Python 2 and typing wasn't always
-available. This code is being kept for now for backwards compatibility.
 
-"""
-import warnings
-from typing import *  # pylint: disable=wildcard-import, unused-wildcard-import
-from typing import Collection, IO  # type: ignore
-
-warnings.warn("acme.magic_typing is deprecated and will be removed in a future release.",
-              DeprecationWarning)
-
-class TypingClass:
+class TypingClass(object):
     """Ignore import errors by getting anything"""
     def __getattr__(self, name):
-        return None  # pragma: no cover
+        return None
+
+try:
+    # mypy doesn't respect modifying sys.modules
+    from typing import *  # pylint: disable=wildcard-import, unused-wildcard-import
+    from typing import Collection, IO  # type: ignore
+except ImportError:
+    sys.modules[__name__] = TypingClass()

acme/acme/messages.py

@@ -1,11 +1,8 @@
 """ACME protocol messages."""
-from collections.abc import Hashable
 import json
-from typing import Any
-from typing import Dict
-from typing import Type
 
 import josepy as jose
+import six
 
 from acme import challenges
 from acme import errors
@@ -14,6 +11,13 @@ from acme import jws
 from acme import util
 from acme.mixins import ResourceMixin
 
+try:
+    from collections.abc import Hashable
+except ImportError:  # pragma: no cover
+    from collections import Hashable
+
+
+
 OLD_ERROR_PREFIX = "urn:acme:error:"
 ERROR_PREFIX = "urn:ietf:params:acme:error:"
 
@@ -64,6 +68,7 @@ def is_acme_error(err):
     return False
 
 
+@six.python_2_unicode_compatible
 class Error(jose.JSONObjectWithFields, errors.Error):
     """ACME error.
 
@@ -90,9 +95,7 @@ class Error(jose.JSONObjectWithFields, errors.Error):
             raise ValueError("The supplied code: %s is not a known ACME error"
                              " code" % code)
         typ = ERROR_PREFIX + code
-        # Mypy will not understand that the Error constructor accepts a named argument
-        # "typ" because of josepy magic. Let's ignore the type check here.
-        return cls(typ=typ, **kwargs)  # type: ignore
+        return cls(typ=typ, **kwargs)
 
     @property
     def description(self):
@@ -129,7 +132,7 @@ class Error(jose.JSONObjectWithFields, errors.Error):
 class _Constant(jose.JSONDeSerializable, Hashable):  # type: ignore
     """ACME constant."""
     __slots__ = ('name',)
-    POSSIBLE_NAMES: Dict[str, '_Constant'] = NotImplemented
+    POSSIBLE_NAMES = NotImplemented
 
     def __init__(self, name):
         super(_Constant, self).__init__()
@@ -155,10 +158,13 @@ class _Constant(jose.JSONDeSerializable, Hashable):  # type: ignore
     def __hash__(self):
         return hash((self.__class__, self.name))
 
+    def __ne__(self, other):
+        return not self == other
+
 
 class Status(_Constant):
     """ACME "status" field."""
-    POSSIBLE_NAMES: dict = {}
+    POSSIBLE_NAMES = {}  # type: dict
 STATUS_UNKNOWN = Status('unknown')
 STATUS_PENDING = Status('pending')
 STATUS_PROCESSING = Status('processing')
@@ -171,7 +177,7 @@ STATUS_DEACTIVATED = Status('deactivated')
 
 class IdentifierType(_Constant):
     """ACME identifier type."""
-    POSSIBLE_NAMES: Dict[str, 'IdentifierType'] = {}
+    POSSIBLE_NAMES = {}  # type: dict
 IDENTIFIER_FQDN = IdentifierType('dns')  # IdentifierDNS in Boulder
 
 
@@ -189,7 +195,7 @@ class Identifier(jose.JSONObjectWithFields):
 class Directory(jose.JSONDeSerializable):
     """Directory."""
 
-    _REGISTERED_TYPES: Dict[str, Type[Any]] = {}
+    _REGISTERED_TYPES = {}  # type: dict
 
     class Meta(jose.JSONObjectWithFields):
         """Directory Meta."""
@@ -223,7 +229,7 @@ class Directory(jose.JSONDeSerializable):
         return getattr(key, 'resource_type', key)
 
     @classmethod
-    def register(cls, resource_body_cls: Type[Any]) -> Type[Any]:
+    def register(cls, resource_body_cls):
         """Register resource."""
         resource_type = resource_body_cls.resource_type
         assert resource_type not in cls._REGISTERED_TYPES
@@ -269,7 +275,7 @@ class Resource(jose.JSONObjectWithFields):
 class ResourceWithURI(Resource):
     """ACME Resource with URI.
 
-    :ivar unicode ~.uri: Location of the resource.
+    :ivar unicode uri: Location of the resource.
 
     """
     uri = jose.Field('uri')  # no ChallengeResource.uri
@@ -279,7 +285,7 @@ class ResourceBody(jose.JSONObjectWithFields):
     """ACME Resource Body."""
 
 
-class ExternalAccountBinding:
+class ExternalAccountBinding(object):
     """ACME External Account Binding"""
 
     @classmethod
@@ -533,9 +539,7 @@ class Authorization(ResourceBody):
     expires = fields.RFC3339Field('expires', omitempty=True)
     wildcard = jose.Field('wildcard', omitempty=True)
 
-    # Mypy does not understand the josepy magic happening here, and falsely claims
-    # that challenge is redefined. Let's ignore the type check here.
-    @challenges.decoder  # type: ignore
+    @challenges.decoder
     def challenges(value):  # pylint: disable=no-self-argument,missing-function-docstring
         return tuple(ChallengeBody.from_json(chall) for chall in value)
 
@@ -623,7 +627,7 @@ class Order(ResourceBody):
     :ivar str finalize: URL to POST to to request issuance once all
         authorizations have "valid" status.
     :ivar datetime.datetime expires: When the order expires.
-    :ivar ~.Error error: Any error that occurred during finalization, if applicable.
+    :ivar .Error error: Any error that occurred during finalization, if applicable.
     """
     identifiers = jose.Field('identifiers', omitempty=True)
     status = jose.Field('status', decoder=Status.from_json,
@@ -634,9 +638,7 @@ class Order(ResourceBody):
     expires = fields.RFC3339Field('expires', omitempty=True)
     error = jose.Field('error', omitempty=True, decoder=Error.from_json)
 
-    # Mypy does not understand the josepy magic happening here, and falsely claims
-    # that identifiers is redefined. Let's ignore the type check here.
-    @identifiers.decoder  # type: ignore
+    @identifiers.decoder
     def identifiers(value):  # pylint: disable=no-self-argument,missing-function-docstring
         return tuple(Identifier.from_json(identifier) for identifier in value)
 

acme/acme/mixins.py

@@ -1,7 +1,7 @@
 """Useful mixins for Challenge and Resource objects"""
 
 
-class VersionedLEACMEMixin:
+class VersionedLEACMEMixin(object):
     """This mixin stores the version of Let's Encrypt's endpoint being used."""
     @property
     def le_acme_version(self):

acme/acme/standalone.py

@@ -1,16 +1,17 @@
 """Support for standalone client challenge solvers. """
 import collections
 import functools
-import http.client as http_client
-import http.server as BaseHTTPServer
 import logging
 import socket
-import socketserver
 import threading
-from typing import List
+
+from six.moves import BaseHTTPServer  # type: ignore
+from six.moves import http_client
+from six.moves import socketserver  # type: ignore
 
 from acme import challenges
 from acme import crypto_util
+from acme.magic_typing import List
 
 logger = logging.getLogger(__name__)
 
@@ -53,7 +54,7 @@ class ACMEServerMixin:
     allow_reuse_address = True
 
 
-class BaseDualNetworkedServers:
+class BaseDualNetworkedServers(object):
     """Base class for a pair of IPv6 and IPv4 servers that tries to do everything
        it's asked for both servers, but where failures in one server don't
        affect the other.
@@ -63,8 +64,8 @@ class BaseDualNetworkedServers:
 
     def __init__(self, ServerClass, server_address, *remaining_args, **kwargs):
         port = server_address[1]
-        self.threads: List[threading.Thread] = []
-        self.servers: List[socketserver.BaseServer] = []
+        self.threads = [] # type: List[threading.Thread]
+        self.servers = [] # type: List[ACMEServerMixin]
 
         # Must try True first.
         # Ubuntu, for example, will fail to bind to IPv4 if we've already bound
@@ -203,24 +204,8 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 
     def __init__(self, *args, **kwargs):
         self.simple_http_resources = kwargs.pop("simple_http_resources", set())
-        self._timeout = kwargs.pop('timeout', 30)
+        self.timeout = kwargs.pop('timeout', 30)
         BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
-        self.server: HTTP01Server
-
-    # In parent class BaseHTTPRequestHandler, 'timeout' is a class-level property but we
-    # need to define its value during the initialization phase in HTTP01RequestHandler.
-    # However MyPy does not appreciate that we dynamically shadow a class-level property
-    # with an instance-level property (eg. self.timeout = ... in __init__()). So to make
-    # everyone happy, we statically redefine 'timeout' as a method property, and set the
-    # timeout value in a new internal instance-level property _timeout.
-    @property
-    def timeout(self):
-        """
-        The default timeout this server should apply to requests.
-        :return: timeout to apply
-        :rtype: int
-        """
-        return self._timeout
 
     def log_message(self, format, *args):  # pylint: disable=redefined-builtin
         """Log arbitrary message."""

acme/acme/util.py

@@ -1,6 +1,7 @@
 """ACME utilities."""
+import six
 
 
 def map_keys(dikt, func):
     """Map dictionary keys."""
-    return {func(key): value for key, value in dikt.items()}
+    return {func(key): value for key, value in six.iteritems(dikt)}

acme/docs/conf.py

@@ -85,9 +85,7 @@ language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = [
-    '_build',
-]
+exclude_patterns = ['_build']
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.

acme/docs/man/jws.rst

@@ -1,3 +1 @@
-:orphan:
-
 .. literalinclude:: ../jws-help.txt

acme/setup.py

@@ -1,25 +1,40 @@
+from distutils.version import LooseVersion
 import sys
 
+from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.14.0.dev0'
+version = '1.10.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
-    'cryptography>=2.1.4',
+    # load_pem_private/public_key (>=0.6)
+    # rsa_recover_prime_factors (>=0.8)
+    'cryptography>=1.2.3',
     # formerly known as acme.jose:
     # 1.1.0+ is required to avoid the warnings described at
     # https://github.com/certbot/josepy/issues/13.
     'josepy>=1.1.0',
-    'PyOpenSSL>=17.3.0',
+    # Connection.set_tlsext_host_name (>=0.13) + matching Xenial requirements (>=0.15.1)
+    'PyOpenSSL>=0.15.1',
     'pyrfc3339',
     'pytz',
-    'requests>=2.6.0',
+    'requests[security]>=2.6.0',  # security extras added in 2.4.1
     'requests-toolbelt>=0.3.0',
-    'setuptools>=39.0.1',
+    'setuptools',
+    'six>=1.9.0',  # needed for python_2_unicode_compatible
 ]
 
+setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
+if setuptools_known_environment_markers:
+    install_requires.append('mock ; python_version < "3.3"')
+elif 'bdist_wheel' in sys.argv[1:]:
+    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
+                       'of setuptools. Version 36.2+ of setuptools is required.')
+elif sys.version_info < (3,3):
+    install_requires.append('mock')
+
 dev_extras = [
     'pytest',
     'pytest-xdist',
@@ -39,17 +54,18 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
-    python_requires='>=3.6',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*',
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Intended Audience :: Developers',
         'License :: OSI Approved :: Apache Software License',
         'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
     ],

acme/tests/challenges_test.py

@@ -1,11 +1,14 @@
 """Tests for acme.challenges."""
-import urllib.parse as urllib_parse
 import unittest
-from unittest import mock
 
 import josepy as jose
 import OpenSSL
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock # type: ignore
 import requests
+from six.moves.urllib import parse as urllib_parse
 
 from acme import errors
 

acme/tests/client_test.py

@@ -2,15 +2,17 @@
 # pylint: disable=too-many-lines
 import copy
 import datetime
-import http.client as http_client
 import json
 import unittest
-from typing import Dict
-from unittest import mock
 
 import josepy as jose
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock # type: ignore
 import OpenSSL
 import requests
+from six.moves import http_client  # pylint: disable=import-error
 
 from acme import challenges
 from acme import errors
@@ -62,7 +64,7 @@ class ClientTestBase(unittest.TestCase):
         self.contact = ('mailto:cert-admin@example.com', 'tel:+12025551212')
         reg = messages.Registration(
             contact=self.contact, key=KEY.public_key())
-        the_arg: Dict = dict(reg)
+        the_arg = dict(reg)  # type: Dict
         self.new_reg = messages.NewRegistration(**the_arg)
         self.regr = messages.RegistrationResource(
             body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1')

acme/tests/crypto_util_test.py

@@ -1,14 +1,14 @@
 """Tests for acme.crypto_util."""
 import itertools
 import socket
-import socketserver
 import threading
 import time
 import unittest
-from typing import List
 
 import josepy as jose
 import OpenSSL
+import six
+from six.moves import socketserver  # type: ignore  # pylint: disable=import-error
 
 from acme import errors
 import test_util
@@ -27,6 +27,8 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
 
         class _TestServer(socketserver.TCPServer):
 
+            # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init
+
             def server_bind(self):  # pylint: disable=missing-docstring
                 self.socket = SSLSocket(socket.socket(),
                         certs)
@@ -60,6 +62,7 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
         self.assertRaises(errors.Error, self._probe, b'bar')
 
     def test_probe_connection_error(self):
+        # pylint has a hard time with six
         self.server.server_close()
         original_timeout = socket.getdefaulttimeout()
         try:
@@ -118,9 +121,9 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase):
     @classmethod
     def _get_idn_names(cls):
         """Returns expected names from '{cert,csr}-idnsans.pem'."""
-        chars = [chr(i) for i in itertools.chain(range(0x3c3, 0x400),
-                                                 range(0x641, 0x6fc),
-                                                 range(0x1820, 0x1877))]
+        chars = [six.unichr(i) for i in itertools.chain(range(0x3c3, 0x400),
+                                                        range(0x641, 0x6fc),
+                                                        range(0x1820, 0x1877))]
         return [''.join(chars[i: i + 45]) + '.invalid'
                 for i in range(0, len(chars), 45)]
 
@@ -181,7 +184,7 @@ class RandomSnTest(unittest.TestCase):
 
     def setUp(self):
         self.cert_count = 5
-        self.serial_num: List[int] = []
+        self.serial_num = [] # type: List[int]
         self.key = OpenSSL.crypto.PKey()
         self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
 

acme/tests/errors_test.py

@@ -1,6 +1,10 @@
 """Tests for acme.errors."""
 import unittest
-from unittest import mock
+
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock # type: ignore
 
 
 class BadNonceTest(unittest.TestCase):

acme/tests/magic_typing_test.py

@@ -1,8 +1,11 @@
 """Tests for acme.magic_typing."""
 import sys
 import unittest
-import warnings
-from unittest import mock
+
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock # type: ignore
 
 
 class MagicTypingTest(unittest.TestCase):
@@ -10,21 +13,32 @@ class MagicTypingTest(unittest.TestCase):
     def test_import_success(self):
         try:
             import typing as temp_typing
-        except ImportError:  # pragma: no cover
-            temp_typing = None  # pragma: no cover
+        except ImportError: # pragma: no cover
+            temp_typing = None # pragma: no cover
         typing_class_mock = mock.MagicMock()
         text_mock = mock.MagicMock()
         typing_class_mock.Text = text_mock
         sys.modules['typing'] = typing_class_mock
         if 'acme.magic_typing' in sys.modules:
-            del sys.modules['acme.magic_typing']  # pragma: no cover
-        with warnings.catch_warnings():
-            warnings.filterwarnings("ignore", category=DeprecationWarning)
-            from acme.magic_typing import Text
+            del sys.modules['acme.magic_typing'] # pragma: no cover
+        from acme.magic_typing import Text
         self.assertEqual(Text, text_mock)
         del sys.modules['acme.magic_typing']
         sys.modules['typing'] = temp_typing
 
+    def test_import_failure(self):
+        try:
+            import typing as temp_typing
+        except ImportError: # pragma: no cover
+            temp_typing = None # pragma: no cover
+        sys.modules['typing'] = None
+        if 'acme.magic_typing' in sys.modules:
+            del sys.modules['acme.magic_typing'] # pragma: no cover
+        from acme.magic_typing import Text
+        self.assertTrue(Text is None)
+        del sys.modules['acme.magic_typing']
+        sys.modules['typing'] = temp_typing
+
 
 if __name__ == '__main__':
     unittest.main()  # pragma: no cover

acme/tests/messages_test.py

@@ -1,9 +1,11 @@
 """Tests for acme.messages."""
-from typing import Dict
 import unittest
-from unittest import mock
 
 import josepy as jose
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock # type: ignore
 
 from acme import challenges
 import test_util
@@ -82,7 +84,7 @@ class ConstantTest(unittest.TestCase):
         from acme.messages import _Constant
 
         class MockConstant(_Constant):  # pylint: disable=missing-docstring
-            POSSIBLE_NAMES: Dict = {}
+            POSSIBLE_NAMES = {} # type: Dict
 
         self.MockConstant = MockConstant  # pylint: disable=invalid-name
         self.const_a = MockConstant('a')
@@ -106,11 +108,11 @@ class ConstantTest(unittest.TestCase):
 
     def test_equality(self):
         const_a_prime = self.MockConstant('a')
-        self.assertNotEqual(self.const_a, self.const_b)
-        self.assertEqual(self.const_a, const_a_prime)
+        self.assertFalse(self.const_a == self.const_b)
+        self.assertTrue(self.const_a == const_a_prime)
 
-        self.assertNotEqual(self.const_a, self.const_b)
-        self.assertEqual(self.const_a, const_a_prime)
+        self.assertTrue(self.const_a != self.const_b)
+        self.assertFalse(self.const_a != const_a_prime)
 
 
 class DirectoryTest(unittest.TestCase):

acme/tests/standalone_test.py

@@ -1,14 +1,16 @@
 """Tests for acme.standalone."""
-import http.client as http_client
 import socket
-import socketserver
 import threading
 import unittest
-from typing import Set
-from unittest import mock
 
 import josepy as jose
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock # type: ignore
 import requests
+from six.moves import http_client  # pylint: disable=import-error
+from six.moves import socketserver  # type: ignore  # pylint: disable=import-error
 
 from acme import challenges
 from acme import crypto_util
@@ -42,7 +44,7 @@ class HTTP01ServerTest(unittest.TestCase):
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))
-        self.resources: Set = set()
+        self.resources = set() # type: Set
 
         from acme.standalone import HTTP01Server
         self.server = HTTP01Server(('', 0), resources=self.resources)
@@ -219,7 +221,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase):
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))
-        self.resources: Set = set()
+        self.resources = set() # type: Set
 
         from acme.standalone import HTTP01DualNetworkedServers
         self.servers = HTTP01DualNetworkedServers(('', 0), resources=self.resources)

certbot-apache/certbot_apache/_internal/apache_util.py

@@ -9,6 +9,7 @@ import pkg_resources
 
 from certbot import errors
 from certbot import util
+
 from certbot.compat import os
 
 logger = logging.getLogger(__name__)

certbot-apache/certbot_apache/_internal/apacheparser.py

@@ -1,5 +1,4 @@
 """ apacheconfig implementation of the ParserNode interfaces """
-from typing import Tuple
 
 from certbot_apache._internal import assertions
 from certbot_apache._internal import interfaces
@@ -22,7 +21,7 @@ class ApacheParserNode(interfaces.ParserNode):
         self.metadata = metadata
         self._raw = self.metadata["ac_ast"]
 
-    def save(self, msg):  # pragma: no cover
+    def save(self, msg): # pragma: no cover
         pass
 
     def find_ancestors(self, name):  # pylint: disable=unused-variable
@@ -84,7 +83,7 @@ class ApacheBlockNode(ApacheDirectiveNode):
 
     def __init__(self, **kwargs):
         super(ApacheBlockNode, self).__init__(**kwargs)
-        self.children: Tuple[ApacheParserNode, ...] = ()
+        self.children = ()
 
     def __eq__(self, other):  # pragma: no cover
         if isinstance(other, self.__class__):

certbot-apache/certbot_apache/_internal/assertions.py

@@ -3,6 +3,7 @@ import fnmatch
 
 from certbot_apache._internal import interfaces
 
+
 PASS = "CERTBOT_PASS_ASSERT"
 
 

certbot-apache/certbot_apache/_internal/augeasparser.py

@@ -64,10 +64,10 @@ Translates over to:
     "/files/etc/apache2/apache2.conf/bLoCk[1]",
 ]
 """
-from typing import Set
-
+from acme.magic_typing import Set
 from certbot import errors
 from certbot.compat import os
+
 from certbot_apache._internal import apache_util
 from certbot_apache._internal import assertions
 from certbot_apache._internal import interfaces
@@ -355,7 +355,7 @@ class AugeasBlockNode(AugeasDirectiveNode):
         ownpath = self.metadata.get("augeaspath")
 
         directives = self.parser.find_dir(name, start=ownpath, exclude=exclude)
-        already_parsed: Set[str] = set()
+        already_parsed = set()  # type: Set[str]
         for directive in directives:
             # Remove the /arg part from the Augeas path
             directive = directive.partition("/arg")[0]

certbot-apache/certbot_apache/_internal/configurator.py

@@ -1,24 +1,29 @@
 """Apache Configurator."""
 # pylint: disable=too-many-lines
 from collections import defaultdict
-import copy
 from distutils.version import LooseVersion
+import copy
 import fnmatch
 import logging
 import re
 import socket
 import time
-from typing import cast
-from typing import DefaultDict
-from typing import Dict
-from typing import List
-from typing import Set
-from typing import Union
 
+import six
 import zope.component
 import zope.interface
+try:
+    import apacheconfig
+    HAS_APACHECONFIG = True
+except ImportError:  # pragma: no cover
+    HAS_APACHECONFIG = False
 
 from acme import challenges
+from acme.magic_typing import DefaultDict
+from acme.magic_typing import Dict
+from acme.magic_typing import List
+from acme.magic_typing import Set
+from acme.magic_typing import Union
 from certbot import errors
 from certbot import interfaces
 from certbot import util
@@ -37,13 +42,6 @@ from certbot_apache._internal import http_01
 from certbot_apache._internal import obj
 from certbot_apache._internal import parser
 
-try:
-    import apacheconfig
-    HAS_APACHECONFIG = True
-except ImportError:  # pragma: no cover
-    HAS_APACHECONFIG = False
-
-
 logger = logging.getLogger(__name__)
 
 
@@ -157,9 +155,9 @@ class ApacheConfigurator(common.Installer):
                 self.options[o] = self.OS_DEFAULTS[o]
 
         # Special cases
-        cast(List[str], self.options["version_cmd"])[0] = self.option("ctl")
-        cast(List[str], self.options["restart_cmd"])[0] = self.option("ctl")
-        cast(List[str], self.options["conftest_cmd"])[0] = self.option("ctl")
+        self.options["version_cmd"][0] = self.option("ctl")
+        self.options["restart_cmd"][0] = self.option("ctl")
+        self.options["conftest_cmd"][0] = self.option("ctl")
 
     @classmethod
     def add_parser_arguments(cls, add):
@@ -213,23 +211,23 @@ class ApacheConfigurator(common.Installer):
         super(ApacheConfigurator, self).__init__(*args, **kwargs)
 
         # Add name_server association dict
-        self.assoc: Dict[str, obj.VirtualHost] = {}
+        self.assoc = {}  # type: Dict[str, obj.VirtualHost]
         # Outstanding challenges
-        self._chall_out: Set[KeyAuthorizationAnnotatedChallenge] = set()
+        self._chall_out = set()  # type: Set[KeyAuthorizationAnnotatedChallenge]
         # List of vhosts configured per wildcard domain on this run.
         # used by deploy_cert() and enhance()
-        self._wildcard_vhosts: Dict[str, List[obj.VirtualHost]] = {}
+        self._wildcard_vhosts = {}  # type: Dict[str, List[obj.VirtualHost]]
         # Maps enhancements to vhosts we've enabled the enhancement for
-        self._enhanced_vhosts: DefaultDict[str, Set[obj.VirtualHost]] = defaultdict(set)
+        self._enhanced_vhosts = defaultdict(set)  # type: DefaultDict[str, Set[obj.VirtualHost]]
         # Temporary state for AutoHSTS enhancement
-        self._autohsts: Dict[str, Dict[str, Union[int, float]]] = {}
+        self._autohsts = {}  # type: Dict[str, Dict[str, Union[int, float]]]
         # Reverter save notes
         self.save_notes = ""
         # Should we use ParserNode implementation instead of the old behavior
         self.USE_PARSERNODE = use_parsernode
         # Saves the list of file paths that were parsed initially, and
         # not added to parser tree by self.conf("vhost-root") for example.
-        self.parsed_paths: List[str] = []
+        self.parsed_paths = []  # type: List[str]
         # These will be set in the prepare function
         self._prepared = False
         self.parser = None
@@ -330,9 +328,6 @@ class ApacheConfigurator(common.Installer):
         if self.version < (2, 2):
             raise errors.NotSupportedError(
                 "Apache Version {0} not supported.".format(str(self.version)))
-        elif self.version < (2, 4):
-            logger.warning('Support for Apache 2.2 is deprecated and will be removed in a '
-                           'future release.')
 
         # Recover from previous crash before Augeas initialization to have the
         # correct parse tree from the get go.
@@ -469,6 +464,21 @@ class ApacheConfigurator(common.Installer):
             metadata=metadata
         )
 
+    def _wildcard_domain(self, domain):
+        """
+        Checks if domain is a wildcard domain
+
+        :param str domain: Domain to check
+
+        :returns: If the domain is wildcard domain
+        :rtype: bool
+        """
+        if isinstance(domain, six.text_type):
+            wildcard_marker = u"*."
+        else:
+            wildcard_marker = b"*."
+        return domain.startswith(wildcard_marker)
+
     def deploy_cert(self, domain, cert_path, key_path,
                     chain_path=None, fullchain_path=None):
         """Deploys certificate to specified virtual host.
@@ -503,7 +513,7 @@ class ApacheConfigurator(common.Installer):
         :rtype: `list` of :class:`~certbot_apache._internal.obj.VirtualHost`
         """
 
-        if util.is_wildcard_domain(domain):
+        if self._wildcard_domain(domain):
             if domain in self._wildcard_vhosts:
                 # Vhosts for a wildcard domain were already selected
                 return self._wildcard_vhosts[domain]
@@ -835,7 +845,7 @@ class ApacheConfigurator(common.Installer):
         :rtype: set
 
         """
-        all_names: Set[str] = set()
+        all_names = set()  # type: Set[str]
 
         vhost_macro = []
 
@@ -999,8 +1009,8 @@ class ApacheConfigurator(common.Installer):
 
         """
         # Search base config, and all included paths for VirtualHosts
-        file_paths: Dict[str, str] = {}
-        internal_paths: DefaultDict[str, Set[str]] = defaultdict(set)
+        file_paths = {}  # type: Dict[str, str]
+        internal_paths = defaultdict(set)  # type: DefaultDict[str, Set[str]]
         vhs = []
         # Make a list of parser paths because the parser_paths
         # dictionary may be modified during the loop.
@@ -2159,7 +2169,7 @@ class ApacheConfigurator(common.Installer):
         # There can be other RewriteRule directive lines in vhost config.
         # rewrite_args_dict keys are directive ids and the corresponding value
         # for each is a list of arguments to that directive.
-        rewrite_args_dict: DefaultDict[str, List[str]] = defaultdict(list)
+        rewrite_args_dict = defaultdict(list)  # type: DefaultDict[str, List[str]]
         pat = r'(.*directive\[\d+\]).*'
         for match in rewrite_path:
             m = re.match(pat, match)
@@ -2253,7 +2263,7 @@ class ApacheConfigurator(common.Installer):
         if ssl_vhost.aliases:
             serveralias = "ServerAlias " + " ".join(ssl_vhost.aliases)
 
-        rewrite_rule_args: List[str] = []
+        rewrite_rule_args = []  # type: List[str]
         if self.get_version() >= (2, 3, 9):
             rewrite_rule_args = constants.REWRITE_HTTPS_ARGS_WITH_END
         else:

certbot-apache/certbot_apache/_internal/dualparser.py

@@ -1,10 +1,10 @@
 """ Dual ParserNode implementation """
-from certbot_apache._internal import apacheparser
 from certbot_apache._internal import assertions
 from certbot_apache._internal import augeasparser
+from certbot_apache._internal import apacheparser
 
 
-class DualNodeBase:
+class DualNodeBase(object):
     """ Dual parser interface for in development testing. This is used as the
     base class for dual parser interface classes. This class handles runtime
     attribute value assertions."""

certbot-apache/certbot_apache/_internal/http_01.py

@@ -1,9 +1,9 @@
 """A class that performs HTTP-01 challenges for Apache"""
-import errno
 import logging
-from typing import List
-from typing import Set
+import errno
 
+from acme.magic_typing import List
+from acme.magic_typing import Set
 from certbot import errors
 from certbot.compat import filesystem
 from certbot.compat import os
@@ -57,7 +57,7 @@ class ApacheHttp01(common.ChallengePerformer):
         self.challenge_dir = os.path.join(
             self.configurator.config.work_dir,
             "http_challenges")
-        self.moded_vhosts: Set[VirtualHost] = set()
+        self.moded_vhosts = set()  # type: Set[VirtualHost]
 
     def perform(self):
         """Perform all HTTP-01 challenges."""
@@ -93,7 +93,7 @@ class ApacheHttp01(common.ChallengePerformer):
                     self.configurator.enable_mod(mod, temp=True)
 
     def _mod_config(self):
-        selected_vhosts: List[VirtualHost] = []
+        selected_vhosts = []  # type: List[VirtualHost]
         http_port = str(self.configurator.config.http01_port)
         for chall in self.achalls:
             # Search for matching VirtualHosts

certbot-apache/certbot_apache/_internal/interfaces.py

@@ -100,9 +100,12 @@ For this reason the internal representation of data should not ignore the case.
 """
 
 import abc
+import six
 
 
-class ParserNode(object, metaclass=abc.ABCMeta):
+
+@six.add_metaclass(abc.ABCMeta)
+class ParserNode(object):
     """
     ParserNode is the basic building block of the tree of such nodes,
     representing the structure of the configuration. It is largely meant to keep
@@ -201,7 +204,9 @@ class ParserNode(object, metaclass=abc.ABCMeta):
         """
 
 
-class CommentNode(ParserNode, metaclass=abc.ABCMeta):
+# Linter rule exclusion done because of https://github.com/PyCQA/pylint/issues/179
+@six.add_metaclass(abc.ABCMeta)  # pylint: disable=abstract-method
+class CommentNode(ParserNode):
     """
     CommentNode class is used for representation of comments within the parsed
     configuration structure. Because of the nature of comments, it is not able
@@ -244,7 +249,8 @@ class CommentNode(ParserNode, metaclass=abc.ABCMeta):
                                           metadata=kwargs.get('metadata', {}))  # pragma: no cover
 
 
-class DirectiveNode(ParserNode, metaclass=abc.ABCMeta):
+@six.add_metaclass(abc.ABCMeta)
+class DirectiveNode(ParserNode):
     """
     DirectiveNode class represents a configuration directive within the configuration.
     It can have zero or more parameters attached to it. Because of the nature of
@@ -319,7 +325,8 @@ class DirectiveNode(ParserNode, metaclass=abc.ABCMeta):
         """
 
 
-class BlockNode(DirectiveNode, metaclass=abc.ABCMeta):
+@six.add_metaclass(abc.ABCMeta)
+class BlockNode(DirectiveNode):
     """
     BlockNode class represents a block of nested configuration directives, comments
     and other blocks as its children. A BlockNode can have zero or more parameters

certbot-apache/certbot_apache/_internal/obj.py

@@ -1,7 +1,7 @@
 """Module contains classes used by the Apache Configurator."""
 import re
-from typing import Set
 
+from acme.magic_typing import Set
 from certbot.plugins import common
 
 
@@ -20,6 +20,9 @@ class Addr(common.Addr):
                      self.is_wildcard() and other.is_wildcard()))
         return False
 
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
     def __repr__(self):
         return "certbot_apache._internal.obj.Addr(" + repr(self.tup) + ")"
 
@@ -95,7 +98,7 @@ class Addr(common.Addr):
         return self.get_addr_obj(port)
 
 
-class VirtualHost:
+class VirtualHost(object):
     """Represents an Apache Virtualhost.
 
     :ivar str filep: file path of VH
@@ -137,7 +140,7 @@ class VirtualHost:
 
     def get_names(self):
         """Return a set of all names."""
-        all_names: Set[str] = set()
+        all_names = set()  # type: Set[str]
         all_names.update(self.aliases)
         # Strip out any scheme:// and <port> field from servername
         if self.name is not None:
@@ -188,6 +191,9 @@ class VirtualHost:
 
         return False
 
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
     def __hash__(self):
         return hash((self.filep, self.path,
                      tuple(self.addrs), tuple(self.get_names()),
@@ -245,7 +251,7 @@ class VirtualHost:
 
         # already_found acts to keep everything very conservative.
         # Don't allow multiple ip:ports in same set.
-        already_found: Set[str] = set()
+        already_found = set()  # type: Set[str]
 
         for addr in vhost.addrs:
             for local_addr in self.addrs:

certbot-apache/certbot_apache/_internal/override_centos.py

@@ -1,10 +1,9 @@
 """ Distribution specific override class for CentOS family (RHEL, Fedora) """
 import logging
-from typing import cast
-from typing import List
 
 import zope.interface
 
+from acme.magic_typing import List
 from certbot import errors
 from certbot import interfaces
 from certbot import util
@@ -77,7 +76,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
         alternative restart cmd used in CentOS.
         """
         super(CentOSConfigurator, self)._prepare_options()
-        cast(List[str], self.options["restart_cmd_alt"])[0] = self.option("ctl")
+        self.options["restart_cmd_alt"][0] = self.option("ctl")
 
     def get_parser(self):
         """Initializes the ApacheParser"""
@@ -103,9 +102,9 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
 
         loadmods = self.parser.find_dir("LoadModule", "ssl_module", exclude=False)
 
-        correct_ifmods: List[str] = []
-        loadmod_args: List[str] = []
-        loadmod_paths: List[str] = []
+        correct_ifmods = []  # type: List[str]
+        loadmod_args = []  # type: List[str]
+        loadmod_paths = []  # type: List[str]
         for m in loadmods:
             noarg_path = m.rpartition("/")[0]
             path_args = self.parser.get_all_args(noarg_path)

certbot-apache/certbot_apache/_internal/override_fedora.py

@@ -1,7 +1,4 @@
 """ Distribution specific override class for Fedora 29+ """
-from typing import cast
-from typing import List
-
 import zope.interface
 
 from certbot import errors
@@ -72,9 +69,9 @@ class FedoraConfigurator(configurator.ApacheConfigurator):
         of Fedora to restart httpd.
         """
         super(FedoraConfigurator, self)._prepare_options()
-        cast(List[str], self.options["restart_cmd"])[0] = 'apachectl'
-        cast(List[str], self.options["restart_cmd_alt"])[0] = 'apachectl'
-        cast(List[str], self.options["conftest_cmd"])[0] = 'apachectl'
+        self.options["restart_cmd"][0] = 'apachectl'
+        self.options["restart_cmd_alt"][0] = 'apachectl'
+        self.options["conftest_cmd"][0] = 'apachectl'
 
 
 class FedoraParser(parser.ApacheParser):

certbot-apache/certbot_apache/_internal/override_gentoo.py

@@ -1,7 +1,4 @@
 """ Distribution specific override class for Gentoo Linux """
-from typing import cast
-from typing import List
-
 import zope.interface
 
 from certbot import interfaces
@@ -39,7 +36,7 @@ class GentooConfigurator(configurator.ApacheConfigurator):
         alternative restart cmd used in Gentoo.
         """
         super(GentooConfigurator, self)._prepare_options()
-        cast(List[str], self.options["restart_cmd_alt"])[0] = self.option("ctl")
+        self.options["restart_cmd_alt"][0] = self.option("ctl")
 
     def get_parser(self):
         """Initializes the ApacheParser"""

certbot-apache/certbot_apache/_internal/override_suse.py

@@ -14,10 +14,10 @@ class OpenSUSEConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/apache2/vhosts.d",
         vhost_files="*.conf",
         logs_root="/var/log/apache2",
-        ctl="apachectl",
-        version_cmd=['apachectl', '-v'],
-        restart_cmd=['apachectl', 'graceful'],
-        conftest_cmd=['apachectl', 'configtest'],
+        ctl="apache2ctl",
+        version_cmd=['apache2ctl', '-v'],
+        restart_cmd=['apache2ctl', 'graceful'],
+        conftest_cmd=['apache2ctl', 'configtest'],
         enmod="a2enmod",
         dismod="a2dismod",
         le_vhost_ext="-le-ssl.conf",

certbot-apache/certbot_apache/_internal/parser.py

@@ -3,9 +3,12 @@ import copy
 import fnmatch
 import logging
 import re
-from typing import Dict
-from typing import List
+import sys
 
+import six
+
+from acme.magic_typing import Dict
+from acme.magic_typing import List
 from certbot import errors
 from certbot.compat import os
 from certbot_apache._internal import apache_util
@@ -14,7 +17,7 @@ from certbot_apache._internal import constants
 logger = logging.getLogger(__name__)
 
 
-class ApacheParser:
+class ApacheParser(object):
     """Class handles the fine details of parsing the Apache Configuration.
 
     .. todo:: Make parsing general... remove sites-available etc...
@@ -48,9 +51,9 @@ class ApacheParser:
                 "version 1.2.0 or higher, please make sure you have you have "
                 "those installed.")
 
-        self.modules: Dict[str, str] = {}
-        self.parser_paths: Dict[str, List[str]] = {}
-        self.variables: Dict[str, str] = {}
+        self.modules = {}  # type: Dict[str, str]
+        self.parser_paths = {}  # type: Dict[str, List[str]]
+        self.variables = {}  # type: Dict[str, str]
 
         # Find configuration root and make sure augeas can parse it.
         self.root = os.path.abspath(root)
@@ -263,7 +266,7 @@ class ApacheParser:
             the iteration issue.  Else... parse and enable mods at same time.
 
         """
-        mods: Dict[str, str] = {}
+        mods = {}  # type: Dict[str, str]
         matches = self.find_dir("LoadModule")
         iterator = iter(matches)
         # Make sure prev_size != cur_size for do: while: iteration
@@ -272,7 +275,7 @@ class ApacheParser:
         while len(mods) != prev_size:
             prev_size = len(mods)
 
-            for match_name, match_filename in zip(
+            for match_name, match_filename in six.moves.zip(
                     iterator, iterator):
                 mod_name = self.get_arg(match_name)
                 mod_filename = self.get_arg(match_filename)
@@ -550,7 +553,7 @@ class ApacheParser:
         else:
             arg_suffix = "/*[self::arg=~regexp('%s')]" % case_i(arg)
 
-        ordered_matches: List[str] = []
+        ordered_matches = []  # type: List[str]
 
         # TODO: Wildcards should be included in alphabetical order
         # https://httpd.apache.org/docs/2.4/mod/core.html#include
@@ -735,6 +738,9 @@ class ApacheParser:
         :rtype: str
 
         """
+        if sys.version_info < (3, 6):
+            # This strips off final /Z(?ms)
+            return fnmatch.translate(clean_fn_match)[:-7]  # pragma: no cover
         # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z
         return fnmatch.translate(clean_fn_match)[4:-3]  # pragma: no cover
 

certbot-apache/setup.py

@@ -1,7 +1,11 @@
+from distutils.version import LooseVersion
+import sys
+
+from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.14.0.dev0'
+version = '1.10.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -9,11 +13,20 @@ install_requires = [
     'acme>=0.29.0',
     'certbot>=1.6.0',
     'python-augeas',
-    'setuptools>=39.0.1',
+    'setuptools',
     'zope.component',
     'zope.interface',
 ]
 
+setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
+if setuptools_known_environment_markers:
+    install_requires.append('mock ; python_version < "3.3"')
+elif 'bdist_wheel' in sys.argv[1:]:
+    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
+                       'of setuptools. Version 36.2+ of setuptools is required.')
+elif sys.version_info < (3,3):
+    install_requires.append('mock')
+
 dev_extras = [
     'apacheconfig>=0.3.2',
 ]
@@ -26,7 +39,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
-    python_requires='>=3.6',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*',
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Environment :: Plugins',
@@ -34,11 +47,12 @@ setup(
         'License :: OSI Approved :: Apache Software License',
         'Operating System :: POSIX :: Linux',
         'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-apache/tests/apache-conf-files/apache-conf-test

@@ -52,7 +52,7 @@ function Cleanup() {
 # if our environment asks us to enable modules, do our best!
 if [ "$1" = --debian-modules ] ; then
     sudo apt-get install -y apache2
-    sudo apt-get install -y libapache2-mod-wsgi-py3
+    sudo apt-get install -y libapache2-mod-wsgi
     sudo apt-get install -y libapache2-mod-macro
 
     for mod in  ssl rewrite macro wsgi deflate userdir version mime setenvif ; do

certbot-apache/tests/augeasnode_test.py

@@ -1,6 +1,4 @@
 """Tests for AugeasParserNode classes"""
-from typing import List
-
 try:
     import mock
 except ImportError: # pragma: no cover
@@ -109,7 +107,7 @@ class AugeasParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-
 
     def test_set_parameters(self):
         servernames = self.config.parser_root.find_directives("servername")
-        names: List[str] = []
+        names = []  # type: List[str]
         for servername in servernames:
             names += servername.parameters
         self.assertFalse("going_to_set_this" in names)

certbot-apache/tests/autohsts_test.py

@@ -7,6 +7,7 @@ try:
     import mock
 except ImportError: # pragma: no cover
     from unittest import mock # type: ignore
+import six  # pylint: disable=unused-import  # six is used in mock.patch()
 
 from certbot import errors
 from certbot_apache._internal import constants

certbot-apache/tests/configurator_test.py

@@ -10,6 +10,7 @@ try:
     import mock
 except ImportError: # pragma: no cover
     from unittest import mock # type: ignore
+import six  # pylint: disable=unused-import  # six is used in mock.patch()
 
 from acme import challenges
 from certbot import achallenges
@@ -725,7 +726,7 @@ class MultipleVhostsTest(util.ApacheTest):
         # This calls open
         self.config.reverter.register_file_creation = mock.Mock()
         mock_open.side_effect = IOError
-        with mock.patch("builtins.open", mock_open):
+        with mock.patch("six.moves.builtins.open", mock_open):
             self.assertRaises(
                 errors.PluginError,
                 self.config.make_vhost_ssl, self.vh_truth[0])
@@ -1336,6 +1337,13 @@ class MultipleVhostsTest(util.ApacheTest):
                           self.config.enable_mod,
                           "whatever")
 
+    def test_wildcard_domain(self):
+        # pylint: disable=protected-access
+        cases = {u"*.example.org": True, b"*.x.example.org": True,
+                 u"a.example.org": False, b"a.x.example.org": False}
+        for key in cases:
+            self.assertEqual(self.config._wildcard_domain(key), cases[key])
+
     def test_choose_vhosts_wildcard(self):
         # pylint: disable=protected-access
         mock_path = "certbot_apache._internal.display_ops.select_vhost_multiple"
@@ -1349,10 +1357,10 @@ class MultipleVhostsTest(util.ApacheTest):
 
             # And the actual returned values
             self.assertEqual(len(vhs), 1)
-            self.assertEqual(vhs[0].name, "certbot.demo")
+            self.assertTrue(vhs[0].name == "certbot.demo")
             self.assertTrue(vhs[0].ssl)
 
-            self.assertNotEqual(vhs[0], self.vh_truth[3])
+            self.assertFalse(vhs[0] == self.vh_truth[3])
 
     @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.make_vhost_ssl")
     def test_choose_vhosts_wildcard_no_ssl(self, mock_makessl):
@@ -1463,10 +1471,10 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.parser.aug.match = mock_match
         vhs = self.config.get_virtual_hosts()
         self.assertEqual(len(vhs), 2)
-        self.assertEqual(vhs[0], self.vh_truth[1])
+        self.assertTrue(vhs[0] == self.vh_truth[1])
         # mock_vhost should have replaced the vh_truth[0], because its filepath
         # isn't a symlink
-        self.assertEqual(vhs[1], mock_vhost)
+        self.assertTrue(vhs[1] == mock_vhost)
 
 
 class AugeasVhostsTest(util.ApacheTest):
@@ -1833,7 +1841,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
 
     def test_open_module_file(self):
         mock_open = mock.mock_open(read_data="testing 12 3")
-        with mock.patch("builtins.open", mock_open):
+        with mock.patch("six.moves.builtins.open", mock_open):
             self.assertEqual(self.config._open_module_file("/nonsense/"), "testing 12 3")
 
 if __name__ == "__main__":

certbot-apache/tests/dualnode_test.py

@@ -412,9 +412,9 @@ class DualParserNodeTest(unittest.TestCase):  # pylint: disable=too-many-public-
                                                     ancestor=self.block,
                                                     filepath="/path/to/whatever",
                                                     metadata=self.metadata)
-        self.assertNotEqual(self.block, ne_block)
-        self.assertNotEqual(self.directive, ne_directive)
-        self.assertNotEqual(self.comment, ne_comment)
+        self.assertFalse(self.block == ne_block)
+        self.assertFalse(self.directive == ne_directive)
+        self.assertFalse(self.comment == ne_comment)
 
     def test_parsed_paths(self):
         mock_p = mock.MagicMock(return_value=['/path/file.conf',

certbot-apache/tests/http_01_test.py

@@ -1,7 +1,6 @@
 """Test for certbot_apache._internal.http_01."""
 import unittest
 import errno
-from typing import List
 
 try:
     import mock
@@ -27,7 +26,7 @@ class ApacheHttp01Test(util.ApacheTest):
         super(ApacheHttp01Test, self).setUp(*args, **kwargs)
 
         self.account_key = self.rsa512jwk
-        self.achalls: List[achallenges.KeyAuthorizationAnnotatedChallenge] = []
+        self.achalls = []  # type: List[achallenges.KeyAuthorizationAnnotatedChallenge]
         vh_truth = util.get_vh_truth(
             self.temp_dir, "debian_apache_2_4/multiple_vhosts")
         # Takes the vhosts for encryption-example.demo, certbot.demo

certbot-apache/tests/obj_test.py

@@ -27,14 +27,14 @@ class VirtualHostTest(unittest.TestCase):
             "certbot_apache._internal.obj.Addr(('127.0.0.1', '443'))")
 
     def test_eq(self):
-        self.assertEqual(self.vhost1b, self.vhost1)
-        self.assertNotEqual(self.vhost1, self.vhost2)
+        self.assertTrue(self.vhost1b == self.vhost1)
+        self.assertFalse(self.vhost1 == self.vhost2)
         self.assertEqual(str(self.vhost1b), str(self.vhost1))
-        self.assertNotEqual(self.vhost1b, 1234)
+        self.assertFalse(self.vhost1b == 1234)
 
     def test_ne(self):
-        self.assertNotEqual(self.vhost1, self.vhost2)
-        self.assertEqual(self.vhost1, self.vhost1b)
+        self.assertTrue(self.vhost1 != self.vhost2)
+        self.assertFalse(self.vhost1 != self.vhost1b)
 
     def test_conflicts(self):
         from certbot_apache._internal.obj import Addr
@@ -128,13 +128,13 @@ class AddrTest(unittest.TestCase):
         self.assertTrue(self.addr1.conflicts(self.addr2))
 
     def test_equal(self):
-        self.assertEqual(self.addr1, self.addr2)
-        self.assertNotEqual(self.addr, self.addr1)
-        self.assertNotEqual(self.addr, 123)
+        self.assertTrue(self.addr1 == self.addr2)
+        self.assertFalse(self.addr == self.addr1)
+        self.assertFalse(self.addr == 123)
 
     def test_not_equal(self):
-        self.assertEqual(self.addr1, self.addr2)
-        self.assertNotEqual(self.addr, self.addr1)
+        self.assertFalse(self.addr1 != self.addr2)
+        self.assertTrue(self.addr != self.addr1)
 
 
 if __name__ == "__main__":

certbot-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="1.13.0"
+LE_AUTO_VERSION="1.9.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -799,14 +799,15 @@ BootstrapMageiaCommon() {
 # that function. If Bootstrap is set to a function that doesn't install any
 # packages BOOTSTRAP_VERSION is not set.
 if [ -f /etc/debian_version ]; then
-  DEPRECATED_OS=1
+  Bootstrap() {
+    BootstrapMessage "Debian-based OSes"
+    BootstrapDebCommon
+  }
+  BOOTSTRAP_VERSION="BootstrapDebCommon $BOOTSTRAP_DEB_COMMON_VERSION"
 elif [ -f /etc/mageia-release ]; then
   # Mageia has both /etc/mageia-release and /etc/redhat-release
   DEPRECATED_OS=1
-  NO_SELF_UPGRADE=1
 elif [ -f /etc/redhat-release ]; then
-  DEPRECATED_OS=1
-  NO_SELF_UPGRADE=1
   # Run DeterminePythonVersion to decide on the basis of available Python versions
   # whether to use 2.x or 3.x on RedHat-like systems.
   # Then, revert LE_PYTHON to its previous state.
@@ -839,7 +840,12 @@ elif [ -f /etc/redhat-release ]; then
       INTERACTIVE_BOOTSTRAP=1
     fi
 
+    Bootstrap() {
+      BootstrapMessage "Legacy RedHat-based OSes that will use Python3"
+      BootstrapRpmPython3Legacy
+    }
     USE_PYTHON_3=1
+    BOOTSTRAP_VERSION="BootstrapRpmPython3Legacy $BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION"
 
     # Try now to enable SCL rh-python36 for systems already bootstrapped
     # NB: EnablePython36SCL has been defined along with BootstrapRpmPython3Legacy in certbot-auto
@@ -858,38 +864,43 @@ elif [ -f /etc/redhat-release ]; then
     fi
 
     if [ "$RPM_USE_PYTHON_3" = 1 ]; then
+      Bootstrap() {
+        BootstrapMessage "RedHat-based OSes that will use Python3"
+        BootstrapRpmPython3
+      }
       USE_PYTHON_3=1
+      BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+    else
+      Bootstrap() {
+        BootstrapMessage "RedHat-based OSes"
+        BootstrapRpmCommon
+      }
+      BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
     fi
   fi
 
   LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
   DEPRECATED_OS=1
-  NO_SELF_UPGRADE=1
 elif [ -f /etc/arch-release ]; then
   DEPRECATED_OS=1
-  NO_SELF_UPGRADE=1
 elif [ -f /etc/manjaro-release ]; then
   DEPRECATED_OS=1
-  NO_SELF_UPGRADE=1
 elif [ -f /etc/gentoo-release ]; then
   DEPRECATED_OS=1
-  NO_SELF_UPGRADE=1
 elif uname | grep -iq FreeBSD ; then
   DEPRECATED_OS=1
-  NO_SELF_UPGRADE=1
 elif uname | grep -iq Darwin ; then
   DEPRECATED_OS=1
-  NO_SELF_UPGRADE=1
 elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then
-  DEPRECATED_OS=1
-  NO_SELF_UPGRADE=1
+  Bootstrap() {
+    ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon
+  }
+  BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
 elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then
   DEPRECATED_OS=1
-  NO_SELF_UPGRADE=1
 else
   DEPRECATED_OS=1
-  NO_SELF_UPGRADE=1
 fi
 
 # We handle this case after determining the normal bootstrap version to allow
@@ -1118,9 +1129,7 @@ if [ "$1" = "--le-auto-phase2" ]; then
     fi
 
     if [ -f "$VENV_BIN/letsencrypt" -a "$INSTALL_ONLY" != 1 ]; then
-      error "certbot-auto and its Certbot installation will no longer receive updates."
-      error "You will not receive any bug fixes including those fixing server compatibility"
-      error "or security problems."
+      error "Certbot will no longer receive updates."
       error "Please visit https://certbot.eff.org/ to check for other alternatives."
       "$VENV_BIN/letsencrypt" "$@"
       exit 0
@@ -1488,18 +1497,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==1.13.0 \
-    --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \
-    --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458
-acme==1.13.0 \
-    --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \
-    --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088
-certbot-apache==1.13.0 \
-    --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \
-    --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1
-certbot-nginx==1.13.0 \
-    --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \
-    --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08
+certbot==1.9.0 \
+    --hash=sha256:d5a804d32e471050921f7b39ed9859e2e9de02824176ed78f57266222036b53a \
+    --hash=sha256:2ff9bf7d9af381c7efee22dec2dd6938d9d8fddcc9e11682b86e734164a30b57
+acme==1.9.0 \
+    --hash=sha256:d8061b396a22b21782c9b23ff9a945b23e50fca2573909a42f845e11d5658ac5 \
+    --hash=sha256:38a1630c98e144136c62eec4d2c545a1bdb1a3cd4eca82214be6b83a1f5a161f
+certbot-apache==1.9.0 \
+    --hash=sha256:09528a820d57e54984d490100644cd8a6603db97bf5776f86e95795ecfacf23d \
+    --hash=sha256:f47fb3f4a9bd927f4812121a0beefe56b163475a28f4db34c64dc838688d9e9e
+certbot-nginx==1.9.0 \
+    --hash=sha256:bb2e3f7fe17f071f350a3efa48571b8ef40a8e4b6db9c6da72539206a20b70be \
+    --hash=sha256:ab26a4f49d53b0e8bf0f903e58e2a840cda233fe1cbbc54c36ff17f973e57d65
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------

certbot-ci/certbot_integration_tests/assets/bind-config/conf/named.conf

@@ -1,60 +0,0 @@
-options {
-  directory "/var/cache/bind";
-
-  // Running inside Docker. Bind address on Docker host is 127.0.0.1.
-  listen-on { any; };
-  listen-on-v6 { any; };
-
-  // We are allowing BIND to service recursive queries, but only in an extremely limimited sense
-  // where it is entirely disconnected from public DNS:
-  // - Iterative queries are disabled. Only forwarding to a non-existent forwarder.
-  // - The only recursive answers we can get (that will not be a SERVFAIL) will come from the
-  //   RPZ "mock-recursion" zone. Effectively this means we are mocking out the entirety of
-  //   public DNS.
-  allow-recursion { any; };       // BIND will only answer using RPZ if recursion is enabled
-  forwarders { 192.0.2.254; };    // Nobody is listening, this is TEST-NET-1
-  forward only;                   // Do NOT perform iterative queries from the root zone
-  dnssec-validation no;           // Do not bother fetching the root DNSKEY set (performance)
-  response-policy {               // All recursive queries will be served from here.
-    zone "mock-recursion"
-    log yes;
-  } recursive-only no             // Allow RPZs to affect authoritative zones too.
-    qname-wait-recurse no         // No real recursion.
-    nsip-wait-recurse no;         // No real recursion.
-
-  allow-transfer { none; };
-  allow-update { none; };
-};
-
-key "default-key." {
-  algorithm hmac-sha512;
-  secret "91CgOwzihr0nAVEHKFXJPQCbuBBbBI19Ks5VAweUXgbF40NWTD83naeg3c5y2MPdEiFRXnRLJxL6M+AfHCGLNw==";
-};
-
-zone "mock-recursion" {
-  type primary;
-  file "/var/lib/bind/rpz.mock-recursion";
-  allow-query {
-    none;
-  };
-};
-
-zone "example.com." {
-  type primary;
-  file "/var/lib/bind/db.example.com";
-  journal "/var/cache/bind/db.example.com.jnl";
-
-  update-policy {
-    grant default-key zonesub TXT;
-  };
-};
-
-zone "sub.example.com." {
-  type primary;
-  file "/var/lib/bind/db.sub.example.com";
-  journal "/var/cache/bind/db.sub.example.com.jnl";
-
-  update-policy {
-    grant default-key zonesub TXT;
-  };
-};

certbot-ci/certbot_integration_tests/assets/bind-config/rfc2136-credentials-default.ini.tpl

@@ -1,10 +0,0 @@
-# Target DNS server
-dns_rfc2136_server = {server_address}
-# Target DNS port
-dns_rfc2136_port = {server_port}
-# TSIG key name
-dns_rfc2136_name = default-key.
-# TSIG key secret
-dns_rfc2136_secret = 91CgOwzihr0nAVEHKFXJPQCbuBBbBI19Ks5VAweUXgbF40NWTD83naeg3c5y2MPdEiFRXnRLJxL6M+AfHCGLNw==
-# TSIG key algorithm
-dns_rfc2136_algorithm = HMAC-SHA512

certbot-ci/certbot_integration_tests/assets/bind-config/zones/db.example.com

@@ -1,11 +0,0 @@
-$ORIGIN example.com.
-$TTL 3600
-example.com.  IN  SOA   ns1.example.com. admin.example.com. ( 2020091025 7200 3600 1209600 3600 )
-
-example.com.  IN  NS    ns1
-example.com.  IN  NS    ns2
-
-ns1           IN  A     192.0.2.2
-ns2           IN  A     192.0.2.3
-
-@         IN  A     192.0.2.1

certbot-ci/certbot_integration_tests/assets/bind-config/zones/db.sub.example.com

@@ -1,9 +0,0 @@
-$ORIGIN sub.example.com.
-$TTL 3600
-sub.example.com.  IN  SOA   ns1.example.com. admin.example.com. ( 2020091025 7200 3600 1209600 3600 )
-
-sub.example.com.  IN  NS    ns1
-sub.example.com.  IN  NS    ns2
-
-ns1           IN  A     192.0.2.2
-ns2           IN  A     192.0.2.3

certbot-ci/certbot_integration_tests/assets/bind-config/zones/rpz.mock-recursion

@@ -1,6 +0,0 @@
-$TTL 3600
-
-@        SOA ns1.example.test. dummy.example.test. 1 12h 15m 3w 2h
-         NS  ns1.example.test.
-
-_acme-challenge.aliased.example   IN     CNAME   _acme-challenge.example.com.

certbot-ci/certbot_integration_tests/assets/hook.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import print_function
 import os
 import sys
 

certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/README

@@ -1,14 +0,0 @@
-This directory contains your keys and certificates.
-
-`privkey.pem`  : the private key for your certificate.
-`fullchain.pem`: the certificate file used in most server software.
-`chain.pem`    : used for OCSP stapling in Nginx >=1.3.7.
-`cert.pem`     : will break many server configurations, and should not be used
-                 without reading further documentation (see link below).
-
-WARNING: DO NOT MOVE OR RENAME THESE FILES!
-         Certbot expects these files to remain in this location in order
-         to function properly!
-
-We recommend not moving these files. For more information, see the Certbot
-User Guide at https://certbot.eff.org/docs/using.html#where-are-my-certificates.

certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/cert.pem

@@ -1,18 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIC2zCCAcOgAwIBAgIIBvrEnbPRYu8wDQYJKoZIhvcNAQELBQAwKDEmMCQGA1UE
-AxMdUGViYmxlIEludGVybWVkaWF0ZSBDQSAxMjZjNGIwHhcNMjAxMDEyMjEwNzQw
-WhcNMjUxMDEyMjEwNzQwWjAjMSEwHwYDVQQDExhjLmVuY3J5cHRpb24tZXhhbXBs
-ZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARjMhuW0ENPPC33PjB5XsYU
-CRw640kPQENIDatcTJaENZIZdqKd6rI6jc+lpbmXot7Zi52clJlSJS+V6oDAt2Lh
-o4HYMIHVMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB
-BQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUj7Kd3ENqxlPf8B2bIGhsjydX
-mPswHwYDVR0jBBgwFoAUEiGxlkRsi+VvcogH5dVD3h1laAcwMQYIKwYBBQUHAQEE
-JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vMTI3LjAuMC4xOjQwMDIwIwYDVR0RBBww
-GoIYYy5lbmNyeXB0aW9uLWV4YW1wbGUuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQCl
-k0JXsa8y7fg41WWMDhw60bPW77O0FtOmTcnhdI5daYNemQVk+Q5EMaBLQ/oGjgXd
-9QXFzXH1PL904YEnSLt+iTpXn++7rQSNzQsdYqw0neWk4f5pEBiN+WORpb6mwobV
-ifMtBOkNEHvrJ2Pkci9U1lLwtKD/DSew6QtJU5DSkmH1XdGuMJiubygEIvELtvgq
-cP9S368ZvPmPGmKaJQXBiuaR8MTjY/Bkr79aXQMjKbf+mpn7h0POCcePk1DY/rm6
-Da+X16lf0hHyQhSUa7Vgyim6rK1/hlw+Z00i+sQCKD9Ih7kXuuGqfSDC33cfO8Tj
-o/MXO8lcxkrem5zU5QWP
------END CERTIFICATE-----

certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/chain.pem

@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDUDCCAjigAwIBAgIIbi787yVrcMAwDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE
-AxMVUGViYmxlIFJvb3QgQ0EgMGM1MjI1MCAXDTIwMTAxMjIwMjI0NloYDzIwNTAx
-MDEyMjEyMjQ2WjAoMSYwJAYDVQQDEx1QZWJibGUgSW50ZXJtZWRpYXRlIENBIDEy
-NmM0YjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALGeVk1BMJraeqRq
-mJ2+hgso8VOAv2s2CVxUJjIVcn7f2adE8NyTsSQ1brlsnKCUYUw7yLTQH0izLQRB
-qKVIDFkUqo5/FuTJ2QlfA2EwBL8J7s/7L7vj3L0DiVpwgxPSyFEwdl/Y5y7ofsX5
-CIhCFcaMAmTIuKLiSfCJjGwkbEMuolm+lO8Mikxxc/JtDVUC479ugU7PU9O09bMH
-nm+sD6Bgd+KMoPkCCCoeShJS9X3Ziq9HGc7Z6nhM/zirFARt2XkonEdAZ8br01zY
-MRiY9txhlWQ7mUkOtzOSoEuYJNoUbvMUf0+tNzto26WRyF7dJmh7lTBsYrvAwUTx
-PzNyst0CAwEAAaOBgzCBgDAOBgNVHQ8BAf8EBAMCAoQwHQYDVR0lBBYwFAYIKwYB
-BQUHAwEGCCsGAQUFBwMCMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFBIhsZZE
-bIvlb3KIB+XVQ94dZWgHMB8GA1UdIwQYMBaAFOaKTaXg37vKgRt7d79YOjAoAtJT
-MA0GCSqGSIb3DQEBCwUAA4IBAQAU2mZii7PH2pkw2lNM0QqPbcW/UYyvFoUeM8Aq
-uCtsI2s+oxCJTqzfLsA0N8NY4nHLQ5wAlNJfJekngni8hbmJTKU4JFTMe7kLQO8P
-fJbk0pTzhhHVQw7CVwB6Pwq3u2m/JV+d6xDIDc+AVkuEl19ZJU0rTWyooClfFLZV
-EdZmEiUtA3PGlxoYwYhoGHYlhFxsoFONhCsBEdN7k7FKtFGVxN7oc5SKmKp0YZTW
-fcrEtrdNThATO4ymhCC2zh33NI/MT1O74fpaAc2k6LcTl57MKiLfTYX4LTL6v9JG
-9tlNqjFVRRmzEbtXTPcCb+w9g1VqoOGok7mGXYLTYtShCuvE
------END CERTIFICATE-----

certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/fullchain.pem

@@ -1,38 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIC2zCCAcOgAwIBAgIILlmGtZhUFEwwDQYJKoZIhvcNAQELBQAwKDEmMCQGA1UE
-AxMdUGViYmxlIEludGVybWVkaWF0ZSBDQSAxMjZjNGIwHhcNMjAxMDEyMjA1MDM0
-WhcNMjUxMDEyMjA1MDM0WjAjMSEwHwYDVQQDExhjLmVuY3J5cHRpb24tZXhhbXBs
-ZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARHEzR8JPWrEmpmgM+F2bk5
-9mT0u6CjzmJG0QpbaqprLiG5NGpW84VQ5TFCrmC4KxYfigCfMhfHRNfFYvNUK3V/
-o4HYMIHVMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB
-BQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU1CsVL+bPnzaxxQ5jUENmQJIO
-lKwwHwYDVR0jBBgwFoAUEiGxlkRsi+VvcogH5dVD3h1laAcwMQYIKwYBBQUHAQEE
-JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vMTI3LjAuMC4xOjQwMDIwIwYDVR0RBBww
-GoIYYy5lbmNyeXB0aW9uLWV4YW1wbGUuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQBn
-2D8loC7pfk28JYpFLr5lmFKJWWmtLGlpsWDj61fVjtTfGKLziJz+MM6il4Y3hIz5
-58qiFK0ue0M63dIBJ33N+XxSEXon4Q0gy/zRWfH9jtPJ3FwfjkU/RT9PAUClYi0G
-ptNWnTmgQkNzousbcAtRNXuuShH3856vhUnwkX+xM+cbIDi1JVmFjcGrEEQJ0rUF
-mv2ZTyfbWbUs3v4rReETi2NVzr1Ql6J+ByNcMvHODzFy3t0L6yelAw2ca1I+c9HU
-+Z0tnp/ykR7eXNuVLivok8UBf5OC413lh8ZO5g+Bgzh/LdtkUuavg1MYtEX0H6mX
-9U7y3nVI8WEbPGf+HDeu
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-MIIDUDCCAjigAwIBAgIIbi787yVrcMAwDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE
-AxMVUGViYmxlIFJvb3QgQ0EgMGM1MjI1MCAXDTIwMTAxMjIwMjI0NloYDzIwNTAx
-MDEyMjEyMjQ2WjAoMSYwJAYDVQQDEx1QZWJibGUgSW50ZXJtZWRpYXRlIENBIDEy
-NmM0YjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALGeVk1BMJraeqRq
-mJ2+hgso8VOAv2s2CVxUJjIVcn7f2adE8NyTsSQ1brlsnKCUYUw7yLTQH0izLQRB
-qKVIDFkUqo5/FuTJ2QlfA2EwBL8J7s/7L7vj3L0DiVpwgxPSyFEwdl/Y5y7ofsX5
-CIhCFcaMAmTIuKLiSfCJjGwkbEMuolm+lO8Mikxxc/JtDVUC479ugU7PU9O09bMH
-nm+sD6Bgd+KMoPkCCCoeShJS9X3Ziq9HGc7Z6nhM/zirFARt2XkonEdAZ8br01zY
-MRiY9txhlWQ7mUkOtzOSoEuYJNoUbvMUf0+tNzto26WRyF7dJmh7lTBsYrvAwUTx
-PzNyst0CAwEAAaOBgzCBgDAOBgNVHQ8BAf8EBAMCAoQwHQYDVR0lBBYwFAYIKwYB
-BQUHAwEGCCsGAQUFBwMCMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFBIhsZZE
-bIvlb3KIB+XVQ94dZWgHMB8GA1UdIwQYMBaAFOaKTaXg37vKgRt7d79YOjAoAtJT
-MA0GCSqGSIb3DQEBCwUAA4IBAQAU2mZii7PH2pkw2lNM0QqPbcW/UYyvFoUeM8Aq
-uCtsI2s+oxCJTqzfLsA0N8NY4nHLQ5wAlNJfJekngni8hbmJTKU4JFTMe7kLQO8P
-fJbk0pTzhhHVQw7CVwB6Pwq3u2m/JV+d6xDIDc+AVkuEl19ZJU0rTWyooClfFLZV
-EdZmEiUtA3PGlxoYwYhoGHYlhFxsoFONhCsBEdN7k7FKtFGVxN7oc5SKmKp0YZTW
-fcrEtrdNThATO4ymhCC2zh33NI/MT1O74fpaAc2k6LcTl57MKiLfTYX4LTL6v9JG
-9tlNqjFVRRmzEbtXTPcCb+w9g1VqoOGok7mGXYLTYtShCuvE
------END CERTIFICATE-----

certbot-ci/certbot_integration_tests/assets/sample-config/archive/c.encryption-example.com/privkey.pem

@@ -1,5 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgNgefv2dad4U1VYEi
-0WkdHuqywi5QXAe30OwNTTGjhbihRANCAARHEzR8JPWrEmpmgM+F2bk59mT0u6Cj
-zmJG0QpbaqprLiG5NGpW84VQ5TFCrmC4KxYfigCfMhfHRNfFYvNUK3V/
------END PRIVATE KEY-----

certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/README

@@ -1,14 +0,0 @@
-This directory contains your keys and certificates.
-
-`privkey.pem`  : the private key for your certificate.
-`fullchain.pem`: the certificate file used in most server software.
-`chain.pem`    : used for OCSP stapling in Nginx >=1.3.7.
-`cert.pem`     : will break many server configurations, and should not be used
-                 without reading further documentation (see link below).
-
-WARNING: DO NOT MOVE OR RENAME THESE FILES!
-         Certbot expects these files to remain in this location in order
-         to function properly!
-
-We recommend not moving these files. For more information, see the Certbot
-User Guide at https://certbot.eff.org/docs/using.html#where-are-my-certificates.

certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/cert.pem

@@ -1 +0,0 @@
-../../archive/c.encryption-example.com/cert.pem

certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/chain.pem

@@ -1 +0,0 @@
-../../archive/c.encryption-example.com/chain.pem

certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/fullchain.pem

@@ -1 +0,0 @@
-../../archive/c.encryption-example.com/fullchain.pem

certbot-ci/certbot_integration_tests/assets/sample-config/live/c.encryption-example.com/privkey.pem

@@ -1 +0,0 @@
-../../archive/c.encryption-example.com/privkey.pem

certbot-ci/certbot_integration_tests/assets/sample-config/renewal/c.encryption-example.com.conf

@@ -1,17 +0,0 @@
-# renew_before_expiry = 30 days
-version = 1.10.0.dev0
-archive_dir = sample-config/archive/c.encryption-example.com
-cert = sample-config/live/c.encryption-example.com/cert.pem
-privkey = sample-config/live/c.encryption-example.com/privkey.pem
-chain = sample-config/live/c.encryption-example.com/chain.pem
-fullchain = sample-config/live/c.encryption-example.com/fullchain.pem
-
-# Options used in the renewal process
-[renewalparams]
-authenticator = apache
-installer = apache
-account = 48d6b9e8d767eccf7e4d877d6ffa81e3
-key_type = ecdsa
-config_dir = sample-config-ec
-elliptic_curve = secp256r1
-manual_public_ip_logging_ok = True

certbot-ci/certbot_integration_tests/certbot_tests/__init__.py

@@ -1,4 +1,3 @@
-# pylint: disable=missing-module-docstring
 import pytest
 
 # Custom assertions defined in the following package need to be registered to be properly

certbot-ci/certbot_integration_tests/certbot_tests/assertions.py

@@ -2,11 +2,6 @@
 import io
 import os
 
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePrivateKey
-from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey
-from cryptography.hazmat.primitives.serialization import load_pem_private_key
-
 try:
     import grp
     POSIX_MODE = True
@@ -21,33 +16,6 @@ SYSTEM_SID = 'S-1-5-18'
 ADMINS_SID = 'S-1-5-32-544'
 
 
-def assert_elliptic_key(key, curve):
-    """
-    Asserts that the key at the given path is an EC key using the given curve.
-    :param key: path to key
-    :param curve: name of the expected elliptic curve
-    """
-    with open(key, 'rb') as file:
-        privkey1 = file.read()
-
-    key = load_pem_private_key(data=privkey1, password=None, backend=default_backend())
-
-    assert isinstance(key, EllipticCurvePrivateKey)
-    assert isinstance(key.curve, curve)
-
-
-def assert_rsa_key(key):
-    """
-    Asserts that the key at the given path is an RSA key.
-    :param key: path to key
-    """
-    with open(key, 'rb') as file:
-        privkey1 = file.read()
-
-    key = load_pem_private_key(data=privkey1, password=None, backend=default_backend())
-    assert isinstance(key, RSAPrivateKey)
-
-
 def assert_hook_execution(probe_path, probe_content):
     """
     Assert that a certbot hook has been executed

certbot-ci/certbot_integration_tests/certbot_tests/context.py

@@ -7,14 +7,14 @@ import tempfile
 from certbot_integration_tests.utils import certbot_call
 
 
-class IntegrationTestsContext:
+class IntegrationTestsContext(object):
     """General fixture describing a certbot integration tests context"""
     def __init__(self, request):
         self.request = request
 
-        if hasattr(request.config, 'workerinput'):  # Worker node
-            self.worker_id = request.config.workerinput['workerid']
-            acme_xdist = request.config.workerinput['acme_xdist']
+        if hasattr(request.config, 'slaveinput'):  # Worker node
+            self.worker_id = request.config.slaveinput['slaveid']
+            acme_xdist = request.config.slaveinput['acme_xdist']
         else:  # Primary node
             self.worker_id = 'primary'
             acme_xdist = request.config.acme_xdist
@@ -77,6 +77,6 @@ class IntegrationTestsContext:
         appending the pytest worker id to the subdomain, using this pattern:
         {subdomain}.{worker_id}.wtf
         :param subdomain: the subdomain to use in the generated domain (default 'le')
-        :return: the well-formed domain suitable for redirection on
+        :return: the well-formed domain suitable for redirection on 
         """
         return '{0}.{1}.wtf'.format(subdomain, self.worker_id)

certbot-ci/certbot_integration_tests/certbot_tests/test_main.py

@@ -1,4 +1,5 @@
 """Module executing integration tests against certbot core."""
+from __future__ import print_function
 
 import os
 from os.path import exists
@@ -8,20 +9,16 @@ import shutil
 import subprocess
 import time
 
-from cryptography.hazmat.primitives.asymmetric.ec import SECP256R1
-from cryptography.hazmat.primitives.asymmetric.ec import SECP384R1
-from cryptography.hazmat.primitives.asymmetric.ec import SECP521R1
 from cryptography.x509 import NameOID
+
 import pytest
 
 from certbot_integration_tests.certbot_tests import context as certbot_context
 from certbot_integration_tests.certbot_tests.assertions import assert_cert_count_for_lineage
-from certbot_integration_tests.certbot_tests.assertions import assert_elliptic_key
 from certbot_integration_tests.certbot_tests.assertions import assert_equals_group_owner
 from certbot_integration_tests.certbot_tests.assertions import assert_equals_group_permissions
 from certbot_integration_tests.certbot_tests.assertions import assert_equals_world_read_permissions
 from certbot_integration_tests.certbot_tests.assertions import assert_hook_execution
-from certbot_integration_tests.certbot_tests.assertions import assert_rsa_key
 from certbot_integration_tests.certbot_tests.assertions import assert_saved_renew_hook
 from certbot_integration_tests.certbot_tests.assertions import assert_world_no_permissions
 from certbot_integration_tests.certbot_tests.assertions import assert_world_read_permissions
@@ -29,9 +26,8 @@ from certbot_integration_tests.certbot_tests.assertions import EVERYBODY_SID
 from certbot_integration_tests.utils import misc
 
 
-@pytest.fixture(name='context')
-def test_context(request):
-    # pylint: disable=missing-function-docstring
+@pytest.fixture()
+def context(request):
     # Fixture request is a built-in pytest fixture describing current test request.
     integration_test_context = certbot_context.IntegrationTestsContext(request)
     try:
@@ -148,17 +144,6 @@ def test_certonly(context):
     """Test the certonly verb on certbot."""
     context.certbot(['certonly', '--cert-name', 'newname', '-d', context.get_domain('newname')])
 
-    assert_cert_count_for_lineage(context.config_dir, 'newname', 1)
-
-
-def test_certonly_webroot(context):
-    """Test the certonly verb with webroot plugin"""
-    with misc.create_http_server(context.http_01_port) as webroot:
-        certname = context.get_domain('webroot')
-        context.certbot(['certonly', '-a', 'webroot', '--webroot-path', webroot, '-d', certname])
-
-    assert_cert_count_for_lineage(context.config_dir, certname, 1)
-
 
 def test_auth_and_install_with_csr(context):
     """Test certificate issuance and install using an existing CSR."""
@@ -234,16 +219,14 @@ def test_renew_files_propagate_permissions(context):
     if os.name != 'nt':
         os.chmod(privkey1, 0o444)
     else:
-        import win32security  # pylint: disable=import-error
-        import ntsecuritycon  # pylint: disable=import-error
+        import win32security
+        import ntsecuritycon
         # Get the current DACL of the private key
         security = win32security.GetFileSecurity(privkey1, win32security.DACL_SECURITY_INFORMATION)
         dacl = security.GetSecurityDescriptorDacl()
         # Create a read permission for Everybody group
         everybody = win32security.ConvertStringSidToSid(EVERYBODY_SID)
-        dacl.AddAccessAllowedAce(
-            win32security.ACL_REVISION, ntsecuritycon.FILE_GENERIC_READ, everybody
-        )
+        dacl.AddAccessAllowedAce(win32security.ACL_REVISION, ntsecuritycon.FILE_GENERIC_READ, everybody)
         # Apply the updated DACL to the private key
         security.SetSecurityDescriptorDacl(1, dacl, 0)
         win32security.SetFileSecurity(privkey1, win32security.DACL_SECURITY_INFORMATION, security)
@@ -252,14 +235,12 @@ def test_renew_files_propagate_permissions(context):
 
     assert_cert_count_for_lineage(context.config_dir, certname, 2)
     if os.name != 'nt':
-        # On Linux, read world permissions + all group permissions
-        # will be copied from the previous private key
+        # On Linux, read world permissions + all group permissions will be copied from the previous private key
         assert_world_read_permissions(privkey2)
         assert_equals_world_read_permissions(privkey1, privkey2)
         assert_equals_group_permissions(privkey1, privkey2)
     else:
-        # On Windows, world will never have any permissions, and
-        # group permission is irrelevant for this platform
+        # On Windows, world will never have any permissions, and group permission is irrelevant for this platform
         assert_world_no_permissions(privkey2)
 
 
@@ -308,7 +289,7 @@ def test_renew_with_changed_private_key_complexity(context):
     assert_cert_count_for_lineage(context.config_dir, certname, 1)
 
     context.certbot(['renew'])
-
+    
     assert_cert_count_for_lineage(context.config_dir, certname, 2)
     key2 = join(context.config_dir, 'archive', certname, 'privkey2.pem')
     assert os.stat(key2).st_size > 3000
@@ -440,115 +421,20 @@ def test_reuse_key(context):
     assert len({cert1, cert2, cert3}) == 3
 
 
-def test_incorrect_key_type(context):
-    with pytest.raises(subprocess.CalledProcessError):
-        context.certbot(['--key-type="failwhale"'])
-
-
 def test_ecdsa(context):
-    """Test issuance for ECDSA CSR based request (legacy supported mode)."""
+    """Test certificate issuance with ECDSA key."""
     key_path = join(context.workspace, 'privkey-p384.pem')
     csr_path = join(context.workspace, 'csr-p384.der')
     cert_path = join(context.workspace, 'cert-p384.pem')
     chain_path = join(context.workspace, 'chain-p384.pem')
 
-    misc.generate_csr(
-        [context.get_domain('ecdsa')],
-        key_path, csr_path,
-        key_type=misc.ECDSA_KEY_TYPE
-    )
-    context.certbot([
-        'auth', '--csr', csr_path, '--cert-path', cert_path,
-        '--chain-path', chain_path,
-    ])
+    misc.generate_csr([context.get_domain('ecdsa')], key_path, csr_path, key_type=misc.ECDSA_KEY_TYPE)
+    context.certbot(['auth', '--csr', csr_path, '--cert-path', cert_path, '--chain-path', chain_path])
 
     certificate = misc.read_certificate(cert_path)
     assert 'ASN1 OID: secp384r1' in certificate
 
 
-def test_default_key_type(context):
-    """Test default key type is RSA"""
-    certname = context.get_domain('renew')
-    context.certbot([
-        'certonly',
-        '--cert-name', certname, '-d', certname
-    ])
-    filename = join(context.config_dir, 'archive/{0}/privkey1.pem').format(certname)
-    assert_rsa_key(filename)
-
-
-def test_default_curve_type(context):
-    """test that the curve used when not specifying any is secp256r1"""
-    certname = context.get_domain('renew')
-    context.certbot([
-        '--key-type', 'ecdsa', '--cert-name', certname, '-d', certname
-    ])
-    key1 = join(context.config_dir, 'archive/{0}/privkey1.pem'.format(certname))
-    assert_elliptic_key(key1, SECP256R1)
-
-
-@pytest.mark.parametrize('curve,curve_cls,skip_servers', [
-    # Curve name, Curve class, ACME servers to skip
-    ('secp256r1', SECP256R1, []),
-    ('secp384r1', SECP384R1, []),
-    ('secp521r1', SECP521R1, ['boulder-v1', 'boulder-v2'])]
-)
-def test_ecdsa_curves(context, curve, curve_cls, skip_servers):
-    """Test issuance for each supported ECDSA curve"""
-    if context.acme_server in skip_servers:
-        pytest.skip('ACME server {} does not support ECDSA curve {}'
-                    .format(context.acme_server, curve))
-
-    domain = context.get_domain('curve')
-    context.certbot([
-        'certonly',
-        '--key-type', 'ecdsa', '--elliptic-curve', curve,
-        '--force-renewal', '-d', domain,
-    ])
-    key = join(context.config_dir, "live", domain, 'privkey.pem')
-    assert_elliptic_key(key, curve_cls)
-
-
-def test_renew_with_ec_keys(context):
-    """Test proper renew with updated private key complexity."""
-    certname = context.get_domain('renew')
-    context.certbot([
-        'certonly',
-        '--cert-name', certname,
-        '--key-type', 'ecdsa', '--elliptic-curve', 'secp256r1',
-        '--force-renewal', '-d', certname,
-    ])
-
-    key1 = join(context.config_dir, "archive", certname, 'privkey1.pem')
-    assert 200 < os.stat(key1).st_size < 250  # ec keys of 256 bits are ~225 bytes
-    assert_elliptic_key(key1, SECP256R1)
-    assert_cert_count_for_lineage(context.config_dir, certname, 1)
-
-    context.certbot(['renew', '--elliptic-curve', 'secp384r1'])
-
-    assert_cert_count_for_lineage(context.config_dir, certname, 2)
-    key2 = join(context.config_dir, 'archive', certname, 'privkey2.pem')
-    assert_elliptic_key(key2, SECP384R1)
-    assert 280 < os.stat(key2).st_size < 320  # ec keys of 384 bits are ~310 bytes
-
-    # We expect here that the command will fail because without --key-type specified,
-    # Certbot must error out to prevent changing an existing certificate key type,
-    # without explicit user consent (by specifying both --cert-name and --key-type).
-    with pytest.raises(subprocess.CalledProcessError):
-        context.certbot([
-            'certonly',
-            '--force-renewal',
-            '-d', certname
-        ])
-
-    # We expect that the previous behavior of requiring both --cert-name and
-    # --key-type to be set to not apply to the renew subcommand.
-    context.certbot(['renew', '--force-renewal', '--key-type', 'rsa'])
-    assert_cert_count_for_lineage(context.config_dir, certname, 3)
-    key3 = join(context.config_dir, 'archive', certname, 'privkey3.pem')
-    assert_rsa_key(key3)
-
-
 def test_ocsp_must_staple(context):
     """Test that OCSP Must-Staple is correctly set in the generated certificate."""
     if context.acme_server == 'pebble':
@@ -647,22 +533,18 @@ def test_revoke_multiple_lineages(context):
     with open(join(context.config_dir, 'renewal', '{0}.conf'.format(cert2)), 'r') as file:
         data = file.read()
 
-    data = re.sub(
-        'archive_dir = .*\n',
-        'archive_dir = {0}\n'.format(
-            join(context.config_dir, 'archive', cert1).replace('\\', '\\\\')
-        ), data
-    )
+    data = re.sub('archive_dir = .*\n',
+                  'archive_dir = {0}\n'.format(join(context.config_dir, 'archive', cert1).replace('\\', '\\\\')),
+                  data)
 
     with open(join(context.config_dir, 'renewal', '{0}.conf'.format(cert2)), 'w') as file:
         file.write(data)
 
-    context.certbot([
+    output = context.certbot([
         'revoke', '--cert-path', join(context.config_dir, 'live', cert1, 'cert.pem')
     ])
 
-    with open(join(context.workspace, 'logs', 'letsencrypt.log'), 'r') as f:
-        assert 'Not deleting revoked certificates due to overlapping archive dirs' in f.read()
+    assert 'Not deleting revoked certs due to overlapping archive dirs' in output
 
 
 def test_wildcard_certificates(context):
@@ -775,4 +657,4 @@ def test_preferred_chain(context):
 
         with open(conf_path, 'r') as f:
             assert 'preferred_chain = {}'.format(requested) in f.read(), \
-                   'Expected preferred_chain to be set in renewal config'
+                   'Expected preferred_chain to be set in renewal config'

certbot-ci/certbot_integration_tests/conftest.py

@@ -6,12 +6,12 @@ for a directory a specific configuration using built-in pytest hooks.
 
 See https://docs.pytest.org/en/latest/reference.html#hook-reference
 """
+from __future__ import print_function
 import contextlib
 import subprocess
 import sys
 
 from certbot_integration_tests.utils import acme_server as acme_lib
-from certbot_integration_tests.utils import dns_server as dns_lib
 
 
 def pytest_addoption(parser):
@@ -23,10 +23,6 @@ def pytest_addoption(parser):
                      choices=['boulder-v1', 'boulder-v2', 'pebble'],
                      help='select the ACME server to use (boulder-v1, boulder-v2, '
                           'pebble), defaulting to pebble')
-    parser.addoption('--dns-server', default='challtestsrv',
-                     choices=['bind', 'challtestsrv'],
-                     help='select the DNS server to use (bind, challtestsrv), '
-                          'defaulting to challtestsrv')
 
 
 def pytest_configure(config):
@@ -34,9 +30,9 @@ def pytest_configure(config):
     Standard pytest hook used to add a configuration logic for each node of a pytest run.
     :param config: the current pytest configuration
     """
-    if not hasattr(config, 'workerinput'):  # If true, this is the primary node
+    if not hasattr(config, 'slaveinput'):  # If true, this is the primary node
         with _print_on_err():
-            _setup_primary_node(config)
+            config.acme_xdist = _setup_primary_node(config)
 
 
 def pytest_configure_node(node):
@@ -44,8 +40,7 @@ def pytest_configure_node(node):
     Standard pytest-xdist hook used to configure a worker node.
     :param node: current worker node
     """
-    node.workerinput['acme_xdist'] = node.config.acme_xdist
-    node.workerinput['dns_xdist'] = node.config.dns_xdist
+    node.slaveinput['acme_xdist'] = node.config.acme_xdist
 
 
 @contextlib.contextmanager
@@ -66,18 +61,12 @@ def _print_on_err():
 def _setup_primary_node(config):
     """
     Setup the environment for integration tests.
-
-    This function will:
+    Will:
         - check runtime compatibility (Docker, docker-compose, Nginx)
         - create a temporary workspace and the persistent GIT repositories space
-        - configure and start a DNS server using Docker, if configured
         - configure and start paralleled ACME CA servers using Docker
-        - transfer ACME CA and DNS servers configurations to pytest nodes using env variables
-
-    This function modifies `config` by injecting the ACME CA and DNS server configurations,
-    in addition to cleanup functions for those servers.
-
-    :param config: Configuration of the pytest primary node. Is modified by this function.
+        - transfer ACME CA servers configurations to pytest nodes using env variables
+    :param config: Configuration of the pytest primary node
     """
     # Check for runtime compatibility: some tools are required to be available in PATH
     if 'boulder' in config.option.acme_server:
@@ -90,35 +79,18 @@ def _setup_primary_node(config):
         try:
             subprocess.check_output(['docker-compose', '-v'], stderr=subprocess.STDOUT)
         except (subprocess.CalledProcessError, OSError):
-            raise ValueError(
-                'Error: docker-compose is required in PATH to launch the integration tests, '
-                'but is not installed or not available for current user.'
-            )
+            raise ValueError('Error: docker-compose is required in PATH to launch the integration tests, '
+                             'but is not installed or not available for current user.')
 
     # Parameter numprocesses is added to option by pytest-xdist
     workers = ['primary'] if not config.option.numprocesses\
         else ['gw{0}'.format(i) for i in range(config.option.numprocesses)]
 
-    # If a non-default DNS server is configured, start it and feed it to the ACME server
-    dns_server = None
-    acme_dns_server = None
-    if config.option.dns_server == 'bind':
-        dns_server = dns_lib.DNSServer(workers)
-        config.add_cleanup(dns_server.stop)
-        print('DNS xdist config:\n{0}'.format(dns_server.dns_xdist))
-        dns_server.start()
-        acme_dns_server = '{}:{}'.format(
-            dns_server.dns_xdist['address'],
-            dns_server.dns_xdist['port']
-        )
-
     # By calling setup_acme_server we ensure that all necessary acme server instances will be
     # fully started. This runtime is reflected by the acme_xdist returned.
-    acme_server = acme_lib.ACMEServer(config.option.acme_server, workers,
-                                      dns_server=acme_dns_server)
+    acme_server = acme_lib.ACMEServer(config.option.acme_server, workers)
     config.add_cleanup(acme_server.stop)
     print('ACME xdist config:\n{0}'.format(acme_server.acme_xdist))
     acme_server.start()
 
-    config.acme_xdist = acme_server.acme_xdist
-    config.dns_xdist = dns_server.dns_xdist if dns_server else None
+    return acme_server.acme_xdist

certbot-ci/certbot_integration_tests/nginx_tests/context.py

@@ -1,4 +1,3 @@
-"""Module to handle the context of nginx integration tests."""
 import os
 import subprocess
 

certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """General purpose nginx test configuration generator."""
 import getpass
 
@@ -43,8 +42,6 @@ events {{
   worker_connections 1024;
 }}
 
-# “This comment contains valid Unicode”.
-
 http {{
   # Set an array of temp, cache and log file options that will otherwise default to
   # restricted locations accessible only to root.
@@ -54,61 +51,61 @@ http {{
   #scgi_temp_path {nginx_root}/scgi_temp;
   #uwsgi_temp_path {nginx_root}/uwsgi_temp;
   access_log {nginx_root}/error.log;
-
+  
   # This should be turned off in a Virtualbox VM, as it can cause some
   # interesting issues with data corruption in delivered files.
   sendfile off;
-
+  
   tcp_nopush on;
   tcp_nodelay on;
   keepalive_timeout 65;
   types_hash_max_size 2048;
-
+  
   #include /etc/nginx/mime.types;
   index index.html index.htm index.php;
-
+  
   log_format   main '$remote_addr - $remote_user [$time_local] $status '
     '"$request" $body_bytes_sent "$http_referer" '
     '"$http_user_agent" "$http_x_forwarded_for"';
-
+    
   default_type application/octet-stream;
-
+  
   server {{
     # IPv4.
     listen {http_port} {default_server};
     # IPv6.
     listen [::]:{http_port} {default_server};
     server_name nginx.{wtf_prefix}.wtf nginx2.{wtf_prefix}.wtf;
-
+    
     root {nginx_webroot};
-
+    
     location / {{
       # First attempt to serve request as file, then as directory, then fall
       # back to index.html.
       try_files $uri $uri/ /index.html;
     }}
   }}
-
+  
   server {{
     listen {http_port};
     listen [::]:{http_port};
     server_name nginx3.{wtf_prefix}.wtf;
-
+    
     root {nginx_webroot};
-
+    
     location /.well-known/ {{
       return 404;
     }}
-
+    
     return 301 https://$host$request_uri;
   }}
-
+  
   server {{
     listen {other_port};
     listen [::]:{other_port};
     server_name nginx4.{wtf_prefix}.wtf nginx5.{wtf_prefix}.wtf;
   }}
-
+  
   server {{
     listen {http_port};
     listen [::]:{http_port};

certbot-ci/certbot_integration_tests/nginx_tests/test_main.py

@@ -1,15 +1,14 @@
 """Module executing integration tests against certbot with nginx plugin."""
 import os
 import ssl
-from typing import List
 
 import pytest
 
 from certbot_integration_tests.nginx_tests import context as nginx_context
 
 
-@pytest.fixture(name='context')
-def test_context(request):
+@pytest.fixture()
+def context(request):
     # Fixture request is a built-in pytest fixture describing current test request.
     integration_test_context = nginx_context.IntegrationTestsContext(request)
     try:
@@ -28,12 +27,10 @@ def test_context(request):
     # No matching server block; default_server does not exist
     ('nginx5.{0}.wtf', ['--preferred-challenges', 'http'], {'default_server': False}),
     # Multiple domains, mix of matching and not
-    ('nginx6.{0}.wtf,nginx7.{0}.wtf', [
-        '--preferred-challenges', 'http'
-    ], {'default_server': False}),
+    ('nginx6.{0}.wtf,nginx7.{0}.wtf', ['--preferred-challenges', 'http'], {'default_server': False}),
 ], indirect=['context'])
-def test_certificate_deployment(certname_pattern: str, params: List[str],
-                                context: nginx_context.IntegrationTestsContext) -> None:
+def test_certificate_deployment(certname_pattern, params, context):
+    # type: (str, list, nginx_context.IntegrationTestsContext) -> None
     """
     Test various scenarios to deploy a certificate to nginx using certbot.
     """
@@ -44,9 +41,7 @@ def test_certificate_deployment(certname_pattern: str, params: List[str],
 
     lineage = domains.split(',')[0]
     server_cert = ssl.get_server_certificate(('localhost', context.tls_alpn_01_port))
-    with open(os.path.join(
-        context.workspace, 'conf/live/{0}/cert.pem'.format(lineage)), 'r'
-    ) as file:
+    with open(os.path.join(context.workspace, 'conf/live/{0}/cert.pem'.format(lineage)), 'r') as file:
         certbot_cert = file.read()
 
     assert server_cert == certbot_cert

certbot-ci/certbot_integration_tests/rfc2136_tests/context.py

@@ -1,66 +0,0 @@
-"""Module to handle the context of RFC2136 integration tests."""
-
-from contextlib import contextmanager
-import tempfile
-
-from pkg_resources import resource_filename
-from pytest import skip
-
-from certbot_integration_tests.certbot_tests import context as certbot_context
-from certbot_integration_tests.utils import certbot_call
-
-
-class IntegrationTestsContext(certbot_context.IntegrationTestsContext):
-    """Integration test context for certbot-dns-rfc2136"""
-    def __init__(self, request):
-        super(IntegrationTestsContext, self).__init__(request)
-
-        self.request = request
-
-        self._dns_xdist = None
-        if hasattr(request.config, 'workerinput'):  # Worker node
-            self._dns_xdist = request.config.workerinput['dns_xdist']
-        else:  # Primary node
-            self._dns_xdist = request.config.dns_xdist
-
-    def certbot_test_rfc2136(self, args):
-        """
-        Main command to execute certbot using the RFC2136 DNS authenticator.
-        :param list args: list of arguments to pass to Certbot
-        """
-        command = ['--authenticator', 'dns-rfc2136', '--dns-rfc2136-propagation-seconds', '2']
-        command.extend(args)
-        return certbot_call.certbot_test(
-            command, self.directory_url, self.http_01_port, self.tls_alpn_01_port,
-            self.config_dir, self.workspace, force_renew=True)
-
-    @contextmanager
-    def rfc2136_credentials(self, label='default'):
-        """
-        Produces the contents of a certbot-dns-rfc2136 credentials file.
-        :param str label: which RFC2136 credential to use
-        :yields: Path to credentials file
-        :rtype: str
-        """
-        src_file = resource_filename('certbot_integration_tests',
-                                     'assets/bind-config/rfc2136-credentials-{}.ini.tpl'
-                                     .format(label))
-        contents = None
-
-        with open(src_file, 'r') as f:
-            contents = f.read().format(
-                server_address=self._dns_xdist['address'],
-                server_port=self._dns_xdist['port']
-            )
-
-        with tempfile.NamedTemporaryFile('w+', prefix='rfc2136-creds-{}'.format(label),
-                                         suffix='.ini', dir=self.workspace) as fp:
-            fp.write(contents)
-            fp.flush()
-            yield fp.name
-
-    def skip_if_no_bind9_server(self):
-        """Skips the test if there was no RFC2136-capable DNS server configured
-        in the test environment"""
-        if not self._dns_xdist:
-            skip('No RFC2136-capable DNS server is configured')

certbot-ci/certbot_integration_tests/rfc2136_tests/test_main.py

@@ -1,26 +0,0 @@
-"""Module executing integration tests against Certbot with the RFC2136 DNS authenticator."""
-import pytest
-
-from certbot_integration_tests.rfc2136_tests import context as rfc2136_context
-
-
-@pytest.fixture(name="context")
-def pytest_context(request):
-    # pylint: disable=missing-function-docstring
-    # Fixture request is a built-in pytest fixture describing current test request.
-    integration_test_context = rfc2136_context.IntegrationTestsContext(request)
-    try:
-        yield integration_test_context
-    finally:
-        integration_test_context.cleanup()
-
-
-@pytest.mark.parametrize('domain', [('example.com'), ('sub.example.com')])
-def test_get_certificate(domain, context):
-    context.skip_if_no_bind9_server()
-
-    with context.rfc2136_credentials() as creds:
-        context.certbot_test_rfc2136([
-            'certonly', '--dns-rfc2136-credentials', creds,
-            '-d', domain, '-d', '*.{}'.format(domain)
-        ])

certbot-ci/certbot_integration_tests/utils/acme_server.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 """Module to setup an ACME CA server environment able to run multiple tests in parallel"""
+from __future__ import print_function
 
-import argparse
 import errno
 import json
 import os
@@ -11,18 +11,16 @@ import subprocess
 import sys
 import tempfile
 import time
-from typing import List
 
 import requests
 
-# pylint: disable=wildcard-import,unused-wildcard-import
 from certbot_integration_tests.utils import misc
 from certbot_integration_tests.utils import pebble_artifacts
 from certbot_integration_tests.utils import proxy
 from certbot_integration_tests.utils.constants import *
 
 
-class ACMEServer:
+class ACMEServer(object):
     """
     ACMEServer configures and handles the lifecycle of an ACME CA server and an HTTP reverse proxy
     instance, to allow parallel execution of integration tests against the unique http-01 port
@@ -31,34 +29,24 @@ class ACMEServer:
     ACMEServer gives access the acme_xdist parameter, listing the ports and directory url to use
     for each pytest node. It exposes also start and stop methods in order to start the stack, and
     stop it with proper resources cleanup.
-    ACMEServer is also a context manager, and so can be used to ensure ACME server is
-    started/stopped upon context enter/exit.
+    ACMEServer is also a context manager, and so can be used to ensure ACME server is started/stopped
+    upon context enter/exit.
     """
-    def __init__(self, acme_server, nodes, http_proxy=True, stdout=False,
-                 dns_server=None, http_01_port=DEFAULT_HTTP_01_PORT):
+    def __init__(self, acme_server, nodes, http_proxy=True, stdout=False):
         """
         Create an ACMEServer instance.
         :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble)
         :param list nodes: list of node names that will be setup by pytest xdist
         :param bool http_proxy: if False do not start the HTTP proxy
         :param bool stdout: if True stream all subprocesses stdout to standard stdout
-        :param str dns_server: if set, Pebble/Boulder will use it to resolve domains
-        :param int http_01_port: port to use for http-01 validation; currently
-            only supported for pebble without an HTTP proxy
         """
         self._construct_acme_xdist(acme_server, nodes)
 
         self._acme_type = 'pebble' if acme_server == 'pebble' else 'boulder'
         self._proxy = http_proxy
         self._workspace = tempfile.mkdtemp()
-        self._processes: List[subprocess.Popen] = []
+        self._processes = []
         self._stdout = sys.stdout if stdout else open(os.devnull, 'w')
-        self._dns_server = dns_server
-        self._http_01_port = http_01_port
-        if http_01_port != DEFAULT_HTTP_01_PORT:
-            if self._acme_type != 'pebble' or self._proxy:
-                raise ValueError('setting http_01_port is not currently supported '
-                                  'with boulder or the HTTP proxy')
 
     def start(self):
         """Start the test stack"""
@@ -115,34 +103,26 @@ class ACMEServer:
         """Generate and return the acme_xdist dict"""
         acme_xdist = {'acme_server': acme_server, 'challtestsrv_port': CHALLTESTSRV_PORT}
 
-        # Directory and ACME port are set implicitly in the docker-compose.yml
-        # files of Boulder/Pebble.
+        # Directory and ACME port are set implicitly in the docker-compose.yml files of Boulder/Pebble.
         if acme_server == 'pebble':
             acme_xdist['directory_url'] = PEBBLE_DIRECTORY_URL
         else:  # boulder
             acme_xdist['directory_url'] = BOULDER_V2_DIRECTORY_URL \
                 if acme_server == 'boulder-v2' else BOULDER_V1_DIRECTORY_URL
 
-        acme_xdist['http_port'] = {
-            node: port for (node, port) in  # pylint: disable=unnecessary-comprehension
-            zip(nodes, range(5200, 5200 + len(nodes)))
-        }
-        acme_xdist['https_port'] = {
-            node: port for (node, port) in  # pylint: disable=unnecessary-comprehension
-            zip(nodes, range(5100, 5100 + len(nodes)))
-        }
-        acme_xdist['other_port'] = {
-            node: port for (node, port) in  # pylint: disable=unnecessary-comprehension
-            zip(nodes, range(5300, 5300 + len(nodes)))
-        }
+        acme_xdist['http_port'] = {node: port for (node, port)
+                                   in zip(nodes, range(5200, 5200 + len(nodes)))}
+        acme_xdist['https_port'] = {node: port for (node, port)
+                                    in zip(nodes, range(5100, 5100 + len(nodes)))}
+        acme_xdist['other_port'] = {node: port for (node, port)
+                                    in zip(nodes, range(5300, 5300 + len(nodes)))}
 
         self.acme_xdist = acme_xdist
 
     def _prepare_pebble_server(self):
         """Configure and launch the Pebble server"""
         print('=> Starting pebble instance deployment...')
-        pebble_artifacts_rv = pebble_artifacts.fetch(self._workspace, self._http_01_port)
-        pebble_path, challtestsrv_path, pebble_config_path = pebble_artifacts_rv
+        pebble_path, challtestsrv_path, pebble_config_path = pebble_artifacts.fetch(self._workspace)
 
         # Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid
         # nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment.
@@ -152,23 +132,18 @@ class ACMEServer:
         environ['PEBBLE_AUTHZREUSE'] = '100'
         environ['PEBBLE_ALTERNATE_ROOTS'] = str(PEBBLE_ALTERNATE_ROOTS)
 
-        if self._dns_server:
-            dns_server = self._dns_server
-        else:
-            dns_server = '127.0.0.1:8053'
-            self._launch_process(
-                [challtestsrv_path, '-management', ':{0}'.format(CHALLTESTSRV_PORT),
-                 '-defaultIPv6', '""', '-defaultIPv4', '127.0.0.1', '-http01', '""',
-                 '-tlsalpn01', '""', '-https01', '""'])
-
         self._launch_process(
-            [pebble_path, '-config', pebble_config_path, '-dnsserver', dns_server, '-strict'],
+            [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053', '-strict'],
             env=environ)
 
-        # pebble_ocsp_server is imported here and not at the top of module in order to avoid a
-        # useless ImportError, in the case where cryptography dependency is too old to support
-        # ocsp, but Boulder is used instead of Pebble, so pebble_ocsp_server is not used. This is
-        # the typical situation of integration-certbot-oldest tox testenv.
+        self._launch_process(
+            [challtestsrv_path, '-management', ':{0}'.format(CHALLTESTSRV_PORT), '-defaultIPv6', '""',
+             '-defaultIPv4', '127.0.0.1', '-http01', '""', '-tlsalpn01', '""', '-https01', '""'])
+
+        # pebble_ocsp_server is imported here and not at the top of module in order to avoid a useless
+        # ImportError, in the case where cryptography dependency is too old to support ocsp, but
+        # Boulder is used instead of Pebble, so pebble_ocsp_server is not used. This is the typical
+        # situation of integration-certbot-oldest tox testenv.
         from certbot_integration_tests.utils import pebble_ocsp_server
         self._launch_process([sys.executable, pebble_ocsp_server.__file__])
 
@@ -192,15 +167,6 @@ class ACMEServer:
         os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'),
                   join(instance_path, 'test/rate-limit-policies.yml'))
 
-        if self._dns_server:
-            # Change Boulder config to use the provided DNS server
-            for suffix in ["", "-remote-a", "-remote-b"]:
-                with open(join(instance_path, 'test/config/va{}.json'.format(suffix)), 'r') as f:
-                    config = json.loads(f.read())
-                config['va']['dnsResolvers'] = [self._dns_server]
-                with open(join(instance_path, 'test/config/va{}.json'.format(suffix)), 'w') as f:
-                    f.write(json.dumps(config, indent=2, separators=(',', ': ')))
-
         try:
             # Launch the Boulder server
             self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path)
@@ -209,18 +175,14 @@ class ACMEServer:
             print('=> Waiting for boulder instance to respond...')
             misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=300)
 
-            if not self._dns_server:
-                # Configure challtestsrv to answer any A record request with ip of the docker host.
-                response = requests.post('http://localhost:{0}/set-default-ipv4'.format(
-                    CHALLTESTSRV_PORT), json={'ip': '10.77.77.1'}
-                )
-                response.raise_for_status()
+            # Configure challtestsrv to answer any A record request with ip of the docker host.
+            response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT),
+                                     json={'ip': '10.77.77.1'})
+            response.raise_for_status()
         except BaseException:
             # If we failed to set up boulder, print its logs.
             print('=> Boulder setup failed. Boulder logs are:')
-            process = self._launch_process([
-                'docker-compose', 'logs'], cwd=instance_path, force_stderr=True
-            )
+            process = self._launch_process(['docker-compose', 'logs'], cwd=instance_path, force_stderr=True)
             process.wait()
             raise
 
@@ -231,7 +193,7 @@ class ACMEServer:
         print('=> Configuring the HTTP proxy...')
         mapping = {r'.+\.{0}\.wtf'.format(node): 'http://127.0.0.1:{0}'.format(port)
                    for node, port in self.acme_xdist['http_port'].items()}
-        command = [sys.executable, proxy.__file__, str(DEFAULT_HTTP_01_PORT), json.dumps(mapping)]
+        command = [sys.executable, proxy.__file__, str(HTTP_01_PORT), json.dumps(mapping)]
         self._launch_process(command)
         print('=> Finished configuring the HTTP proxy.')
 
@@ -240,34 +202,20 @@ class ACMEServer:
         if not env:
             env = os.environ
         stdout = sys.stderr if force_stderr else self._stdout
-        process = subprocess.Popen(
-            command, stdout=stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env
-        )
+        process = subprocess.Popen(command, stdout=stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env)
         self._processes.append(process)
         return process
 
 
 def main():
-    # pylint: disable=missing-function-docstring
-    parser = argparse.ArgumentParser(
-        description='CLI tool to start a local instance of Pebble or Boulder CA server.')
-    parser.add_argument('--server-type', '-s',
-                        choices=['pebble', 'boulder-v1', 'boulder-v2'], default='pebble',
-                        help='type of CA server to start: can be Pebble or Boulder '
-                             '(in ACMEv1 or ACMEv2 mode), Pebble is used if not set.')
-    parser.add_argument('--dns-server', '-d',
-                        help='specify the DNS server as `IP:PORT` to use by '
-                             'Pebble; if not specified, a local mock DNS server will be used to '
-                             'resolve domains to localhost.')
-    parser.add_argument('--http-01-port', type=int, default=DEFAULT_HTTP_01_PORT,
-                        help='specify the port to use for http-01 validation; '
-                             'this is currently only supported for Pebble.')
-    args = parser.parse_args()
+    args = sys.argv[1:]
+    server_type = args[0] if args else 'pebble'
+    possible_values = ('pebble', 'boulder-v1', 'boulder-v2')
+    if server_type not in possible_values:
+        raise ValueError('Invalid server value {0}, should be one of {1}'
+                         .format(server_type, possible_values))
 
-    acme_server = ACMEServer(
-        args.server_type, [], http_proxy=False, stdout=True,
-        dns_server=args.dns_server, http_01_port=args.http_01_port,
-    )
+    acme_server = ACMEServer(server_type, [], http_proxy=False, stdout=True)
 
     try:
         with acme_server as acme_xdist:

certbot-ci/certbot_integration_tests/utils/certbot_call.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 """Module to call certbot in test mode"""
+from __future__ import absolute_import
 
 from distutils.version import LooseVersion
 import os
@@ -7,7 +8,6 @@ import subprocess
 import sys
 
 import certbot_integration_tests
-# pylint: disable=wildcard-import,unused-wildcard-import
 from certbot_integration_tests.utils.constants import *
 
 
@@ -35,8 +35,6 @@ def certbot_test(certbot_args, directory_url, http_01_port, tls_alpn_01_port,
 
 
 def _prepare_environ(workspace):
-    # pylint: disable=missing-function-docstring
-
     new_environ = os.environ.copy()
     new_environ['TMPDIR'] = workspace
 
@@ -60,13 +58,8 @@ def _prepare_environ(workspace):
         # certbot_integration_tests.__file__ is:
         # '/path/to/certbot/certbot-ci/certbot_integration_tests/__init__.pyc'
         # ... and we want '/path/to/certbot'
-        certbot_root = os.path.dirname(os.path.dirname(
-            os.path.dirname(certbot_integration_tests.__file__))
-        )
-        python_paths = [
-            path for path in new_environ['PYTHONPATH'].split(':')
-            if path != certbot_root
-        ]
+        certbot_root = os.path.dirname(os.path.dirname(os.path.dirname(certbot_integration_tests.__file__)))
+        python_paths = [path for path in new_environ['PYTHONPATH'].split(':') if path != certbot_root]
         new_environ['PYTHONPATH'] = ':'.join(python_paths)
 
     return new_environ
@@ -77,8 +70,7 @@ def _compute_additional_args(workspace, environ, force_renew):
     output = subprocess.check_output(['certbot', '--version'],
                                      universal_newlines=True, stderr=subprocess.STDOUT,
                                      cwd=workspace, env=environ)
-    # Typical response is: output = 'certbot 0.31.0.dev0'
-    version_str = output.split(' ')[1].strip()
+    version_str = output.split(' ')[1].strip()  # Typical response is: output = 'certbot 0.31.0.dev0'
     if LooseVersion(version_str) >= LooseVersion('0.30.0'):
         additional_args.append('--no-random-sleep-on-renew')
 
@@ -121,12 +113,11 @@ def _prepare_args_env(certbot_args, directory_url, http_01_port, tls_alpn_01_por
 
 
 def main():
-    # pylint: disable=missing-function-docstring
     args = sys.argv[1:]
 
     # Default config is pebble
     directory_url = os.environ.get('SERVER', PEBBLE_DIRECTORY_URL)
-    http_01_port = int(os.environ.get('HTTP_01_PORT', DEFAULT_HTTP_01_PORT))
+    http_01_port = int(os.environ.get('HTTP_01_PORT', HTTP_01_PORT))
     tls_alpn_01_port = int(os.environ.get('TLS_ALPN_01_PORT', TLS_ALPN_01_PORT))
 
     # Execution of certbot in a self-contained workspace

certbot-ci/certbot_integration_tests/utils/constants.py

@@ -1,5 +1,5 @@
 """Some useful constants to use throughout certbot-ci integration tests"""
-DEFAULT_HTTP_01_PORT = 5002
+HTTP_01_PORT = 5002
 TLS_ALPN_01_PORT = 5001
 CHALLTESTSRV_PORT = 8055
 BOULDER_V1_DIRECTORY_URL = 'http://localhost:4000/directory'
@@ -7,4 +7,4 @@ BOULDER_V2_DIRECTORY_URL = 'http://localhost:4001/directory'
 PEBBLE_DIRECTORY_URL = 'https://localhost:14000/dir'
 PEBBLE_MANAGEMENT_URL = 'https://localhost:15000'
 MOCK_OCSP_SERVER_PORT = 4002
-PEBBLE_ALTERNATE_ROOTS = 2
+PEBBLE_ALTERNATE_ROOTS = 2

certbot-ci/certbot_integration_tests/utils/dns_server.py

@@ -1,152 +0,0 @@
-#!/usr/bin/env python
-"""Module to setup an RFC2136-capable DNS server"""
-import os
-import os.path
-import shutil
-import socket
-import subprocess
-import sys
-import tempfile
-import time
-
-from pkg_resources import resource_filename
-
-BIND_DOCKER_IMAGE = "internetsystemsconsortium/bind9:9.16"
-BIND_BIND_ADDRESS = ("127.0.0.1", 45953)
-
-# A TCP DNS message which is a query for '. CH A' transaction ID 0xcb37. This is used
-# by _wait_until_ready to check that BIND is responding without depending on dnspython.
-BIND_TEST_QUERY = bytearray.fromhex("0011cb37000000010000000000000000010003")
-
-
-class DNSServer:
-    """
-    DNSServer configures and handles the lifetime of an RFC2136-capable server.
-    DNServer provides access to the dns_xdist parameter, listing the address and port
-    to use for each pytest node.
-
-    At this time, DNSServer should only be used with a single node, but may be expanded in
-    future to support parallelization (https://github.com/certbot/certbot/issues/8455).
-    """
-
-    def __init__(self, unused_nodes, show_output=False):
-        """
-        Create an DNSServer instance.
-        :param list nodes: list of node names that will be setup by pytest xdist
-        :param bool show_output: if True, print the output of the DNS server
-        """
-
-        self.bind_root = tempfile.mkdtemp()
-
-        self.process: subprocess.Popen = None
-
-        self.dns_xdist = {"address": BIND_BIND_ADDRESS[0], "port": BIND_BIND_ADDRESS[1]}
-
-        # Unfortunately the BIND9 image forces everything to stderr with -g and we can't
-        # modify the verbosity.
-        self._output = sys.stderr if show_output else open(os.devnull, "w")
-
-    def start(self):
-        """Start the DNS server"""
-        try:
-            self._configure_bind()
-            self._start_bind()
-        except:
-            self.stop()
-            raise
-
-    def stop(self):
-        """Stop the DNS server, and clean its resources"""
-        if self.process:
-            try:
-                self.process.terminate()
-                self.process.wait()
-            except BaseException as e:
-                print("BIND9 did not stop cleanly: {}".format(e), file=sys.stderr)
-
-        shutil.rmtree(self.bind_root, ignore_errors=True)
-
-        if self._output != sys.stderr:
-            self._output.close()
-
-    def _configure_bind(self):
-        """Configure the BIND9 server based on the prebaked configuration"""
-        bind_conf_src = resource_filename(
-            "certbot_integration_tests", "assets/bind-config"
-        )
-        for directory in ("conf", "zones"):
-            shutil.copytree(
-                os.path.join(bind_conf_src, directory), os.path.join(self.bind_root, directory)
-            )
-
-    def _start_bind(self):
-        """Launch the BIND9 server as a Docker container"""
-        addr_str = "{}:{}".format(BIND_BIND_ADDRESS[0], BIND_BIND_ADDRESS[1])
-        self.process = subprocess.Popen(
-            [
-                "docker",
-                "run",
-                "--rm",
-                "-p",
-                "{}:53/udp".format(addr_str),
-                "-p",
-                "{}:53/tcp".format(addr_str),
-                "-v",
-                "{}/conf:/etc/bind".format(self.bind_root),
-                "-v",
-                "{}/zones:/var/lib/bind".format(self.bind_root),
-                BIND_DOCKER_IMAGE,
-            ],
-            stdout=self._output,
-            stderr=self._output,
-        )
-
-        if self.process.poll():
-            raise ValueError("BIND9 server stopped unexpectedly")
-
-        try:
-            self._wait_until_ready()
-        except:
-            # The container might be running even if we think it isn't
-            self.stop()
-            raise
-
-    def _wait_until_ready(self, attempts: int = 30) -> None:
-        """
-        Polls the DNS server over TCP until it gets a response, or until
-        it runs out of attempts and raises a ValueError.
-        The DNS response message must match the txn_id of the DNS query message,
-        but otherwise the contents are ignored.
-        :param int attempts: The number of attempts to make.
-        """
-        for _ in range(attempts):
-            if self.process.poll():
-                raise ValueError("BIND9 server stopped unexpectedly")
-
-            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            sock.settimeout(5.0)
-            try:
-                sock.connect(BIND_BIND_ADDRESS)
-                sock.sendall(BIND_TEST_QUERY)
-                buf = sock.recv(1024)
-                # We should receive a DNS message with the same tx_id
-                if buf and len(buf) > 4 and buf[2:4] == BIND_TEST_QUERY[2:4]:
-                    return
-                # If we got a response but it wasn't the one we wanted, wait a little
-                time.sleep(1)
-            except:  # pylint: disable=bare-except
-                # If there was a network error, wait a little
-                time.sleep(1)
-            finally:
-                sock.close()
-
-        raise ValueError(
-            "Gave up waiting for DNS server {} to respond".format(BIND_BIND_ADDRESS)
-        )
-
-    def __enter__(self):
-        self.start()
-        return self.dns_xdist
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.stop()

certbot-ci/certbot_integration_tests/utils/misc.py

@@ -4,12 +4,10 @@ or outside during setup/teardown of the integration tests environment.
 """
 import contextlib
 import errno
-import http.server as SimpleHTTPServer
 import multiprocessing
 import os
 import re
 import shutil
-import socketserver
 import stat
 import sys
 import tempfile
@@ -25,9 +23,11 @@ from cryptography.x509 import load_pem_x509_certificate
 from OpenSSL import crypto
 import pkg_resources
 import requests
+from six.moves import SimpleHTTPServer
+from six.moves import socketserver
 
-from certbot_integration_tests.utils.constants import PEBBLE_ALTERNATE_ROOTS
-from certbot_integration_tests.utils.constants import PEBBLE_MANAGEMENT_URL
+from certbot_integration_tests.utils.constants import \
+     PEBBLE_ALTERNATE_ROOTS, PEBBLE_MANAGEMENT_URL
 
 RSA_KEY_TYPE = 'rsa'
 ECDSA_KEY_TYPE = 'ecdsa'
@@ -39,7 +39,6 @@ def _suppress_x509_verification_warnings():
         urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
     except ImportError:
         # Handle old versions of request with vendorized urllib3
-        # pylint: disable=no-member
         from requests.packages.urllib3.exceptions import InsecureRequestWarning
         requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 
@@ -257,8 +256,7 @@ def generate_csr(domains, key_path, csr_path, key_type=RSA_KEY_TYPE):
 
 def read_certificate(cert_path):
     """
-    Load the certificate from the provided path, and return a human readable version
-    of it (TEXT mode).
+    Load the certificate from the provided path, and return a human readable version of it (TEXT mode).
     :param str cert_path: the path to the certificate
     :returns: the TEXT version of the certificate, as it would be displayed by openssl binary
     """
@@ -282,11 +280,7 @@ def load_sample_data_path(workspace):
 
     if os.name == 'nt':
         # Fix the symlinks on Windows if GIT is not configured to create them upon checkout
-        for lineage in [
-            'a.encryption-example.com',
-            'b.encryption-example.com',
-            'c.encryption-example.com',
-        ]:
+        for lineage in ['a.encryption-example.com', 'b.encryption-example.com']:
             current_live = os.path.join(copied, 'live', lineage)
             for name in os.listdir(current_live):
                 if name != 'README':
@@ -311,7 +305,7 @@ def echo(keyword, path=None):
     if not re.match(r'^\w+$', keyword):
         raise ValueError('Error, keyword `{0}` is not a single keyword.'
                          .format(keyword))
-    return '{0} -c "print(\'{1}\')"{2}'.format(
+    return '{0} -c "from __future__ import print_function; print(\'{1}\')"{2}'.format(
         os.path.basename(sys.executable), keyword, ' >> "{0}"'.format(path) if path else '')
 
 

certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py

@@ -1,5 +1,3 @@
-# pylint: disable=missing-module-docstring
-
 import json
 import os
 import stat
@@ -7,20 +5,18 @@ import stat
 import pkg_resources
 import requests
 
-from certbot_integration_tests.utils.constants import DEFAULT_HTTP_01_PORT
 from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT
 
 PEBBLE_VERSION = 'v2.3.0'
 ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets')
 
 
-def fetch(workspace, http_01_port=DEFAULT_HTTP_01_PORT):
-    # pylint: disable=missing-function-docstring
+def fetch(workspace):
     suffix = 'linux-amd64' if os.name != 'nt' else 'windows-amd64.exe'
 
     pebble_path = _fetch_asset('pebble', suffix)
     challtestsrv_path = _fetch_asset('pebble-challtestsrv', suffix)
-    pebble_config_path = _build_pebble_config(workspace, http_01_port)
+    pebble_config_path = _build_pebble_config(workspace)
 
     return pebble_path, challtestsrv_path, pebble_config_path
 
@@ -39,7 +35,7 @@ def _fetch_asset(asset, suffix):
     return asset_path
 
 
-def _build_pebble_config(workspace, http_01_port):
+def _build_pebble_config(workspace):
     config_path = os.path.join(workspace, 'pebble-config.json')
     with open(config_path, 'w') as file_h:
         file_h.write(json.dumps({
@@ -48,7 +44,7 @@ def _build_pebble_config(workspace, http_01_port):
                 'managementListenAddress': '0.0.0.0:15000',
                 'certificate': os.path.join(ASSETS_PATH, 'cert.pem'),
                 'privateKey': os.path.join(ASSETS_PATH, 'key.pem'),
-                'httpPort': http_01_port,
+                'httpPort': 5002,
                 'tlsPort': 5001,
                 'ocspResponderURL': 'http://127.0.0.1:{0}'.format(MOCK_OCSP_SERVER_PORT),
             },

certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py

@@ -4,7 +4,6 @@ This runnable module interfaces itself with the Pebble management interface in o
 to serve a mock OCSP responder during integration tests against Pebble.
 """
 import datetime
-import http.server as BaseHTTPServer
 import re
 
 from cryptography import x509
@@ -14,6 +13,7 @@ from cryptography.hazmat.primitives import serialization
 from cryptography.x509 import ocsp
 from dateutil import parser
 import requests
+from six.moves import BaseHTTPServer
 
 from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT
 from certbot_integration_tests.utils.constants import PEBBLE_MANAGEMENT_URL
@@ -21,7 +21,6 @@ from certbot_integration_tests.utils.misc import GracefulTCPServer
 
 
 class _ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
-    # pylint: disable=missing-function-docstring
     def do_POST(self):
         request = requests.get(PEBBLE_MANAGEMENT_URL + '/intermediate-keys/0', verify=False)
         issuer_key = serialization.load_pem_private_key(request.content, None, default_backend())
@@ -29,32 +28,27 @@ class _ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
         request = requests.get(PEBBLE_MANAGEMENT_URL + '/intermediates/0', verify=False)
         issuer_cert = x509.load_pem_x509_certificate(request.content, default_backend())
 
-        content_len = int(self.headers.get('Content-Length'))
+        try:
+            content_len = int(self.headers.getheader('content-length', 0))
+        except AttributeError:
+            content_len = int(self.headers.get('Content-Length'))
 
         ocsp_request = ocsp.load_der_ocsp_request(self.rfile.read(content_len))
         response = requests.get('{0}/cert-status-by-serial/{1}'.format(
-            PEBBLE_MANAGEMENT_URL, str(hex(ocsp_request.serial_number)).replace('0x', '')),
-            verify=False
-        )
+            PEBBLE_MANAGEMENT_URL, str(hex(ocsp_request.serial_number)).replace('0x', '')), verify=False)
 
         if not response.ok:
-            ocsp_response = ocsp.OCSPResponseBuilder.build_unsuccessful(
-                ocsp.OCSPResponseStatus.UNAUTHORIZED
-            )
+            ocsp_response = ocsp.OCSPResponseBuilder.build_unsuccessful(ocsp.OCSPResponseStatus.UNAUTHORIZED)
         else:
             data = response.json()
 
             now = datetime.datetime.utcnow()
             cert = x509.load_pem_x509_certificate(data['Certificate'].encode(), default_backend())
             if data['Status'] != 'Revoked':
-                ocsp_status = ocsp.OCSPCertStatus.GOOD
-                revocation_time = None
-                revocation_reason = None
+                ocsp_status, revocation_time, revocation_reason = ocsp.OCSPCertStatus.GOOD, None, None
             else:
-                ocsp_status = ocsp.OCSPCertStatus.REVOKED
-                revocation_reason = x509.ReasonFlags.unspecified
-                # "... +0000 UTC" => "+0000"
-                revoked_at = re.sub(r'( \+\d{4}).*$', r'\1', data['RevokedAt'])
+                ocsp_status, revocation_reason = ocsp.OCSPCertStatus.REVOKED, x509.ReasonFlags.unspecified
+                revoked_at = re.sub(r'( \+\d{4}).*$', r'\1', data['RevokedAt'])  # "... +0000 UTC" => "+0000"
                 revocation_time = parser.parse(revoked_at)
 
             ocsp_response = ocsp.OCSPResponseBuilder().add_response(

certbot-ci/certbot_integration_tests/utils/proxy.py

@@ -1,20 +1,16 @@
 #!/usr/bin/env python
-# pylint: disable=missing-module-docstring
-
-import http.server as BaseHTTPServer
 import json
 import re
 import sys
 
 import requests
+from six.moves import BaseHTTPServer
 
 from certbot_integration_tests.utils.misc import GracefulTCPServer
 
 
 def _create_proxy(mapping):
-    # pylint: disable=missing-function-docstring
     class ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
-        # pylint: disable=missing-class-docstring
         def do_GET(self):
             headers = {key.lower(): value for key, value in self.headers.items()}
             backend = [backend for pattern, backend in mapping.items()

certbot-ci/setup.py

@@ -7,13 +7,6 @@ from setuptools import setup
 
 version = '0.32.0.dev0'
 
-# setuptools 36.2+ is needed for support for environment markers
-min_setuptools_version='36.2'
-# This conditional isn't necessary, but it provides better error messages to
-# people who try to install this package with older versions of setuptools.
-if LooseVersion(setuptools_version) < LooseVersion(min_setuptools_version):
-    raise RuntimeError(f'setuptools {min_setuptools_version}+ is required')
-
 install_requires = [
     'coverage',
     'cryptography',
@@ -21,17 +14,24 @@ install_requires = [
     'pyopenssl',
     'pytest',
     'pytest-cov',
-    # This version is needed for "worker" attributes we currently use like
-    # "workerinput".  See https://github.com/pytest-dev/pytest-xdist/pull/268.
-    'pytest-xdist>=1.22.1',
+    'pytest-xdist',
     'python-dateutil',
-    # This dependency needs to be added using environment markers to avoid its
-    # installation on Linux.
-    'pywin32>=300 ; sys_platform == "win32"',
     'pyyaml',
     'requests',
+    'six',
 ]
 
+# Add pywin32 on Windows platforms to handle low-level system calls.
+# This dependency needs to be added using environment markers to avoid its installation on Linux.
+# However environment markers are supported only with setuptools >= 36.2.
+# So this dependency is not added for old Linux distributions with old setuptools,
+# in order to allow these systems to build certbot from sources.
+if LooseVersion(setuptools_version) >= LooseVersion('36.2'):
+    install_requires.append("pywin32>=224 ; sys_platform == 'win32'")
+elif 'bdist_wheel' in sys.argv[1:]:
+    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
+                       'of setuptools. Version 36.2+ of setuptools is required.')
+
 setup(
     name='certbot-ci',
     version=version,
@@ -40,17 +40,18 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
-    python_requires='>=3.6',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Intended Audience :: Developers',
         'License :: OSI Approved :: Apache Software License',
         'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
     ],

certbot-ci/snap_integration_tests/dns_tests/test_main.py

@@ -1,10 +1,9 @@
 #!/usr/bin/env python3
+import pytest
+import subprocess
 import glob
 import os
 import re
-import subprocess
-
-import pytest
 
 
 @pytest.fixture(autouse=True, scope="module")

certbot-ci/windows_installer_integration_tests/conftest.py

@@ -6,9 +6,11 @@ for a directory a specific configuration using built-in pytest hooks.
 
 See https://docs.pytest.org/en/latest/reference.html#hook-reference
 """
-
+from __future__ import print_function
 import os
 
+import pytest
+
 ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 
 

certbot-ci/windows_installer_integration_tests/test_main.py

@@ -1,8 +1,8 @@
 import os
-import re
-import subprocess
 import time
 import unittest
+import subprocess
+import re
 
 
 @unittest.skipIf(os.name != 'nt', reason='Windows installer tests must be run on Windows.')