test quietly

Co-Authored-By: Brad Warren <bmw@users.noreply.github.com>

.azure-pipelines/advanced-test.yml

@@ -1,13 +0,0 @@
-# Advanced pipeline for running our full test suite on demand.
-trigger:
-  # When changing these triggers, please ensure the documentation under
-  # "Running tests in CI" is still correct.
-  - azure-test-*
-  - test-*
-pr: none
-
-jobs:
-  # Any addition here should be reflected in the advanced and release pipelines.
-  # It is advised to declare all jobs here as templates to improve maintainability.
-  - template: templates/tests-suite.yml
-  - template: templates/installer-tests.yml

.azure-pipelines/advanced.yml

@@ -1,7 +1,9 @@
-# Advanced pipeline for running our full test suite on protected branches.
+# Advanced pipeline for isolated checks and release purpose
 trigger:
+  - test-*
   - '*.x'
-pr: none
+pr:
+  - test-*
 # This pipeline is also nightly run on master
 schedules:
   - cron: "0 4 * * *"
@@ -12,7 +14,7 @@ schedules:
     always: true
 
 jobs:
-  # Any addition here should be reflected in the advanced-test and release pipelines.
+  # Any addition here should be reflected in the release pipeline.
   # It is advised to declare all jobs here as templates to improve maintainability.
   - template: templates/tests-suite.yml
   - template: templates/installer-tests.yml

.azure-pipelines/main.yml

@@ -1,6 +1,10 @@
 trigger:
+  # apache-parser-v2 is a temporary branch for doing work related to
+  # rewriting the parser in the Apache plugin.
+  - apache-parser-v2
   - master
 pr:
+  - apache-parser-v2
   - master
   - '*.x'
 

.azure-pipelines/release.yml

@@ -6,7 +6,7 @@ trigger:
 pr: none
 
 jobs:
-  # Any addition here should be reflected in the advanced and advanced-test pipelines.
+  # Any addition here should be reflected in the advanced pipeline.
   # It is advised to declare all jobs here as templates to improve maintainability.
   - template: templates/tests-suite.yml
   - template: templates/installer-tests.yml

.azure-pipelines/templates/installer-tests.yml

@@ -28,34 +28,27 @@ jobs:
           imageName: windows-2019
         win2016:
           imageName: vs2017-win2016
+        win2012r2:
+          imageName: vs2015-win2012r2
     pool:
       vmImage: $(imageName)
     steps:
-      - powershell: |
-          $currentVersion = $PSVersionTable.PSVersion
-          if ($currentVersion.Major -ne 5) {
-              throw "Powershell version is not 5.x"
-          }
-        condition: eq(variables['imageName'], 'vs2017-win2016')
-        displayName: Check Powershell 5.x is used in vs2017-win2016
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: 3.8
-          addToPath: true
       - task: DownloadPipelineArtifact@2
         inputs:
           artifact: windows-installer
           path: $(Build.SourcesDirectory)/bin
         displayName: Retrieve Windows installer
+      - script: $(Build.SourcesDirectory)\bin\certbot-beta-installer-win32.exe /S
+        displayName: Install Certbot
+      - powershell: Invoke-WebRequest https://www.python.org/ftp/python/3.8.1/python-3.8.1-amd64-webinstall.exe -OutFile C:\py3-setup.exe
+        displayName: Get Python
+      - script: C:\py3-setup.exe /quiet PrependPath=1 InstallAllUsers=1 Include_launcher=1 InstallLauncherAllUsers=1 Include_test=0 Include_doc=0 Include_dev=1 Include_debug=0 Include_tcltk=0 TargetDir=C:\py3
+        displayName: Install Python
       - script: |
           py -3 -m venv venv
           venv\Scripts\python tools\pip_install.py -e certbot-ci
         displayName: Prepare Certbot-CI
-      - script: |
-          set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
-          venv\Scripts\python -m pytest certbot-ci\windows_installer_integration_tests --allow-persistent-changes --installer-path $(Build.SourcesDirectory)\bin\certbot-beta-installer-win32.exe
-        displayName: Run windows installer integration tests
       - script: |
           set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
           venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
-        displayName: Run certbot integration tests
+        displayName: Run integration tests

.azure-pipelines/templates/tests-suite.yml

@@ -1,34 +1,22 @@
 jobs:
   - job: test
+    pool:
+      vmImage: vs2017-win2016
     strategy:
       matrix:
-        macos-py27:
-          IMAGE_NAME: macOS-10.14
-          PYTHON_VERSION: 2.7
-          TOXENV: py27
-        macos-py38:
-          IMAGE_NAME: macOS-10.14
-          PYTHON_VERSION: 3.8
-          TOXENV: py38
-        windows-py35:
-          IMAGE_NAME: vs2017-win2016
+        py35:
           PYTHON_VERSION: 3.5
           TOXENV: py35
-        windows-py37-cover:
-          IMAGE_NAME: vs2017-win2016
+        py37-cover:
           PYTHON_VERSION: 3.7
           TOXENV: py37-cover
-        windows-integration-certbot:
-          IMAGE_NAME: vs2017-win2016
+        integration-certbot:
           PYTHON_VERSION: 3.7
           TOXENV: integration-certbot
           PYTEST_ADDOPTS: --numprocesses 4
-    pool:
-      vmImage: $(IMAGE_NAME)
+    variables:
+    - group: certbot-common
     steps:
-    - bash: brew install augeas
-      condition: startswith(variables['IMAGE_NAME'], 'macOS')
-      displayName: Install Augeas
     - task: UsePythonVersion@0
       inputs:
         versionSpec: $(PYTHON_VERSION)
@@ -37,3 +25,14 @@ jobs:
       displayName: Install dependencies
     - script: python -m tox
       displayName: Run tox
+    # We do not require codecov report upload to succeed. So to avoid to break the pipeline if
+    # something goes wrong, each command is suffixed with a command that hides any non zero exit
+    # codes and echoes an informative message instead.
+    - bash: |
+        curl -s https://codecov.io/bash -o codecov-bash || echo "Failed to download codecov-bash"
+        chmod +x codecov-bash || echo "Failed to apply execute permissions on codecov-bash"
+        ./codecov-bash -F windows  || echo "Codecov did not collect coverage reports"
+      condition: in(variables['TOXENV'], 'py37-cover', 'integration-certbot')
+      env:
+        CODECOV_TOKEN: $(codecov_token)
+      displayName: Publish coverage

.codecov.yml

@@ -0,0 +1,18 @@
+coverage:
+  status:
+    project:
+      default: off
+      linux:
+        flags: linux
+        # Fixed target instead of auto set by #7173, can
+        # be removed when flags in Codecov are added back.
+        target: 97.4
+        threshold: 0.1
+        base: auto
+      windows:
+        flags: windows
+        # Fixed target instead of auto set by #7173, can
+        # be removed when flags in Codecov are added back.
+        target: 97.4
+        threshold: 0.1
+        base: auto

.gitignore

@@ -35,7 +35,6 @@ tags
 tests/letstest/letest-*/
 tests/letstest/*.pem
 tests/letstest/venv/
-tests/letstest/venv3/
 
 .venv
 
@@ -51,10 +50,3 @@ tests/letstest/venv3/
 .certbot_test_workspace
 **/assets/pebble*
 **/assets/challtestsrv*
-
-# snap files
-.snapcraft
-parts
-prime
-stage
-*.snap

.travis.yml

@@ -6,46 +6,240 @@ cache:
         - $HOME/.cache/pip
 
 before_script:
+  - 'if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ulimit -n 1024 ; fi'
   # On Travis, the fastest parallelization for integration tests has proved to be 4.
   - 'if [[ "$TOXENV" == *"integration"* ]]; then export PYTEST_ADDOPTS="--numprocesses 4"; fi'
   # Use Travis retry feature for farm tests since they are flaky
   - 'if [[ "$TOXENV" == "travis-test-farm"* ]]; then export TRAVIS_RETRY=travis_retry; fi'
   - export TOX_TESTENV_PASSENV=TRAVIS
-  - 'if [[ "$SNAP" == true ]]; then snap/local/build_and_install.sh; fi'
 
 # Only build pushes to the master branch, PRs, and branches beginning with
-# `test-`, `travis-test-`, or of the form `digit(s).digit(s).x` or
-# `vdigit(s).digit(s).digit(s)`. As documented at
-# https://docs.travis-ci.com/user/customizing-the-build/#safelisting-or-blocklisting-branches,
-# this includes tags so pushing tags of the form `vdigit(s).digit(s).digit(s)`
-# will also trigger tests. This reduces the number of simultaneous Travis runs,
-# which speeds turnaround time on review since there is a cap of on the number
-# of simultaneous runs.
+# `test-` or of the form `digit(s).digit(s).x`. This reduces the number of
+# simultaneous Travis runs, which speeds turnaround time on review since there
+# is a cap of on the number of simultaneous runs.
 branches:
-  # When changing these branches, please ensure the documentation under
-  # "Running tests in CI" is still correct.
   only:
+    # apache-parser-v2 is a temporary branch for doing work related to
+    # rewriting the parser in the Apache plugin.
+    - apache-parser-v2
     - master
-    - /^\d+\.\d+\.x$/  # this matches our point release branches
-    - /^v\d+\.\d+\.\d+$/  # this matches our release tags
-    - /^(travis-)?test-.*$/
+    - /^\d+\.\d+\.x$/
+    - /^test-.*$/
 
 # Jobs for the main test suite are always executed (including on PRs) except for pushes on master.
 not-on-master: &not-on-master
   if: NOT (type = push AND branch = master)
 
 # Jobs for the extended test suite are executed for cron jobs and pushes to
-# non-development branches.
+# non-development branches. See the explanation for apache-parser-v2 above.
 extended-test-suite: &extended-test-suite
-  if: type = cron OR (type = push AND branch != master)
+  if: type = cron OR (type = push AND branch NOT IN (apache-parser-v2, master))
 
 matrix:
   include:
+    # Main test suite
+    - python: "2.7"
+      env: ACME_SERVER=pebble TOXENV=integration
+      <<: *not-on-master
+
+    # This job is always executed, including on master
+    - python: "2.7"
+      env: TOXENV=py27-cover FYI="py27 tests + code coverage"
+
     - python: "3.7"
+      env: TOXENV=lint
+      <<: *not-on-master
+    - python: "3.5"
+      env: TOXENV=mypy
+      <<: *not-on-master
+    - python: "2.7"
+      # Ubuntu Trusty or older must be used because the oldest version of
+      # cryptography we support cannot be compiled against the version of
+      # OpenSSL in Xenial or newer.
+      dist: trusty
+      env: TOXENV='py27-{acme,apache,certbot,dns,nginx}-oldest'
+      <<: *not-on-master
+    - python: "3.5"
+      env: TOXENV=py35
+      <<: *not-on-master
+    - python: "3.8"
+      env: TOXENV=py38
+      <<: *not-on-master
+    - sudo: required
+      env: TOXENV=apache_compat
+      services: docker
+      before_install:
+      addons:
+      <<: *not-on-master
+    - sudo: required
+      env: TOXENV=le_auto_xenial
+      services: docker
+      <<: *not-on-master
+    - python: "2.7"
+      env: TOXENV=apacheconftest-with-pebble
+      <<: *not-on-master
+    - python: "2.7"
+      env: TOXENV=nginxroundtrip
+      <<: *not-on-master
+
+    # Extended test suite on cron jobs and pushes to tested branches other than master
+    - sudo: required
+      env: TOXENV=nginx_compat
+      services: docker
+      before_install:
+      addons:
+      <<: *extended-test-suite
+    - python: "2.7"
       env:
         - TOXENV=travis-test-farm-apache2
         - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
       <<: *extended-test-suite
+    - python: "2.7"
+      env:
+        - TOXENV=travis-test-farm-leauto-upgrades
+        - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
+      git:
+        depth: false  # This is needed to have the history to checkout old versions of certbot-auto.
+      <<: *extended-test-suite
+    - python: "2.7"
+      env:
+        - TOXENV=travis-test-farm-certonly-standalone
+        - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
+      <<: *extended-test-suite
+    - python: "2.7"
+      env:
+        - TOXENV=travis-test-farm-sdists
+        - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
+      <<: *extended-test-suite
+    - python: "3.7"
+      env: TOXENV=py37 CERTBOT_NO_PIN=1
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration-certbot-oldest
+      # Ubuntu Trusty or older must be used because the oldest version of
+      # cryptography we support cannot be compiled against the version of
+      # OpenSSL in Xenial or newer.
+      dist: trusty
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration-certbot-oldest
+      # Ubuntu Trusty or older must be used because the oldest version of
+      # cryptography we support cannot be compiled against the version of
+      # OpenSSL in Xenial or newer.
+      dist: trusty
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration-nginx-oldest
+      # Ubuntu Trusty or older must be used because the oldest version of
+      # cryptography we support cannot be compiled against the version of
+      # OpenSSL in Xenial or newer.
+      dist: trusty
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration-nginx-oldest
+      # Ubuntu Trusty or older must be used because the oldest version of
+      # cryptography we support cannot be compiled against the version of
+      # OpenSSL in Xenial or newer.
+      dist: trusty
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "3.6"
+      env: TOXENV=py36
+      <<: *extended-test-suite
+    - python: "3.7"
+      env: TOXENV=py37
+      <<: *extended-test-suite
+    - python: "3.5"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "3.5"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "3.6"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "3.6"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "3.7"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "3.7"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "3.8"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
+      <<: *extended-test-suite
+    - python: "3.8"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
+      <<: *extended-test-suite
+    - sudo: required
+      env: TOXENV=le_auto_jessie
+      services: docker
+      <<: *extended-test-suite
+    - sudo: required
+      env: TOXENV=le_auto_centos6
+      services: docker
+      <<: *extended-test-suite
+    - sudo: required
+      env: TOXENV=le_auto_oraclelinux6
+      services: docker
+      <<: *extended-test-suite
+    - sudo: required
+      env: TOXENV=docker_dev
+      services: docker
+      addons:
+        apt:
+          packages:  # don't install nginx and apache
+            - libaugeas0
+      <<: *extended-test-suite
+    - language: generic
+      env: TOXENV=py27
+      os: osx
+      addons:
+        homebrew:
+          packages:
+            - augeas
+            - python2
+      <<: *extended-test-suite
+    - language: generic
+      env: TOXENV=py3
+      os: osx
+      addons:
+        homebrew:
+          packages:
+            - augeas
+            - python3
+      <<: *extended-test-suite
 
 # container-based infrastructure
 sudo: false
@@ -67,26 +261,15 @@ addons:
 # except in tests where the environment variable CERTBOT_NO_PIN is set.
 # virtualenv is listed here explicitly to make sure it is upgraded when
 # CERTBOT_NO_PIN is set to work around failures we've seen when using an older
-# version of virtualenv. The option "-I" is set so when CERTBOT_NO_PIN is also
-# set, pip updates dependencies it thinks are already satisfied to avoid some
-# problems with its lack of real dependency resolution.
-install: 'tools/pip_install.py -I tox virtualenv'
+# version of virtualenv.
+install: 'tools/pip_install.py -U codecov tox virtualenv'
 # Most of the time TRAVIS_RETRY is an empty string, and has no effect on the
 # script command. It is set only to `travis_retry` during farm tests, in
 # order to trigger the Travis retry feature, and compensate the inherent
 # flakiness of these specific tests.
 script: '$TRAVIS_RETRY tox'
 
+after_success: '[ "$TOXENV" == "py27-cover" ] && codecov -F linux'
+
 notifications:
   email: false
-  irc:
-    if: NOT branch =~ ^(travis-)?test-.*$
-    channels:
-      # This is set to a secure variable to prevent forks from sending
-      # notifications. This value was created by installing
-      # https://github.com/travis-ci/travis.rb and running
-      # `travis encrypt "chat.freenode.net#certbot-devel"`.
-      - secure: "EWW66E2+KVPZyIPR8ViENZwfcup4Gx3/dlimmAZE0WuLwxDCshBBOd3O8Rf6pBokEoZlXM5eDT6XdyJj8n0DLslgjO62pExdunXpbcMwdY7l1ELxX2/UbnDTE6UnPYa09qVBHNG7156Z6yE0x2lH4M9Ykvp0G0cubjPQHylAwo0="
-    on_cancel: never
-    on_success: never
-    on_failure: always

AUTHORS.md

@@ -21,7 +21,6 @@ Authors
 * [Andrzej Górski](https://github.com/andrzej3393)
 * [Anselm Levskaya](https://github.com/levskaya)
 * [Antoine Jacoutot](https://github.com/ajacoutot)
-* [April King](https://github.com/april)
 * [asaph](https://github.com/asaph)
 * [Axel Beckert](https://github.com/xtaran)
 * [Bas](https://github.com/Mechazawa)
@@ -37,7 +36,6 @@ Authors
 * [Brad Warren](https://github.com/bmw)
 * [Brandon Kraft](https://github.com/kraftbj)
 * [Brandon Kreisel](https://github.com/kraftbj)
-* [Cameron Steel](https://github.com/Tugzrida)
 * [Ceesjan Luiten](https://github.com/quinox)
 * [Chad Whitacre](https://github.com/whit537)
 * [Chhatoi Pritam Baral](https://github.com/pritambaral)
@@ -102,9 +100,7 @@ Authors
 * [Harlan Lieberman-Berg](https://github.com/hlieberman)
 * [Henri Salo](https://github.com/fgeek)
 * [Henry Chen](https://github.com/henrychen95)
-* [Hugo van Kemenade](https://github.com/hugovk)
 * [Ingolf Becker](https://github.com/watercrossing)
-* [Ivan Nejgebauer](https://github.com/inejge)
 * [Jaap Eldering](https://github.com/eldering)
 * [Jacob Hoffman-Andrews](https://github.com/jsha)
 * [Jacob Sachs](https://github.com/jsachs)
@@ -128,7 +124,6 @@ Authors
 * [Jonathan Herlin](https://github.com/Jonher937)
 * [Jon Walsh](https://github.com/code-tree)
 * [Joona Hoikkala](https://github.com/joohoi)
-* [Josh McCullough](https://github.com/JoshMcCullough)
 * [Josh Soref](https://github.com/jsoref)
 * [Joubin Jabbari](https://github.com/joubin)
 * [Juho Juopperi](https://github.com/jkjuopperi)
@@ -237,7 +232,6 @@ Authors
 * [Stefan Weil](https://github.com/stweil)
 * [Steve Desmond](https://github.com/stevedesmond-ca)
 * [sydneyli](https://github.com/sydneyli)
-* [taixx046](https://github.com/taixx046)
 * [Tan Jay Jun](https://github.com/jayjun)
 * [Tapple Gao](https://github.com/tapple)
 * [Telepenin Nikolay](https://github.com/telepenin)
@@ -269,6 +263,5 @@ Authors
 * [Yomna](https://github.com/ynasser)
 * [Yoni Jah](https://github.com/yonjah)
 * [YourDaddyIsHere](https://github.com/YourDaddyIsHere)
-* [Yuseong Cho](https://github.com/g6123)
 * [Zach Shepherd](https://github.com/zjs)
 * [陈三](https://github.com/chenxsan)

acme/acme/challenges.py

@@ -1,22 +1,15 @@
 """ACME Identifier Validation Challenges."""
 import abc
-import codecs
 import functools
 import hashlib
 import logging
-import socket
 
 from cryptography.hazmat.primitives import hashes  # type: ignore
 import josepy as jose
 import requests
 import six
-from OpenSSL import SSL  # type: ignore # https://github.com/python/typeshed/issues/2052
-from OpenSSL import crypto
 
-from acme import crypto_util
-from acme import errors
 from acme import fields
-from acme.mixins import ResourceMixin, TypeMixin
 
 logger = logging.getLogger(__name__)
 
@@ -35,7 +28,7 @@ class Challenge(jose.TypedJSONObjectWithFields):
             return UnrecognizedChallenge.from_json(jobj)
 
 
-class ChallengeResponse(ResourceMixin, TypeMixin, jose.TypedJSONObjectWithFields):
+class ChallengeResponse(jose.TypedJSONObjectWithFields):
     # _fields_to_partial_json
     """ACME challenge response."""
     TYPES = {}  # type: dict
@@ -310,7 +303,7 @@ class HTTP01Response(KeyAuthorizationChallengeResponse):
         uri = chall.uri(domain)
         logger.debug("Verifying %s at %s...", chall.typ, uri)
         try:
-            http_response = requests.get(uri, verify=False)
+            http_response = requests.get(uri)
         except requests.exceptions.RequestException as error:
             logger.error("Unable to reach %s: %s", uri, error)
             return False
@@ -369,163 +362,29 @@ class HTTP01(KeyAuthorizationChallenge):
 
 @ChallengeResponse.register
 class TLSALPN01Response(KeyAuthorizationChallengeResponse):
-    """ACME tls-alpn-01 challenge response."""
+    """ACME TLS-ALPN-01 challenge response.
+
+    This class only allows initiating a TLS-ALPN-01 challenge returned from the
+    CA. Full support for responding to TLS-ALPN-01 challenges by generating and
+    serving the expected response certificate is not currently provided.
+    """
     typ = "tls-alpn-01"
 
-    PORT = 443
-    """Verification port as defined by the protocol.
 
-    You can override it (e.g. for testing) by passing ``port`` to
-    `simple_verify`.
+@Challenge.register
+class TLSALPN01(KeyAuthorizationChallenge):
+    """ACME tls-alpn-01 challenge.
+
+    This class simply allows parsing the TLS-ALPN-01 challenge returned from
+    the CA. Full TLS-ALPN-01 support is not currently provided.
 
     """
-
-    ID_PE_ACME_IDENTIFIER_V1 = b"1.3.6.1.5.5.7.1.30.1"
-    ACME_TLS_1_PROTOCOL = "acme-tls/1"
-
-    @property
-    def h(self):
-        """Hash value stored in challenge certificate"""
-        return hashlib.sha256(self.key_authorization.encode('utf-8')).digest()
-
-    def gen_cert(self, domain, key=None, bits=2048):
-        """Generate tls-alpn-01 certificate.
-
-        :param unicode domain: Domain verified by the challenge.
-        :param OpenSSL.crypto.PKey key: Optional private key used in
-            certificate generation. If not provided (``None``), then
-            fresh key will be generated.
-        :param int bits: Number of bits for newly generated key.
-
-        :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey`
-
-        """
-        if key is None:
-            key = crypto.PKey()
-            key.generate_key(crypto.TYPE_RSA, bits)
-
-
-        der_value = b"DER:" + codecs.encode(self.h, 'hex')
-        acme_extension = crypto.X509Extension(self.ID_PE_ACME_IDENTIFIER_V1,
-                critical=True, value=der_value)
-
-        return crypto_util.gen_ss_cert(key, [domain], force_san=True,
-                extensions=[acme_extension]), key
-
-    def probe_cert(self, domain, host=None, port=None):
-        """Probe tls-alpn-01 challenge certificate.
-
-        :param unicode domain: domain being validated, required.
-        :param string host: IP address used to probe the certificate.
-        :param int port: Port used to probe the certificate.
-
-        """
-        if host is None:
-            host = socket.gethostbyname(domain)
-            logger.debug('%s resolved to %s', domain, host)
-        if port is None:
-            port = self.PORT
-
-        return crypto_util.probe_sni(host=host, port=port, name=domain,
-                alpn_protocols=[self.ACME_TLS_1_PROTOCOL])
-
-    def verify_cert(self, domain, cert):
-        """Verify tls-alpn-01 challenge certificate.
-
-        :param unicode domain: Domain name being validated.
-        :param OpensSSL.crypto.X509 cert: Challenge certificate.
-
-        :returns: Whether the certificate was successfully verified.
-        :rtype: bool
-
-        """
-        # pylint: disable=protected-access
-        names = crypto_util._pyopenssl_cert_or_req_all_names(cert)
-        logger.debug('Certificate %s. SANs: %s', cert.digest('sha256'), names)
-        if len(names) != 1 or names[0].lower() != domain.lower():
-            return False
-
-        for i in range(cert.get_extension_count()):
-            ext = cert.get_extension(i)
-            # FIXME: assume this is the ACME extension. Currently there is no
-            # way to get full OID of an unknown extension from pyopenssl.
-            if ext.get_short_name() == b'UNDEF':
-                data = ext.get_data()
-                return data == self.h
-
-        return False
-
-    # pylint: disable=too-many-arguments
-    def simple_verify(self, chall, domain, account_public_key,
-                      cert=None, host=None, port=None):
-        """Simple verify.
-
-        Verify ``validation`` using ``account_public_key``, optionally
-        probe tls-alpn-01 certificate and check using `verify_cert`.
-
-        :param .challenges.TLSALPN01 chall: Corresponding challenge.
-        :param str domain: Domain name being validated.
-        :param JWK account_public_key:
-        :param OpenSSL.crypto.X509 cert: Optional certificate. If not
-            provided (``None``) certificate will be retrieved using
-            `probe_cert`.
-        :param string host: IP address used to probe the certificate.
-        :param int port: Port used to probe the certificate.
-
-
-        :returns: ``True`` if and only if client's control of the domain has been verified.
-        :rtype: bool
-
-        """
-        if not self.verify(chall, account_public_key):
-            logger.debug("Verification of key authorization in response failed")
-            return False
-
-        if cert is None:
-            try:
-                cert = self.probe_cert(domain=domain, host=host, port=port)
-            except errors.Error as error:
-                logger.debug(str(error), exc_info=True)
-                return False
-
-        return self.verify_cert(domain, cert)
-
-
-@Challenge.register  # pylint: disable=too-many-ancestors
-class TLSALPN01(KeyAuthorizationChallenge):
-    """ACME tls-alpn-01 challenge."""
+    typ = "tls-alpn-01"
     response_cls = TLSALPN01Response
-    typ = response_cls.typ
 
     def validation(self, account_key, **kwargs):
-        """Generate validation.
-
-        :param JWK account_key:
-        :param unicode domain: Domain verified by the challenge.
-        :param OpenSSL.crypto.PKey cert_key: Optional private key used
-            in certificate generation. If not provided (``None``), then
-            fresh key will be generated.
-
-        :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey`
-
-        """
-        return self.response(account_key).gen_cert(
-            key=kwargs.get('cert_key'),
-            domain=kwargs.get('domain'))
-
-    @staticmethod
-    def is_supported():
-        """
-        Check if TLS-ALPN-01 challenge is supported on this machine.
-        This implies that a recent version of OpenSSL is installed (>= 1.0.2),
-        or a recent cryptography version shipped with the OpenSSL library is installed.
-
-        :returns: ``True`` if TLS-ALPN-01 is supported on this machine, ``False`` otherwise.
-        :rtype: bool
-
-        """
-        return (hasattr(SSL.Connection, "set_alpn_protos")
-                and hasattr(SSL.Context, "set_alpn_select_callback"))
+        """Generate validation for the challenge."""
+        raise NotImplementedError()
 
 
 @Challenge.register

acme/acme/client.py

@@ -15,17 +15,16 @@ import requests
 from requests.adapters import HTTPAdapter
 from requests_toolbelt.adapters.source import SourceAddressAdapter
 import six
-from six.moves import http_client
+from six.moves import http_client  # pylint: disable=import-error
 
 from acme import crypto_util
 from acme import errors
 from acme import jws
 from acme import messages
-from acme.magic_typing import Dict
-from acme.magic_typing import List
-from acme.magic_typing import Set
-from acme.magic_typing import Text
-from acme.mixins import VersionedLEACMEMixin
+from acme.magic_typing import Dict  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Text  # pylint: disable=unused-import, no-name-in-module
 
 logger = logging.getLogger(__name__)
 
@@ -37,7 +36,7 @@ if sys.version_info < (2, 7, 9):  # pragma: no cover
     try:
         requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3()  # type: ignore
     except AttributeError:
-        import urllib3.contrib.pyopenssl
+        import urllib3.contrib.pyopenssl  # pylint: disable=import-error
         urllib3.contrib.pyopenssl.inject_into_urllib3()
 
 DEFAULT_NETWORK_TIMEOUT = 45
@@ -667,7 +666,7 @@ class ClientV2(ClientBase):
         response = self._post(self.directory['newOrder'], order)
         body = messages.Order.from_json(response.json())
         authorizations = []
-        for url in body.authorizations:
+        for url in body.authorizations:  # pylint: disable=not-an-iterable
             authorizations.append(self._authzr_from_response(self._post_as_get(url), uri=url))
         return messages.OrderResource(
             body=body,
@@ -988,8 +987,6 @@ class ClientNetwork(object):
         :rtype: `josepy.JWS`
 
         """
-        if isinstance(obj, VersionedLEACMEMixin):
-            obj.le_acme_version = acme_version
         jobj = obj.json_dumps(indent=2).encode() if obj else b''
         logger.debug('JWS payload:\n%s', jobj)
         kwargs = {
@@ -1025,9 +1022,6 @@ class ClientNetwork(object):
 
         """
         response_ct = response.headers.get('Content-Type')
-        # Strip parameters from the media-type (rfc2616#section-3.7)
-        if response_ct:
-            response_ct = response_ct.split(';')[0].strip()
         try:
             # TODO: response.json() is called twice, once here, and
             # once in _get and _post clients
@@ -1123,8 +1117,8 @@ class ClientNetwork(object):
             debug_content = response.content.decode("utf-8")
         logger.debug('Received response:\nHTTP %d\n%s\n\n%s',
                      response.status_code,
-                     "\n".join("{0}: {1}".format(k, v)
-                                for k, v in response.headers.items()),
+                     "\n".join(["{0}: {1}".format(k, v)
+                                for k, v in response.headers.items()]),
                      debug_content)
         return response
 

acme/acme/crypto_util.py

@@ -11,9 +11,10 @@ from OpenSSL import crypto
 from OpenSSL import SSL  # type: ignore # https://github.com/python/typeshed/issues/2052
 
 from acme import errors
-from acme.magic_typing import Callable
-from acme.magic_typing import Tuple
-from acme.magic_typing import Union
+from acme.magic_typing import Callable  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Optional  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Tuple  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Union  # pylint: disable=unused-import, no-name-in-module
 
 logger = logging.getLogger(__name__)
 
@@ -27,41 +28,19 @@ logger = logging.getLogger(__name__)
 _DEFAULT_SSL_METHOD = SSL.SSLv23_METHOD  # type: ignore
 
 
-class _DefaultCertSelection(object):
-    def __init__(self, certs):
-        self.certs = certs
-
-    def __call__(self, connection):
-        server_name = connection.get_servername()
-        return self.certs.get(server_name, None)
-
-
-class SSLSocket(object):  # pylint: disable=too-few-public-methods
+class SSLSocket(object):
     """SSL wrapper for sockets.
 
     :ivar socket sock: Original wrapped socket.
     :ivar dict certs: Mapping from domain names (`bytes`) to
         `OpenSSL.crypto.X509`.
     :ivar method: See `OpenSSL.SSL.Context` for allowed values.
-    :ivar alpn_selection: Hook to select negotiated ALPN protocol for
-        connection.
-    :ivar cert_selection: Hook to select certificate for connection. If given,
-        `certs` parameter would be ignored, and therefore must be empty.
 
     """
-    def __init__(self, sock, certs=None,
-            method=_DEFAULT_SSL_METHOD, alpn_selection=None,
-            cert_selection=None):
+    def __init__(self, sock, certs, method=_DEFAULT_SSL_METHOD):
         self.sock = sock
-        self.alpn_selection = alpn_selection
+        self.certs = certs
         self.method = method
-        if not cert_selection and not certs:
-            raise ValueError("Neither cert_selection or certs specified.")
-        if cert_selection and certs:
-            raise ValueError("Both cert_selection and certs specified.")
-        if cert_selection is None:
-            cert_selection = _DefaultCertSelection(certs)
-        self.cert_selection = cert_selection
 
     def __getattr__(self, name):
         return getattr(self.sock, name)
@@ -78,25 +57,24 @@ class SSLSocket(object):  # pylint: disable=too-few-public-methods
         :type connection: :class:`OpenSSL.Connection`
 
         """
-        pair = self.cert_selection(connection)
-        if pair is None:
-            logger.debug("Certificate selection for server name %s failed, dropping SSL",
-                         connection.get_servername())
+        server_name = connection.get_servername()
+        try:
+            key, cert = self.certs[server_name]
+        except KeyError:
+            logger.debug("Server name (%s) not recognized, dropping SSL",
+                         server_name)
             return
-        key, cert = pair
         new_context = SSL.Context(self.method)
         new_context.set_options(SSL.OP_NO_SSLv2)
         new_context.set_options(SSL.OP_NO_SSLv3)
         new_context.use_privatekey(key)
         new_context.use_certificate(cert)
-        if self.alpn_selection is not None:
-            new_context.set_alpn_select_callback(self.alpn_selection)
         connection.set_context(new_context)
 
     class FakeConnection(object):
         """Fake OpenSSL.SSL.Connection."""
 
-        # pylint: disable=missing-function-docstring
+        # pylint: disable=missing-docstring
 
         def __init__(self, connection):
             self._wrapped = connection
@@ -108,15 +86,13 @@ class SSLSocket(object):  # pylint: disable=too-few-public-methods
             # OpenSSL.SSL.Connection.shutdown doesn't accept any args
             return self._wrapped.shutdown()
 
-    def accept(self):  # pylint: disable=missing-function-docstring
+    def accept(self):  # pylint: disable=missing-docstring
         sock, addr = self.sock.accept()
 
         context = SSL.Context(self.method)
         context.set_options(SSL.OP_NO_SSLv2)
         context.set_options(SSL.OP_NO_SSLv3)
         context.set_tlsext_servername_callback(self._pick_certificate_cb)
-        if self.alpn_selection is not None:
-            context.set_alpn_select_callback(self.alpn_selection)
 
         ssl_sock = self.FakeConnection(SSL.Connection(context, sock))
         ssl_sock.set_accept_state()
@@ -132,9 +108,8 @@ class SSLSocket(object):  # pylint: disable=too-few-public-methods
         return ssl_sock, addr
 
 
-def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-arguments
-              method=_DEFAULT_SSL_METHOD, source_address=('', 0),
-              alpn_protocols=None):
+def probe_sni(name, host, port=443, timeout=300,
+              method=_DEFAULT_SSL_METHOD, source_address=('', 0)):
     """Probe SNI server for SSL certificate.
 
     :param bytes name: Byte string to send as the server name in the
@@ -146,8 +121,6 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu
     :param tuple source_address: Enables multi-path probing (selection
         of source interface). See `socket.creation_connection` for more
         info. Available only in Python 2.7+.
-    :param alpn_protocols: Protocols to request using ALPN.
-    :type alpn_protocols: `list` of `bytes`
 
     :raises acme.errors.Error: In case of any problems.
 
@@ -177,8 +150,6 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu
         client_ssl = SSL.Connection(context, client)
         client_ssl.set_connect_state()
         client_ssl.set_tlsext_host_name(name)  # pyOpenSSL>=0.13
-        if alpn_protocols is not None:
-            client_ssl.set_alpn_protos(alpn_protocols)
         try:
             client_ssl.do_handshake()
             client_ssl.shutdown()
@@ -269,14 +240,12 @@ def _pyopenssl_cert_or_req_san(cert_or_req):
 
 
 def gen_ss_cert(key, domains, not_before=None,
-                validity=(7 * 24 * 60 * 60), force_san=True, extensions=None):
+                validity=(7 * 24 * 60 * 60), force_san=True):
     """Generate new self-signed certificate.
 
     :type domains: `list` of `unicode`
     :param OpenSSL.crypto.PKey key:
     :param bool force_san:
-    :param extensions: List of additional extensions to include in the cert.
-    :type extensions: `list` of `OpenSSL.crypto.X509Extension`
 
     If more than one domain is provided, all of the domains are put into
     ``subjectAltName`` X.509 extension and first domain is set as the
@@ -289,13 +258,10 @@ def gen_ss_cert(key, domains, not_before=None,
     cert.set_serial_number(int(binascii.hexlify(os.urandom(16)), 16))
     cert.set_version(2)
 
-    if extensions is None:
-        extensions = []
-
-    extensions.append(
+    extensions = [
         crypto.X509Extension(
             b"basicConstraints", True, b"CA:TRUE, pathlen:0"),
-    )
+    ]
 
     cert.get_subject().CN = domains[0]
     # TODO: what to put into cert.get_subject()?
@@ -332,6 +298,7 @@ def dump_pyopenssl_chain(chain, filetype=crypto.FILETYPE_PEM):
 
     def _dump_cert(cert):
         if isinstance(cert, jose.ComparableX509):
+            # pylint: disable=protected-access
             cert = cert.wrapped
         return crypto.dump_certificate(filetype, cert)
 

acme/acme/jws.py

@@ -15,7 +15,7 @@ class Header(jose.Header):
     url = jose.Field('url', omitempty=True)
 
     @nonce.decoder
-    def nonce(value):  # pylint: disable=no-self-argument,missing-function-docstring
+    def nonce(value):  # pylint: disable=missing-docstring,no-self-argument
         try:
             return jose.decode_b64jose(value)
         except jose.DeserializationError as error:

acme/acme/magic_typing.py

@@ -10,6 +10,8 @@ class TypingClass(object):
 try:
     # mypy doesn't respect modifying sys.modules
     from typing import *  # pylint: disable=wildcard-import, unused-wildcard-import
+    # pylint: disable=unused-import
     from typing import Collection, IO  # type: ignore
+    # pylint: enable=unused-import
 except ImportError:
     sys.modules[__name__] = TypingClass()

acme/acme/messages.py

@@ -9,10 +9,9 @@ from acme import errors
 from acme import fields
 from acme import jws
 from acme import util
-from acme.mixins import ResourceMixin
 
 try:
-    from collections.abc import Hashable
+    from collections.abc import Hashable  # pylint: disable=no-name-in-module
 except ImportError:  # pragma: no cover
     from collections import Hashable
 
@@ -357,13 +356,13 @@ class Registration(ResourceBody):
 
 
 @Directory.register
-class NewRegistration(ResourceMixin, Registration):
+class NewRegistration(Registration):
     """New registration."""
     resource_type = 'new-reg'
     resource = fields.Resource(resource_type)
 
 
-class UpdateRegistration(ResourceMixin, Registration):
+class UpdateRegistration(Registration):
     """Update registration."""
     resource_type = 'reg'
     resource = fields.Resource(resource_type)
@@ -461,6 +460,7 @@ class ChallengeResource(Resource):
     @property
     def uri(self):
         """The URL of the challenge body."""
+        # pylint: disable=function-redefined,no-member
         return self.body.uri
 
 
@@ -488,7 +488,7 @@ class Authorization(ResourceBody):
     wildcard = jose.Field('wildcard', omitempty=True)
 
     @challenges.decoder
-    def challenges(value):  # pylint: disable=no-self-argument,missing-function-docstring
+    def challenges(value):  # pylint: disable=missing-docstring,no-self-argument
         return tuple(ChallengeBody.from_json(chall) for chall in value)
 
     @property
@@ -499,13 +499,13 @@ class Authorization(ResourceBody):
 
 
 @Directory.register
-class NewAuthorization(ResourceMixin, Authorization):
+class NewAuthorization(Authorization):
     """New authorization."""
     resource_type = 'new-authz'
     resource = fields.Resource(resource_type)
 
 
-class UpdateAuthorization(ResourceMixin, Authorization):
+class UpdateAuthorization(Authorization):
     """Update authorization."""
     resource_type = 'authz'
     resource = fields.Resource(resource_type)
@@ -523,7 +523,7 @@ class AuthorizationResource(ResourceWithURI):
 
 
 @Directory.register
-class CertificateRequest(ResourceMixin, jose.JSONObjectWithFields):
+class CertificateRequest(jose.JSONObjectWithFields):
     """ACME new-cert request.
 
     :ivar josepy.util.ComparableX509 csr:
@@ -549,7 +549,7 @@ class CertificateResource(ResourceWithURI):
 
 
 @Directory.register
-class Revocation(ResourceMixin, jose.JSONObjectWithFields):
+class Revocation(jose.JSONObjectWithFields):
     """Revocation message.
 
     :ivar .ComparableX509 certificate: `OpenSSL.crypto.X509` wrapped in
@@ -585,7 +585,7 @@ class Order(ResourceBody):
     error = jose.Field('error', omitempty=True, decoder=Error.from_json)
 
     @identifiers.decoder
-    def identifiers(value):  # pylint: disable=no-self-argument,missing-function-docstring
+    def identifiers(value):  # pylint: disable=missing-docstring,no-self-argument
         return tuple(Identifier.from_json(identifier) for identifier in value)
 
 class OrderResource(ResourceWithURI):

acme/acme/mixins.py

@@ -1,65 +0,0 @@
-"""Useful mixins for Challenge and Resource objects"""
-
-
-class VersionedLEACMEMixin(object):
-    """This mixin stores the version of Let's Encrypt's endpoint being used."""
-    @property
-    def le_acme_version(self):
-        """Define the version of ACME protocol to use"""
-        return getattr(self, '_le_acme_version', 1)
-
-    @le_acme_version.setter
-    def le_acme_version(self, version):
-        # We need to use object.__setattr__ to not depend on the specific implementation of
-        # __setattr__  in current class (eg. jose.TypedJSONObjectWithFields raises AttributeError
-        # for any attempt to set an attribute to make objects immutable).
-        object.__setattr__(self, '_le_acme_version', version)
-
-    def __setattr__(self, key, value):
-        if key == 'le_acme_version':
-            # Required for @property to operate properly. See comment above.
-            object.__setattr__(self, key, value)
-        else:
-            super(VersionedLEACMEMixin, self).__setattr__(key, value)  # pragma: no cover
-
-
-class ResourceMixin(VersionedLEACMEMixin):
-    """
-    This mixin generates a RFC8555 compliant JWS payload
-    by removing the `resource` field if needed (eg. ACME v2 protocol).
-    """
-    def to_partial_json(self):
-        """See josepy.JSONDeserializable.to_partial_json()"""
-        return _safe_jobj_compliance(super(ResourceMixin, self),
-                                     'to_partial_json', 'resource')
-
-    def fields_to_partial_json(self):
-        """See josepy.JSONObjectWithFields.fields_to_partial_json()"""
-        return _safe_jobj_compliance(super(ResourceMixin, self),
-                                     'fields_to_partial_json', 'resource')
-
-
-class TypeMixin(VersionedLEACMEMixin):
-    """
-    This mixin allows generation of a RFC8555 compliant JWS payload
-    by removing the `type` field if needed (eg. ACME v2 protocol).
-    """
-    def to_partial_json(self):
-        """See josepy.JSONDeserializable.to_partial_json()"""
-        return _safe_jobj_compliance(super(TypeMixin, self),
-                                     'to_partial_json', 'type')
-
-    def fields_to_partial_json(self):
-        """See josepy.JSONObjectWithFields.fields_to_partial_json()"""
-        return _safe_jobj_compliance(super(TypeMixin, self),
-                                     'fields_to_partial_json', 'type')
-
-
-def _safe_jobj_compliance(instance, jobj_method, uncompliant_field):
-    if hasattr(instance, jobj_method):
-        jobj = getattr(instance, jobj_method)()
-        if instance.le_acme_version == 2:
-            jobj.pop(uncompliant_field, None)
-        return jobj
-
-    raise AttributeError('Method {0}() is not implemented.'.format(jobj_method))  # pragma: no cover

acme/acme/standalone.py

@@ -5,16 +5,19 @@ import logging
 import socket
 import threading
 
-from six.moves import BaseHTTPServer  # type: ignore
-from six.moves import http_client
-from six.moves import socketserver  # type: ignore
+from six.moves import BaseHTTPServer  # type: ignore  # pylint: disable=import-error
+from six.moves import http_client  # pylint: disable=import-error
+from six.moves import socketserver  # type: ignore  # pylint: disable=import-error
 
 from acme import challenges
 from acme import crypto_util
-from acme.magic_typing import List
+from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
 
 logger = logging.getLogger(__name__)
 
+# six.moves.* | pylint: disable=no-member,attribute-defined-outside-init
+# pylint: disable=no-init
+
 
 class TLSServer(socketserver.TCPServer):
     """Generic TLS Server."""
@@ -27,22 +30,16 @@ class TLSServer(socketserver.TCPServer):
             self.address_family = socket.AF_INET
         self.certs = kwargs.pop("certs", {})
         self.method = kwargs.pop(
+            # pylint: disable=protected-access
             "method", crypto_util._DEFAULT_SSL_METHOD)
         self.allow_reuse_address = kwargs.pop("allow_reuse_address", True)
         socketserver.TCPServer.__init__(self, *args, **kwargs)
 
     def _wrap_sock(self):
         self.socket = crypto_util.SSLSocket(
-            self.socket, cert_selection=self._cert_selection,
-            alpn_selection=getattr(self, '_alpn_selection', None),
-            method=self.method)
+            self.socket, certs=self.certs, method=self.method)
 
-    def _cert_selection(self, connection):  # pragma: no cover
-        """Callback selecting certificate for connection."""
-        server_name = connection.get_servername()
-        return self.certs.get(server_name, None)
-
-    def server_bind(self):
+    def server_bind(self):  # pylint: disable=missing-docstring
         self._wrap_sock()
         return socketserver.TCPServer.server_bind(self)
 
@@ -127,40 +124,6 @@ class BaseDualNetworkedServers(object):
         self.threads = []
 
 
-class TLSALPN01Server(TLSServer, ACMEServerMixin):
-    """TLSALPN01 Server."""
-
-    ACME_TLS_1_PROTOCOL = b"acme-tls/1"
-
-    def __init__(self, server_address, certs, challenge_certs, ipv6=False):
-        TLSServer.__init__(
-            self, server_address, _BaseRequestHandlerWithLogging, certs=certs,
-            ipv6=ipv6)
-        self.challenge_certs = challenge_certs
-
-    def _cert_selection(self, connection):
-        # TODO: We would like to serve challenge cert only if asked for it via
-        # ALPN. To do this, we need to retrieve the list of protos from client
-        # hello, but this is currently impossible with openssl [0], and ALPN
-        # negotiation is done after cert selection.
-        # Therefore, currently we always return challenge cert, and terminate
-        # handshake in alpn_selection() if ALPN protos are not what we expect.
-        # [0] https://github.com/openssl/openssl/issues/4952
-        server_name = connection.get_servername()
-        logger.debug("Serving challenge cert for server name %s", server_name)
-        return self.challenge_certs.get(server_name, None)
-
-    def _alpn_selection(self, _connection, alpn_protos):
-        """Callback to select alpn protocol."""
-        if len(alpn_protos) == 1 and alpn_protos[0] == self.ACME_TLS_1_PROTOCOL:
-            logger.debug("Agreed on %s ALPN", self.ACME_TLS_1_PROTOCOL)
-            return self.ACME_TLS_1_PROTOCOL
-        logger.debug("Cannot agree on ALPN proto. Got: %s", str(alpn_protos))
-        # Explicitly close the connection now, by returning an empty string.
-        # See https://www.pyopenssl.org/en/stable/api/ssl.html#OpenSSL.SSL.Context.set_alpn_select_callback  # pylint: disable=line-too-long
-        return b""
-
-
 class HTTPServer(BaseHTTPServer.HTTPServer):
     """Generic HTTP Server."""
 
@@ -176,10 +139,10 @@ class HTTPServer(BaseHTTPServer.HTTPServer):
 class HTTP01Server(HTTPServer, ACMEServerMixin):
     """HTTP01 Server."""
 
-    def __init__(self, server_address, resources, ipv6=False, timeout=30):
+    def __init__(self, server_address, resources, ipv6=False):
         HTTPServer.__init__(
             self, server_address, HTTP01RequestHandler.partial_init(
-                simple_http_resources=resources, timeout=timeout), ipv6=ipv6)
+                simple_http_resources=resources), ipv6=ipv6)
 
 
 class HTTP01DualNetworkedServers(BaseDualNetworkedServers):
@@ -204,7 +167,6 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 
     def __init__(self, *args, **kwargs):
         self.simple_http_resources = kwargs.pop("simple_http_resources", set())
-        self.timeout = kwargs.pop('timeout', 30)
         BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
 
     def log_message(self, format, *args):  # pylint: disable=redefined-builtin
@@ -216,7 +178,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
         self.log_message("Incoming request")
         BaseHTTPServer.BaseHTTPRequestHandler.handle(self)
 
-    def do_GET(self):  # pylint: disable=invalid-name,missing-function-docstring
+    def do_GET(self):  # pylint: disable=invalid-name,missing-docstring
         if self.path == "/":
             self.handle_index()
         elif self.path.startswith("/" + challenges.HTTP01.URI_ROOT_PATH):
@@ -254,7 +216,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
                          self.path)
 
     @classmethod
-    def partial_init(cls, simple_http_resources, timeout):
+    def partial_init(cls, simple_http_resources):
         """Partially initialize this handler.
 
         This is useful because `socketserver.BaseServer` takes
@@ -263,18 +225,4 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 
         """
         return functools.partial(
-            cls, simple_http_resources=simple_http_resources,
-            timeout=timeout)
-
-
-class _BaseRequestHandlerWithLogging(socketserver.BaseRequestHandler):
-    """BaseRequestHandler with logging."""
-
-    def log_message(self, format, *args):  # pylint: disable=redefined-builtin
-        """Log arbitrary message."""
-        logger.debug("%s - - %s", self.client_address[0], format % args)
-
-    def handle(self):
-        """Handle request."""
-        self.log_message("Incoming request")
-        socketserver.BaseRequestHandler.handle(self)
+            cls, simple_http_resources=simple_http_resources)

acme/docs/conf.py

@@ -13,6 +13,7 @@
 # serve to show the default.
 
 import os
+import shlex
 import sys
 
 here = os.path.abspath(os.path.dirname(__file__))
@@ -112,7 +113,7 @@ pygments_style = 'sphinx'
 #keep_warnings = False
 
 # If true, `todo` and `todoList` produce output, else they produce nothing.
-todo_include_todos = False
+todo_include_todos = True
 
 
 # -- Options for HTML output ----------------------------------------------

acme/setup.py

@@ -1,12 +1,10 @@
-from distutils.version import StrictVersion
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.6.0.dev0'
+version = '1.2.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
@@ -17,6 +15,7 @@ install_requires = [
     # 1.1.0+ is required to avoid the warnings described at
     # https://github.com/certbot/josepy/issues/13.
     'josepy>=1.1.0',
+    'mock',
     # Connection.set_tlsext_host_name (>=0.13)
     'PyOpenSSL>=0.13.1',
     'pyrfc3339',
@@ -27,15 +26,6 @@ install_requires = [
     'six>=1.9.0',  # needed for python_2_unicode_compatible
 ]
 
-setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 dev_extras = [
     'pytest',
     'pytest-xdist',
@@ -71,7 +61,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
-    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Intended Audience :: Developers',
@@ -80,6 +70,7 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',

acme/tests/challenges_test.py

@@ -2,16 +2,10 @@
 import unittest
 
 import josepy as jose
-import OpenSSL
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 import requests
 from six.moves.urllib import parse as urllib_parse
 
-from acme import errors
-
 import test_util
 
 CERT = test_util.load_comparable_cert('cert.pem')
@@ -187,7 +181,7 @@ class HTTP01ResponseTest(unittest.TestCase):
         mock_get.return_value = mock.MagicMock(text=validation)
         self.assertTrue(self.response.simple_verify(
             self.chall, "local", KEY.public_key()))
-        mock_get.assert_called_once_with(self.chall.uri("local"), verify=False)
+        mock_get.assert_called_once_with(self.chall.uri("local"))
 
     @mock.patch("acme.challenges.requests.get")
     def test_simple_verify_bad_validation(self, mock_get):
@@ -203,7 +197,7 @@ class HTTP01ResponseTest(unittest.TestCase):
                   HTTP01Response.WHITESPACE_CUTSET))
         self.assertTrue(self.response.simple_verify(
             self.chall, "local", KEY.public_key()))
-        mock_get.assert_called_once_with(self.chall.uri("local"), verify=False)
+        mock_get.assert_called_once_with(self.chall.uri("local"))
 
     @mock.patch("acme.challenges.requests.get")
     def test_simple_verify_connection_error(self, mock_get):
@@ -262,87 +256,30 @@ class HTTP01Test(unittest.TestCase):
 class TLSALPN01ResponseTest(unittest.TestCase):
 
     def setUp(self):
-        from acme.challenges import TLSALPN01
-        self.chall = TLSALPN01(
-            token=jose.b64decode(b'a82d5ff8ef740d12881f6d3c2277ab2e'))
-        self.domain = u'example.com'
-        self.domain2 = u'example2.com'
-
-        self.response = self.chall.response(KEY)
+        from acme.challenges import TLSALPN01Response
+        self.msg = TLSALPN01Response(key_authorization=u'foo')
         self.jmsg = {
             'resource': 'challenge',
             'type': 'tls-alpn-01',
-            'keyAuthorization': self.response.key_authorization,
+            'keyAuthorization': u'foo',
         }
 
+        from acme.challenges import TLSALPN01
+        self.chall = TLSALPN01(token=(b'x' * 16))
+        self.response = self.chall.response(KEY)
+
     def test_to_partial_json(self):
         self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'},
-                         self.response.to_partial_json())
+                         self.msg.to_partial_json())
 
     def test_from_json(self):
         from acme.challenges import TLSALPN01Response
-        self.assertEqual(self.response, TLSALPN01Response.from_json(self.jmsg))
+        self.assertEqual(self.msg, TLSALPN01Response.from_json(self.jmsg))
 
     def test_from_json_hashable(self):
         from acme.challenges import TLSALPN01Response
         hash(TLSALPN01Response.from_json(self.jmsg))
 
-    def test_gen_verify_cert(self):
-        key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem')
-        cert, key2 = self.response.gen_cert(self.domain, key1)
-        self.assertEqual(key1, key2)
-        self.assertTrue(self.response.verify_cert(self.domain, cert))
-
-    def test_gen_verify_cert_gen_key(self):
-        cert, key = self.response.gen_cert(self.domain)
-        self.assertTrue(isinstance(key, OpenSSL.crypto.PKey))
-        self.assertTrue(self.response.verify_cert(self.domain, cert))
-
-    def test_verify_bad_cert(self):
-        self.assertFalse(self.response.verify_cert(self.domain,
-            test_util.load_cert('cert.pem')))
-
-    def test_verify_bad_domain(self):
-        key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem')
-        cert, key2 = self.response.gen_cert(self.domain, key1)
-        self.assertEqual(key1, key2)
-        self.assertFalse(self.response.verify_cert(self.domain2, cert))
-
-    def test_simple_verify_bad_key_authorization(self):
-        key2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem'))
-        self.response.simple_verify(self.chall, "local", key2.public_key())
-
-    @mock.patch('acme.challenges.TLSALPN01Response.verify_cert', autospec=True)
-    def test_simple_verify(self, mock_verify_cert):
-        mock_verify_cert.return_value = mock.sentinel.verification
-        self.assertEqual(
-            mock.sentinel.verification, self.response.simple_verify(
-                self.chall, self.domain, KEY.public_key(),
-                cert=mock.sentinel.cert))
-        mock_verify_cert.assert_called_once_with(
-            self.response, self.domain, mock.sentinel.cert)
-
-    @mock.patch('acme.challenges.socket.gethostbyname')
-    @mock.patch('acme.challenges.crypto_util.probe_sni')
-    def test_probe_cert(self, mock_probe_sni, mock_gethostbyname):
-        mock_gethostbyname.return_value = '127.0.0.1'
-        self.response.probe_cert('foo.com')
-        mock_gethostbyname.assert_called_once_with('foo.com')
-        mock_probe_sni.assert_called_once_with(
-            host='127.0.0.1', port=self.response.PORT, name='foo.com',
-            alpn_protocols=['acme-tls/1'])
-
-        self.response.probe_cert('foo.com', host='8.8.8.8')
-        mock_probe_sni.assert_called_with(
-            host='8.8.8.8', port=mock.ANY, name='foo.com',
-            alpn_protocols=['acme-tls/1'])
-
-    @mock.patch('acme.challenges.TLSALPN01Response.probe_cert')
-    def test_simple_verify_false_on_probe_error(self, mock_probe_cert):
-        mock_probe_cert.side_effect = errors.Error
-        self.assertFalse(self.response.simple_verify(
-            self.chall, self.domain, KEY.public_key()))
-
 
 class TLSALPN01Test(unittest.TestCase):
 
@@ -372,13 +309,8 @@ class TLSALPN01Test(unittest.TestCase):
         self.assertRaises(
             jose.DeserializationError, TLSALPN01.from_json, self.jmsg)
 
-    @mock.patch('acme.challenges.TLSALPN01Response.gen_cert')
-    def test_validation(self, mock_gen_cert):
-        mock_gen_cert.return_value = ('cert', 'key')
-        self.assertEqual(('cert', 'key'), self.msg.validation(
-            KEY, cert_key=mock.sentinel.cert_key, domain=mock.sentinel.domain))
-        mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key,
-                                              domain=mock.sentinel.domain)
+    def test_validation(self):
+        self.assertRaises(NotImplementedError, self.msg.validation, KEY)
 
 
 class DNSTest(unittest.TestCase):
@@ -481,18 +413,5 @@ class DNSResponseTest(unittest.TestCase):
             self.msg.check_validation(self.chall, KEY.public_key()))
 
 
-class JWSPayloadRFC8555Compliant(unittest.TestCase):
-    """Test for RFC8555 compliance of JWS generated from resources/challenges"""
-    def test_challenge_payload(self):
-        from acme.challenges import HTTP01Response
-
-        challenge_body = HTTP01Response()
-        challenge_body.le_acme_version = 2
-
-        jobj = challenge_body.json_dumps(indent=2).encode()
-        # RFC8555 states that challenge responses must have an empty payload.
-        self.assertEqual(jobj, b'{}')
-
-
 if __name__ == '__main__':
     unittest.main()  # pragma: no cover

acme/tests/client_test.py

@@ -6,10 +6,7 @@ import json
 import unittest
 
 import josepy as jose
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 import OpenSSL
 import requests
 from six.moves import http_client  # pylint: disable=import-error
@@ -18,7 +15,7 @@ from acme import challenges
 from acme import errors
 from acme import jws as acme_jws
 from acme import messages
-from acme.mixins import VersionedLEACMEMixin
+from acme.magic_typing import Dict  # pylint: disable=unused-import, no-name-in-module
 import messages_test
 import test_util
 
@@ -889,7 +886,7 @@ class ClientV2Test(ClientTestBase):
             self.client.net.get.assert_not_called()
 
 
-class MockJSONDeSerializable(VersionedLEACMEMixin, jose.JSONDeSerializable):
+class MockJSONDeSerializable(jose.JSONDeSerializable):
     # pylint: disable=missing-docstring
     def __init__(self, value):
         self.value = value
@@ -983,35 +980,6 @@ class ClientNetworkTest(unittest.TestCase):
             self.assertEqual(
                 self.response, self.net._check_response(self.response))
 
-    @mock.patch('acme.client.logger')
-    def test_check_response_ok_ct_with_charset(self, mock_logger):
-        self.response.json.return_value = {}
-        self.response.headers['Content-Type'] = 'application/json; charset=utf-8'
-        # pylint: disable=protected-access
-        self.assertEqual(self.response, self.net._check_response(
-            self.response, content_type='application/json'))
-        try:
-            mock_logger.debug.assert_called_with(
-                'Ignoring wrong Content-Type (%r) for JSON decodable response',
-                'application/json; charset=utf-8'
-            )
-        except AssertionError:
-            return
-        raise AssertionError('Expected Content-Type warning ' #pragma: no cover
-            'to not have been logged')
-
-    @mock.patch('acme.client.logger')
-    def test_check_response_ok_bad_ct(self, mock_logger):
-        self.response.json.return_value = {}
-        self.response.headers['Content-Type'] = 'text/plain'
-        # pylint: disable=protected-access
-        self.assertEqual(self.response, self.net._check_response(
-            self.response, content_type='application/json'))
-        mock_logger.debug.assert_called_with(
-            'Ignoring wrong Content-Type (%r) for JSON decodable response',
-            'text/plain'
-        )
-
     def test_check_response_conflict(self):
         self.response.ok = False
         self.response.status_code = 409

acme/tests/crypto_util_test.py

@@ -11,12 +11,14 @@ import six
 from six.moves import socketserver  # type: ignore  # pylint: disable=import-error
 
 from acme import errors
+from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
 import test_util
 
 
 class SSLSocketAndProbeSNITest(unittest.TestCase):
     """Tests for acme.crypto_util.SSLSocket/probe_sni."""
 
+
     def setUp(self):
         self.cert = test_util.load_comparable_cert('rsa2048_cert.pem')
         key = test_util.load_pyopenssl_private_key('rsa2048_key.pem')
@@ -30,8 +32,7 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
             # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init
 
             def server_bind(self):  # pylint: disable=missing-docstring
-                self.socket = SSLSocket(socket.socket(),
-                        certs)
+                self.socket = SSLSocket(socket.socket(), certs=certs)
                 socketserver.TCPServer.server_bind(self)
 
         self.server = _TestServer(('', 0), socketserver.BaseRequestHandler)
@@ -72,18 +73,6 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
             socket.setdefaulttimeout(original_timeout)
 
 
-class SSLSocketTest(unittest.TestCase):
-    """Tests for acme.crypto_util.SSLSocket."""
-
-    def test_ssl_socket_invalid_arguments(self):
-        from acme.crypto_util import SSLSocket
-        with self.assertRaises(ValueError):
-            _ = SSLSocket(None, {'sni': ('key', 'cert')},
-                    cert_selection=lambda _: None)
-        with self.assertRaises(ValueError):
-            _ = SSLSocket(None)
-
-
 class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase):
     """Test for acme.crypto_util._pyopenssl_cert_or_req_all_names."""
 

acme/tests/errors_test.py

@@ -1,10 +1,7 @@
 """Tests for acme.errors."""
 import unittest
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 
 
 class BadNonceTest(unittest.TestCase):
@@ -38,7 +35,7 @@ class PollErrorTest(unittest.TestCase):
     def setUp(self):
         from acme.errors import PollError
         self.timeout = PollError(
-            exhausted={mock.sentinel.AR},
+            exhausted=set([mock.sentinel.AR]),
             updated={})
         self.invalid = PollError(exhausted=set(), updated={
             mock.sentinel.AR: mock.sentinel.AR2})

acme/tests/magic_typing_test.py

@@ -2,10 +2,7 @@
 import sys
 import unittest
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 
 
 class MagicTypingTest(unittest.TestCase):
@@ -21,7 +18,7 @@ class MagicTypingTest(unittest.TestCase):
         sys.modules['typing'] = typing_class_mock
         if 'acme.magic_typing' in sys.modules:
             del sys.modules['acme.magic_typing'] # pragma: no cover
-        from acme.magic_typing import Text
+        from acme.magic_typing import Text # pylint: disable=no-name-in-module
         self.assertEqual(Text, text_mock)
         del sys.modules['acme.magic_typing']
         sys.modules['typing'] = temp_typing
@@ -34,7 +31,7 @@ class MagicTypingTest(unittest.TestCase):
         sys.modules['typing'] = None
         if 'acme.magic_typing' in sys.modules:
             del sys.modules['acme.magic_typing'] # pragma: no cover
-        from acme.magic_typing import Text
+        from acme.magic_typing import Text # pylint: disable=no-name-in-module
         self.assertTrue(Text is None)
         del sys.modules['acme.magic_typing']
         sys.modules['typing'] = temp_typing

acme/tests/messages_test.py

@@ -2,12 +2,10 @@
 import unittest
 
 import josepy as jose
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 
 from acme import challenges
+from acme.magic_typing import Dict  # pylint: disable=unused-import, no-name-in-module
 import test_util
 
 CERT = test_util.load_comparable_cert('cert.der')
@@ -455,7 +453,6 @@ class OrderResourceTest(unittest.TestCase):
             'authorizations': None,
         })
 
-
 class NewOrderTest(unittest.TestCase):
     """Tests for acme.messages.NewOrder."""
 
@@ -470,18 +467,5 @@ class NewOrderTest(unittest.TestCase):
         })
 
 
-class JWSPayloadRFC8555Compliant(unittest.TestCase):
-    """Test for RFC8555 compliance of JWS generated from resources/challenges"""
-    def test_message_payload(self):
-        from acme.messages import NewAuthorization
-
-        new_order = NewAuthorization()
-        new_order.le_acme_version = 2
-
-        jobj = new_order.json_dumps(indent=2).encode()
-        # RFC8555 states that JWS bodies must not have a resource field.
-        self.assertEqual(jobj, b'{}')
-
-
 if __name__ == '__main__':
     unittest.main()  # pragma: no cover

acme/tests/standalone_test.py

@@ -4,18 +4,13 @@ import threading
 import unittest
 
 import josepy as jose
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 import requests
 from six.moves import http_client  # pylint: disable=import-error
 from six.moves import socketserver  # type: ignore  # pylint: disable=import-error
 
 from acme import challenges
-from acme import crypto_util
-from acme import errors
-
+from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
 import test_util
 
 
@@ -88,81 +83,6 @@ class HTTP01ServerTest(unittest.TestCase):
     def test_http01_not_found(self):
         self.assertFalse(self._test_http01(add=False))
 
-    def test_timely_shutdown(self):
-        from acme.standalone import HTTP01Server
-        server = HTTP01Server(('', 0), resources=set(), timeout=0.05)
-        server_thread = threading.Thread(target=server.serve_forever)
-        server_thread.start()
-
-        client = socket.socket()
-        client.connect(('localhost', server.socket.getsockname()[1]))
-
-        stop_thread = threading.Thread(target=server.shutdown)
-        stop_thread.start()
-        server_thread.join(5.)
-
-        is_hung = server_thread.is_alive()
-        try:
-            client.shutdown(socket.SHUT_RDWR)
-        except: # pragma: no cover, pylint: disable=bare-except
-            # may raise error because socket could already be closed
-            pass
-
-        self.assertFalse(is_hung, msg='Server shutdown should not be hung')
-
-
-@unittest.skipIf(not challenges.TLSALPN01.is_supported(), "pyOpenSSL too old")
-class TLSALPN01ServerTest(unittest.TestCase):
-    """Test for acme.standalone.TLSALPN01Server."""
-
-    def setUp(self):
-        self.certs = {b'localhost': (
-            test_util.load_pyopenssl_private_key('rsa2048_key.pem'),
-            test_util.load_cert('rsa2048_cert.pem'),
-        )}
-        # Use different certificate for challenge.
-        self.challenge_certs = {b'localhost': (
-            test_util.load_pyopenssl_private_key('rsa4096_key.pem'),
-            test_util.load_cert('rsa4096_cert.pem'),
-        )}
-        from acme.standalone import TLSALPN01Server
-        self.server = TLSALPN01Server(("localhost", 0), certs=self.certs,
-                challenge_certs=self.challenge_certs)
-        # pylint: disable=no-member
-        self.thread = threading.Thread(target=self.server.serve_forever)
-        self.thread.start()
-
-    def tearDown(self):
-        self.server.shutdown()  # pylint: disable=no-member
-        self.thread.join()
-
-    # TODO: This is not implemented yet, see comments in standalone.py
-    # def test_certs(self):
-    #    host, port = self.server.socket.getsockname()[:2]
-    #    cert = crypto_util.probe_sni(
-    #        b'localhost', host=host, port=port, timeout=1)
-    #    # Expect normal cert when connecting without ALPN.
-    #    self.assertEqual(jose.ComparableX509(cert),
-    #                     jose.ComparableX509(self.certs[b'localhost'][1]))
-
-    def test_challenge_certs(self):
-        host, port = self.server.socket.getsockname()[:2]
-        cert = crypto_util.probe_sni(
-            b'localhost', host=host, port=port, timeout=1,
-            alpn_protocols=[b"acme-tls/1"])
-        #  Expect challenge cert when connecting with ALPN.
-        self.assertEqual(
-                jose.ComparableX509(cert),
-                jose.ComparableX509(self.challenge_certs[b'localhost'][1])
-        )
-
-    def test_bad_alpn(self):
-        host, port = self.server.socket.getsockname()[:2]
-        with self.assertRaises(errors.Error):
-            crypto_util.probe_sni(
-                b'localhost', host=host, port=port, timeout=1,
-                alpn_protocols=[b"bad-alpn"])
-
 
 class BaseDualNetworkedServersTest(unittest.TestCase):
     """Test for acme.standalone.BaseDualNetworkedServers."""
@@ -218,6 +138,7 @@ class BaseDualNetworkedServersTest(unittest.TestCase):
 class HTTP01DualNetworkedServersTest(unittest.TestCase):
     """Tests for acme.standalone.HTTP01DualNetworkedServers."""
 
+
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))

acme/tests/testdata/README

@@ -4,14 +4,12 @@ to use appropriate extension for vector filenames: .pem for PEM and
 
 The following command has been used to generate test keys:
 
-  for k in 256 512 1024 2048 4096; do openssl genrsa -out rsa${k}_key.pem $k; done
+  for x in 256 512 1024 2048; do openssl genrsa -out rsa${k}_key.pem $k; done
 
 and for the CSR:
 
   openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -outform DER > csr.der
 
-and for the certificates:
+and for the certificate:
 
-  openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der
-  openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 > rsa2048_cert.pem
-  openssl req -key rsa1024_key.pem -new -subj '/CN=example.com' -x509 > rsa1024_cert.pem
+  openssl req -key rsa2047_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der

acme/tests/testdata/rsa1024_cert.pem

@@ -1,13 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIB/TCCAWagAwIBAgIJAOyRIBs3QT8QMA0GCSqGSIb3DQEBCwUAMBYxFDASBgNV
-BAMMC2V4YW1wbGUuY29tMB4XDTE4MDQyMzEwMzE0NFoXDTE4MDUyMzEwMzE0NFow
-FjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAJqJ87R8aVwByONxgQA9hwgvQd/QqI1r1UInXhEF2VnEtZGtUWLi100IpIqr
-Mq4qusDwNZ3g8cUPtSkvJGs89djoajMDIJP7lQUEKUYnYrI0q755Tr/DgLWSk7iW
-l5ezym0VzWUD0/xXUz8yRbNMTjTac80rS5SZk2ja2wWkYlRJAgMBAAGjUzBRMB0G
-A1UdDgQWBBSsaX0IVZ4XXwdeffVAbG7gnxSYjTAfBgNVHSMEGDAWgBSsaX0IVZ4X
-XwdeffVAbG7gnxSYjTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GB
-ADe7SVmvGH2nkwVfONk8TauRUDkePN1CJZKFb2zW1uO9ANJ2v5Arm/OQp0BG/xnI
-Djw/aLTNVESF89oe15dkrUErtcaF413MC1Ld5lTCaJLHLGqDKY69e02YwRuxW7jY
-qarpt7k7aR5FbcfO5r4V/FK/Gvp4Dmoky8uap7SJIW6x
------END CERTIFICATE-----

acme/tests/testdata/rsa4096_cert.pem

@@ -1,30 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIFDTCCAvWgAwIBAgIUImqDrP53V69vFROsjP/gL0YtoA4wDQYJKoZIhvcNAQEL
-BQAwFjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wHhcNMjAwNTI3MjMyNDE0WhcNMjAw
-NjI2MjMyNDE0WjAWMRQwEgYDVQQDDAtleGFtcGxlLmNvbTCCAiIwDQYJKoZIhvcN
-AQEBBQADggIPADCCAgoCggIBANY9LKLk9Dxn0MUMQFHwBoTN4ehDSWBws2KcytpF
-mc8m9Mfk1wmb4fQSKYtK3wIFMfIyo9HQu0nKqMkkUw52o3ZXyOv+oWwF5qNy2BKu
-lh5OMSkaZ0o13zoPpW42e+IUnyxvg70+0urD+sUue4cyTHh/nBIUjrM/05ZJ/ac8
-HR0RK3H41YoqBjq69JjMZczZZhbNFit3s6p0R1TbVAgc3ckqbtX5BDyQMQQCP4Ed
-m4DgbAFVqdcPUCC5W3F3fmuQiPKHiADzONZnXpy6lUvLDWqcd6loKp+nKHM6OkXX
-8hmD7pE1PYMQo4hqOfhBR2IgMjAShwd5qUFjl1m2oo0Qm3PFXOk6i2ZQdS6AA/yd
-B5/mX0RnM2oIdFZPb6UZFSmtEgs9sTzn+hMUyNSZQRE54px1ur1xws2R+vbsCyM5
-+KoFVxDjVjU9TlZx3GvDvnqz/tbHjji6l8VHZYOBMBUXbKHu2U6pJFZ5Zp7k68/z
-a3Fb9Pjtn3iRkXEyC0N5kLgqO4QTlExnxebV8aMvQpWd/qefnMn9qPYIZPEXSQAR
-mEBIahkcACb60s+acG0WFFluwBPtBqEr8Q67XlSF0Ibf4iBiRzpPobhlWta1nrFg
-4IWHMSoZ0PE75bhIGBEkhrpcXQCAxXmAfxfjKDH7jdJ1fRdnZ/9+OzwYGVX5GH/l
-0QDtAgMBAAGjUzBRMB0GA1UdDgQWBBQh3xiz/o1nEU2ySylZ9gxCXvIPGzAfBgNV
-HSMEGDAWgBQh3xiz/o1nEU2ySylZ9gxCXvIPGzAPBgNVHRMBAf8EBTADAQH/MA0G
-CSqGSIb3DQEBCwUAA4ICAQAELoXz31oR9pdAwidlv9ZBOKiC7KBWy8VMqXNVkfTn
-bVRxAUex7zleLFIOkWnqadsMesU9sIwrbLzBcZ8Q/vBY+z2xOPdXcgcAoAmdKWoq
-YBQNiqng9r54sqlzB/77QZCf5fdktESe7NTxhCifgx5SAWq7IUQs/lm3tnMUSAfE
-5ctuN6M+w8K54y3WDprcfMHpnc3ZHeSPhVQApHM0h/bDvXq0bRS7kmq27Hb153Qm
-nH3TwYB5pPSWW38NbUc+s/a7mItO7S8ly8yGbA0j9c/IbN5lM+OCdk06asz3+c8E
-uo8nuCBoYO5+6AqC2N7WJ3Tdr/pFA8jTbd6VNVlgCWTIR8ZosL5Fgkfv+4fUBrHt
-zdVUqMUzvga5rvZnwnJ5Qfu/drHeAAo9MTNFQNe2QgDlYfWBh5GweolgmFSwrpkY
-v/5wLtIyv/ASHKswybbqMIlpttcLTXjx5yuh8swttT6Wh+FQqqQ32KSRB3StiwyK
-oH0ZhrwYHiFYNlPxecGX6XUta6rFtTlEdkBGSnXzgiTzL2l+Nc0as0V5B9RninZG
-qJ+VOChSQ0OFvg1riSXv7tMvbLdGQnxwTRL3t6BMS8I4LA2m3ZfWUcuXT783ODTH
-16f1Q1AgXd2csstTWO9cv+N/0fpX31nqrm6+CrGduSr2u4HjYYnlLIUhmdTvK3fX
-Fg==
------END CERTIFICATE-----

acme/tests/testdata/rsa4096_key.pem

@@ -1,51 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIJKgIBAAKCAgEA1j0souT0PGfQxQxAUfAGhM3h6ENJYHCzYpzK2kWZzyb0x+TX
-CZvh9BIpi0rfAgUx8jKj0dC7ScqoySRTDnajdlfI6/6hbAXmo3LYEq6WHk4xKRpn
-SjXfOg+lbjZ74hSfLG+DvT7S6sP6xS57hzJMeH+cEhSOsz/Tlkn9pzwdHRErcfjV
-iioGOrr0mMxlzNlmFs0WK3ezqnRHVNtUCBzdySpu1fkEPJAxBAI/gR2bgOBsAVWp
-1w9QILlbcXd+a5CI8oeIAPM41mdenLqVS8sNapx3qWgqn6coczo6RdfyGYPukTU9
-gxCjiGo5+EFHYiAyMBKHB3mpQWOXWbaijRCbc8Vc6TqLZlB1LoAD/J0Hn+ZfRGcz
-agh0Vk9vpRkVKa0SCz2xPOf6ExTI1JlBETninHW6vXHCzZH69uwLIzn4qgVXEONW
-NT1OVnHca8O+erP+1seOOLqXxUdlg4EwFRdsoe7ZTqkkVnlmnuTrz/NrcVv0+O2f
-eJGRcTILQ3mQuCo7hBOUTGfF5tXxoy9ClZ3+p5+cyf2o9ghk8RdJABGYQEhqGRwA
-JvrSz5pwbRYUWW7AE+0GoSvxDrteVIXQht/iIGJHOk+huGVa1rWesWDghYcxKhnQ
-8TvluEgYESSGulxdAIDFeYB/F+MoMfuN0nV9F2dn/347PBgZVfkYf+XRAO0CAwEA
-AQKCAgEA0hZdTkQtCYtYm9LexDsXeWYX8VcCfrMmBj7xYcg9A3oVMmzDPuYBVwH0
-gWbjd6y2hOaJ5TfGYZ99kvmvBRDsTSHaoyopC7BhssjtAKz6Ay/0X3VH8usPQ3WS
-aZi+NT65tK6KRqtz08ppgLGLa1G00bl5x/Um1rpxeACI4FU/y4BJ1VMJvJpnT3KE
-Z86Qyagqx5NH+UpCApZSWPFX3zjHePzGgcfXErjniCHYOnpZQrFQ2KIzkfSvQ9fg
-x01ByKOM2CB2C1B33TCzBAioXRH6zyAu7A59NeCK9ywTduhDvie1a+oEryFC7IQW
-4s7I/H3MGX4hsf/pLXlHMy+5CZJOjRaC2h+pypfbbcuiXu6Sn64kHNpiI7SxI5DI
-MIRjyG7MdUcrzq0Rt8ogwwpbCoRqrl/w3bhxtqmeZaEZtyxbjlm7reK2YkIFDgyz
-JMqiJK5ZAi+9L/8c0xhjjAQQ0sIzrjmjA8U+6YnWL9jU5qXTVnBB8XQucyeeZGgk
-yRHyMur71qOXN8z3UEva7MHkDTUBlj8DgTz6sEjqCipaWl0CXfDNa4IhHIXD5qiF
-wplhq7OeS0v6EGG/UFa3Q/lFntxtrayxJX7uvvSccGzjPKXTjpWUELLi/FdnIsum
-eXT3RgIEYozj4BibDXaBLfHTCVzxOr7AAEvKM9XWSUgLA0paSWECggEBAO9ZBeE1
-GWzd1ejTTkcxBC9AK2rNsYG8PdNqiof/iTbuJWNeRqpG+KB/0CNIpjZ2X5xZd0tM
-FDpHTFehlP26Roxuq50iRAFc+SN5KoiO0A3JuJAidreIgRTia1saUUrypHqWrYEA
-VZVj2AI8Pyg3s1OkR2frFskY7hXBVb/pJNDP/m9xTXXIYiIXYkHYe+4RIJCnAxRv
-q5YHKaX+0Ull9YCZJCxmwvcHat8sgu8qkiwUMEM6QSNEkrEbdnWYBABvC1AR6sws
-7MP1h9+j22n4Zc/3D6kpFZEL9Erx8nNyhbOZ6q2Tdnf6YKVVjZdyVa8VyNnR0ROl
-3BjkFaHb/bg4e4kCggEBAOUk8ZJS3qBeGCOjug384zbHGcnhUBYtYJiOz+RXBtP+
-PRksbFtTkgk1sHuSGO8YRddU4Qv7Av1xL8o+DEsLBSD0YQ7pmLrR/LK+iDQ5N63O
-Fve9uJH0ybxAOkiua7G24+lTsVUP//KWToL4Wh5zbHBBjL5D2Z9zoeVbcE87xhva
-lImMVr4Ex252DqNP9wkZxBjudFyJ/C/TnXrjPcgwhxWTC7sLQMhE5p+490G7c4hX
-PywkIKrANbu37KDiAvVS+dC66ZgpL/NUDkeloAmGNO08LGzbV6YKchlvDyWU/AvW
-0hYjbL0FUq7K/wp1G9fumolB+fbI25K9c13X93STzUUCggEBAJDsNFUyk5yJjbYW
-C/WrRj9d+WwH9Az77+uNPSgvn+O0usq6EMuVgYGdImfa21lqv2Wp/kOHY1AOT7lX
-yyD+oyzw7dSNJOQ2aVwDR6+72Vof5DLRy1RBwPbmSd61xrc8yD658YCEtU1pUSe5
-VvyBDYH9nIbdn8RP5gkiMUusXXBaIFNWJXLFzDWcNxBrhk6V7EPp/EFphFmpKJyr
-+AkbRVWCZJbF+hMdWKadCwLJogwyhS6PnVU/dhrq6AU38GRa2Fy5HJRYN1xH1Oej
-DX3Su8L6c28Xw0k6FcczTHx+wVoIPkKvYTIwVkiFzt/+iMckx6KsGo5tBSHFKRwC
-WlQrTxECggEBALjUruLnY1oZ7AC7bTUhOimSOfQEgTQSUCtebsRxijlvhtsKYTDd
-XRt+qidStjgN7S/+8DRYuZWzOeg5WnMhpXZqiOudcyume922IGl3ibjxVsdoyjs5
-J4xohlrgDlBgBMDNWGoTqNGFejjcmNydH+gAh8VlN2INxJYbxqCyx17qVgwJHmLR
-uggYxD/pHYvCs9GkbknCp5/wYsOgDtKuihfV741lS1D/esN1UEQ+LrfYIEW7snno
-5q7Pcdhn1hkKYCWEzy2Ec4Aj2gzixQ9JqOF/OxpnZvCw1k47rg0TeqcWFYnz8x8Y
-7xO8/DH0OoxXk2GJzVXJuItJs4gLzzfCjL0CggEAJFHfC9jisdy7CoWiOpNCSF1B
-S0/CWDz77cZdlWkpTdaXGGp1MA/UKUFPIH8sOHfvpKS660+X4G/1ZBHmFb4P5kFF
-Qy8UyUMKtSOEdZS6KFlRlfSCAMd5aSTmCvq4OSjYEpMRwUhU/iEJNkn9Z1Soehe0
-U3dxJ8KiT1071geO6rRquSHoSJs6Y0WQKriYYQJOhh4Axs3PQihER2eyh+WGk8YJ
-02m0mMsjntqnXtdc6IcdKaHp9ko+OpM9QZLsvt19fxBcrXj/i21uUXrzuNtKfO6M
-JqGhsOrO2dh8lMhvodENvgKA0DmYDC9N7ogo7bxTNSedcjBF46FhJoqii8m70Q==
------END RSA PRIVATE KEY-----

certbot-apache/MANIFEST.in

@@ -1,7 +1,8 @@
 include LICENSE.txt
 include README.rst
 recursive-include tests *
+include certbot_apache/_internal/centos-options-ssl-apache.conf
+include certbot_apache/_internal/options-ssl-apache.conf
 recursive-include certbot_apache/_internal/augeas_lens *.aug
-recursive-include certbot_apache/_internal/tls_configs *.conf
 global-exclude __pycache__
 global-exclude *.py[cod]

certbot-apache/certbot_apache/_internal/apache_util.py

@@ -1,18 +1,102 @@
 """ Utility functions for certbot-apache plugin """
 import binascii
-import fnmatch
-import logging
-import re
-import subprocess
-
-import pkg_resources
+import hashlib
+import struct
+import shutil
+import time
 
+from certbot import crypto_util
 from certbot import errors
 from certbot import util
-
 from certbot.compat import os
 
-logger = logging.getLogger(__name__)
+
+def get_apache_ocsp_struct(ttl, ocsp_response):
+    """Create Apache OCSP response structure to be used in response cache
+
+    :param int ttl: Time-To-Live in seconds
+    :param str ocsp_response: OCSP response data
+
+    :returns: Apache OCSP structure
+    :rtype: `str`
+
+    """
+    ttl = time.time() + ttl
+    # As microseconds
+    ttl_struct = struct.pack('l', int(ttl*1000000))
+    return b'\x01'.join([ttl_struct, ocsp_response])
+
+
+def certid_sha1_hex(cert_path):
+    """Hex representation of certificate SHA1 fingerprint
+
+    :param str cert_path: File path to certificate
+
+    :returns: Hex representation SHA1 fingerprint of certificate
+    :rtype: `str`
+
+    """
+    sha1_hex = binascii.hexlify(certid_sha1(cert_path))
+    return sha1_hex.decode('utf-8')
+
+
+def certid_sha1(cert_path):
+    """SHA1 fingerprint of certificate
+
+    :param str cert_path: File path to certificate
+
+    :returns: SHA1 fingerprint bytestring
+    :rtype: `str`
+
+    """
+    return crypto_util.cert_sha1_fingerprint(cert_path)
+
+
+def safe_copy(source, target):
+    """Copies a file, while verifying the target integrity
+    with the source. Retries twice if the initial
+    copy fails.
+
+    :param str source: File path of the source file
+    :param str target: File path of the target file
+
+    :raises: .errors.PluginError: If file cannot be
+        copied or the target file hash does not match
+        with the source file.
+    """
+    for _ in range(3):
+        try:
+            shutil.copy2(source, target)
+        except IOError as e:
+            emsg = "Could not copy {} to {}: {}".format(
+                source, target, e
+            )
+            raise errors.PluginError(emsg)
+        try:
+            source_hash = _file_hash(source)
+            target_hash = _file_hash(target)
+        except IOError:
+            continue
+        if source_hash == target_hash:
+            return
+    raise errors.PluginError(
+        "Safe copy failed. The file integrity does not match"
+    )
+
+
+def _file_hash(filepath):
+    """Helper function for safe_copy that calculates a
+    sha-256 hash of file.
+
+    :param str filepath: Path of file to calculate hash for
+
+    :returns: File sha-256 hash
+    :rtype: str
+    """
+    fhash = hashlib.sha256()
+    with open(filepath, 'rb') as fh:
+        fhash.update(fh.read())
+    return fhash.hexdigest()
 
 
 def get_mod_deps(mod_name):
@@ -115,142 +199,3 @@ def parse_define_file(filepath, varname):
 def unique_id():
     """ Returns an unique id to be used as a VirtualHost identifier"""
     return binascii.hexlify(os.urandom(16)).decode("utf-8")
-
-
-def included_in_paths(filepath, paths):
-    """
-    Returns true if the filepath is included in the list of paths
-    that may contain full paths or wildcard paths that need to be
-    expanded.
-
-    :param str filepath: Filepath to check
-    :params list paths: List of paths to check against
-
-    :returns: True if included
-    :rtype: bool
-    """
-
-    return any(fnmatch.fnmatch(filepath, path) for path in paths)
-
-
-def parse_defines(apachectl):
-    """
-    Gets Defines from httpd process and returns a dictionary of
-    the defined variables.
-
-    :param str apachectl: Path to apachectl executable
-
-    :returns: dictionary of defined variables
-    :rtype: dict
-    """
-
-    variables = {}
-    define_cmd = [apachectl, "-t", "-D",
-                  "DUMP_RUN_CFG"]
-    matches = parse_from_subprocess(define_cmd, r"Define: ([^ \n]*)")
-    try:
-        matches.remove("DUMP_RUN_CFG")
-    except ValueError:
-        return {}
-
-    for match in matches:
-        if match.count("=") > 1:
-            logger.error("Unexpected number of equal signs in "
-                         "runtime config dump.")
-            raise errors.PluginError(
-                "Error parsing Apache runtime variables")
-        parts = match.partition("=")
-        variables[parts[0]] = parts[2]
-
-    return variables
-
-
-def parse_includes(apachectl):
-    """
-    Gets Include directives from httpd process and returns a list of
-    their values.
-
-    :param str apachectl: Path to apachectl executable
-
-    :returns: list of found Include directive values
-    :rtype: list of str
-    """
-
-    inc_cmd = [apachectl, "-t", "-D",
-               "DUMP_INCLUDES"]
-    return parse_from_subprocess(inc_cmd, r"\(.*\) (.*)")
-
-
-def parse_modules(apachectl):
-    """
-    Get loaded modules from httpd process, and return the list
-    of loaded module names.
-
-    :param str apachectl: Path to apachectl executable
-
-    :returns: list of found LoadModule module names
-    :rtype: list of str
-    """
-
-    mod_cmd = [apachectl, "-t", "-D",
-               "DUMP_MODULES"]
-    return parse_from_subprocess(mod_cmd, r"(.*)_module")
-
-
-def parse_from_subprocess(command, regexp):
-    """Get values from stdout of subprocess command
-
-    :param list command: Command to run
-    :param str regexp: Regexp for parsing
-
-    :returns: list parsed from command output
-    :rtype: list
-
-    """
-    stdout = _get_runtime_cfg(command)
-    return re.compile(regexp).findall(stdout)
-
-
-def _get_runtime_cfg(command):
-    """
-    Get runtime configuration info.
-
-    :param command: Command to run
-
-    :returns: stdout from command
-
-    """
-    try:
-        proc = subprocess.Popen(
-            command,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            universal_newlines=True)
-        stdout, stderr = proc.communicate()
-
-    except (OSError, ValueError):
-        logger.error(
-            "Error running command %s for runtime parameters!%s",
-            command, os.linesep)
-        raise errors.MisconfigurationError(
-            "Error accessing loaded Apache parameters: {0}".format(
-                command))
-    # Small errors that do not impede
-    if proc.returncode != 0:
-        logger.warning("Error in checking parameter list: %s", stderr)
-        raise errors.MisconfigurationError(
-            "Apache is unable to check whether or not the module is "
-            "loaded because Apache is misconfigured.")
-
-    return stdout
-
-def find_ssl_apache_conf(prefix):
-    """
-    Find a TLS Apache config file in the dedicated storage.
-    :param str prefix: prefix of the TLS Apache config file to find
-    :return: the path the TLS Apache config file
-    :rtype: str
-    """
-    return pkg_resources.resource_filename(
-        "certbot_apache",
-        os.path.join("_internal", "tls_configs", "{0}-options-ssl-apache.conf".format(prefix)))

certbot-apache/certbot_apache/_internal/apacheparser.py

@@ -1,172 +0,0 @@
-""" apacheconfig implementation of the ParserNode interfaces """
-
-from certbot_apache._internal import assertions
-from certbot_apache._internal import interfaces
-from certbot_apache._internal import parsernode_util as util
-
-
-class ApacheParserNode(interfaces.ParserNode):
-    """ apacheconfig implementation of ParserNode interface.
-
-        Expects metadata `ac_ast` to be passed in, where `ac_ast` is the AST provided
-        by parsing the equivalent configuration text using the apacheconfig library.
-    """
-
-    def __init__(self, **kwargs):
-        ancestor, dirty, filepath, metadata = util.parsernode_kwargs(kwargs)  # pylint: disable=unused-variable
-        super(ApacheParserNode, self).__init__(**kwargs)
-        self.ancestor = ancestor
-        self.filepath = filepath
-        self.dirty = dirty
-        self.metadata = metadata
-        self._raw = self.metadata["ac_ast"]
-
-    def save(self, msg): # pragma: no cover
-        pass
-
-    def find_ancestors(self, name):  # pylint: disable=unused-variable
-        """Find ancestor BlockNodes with a given name"""
-        return [ApacheBlockNode(name=assertions.PASS,
-                                parameters=assertions.PASS,
-                                ancestor=self,
-                                filepath=assertions.PASS,
-                                metadata=self.metadata)]
-
-
-class ApacheCommentNode(ApacheParserNode):
-    """ apacheconfig implementation of CommentNode interface """
-
-    def __init__(self, **kwargs):
-        comment, kwargs = util.commentnode_kwargs(kwargs)  # pylint: disable=unused-variable
-        super(ApacheCommentNode, self).__init__(**kwargs)
-        self.comment = comment
-
-    def __eq__(self, other):  # pragma: no cover
-        if isinstance(other, self.__class__):
-            return (self.comment == other.comment and
-                    self.dirty == other.dirty and
-                    self.ancestor == other.ancestor and
-                    self.metadata == other.metadata and
-                    self.filepath == other.filepath)
-        return False
-
-
-class ApacheDirectiveNode(ApacheParserNode):
-    """ apacheconfig implementation of DirectiveNode interface """
-
-    def __init__(self, **kwargs):
-        name, parameters, enabled, kwargs = util.directivenode_kwargs(kwargs)
-        super(ApacheDirectiveNode, self).__init__(**kwargs)
-        self.name = name
-        self.parameters = parameters
-        self.enabled = enabled
-        self.include = None
-
-    def __eq__(self, other):  # pragma: no cover
-        if isinstance(other, self.__class__):
-            return (self.name == other.name and
-                    self.filepath == other.filepath and
-                    self.parameters == other.parameters and
-                    self.enabled == other.enabled and
-                    self.dirty == other.dirty and
-                    self.ancestor == other.ancestor and
-                    self.metadata == other.metadata)
-        return False
-
-    def set_parameters(self, _parameters):  # pragma: no cover
-        """Sets the parameters for DirectiveNode"""
-        return
-
-
-class ApacheBlockNode(ApacheDirectiveNode):
-    """ apacheconfig implementation of BlockNode interface """
-
-    def __init__(self, **kwargs):
-        super(ApacheBlockNode, self).__init__(**kwargs)
-        self.children = ()
-
-    def __eq__(self, other):  # pragma: no cover
-        if isinstance(other, self.__class__):
-            return (self.name == other.name and
-                    self.filepath == other.filepath and
-                    self.parameters == other.parameters and
-                    self.children == other.children and
-                    self.enabled == other.enabled and
-                    self.dirty == other.dirty and
-                    self.ancestor == other.ancestor and
-                    self.metadata == other.metadata)
-        return False
-
-    # pylint: disable=unused-argument
-    def add_child_block(self, name, parameters=None, position=None):  # pragma: no cover
-        """Adds a new BlockNode to the sequence of children"""
-        new_block = ApacheBlockNode(name=assertions.PASS,
-                                    parameters=assertions.PASS,
-                                    ancestor=self,
-                                    filepath=assertions.PASS,
-                                    metadata=self.metadata)
-        self.children += (new_block,)
-        return new_block
-
-    # pylint: disable=unused-argument
-    def add_child_directive(self, name, parameters=None, position=None):  # pragma: no cover
-        """Adds a new DirectiveNode to the sequence of children"""
-        new_dir = ApacheDirectiveNode(name=assertions.PASS,
-                                      parameters=assertions.PASS,
-                                      ancestor=self,
-                                      filepath=assertions.PASS,
-                                      metadata=self.metadata)
-        self.children += (new_dir,)
-        return new_dir
-
-    # pylint: disable=unused-argument
-    def add_child_comment(self, comment="", position=None):  # pragma: no cover
-
-        """Adds a new CommentNode to the sequence of children"""
-        new_comment = ApacheCommentNode(comment=assertions.PASS,
-                                        ancestor=self,
-                                        filepath=assertions.PASS,
-                                        metadata=self.metadata)
-        self.children += (new_comment,)
-        return new_comment
-
-    def find_blocks(self, name, exclude=True): # pylint: disable=unused-argument
-        """Recursive search of BlockNodes from the sequence of children"""
-        return [ApacheBlockNode(name=assertions.PASS,
-                                parameters=assertions.PASS,
-                                ancestor=self,
-                                filepath=assertions.PASS,
-                                metadata=self.metadata)]
-
-    def find_directives(self, name, exclude=True): # pylint: disable=unused-argument
-        """Recursive search of DirectiveNodes from the sequence of children"""
-        return [ApacheDirectiveNode(name=assertions.PASS,
-                                    parameters=assertions.PASS,
-                                    ancestor=self,
-                                    filepath=assertions.PASS,
-                                    metadata=self.metadata)]
-
-    # pylint: disable=unused-argument
-    def find_comments(self, comment, exact=False):  # pragma: no cover
-        """Recursive search of DirectiveNodes from the sequence of children"""
-        return [ApacheCommentNode(comment=assertions.PASS,
-                                  ancestor=self,
-                                  filepath=assertions.PASS,
-                                  metadata=self.metadata)]
-
-    def delete_child(self, child):  # pragma: no cover
-        """Deletes a ParserNode from the sequence of children"""
-        return
-
-    def unsaved_files(self):  # pragma: no cover
-        """Returns a list of unsaved filepaths"""
-        return [assertions.PASS]
-
-    def parsed_paths(self):  # pragma: no cover
-        """Returns a list of parsed configuration file paths"""
-        return [assertions.PASS]
-
-
-interfaces.CommentNode.register(ApacheCommentNode)
-interfaces.DirectiveNode.register(ApacheDirectiveNode)
-interfaces.BlockNode.register(ApacheBlockNode)

certbot-apache/certbot_apache/_internal/assertions.py

@@ -1,142 +0,0 @@
-"""Dual parser node assertions"""
-import fnmatch
-
-from certbot_apache._internal import interfaces
-
-
-PASS = "CERTBOT_PASS_ASSERT"
-
-
-def assertEqual(first, second):
-    """ Equality assertion """
-
-    if isinstance(first, interfaces.CommentNode):
-        assertEqualComment(first, second)
-    elif isinstance(first, interfaces.DirectiveNode):
-        assertEqualDirective(first, second)
-
-    # Do an extra interface implementation assertion, as the contents were
-    # already checked for BlockNode in the assertEqualDirective
-    if isinstance(first, interfaces.BlockNode):
-        assert isinstance(second, interfaces.BlockNode)
-
-    # Skip tests if filepath includes the pass value. This is done
-    # because filepath is variable of the base ParserNode interface, and
-    # unless the implementation is actually done, we cannot assume getting
-    # correct results from boolean assertion for dirty
-    if not isPass(first.filepath) and not isPass(second.filepath):
-        assert first.dirty == second.dirty
-        # We might want to disable this later if testing with two separate
-        # (but identical) directory structures.
-        assert first.filepath == second.filepath
-
-def assertEqualComment(first, second): # pragma: no cover
-    """ Equality assertion for CommentNode """
-
-    assert isinstance(first, interfaces.CommentNode)
-    assert isinstance(second, interfaces.CommentNode)
-
-    if not isPass(first.comment) and not isPass(second.comment):  # type: ignore
-        assert first.comment == second.comment  # type: ignore
-
-def _assertEqualDirectiveComponents(first, second): # pragma: no cover
-    """ Handles assertion for instance variables for DirectiveNode and BlockNode"""
-
-    # Enabled value cannot be asserted, because Augeas implementation
-    # is unable to figure that out.
-    # assert first.enabled == second.enabled
-    if not isPass(first.name) and not isPass(second.name):
-        assert first.name == second.name
-
-    if not isPass(first.parameters) and not isPass(second.parameters):
-        assert first.parameters == second.parameters
-
-def assertEqualDirective(first, second):
-    """ Equality assertion for DirectiveNode """
-
-    assert isinstance(first, interfaces.DirectiveNode)
-    assert isinstance(second, interfaces.DirectiveNode)
-    _assertEqualDirectiveComponents(first, second)
-
-def isPass(value): # pragma: no cover
-    """Checks if the value is set to PASS"""
-    if isinstance(value, bool):
-        return True
-    return PASS in value
-
-def isPassDirective(block):
-    """ Checks if BlockNode or DirectiveNode should pass the assertion """
-
-    if isPass(block.name):
-        return True
-    if isPass(block.parameters): # pragma: no cover
-        return True
-    if isPass(block.filepath): # pragma: no cover
-        return True
-    return False
-
-def isPassComment(comment):
-    """ Checks if CommentNode should pass the assertion """
-
-    if isPass(comment.comment):
-        return True
-    if isPass(comment.filepath): # pragma: no cover
-        return True
-    return False
-
-def isPassNodeList(nodelist): # pragma: no cover
-    """ Checks if a ParserNode in the nodelist should pass the assertion,
-    this function is used for results of find_* methods. Unimplemented find_*
-    methods should return a sequence containing a single ParserNode instance
-    with assertion pass string."""
-
-    try:
-        node = nodelist[0]
-    except IndexError:
-        node = None
-
-    if not node: # pragma: no cover
-        return False
-
-    if isinstance(node, interfaces.DirectiveNode):
-        return isPassDirective(node)
-    return isPassComment(node)
-
-def assertEqualSimple(first, second):
-    """ Simple assertion """
-    if not isPass(first) and not isPass(second):
-        assert first == second
-
-def isEqualVirtualHost(first, second):
-    """
-    Checks that two VirtualHost objects are similar. There are some built
-    in differences with the implementations: VirtualHost created by ParserNode
-    implementation doesn't have "path" defined, as it was used for Augeas path
-    and that cannot obviously be used in the future. Similarly the legacy
-    version lacks "node" variable, that has a reference to the BlockNode for the
-    VirtualHost.
-    """
-    return (
-        first.name == second.name and
-        first.aliases == second.aliases and
-        first.filep == second.filep and
-        first.addrs == second.addrs and
-        first.ssl == second.ssl and
-        first.enabled == second.enabled and
-        first.modmacro == second.modmacro and
-        first.ancestor == second.ancestor
-    )
-
-def assertEqualPathsList(first, second):  # pragma: no cover
-    """
-    Checks that the two lists of file paths match. This assertion allows for wildcard
-    paths.
-    """
-    if any(isPass(path) for path in first):
-        return
-    if any(isPass(path) for path in second):
-        return
-    for fpath in first:
-        assert any([fnmatch.fnmatch(fpath, spath) for spath in second])
-    for spath in second:
-        assert any([fnmatch.fnmatch(fpath, spath) for fpath in first])

certbot-apache/certbot_apache/_internal/augeasparser.py

@@ -1,538 +0,0 @@
-"""
-Augeas implementation of the ParserNode interfaces.
-
-Augeas works internally by using XPATH notation. The following is a short example
-of how this all works internally, to better understand what's going on under the
-hood.
-
-A configuration file /etc/apache2/apache2.conf with the following content:
-
-    # First comment line
-    # Second comment line
-    WhateverDirective whatevervalue
-    <ABlock>
-        DirectiveInABlock dirvalue
-    </ABlock>
-    SomeDirective somedirectivevalue
-    <ABlock>
-        AnotherDirectiveInABlock dirvalue
-    </ABlock>
-    # Yet another comment
-
-
-Translates over to Augeas path notation (of immediate children), when calling
-for example: aug.match("/files/etc/apache2/apache2.conf/*")
-
-[
-    "/files/etc/apache2/apache2.conf/#comment[1]",
-    "/files/etc/apache2/apache2.conf/#comment[2]",
-    "/files/etc/apache2/apache2.conf/directive[1]",
-    "/files/etc/apache2/apache2.conf/ABlock[1]",
-    "/files/etc/apache2/apache2.conf/directive[2]",
-    "/files/etc/apache2/apache2.conf/ABlock[2]",
-    "/files/etc/apache2/apache2.conf/#comment[3]"
-]
-
-Regardless of directives name, its key in the Augeas tree is always "directive",
-with index where needed of course. Comments work similarly, while blocks
-have their own key in the Augeas XPATH notation.
-
-It's important to note that all of the unique keys have their own indices.
-
-Augeas paths are case sensitive, while Apache configuration is case insensitive.
-It looks like this:
-
-    <block>
-        directive value
-    </block>
-    <Block>
-        Directive Value
-    </Block>
-    <block>
-        directive value
-    </block>
-    <bLoCk>
-        DiReCtiVe VaLuE
-    </bLoCk>
-
-Translates over to:
-
-[
-    "/files/etc/apache2/apache2.conf/block[1]",
-    "/files/etc/apache2/apache2.conf/Block[1]",
-    "/files/etc/apache2/apache2.conf/block[2]",
-    "/files/etc/apache2/apache2.conf/bLoCk[1]",
-]
-"""
-from acme.magic_typing import Set
-from certbot import errors
-from certbot.compat import os
-
-from certbot_apache._internal import apache_util
-from certbot_apache._internal import assertions
-from certbot_apache._internal import interfaces
-from certbot_apache._internal import parser
-from certbot_apache._internal import parsernode_util as util
-
-
-class AugeasParserNode(interfaces.ParserNode):
-    """ Augeas implementation of ParserNode interface """
-
-    def __init__(self, **kwargs):
-        ancestor, dirty, filepath, metadata = util.parsernode_kwargs(kwargs)  # pylint: disable=unused-variable
-        super(AugeasParserNode, self).__init__(**kwargs)
-        self.ancestor = ancestor
-        self.filepath = filepath
-        self.dirty = dirty
-        self.metadata = metadata
-        self.parser = self.metadata.get("augeasparser")
-        try:
-            if self.metadata["augeaspath"].endswith("/"):
-                raise errors.PluginError(
-                    "Augeas path: {} has a trailing slash".format(
-                        self.metadata["augeaspath"]
-                    )
-                )
-        except KeyError:
-            raise errors.PluginError("Augeas path is required")
-
-    def save(self, msg):
-        self.parser.save(msg)
-
-    def find_ancestors(self, name):
-        """
-        Searches for ancestor BlockNodes with a given name.
-
-        :param str name: Name of the BlockNode parent to search for
-
-        :returns: List of matching ancestor nodes.
-        :rtype: list of AugeasBlockNode
-        """
-
-        ancestors = []
-
-        parent = self.metadata["augeaspath"]
-        while True:
-            # Get the path of ancestor node
-            parent = parent.rpartition("/")[0]
-            # Root of the tree
-            if not parent or parent == "/files":
-                break
-            anc = self._create_blocknode(parent)
-            if anc.name.lower() == name.lower():
-                ancestors.append(anc)
-
-        return ancestors
-
-    def _create_blocknode(self, path):
-        """
-        Helper function to create a BlockNode from Augeas path. This is used by
-        AugeasParserNode.find_ancestors and AugeasBlockNode.
-        and AugeasBlockNode.find_blocks
-
-        """
-
-        name = self._aug_get_name(path)
-        metadata = {"augeasparser": self.parser, "augeaspath": path}
-
-        # Check if the file was included from the root config or initial state
-        enabled = self.parser.parsed_in_original(
-            apache_util.get_file_path(path)
-        )
-
-        return AugeasBlockNode(name=name,
-                               enabled=enabled,
-                               ancestor=assertions.PASS,
-                               filepath=apache_util.get_file_path(path),
-                               metadata=metadata)
-
-    def _aug_get_name(self, path):
-        """
-        Helper function to get name of a configuration block or variable from path.
-        """
-
-        # Remove the ending slash if any
-        if path[-1] == "/":  # pragma: no cover
-            path = path[:-1]
-
-        # Get the block name
-        name = path.split("/")[-1]
-
-        # remove [...], it's not allowed in Apache configuration and is used
-        # for indexing within Augeas
-        name = name.split("[")[0]
-        return name
-
-
-class AugeasCommentNode(AugeasParserNode):
-    """ Augeas implementation of CommentNode interface """
-
-    def __init__(self, **kwargs):
-        comment, kwargs = util.commentnode_kwargs(kwargs)  # pylint: disable=unused-variable
-        super(AugeasCommentNode, self).__init__(**kwargs)
-        # self.comment = comment
-        self.comment = comment
-
-    def __eq__(self, other):
-        if isinstance(other, self.__class__):
-            return (self.comment == other.comment and
-                    self.filepath == other.filepath and
-                    self.dirty == other.dirty and
-                    self.ancestor == other.ancestor and
-                    self.metadata == other.metadata)
-        return False
-
-
-class AugeasDirectiveNode(AugeasParserNode):
-    """ Augeas implementation of DirectiveNode interface """
-
-    def __init__(self, **kwargs):
-        name, parameters, enabled, kwargs = util.directivenode_kwargs(kwargs)
-        super(AugeasDirectiveNode, self).__init__(**kwargs)
-        self.name = name
-        self.enabled = enabled
-        if parameters:
-            self.set_parameters(parameters)
-
-    def __eq__(self, other):
-        if isinstance(other, self.__class__):
-            return (self.name == other.name and
-                    self.filepath == other.filepath and
-                    self.parameters == other.parameters and
-                    self.enabled == other.enabled and
-                    self.dirty == other.dirty and
-                    self.ancestor == other.ancestor and
-                    self.metadata == other.metadata)
-        return False
-
-    def set_parameters(self, parameters):
-        """
-        Sets parameters of a DirectiveNode or BlockNode object.
-
-        :param list parameters: List of all parameters for the node to set.
-        """
-        orig_params = self._aug_get_params(self.metadata["augeaspath"])
-
-        # Clear out old parameters
-        for _ in orig_params:
-            # When the first parameter is removed, the indices get updated
-            param_path = "{}/arg[1]".format(self.metadata["augeaspath"])
-            self.parser.aug.remove(param_path)
-        # Insert new ones
-        for pi, param in enumerate(parameters):
-            param_path = "{}/arg[{}]".format(self.metadata["augeaspath"], pi+1)
-            self.parser.aug.set(param_path, param)
-
-    @property
-    def parameters(self):
-        """
-        Fetches the parameters from Augeas tree, ensuring that the sequence always
-        represents the current state
-
-        :returns: Tuple of parameters for this DirectiveNode
-        :rtype: tuple:
-        """
-        return tuple(self._aug_get_params(self.metadata["augeaspath"]))
-
-    def _aug_get_params(self, path):
-        """Helper function to get parameters for DirectiveNodes and BlockNodes"""
-
-        arg_paths = self.parser.aug.match(path + "/arg")
-        return [self.parser.get_arg(apath) for apath in arg_paths]
-
-
-class AugeasBlockNode(AugeasDirectiveNode):
-    """ Augeas implementation of BlockNode interface """
-
-    def __init__(self, **kwargs):
-        super(AugeasBlockNode, self).__init__(**kwargs)
-        self.children = ()
-
-    def __eq__(self, other):
-        if isinstance(other, self.__class__):
-            return (self.name == other.name and
-                    self.filepath == other.filepath and
-                    self.parameters == other.parameters and
-                    self.children == other.children and
-                    self.enabled == other.enabled and
-                    self.dirty == other.dirty and
-                    self.ancestor == other.ancestor and
-                    self.metadata == other.metadata)
-        return False
-
-    # pylint: disable=unused-argument
-    def add_child_block(self, name, parameters=None, position=None):  # pragma: no cover
-        """Adds a new BlockNode to the sequence of children"""
-
-        insertpath, realpath, before = self._aug_resolve_child_position(
-            name,
-            position
-        )
-        new_metadata = {"augeasparser": self.parser, "augeaspath": realpath}
-
-        # Create the new block
-        self.parser.aug.insert(insertpath, name, before)
-        # Check if the file was included from the root config or initial state
-        enabled = self.parser.parsed_in_original(
-            apache_util.get_file_path(realpath)
-        )
-
-        # Parameters will be set at the initialization of the new object
-        new_block = AugeasBlockNode(name=name,
-                                    parameters=parameters,
-                                    enabled=enabled,
-                                    ancestor=assertions.PASS,
-                                    filepath=apache_util.get_file_path(realpath),
-                                    metadata=new_metadata)
-        return new_block
-
-    # pylint: disable=unused-argument
-    def add_child_directive(self, name, parameters=None, position=None):  # pragma: no cover
-        """Adds a new DirectiveNode to the sequence of children"""
-
-        if not parameters:
-            raise errors.PluginError("Directive requires parameters and none were set.")
-
-        insertpath, realpath, before = self._aug_resolve_child_position(
-            "directive",
-            position
-        )
-        new_metadata = {"augeasparser": self.parser, "augeaspath": realpath}
-
-        # Create the new directive
-        self.parser.aug.insert(insertpath, "directive", before)
-        # Set the directive key
-        self.parser.aug.set(realpath, name)
-        # Check if the file was included from the root config or initial state
-        enabled = self.parser.parsed_in_original(
-            apache_util.get_file_path(realpath)
-        )
-
-        new_dir = AugeasDirectiveNode(name=name,
-                                      parameters=parameters,
-                                      enabled=enabled,
-                                      ancestor=assertions.PASS,
-                                      filepath=apache_util.get_file_path(realpath),
-                                      metadata=new_metadata)
-        return new_dir
-
-    def add_child_comment(self, comment="", position=None):
-        """Adds a new CommentNode to the sequence of children"""
-
-        insertpath, realpath, before = self._aug_resolve_child_position(
-            "#comment",
-            position
-        )
-        new_metadata = {"augeasparser": self.parser, "augeaspath": realpath}
-
-        # Create the new comment
-        self.parser.aug.insert(insertpath, "#comment", before)
-        # Set the comment content
-        self.parser.aug.set(realpath, comment)
-
-        new_comment = AugeasCommentNode(comment=comment,
-                                        ancestor=assertions.PASS,
-                                        filepath=apache_util.get_file_path(realpath),
-                                        metadata=new_metadata)
-        return new_comment
-
-    def find_blocks(self, name, exclude=True):
-        """Recursive search of BlockNodes from the sequence of children"""
-
-        nodes = []
-        paths = self._aug_find_blocks(name)
-        if exclude:
-            paths = self.parser.exclude_dirs(paths)
-        for path in paths:
-            nodes.append(self._create_blocknode(path))
-
-        return nodes
-
-    def find_directives(self, name, exclude=True):
-        """Recursive search of DirectiveNodes from the sequence of children"""
-
-        nodes = []
-        ownpath = self.metadata.get("augeaspath")
-
-        directives = self.parser.find_dir(name, start=ownpath, exclude=exclude)
-        already_parsed = set()  # type: Set[str]
-        for directive in directives:
-            # Remove the /arg part from the Augeas path
-            directive = directive.partition("/arg")[0]
-            # find_dir returns an object for each _parameter_ of a directive
-            # so we need to filter out duplicates.
-            if directive not in already_parsed:
-                nodes.append(self._create_directivenode(directive))
-                already_parsed.add(directive)
-
-        return nodes
-
-    def find_comments(self, comment):
-        """
-        Recursive search of DirectiveNodes from the sequence of children.
-
-        :param str comment: Comment content to search for.
-        """
-
-        nodes = []
-        ownpath = self.metadata.get("augeaspath")
-
-        comments = self.parser.find_comments(comment, start=ownpath)
-        for com in comments:
-            nodes.append(self._create_commentnode(com))
-
-        return nodes
-
-    def delete_child(self, child):
-        """
-        Deletes a ParserNode from the sequence of children, and raises an
-        exception if it's unable to do so.
-        :param AugeasParserNode: child: A node to delete.
-        """
-        if not self.parser.aug.remove(child.metadata["augeaspath"]):
-
-            raise errors.PluginError(
-                ("Could not delete child node, the Augeas path: {} doesn't " +
-                 "seem to exist.").format(child.metadata["augeaspath"])
-            )
-
-    def unsaved_files(self):
-        """Returns a list of unsaved filepaths"""
-        return self.parser.unsaved_files()
-
-    def parsed_paths(self):
-        """
-        Returns a list of file paths that have currently been parsed into the parser
-        tree. The returned list may include paths with wildcard characters, for
-        example: ['/etc/apache2/conf.d/*.load']
-
-        This is typically called on the root node of the ParserNode tree.
-
-        :returns: list of file paths of files that have been parsed
-        """
-
-        res_paths = []
-
-        paths = self.parser.existing_paths
-        for directory in paths:
-            for filename in paths[directory]:
-                res_paths.append(os.path.join(directory, filename))
-
-        return res_paths
-
-    def _create_commentnode(self, path):
-        """Helper function to create a CommentNode from Augeas path"""
-
-        comment = self.parser.aug.get(path)
-        metadata = {"augeasparser": self.parser, "augeaspath": path}
-
-        # Because of the dynamic nature of AugeasParser and the fact that we're
-        # not populating the complete node tree, the ancestor has a dummy value
-        return AugeasCommentNode(comment=comment,
-                                 ancestor=assertions.PASS,
-                                 filepath=apache_util.get_file_path(path),
-                                 metadata=metadata)
-
-    def _create_directivenode(self, path):
-        """Helper function to create a DirectiveNode from Augeas path"""
-
-        name = self.parser.get_arg(path)
-        metadata = {"augeasparser": self.parser, "augeaspath": path}
-
-        # Check if the file was included from the root config or initial state
-        enabled = self.parser.parsed_in_original(
-            apache_util.get_file_path(path)
-        )
-        return AugeasDirectiveNode(name=name,
-                                   ancestor=assertions.PASS,
-                                   enabled=enabled,
-                                   filepath=apache_util.get_file_path(path),
-                                   metadata=metadata)
-
-    def _aug_find_blocks(self, name):
-        """Helper function to perform a search to Augeas DOM tree to search
-        configuration blocks with a given name"""
-
-        # The code here is modified from configurator.get_virtual_hosts()
-        blk_paths = set()
-        for vhost_path in list(self.parser.parser_paths):
-            paths = self.parser.aug.match(
-                ("/files%s//*[label()=~regexp('%s')]" %
-                    (vhost_path, parser.case_i(name))))
-            blk_paths.update([path for path in paths if
-                              name.lower() in os.path.basename(path).lower()])
-        return blk_paths
-
-    def _aug_resolve_child_position(self, name, position):
-        """
-        Helper function that iterates through the immediate children and figures
-        out the insertion path for a new AugeasParserNode.
-
-        Augeas also generalizes indices for directives and comments, simply by
-        using "directive" or "comment" respectively as their names.
-
-        This function iterates over the existing children of the AugeasBlockNode,
-        returning their insertion path, resulting Augeas path and if the new node
-        should be inserted before or after the returned insertion path.
-
-        Note: while Apache is case insensitive, Augeas is not, and blocks like
-        Nameofablock and NameOfABlock have different indices.
-
-        :param str name: Name of the AugeasBlockNode to insert, "directive" for
-            AugeasDirectiveNode or "comment" for AugeasCommentNode
-        :param int position: The position to insert the child AugeasParserNode to
-
-        :returns: Tuple of insert path, resulting path and a boolean if the new
-            node should be inserted before it.
-        :rtype: tuple of str, str, bool
-        """
-
-        # Default to appending
-        before = False
-
-        all_children = self.parser.aug.match("{}/*".format(
-            self.metadata["augeaspath"])
-        )
-
-        # Calculate resulting_path
-        # Augeas indices start at 1. We use counter to calculate the index to
-        # be used in resulting_path.
-        counter = 1
-        for i, child in enumerate(all_children):
-            if position is not None and i >= position:
-                # We're not going to insert the new node to an index after this
-                break
-            childname = self._aug_get_name(child)
-            if name == childname:
-                counter += 1
-
-        resulting_path = "{}/{}[{}]".format(
-            self.metadata["augeaspath"],
-            name,
-            counter
-        )
-
-        # Form the correct insert_path
-        # Inserting the only child and appending as the last child work
-        # similarly in Augeas.
-        append = not all_children or position is None or position >= len(all_children)
-        if append:
-            insert_path = "{}/*[last()]".format(
-                self.metadata["augeaspath"]
-            )
-        elif position == 0:
-            # Insert as the first child, before the current first one.
-            insert_path = all_children[0]
-            before = True
-        else:
-            insert_path = "{}/*[{}]".format(
-                self.metadata["augeaspath"],
-                position
-            )
-
-        return (insert_path, resulting_path, before)
-
-
-interfaces.CommentNode.register(AugeasCommentNode)
-interfaces.DirectiveNode.register(AugeasDirectiveNode)
-interfaces.BlockNode.register(AugeasBlockNode)

certbot-apache/certbot_apache/_internal/centos-options-ssl-apache.conf

@@ -0,0 +1,25 @@
+# This file contains important security parameters. If you modify this file
+# manually, Certbot will be unable to automatically provide future security
+# updates. Instead, Certbot will print and log an error message with a path to
+# the up-to-date file that you will need to refer to when manually updating
+# this file.
+
+SSLEngine on
+
+# Intermediate configuration, tweak to your needs
+SSLProtocol             all -SSLv2 -SSLv3
+SSLCipherSuite          ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
+SSLHonorCipherOrder     on
+
+SSLOptions +StrictRequire
+
+# Add vhost name to log entries:
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" vhost_combined
+LogFormat "%v %h %l %u %t \"%r\" %>s %b" vhost_common
+
+#CustomLog /var/log/apache2/access.log vhost_combined
+#LogLevel warn
+#ErrorLog /var/log/apache2/error.log
+
+# Always ensure Cookies have "Secure" set (JAH 2012/1)
+#Header edit Set-Cookie (?i)^(.*)(;\s*secure)??((\s*;)?(.*)) "$1; Secure$3$4"

certbot-apache/certbot_apache/_internal/configurator.py

@@ -1,7 +1,6 @@
 """Apache Configurator."""
 # pylint: disable=too-many-lines
 from collections import defaultdict
-from distutils.version import LooseVersion
 import copy
 import fnmatch
 import logging
@@ -9,24 +8,21 @@ import re
 import socket
 import time
 
+import pkg_resources
 import six
 import zope.component
 import zope.interface
-try:
-    import apacheconfig
-    HAS_APACHECONFIG = True
-except ImportError:  # pragma: no cover
-    HAS_APACHECONFIG = False
 
 from acme import challenges
-from acme.magic_typing import DefaultDict
-from acme.magic_typing import Dict
-from acme.magic_typing import List
-from acme.magic_typing import Set
-from acme.magic_typing import Union
+from acme.magic_typing import DefaultDict  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Dict  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Union  # pylint: disable=unused-import, no-name-in-module
 from certbot import errors
 from certbot import interfaces
 from certbot import util
+
 from certbot.achallenges import KeyAuthorizationAnnotatedChallenge  # pylint: disable=unused-import
 from certbot.compat import filesystem
 from certbot.compat import os
@@ -34,10 +30,8 @@ from certbot.plugins import common
 from certbot.plugins.enhancements import AutoHSTSEnhancement
 from certbot.plugins.util import path_surgery
 from certbot_apache._internal import apache_util
-from certbot_apache._internal import assertions
 from certbot_apache._internal import constants
 from certbot_apache._internal import display_ops
-from certbot_apache._internal import dualparser
 from certbot_apache._internal import http_01
 from certbot_apache._internal import obj
 from certbot_apache._internal import parser
@@ -115,30 +109,14 @@ class ApacheConfigurator(common.Installer):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2",
-        bin=None
+        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
+            "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf"))
     )
 
     def option(self, key):
         """Get a value from options"""
         return self.options.get(key)
 
-    def pick_apache_config(self, warn_on_no_mod_ssl=True):
-        """
-        Pick the appropriate TLS Apache configuration file for current version of Apache and OS.
-
-        :param bool warn_on_no_mod_ssl: True if we should warn if mod_ssl is not found.
-
-        :return: the path to the TLS Apache configuration file to use
-        :rtype: str
-        """
-        # Disabling TLS session tickets is supported by Apache 2.4.11+ and OpenSSL 1.0.2l+.
-        # So for old versions of Apache we pick a configuration without this option.
-        openssl_version = self.openssl_version(warn_on_no_mod_ssl)
-        if self.version < (2, 4, 11) or not openssl_version or\
-            LooseVersion(openssl_version) < LooseVersion('1.0.2l'):
-            return apache_util.find_ssl_apache_conf("old")
-        return apache_util.find_ssl_apache_conf("current")
-
     def _prepare_options(self):
         """
         Set the values possibly changed by command line parameters to
@@ -146,7 +124,7 @@ class ApacheConfigurator(common.Installer):
         """
         opts = ["enmod", "dismod", "le_vhost_ext", "server_root", "vhost_root",
                 "logs_root", "challenge_location", "handle_modules", "handle_sites",
-                "ctl", "bin"]
+                "ctl"]
         for o in opts:
             # Config options use dashes instead of underscores
             if self.conf(o.replace("_", "-")) is not None:
@@ -195,8 +173,6 @@ class ApacheConfigurator(common.Installer):
                  "(Only Ubuntu/Debian currently)")
         add("ctl", default=DEFAULTS["ctl"],
             help="Full path to Apache control script")
-        add("bin", default=DEFAULTS["bin"],
-            help="Full path to apache2/httpd binary")
 
     def __init__(self, *args, **kwargs):
         """Initialize an Apache Configurator.
@@ -206,34 +182,26 @@ class ApacheConfigurator(common.Installer):
 
         """
         version = kwargs.pop("version", None)
-        use_parsernode = kwargs.pop("use_parsernode", False)
-        openssl_version = kwargs.pop("openssl_version", None)
         super(ApacheConfigurator, self).__init__(*args, **kwargs)
 
         # Add name_server association dict
-        self.assoc = {}  # type: Dict[str, obj.VirtualHost]
+        self.assoc = dict()  # type: Dict[str, obj.VirtualHost]
         # Outstanding challenges
         self._chall_out = set()  # type: Set[KeyAuthorizationAnnotatedChallenge]
         # List of vhosts configured per wildcard domain on this run.
         # used by deploy_cert() and enhance()
-        self._wildcard_vhosts = {}  # type: Dict[str, List[obj.VirtualHost]]
+        self._wildcard_vhosts = dict()  # type: Dict[str, List[obj.VirtualHost]]
         # Maps enhancements to vhosts we've enabled the enhancement for
         self._enhanced_vhosts = defaultdict(set)  # type: DefaultDict[str, Set[obj.VirtualHost]]
         # Temporary state for AutoHSTS enhancement
         self._autohsts = {}  # type: Dict[str, Dict[str, Union[int, float]]]
         # Reverter save notes
         self.save_notes = ""
-        # Should we use ParserNode implementation instead of the old behavior
-        self.USE_PARSERNODE = use_parsernode
-        # Saves the list of file paths that were parsed initially, and
-        # not added to parser tree by self.conf("vhost-root") for example.
-        self.parsed_paths = []  # type: List[str]
+
         # These will be set in the prepare function
         self._prepared = False
         self.parser = None
-        self.parser_root = None
         self.version = version
-        self._openssl_version = openssl_version
         self.vhosts = None
         self.options = copy.deepcopy(self.OS_DEFAULTS)
         self._enhance_func = {"redirect": self._enable_redirect,
@@ -250,59 +218,6 @@ class ApacheConfigurator(common.Installer):
         """Full absolute path to digest of updated SSL configuration file."""
         return os.path.join(self.config.config_dir, constants.UPDATED_MOD_SSL_CONF_DIGEST)
 
-    def _open_module_file(self, ssl_module_location):
-        """Extract the open lines of openssl_version for testing purposes"""
-        try:
-            with open(ssl_module_location, mode="rb") as f:
-                contents = f.read()
-        except IOError as error:
-            logger.debug(str(error), exc_info=True)
-            return None
-        return contents
-
-    def openssl_version(self, warn_on_no_mod_ssl=True):
-        """Lazily retrieve openssl version
-
-        :param bool warn_on_no_mod_ssl: `True` if we should warn if mod_ssl is not found. Set to
-            `False` when we know we'll try to enable mod_ssl later. This is currently debian/ubuntu,
-            when called from `prepare`.
-
-        :return: the OpenSSL version as a string, or None.
-        :rtype: str or None
-        """
-        if self._openssl_version:
-            return self._openssl_version
-        # Step 1. Determine the location of ssl_module
-        try:
-            ssl_module_location = self.parser.modules['ssl_module']
-        except KeyError:
-            if warn_on_no_mod_ssl:
-                logger.warning("Could not find ssl_module; not disabling session tickets.")
-            return None
-        if ssl_module_location:
-            # Possibility A: ssl_module is a DSO
-            ssl_module_location = self.parser.standard_path_from_server_root(ssl_module_location)
-        else:
-            # Possibility B: ssl_module is statically linked into Apache
-            if self.option("bin"):
-                ssl_module_location = self.option("bin")
-            else:
-                logger.warning("ssl_module is statically linked but --apache-bin is "
-                               "missing; not disabling session tickets.")
-                return None
-        # Step 2. Grep in the binary for openssl version
-        contents = self._open_module_file(ssl_module_location)
-        if not contents:
-            logger.warning("Unable to read ssl_module file; not disabling session tickets.")
-            return None
-        # looks like: OpenSSL 1.0.2s  28 May 2019
-        matches = re.findall(br"OpenSSL ([0-9]\.[^ ]+) ", contents)
-        if not matches:
-            logger.warning("Could not find OpenSSL version; not disabling session tickets.")
-            return None
-        self._openssl_version = matches[0].decode('UTF-8')
-        return self._openssl_version
-
     def prepare(self):
         """Prepare the authenticator/installer.
 
@@ -335,26 +250,14 @@ class ApacheConfigurator(common.Installer):
         # Perform the actual Augeas initialization to be able to react
         self.parser = self.get_parser()
 
-        # Set up ParserNode root
-        pn_meta = {"augeasparser": self.parser,
-                   "augeaspath": self.parser.get_root_augpath(),
-                   "ac_ast": None}
-        if self.USE_PARSERNODE:
-            self.parser_root = self.get_parsernode_root(pn_meta)
-            self.parsed_paths = self.parser_root.parsed_paths()
-
         # Check for errors in parsing files with Augeas
         self.parser.check_parsing_errors("httpd.aug")
 
         # Get all of the available vhosts
         self.vhosts = self.get_virtual_hosts()
 
-        # We may try to enable mod_ssl later. If so, we shouldn't warn if we can't find it now.
-        # This is currently only true for debian/ubuntu.
-        warn_on_no_mod_ssl = not self.option("handle_modules")
         self.install_ssl_options_conf(self.mod_ssl_conf,
-                                      self.updated_mod_ssl_conf_digest,
-                                      warn_on_no_mod_ssl)
+                                      self.updated_mod_ssl_conf_digest)
 
         # Prevent two Apache plugins from modifying a config at once
         try:
@@ -442,28 +345,6 @@ class ApacheConfigurator(common.Installer):
             self.option("server_root"), self.conf("vhost-root"),
             self.version, configurator=self)
 
-    def get_parsernode_root(self, metadata):
-        """Initializes the ParserNode parser root instance."""
-
-        if HAS_APACHECONFIG:
-            apache_vars = {}
-            apache_vars["defines"] = apache_util.parse_defines(self.option("ctl"))
-            apache_vars["includes"] = apache_util.parse_includes(self.option("ctl"))
-            apache_vars["modules"] = apache_util.parse_modules(self.option("ctl"))
-            metadata["apache_vars"] = apache_vars
-
-            with open(self.parser.loc["root"]) as f:
-                with apacheconfig.make_loader(writable=True,
-                      **apacheconfig.flavors.NATIVE_APACHE) as loader:
-                    metadata["ac_ast"] = loader.loads(f.read())
-
-        return dualparser.DualBlockNode(
-            name=assertions.PASS,
-            ancestor=None,
-            filepath=self.parser.loc["root"],
-            metadata=metadata
-        )
-
     def _wildcard_domain(self, domain):
         """
         Checks if domain is a wildcard domain
@@ -558,7 +439,7 @@ class ApacheConfigurator(common.Installer):
 
         # Go through the vhosts, making sure that we cover all the names
         # present, but preferring the SSL vhosts
-        filtered_vhosts = {}
+        filtered_vhosts = dict()
         for vhost in vhosts:
             for name in vhost.get_names():
                 if vhost.ssl:
@@ -584,7 +465,7 @@ class ApacheConfigurator(common.Installer):
 
         # Make sure we create SSL vhosts for the ones that are HTTP only
         # if requested.
-        return_vhosts = []
+        return_vhosts = list()
         for vhost in dialog_output:
             if not vhost.ssl:
                 return_vhosts.append(self.make_vhost_ssl(vhost))
@@ -605,11 +486,6 @@ class ApacheConfigurator(common.Installer):
         # cert_key... can all be parsed appropriately
         self.prepare_server_https("443")
 
-        # If we haven't managed to enable mod_ssl by this point, error out
-        if "ssl_module" not in self.parser.modules:
-            raise errors.MisconfigurationError("Could not find ssl_module; "
-                "not installing certificate.")
-
         # Add directives and remove duplicates
         self._add_dummy_ssl_directives(vhost.path)
         self._clean_vhost(vhost)
@@ -624,6 +500,21 @@ class ApacheConfigurator(common.Installer):
             path["chain_path"] = self.parser.find_dir(
                 "SSLCertificateChainFile", None, vhost.path)
 
+        # Handle errors when certificate/key directives cannot be found
+        if not path["cert_path"]:
+            logger.warning(
+                "Cannot find an SSLCertificateFile directive in %s. "
+                "VirtualHost was not modified", vhost.path)
+            raise errors.PluginError(
+                "Unable to find an SSLCertificateFile directive")
+        elif not path["cert_key"]:
+            logger.warning(
+                "Cannot find an SSLCertificateKeyFile directive for "
+                "certificate in %s. VirtualHost was not modified", vhost.path)
+            raise errors.PluginError(
+                "Unable to find an SSLCertificateKeyFile directive for "
+                "certificate")
+
         logger.info("Deploying Certificate to VirtualHost %s", vhost.filep)
 
         if self.version < (2, 4, 8) or (chain_path and not fullchain_path):
@@ -870,7 +761,7 @@ class ApacheConfigurator(common.Installer):
 
         return util.get_filtered_names(all_names)
 
-    def get_name_from_ip(self, addr):
+    def get_name_from_ip(self, addr):  # pylint: disable=no-self-use
         """Returns a reverse dns name if available.
 
         :param addr: IP Address
@@ -978,29 +869,6 @@ class ApacheConfigurator(common.Installer):
         return vhost
 
     def get_virtual_hosts(self):
-        """
-        Temporary wrapper for legacy and ParserNode version for
-        get_virtual_hosts. This should be replaced with the ParserNode
-        implementation when ready.
-        """
-
-        v1_vhosts = self.get_virtual_hosts_v1()
-        if self.USE_PARSERNODE and HAS_APACHECONFIG:
-            v2_vhosts = self.get_virtual_hosts_v2()
-
-            for v1_vh in v1_vhosts:
-                found = False
-                for v2_vh in v2_vhosts:
-                    if assertions.isEqualVirtualHost(v1_vh, v2_vh):
-                        found = True
-                        break
-                if not found:
-                    raise AssertionError("Equivalent for {} was not found".format(v1_vh.path))
-
-            return v2_vhosts
-        return v1_vhosts
-
-    def get_virtual_hosts_v1(self):
         """Returns list of virtual hosts found in the Apache configuration.
 
         :returns: List of :class:`~certbot_apache._internal.obj.VirtualHost`
@@ -1053,80 +921,6 @@ class ApacheConfigurator(common.Installer):
                     vhs.append(new_vhost)
         return vhs
 
-    def get_virtual_hosts_v2(self):
-        """Returns list of virtual hosts found in the Apache configuration using
-        ParserNode interface.
-        :returns: List of :class:`~certbot_apache.obj.VirtualHost`
-            objects found in configuration
-        :rtype: list
-        """
-
-        vhs = []
-        vhosts = self.parser_root.find_blocks("VirtualHost", exclude=False)
-        for vhblock in vhosts:
-            vhs.append(self._create_vhost_v2(vhblock))
-        return vhs
-
-    def _create_vhost_v2(self, node):
-        """Used by get_virtual_hosts_v2 to create vhost objects using ParserNode
-        interfaces.
-        :param interfaces.BlockNode node: The BlockNode object of VirtualHost block
-        :returns: newly created vhost
-        :rtype: :class:`~certbot_apache.obj.VirtualHost`
-        """
-        addrs = set()
-        for param in node.parameters:
-            addrs.add(obj.Addr.fromstring(param))
-
-        is_ssl = False
-        # Exclusion to match the behavior in get_virtual_hosts_v2
-        sslengine = node.find_directives("SSLEngine", exclude=False)
-        if sslengine:
-            for directive in sslengine:
-                if directive.parameters[0].lower() == "on":
-                    is_ssl = True
-                    break
-
-        # "SSLEngine on" might be set outside of <VirtualHost>
-        # Treat vhosts with port 443 as ssl vhosts
-        for addr in addrs:
-            if addr.get_port() == "443":
-                is_ssl = True
-
-        enabled = apache_util.included_in_paths(node.filepath, self.parsed_paths)
-
-        macro = False
-        # Check if the VirtualHost is contained in a mod_macro block
-        if node.find_ancestors("Macro"):
-            macro = True
-        vhost = obj.VirtualHost(
-            node.filepath, None, addrs, is_ssl, enabled, modmacro=macro, node=node
-        )
-        self._populate_vhost_names_v2(vhost)
-        return vhost
-
-    def _populate_vhost_names_v2(self, vhost):
-        """Helper function that populates the VirtualHost names.
-        :param host: In progress vhost whose names will be added
-        :type host: :class:`~certbot_apache.obj.VirtualHost`
-        """
-
-        servername_match = vhost.node.find_directives("ServerName",
-                                                      exclude=False)
-        serveralias_match = vhost.node.find_directives("ServerAlias",
-                                                       exclude=False)
-
-        servername = None
-        if servername_match:
-            servername = servername_match[-1].parameters[-1]
-
-        if not vhost.modmacro:
-            for alias in serveralias_match:
-                for serveralias in alias.parameters:
-                    vhost.aliases.add(serveralias)
-            vhost.name = servername
-
-
     def is_name_vhost(self, target_addr):
         """Returns if vhost is a name based vhost
 
@@ -1319,14 +1113,6 @@ class ApacheConfigurator(common.Installer):
                 self.enable_mod("socache_shmcb", temp=temp)
             if "ssl_module" not in self.parser.modules:
                 self.enable_mod("ssl", temp=temp)
-                # Make sure we're not throwing away any unwritten changes to the config
-                self.parser.ensure_augeas_state()
-                self.parser.aug.load()
-                self.parser.reset_modules() # Reset to load the new ssl_module path
-                # Call again because now we can gate on openssl version
-                self.install_ssl_options_conf(self.mod_ssl_conf,
-                                              self.updated_mod_ssl_conf_digest,
-                                              warn_on_no_mod_ssl=True)
 
     def make_vhost_ssl(self, nonssl_vhost):
         """Makes an ssl_vhost version of a nonssl_vhost.
@@ -1579,7 +1365,7 @@ class ApacheConfigurator(common.Installer):
                         result.append(comment)
                         sift = True
 
-                    result.append('\n'.join('# ' + l for l in chunk))
+                    result.append('\n'.join(['# ' + l for l in chunk]))
                 else:
                     result.append('\n'.join(chunk))
         return result, sift
@@ -1719,7 +1505,7 @@ class ApacheConfigurator(common.Installer):
         for addr in vhost.addrs:
             # In Apache 2.2, when a NameVirtualHost directive is not
             # set, "*" and "_default_" will conflict when sharing a port
-            addrs = {addr,}
+            addrs = set((addr,))
             if addr.get_addr() in ("*", "_default_"):
                 addrs.update(obj.Addr((a, addr.get_port(),))
                              for a in ("*", "_default_"))
@@ -1812,7 +1598,7 @@ class ApacheConfigurator(common.Installer):
     ######################################################################
     # Enhancements
     ######################################################################
-    def supported_enhancements(self):
+    def supported_enhancements(self):  # pylint: disable=no-self-use
         """Returns currently supported enhancements."""
         return ["redirect", "ensure-http-header", "staple-ocsp"]
 
@@ -1910,7 +1696,7 @@ class ApacheConfigurator(common.Installer):
         try:
             self._autohsts = self.storage.fetch("autohsts")
         except KeyError:
-            self._autohsts = {}
+            self._autohsts = dict()
 
     def _autohsts_save_state(self):
         """
@@ -1929,7 +1715,7 @@ class ApacheConfigurator(common.Installer):
             self.parser.find_dir("SSLCertificateKeyFile",
                                  lineage.key_path, vhost.path))
 
-    def _enable_ocsp_stapling(self, ssl_vhost, unused_options):
+    def _enable_ocsp_stapling(self, ssl_vhost, unused_options, prefetch=False):
         """Enables OCSP Stapling
 
         In OCSP, each client (e.g. browser) would have to query the
@@ -1950,6 +1736,9 @@ class ApacheConfigurator(common.Installer):
         :param unused_options: Not currently used
         :type unused_options: Not Available
 
+        :param prefetch: Use OCSP prefetching
+        :type prefetch: bool
+
         :returns: Success, general_vhost (HTTP vhost)
         :rtype: (bool, :class:`~certbot_apache._internal.obj.VirtualHost`)
 
@@ -1960,8 +1749,15 @@ class ApacheConfigurator(common.Installer):
                 "Unable to set OCSP directives.\n"
                 "Apache version is below 2.3.3.")
 
-        if "socache_shmcb_module" not in self.parser.modules:
-            self.enable_mod("socache_shmcb")
+        if prefetch:
+            if "socache_dbm_module" not in self.parser.modules:
+                self.enable_mod("socache_dbm")
+            cache_path = os.path.join(self.config.work_dir, "ocsp", "ocsp_cache.db")
+            cache_dir = ["dbm:"+cache_path]
+        else:
+            if "socache_shmcb_module" not in self.parser.modules:
+                self.enable_mod("socache_shmcb")
+            cache_dir = ["shmcb:/var/run/apache2/stapling_cache(128000)"]
 
         # Check if there's an existing SSLUseStapling directive on.
         use_stapling_aug_path = self.parser.find_dir("SSLUseStapling",
@@ -1981,9 +1777,7 @@ class ApacheConfigurator(common.Installer):
             self.parser.aug.remove(
                     re.sub(r"/\w*$", "", stapling_cache_aug_path[0]))
 
-        self.parser.add_dir_to_ifmodssl(ssl_vhost_aug_path,
-                "SSLStaplingCache",
-                ["shmcb:/var/run/apache2/stapling_cache(128000)"])
+        self.parser.add_dir_to_ifmodssl(ssl_vhost_aug_path, "SSLStaplingCache", cache_dir)
 
         msg = "OCSP Stapling was enabled on SSL Vhost: %s.\n"%(
                 ssl_vhost.filep)
@@ -2378,7 +2172,7 @@ class ApacheConfigurator(common.Installer):
             vhost.enabled = True
         return
 
-    def enable_mod(self, mod_name, temp=False):
+    def enable_mod(self, mod_name, temp=False):  # pylint: disable=unused-argument
         """Enables module in Apache.
 
         Both enables and reloads Apache so module is active.
@@ -2436,7 +2230,7 @@ class ApacheConfigurator(common.Installer):
                 error = str(err)
             raise errors.MisconfigurationError(error)
 
-    def config_test(self):
+    def config_test(self):  # pylint: disable=no-self-use
         """Check the configuration of Apache for errors.
 
         :raises .errors.MisconfigurationError: If config_test fails
@@ -2471,7 +2265,7 @@ class ApacheConfigurator(common.Installer):
         if len(matches) != 1:
             raise errors.PluginError("Unable to find Apache version")
 
-        return tuple(int(i) for i in matches[0].split("."))
+        return tuple([int(i) for i in matches[0].split(".")])
 
     def more_info(self):
         """Human-readable string to help understand the module"""
@@ -2486,7 +2280,7 @@ class ApacheConfigurator(common.Installer):
     ###########################################################################
     # Challenges Section
     ###########################################################################
-    def get_chall_pref(self, unused_domain):
+    def get_chall_pref(self, unused_domain):  # pylint: disable=no-self-use
         """Return list of challenge preferences."""
         return [challenges.HTTP01]
 
@@ -2540,19 +2334,14 @@ class ApacheConfigurator(common.Installer):
             self.restart()
             self.parser.reset_modules()
 
-    def install_ssl_options_conf(self, options_ssl, options_ssl_digest, warn_on_no_mod_ssl=True):
-        """Copy Certbot's SSL options file into the system's config dir if required.
-
-        :param bool warn_on_no_mod_ssl: True if we should warn if mod_ssl is not found.
-        """
+    def install_ssl_options_conf(self, options_ssl, options_ssl_digest):
+        """Copy Certbot's SSL options file into the system's config dir if required."""
 
         # XXX if we ever try to enforce a local privilege boundary (eg, running
         # certbot for unprivileged users via setuid), this function will need
         # to be modified.
-        apache_config_path = self.pick_apache_config(warn_on_no_mod_ssl)
-
-        return common.install_version_controlled_file(
-            options_ssl, options_ssl_digest, apache_config_path, constants.ALL_SSL_OPTIONS_HASHES)
+        return common.install_version_controlled_file(options_ssl, options_ssl_digest,
+            self.option("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES)
 
     def enable_autohsts(self, _unused_lineage, domains):
         """
@@ -2562,7 +2351,7 @@ class ApacheConfigurator(common.Installer):
         :type _unused_lineage: certbot._internal.storage.RenewableCert
 
         :param domains: List of domains in certificate to enhance
-        :type domains: `list` of `str`
+        :type domains: str
         """
 
         self._autohsts_fetch_state()

certbot-apache/certbot_apache/_internal/constants.py

@@ -24,9 +24,6 @@ ALL_SSL_OPTIONS_HASHES = [
     '0fcdc81280cd179a07ec4d29d3595068b9326b455c488de4b09f585d5dafc137',
     '86cc09ad5415cd6d5f09a947fe2501a9344328b1e8a8b458107ea903e80baa6c',
     '06675349e457eae856120cdebb564efe546f0b87399f2264baeb41e442c724c7',
-    '5cc003edd93fb9cd03d40c7686495f8f058f485f75b5e764b789245a386e6daf',
-    '007cd497a56a3bb8b6a2c1aeb4997789e7e38992f74e44cc5d13a625a738ac73',
-    '34783b9e2210f5c4a23bced2dfd7ec289834716673354ed7c7abf69fe30192a3',
 ]
 """SHA256 hashes of the contents of previous versions of all versions of MOD_SSL_CONF_SRC"""
 
@@ -70,3 +67,9 @@ AUTOHSTS_FREQ = 172800
 MANAGED_COMMENT = "DO NOT REMOVE - Managed by Certbot"
 MANAGED_COMMENT_ID = MANAGED_COMMENT+", VirtualHost id: {0}"
 """Managed by Certbot comments and the VirtualHost identification template"""
+
+OCSP_APACHE_TTL = 432000
+"""Apache TTL for OCSP response in seconds: 5 days"""
+
+OCSP_INTERNAL_TTL = 86400
+"""Internal TTL for OCSP response in seconds: 1 day"""

certbot-apache/certbot_apache/_internal/display_ops.py

@@ -21,7 +21,7 @@ def select_vhost_multiple(vhosts):
     :rtype: :class:`list`of type `~obj.Vhost`
     """
     if not vhosts:
-        return []
+        return list()
     tags_list = [vhost.display_repr()+"\n" for vhost in vhosts]
     # Remove the extra newline from the last entry
     if tags_list:
@@ -37,7 +37,7 @@ def select_vhost_multiple(vhosts):
 def _reversemap_vhosts(names, vhosts):
     """Helper function for select_vhost_multiple for mapping string
     representations back to actual vhost objects"""
-    return_vhosts = []
+    return_vhosts = list()
 
     for selection in names:
         for vhost in vhosts:

certbot-apache/certbot_apache/_internal/dualparser.py

@@ -1,306 +0,0 @@
-""" Dual ParserNode implementation """
-from certbot_apache._internal import assertions
-from certbot_apache._internal import augeasparser
-from certbot_apache._internal import apacheparser
-
-
-class DualNodeBase(object):
-    """ Dual parser interface for in development testing. This is used as the
-    base class for dual parser interface classes. This class handles runtime
-    attribute value assertions."""
-
-    def save(self, msg):  # pragma: no cover
-        """ Call save for both parsers """
-        self.primary.save(msg)
-        self.secondary.save(msg)
-
-    def __getattr__(self, aname):
-        """ Attribute value assertion """
-        firstval = getattr(self.primary, aname)
-        secondval = getattr(self.secondary, aname)
-        exclusions = [
-            # Metadata will inherently be different, as ApacheParserNode does
-            # not have Augeas paths and so on.
-            aname == "metadata",
-            callable(firstval)
-        ]
-        if not any(exclusions):
-            assertions.assertEqualSimple(firstval, secondval)
-        return firstval
-
-    def find_ancestors(self, name):
-        """ Traverses the ancestor tree and returns ancestors matching name """
-        return self._find_helper(DualBlockNode, "find_ancestors", name)
-
-    def _find_helper(self, nodeclass, findfunc, search, **kwargs):
-        """A helper for find_* functions. The function specific attributes should
-        be passed as keyword arguments.
-
-        :param interfaces.ParserNode nodeclass: The node class for results.
-        :param str findfunc: Name of the find function to call
-        :param str search: The search term
-        """
-
-        primary_res = getattr(self.primary, findfunc)(search, **kwargs)
-        secondary_res = getattr(self.secondary, findfunc)(search, **kwargs)
-
-        # The order of search results for Augeas implementation cannot be
-        # assured.
-
-        pass_primary = assertions.isPassNodeList(primary_res)
-        pass_secondary = assertions.isPassNodeList(secondary_res)
-        new_nodes = []
-
-        if pass_primary and pass_secondary:
-            # Both unimplemented
-            new_nodes.append(nodeclass(primary=primary_res[0],
-                                       secondary=secondary_res[0]))  # pragma: no cover
-        elif pass_primary:
-            for c in secondary_res:
-                new_nodes.append(nodeclass(primary=primary_res[0],
-                                           secondary=c))
-        elif pass_secondary:
-            for c in primary_res:
-                new_nodes.append(nodeclass(primary=c,
-                                           secondary=secondary_res[0]))
-        else:
-            assert len(primary_res) == len(secondary_res)
-            matches = self._create_matching_list(primary_res, secondary_res)
-            for p, s in matches:
-                new_nodes.append(nodeclass(primary=p, secondary=s))
-
-        return new_nodes
-
-
-class DualCommentNode(DualNodeBase):
-    """ Dual parser implementation of CommentNode interface """
-
-    def __init__(self, **kwargs):
-        """ This initialization implementation allows ordinary initialization
-        of CommentNode objects as well as creating a DualCommentNode object
-        using precreated or fetched CommentNode objects if provided as optional
-        arguments primary and secondary.
-
-        Parameters other than the following are from interfaces.CommentNode:
-
-        :param CommentNode primary: Primary pre-created CommentNode, mainly
-            used when creating new DualParser nodes using add_* methods.
-        :param CommentNode secondary: Secondary pre-created CommentNode
-        """
-
-        kwargs.setdefault("primary", None)
-        kwargs.setdefault("secondary", None)
-        primary = kwargs.pop("primary")
-        secondary = kwargs.pop("secondary")
-
-        if primary or secondary:
-            assert primary and secondary
-            self.primary = primary
-            self.secondary = secondary
-        else:
-            self.primary = augeasparser.AugeasCommentNode(**kwargs)
-            self.secondary = apacheparser.ApacheCommentNode(**kwargs)
-
-        assertions.assertEqual(self.primary, self.secondary)
-
-
-class DualDirectiveNode(DualNodeBase):
-    """ Dual parser implementation of DirectiveNode interface """
-
-    def __init__(self, **kwargs):
-        """ This initialization implementation allows ordinary initialization
-        of DirectiveNode objects as well as creating a DualDirectiveNode object
-        using precreated or fetched DirectiveNode objects if provided as optional
-        arguments primary and secondary.
-
-        Parameters other than the following are from interfaces.DirectiveNode:
-
-        :param DirectiveNode primary: Primary pre-created DirectiveNode, mainly
-            used when creating new DualParser nodes using add_* methods.
-        :param DirectiveNode secondary: Secondary pre-created DirectiveNode
-
-
-        """
-
-        kwargs.setdefault("primary", None)
-        kwargs.setdefault("secondary", None)
-        primary = kwargs.pop("primary")
-        secondary = kwargs.pop("secondary")
-
-        if primary or secondary:
-            assert primary and secondary
-            self.primary = primary
-            self.secondary = secondary
-        else:
-            self.primary = augeasparser.AugeasDirectiveNode(**kwargs)
-            self.secondary = apacheparser.ApacheDirectiveNode(**kwargs)
-
-        assertions.assertEqual(self.primary, self.secondary)
-
-    def set_parameters(self, parameters):
-        """ Sets parameters and asserts that both implementation successfully
-        set the parameter sequence """
-
-        self.primary.set_parameters(parameters)
-        self.secondary.set_parameters(parameters)
-        assertions.assertEqual(self.primary, self.secondary)
-
-
-class DualBlockNode(DualNodeBase):
-    """ Dual parser implementation of BlockNode interface """
-
-    def __init__(self, **kwargs):
-        """ This initialization implementation allows ordinary initialization
-        of BlockNode objects as well as creating a DualBlockNode object
-        using precreated or fetched BlockNode objects if provided as optional
-        arguments primary and secondary.
-
-        Parameters other than the following are from interfaces.BlockNode:
-
-        :param BlockNode primary: Primary pre-created BlockNode, mainly
-            used when creating new DualParser nodes using add_* methods.
-        :param BlockNode secondary: Secondary pre-created BlockNode
-        """
-
-        kwargs.setdefault("primary", None)
-        kwargs.setdefault("secondary", None)
-        primary = kwargs.pop("primary")
-        secondary = kwargs.pop("secondary")
-
-        if primary or secondary:
-            assert primary and secondary
-            self.primary = primary
-            self.secondary = secondary
-        else:
-            self.primary = augeasparser.AugeasBlockNode(**kwargs)
-            self.secondary = apacheparser.ApacheBlockNode(**kwargs)
-
-        assertions.assertEqual(self.primary, self.secondary)
-
-    def add_child_block(self, name, parameters=None, position=None):
-        """ Creates a new child BlockNode, asserts that both implementations
-        did it in a similar way, and returns a newly created DualBlockNode object
-        encapsulating both of the newly created objects """
-
-        primary_new = self.primary.add_child_block(name, parameters, position)
-        secondary_new = self.secondary.add_child_block(name, parameters, position)
-        assertions.assertEqual(primary_new, secondary_new)
-        new_block = DualBlockNode(primary=primary_new, secondary=secondary_new)
-        return new_block
-
-    def add_child_directive(self, name, parameters=None, position=None):
-        """ Creates a new child DirectiveNode, asserts that both implementations
-        did it in a similar way, and returns a newly created DualDirectiveNode
-        object encapsulating both of the newly created objects """
-
-        primary_new = self.primary.add_child_directive(name, parameters, position)
-        secondary_new = self.secondary.add_child_directive(name, parameters, position)
-        assertions.assertEqual(primary_new, secondary_new)
-        new_dir = DualDirectiveNode(primary=primary_new, secondary=secondary_new)
-        return new_dir
-
-    def add_child_comment(self, comment="", position=None):
-        """ Creates a new child CommentNode, asserts that both implementations
-        did it in a similar way, and returns a newly created DualCommentNode
-        object encapsulating both of the newly created objects """
-
-        primary_new = self.primary.add_child_comment(comment, position)
-        secondary_new = self.secondary.add_child_comment(comment, position)
-        assertions.assertEqual(primary_new, secondary_new)
-        new_comment = DualCommentNode(primary=primary_new, secondary=secondary_new)
-        return new_comment
-
-    def _create_matching_list(self, primary_list, secondary_list):
-        """ Matches the list of primary_list to a list of secondary_list and
-        returns a list of tuples. This is used to create results for find_
-        methods.
-
-        This helper function exists, because we cannot ensure that the list of
-        search results returned by primary.find_* and secondary.find_* are ordered
-        in a same way. The function pairs the same search results from both
-        implementations to a list of tuples.
-        """
-
-        matched = []
-        for p in primary_list:
-            match = None
-            for s in secondary_list:
-                try:
-                    assertions.assertEqual(p, s)
-                    match = s
-                    break
-                except AssertionError:
-                    continue
-            if match:
-                matched.append((p, match))
-            else:
-                raise AssertionError("Could not find a matching node.")
-        return matched
-
-    def find_blocks(self, name, exclude=True):
-        """
-        Performs a search for BlockNodes using both implementations and does simple
-        checks for results. This is built upon the assumption that unimplemented
-        find_* methods return a list with a single assertion passing object.
-        After the assertion, it creates a list of newly created DualBlockNode
-        instances that encapsulate the pairs of returned BlockNode objects.
-        """
-
-        return self._find_helper(DualBlockNode, "find_blocks", name,
-                                 exclude=exclude)
-
-    def find_directives(self, name, exclude=True):
-        """
-        Performs a search for DirectiveNodes using both implementations and
-        checks the results. This is built upon the assumption that unimplemented
-        find_* methods return a list with a single assertion passing object.
-        After the assertion, it creates a list of newly created DualDirectiveNode
-        instances that encapsulate the pairs of returned DirectiveNode objects.
-        """
-
-        return self._find_helper(DualDirectiveNode, "find_directives", name,
-                                 exclude=exclude)
-
-    def find_comments(self, comment):
-        """
-        Performs a search for CommentNodes using both implementations and
-        checks the results. This is built upon the assumption that unimplemented
-        find_* methods return a list with a single assertion passing object.
-        After the assertion, it creates a list of newly created DualCommentNode
-        instances that encapsulate the pairs of returned CommentNode objects.
-        """
-
-        return self._find_helper(DualCommentNode, "find_comments", comment)
-
-    def delete_child(self, child):
-        """Deletes a child from the ParserNode implementations. The actual
-        ParserNode implementations are used here directly in order to be able
-        to match a child to the list of children."""
-
-        self.primary.delete_child(child.primary)
-        self.secondary.delete_child(child.secondary)
-
-    def unsaved_files(self):
-        """ Fetches the list of unsaved file paths and asserts that the lists
-        match """
-        primary_files = self.primary.unsaved_files()
-        secondary_files = self.secondary.unsaved_files()
-        assertions.assertEqualSimple(primary_files, secondary_files)
-
-        return primary_files
-
-    def parsed_paths(self):
-        """
-        Returns a list of file paths that have currently been parsed into the parser
-        tree. The returned list may include paths with wildcard characters, for
-        example: ['/etc/apache2/conf.d/*.load']
-
-        This is typically called on the root node of the ParserNode tree.
-
-        :returns: list of file paths of files that have been parsed
-        """
-
-        primary_paths = self.primary.parsed_paths()
-        secondary_paths = self.secondary.parsed_paths()
-        assertions.assertEqualPathsList(primary_paths, secondary_paths)
-        return primary_paths

certbot-apache/certbot_apache/_internal/entrypoint.py

@@ -1,5 +1,7 @@
 """ Entry point for Apache Plugin """
-from distutils.version import LooseVersion
+# Pylint does not like disutils.version when running inside a venv.
+# See: https://github.com/PyCQA/pylint/issues/73
+from distutils.version import LooseVersion  # pylint: disable=no-name-in-module,import-error
 
 from certbot import util
 from certbot_apache._internal import configurator

certbot-apache/certbot_apache/_internal/http_01.py

@@ -1,9 +1,8 @@
 """A class that performs HTTP-01 challenges for Apache"""
 import logging
-import errno
 
-from acme.magic_typing import List
-from acme.magic_typing import Set
+from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
 from certbot import errors
 from certbot.compat import filesystem
 from certbot.compat import os
@@ -169,15 +168,7 @@ class ApacheHttp01(common.ChallengePerformer):
 
     def _set_up_challenges(self):
         if not os.path.isdir(self.challenge_dir):
-            old_umask = os.umask(0o022)
-            try:
-                filesystem.makedirs(self.challenge_dir, 0o755)
-            except OSError as exception:
-                if exception.errno not in (errno.EEXIST, errno.EISDIR):
-                    raise errors.PluginError(
-                        "Couldn't create root for http-01 challenge")
-            finally:
-                os.umask(old_umask)
+            filesystem.makedirs(self.challenge_dir, 0o755)
 
         responses = []
         for achall in self.achalls:

certbot-apache/certbot_apache/_internal/interfaces.py

@@ -1,515 +0,0 @@
-"""ParserNode interface for interacting with configuration tree.
-
-General description
--------------------
-
-The ParserNode interfaces are designed to be able to contain all the parsing logic,
-while allowing their users to interact with the configuration tree in a Pythonic
-and well structured manner.
-
-The structure allows easy traversal of the tree of ParserNodes. Each ParserNode
-stores a reference to its ancestor and immediate children, allowing the user to
-traverse the tree using built in interface methods as well as accessing the interface
-properties directly.
-
-ParserNode interface implementation should stand between the actual underlying
-parser functionality and the business logic within Configurator code, interfacing
-with both. The ParserNode tree is a result of configuration parsing action.
-
-ParserNode tree will be in charge of maintaining the parser state and hence the
-abstract syntax tree (AST). Interactions between ParserNode tree and underlying
-parser should involve only parsing the configuration files to this structure, and
-writing it back to the filesystem - while preserving the format including whitespaces.
-
-For some implementations (Apache for example) it's important to keep track of and
-to use state information while parsing conditional blocks and directives. This
-allows the implementation to set a flag to parts of the parsed configuration
-structure as not being in effect in a case of unmatched conditional block. It's
-important to store these blocks in the tree as well in order to not to conduct
-destructive actions (failing to write back parts of the configuration) while writing
-the AST back to the filesystem.
-
-The ParserNode tree is in charge of maintaining the its own structure while every
-child node fetched with find - methods or by iterating its list of children can be
-changed in place. When making changes the affected nodes should be flagged as "dirty"
-in order for the parser implementation to figure out the parts of the configuration
-that need to be written back to disk during the save() operation.
-
-
-Metadata
---------
-
-The metadata holds all the implementation specific attributes of the ParserNodes -
-things like the positional information related to the AST, file paths, whitespacing,
-and any other information relevant to the underlying parser engine.
-
-Access to the metadata should be handled by implementation specific methods, allowing
-the Configurator functionality to access the underlying information where needed.
-
-For some implementations the node can be initialized using the information carried
-in metadata alone. This is useful especially when populating the ParserNode tree
-while parsing the configuration.
-
-
-Apache implementation
----------------------
-
-The Apache implementation of ParserNode interface requires some implementation
-specific functionalities that are not described by the interface itself.
-
-Initialization
-
-When the user of a ParserNode class is creating these objects, they must specify
-the parameters as described in the documentation for the __init__ methods below.
-When these objects are created internally, however, some parameters may not be
-needed because (possibly more detailed) information is included in the metadata
-parameter. In this case, implementations can deviate from the required parameters
-from __init__, however, they should still behave the same when metadata is not
-provided.
-
-For consistency internally, if an argument is provided directly in the ParserNode
-initialization parameters as well as within metadata it's recommended to establish
-clear behavior around this scenario within the implementation.
-
-Conditional blocks
-
-Apache configuration can have conditional blocks, for example: <IfModule ...>,
-resulting the directives and subblocks within it being either enabled or disabled.
-While find_* interface methods allow including the disabled parts of the configuration
-tree in searches a special care needs to be taken while parsing the structure in
-order to reflect the active state of configuration.
-
-Whitespaces
-
-Each ParserNode object is responsible of storing its prepending whitespace characters
-in order to be able to write the AST back to filesystem like it was, preserving the
-format, this applies for parameters of BlockNode and DirectiveNode as well.
-When parameters of ParserNode are changed, the pre-existing whitespaces in the
-parameter sequence are discarded, as the general reason for storing them is to
-maintain the ability to write the configuration back to filesystem exactly like
-it was. This loses its meaning when we have to change the directives or blocks
-parameters for other reasons.
-
-Searches and matching
-
-Apache configuration is largely case insensitive, so the Apache implementation of
-ParserNode interface needs to provide the user means to match block and directive
-names and parameters in case insensitive manner. This does not apply to everything
-however, for example the parameters of a conditional statement may be case sensitive.
-For this reason the internal representation of data should not ignore the case.
-"""
-
-import abc
-import six
-
-
-
-@six.add_metaclass(abc.ABCMeta)
-class ParserNode(object):
-    """
-    ParserNode is the basic building block of the tree of such nodes,
-    representing the structure of the configuration. It is largely meant to keep
-    the structure information intact and idiomatically accessible.
-
-    The root node as well as the child nodes of it should be instances of ParserNode.
-    Nodes keep track of their differences to on-disk representation of configuration
-    by marking modified ParserNodes as dirty to enable partial write-to-disk for
-    different files in the configuration structure.
-
-    While for the most parts the usage and the child types are obvious, "include"-
-    and similar directives are an exception to this rule. This is because of the
-    nature of include directives - which unroll the contents of another file or
-    configuration block to their place. While we could unroll the included nodes
-    to the parent tree, it remains important to keep the context of include nodes
-    separate in order to write back the original configuration as it was.
-
-    For parsers that require the implementation to keep track of the whitespacing,
-    it's responsibility of each ParserNode object itself to store its prepending
-    whitespaces in order to be able to reconstruct the complete configuration file
-    as it was when originally read from the disk.
-
-    ParserNode objects should have the following attributes:
-
-    # Reference to ancestor node, or None if the node is the root node of the
-    # configuration tree.
-    ancestor: Optional[ParserNode]
-
-    # True if this node has been modified since last save.
-    dirty: bool
-
-    # Filepath of the file where the configuration element for this ParserNode
-    # object resides. For root node, the value for filepath is the httpd root
-    # configuration file. Filepath can be None if a configuration directive is
-    # defined in for example the httpd command line.
-    filepath: Optional[str]
-
-    # Metadata dictionary holds all the implementation specific key-value pairs
-    # for the ParserNode instance.
-    metadata: Dict[str, Any]
-    """
-
-    @abc.abstractmethod
-    def __init__(self, **kwargs):
-        """
-        Initializes the ParserNode instance, and sets the ParserNode specific
-        instance variables. This is not meant to be used directly, but through
-        specific classes implementing ParserNode interface.
-
-        :param ancestor: BlockNode ancestor for this CommentNode. Required.
-        :type ancestor: BlockNode or None
-
-        :param filepath: Filesystem path for the file where this CommentNode
-            does or should exist in the filesystem. Required.
-        :type filepath: str or None
-
-        :param dirty: Boolean flag for denoting if this CommentNode has been
-            created or changed after the last save. Default: False.
-        :type dirty: bool
-
-        :param metadata: Dictionary of metadata values for this ParserNode object.
-            Metadata information should be used only internally in the implementation.
-            Default: {}
-        :type metadata: dict
-        """
-
-    @abc.abstractmethod
-    def save(self, msg):
-        """
-        Save traverses the children, and attempts to write the AST to disk for
-        all the objects that are marked dirty. The actual operation of course
-        depends on the underlying implementation. save() shouldn't be called
-        from the Configurator outside of its designated save() method in order
-        to ensure that the Reverter checkpoints are created properly.
-
-        Note: this approach of keeping internal structure of the configuration
-        within the ParserNode tree does not represent the file inclusion structure
-        of actual configuration files that reside in the filesystem. To handle
-        file writes properly, the file specific temporary trees should be extracted
-        from the full ParserNode tree where necessary when writing to disk.
-
-        :param str msg: Message describing the reason for the save.
-
-        """
-
-    @abc.abstractmethod
-    def find_ancestors(self, name):
-        """
-        Traverses the ancestor tree up, searching for BlockNodes with a specific
-        name.
-
-        :param str name: Name of the ancestor BlockNode to search for
-
-        :returns: A list of ancestor BlockNodes that match the name
-        :rtype: list of BlockNode
-        """
-
-
-# Linter rule exclusion done because of https://github.com/PyCQA/pylint/issues/179
-@six.add_metaclass(abc.ABCMeta)  # pylint: disable=abstract-method
-class CommentNode(ParserNode):
-    """
-    CommentNode class is used for representation of comments within the parsed
-    configuration structure. Because of the nature of comments, it is not able
-    to have child nodes and hence it is always treated as a leaf node.
-
-    CommentNode stores its contents in class variable 'comment' and does not
-    have a specific name.
-
-    CommentNode objects should have the following attributes in addition to
-    the ones described in ParserNode:
-
-    # Contains the contents of the comment without the directive notation
-    # (typically # or /* ... */).
-    comment: str
-
-    """
-
-    @abc.abstractmethod
-    def __init__(self, **kwargs):
-        """
-        Initializes the CommentNode instance and sets its instance variables.
-
-        :param comment: Contents of the comment. Required.
-        :type comment: str
-
-        :param ancestor: BlockNode ancestor for this CommentNode. Required.
-        :type ancestor: BlockNode or None
-
-        :param filepath: Filesystem path for the file where this CommentNode
-            does or should exist in the filesystem. Required.
-        :type filepath: str or None
-
-        :param dirty: Boolean flag for denoting if this CommentNode has been
-            created or changed after the last save. Default: False.
-        :type dirty: bool
-        """
-        super(CommentNode, self).__init__(ancestor=kwargs['ancestor'],
-                                          dirty=kwargs.get('dirty', False),
-                                          filepath=kwargs['filepath'],
-                                          metadata=kwargs.get('metadata', {}))  # pragma: no cover
-
-
-@six.add_metaclass(abc.ABCMeta)
-class DirectiveNode(ParserNode):
-    """
-    DirectiveNode class represents a configuration directive within the configuration.
-    It can have zero or more parameters attached to it. Because of the nature of
-    single directives, it is not able to have child nodes and hence it is always
-    treated as a leaf node.
-
-    If a this directive was defined on the httpd command line, the ancestor instance
-    variable for this DirectiveNode should be None, and it should be inserted to the
-    beginning of root BlockNode children sequence.
-
-    DirectiveNode objects should have the following attributes in addition to
-    the ones described in ParserNode:
-
-    # True if this DirectiveNode is enabled and False if it is inside of an
-    # inactive conditional block.
-    enabled: bool
-
-    # Name, or key of the configuration directive. If BlockNode subclass of
-    # DirectiveNode is the root configuration node, the name should be None.
-    name: Optional[str]
-
-    # Tuple of parameters of this ParserNode object, excluding whitespaces.
-    parameters: Tuple[str, ...]
-
-    """
-
-    @abc.abstractmethod
-    def __init__(self, **kwargs):
-        """
-        Initializes the DirectiveNode instance and sets its instance variables.
-
-        :param name: Name or key of the DirectiveNode object. Required.
-        :type name: str or None
-
-        :param tuple parameters: Tuple of str parameters for this DirectiveNode.
-            Default: ().
-        :type parameters: tuple
-
-        :param ancestor: BlockNode ancestor for this DirectiveNode, or None for
-            root configuration node. Required.
-        :type ancestor: BlockNode or None
-
-        :param filepath: Filesystem path for the file where this DirectiveNode
-            does or should exist in the filesystem, or None for directives introduced
-            in the httpd command line. Required.
-        :type filepath: str or None
-
-        :param dirty: Boolean flag for denoting if this DirectiveNode has been
-            created or changed after the last save. Default: False.
-        :type dirty: bool
-
-        :param enabled: True if this DirectiveNode object is parsed in the active
-            configuration of the httpd. False if the DirectiveNode exists within a
-            unmatched conditional configuration block. Default: True.
-        :type enabled: bool
-
-        """
-        super(DirectiveNode, self).__init__(ancestor=kwargs['ancestor'],
-                                            dirty=kwargs.get('dirty', False),
-                                            filepath=kwargs['filepath'],
-                                            metadata=kwargs.get('metadata', {}))  # pragma: no cover
-
-    @abc.abstractmethod
-    def set_parameters(self, parameters):
-        """
-        Sets the sequence of parameters for this ParserNode object without
-        whitespaces. While the whitespaces for parameters are discarded when using
-        this method, the whitespacing preceeding the ParserNode itself should be
-        kept intact.
-
-        :param list parameters: sequence of parameters
-        """
-
-
-@six.add_metaclass(abc.ABCMeta)
-class BlockNode(DirectiveNode):
-    """
-    BlockNode class represents a block of nested configuration directives, comments
-    and other blocks as its children. A BlockNode can have zero or more parameters
-    attached to it.
-
-    Configuration blocks typically consist of one or more child nodes of all possible
-    types. Because of this, the BlockNode class has various discovery and structure
-    management methods.
-
-    Lists of parameters used as an optional argument for some of the methods should
-    be lists of strings that are applicable parameters for each specific BlockNode
-    or DirectiveNode type. As an example, for a following configuration example:
-
-        <VirtualHost *:80>
-           ...
-        </VirtualHost>
-
-    The node type would be BlockNode, name would be 'VirtualHost' and its parameters
-    would be: ['*:80'].
-
-    While for the following example:
-
-        LoadModule alias_module /usr/lib/apache2/modules/mod_alias.so
-
-    The node type would be DirectiveNode, name would be 'LoadModule' and its
-    parameters would be: ['alias_module', '/usr/lib/apache2/modules/mod_alias.so']
-
-    The applicable parameters are dependent on the underlying configuration language
-    and its grammar.
-
-    BlockNode objects should have the following attributes in addition to
-    the ones described in DirectiveNode:
-
-    # Tuple of direct children of this BlockNode object. The order of children
-    # in this tuple retain the order of elements in the parsed configuration
-    # block.
-    children: Tuple[ParserNode, ...]
-
-    """
-
-    @abc.abstractmethod
-    def add_child_block(self, name, parameters=None, position=None):
-        """
-        Adds a new BlockNode child node with provided values and marks the callee
-        BlockNode dirty. This is used to add new children to the AST. The preceeding
-        whitespaces should not be added based on the ancestor or siblings for the
-        newly created object. This is to match the current behavior of the legacy
-        parser implementation.
-
-        :param str name: The name of the child node to add
-        :param list parameters: list of parameters for the node
-        :param int position: Position in the list of children to add the new child
-            node to. Defaults to None, which appends the newly created node to the list.
-            If an integer is given, the child is inserted before that index in the
-            list similar to list().insert.
-
-        :returns: BlockNode instance of the created child block
-
-        """
-
-    @abc.abstractmethod
-    def add_child_directive(self, name, parameters=None, position=None):
-        """
-        Adds a new DirectiveNode child node with provided values and marks the
-        callee BlockNode dirty. This is used to add new children to the AST. The
-        preceeding whitespaces should not be added based on the ancestor or siblings
-        for the newly created object. This is to match the current behavior of the
-        legacy parser implementation.
-
-
-        :param str name: The name of the child node to add
-        :param list parameters: list of parameters for the node
-        :param int position: Position in the list of children to add the new child
-            node to. Defaults to None, which appends the newly created node to the list.
-            If an integer is given, the child is inserted before that index in the
-            list similar to list().insert.
-
-        :returns: DirectiveNode instance of the created child directive
-
-        """
-
-    @abc.abstractmethod
-    def add_child_comment(self, comment="", position=None):
-        """
-        Adds a new CommentNode child node with provided value and marks the
-        callee BlockNode dirty. This is used to add new children to the AST. The
-        preceeding whitespaces should not be added based on the ancestor or siblings
-        for the newly created object. This is to match the current behavior of the
-        legacy parser implementation.
-
-
-        :param str comment: Comment contents
-        :param int position: Position in the list of children to add the new child
-            node to. Defaults to None, which appends the newly created node to the list.
-            If an integer is given, the child is inserted before that index in the
-            list similar to list().insert.
-
-        :returns: CommentNode instance of the created child comment
-
-        """
-
-    @abc.abstractmethod
-    def find_blocks(self, name, exclude=True):
-        """
-        Find a configuration block by name. This method walks the child tree of
-        ParserNodes under the instance it was called from. This way it is possible
-        to search for the whole configuration tree, when starting from root node or
-        to do a partial search when starting from a specified branch. The lookup
-        should be case insensitive.
-
-        :param str name: The name of the directive to search for
-        :param bool exclude: If the search results should exclude the contents of
-            ParserNode objects that reside within conditional blocks and because
-            of current state are not enabled.
-
-        :returns: A list of found BlockNode objects.
-        """
-
-    @abc.abstractmethod
-    def find_directives(self, name, exclude=True):
-        """
-        Find a directive by name. This method walks the child tree of ParserNodes
-        under the instance it was called from. This way it is possible to search
-        for the whole configuration tree, when starting from root node, or to do
-        a partial search when starting from a specified branch. The lookup should
-        be case insensitive.
-
-        :param str name: The name of the directive to search for
-        :param bool exclude: If the search results should exclude the contents of
-            ParserNode objects that reside within conditional blocks and because
-            of current state are not enabled.
-
-        :returns: A list of found DirectiveNode objects.
-
-        """
-
-    @abc.abstractmethod
-    def find_comments(self, comment):
-        """
-        Find comments with value containing the search term.
-
-        This method walks the child tree of ParserNodes under the instance it was
-        called from. This way it is possible to search for the whole configuration
-        tree, when starting from root node, or to do a partial search when starting
-        from a specified branch. The lookup should be case sensitive.
-
-        :param str comment: The content of comment to search for
-
-        :returns: A list of found CommentNode objects.
-
-        """
-
-    @abc.abstractmethod
-    def delete_child(self, child):
-        """
-        Remove a specified child node from the list of children of the called
-        BlockNode object.
-
-        :param ParserNode child: Child ParserNode object to remove from the list
-            of children of the callee.
-        """
-
-    @abc.abstractmethod
-    def unsaved_files(self):
-        """
-        Returns a list of file paths that have been changed since the last save
-        (or the initial configuration parse). The intended use for this method
-        is to tell the Reverter which files need to be included in a checkpoint.
-
-        This is typically called for the root of the ParserNode tree.
-
-        :returns: list of file paths of files that have been changed but not yet
-            saved to disk.
-        """
-
-    @abc.abstractmethod
-    def parsed_paths(self):
-        """
-        Returns a list of file paths that have currently been parsed into the parser
-        tree. The returned list may include paths with wildcard characters, for
-        example: ['/etc/apache2/conf.d/*.load']
-
-        This is typically called on the root node of the ParserNode tree.
-
-        :returns: list of file paths of files that have been parsed
-        """

certbot-apache/certbot_apache/_internal/obj.py

@@ -1,7 +1,7 @@
 """Module contains classes used by the Apache Configurator."""
 import re
 
-from acme.magic_typing import Set
+from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
 from certbot.plugins import common
 
 
@@ -124,7 +124,7 @@ class VirtualHost(object):
     strip_name = re.compile(r"^(?:.+://)?([^ :$]*)")
 
     def __init__(self, filep, path, addrs, ssl, enabled, name=None,
-                 aliases=None, modmacro=False, ancestor=None, node=None):
+                 aliases=None, modmacro=False, ancestor=None):
 
         """Initialize a VH."""
         self.filep = filep
@@ -136,7 +136,6 @@ class VirtualHost(object):
         self.enabled = enabled
         self.modmacro = modmacro
         self.ancestor = ancestor
-        self.node = node
 
     def get_names(self):
         """Return a set of all names."""

certbot-apache/certbot_apache/_internal/options-ssl-apache.conf

@@ -0,0 +1,26 @@
+# This file contains important security parameters. If you modify this file
+# manually, Certbot will be unable to automatically provide future security
+# updates. Instead, Certbot will print and log an error message with a path to
+# the up-to-date file that you will need to refer to when manually updating
+# this file.
+
+SSLEngine on
+
+# Intermediate configuration, tweak to your needs
+SSLProtocol             all -SSLv2 -SSLv3
+SSLCipherSuite          ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
+SSLHonorCipherOrder     on
+SSLCompression          off
+
+SSLOptions +StrictRequire
+
+# Add vhost name to log entries:
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" vhost_combined
+LogFormat "%v %h %l %u %t \"%r\" %>s %b" vhost_common
+
+#CustomLog /var/log/apache2/access.log vhost_combined
+#LogLevel warn
+#ErrorLog /var/log/apache2/error.log
+
+# Always ensure Cookies have "Secure" set (JAH 2012/1)
+#Header edit Set-Cookie (?i)^(.*)(;\s*secure)??((\s*;)?(.*)) "$1; Secure$3$4"

certbot-apache/certbot_apache/_internal/override_arch.py

@@ -1,7 +1,9 @@
 """ Distribution specific override class for Arch Linux """
+import pkg_resources
 import zope.interface
 
 from certbot import interfaces
+from certbot.compat import os
 from certbot_apache._internal import configurator
 
 
@@ -24,5 +26,6 @@ class ArchConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/httpd/conf",
-        bin=None,
+        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
+            "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf"))
     )

certbot-apache/certbot_apache/_internal/override_centos.py

@@ -1,12 +1,14 @@
 """ Distribution specific override class for CentOS family (RHEL, Fedora) """
 import logging
 
+import pkg_resources
 import zope.interface
 
-from acme.magic_typing import List
+from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
 from certbot import errors
 from certbot import interfaces
 from certbot import util
+from certbot.compat import os
 from certbot.errors import MisconfigurationError
 from certbot_apache._internal import apache_util
 from certbot_apache._internal import configurator
@@ -35,7 +37,8 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/httpd/conf.d",
-        bin=None,
+        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
+            "certbot_apache", os.path.join("_internal", "centos-options-ssl-apache.conf"))
     )
 
     def config_test(self):

certbot-apache/certbot_apache/_internal/override_darwin.py

@@ -1,7 +1,9 @@
 """ Distribution specific override class for macOS """
+import pkg_resources
 import zope.interface
 
 from certbot import interfaces
+from certbot.compat import os
 from certbot_apache._internal import configurator
 
 
@@ -24,5 +26,6 @@ class DarwinConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/other",
-        bin=None,
+        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
+            "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf"))
     )

certbot-apache/certbot_apache/_internal/override_debian.py

@@ -1,6 +1,7 @@
 """ Distribution specific override class for Debian family (Ubuntu/Debian) """
 import logging
 
+import pkg_resources
 import zope.interface
 
 from certbot import errors
@@ -8,14 +9,16 @@ from certbot import interfaces
 from certbot import util
 from certbot.compat import filesystem
 from certbot.compat import os
+
 from certbot_apache._internal import apache_util
 from certbot_apache._internal import configurator
+from certbot_apache._internal import prefetch_ocsp
 
 logger = logging.getLogger(__name__)
 
 
 @zope.interface.provider(interfaces.IPluginFactory)
-class DebianConfigurator(configurator.ApacheConfigurator):
+class DebianConfigurator(prefetch_ocsp.OCSPPrefetchMixin, configurator.ApacheConfigurator):
     """Debian specific ApacheConfigurator override class"""
 
     OS_DEFAULTS = dict(
@@ -33,7 +36,8 @@ class DebianConfigurator(configurator.ApacheConfigurator):
         handle_modules=True,
         handle_sites=True,
         challenge_location="/etc/apache2",
-        bin=None,
+        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
+            "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf"))
     )
 
     def enable_site(self, vhost):

certbot-apache/certbot_apache/_internal/override_fedora.py

@@ -1,9 +1,11 @@
 """ Distribution specific override class for Fedora 29+ """
+import pkg_resources
 import zope.interface
 
 from certbot import errors
 from certbot import interfaces
 from certbot import util
+from certbot.compat import os
 from certbot_apache._internal import apache_util
 from certbot_apache._internal import configurator
 from certbot_apache._internal import parser
@@ -29,7 +31,9 @@ class FedoraConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/httpd/conf.d",
-        bin=None,
+        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
+            # TODO: eventually newest version of Fedora will need their own config
+            "certbot_apache", os.path.join("_internal", "centos-options-ssl-apache.conf"))
     )
 
     def config_test(self):

certbot-apache/certbot_apache/_internal/override_gentoo.py

@@ -1,7 +1,9 @@
 """ Distribution specific override class for Gentoo Linux """
+import pkg_resources
 import zope.interface
 
 from certbot import interfaces
+from certbot.compat import os
 from certbot_apache._internal import apache_util
 from certbot_apache._internal import configurator
 from certbot_apache._internal import parser
@@ -27,7 +29,8 @@ class GentooConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/vhosts.d",
-        bin=None,
+        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
+            "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf"))
     )
 
     def _prepare_options(self):
@@ -67,6 +70,6 @@ class GentooParser(parser.ApacheParser):
     def update_modules(self):
         """Get loaded modules from httpd process, and add them to DOM"""
         mod_cmd = [self.configurator.option("ctl"), "modules"]
-        matches = apache_util.parse_from_subprocess(mod_cmd, r"(.*)_module")
+        matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module")
         for mod in matches:
             self.add_mod(mod.strip())

certbot-apache/certbot_apache/_internal/override_suse.py

@@ -1,7 +1,9 @@
 """ Distribution specific override class for OpenSUSE """
+import pkg_resources
 import zope.interface
 
 from certbot import interfaces
+from certbot.compat import os
 from certbot_apache._internal import configurator
 
 
@@ -24,5 +26,6 @@ class OpenSUSEConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/vhosts.d",
-        bin=None,
+        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
+            "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf"))
     )

certbot-apache/certbot_apache/_internal/parser.py

@@ -3,15 +3,16 @@ import copy
 import fnmatch
 import logging
 import re
+import subprocess
 import sys
 
 import six
 
-from acme.magic_typing import Dict
-from acme.magic_typing import List
+from acme.magic_typing import Dict  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
 from certbot import errors
 from certbot.compat import os
-from certbot_apache._internal import apache_util
 from certbot_apache._internal import constants
 
 logger = logging.getLogger(__name__)
@@ -30,7 +31,7 @@ class ApacheParser(object):
 
     """
     arg_var_interpreter = re.compile(r"\$\{[^ \}]*}")
-    fnmatch_chars = {"*", "?", "\\", "[", "]"}
+    fnmatch_chars = set(["*", "?", "\\", "[", "]"])
 
     def __init__(self, root, vhostroot=None, version=(2, 4),
                  configurator=None):
@@ -51,7 +52,7 @@ class ApacheParser(object):
                 "version 1.2.0 or higher, please make sure you have you have "
                 "those installed.")
 
-        self.modules = {}  # type: Dict[str, str]
+        self.modules = set()  # type: Set[str]
         self.parser_paths = {}  # type: Dict[str, List[str]]
         self.variables = {}  # type: Dict[str, str]
 
@@ -248,14 +249,14 @@ class ApacheParser(object):
     def add_mod(self, mod_name):
         """Shortcut for updating parser modules."""
         if mod_name + "_module" not in self.modules:
-            self.modules[mod_name + "_module"] = None
+            self.modules.add(mod_name + "_module")
         if "mod_" + mod_name + ".c" not in self.modules:
-            self.modules["mod_" + mod_name + ".c"] = None
+            self.modules.add("mod_" + mod_name + ".c")
 
     def reset_modules(self):
         """Reset the loaded modules list. This is called from cleanup to clear
         temporarily loaded modules."""
-        self.modules = {}
+        self.modules = set()
         self.update_modules()
         self.parse_modules()
 
@@ -266,7 +267,7 @@ class ApacheParser(object):
             the iteration issue.  Else... parse and enable mods at same time.
 
         """
-        mods = {}  # type: Dict[str, str]
+        mods = set()  # type: Set[str]
         matches = self.find_dir("LoadModule")
         iterator = iter(matches)
         # Make sure prev_size != cur_size for do: while: iteration
@@ -280,8 +281,8 @@ class ApacheParser(object):
                 mod_name = self.get_arg(match_name)
                 mod_filename = self.get_arg(match_filename)
                 if mod_name and mod_filename:
-                    mods[mod_name] = mod_filename
-                    mods[os.path.basename(mod_filename)[:-2] + "c"] = mod_filename
+                    mods.add(mod_name)
+                    mods.add(os.path.basename(mod_filename)[:-2] + "c")
                 else:
                     logger.debug("Could not read LoadModule directive from Augeas path: %s",
                                  match_name[6:])
@@ -289,15 +290,32 @@ class ApacheParser(object):
 
     def update_runtime_variables(self):
         """Update Includes, Defines and Includes from httpd config dump data"""
-
         self.update_defines()
         self.update_includes()
         self.update_modules()
 
     def update_defines(self):
-        """Updates the dictionary of known variables in the configuration"""
+        """Get Defines from httpd process"""
 
-        self.variables = apache_util.parse_defines(self.configurator.option("ctl"))
+        variables = dict()
+        define_cmd = [self.configurator.option("ctl"), "-t", "-D",
+                      "DUMP_RUN_CFG"]
+        matches = self.parse_from_subprocess(define_cmd, r"Define: ([^ \n]*)")
+        try:
+            matches.remove("DUMP_RUN_CFG")
+        except ValueError:
+            return
+
+        for match in matches:
+            if match.count("=") > 1:
+                logger.error("Unexpected number of equal signs in "
+                             "runtime config dump.")
+                raise errors.PluginError(
+                    "Error parsing Apache runtime variables")
+            parts = match.partition("=")
+            variables[parts[0]] = parts[2]
+
+        self.variables = variables
 
     def update_includes(self):
         """Get includes from httpd process, and add them to DOM if needed"""
@@ -307,7 +325,9 @@ class ApacheParser(object):
         # configuration files
         _ = self.find_dir("Include")
 
-        matches = apache_util.parse_includes(self.configurator.option("ctl"))
+        inc_cmd = [self.configurator.option("ctl"), "-t", "-D",
+                   "DUMP_INCLUDES"]
+        matches = self.parse_from_subprocess(inc_cmd, r"\(.*\) (.*)")
         if matches:
             for i in matches:
                 if not self.parsed_in_current(i):
@@ -316,11 +336,57 @@ class ApacheParser(object):
     def update_modules(self):
         """Get loaded modules from httpd process, and add them to DOM"""
 
-        matches = apache_util.parse_modules(self.configurator.option("ctl"))
+        mod_cmd = [self.configurator.option("ctl"), "-t", "-D",
+                       "DUMP_MODULES"]
+        matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module")
         for mod in matches:
             self.add_mod(mod.strip())
 
-    def filter_args_num(self, matches, args):
+    def parse_from_subprocess(self, command, regexp):
+        """Get values from stdout of subprocess command
+
+        :param list command: Command to run
+        :param str regexp: Regexp for parsing
+
+        :returns: list parsed from command output
+        :rtype: list
+
+        """
+        stdout = self._get_runtime_cfg(command)
+        return re.compile(regexp).findall(stdout)
+
+    def _get_runtime_cfg(self, command):  # pylint: disable=no-self-use
+        """Get runtime configuration info.
+        :param command: Command to run
+
+        :returns: stdout from command
+
+        """
+        try:
+            proc = subprocess.Popen(
+                command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                universal_newlines=True)
+            stdout, stderr = proc.communicate()
+
+        except (OSError, ValueError):
+            logger.error(
+                "Error running command %s for runtime parameters!%s",
+                command, os.linesep)
+            raise errors.MisconfigurationError(
+                "Error accessing loaded Apache parameters: {0}".format(
+                command))
+        # Small errors that do not impede
+        if proc.returncode != 0:
+            logger.warning("Error in checking parameter list: %s", stderr)
+            raise errors.MisconfigurationError(
+                "Apache is unable to check whether or not the module is "
+                "loaded because Apache is misconfigured.")
+
+        return stdout
+
+    def filter_args_num(self, matches, args):  # pylint: disable=no-self-use
         """Filter out directives with specific number of arguments.
 
         This function makes the assumption that all related arguments are given
@@ -546,7 +612,7 @@ class ApacheParser(object):
             "%s//*[self::directive=~regexp('%s')]" % (start, regex))
 
         if exclude:
-            matches = self.exclude_dirs(matches)
+            matches = self._exclude_dirs(matches)
 
         if arg is None:
             arg_suffix = "/arg"
@@ -612,15 +678,9 @@ class ApacheParser(object):
 
         return value
 
-    def get_root_augpath(self):
-        """
-        Returns the Augeas path of root configuration.
-        """
-        return get_aug_path(self.loc["root"])
-
-    def exclude_dirs(self, matches):
+    def _exclude_dirs(self, matches):
         """Exclude directives that are not loaded into the configuration."""
-        filters = [("ifmodule", self.modules.keys()), ("ifdefine", self.variables)]
+        filters = [("ifmodule", self.modules), ("ifdefine", self.variables)]
 
         valid_matches = []
 
@@ -661,25 +721,6 @@ class ApacheParser(object):
 
         return True
 
-    def standard_path_from_server_root(self, arg):
-        """Ensure paths are consistent and absolute
-
-        :param str arg: Argument of directive
-
-        :returns: Standardized argument path
-        :rtype: str
-        """
-        # Remove beginning and ending quotes
-        arg = arg.strip("'\"")
-
-        # Standardize the include argument based on server root
-        if not arg.startswith("/"):
-            # Normpath will condense ../
-            arg = os.path.normpath(os.path.join(self.root, arg))
-        else:
-            arg = os.path.normpath(arg)
-        return arg
-
     def _get_include_path(self, arg):
         """Converts an Apache Include directive into Augeas path.
 
@@ -700,7 +741,16 @@ class ApacheParser(object):
         # if matchObj.group() != arg:
         #     logger.error("Error: Invalid regexp characters in %s", arg)
         #     return []
-        arg = self.standard_path_from_server_root(arg)
+
+        # Remove beginning and ending quotes
+        arg = arg.strip("'\"")
+
+        # Standardize the include argument based on server root
+        if not arg.startswith("/"):
+            # Normpath will condense ../
+            arg = os.path.normpath(os.path.join(self.root, arg))
+        else:
+            arg = os.path.normpath(arg)
 
         # Attempts to add a transform to the file if one does not already exist
         if os.path.isdir(arg):
@@ -724,7 +774,7 @@ class ApacheParser(object):
 
         return get_aug_path(arg)
 
-    def fnmatch_to_re(self, clean_fn_match):
+    def fnmatch_to_re(self, clean_fn_match):  # pylint: disable=no-self-use
         """Method converts Apache's basic fnmatch to regular expression.
 
         Assumption - Configs are assumed to be well-formed and only writable by
@@ -741,7 +791,7 @@ class ApacheParser(object):
         """
         if sys.version_info < (3, 6):
             # This strips off final /Z(?ms)
-            return fnmatch.translate(clean_fn_match)[:-7]  # pragma: no cover
+            return fnmatch.translate(clean_fn_match)[:-7]
         # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z
         return fnmatch.translate(clean_fn_match)[4:-3]  # pragma: no cover
 
@@ -945,8 +995,8 @@ def case_i(string):
     :param str string: string to make case i regex
 
     """
-    return "".join("[" + c.upper() + c.lower() + "]"
-                    if c.isalpha() else c for c in re.escape(string))
+    return "".join(["[" + c.upper() + c.lower() + "]"
+                    if c.isalpha() else c for c in re.escape(string)])
 
 
 def get_aug_path(file_path):

certbot-apache/certbot_apache/_internal/parsernode_util.py

@@ -1,129 +0,0 @@
-"""ParserNode utils"""
-
-
-def validate_kwargs(kwargs, required_names):
-    """
-    Ensures that the kwargs dict has all the expected values. This function modifies
-    the kwargs dictionary, and hence the returned dictionary should be used instead
-    in the caller function instead of the original kwargs.
-
-    :param dict kwargs: Dictionary of keyword arguments to validate.
-    :param list required_names: List of required parameter names.
-    """
-
-    validated_kwargs = {}
-    for name in required_names:
-        try:
-            validated_kwargs[name] = kwargs.pop(name)
-        except KeyError:
-            raise TypeError("Required keyword argument: {} undefined.".format(name))
-
-    # Raise exception if unknown key word arguments are found.
-    if kwargs:
-        unknown = ", ".join(kwargs.keys())
-        raise TypeError("Unknown keyword argument(s): {}".format(unknown))
-    return validated_kwargs
-
-
-def parsernode_kwargs(kwargs):
-    """
-    Validates keyword arguments for ParserNode. This function modifies the kwargs
-    dictionary, and hence the returned dictionary should be used instead in the
-    caller function instead of the original kwargs.
-
-    If metadata is provided, the otherwise required argument "filepath" may be
-    omitted if the implementation is able to extract its value from the metadata.
-    This usecase is handled within this function. Filepath defaults to None.
-
-    :param dict kwargs: Keyword argument dictionary to validate.
-
-    :returns: Tuple of validated and prepared arguments.
-    """
-
-    # As many values of ParserNode instances can be derived from the metadata,
-    # (ancestor being a common exception here) make sure we permit it here as well.
-    if "metadata" in kwargs:
-        # Filepath can be derived from the metadata in Augeas implementation.
-        # Default is None, as in this case the responsibility of populating this
-        # variable lies on the implementation.
-        kwargs.setdefault("filepath", None)
-
-    kwargs.setdefault("dirty", False)
-    kwargs.setdefault("metadata", {})
-
-    kwargs = validate_kwargs(kwargs, ["ancestor", "dirty", "filepath", "metadata"])
-    return kwargs["ancestor"], kwargs["dirty"], kwargs["filepath"], kwargs["metadata"]
-
-
-def commentnode_kwargs(kwargs):
-    """
-    Validates keyword arguments for CommentNode and sets the default values for
-    optional kwargs. This function modifies the kwargs dictionary, and hence the
-    returned dictionary should be used instead in the caller function instead of
-    the original kwargs.
-
-    If metadata is provided, the otherwise required argument "comment" may be
-    omitted if the implementation is able to extract its value from the metadata.
-    This usecase is handled within this function.
-
-    :param dict kwargs: Keyword argument dictionary to validate.
-
-    :returns: Tuple of validated and prepared arguments and ParserNode kwargs.
-    """
-
-    # As many values of ParserNode instances can be derived from the metadata,
-    # (ancestor being a common exception here) make sure we permit it here as well.
-    if "metadata" in kwargs:
-        kwargs.setdefault("comment", None)
-        # Filepath can be derived from the metadata in Augeas implementation.
-        # Default is None, as in this case the responsibility of populating this
-        # variable lies on the implementation.
-        kwargs.setdefault("filepath", None)
-
-    kwargs.setdefault("dirty", False)
-    kwargs.setdefault("metadata", {})
-
-    kwargs = validate_kwargs(kwargs, ["ancestor", "dirty", "filepath", "comment",
-                                      "metadata"])
-
-    comment = kwargs.pop("comment")
-    return comment, kwargs
-
-
-def directivenode_kwargs(kwargs):
-    """
-    Validates keyword arguments for DirectiveNode and BlockNode and sets the
-    default values for optional kwargs. This function modifies the kwargs
-    dictionary, and hence the returned dictionary should be used instead in the
-    caller function instead of the original kwargs.
-
-    If metadata is provided, the otherwise required argument "name" may be
-    omitted if the implementation is able to extract its value from the metadata.
-    This usecase is handled within this function.
-
-    :param dict kwargs: Keyword argument dictionary to validate.
-
-    :returns: Tuple of validated and prepared arguments and ParserNode kwargs.
-    """
-
-    # As many values of ParserNode instances can be derived from the metadata,
-    # (ancestor being a common exception here) make sure we permit it here as well.
-    if "metadata" in kwargs:
-        kwargs.setdefault("name", None)
-        # Filepath can be derived from the metadata in Augeas implementation.
-        # Default is None, as in this case the responsibility of populating this
-        # variable lies on the implementation.
-        kwargs.setdefault("filepath", None)
-
-    kwargs.setdefault("dirty", False)
-    kwargs.setdefault("enabled", True)
-    kwargs.setdefault("parameters", ())
-    kwargs.setdefault("metadata", {})
-
-    kwargs = validate_kwargs(kwargs, ["ancestor", "dirty", "filepath", "name",
-                                      "parameters", "enabled", "metadata"])
-
-    name = kwargs.pop("name")
-    parameters = kwargs.pop("parameters")
-    enabled = kwargs.pop("enabled")
-    return name, parameters, enabled, kwargs

certbot-apache/certbot_apache/_internal/prefetch_ocsp.py

@@ -0,0 +1,394 @@
+"""A mixin class for OCSP response prefetching for Apache plugin.
+
+The OCSP prefetching functionality solves multiple issues in Apache httpd
+that make using OCSP must-staple error prone.
+
+The prefetching functionality works by storing a value to PluginStorage,
+noting certificates that Certbot should keep OCSP staples (OCSP responses)
+updated for alongside of the information when the last response was
+updated by Certbot.
+
+When Certbot is invoked, typically by scheduled "certbot renew" and the
+TTL from "lastupdate" value in PluginStorage entry has expired,
+Certbot then proceeds to fetch a new OCSP response from the OCSP servers
+pointed by the certificate.
+
+The OCSP response is validated and if valid, stored to Apache DBM
+cache. A high internal cache expiry value is set for Apache in order
+to make it to not to discard the stored response and try to renew
+the staple itself letting Certbot to renew it on its subsequent run
+instead.
+
+The DBM cache file used by Apache is a lightweight key-value storage.
+For OCSP response caching, the sha1 hash of certificate fingerprint
+is used as a key. The value consists of expiry time as timestamp
+in microseconds, \x01 delimiter and the raw OCSP response.
+
+When restarting Apache, Certbot backups the current OCSP response
+cache, and restores it after the restart has happened. This is
+done because Apache deletes and then recreates the file upon
+restart.
+"""
+
+from datetime import datetime
+import logging
+import shutil
+import time
+
+from acme.magic_typing import Dict  # pylint: disable=unused-import, no-name-in-module
+
+from certbot import errors
+from certbot._internal import ocsp
+from certbot.plugins.enhancements import OCSPPrefetchEnhancement
+
+from certbot.compat import filesystem
+from certbot.compat import os
+from certbot_apache._internal import apache_util
+from certbot_apache._internal import constants
+
+logger = logging.getLogger(__name__)
+
+
+class DBMHandler(object):
+    """Context manager to handle DBM file reads and writes"""
+
+    def __init__(self, filename, mode):
+        self.filename = filename
+        self.filemode = mode
+        self.bsddb = False
+        self.database = None
+
+    def __enter__(self):
+        """Open the DBM file and return the filehandle"""
+
+        try:
+            import bsddb
+            self.bsddb = True
+            try:
+                self.database = bsddb.hashopen(self.filename, self.filemode)
+            except Exception:
+                raise errors.PluginError("Unable to open dbm database file.")
+        except ImportError:
+            # Python3 doesn't have bsddb module, so we use dbm.ndbm instead
+            import dbm
+            if self.filename.endswith(".db"):
+                self.filename = self.filename[:-3]
+            try:
+                self.database = dbm.ndbm.open(self.filename, self.filemode)  # pylint: disable=no-member
+            except Exception:
+                # This is raised if a file cannot be found
+                raise errors.PluginError("Unable to open dbm database file.")
+        return self.database
+
+    def __exit__(self, *args):
+        """Close the DBM file"""
+        if self.bsddb:
+            self.database.sync()
+        self.database.close()
+
+
+class OCSPPrefetchMixin(object):
+    """OCSPPrefetchMixin implements OCSP response prefetching"""
+
+    def __init__(self, *args, **kwargs):
+        self._ocsp_prefetch = {}  # type: Dict[str, str]
+        # This is required because of python super() call chain.
+        # Additionally, mypy isn't able to figure the chain out and needs to be
+        # disabled for this line. See https://github.com/python/mypy/issues/5887
+        super(OCSPPrefetchMixin, self).__init__(*args, **kwargs)  # type: ignore
+
+    def _ensure_ocsp_dirs(self):
+        """Makes sure that the OCSP directory paths exist."""
+        ocsp_work = os.path.join(self.config.work_dir, "ocsp_work")
+        ocsp_save = os.path.join(self.config.work_dir, "ocsp")
+        for path in [ocsp_work, ocsp_save]:
+            if not os.path.isdir(path):
+                filesystem.makedirs(path, 0o755)
+
+    def _ensure_ocsp_prefetch_compatibility(self):
+        """Make sure that the operating system supports the required libraries
+        to manage Apache DBM files.
+
+        :raises: errors.NotSupportedError
+        """
+        try:
+            import bsddb  # pylint: disable=unused-variable
+        except ImportError:
+            import dbm
+            if not hasattr(dbm, 'ndbm') or dbm.ndbm.library != 'Berkeley DB':  # pylint: disable=no-member
+                msg = ("Unfortunately your operating system does not have a "
+                       "compatible database module available for managing "
+                       "Apache OCSP stapling cache database.")
+                raise errors.NotSupportedError(msg)
+
+    def _ocsp_refresh_needed(self, pf_obj):
+        """Refreshes OCSP response for a certiifcate if it's due
+
+        :param dict pf_obj: OCSP prefetch object from pluginstorage
+
+        :returns: If OCSP response was updated
+        :rtype: bool
+
+        """
+        ttl = pf_obj["lastupdate"] + constants.OCSP_INTERNAL_TTL
+        if ttl < time.time():
+            self._ocsp_refresh(pf_obj["cert_path"], pf_obj["chain_path"])
+            return True
+        return False
+
+
+    def _ocsp_refresh(self, cert_path, chain_path):
+        """Refresh the OCSP response for a certificate
+
+        :param str cert_path: Filesystem path to certificate file
+        :param str chain_path: Filesystem path to certificate chain file
+
+        """
+
+        self._ensure_ocsp_dirs()
+        ocsp_workfile = os.path.join(
+            self.config.work_dir, "ocsp_work",
+            apache_util.certid_sha1_hex(cert_path))
+        handler = ocsp.RevocationChecker()
+        if not handler.ocsp_revoked_by_paths(cert_path, chain_path, ocsp_workfile):
+            # Guaranteed good response
+            cache_path = os.path.join(self.config.work_dir, "ocsp", "ocsp_cache.db")
+            cert_sha = apache_util.certid_sha1(cert_path)
+            # dbm.open automatically adds the file extension
+            self._write_to_dbm(cache_path, cert_sha, self._ocsp_response_dbm(ocsp_workfile))
+        else:
+            logger.warning("Encountered an issue while trying to prefetch OCSP "
+                           "response for certificate: %s", cert_path)
+        # Clean up
+        try:
+            os.remove(ocsp_workfile)
+        except OSError:
+            # The OCSP workfile did not exist because of an OCSP response fetching error
+            return
+
+    def _write_to_dbm(self, filename, key, value):
+        """Helper method to write an OCSP response cache value to DBM.
+
+        :param filename: DBM database filename
+        :param bytes key: Database key name
+        :param bytes value: Database entry value
+        """
+        tmp_file = os.path.join(
+            self.config.work_dir,
+            "ocsp_work",
+            "tmp_" + os.path.basename(filename)
+        )
+
+        apache_util.safe_copy(filename, tmp_file)
+
+        with DBMHandler(tmp_file, 'w') as db:
+            db[key] = value
+
+        shutil.copy2(tmp_file, filename)
+        os.remove(tmp_file)
+
+    def _read_dbm(self, filename):
+        """Helper method for reading the dbm using context manager.
+        Used for tests.
+
+        :param str filename: DBM database filename
+
+        :returns: Dictionary of database keys and values
+        :rtype: dict
+        """
+
+        ret = dict()
+        with DBMHandler(filename, 'r') as db:
+            for k in db.keys():
+                ret[k] = db[k]
+        return ret
+
+    def _ocsp_ttl(self, next_update):
+        """Calculates Apache internal TTL for the next OCSP staple
+        update.
+
+        The resulting TTL is half of the time between now
+        and the time noted by nextUpdate field in OCSP response.
+
+        If nextUpdate value is None, a default value will be
+        used instead.
+
+        :param next_update: datetime value for nextUpdate or None
+
+        :returns: TTL in seconds.
+        :rtype: int
+        """
+
+        if next_update is not None:
+            now = datetime.fromtimestamp(time.time())
+            res_ttl = int((next_update - now).total_seconds())
+            if res_ttl > 0:
+                return res_ttl/2
+        return constants.OCSP_APACHE_TTL
+
+    def _ocsp_response_dbm(self, workfile):
+        """Creates a dbm entry for OCSP response data
+
+        :param str workfile: File path for raw OCSP response
+
+        :returns: OCSP response cache data that Apache can use
+        :rtype: string
+
+        """
+
+        handler = ocsp.RevocationChecker()
+        _, _, next_update = handler.ocsp_times(workfile)
+        ttl = self._ocsp_ttl(next_update)
+
+        with open(workfile, 'rb') as fh:
+            response = fh.read()
+        return apache_util.get_apache_ocsp_struct(ttl, response)
+
+    def _ocsp_prefetch_save(self, cert_path, chain_path):
+        """Saves status of current OCSP prefetch, including the last update
+        time to determine if an update is needed on later run.
+
+        :param str cert_path: Filesystem path to certificate
+        :param str chain_path: Filesystem path to certificate chain file
+
+        """
+        status = {
+            "lastupdate": time.time(),
+            "cert_path": cert_path,
+            "chain_path": chain_path
+        }
+        cert_id = apache_util.certid_sha1_hex(cert_path)
+        self._ocsp_prefetch[cert_id] = status
+        self.storage.put("ocsp_prefetch", self._ocsp_prefetch)
+        self.storage.save()
+
+    def _ocsp_prefetch_fetch_state(self):
+        """
+        Populates the OCSP prefetch state from the pluginstorage.
+        """
+        try:
+            self._ocsp_prefetch = self.storage.fetch("ocsp_prefetch")
+        except KeyError:
+            self._ocsp_prefetch = dict()
+
+    def _ocsp_prefetch_backup_db(self):
+        """
+        Copies the active dbm file to work directory. Logs a debug error
+        message if unable to copy, but does not error out as it would
+        prevent other critical functions that need to be carried out for
+        Apache httpd.
+        """
+        self._ensure_ocsp_dirs()
+        cache_path = os.path.join(self.config.work_dir, "ocsp", "ocsp_cache.db")
+        try:
+            shutil.copy2(cache_path, os.path.join(self.config.work_dir, "ocsp_work"))
+        except IOError:
+            logger.debug("Encountered an issue while trying to backup OCSP dbm file")
+
+    def _ocsp_prefetch_restore_db(self):
+        """
+        Restores the active dbm file from work directory. Logs a debug error
+        message if unable to restore, but does not error out as it would
+        prevent other critical functions that need to be carried out for
+        Apache httpd.
+
+        """
+        self._ensure_ocsp_dirs()
+        cache_path = os.path.join(self.config.work_dir, "ocsp", "ocsp_cache.db")
+        work_file_path = os.path.join(self.config.work_dir, "ocsp_work", "ocsp_cache.db")
+        try:
+            shutil.copy2(work_file_path, cache_path)
+        except IOError:
+            logger.debug("Encountered an issue when trying to restore OCSP dbm file")
+
+    def enable_ocsp_prefetch(self, lineage, domains):
+        """Enable OCSP Stapling and prefetching of the responses.
+
+        In OCSP, each client (e.g. browser) would have to query the
+        OCSP Responder to validate that the site certificate was not revoked.
+
+        Enabling OCSP Stapling, would allow the web-server to query the OCSP
+        Responder, and staple its response to the offered certificate during
+        TLS. i.e. clients would not have to query the OCSP responder.
+
+        """
+
+        # Fail early if we are not able to support this
+        self._ensure_ocsp_prefetch_compatibility()
+        prefetch_vhosts = set()
+        for domain in domains:
+            matched_vhosts = self.choose_vhosts(domain, create_if_no_ssl=False)
+            # We should be handling only SSL vhosts
+            for vh in matched_vhosts:
+                if vh.ssl:
+                    prefetch_vhosts.add(vh)
+
+        if not prefetch_vhosts:
+            raise errors.MisconfigurationError(
+                "Could not find VirtualHost to enable OCSP prefetching on."
+            )
+
+        try:
+            # The try - block is huge, but required for handling rollback properly.
+            for vh in prefetch_vhosts:
+                self._enable_ocsp_stapling(vh, None, prefetch=True)
+
+            self._ensure_ocsp_dirs()
+            self.restart()
+            # Ensure Apache has enough time to properly restart and create the file
+            time.sleep(2)
+            self._ocsp_refresh(lineage.cert_path, lineage.chain_path)
+            self._ocsp_prefetch_save(lineage.cert_path, lineage.chain_path)
+            self.save("Enabled OCSP prefetching")
+        except errors.PluginError as err:
+            # Revert the OCSP prefetch configuration
+            self.recovery_routine()
+            self.restart()
+            msg = ("Encountered an error while trying to enable OCSP prefetch "
+                "enhancement: %s\nOCSP prefetch was not enabled.")
+            raise errors.PluginError(msg % str(err))
+
+    def update_ocsp_prefetch(self, _unused_lineage):
+        """Checks all certificates that are managed by OCSP prefetch, and
+        refreshes OCSP responses for them if required."""
+
+        self._ocsp_prefetch_fetch_state()
+        if not self._ocsp_prefetch:
+            # No OCSP prefetching enabled for any certificate
+            return
+
+        for _, pf in self._ocsp_prefetch.items():
+            if not self._ocsp_refresh_needed(pf):
+                continue
+            # Save the status to pluginstorage
+            self._ocsp_prefetch_save(pf["cert_path"], pf["chain_path"])
+
+    def restart(self):
+        """Reloads the Apache server. When restarting, Apache deletes
+        the DBM cache file used to store OCSP staples. In this override
+        function, Certbot checks the pluginstorage if we're supposed to
+        manage OCSP prefetching. If needed, Certbot will backup the DBM
+        file, restoring it after calling restart.
+
+        :raises .errors.MisconfigurationError: If either the config test
+            or reload fails.
+
+        """
+        if not self._ocsp_prefetch:
+            # Try to populate OCSP prefetch structure from pluginstorage
+            self._ocsp_prefetch_fetch_state()
+        if self._ocsp_prefetch:
+            # OCSP prefetching is enabled, so back up the db
+            self._ocsp_prefetch_backup_db()
+
+        try:
+            # Ignored because mypy doesn't know that this class is used as
+            # a mixin and fails because object has no restart method.
+            super(OCSPPrefetchMixin, self).restart()  # type: ignore
+        finally:
+            if self._ocsp_prefetch:
+                # Restore the backed up dbm database
+                self._ocsp_prefetch_restore_db()
+
+
+OCSPPrefetchEnhancement.register(OCSPPrefetchMixin)  # pylint: disable=no-member

certbot-apache/certbot_apache/_internal/tls_configs/current-options-ssl-apache.conf

@@ -1,19 +0,0 @@
-# This file contains important security parameters. If you modify this file
-# manually, Certbot will be unable to automatically provide future security
-# updates. Instead, Certbot will print and log an error message with a path to
-# the up-to-date file that you will need to refer to when manually updating
-# this file.
-
-SSLEngine on
-
-# Intermediate configuration, tweak to your needs
-SSLProtocol             all -SSLv2 -SSLv3 -TLSv1 -TLSv1.1
-SSLCipherSuite          ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
-SSLHonorCipherOrder     off
-SSLSessionTickets       off
-
-SSLOptions +StrictRequire
-
-# Add vhost name to log entries:
-LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" vhost_combined
-LogFormat "%v %h %l %u %t \"%r\" %>s %b" vhost_common

certbot-apache/certbot_apache/_internal/tls_configs/old-options-ssl-apache.conf

@@ -1,18 +0,0 @@
-# This file contains important security parameters. If you modify this file
-# manually, Certbot will be unable to automatically provide future security
-# updates. Instead, Certbot will print and log an error message with a path to
-# the up-to-date file that you will need to refer to when manually updating
-# this file.
-
-SSLEngine on
-
-# Intermediate configuration, tweak to your needs
-SSLProtocol             all -SSLv2 -SSLv3 -TLSv1 -TLSv1.1
-SSLCipherSuite          ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
-SSLHonorCipherOrder     off
-
-SSLOptions +StrictRequire
-
-# Add vhost name to log entries:
-LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" vhost_combined
-LogFormat "%v %h %l %u %t \"%r\" %>s %b" vhost_common

certbot-apache/local-oldest-requirements.txt

@@ -1,3 +1,3 @@
 # Remember to update setup.py to match the package versions below.
 acme[dev]==0.29.0
-certbot[dev]==1.1.0
+-e certbot[dev]

certbot-apache/setup.py

@@ -1,36 +1,23 @@
-from distutils.version import StrictVersion
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.6.0.dev0'
+version = '1.2.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
     'acme>=0.29.0',
-    'certbot>=1.1.0',
+    'certbot>=1.2.0.dev0',
+    'mock',
     'python-augeas',
     'setuptools',
     'zope.component',
     'zope.interface',
 ]
 
-setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
-dev_extras = [
-    'apacheconfig>=0.3.2',
-]
 
 class PyTest(TestCommand):
     user_options = []
@@ -55,7 +42,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
-    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Environment :: Plugins',
@@ -66,6 +53,7 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
@@ -81,9 +69,6 @@ setup(
     packages=find_packages(),
     include_package_data=True,
     install_requires=install_requires,
-    extras_require={
-        'dev': dev_extras,
-    },
     entry_points={
         'certbot.plugins': [
             'apache = certbot_apache._internal.entrypoint:ENTRYPOINT',

certbot-apache/tests/augeasnode_test.py

@@ -1,335 +0,0 @@
-"""Tests for AugeasParserNode classes"""
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
-
-import os
-import util
-
-from certbot import errors
-
-from certbot_apache._internal import assertions
-from certbot_apache._internal import augeasparser
-
-
-def _get_augeasnode_mock(filepath):
-    """ Helper function for mocking out DualNode instance with an AugeasNode """
-    def augeasnode_mock(metadata):
-        return augeasparser.AugeasBlockNode(
-            name=assertions.PASS,
-            ancestor=None,
-            filepath=filepath,
-            metadata=metadata)
-    return augeasnode_mock
-
-class AugeasParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-methods
-    """Test AugeasParserNode using available test configurations"""
-
-    def setUp(self):  # pylint: disable=arguments-differ
-        super(AugeasParserNodeTest, self).setUp()
-
-        with mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.get_parsernode_root") as mock_parsernode:
-            mock_parsernode.side_effect = _get_augeasnode_mock(
-                                              os.path.join(self.config_path, "apache2.conf"))
-            self.config = util.get_apache_configurator(
-                self.config_path, self.vhost_path, self.config_dir, self.work_dir, use_parsernode=True)
-        self.vh_truth = util.get_vh_truth(
-            self.temp_dir, "debian_apache_2_4/multiple_vhosts")
-
-    def test_save(self):
-        with mock.patch('certbot_apache._internal.parser.ApacheParser.save') as mock_save:
-            self.config.parser_root.save("A save message")
-        self.assertTrue(mock_save.called)
-        self.assertEqual(mock_save.call_args[0][0], "A save message")
-
-    def test_unsaved_files(self):
-        with mock.patch('certbot_apache._internal.parser.ApacheParser.unsaved_files') as mock_uf:
-            mock_uf.return_value = ["first", "second"]
-            files = self.config.parser_root.unsaved_files()
-        self.assertEqual(files, ["first", "second"])
-
-    def test_get_block_node_name(self):
-        from certbot_apache._internal.augeasparser import AugeasBlockNode
-        block = AugeasBlockNode(
-            name=assertions.PASS,
-            ancestor=None,
-            filepath=assertions.PASS,
-            metadata={"augeasparser": mock.Mock(), "augeaspath": "/files/anything"}
-        )
-        testcases = {
-            "/some/path/FirstNode/SecondNode": "SecondNode",
-            "/some/path/FirstNode/SecondNode/": "SecondNode",
-            "OnlyPathItem": "OnlyPathItem",
-            "/files/etc/apache2/apache2.conf/VirtualHost": "VirtualHost",
-            "/Anything": "Anything",
-        }
-        for test in testcases:
-            self.assertEqual(block._aug_get_name(test), testcases[test])  # pylint: disable=protected-access
-
-    def test_find_blocks(self):
-        blocks = self.config.parser_root.find_blocks("VirtualHost", exclude=False)
-        self.assertEqual(len(blocks), 12)
-
-    def test_find_blocks_case_insensitive(self):
-        vhs = self.config.parser_root.find_blocks("VirtualHost")
-        vhs2 = self.config.parser_root.find_blocks("viRtuAlHoST")
-        self.assertEqual(len(vhs), len(vhs2))
-
-    def test_find_directive_found(self):
-        directives = self.config.parser_root.find_directives("Listen")
-        self.assertEqual(len(directives), 1)
-        self.assertTrue(directives[0].filepath.endswith("/apache2/ports.conf"))
-        self.assertEqual(directives[0].parameters, (u'80',))
-
-    def test_find_directive_notfound(self):
-        directives = self.config.parser_root.find_directives("Nonexistent")
-        self.assertEqual(len(directives), 0)
-
-    def test_find_directive_from_block(self):
-        blocks = self.config.parser_root.find_blocks("virtualhost")
-        found = False
-        for vh in blocks:
-            if vh.filepath.endswith("sites-enabled/certbot.conf"):
-                servername = vh.find_directives("servername")
-                self.assertEqual(servername[0].parameters[0], "certbot.demo")
-                found = True
-        self.assertTrue(found)
-
-    def test_find_comments(self):
-        rootcomment = self.config.parser_root.find_comments(
-            "This is the main Apache server configuration file. "
-        )
-        self.assertEqual(len(rootcomment), 1)
-        self.assertTrue(rootcomment[0].filepath.endswith(
-            "debian_apache_2_4/multiple_vhosts/apache2/apache2.conf"
-        ))
-
-    def test_set_parameters(self):
-        servernames = self.config.parser_root.find_directives("servername")
-        names = []  # type: List[str]
-        for servername in servernames:
-            names += servername.parameters
-        self.assertFalse("going_to_set_this" in names)
-        servernames[0].set_parameters(["something", "going_to_set_this"])
-        servernames = self.config.parser_root.find_directives("servername")
-        names = []
-        for servername in servernames:
-            names += servername.parameters
-        self.assertTrue("going_to_set_this" in names)
-
-    def test_set_parameters_atinit(self):
-        from certbot_apache._internal.augeasparser import AugeasDirectiveNode
-        servernames = self.config.parser_root.find_directives("servername")
-        setparam = "certbot_apache._internal.augeasparser.AugeasDirectiveNode.set_parameters"
-        with mock.patch(setparam) as mock_set:
-            AugeasDirectiveNode(
-                name=servernames[0].name,
-                parameters=["test", "setting", "these"],
-                ancestor=assertions.PASS,
-                metadata=servernames[0].metadata
-            )
-            self.assertTrue(mock_set.called)
-            self.assertEqual(
-                mock_set.call_args_list[0][0][0],
-                ["test", "setting", "these"]
-            )
-
-    def test_set_parameters_delete(self):
-        # Set params
-        servername = self.config.parser_root.find_directives("servername")[0]
-        servername.set_parameters(["thisshouldnotexistpreviously", "another",
-                                   "third"])
-
-        # Delete params
-        servernames = self.config.parser_root.find_directives("servername")
-        found = False
-        for servername in servernames:
-            if "thisshouldnotexistpreviously" in servername.parameters:
-                self.assertEqual(len(servername.parameters), 3)
-                servername.set_parameters(["thisshouldnotexistpreviously"])
-                found = True
-        self.assertTrue(found)
-
-        # Verify params
-        servernames = self.config.parser_root.find_directives("servername")
-        found = False
-        for servername in servernames:
-            if "thisshouldnotexistpreviously" in servername.parameters:
-                self.assertEqual(len(servername.parameters), 1)
-                servername.set_parameters(["thisshouldnotexistpreviously"])
-                found = True
-        self.assertTrue(found)
-
-    def test_add_child_comment(self):
-        newc = self.config.parser_root.add_child_comment("The content")
-        comments = self.config.parser_root.find_comments("The content")
-        self.assertEqual(len(comments), 1)
-        self.assertEqual(
-            newc.metadata["augeaspath"],
-            comments[0].metadata["augeaspath"]
-        )
-        self.assertEqual(newc.comment, comments[0].comment)
-
-    def test_delete_child(self):
-        listens = self.config.parser_root.find_directives("Listen")
-        self.assertEqual(len(listens), 1)
-        self.config.parser_root.delete_child(listens[0])
-
-        listens = self.config.parser_root.find_directives("Listen")
-        self.assertEqual(len(listens), 0)
-
-    def test_delete_child_not_found(self):
-        listen = self.config.parser_root.find_directives("Listen")[0]
-        listen.metadata["augeaspath"] = "/files/something/nonexistent"
-
-        self.assertRaises(
-            errors.PluginError,
-            self.config.parser_root.delete_child,
-            listen
-        )
-
-    def test_add_child_block(self):
-        nb = self.config.parser_root.add_child_block(
-            "NewBlock",
-            ["first", "second"]
-        )
-        rpath, _, directive = nb.metadata["augeaspath"].rpartition("/")
-        self.assertEqual(
-            rpath,
-            self.config.parser_root.metadata["augeaspath"]
-        )
-        self.assertTrue(directive.startswith("NewBlock"))
-
-    def test_add_child_block_beginning(self):
-        self.config.parser_root.add_child_block(
-            "Beginning",
-            position=0
-        )
-        parser = self.config.parser_root.parser
-        root_path = self.config.parser_root.metadata["augeaspath"]
-        # Get first child
-        first = parser.aug.match("{}/*[1]".format(root_path))
-        self.assertTrue(first[0].endswith("Beginning"))
-
-    def test_add_child_block_append(self):
-        self.config.parser_root.add_child_block(
-            "VeryLast",
-        )
-        parser = self.config.parser_root.parser
-        root_path = self.config.parser_root.metadata["augeaspath"]
-        # Get last child
-        last = parser.aug.match("{}/*[last()]".format(root_path))
-        self.assertTrue(last[0].endswith("VeryLast"))
-
-    def test_add_child_block_append_alt(self):
-        self.config.parser_root.add_child_block(
-            "VeryLastAlt",
-            position=99999
-        )
-        parser = self.config.parser_root.parser
-        root_path = self.config.parser_root.metadata["augeaspath"]
-        # Get last child
-        last = parser.aug.match("{}/*[last()]".format(root_path))
-        self.assertTrue(last[0].endswith("VeryLastAlt"))
-
-    def test_add_child_block_middle(self):
-        self.config.parser_root.add_child_block(
-            "Middle",
-            position=5
-        )
-        parser = self.config.parser_root.parser
-        root_path = self.config.parser_root.metadata["augeaspath"]
-        # Augeas indices start at 1 :(
-        middle = parser.aug.match("{}/*[6]".format(root_path))
-        self.assertTrue(middle[0].endswith("Middle"))
-
-    def test_add_child_block_existing_name(self):
-        parser = self.config.parser_root.parser
-        root_path = self.config.parser_root.metadata["augeaspath"]
-        # There already exists a single VirtualHost in the base config
-        new_block = parser.aug.match("{}/VirtualHost[2]".format(root_path))
-        self.assertEqual(len(new_block), 0)
-        vh = self.config.parser_root.add_child_block(
-            "VirtualHost",
-        )
-        new_block = parser.aug.match("{}/VirtualHost[2]".format(root_path))
-        self.assertEqual(len(new_block), 1)
-        self.assertTrue(vh.metadata["augeaspath"].endswith("VirtualHost[2]"))
-
-    def test_node_init_error_bad_augeaspath(self):
-        from certbot_apache._internal.augeasparser import AugeasBlockNode
-        parameters = {
-            "name": assertions.PASS,
-            "ancestor": None,
-            "filepath": assertions.PASS,
-            "metadata": {
-                "augeasparser": mock.Mock(),
-                "augeaspath": "/files/path/endswith/slash/"
-            }
-        }
-        self.assertRaises(
-            errors.PluginError,
-            AugeasBlockNode,
-            **parameters
-        )
-
-    def test_node_init_error_missing_augeaspath(self):
-        from certbot_apache._internal.augeasparser import AugeasBlockNode
-        parameters = {
-            "name": assertions.PASS,
-            "ancestor": None,
-            "filepath": assertions.PASS,
-            "metadata": {
-                "augeasparser": mock.Mock(),
-            }
-        }
-        self.assertRaises(
-            errors.PluginError,
-            AugeasBlockNode,
-            **parameters
-        )
-
-    def test_add_child_directive(self):
-        self.config.parser_root.add_child_directive(
-            "ThisWasAdded",
-            ["with", "parameters"],
-            position=0
-        )
-        dirs = self.config.parser_root.find_directives("ThisWasAdded")
-        self.assertEqual(len(dirs), 1)
-        self.assertEqual(dirs[0].parameters, ("with", "parameters"))
-        # The new directive was added to the very first line of the config
-        self.assertTrue(dirs[0].metadata["augeaspath"].endswith("[1]"))
-
-    def test_add_child_directive_exception(self):
-        self.assertRaises(
-            errors.PluginError,
-            self.config.parser_root.add_child_directive,
-            "ThisRaisesErrorBecauseMissingParameters"
-        )
-
-    def test_parsed_paths(self):
-        paths = self.config.parser_root.parsed_paths()
-        self.assertEqual(len(paths), 6)
-
-    def test_find_ancestors(self):
-        vhsblocks = self.config.parser_root.find_blocks("VirtualHost")
-        macro_test = False
-        nonmacro_test = False
-        for vh in vhsblocks:
-            if "/macro/" in vh.metadata["augeaspath"].lower():
-                ancs = vh.find_ancestors("Macro")
-                self.assertEqual(len(ancs), 1)
-                macro_test = True
-            else:
-                ancs = vh.find_ancestors("Macro")
-                self.assertEqual(len(ancs), 0)
-                nonmacro_test = True
-        self.assertTrue(macro_test)
-        self.assertTrue(nonmacro_test)
-
-    def test_find_ancestors_bad_path(self):
-        self.config.parser_root.metadata["augeaspath"] = ""
-        ancs = self.config.parser_root.find_ancestors("Anything")
-        self.assertEqual(len(ancs), 0)

certbot-apache/tests/autohsts_test.py

@@ -3,10 +3,7 @@
 import re
 import unittest
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 import six  # pylint: disable=unused-import  # six is used in mock.patch()
 
 from certbot import errors
@@ -23,10 +20,10 @@ class AutoHSTSTest(util.ApacheTest):
 
         self.config = util.get_apache_configurator(
             self.config_path, self.vhost_path, self.config_dir, self.work_dir)
-        self.config.parser.modules["headers_module"] = None
-        self.config.parser.modules["mod_headers.c"] = None
-        self.config.parser.modules["ssl_module"] = None
-        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules.add("headers_module")
+        self.config.parser.modules.add("mod_headers.c")
+        self.config.parser.modules.add("ssl_module")
+        self.config.parser.modules.add("mod_ssl.c")
 
         self.vh_truth = util.get_vh_truth(
             self.temp_dir, "debian_apache_2_4/multiple_vhosts")
@@ -45,8 +42,8 @@ class AutoHSTSTest(util.ApacheTest):
     @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.restart")
     @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.enable_mod")
     def test_autohsts_enable_headers_mod(self, mock_enable, _restart):
-        self.config.parser.modules.pop("headers_module", None)
-        self.config.parser.modules.pop("mod_header.c", None)
+        self.config.parser.modules.discard("headers_module")
+        self.config.parser.modules.discard("mod_header.c")
         self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
         self.assertTrue(mock_enable.called)
 

certbot-apache/tests/centos6_test.py

@@ -19,12 +19,12 @@ def get_vh_truth(temp_dir, config_name):
         obj.VirtualHost(
             os.path.join(prefix, "test.example.com.conf"),
             os.path.join(aug_pre, "test.example.com.conf/VirtualHost"),
-            {obj.Addr.fromstring("*:80")},
+            set([obj.Addr.fromstring("*:80")]),
             False, True, "test.example.com"),
         obj.VirtualHost(
             os.path.join(prefix, "ssl.conf"),
             os.path.join(aug_pre, "ssl.conf/VirtualHost"),
-            {obj.Addr.fromstring("_default_:443")},
+            set([obj.Addr.fromstring("_default_:443")]),
             True, True, None)
     ]
     return vh_truth
@@ -104,7 +104,7 @@ class CentOS6Tests(util.ApacheTest):
         pre_loadmods = self.config.parser.find_dir(
             "LoadModule", "ssl_module", exclude=False)
         # LoadModules are not within IfModule blocks
-        self.assertFalse(any("ifmodule" in m.lower() for m in pre_loadmods))
+        self.assertFalse(any(["ifmodule" in m.lower() for m in pre_loadmods]))
         self.config.assoc["test.example.com"] = self.vh_truth[0]
         self.config.deploy_cert(
             "random.demo", "example/cert.pem", "example/key.pem",

certbot-apache/tests/centos_test.py

@@ -1,10 +1,7 @@
 """Test for certbot_apache._internal.configurator for Centos overrides"""
 import unittest
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 
 from certbot import errors
 from certbot.compat import filesystem
@@ -24,12 +21,12 @@ def get_vh_truth(temp_dir, config_name):
         obj.VirtualHost(
             os.path.join(prefix, "centos.example.com.conf"),
             os.path.join(aug_pre, "centos.example.com.conf/VirtualHost"),
-            {obj.Addr.fromstring("*:80")},
+            set([obj.Addr.fromstring("*:80")]),
             False, True, "centos.example.com"),
         obj.VirtualHost(
             os.path.join(prefix, "ssl.conf"),
             os.path.join(aug_pre, "ssl.conf/VirtualHost"),
-            {obj.Addr.fromstring("_default_:443")},
+            set([obj.Addr.fromstring("_default_:443")]),
             True, True, None)
     ]
     return vh_truth
@@ -109,7 +106,7 @@ class MultipleVhostsTestCentOS(util.ApacheTest):
     def test_get_parser(self):
         self.assertIsInstance(self.config.parser, override_centos.CentOSParser)
 
-    @mock.patch("certbot_apache._internal.apache_util._get_runtime_cfg")
+    @mock.patch("certbot_apache._internal.parser.ApacheParser._get_runtime_cfg")
     def test_opportunistic_httpd_runtime_parsing(self, mock_get):
         define_val = (
             'Define: TEST1\n'
@@ -129,7 +126,7 @@ class MultipleVhostsTestCentOS(util.ApacheTest):
                 return mod_val
             return ""
         mock_get.side_effect = mock_get_cfg
-        self.config.parser.modules = {}
+        self.config.parser.modules = set()
         self.config.parser.variables = {}
 
         with mock.patch("certbot.util.get_os_info") as mock_osi:
@@ -158,7 +155,7 @@ class MultipleVhostsTestCentOS(util.ApacheTest):
                 raise Exception("Missed: %s" % vhost)  # pragma: no cover
         self.assertEqual(found, 2)
 
-    @mock.patch("certbot_apache._internal.apache_util._get_runtime_cfg")
+    @mock.patch("certbot_apache._internal.parser.ApacheParser._get_runtime_cfg")
     def test_get_sysconfig_vars(self, mock_cfg):
         """Make sure we read the sysconfig OPTIONS variable correctly"""
         # Return nothing for the process calls

certbot-apache/tests/configurator_reverter_test.py

@@ -2,10 +2,7 @@
 import shutil
 import unittest
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 
 from certbot import errors
 import util

certbot-apache/tests/configurator_test.py

@@ -6,10 +6,7 @@ import socket
 import tempfile
 import unittest
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 import six  # pylint: disable=unused-import  # six is used in mock.patch()
 
 from acme import challenges
@@ -78,8 +75,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
     @mock.patch("certbot_apache._internal.parser.ApacheParser")
     @mock.patch("certbot_apache._internal.configurator.util.exe_exists")
-    @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.get_parsernode_root")
-    def _test_prepare_locked(self, _node, _exists, _parser):
+    def _test_prepare_locked(self, unused_parser, unused_exe_exists):
         try:
             self.config.prepare()
         except errors.PluginError as err:
@@ -102,7 +98,7 @@ class MultipleVhostsTest(util.ApacheTest):
         parserargs = ["server_root", "enmod", "dismod", "le_vhost_ext",
                       "vhost_root", "logs_root", "challenge_location",
                       "handle_modules", "handle_sites", "ctl"]
-        exp = {}
+        exp = dict()
 
         for k in ApacheConfigurator.OS_DEFAULTS:
             if k in parserargs:
@@ -143,9 +139,11 @@ class MultipleVhostsTest(util.ApacheTest):
         mock_utility = mock_getutility()
         mock_utility.notification = mock.MagicMock(return_value=True)
         names = self.config.get_all_names()
-        self.assertEqual(names, {"certbot.demo", "ocspvhost.com", "encryption-example.demo",
+        self.assertEqual(names, set(
+            ["certbot.demo", "ocspvhost.com", "encryption-example.demo",
              "nonsym.link", "vhost.in.rootconf", "www.certbot.demo",
-             "duplicate.example.com"})
+             "duplicate.example.com"]
+        ))
 
     @certbot_util.patch_get_utility()
     @mock.patch("certbot_apache._internal.configurator.socket.gethostbyaddr")
@@ -155,9 +153,9 @@ class MultipleVhostsTest(util.ApacheTest):
         mock_utility.notification.return_value = True
         vhost = obj.VirtualHost(
             "fp", "ap",
-            {obj.Addr(("8.8.8.8", "443")),
+            set([obj.Addr(("8.8.8.8", "443")),
                  obj.Addr(("zombo.com",)),
-                 obj.Addr(("192.168.1.2"))},
+                 obj.Addr(("192.168.1.2"))]),
             True, False)
 
         self.config.vhosts.append(vhost)
@@ -186,7 +184,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
     def test_bad_servername_alias(self):
         ssl_vh1 = obj.VirtualHost(
-            "fp1", "ap1", {obj.Addr(("*", "443"))},
+            "fp1", "ap1", set([obj.Addr(("*", "443"))]),
             True, False)
         # pylint: disable=protected-access
         self.config._add_servernames(ssl_vh1)
@@ -199,7 +197,7 @@ class MultipleVhostsTest(util.ApacheTest):
         # pylint: disable=protected-access
         self.config._add_servernames(self.vh_truth[2])
         self.assertEqual(
-            self.vh_truth[2].get_names(), {"*.le.co", "ip-172-30-0-17"})
+            self.vh_truth[2].get_names(), set(["*.le.co", "ip-172-30-0-17"]))
 
     def test_get_virtual_hosts(self):
         """Make sure all vhosts are being properly found."""
@@ -270,7 +268,7 @@ class MultipleVhostsTest(util.ApacheTest):
     def test_choose_vhost_select_vhost_conflicting_non_ssl(self, mock_select):
         mock_select.return_value = self.vh_truth[3]
         conflicting_vhost = obj.VirtualHost(
-            "path", "aug_path", {obj.Addr.fromstring("*:443")},
+            "path", "aug_path", set([obj.Addr.fromstring("*:443")]),
             True, True)
         self.config.vhosts.append(conflicting_vhost)
 
@@ -279,14 +277,14 @@ class MultipleVhostsTest(util.ApacheTest):
 
     def test_find_best_http_vhost_default(self):
         vh = obj.VirtualHost(
-            "fp", "ap", {obj.Addr.fromstring("_default_:80")}, False, True)
+            "fp", "ap", set([obj.Addr.fromstring("_default_:80")]), False, True)
         self.config.vhosts = [vh]
         self.assertEqual(self.config.find_best_http_vhost("foo.bar", False), vh)
 
     def test_find_best_http_vhost_port(self):
         port = "8080"
         vh = obj.VirtualHost(
-            "fp", "ap", {obj.Addr.fromstring("*:" + port)},
+            "fp", "ap", set([obj.Addr.fromstring("*:" + port)]),
             False, True, "encryption-example.demo")
         self.config.vhosts.append(vh)
         self.assertEqual(self.config.find_best_http_vhost("foo.bar", False, port), vh)
@@ -314,8 +312,8 @@ class MultipleVhostsTest(util.ApacheTest):
     def test_find_best_vhost_variety(self):
         # pylint: disable=protected-access
         ssl_vh = obj.VirtualHost(
-            "fp", "ap", {obj.Addr(("*", "443")),
-                             obj.Addr(("zombo.com",))},
+            "fp", "ap", set([obj.Addr(("*", "443")),
+                             obj.Addr(("zombo.com",))]),
             True, False)
         self.config.vhosts.append(ssl_vh)
         self.assertEqual(self.config._find_best_vhost("zombo.com"), ssl_vh)
@@ -342,9 +340,9 @@ class MultipleVhostsTest(util.ApacheTest):
     def test_deploy_cert_enable_new_vhost(self):
         # Create
         ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])
-        self.config.parser.modules["ssl_module"] = None
-        self.config.parser.modules["mod_ssl.c"] = None
-        self.config.parser.modules["socache_shmcb_module"] = None
+        self.config.parser.modules.add("ssl_module")
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("socache_shmcb_module")
 
         self.assertFalse(ssl_vhost.enabled)
         self.config.deploy_cert(
@@ -378,9 +376,9 @@ class MultipleVhostsTest(util.ApacheTest):
                     # pragma: no cover
 
     def test_deploy_cert(self):
-        self.config.parser.modules["ssl_module"] = None
-        self.config.parser.modules["mod_ssl.c"] = None
-        self.config.parser.modules["socache_shmcb_module"] = None
+        self.config.parser.modules.add("ssl_module")
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("socache_shmcb_module")
         # Patch _add_dummy_ssl_directives to make sure we write them correctly
         # pylint: disable=protected-access
         orig_add_dummy = self.config._add_dummy_ssl_directives
@@ -455,6 +453,41 @@ class MultipleVhostsTest(util.ApacheTest):
             "SSLCertificateChainFile", "two/cert_chain.pem",
             self.vh_truth[1].path))
 
+    def test_deploy_cert_invalid_vhost(self):
+        """For test cases where the `ApacheConfigurator` class' `_deploy_cert`
+        method is called with an invalid vhost parameter. Currently this tests
+        that a PluginError is appropriately raised when important directives
+        are missing in an SSL module."""
+        self.config.parser.modules.add("ssl_module")
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("socache_shmcb_module")
+
+        def side_effect(*args):
+            """Mocks case where an SSLCertificateFile directive can be found
+            but an SSLCertificateKeyFile directive is missing."""
+            if "SSLCertificateFile" in args:
+                return ["example/cert.pem"]
+            return []
+
+        mock_find_dir = mock.MagicMock(return_value=[])
+        mock_find_dir.side_effect = side_effect
+
+        self.config.parser.find_dir = mock_find_dir
+
+        # Get the default 443 vhost
+        self.config.assoc["random.demo"] = self.vh_truth[1]
+
+        self.assertRaises(
+            errors.PluginError, self.config.deploy_cert, "random.demo",
+            "example/cert.pem", "example/key.pem", "example/cert_chain.pem")
+
+        # Remove side_effect to mock case where both SSLCertificateFile
+        # and SSLCertificateKeyFile directives are missing
+        self.config.parser.find_dir.side_effect = None
+        self.assertRaises(
+            errors.PluginError, self.config.deploy_cert, "random.demo",
+            "example/cert.pem", "example/key.pem", "example/cert_chain.pem")
+
     def test_is_name_vhost(self):
         addr = obj.Addr.fromstring("*:80")
         self.assertTrue(self.config.is_name_vhost(addr))
@@ -510,8 +543,7 @@ class MultipleVhostsTest(util.ApacheTest):
                 call_found = True
         self.assertTrue(call_found)
 
-    @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules")
-    def test_prepare_server_https(self, mock_reset):
+    def test_prepare_server_https(self):
         mock_enable = mock.Mock()
         self.config.enable_mod = mock_enable
 
@@ -537,8 +569,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
         self.assertEqual(mock_add_dir.call_count, 2)
 
-    @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules")
-    def test_prepare_server_https_named_listen(self, mock_reset):
+    def test_prepare_server_https_named_listen(self):
         mock_find = mock.Mock()
         mock_find.return_value = ["test1", "test2", "test3"]
         mock_get = mock.Mock()
@@ -576,8 +607,7 @@ class MultipleVhostsTest(util.ApacheTest):
         # self.config.prepare_server_https("8080", temp=True)
         # self.assertEqual(self.listens, 0)
 
-    @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules")
-    def test_prepare_server_https_needed_listen(self, mock_reset):
+    def test_prepare_server_https_needed_listen(self):
         mock_find = mock.Mock()
         mock_find.return_value = ["test1", "test2"]
         mock_get = mock.Mock()
@@ -593,8 +623,8 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.prepare_server_https("443")
         self.assertEqual(mock_add_dir.call_count, 1)
 
-    @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules")
-    def test_prepare_server_https_mixed_listen(self, mock_reset):
+    def test_prepare_server_https_mixed_listen(self):
+
         mock_find = mock.Mock()
         mock_find.return_value = ["test1", "test2"]
         mock_get = mock.Mock()
@@ -651,7 +681,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertEqual(ssl_vhost.path,
                          "/files" + ssl_vhost.filep + "/IfModule/Virtualhost")
         self.assertEqual(len(ssl_vhost.addrs), 1)
-        self.assertEqual({obj.Addr.fromstring("*:443")}, ssl_vhost.addrs)
+        self.assertEqual(set([obj.Addr.fromstring("*:443")]), ssl_vhost.addrs)
         self.assertEqual(ssl_vhost.name, "encryption-example.demo")
         self.assertTrue(ssl_vhost.ssl)
         self.assertFalse(ssl_vhost.enabled)
@@ -769,7 +799,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertEqual(mock_restart.call_count, 1)
 
     @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.restart")
-    @mock.patch("certbot_apache._internal.apache_util._get_runtime_cfg")
+    @mock.patch("certbot_apache._internal.parser.ApacheParser._get_runtime_cfg")
     def test_cleanup(self, mock_cfg, mock_restart):
         mock_cfg.return_value = ""
         _, achalls = self.get_key_and_achalls()
@@ -785,7 +815,7 @@ class MultipleVhostsTest(util.ApacheTest):
                 self.assertFalse(mock_restart.called)
 
     @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.restart")
-    @mock.patch("certbot_apache._internal.apache_util._get_runtime_cfg")
+    @mock.patch("certbot_apache._internal.parser.ApacheParser._get_runtime_cfg")
     def test_cleanup_no_errors(self, mock_cfg, mock_restart):
         mock_cfg.return_value = ""
         _, achalls = self.get_key_and_achalls()
@@ -873,10 +903,10 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot_apache._internal.display_ops.select_vhost")
     @mock.patch("certbot.util.exe_exists")
     def test_enhance_unknown_vhost(self, mock_exe, mock_sel_vhost, mock_get):
-        self.config.parser.modules["rewrite_module"] = None
+        self.config.parser.modules.add("rewrite_module")
         mock_exe.return_value = True
         ssl_vh1 = obj.VirtualHost(
-            "fp1", "ap1", {obj.Addr(("*", "443"))},
+            "fp1", "ap1", set([obj.Addr(("*", "443"))]),
             True, False)
         ssl_vh1.name = "satoshi.com"
         self.config.vhosts.append(ssl_vh1)
@@ -911,8 +941,8 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot.util.exe_exists")
     def test_ocsp_stapling(self, mock_exe):
         self.config.parser.update_runtime_variables = mock.Mock()
-        self.config.parser.modules["mod_ssl.c"] = None
-        self.config.parser.modules["socache_shmcb_module"] = None
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("socache_shmcb_module")
         self.config.get_version = mock.Mock(return_value=(2, 4, 7))
         mock_exe.return_value = True
 
@@ -938,8 +968,8 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot.util.exe_exists")
     def test_ocsp_stapling_twice(self, mock_exe):
         self.config.parser.update_runtime_variables = mock.Mock()
-        self.config.parser.modules["mod_ssl.c"] = None
-        self.config.parser.modules["socache_shmcb_module"] = None
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("socache_shmcb_module")
         self.config.get_version = mock.Mock(return_value=(2, 4, 7))
         mock_exe.return_value = True
 
@@ -966,8 +996,8 @@ class MultipleVhostsTest(util.ApacheTest):
     def test_ocsp_unsupported_apache_version(self, mock_exe):
         mock_exe.return_value = True
         self.config.parser.update_runtime_variables = mock.Mock()
-        self.config.parser.modules["mod_ssl.c"] = None
-        self.config.parser.modules["socache_shmcb_module"] = None
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("socache_shmcb_module")
         self.config.get_version = mock.Mock(return_value=(2, 2, 0))
         self.config.choose_vhost("certbot.demo")
 
@@ -977,7 +1007,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
     def test_get_http_vhost_third_filter(self):
         ssl_vh = obj.VirtualHost(
-            "fp", "ap", {obj.Addr(("*", "443"))},
+            "fp", "ap", set([obj.Addr(("*", "443"))]),
             True, False)
         ssl_vh.name = "satoshi.com"
         self.config.vhosts.append(ssl_vh)
@@ -990,8 +1020,8 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot.util.exe_exists")
     def test_http_header_hsts(self, mock_exe, _):
         self.config.parser.update_runtime_variables = mock.Mock()
-        self.config.parser.modules["mod_ssl.c"] = None
-        self.config.parser.modules["headers_module"] = None
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("headers_module")
         mock_exe.return_value = True
 
         # This will create an ssl vhost for certbot.demo
@@ -1011,9 +1041,9 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertEqual(len(hsts_header), 4)
 
     def test_http_header_hsts_twice(self):
-        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules.add("mod_ssl.c")
         # skip the enable mod
-        self.config.parser.modules["headers_module"] = None
+        self.config.parser.modules.add("headers_module")
 
         # This will create an ssl vhost for encryption-example.demo
         self.config.choose_vhost("encryption-example.demo")
@@ -1029,8 +1059,8 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot.util.exe_exists")
     def test_http_header_uir(self, mock_exe, _):
         self.config.parser.update_runtime_variables = mock.Mock()
-        self.config.parser.modules["mod_ssl.c"] = None
-        self.config.parser.modules["headers_module"] = None
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("headers_module")
 
         mock_exe.return_value = True
 
@@ -1053,9 +1083,9 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertEqual(len(uir_header), 4)
 
     def test_http_header_uir_twice(self):
-        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules.add("mod_ssl.c")
         # skip the enable mod
-        self.config.parser.modules["headers_module"] = None
+        self.config.parser.modules.add("headers_module")
 
         # This will create an ssl vhost for encryption-example.demo
         self.config.choose_vhost("encryption-example.demo")
@@ -1070,7 +1100,7 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot.util.run_script")
     @mock.patch("certbot.util.exe_exists")
     def test_redirect_well_formed_http(self, mock_exe, _):
-        self.config.parser.modules["rewrite_module"] = None
+        self.config.parser.modules.add("rewrite_module")
         self.config.parser.update_runtime_variables = mock.Mock()
         mock_exe.return_value = True
         self.config.get_version = mock.Mock(return_value=(2, 2))
@@ -1096,7 +1126,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
     def test_rewrite_rule_exists(self):
         # Skip the enable mod
-        self.config.parser.modules["rewrite_module"] = None
+        self.config.parser.modules.add("rewrite_module")
         self.config.get_version = mock.Mock(return_value=(2, 3, 9))
         self.config.parser.add_dir(
             self.vh_truth[3].path, "RewriteRule", ["Unknown"])
@@ -1105,7 +1135,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
     def test_rewrite_engine_exists(self):
         # Skip the enable mod
-        self.config.parser.modules["rewrite_module"] = None
+        self.config.parser.modules.add("rewrite_module")
         self.config.get_version = mock.Mock(return_value=(2, 3, 9))
         self.config.parser.add_dir(
             self.vh_truth[3].path, "RewriteEngine", "on")
@@ -1115,7 +1145,7 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot.util.run_script")
     @mock.patch("certbot.util.exe_exists")
     def test_redirect_with_existing_rewrite(self, mock_exe, _):
-        self.config.parser.modules["rewrite_module"] = None
+        self.config.parser.modules.add("rewrite_module")
         self.config.parser.update_runtime_variables = mock.Mock()
         mock_exe.return_value = True
         self.config.get_version = mock.Mock(return_value=(2, 2, 0))
@@ -1149,7 +1179,7 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot.util.run_script")
     @mock.patch("certbot.util.exe_exists")
     def test_redirect_with_old_https_redirection(self, mock_exe, _):
-        self.config.parser.modules["rewrite_module"] = None
+        self.config.parser.modules.add("rewrite_module")
         self.config.parser.update_runtime_variables = mock.Mock()
         mock_exe.return_value = True
         self.config.get_version = mock.Mock(return_value=(2, 2, 0))
@@ -1178,10 +1208,10 @@ class MultipleVhostsTest(util.ApacheTest):
 
 
     def test_redirect_with_conflict(self):
-        self.config.parser.modules["rewrite_module"] = None
+        self.config.parser.modules.add("rewrite_module")
         ssl_vh = obj.VirtualHost(
-            "fp", "ap", {obj.Addr(("*", "443")),
-                             obj.Addr(("zombo.com",))},
+            "fp", "ap", set([obj.Addr(("*", "443")),
+                             obj.Addr(("zombo.com",))]),
             True, False)
         # No names ^ this guy should conflict.
 
@@ -1191,7 +1221,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
     def test_redirect_two_domains_one_vhost(self):
         # Skip the enable mod
-        self.config.parser.modules["rewrite_module"] = None
+        self.config.parser.modules.add("rewrite_module")
         self.config.get_version = mock.Mock(return_value=(2, 3, 9))
 
         # Creates ssl vhost for the domain
@@ -1206,7 +1236,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
     def test_redirect_from_previous_run(self):
         # Skip the enable mod
-        self.config.parser.modules["rewrite_module"] = None
+        self.config.parser.modules.add("rewrite_module")
         self.config.get_version = mock.Mock(return_value=(2, 3, 9))
         self.config.choose_vhost("red.blue.purple.com")
         self.config.enhance("red.blue.purple.com", "redirect")
@@ -1219,22 +1249,22 @@ class MultipleVhostsTest(util.ApacheTest):
             self.config.enhance, "green.blue.purple.com", "redirect")
 
     def test_create_own_redirect(self):
-        self.config.parser.modules["rewrite_module"] = None
+        self.config.parser.modules.add("rewrite_module")
         self.config.get_version = mock.Mock(return_value=(2, 3, 9))
         # For full testing... give names...
         self.vh_truth[1].name = "default.com"
-        self.vh_truth[1].aliases = {"yes.default.com"}
+        self.vh_truth[1].aliases = set(["yes.default.com"])
 
         # pylint: disable=protected-access
         self.config._enable_redirect(self.vh_truth[1], "")
         self.assertEqual(len(self.config.vhosts), 13)
 
     def test_create_own_redirect_for_old_apache_version(self):
-        self.config.parser.modules["rewrite_module"] = None
+        self.config.parser.modules.add("rewrite_module")
         self.config.get_version = mock.Mock(return_value=(2, 2))
         # For full testing... give names...
         self.vh_truth[1].name = "default.com"
-        self.vh_truth[1].aliases = {"yes.default.com"}
+        self.vh_truth[1].aliases = set(["yes.default.com"])
 
         # pylint: disable=protected-access
         self.config._enable_redirect(self.vh_truth[1], "")
@@ -1295,9 +1325,9 @@ class MultipleVhostsTest(util.ApacheTest):
     def test_deploy_cert_not_parsed_path(self):
         # Make sure that we add include to root config for vhosts when
         # handle-sites is false
-        self.config.parser.modules["ssl_module"] = None
-        self.config.parser.modules["mod_ssl.c"] = None
-        self.config.parser.modules["socache_shmcb_module"] = None
+        self.config.parser.modules.add("ssl_module")
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("socache_shmcb_module")
         tmp_path = filesystem.realpath(tempfile.mkdtemp("vhostroot"))
         filesystem.chmod(tmp_path, 0o755)
         mock_p = "certbot_apache._internal.configurator.ApacheConfigurator._get_ssl_vhost_path"
@@ -1314,16 +1344,6 @@ class MultipleVhostsTest(util.ApacheTest):
                 self.assertTrue(mock_add.called)
         shutil.rmtree(tmp_path)
 
-    def test_deploy_cert_no_mod_ssl(self):
-        # Create
-        ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])
-        self.config.parser.modules["socache_shmcb_module"] = None
-        self.config.prepare_server_https = mock.Mock()
-
-        self.assertRaises(errors.MisconfigurationError, self.config.deploy_cert,
-            "encryption-example.demo", "example/cert.pem", "example/key.pem",
-            "example/cert_chain.pem", "example/fullchain.pem")
-
     @mock.patch("certbot_apache._internal.parser.ApacheParser.parsed_in_original")
     def test_choose_vhost_and_servername_addition_parsed(self, mock_parsed):
         ret_vh = self.vh_truth[8]
@@ -1420,8 +1440,8 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator._choose_vhosts_wildcard")
     def test_enhance_wildcard_after_install(self, mock_choose):
         # pylint: disable=protected-access
-        self.config.parser.modules["mod_ssl.c"] = None
-        self.config.parser.modules["headers_module"] = None
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("headers_module")
         self.vh_truth[3].ssl = True
         self.config._wildcard_vhosts["*.certbot.demo"] = [self.vh_truth[3]]
         self.config.enhance("*.certbot.demo", "ensure-http-header",
@@ -1432,8 +1452,8 @@ class MultipleVhostsTest(util.ApacheTest):
     def test_enhance_wildcard_no_install(self, mock_choose):
         self.vh_truth[3].ssl = True
         mock_choose.return_value = [self.vh_truth[3]]
-        self.config.parser.modules["mod_ssl.c"] = None
-        self.config.parser.modules["headers_module"] = None
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("headers_module")
         self.config.enhance("*.certbot.demo", "ensure-http-header",
                             "Upgrade-Insecure-Requests")
         self.assertTrue(mock_choose.called)
@@ -1585,7 +1605,7 @@ class MultiVhostsTest(util.ApacheTest):
         self.assertEqual(ssl_vhost.path,
                          "/files" + ssl_vhost.filep + "/IfModule/VirtualHost")
         self.assertEqual(len(ssl_vhost.addrs), 1)
-        self.assertEqual({obj.Addr.fromstring("*:443")}, ssl_vhost.addrs)
+        self.assertEqual(set([obj.Addr.fromstring("*:443")]), ssl_vhost.addrs)
         self.assertEqual(ssl_vhost.name, "banana.vomit.com")
         self.assertTrue(ssl_vhost.ssl)
         self.assertFalse(ssl_vhost.enabled)
@@ -1617,7 +1637,7 @@ class MultiVhostsTest(util.ApacheTest):
 
     @certbot_util.patch_get_utility()
     def test_make_vhost_ssl_with_existing_rewrite_rule(self, mock_get_utility):
-        self.config.parser.modules["rewrite_module"] = None
+        self.config.parser.modules.add("rewrite_module")
 
         ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[4])
 
@@ -1637,7 +1657,7 @@ class MultiVhostsTest(util.ApacheTest):
 
     @certbot_util.patch_get_utility()
     def test_make_vhost_ssl_with_existing_rewrite_conds(self, mock_get_utility):
-        self.config.parser.modules["rewrite_module"] = None
+        self.config.parser.modules.add("rewrite_module")
 
         ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[3])
 
@@ -1679,7 +1699,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
                                              self.config.updated_mod_ssl_conf_digest)
 
     def _current_ssl_options_hash(self):
-        return crypto_util.sha256sum(self.config.pick_apache_config())
+        return crypto_util.sha256sum(self.config.option("MOD_SSL_CONF_SRC"))
 
     def _assert_current_file(self):
         self.assertTrue(os.path.isfile(self.config.mod_ssl_conf))
@@ -1715,7 +1735,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
             self.assertFalse(mock_logger.warning.called)
         self.assertTrue(os.path.isfile(self.config.mod_ssl_conf))
         self.assertEqual(crypto_util.sha256sum(
-            self.config.pick_apache_config()),
+            self.config.option("MOD_SSL_CONF_SRC")),
             self._current_ssl_options_hash())
         self.assertNotEqual(crypto_util.sha256sum(self.config.mod_ssl_conf),
             self._current_ssl_options_hash())
@@ -1731,118 +1751,19 @@ class InstallSslOptionsConfTest(util.ApacheTest):
                 "%s has been manually modified; updated file "
                 "saved to %s. We recommend updating %s for security purposes.")
         self.assertEqual(crypto_util.sha256sum(
-            self.config.pick_apache_config()),
+            self.config.option("MOD_SSL_CONF_SRC")),
             self._current_ssl_options_hash())
         # only print warning once
         with mock.patch("certbot.plugins.common.logger") as mock_logger:
             self._call()
             self.assertFalse(mock_logger.warning.called)
 
-    def test_ssl_config_files_hash_in_all_hashes(self):
-        """
-        It is really critical that all TLS Apache config files have their SHA256 hash registered in
-        constants.ALL_SSL_OPTIONS_HASHES. Otherwise Certbot will mistakenly assume that the config
-        file has been manually edited by the user, and will refuse to update it.
-        This test ensures that all necessary hashes are present.
-        """
+    def test_current_file_hash_in_all_hashes(self):
         from certbot_apache._internal.constants import ALL_SSL_OPTIONS_HASHES
-        import pkg_resources
+        self.assertTrue(self._current_ssl_options_hash() in ALL_SSL_OPTIONS_HASHES,
+            "Constants.ALL_SSL_OPTIONS_HASHES must be appended"
+            " with the sha256 hash of self.config.mod_ssl_conf when it is updated.")
 
-        tls_configs_dir = pkg_resources.resource_filename(
-            "certbot_apache", os.path.join("_internal", "tls_configs"))
-        all_files = [os.path.join(tls_configs_dir, name) for name in os.listdir(tls_configs_dir)
-                     if name.endswith('options-ssl-apache.conf')]
-        self.assertTrue(all_files)
-        for one_file in all_files:
-            file_hash = crypto_util.sha256sum(one_file)
-            self.assertTrue(file_hash in ALL_SSL_OPTIONS_HASHES,
-                            "Constants.ALL_SSL_OPTIONS_HASHES must be appended with the sha256 "
-                            "hash of {0} when it is updated.".format(one_file))
-
-    def test_openssl_version(self):
-        self.config._openssl_version = None
-        some_string_contents = b"""
-            SSLOpenSSLConfCmd
-            OpenSSL configuration command
-            SSLv3 not supported by this version of OpenSSL
-            '%s': invalid OpenSSL configuration command
-            OpenSSL 1.0.2g  1 Mar 2016
-            OpenSSL
-            AH02407: "SSLOpenSSLConfCmd %s %s" failed for %s
-            AH02556: "SSLOpenSSLConfCmd %s %s" applied to %s
-            OpenSSL 1.0.2g  1 Mar 2016
-            """
-        # ssl_module as a DSO
-        self.config.parser.modules['ssl_module'] = '/fake/path'
-        with mock.patch("certbot_apache._internal.configurator."
-            "ApacheConfigurator._open_module_file") as mock_omf:
-            mock_omf.return_value = some_string_contents
-            self.assertEqual(self.config.openssl_version(), "1.0.2g")
-
-        # ssl_module statically linked
-        self.config._openssl_version = None
-        self.config.parser.modules['ssl_module'] = None
-        self.config.options['bin'] = '/fake/path/to/httpd'
-        with mock.patch("certbot_apache._internal.configurator."
-            "ApacheConfigurator._open_module_file") as mock_omf:
-            mock_omf.return_value = some_string_contents
-            self.assertEqual(self.config.openssl_version(), "1.0.2g")
-
-    def test_current_version(self):
-        self.config.version = (2, 4, 10)
-        self.config._openssl_version = '1.0.2m'
-        self.assertTrue('old' in self.config.pick_apache_config())
-
-        self.config.version = (2, 4, 11)
-        self.config._openssl_version = '1.0.2m'
-        self.assertTrue('current' in self.config.pick_apache_config())
-
-        self.config._openssl_version = '1.0.2a'
-        self.assertTrue('old' in self.config.pick_apache_config())
-
-    def test_openssl_version_warns(self):
-        self.config._openssl_version = '1.0.2a'
-        self.assertEqual(self.config.openssl_version(), '1.0.2a')
-
-        self.config._openssl_version = None
-        with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log:
-            self.assertEqual(self.config.openssl_version(), None)
-            self.assertTrue("Could not find ssl_module" in mock_log.call_args[0][0])
-
-        # When no ssl_module is present at all
-        self.config._openssl_version = None
-        self.assertTrue("ssl_module" not in self.config.parser.modules)
-        with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log:
-            self.assertEqual(self.config.openssl_version(), None)
-            self.assertTrue("Could not find ssl_module" in mock_log.call_args[0][0])
-
-        # When ssl_module is statically linked but --apache-bin not provided
-        self.config._openssl_version = None
-        self.config.options['bin'] = None
-        self.config.parser.modules['ssl_module'] = None
-        with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log:
-            self.assertEqual(self.config.openssl_version(), None)
-            self.assertTrue("ssl_module is statically linked but" in mock_log.call_args[0][0])
-
-        self.config.parser.modules['ssl_module'] = "/fake/path"
-        with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log:
-            # Check that correct logger.warning was printed
-            self.assertEqual(self.config.openssl_version(), None)
-            self.assertTrue("Unable to read" in mock_log.call_args[0][0])
-
-        contents_missing_openssl = b"these contents won't match the regex"
-        with mock.patch("certbot_apache._internal.configurator."
-            "ApacheConfigurator._open_module_file") as mock_omf:
-            mock_omf.return_value = contents_missing_openssl
-            with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log:
-                # Check that correct logger.warning was printed
-                self.assertEqual(self.config.openssl_version(), None)
-                self.assertTrue("Could not find OpenSSL" in mock_log.call_args[0][0])
-
-    def test_open_module_file(self):
-        mock_open = mock.mock_open(read_data="testing 12 3")
-        with mock.patch("six.moves.builtins.open", mock_open):
-            self.assertEqual(self.config._open_module_file("/nonsense/"), "testing 12 3")
 
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot-apache/tests/debian_test.py

@@ -2,10 +2,7 @@
 import shutil
 import unittest
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 
 from certbot import errors
 from certbot.compat import os
@@ -49,7 +46,7 @@ class MultipleVhostsTestDebian(util.ApacheTest):
 
     @mock.patch("certbot.util.run_script")
     @mock.patch("certbot.util.exe_exists")
-    @mock.patch("certbot_apache._internal.apache_util.subprocess.Popen")
+    @mock.patch("certbot_apache._internal.parser.subprocess.Popen")
     def test_enable_mod(self, mock_popen, mock_exe_exists, mock_run_script):
         mock_popen().communicate.return_value = ("Define: DUMP_RUN_CFG", "")
         mock_popen().returncode = 0
@@ -64,8 +61,8 @@ class MultipleVhostsTestDebian(util.ApacheTest):
     def test_deploy_cert_enable_new_vhost(self):
         # Create
         ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])
-        self.config.parser.modules["ssl_module"] = None
-        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules.add("ssl_module")
+        self.config.parser.modules.add("mod_ssl.c")
         self.assertFalse(ssl_vhost.enabled)
         self.config.deploy_cert(
             "encryption-example.demo", "example/cert.pem", "example/key.pem",
@@ -95,8 +92,8 @@ class MultipleVhostsTestDebian(util.ApacheTest):
             self.config_path, self.vhost_path, self.config_dir,
             self.work_dir, version=(2, 4, 16))
         self.config = self.mock_deploy_cert(self.config)
-        self.config.parser.modules["ssl_module"] = None
-        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules.add("ssl_module")
+        self.config.parser.modules.add("mod_ssl.c")
 
         # Get the default 443 vhost
         self.config.assoc["random.demo"] = self.vh_truth[1]
@@ -131,8 +128,8 @@ class MultipleVhostsTestDebian(util.ApacheTest):
             self.config_path, self.vhost_path, self.config_dir,
             self.work_dir, version=(2, 4, 16))
         self.config = self.mock_deploy_cert(self.config)
-        self.config.parser.modules["ssl_module"] = None
-        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules.add("ssl_module")
+        self.config.parser.modules.add("mod_ssl.c")
 
         # Get the default 443 vhost
         self.config.assoc["random.demo"] = self.vh_truth[1]
@@ -146,8 +143,8 @@ class MultipleVhostsTestDebian(util.ApacheTest):
             self.config_path, self.vhost_path, self.config_dir,
             self.work_dir, version=(2, 4, 7))
         self.config = self.mock_deploy_cert(self.config)
-        self.config.parser.modules["ssl_module"] = None
-        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules.add("ssl_module")
+        self.config.parser.modules.add("mod_ssl.c")
 
         # Get the default 443 vhost
         self.config.assoc["random.demo"] = self.vh_truth[1]
@@ -160,7 +157,7 @@ class MultipleVhostsTestDebian(util.ApacheTest):
     @mock.patch("certbot.util.exe_exists")
     def test_ocsp_stapling_enable_mod(self, mock_exe, _):
         self.config.parser.update_runtime_variables = mock.Mock()
-        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules.add("mod_ssl.c")
         self.config.get_version = mock.Mock(return_value=(2, 4, 7))
         mock_exe.return_value = True
         # This will create an ssl vhost for certbot.demo
@@ -172,7 +169,7 @@ class MultipleVhostsTestDebian(util.ApacheTest):
     @mock.patch("certbot.util.exe_exists")
     def test_ensure_http_header_enable_mod(self, mock_exe, _):
         self.config.parser.update_runtime_variables = mock.Mock()
-        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules.add("mod_ssl.c")
         mock_exe.return_value = True
 
         # This will create an ssl vhost for certbot.demo

certbot-apache/tests/display_ops_test.py

@@ -1,10 +1,7 @@
 """Test certbot_apache._internal.display_ops."""
 import unittest
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 
 from certbot import errors
 from certbot.display import util as display_util
@@ -96,9 +93,9 @@ class SelectVhostTest(unittest.TestCase):
 
         self.vhosts.append(
             obj.VirtualHost(
-                "path", "aug_path", {obj.Addr.fromstring("*:80")},
+                "path", "aug_path", set([obj.Addr.fromstring("*:80")]),
                 False, False,
-                "wildcard.com", {"*.wildcard.com"}))
+                "wildcard.com", set(["*.wildcard.com"])))
 
         self.assertEqual(self.vhosts[5], self._call(self.vhosts))
 

certbot-apache/tests/dualnode_test.py

@@ -1,445 +0,0 @@
-"""Tests for DualParserNode implementation"""
-import unittest
-
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
-
-from certbot_apache._internal import assertions
-from certbot_apache._internal import augeasparser
-from certbot_apache._internal import dualparser
-
-
-class DualParserNodeTest(unittest.TestCase):  # pylint: disable=too-many-public-methods
-    """DualParserNode tests"""
-
-    def setUp(self):  # pylint: disable=arguments-differ
-        parser_mock = mock.MagicMock()
-        parser_mock.aug.match.return_value = []
-        parser_mock.get_arg.return_value = []
-        self.metadata = {"augeasparser": parser_mock, "augeaspath": "/invalid", "ac_ast": None}
-        self.block = dualparser.DualBlockNode(name="block",
-                                              ancestor=None,
-                                              filepath="/tmp/something",
-                                              metadata=self.metadata)
-        self.block_two = dualparser.DualBlockNode(name="block",
-                                                  ancestor=self.block,
-                                                  filepath="/tmp/something",
-                                                  metadata=self.metadata)
-        self.directive = dualparser.DualDirectiveNode(name="directive",
-                                                      ancestor=self.block,
-                                                      filepath="/tmp/something",
-                                                      metadata=self.metadata)
-        self.comment = dualparser.DualCommentNode(comment="comment",
-                                                  ancestor=self.block,
-                                                  filepath="/tmp/something",
-                                                  metadata=self.metadata)
-
-    def test_create_with_precreated(self):
-        cnode = dualparser.DualCommentNode(comment="comment",
-                                           ancestor=self.block,
-                                           filepath="/tmp/something",
-                                           primary=self.comment.secondary,
-                                           secondary=self.comment.primary)
-        dnode = dualparser.DualDirectiveNode(name="directive",
-                                             ancestor=self.block,
-                                             filepath="/tmp/something",
-                                             primary=self.directive.secondary,
-                                             secondary=self.directive.primary)
-        bnode = dualparser.DualBlockNode(name="block",
-                                         ancestor=self.block,
-                                         filepath="/tmp/something",
-                                         primary=self.block.secondary,
-                                         secondary=self.block.primary)
-        # Switched around
-        self.assertTrue(cnode.primary is self.comment.secondary)
-        self.assertTrue(cnode.secondary is self.comment.primary)
-        self.assertTrue(dnode.primary is self.directive.secondary)
-        self.assertTrue(dnode.secondary is self.directive.primary)
-        self.assertTrue(bnode.primary is self.block.secondary)
-        self.assertTrue(bnode.secondary is self.block.primary)
-
-    def test_set_params(self):
-        params = ("first", "second")
-        self.directive.primary.set_parameters = mock.Mock()
-        self.directive.secondary.set_parameters = mock.Mock()
-        self.directive.set_parameters(params)
-        self.assertTrue(self.directive.primary.set_parameters.called)
-        self.assertTrue(self.directive.secondary.set_parameters.called)
-
-    def test_set_parameters(self):
-        pparams = mock.MagicMock()
-        sparams = mock.MagicMock()
-        pparams.parameters = ("a", "b")
-        sparams.parameters = ("a", "b")
-        self.directive.primary.set_parameters = pparams
-        self.directive.secondary.set_parameters = sparams
-        self.directive.set_parameters(("param", "seq"))
-        self.assertTrue(pparams.called)
-        self.assertTrue(sparams.called)
-
-    def test_delete_child(self):
-        pdel = mock.MagicMock()
-        sdel = mock.MagicMock()
-        self.block.primary.delete_child = pdel
-        self.block.secondary.delete_child = sdel
-        self.block.delete_child(self.comment)
-        self.assertTrue(pdel.called)
-        self.assertTrue(sdel.called)
-
-    def test_unsaved_files(self):
-        puns = mock.MagicMock()
-        suns = mock.MagicMock()
-        puns.return_value = assertions.PASS
-        suns.return_value = assertions.PASS
-        self.block.primary.unsaved_files = puns
-        self.block.secondary.unsaved_files = suns
-        self.block.unsaved_files()
-        self.assertTrue(puns.called)
-        self.assertTrue(suns.called)
-
-    def test_getattr_equality(self):
-        self.directive.primary.variableexception = "value"
-        self.directive.secondary.variableexception = "not_value"
-        with self.assertRaises(AssertionError):
-            _ = self.directive.variableexception
-
-        self.directive.primary.variable = "value"
-        self.directive.secondary.variable = "value"
-        try:
-            self.directive.variable
-        except AssertionError: # pragma: no cover
-            self.fail("getattr check raised an AssertionError where it shouldn't have")
-
-    def test_parsernode_dirty_assert(self):
-        # disable assertion pass
-        self.comment.primary.comment = "value"
-        self.comment.secondary.comment = "value"
-        self.comment.primary.filepath = "x"
-        self.comment.secondary.filepath = "x"
-
-        self.comment.primary.dirty = False
-        self.comment.secondary.dirty = True
-        with self.assertRaises(AssertionError):
-            assertions.assertEqual(self.comment.primary, self.comment.secondary)
-
-    def test_parsernode_filepath_assert(self):
-        # disable assertion pass
-        self.comment.primary.comment = "value"
-        self.comment.secondary.comment = "value"
-
-        self.comment.primary.filepath = "first"
-        self.comment.secondary.filepath = "second"
-        with self.assertRaises(AssertionError):
-            assertions.assertEqual(self.comment.primary, self.comment.secondary)
-
-    def test_add_child_block(self):
-        mock_first = mock.MagicMock(return_value=self.block.primary)
-        mock_second = mock.MagicMock(return_value=self.block.secondary)
-        self.block.primary.add_child_block = mock_first
-        self.block.secondary.add_child_block = mock_second
-        self.block.add_child_block("Block")
-        self.assertTrue(mock_first.called)
-        self.assertTrue(mock_second.called)
-
-    def test_add_child_directive(self):
-        mock_first = mock.MagicMock(return_value=self.directive.primary)
-        mock_second = mock.MagicMock(return_value=self.directive.secondary)
-        self.block.primary.add_child_directive = mock_first
-        self.block.secondary.add_child_directive = mock_second
-        self.block.add_child_directive("Directive")
-        self.assertTrue(mock_first.called)
-        self.assertTrue(mock_second.called)
-
-    def test_add_child_comment(self):
-        mock_first = mock.MagicMock(return_value=self.comment.primary)
-        mock_second = mock.MagicMock(return_value=self.comment.secondary)
-        self.block.primary.add_child_comment = mock_first
-        self.block.secondary.add_child_comment = mock_second
-        self.block.add_child_comment("Comment")
-        self.assertTrue(mock_first.called)
-        self.assertTrue(mock_second.called)
-
-    def test_find_comments(self):
-        pri_comments = [augeasparser.AugeasCommentNode(comment="some comment",
-                                                       ancestor=self.block,
-                                                       filepath="/path/to/whatever",
-                                                       metadata=self.metadata)]
-        sec_comments = [augeasparser.AugeasCommentNode(comment=assertions.PASS,
-                                                       ancestor=self.block,
-                                                       filepath=assertions.PASS,
-                                                       metadata=self.metadata)]
-        find_coms_primary = mock.MagicMock(return_value=pri_comments)
-        find_coms_secondary = mock.MagicMock(return_value=sec_comments)
-        self.block.primary.find_comments = find_coms_primary
-        self.block.secondary.find_comments = find_coms_secondary
-
-        dcoms = self.block.find_comments("comment")
-        p_dcoms = [d.primary for d in dcoms]
-        s_dcoms = [d.secondary for d in dcoms]
-        p_coms = self.block.primary.find_comments("comment")
-        s_coms = self.block.secondary.find_comments("comment")
-        # Check that every comment response is represented in the list of
-        # DualParserNode instances.
-        for p in p_dcoms:
-            self.assertTrue(p in p_coms)
-        for s in s_dcoms:
-            self.assertTrue(s in s_coms)
-
-    def test_find_blocks_first_passing(self):
-        youshallnotpass = [augeasparser.AugeasBlockNode(name="notpassing",
-                                                        ancestor=self.block,
-                                                        filepath="/path/to/whatever",
-                                                        metadata=self.metadata)]
-        youshallpass = [augeasparser.AugeasBlockNode(name=assertions.PASS,
-                                                     ancestor=self.block,
-                                                     filepath=assertions.PASS,
-                                                     metadata=self.metadata)]
-        find_blocks_primary = mock.MagicMock(return_value=youshallpass)
-        find_blocks_secondary = mock.MagicMock(return_value=youshallnotpass)
-        self.block.primary.find_blocks = find_blocks_primary
-        self.block.secondary.find_blocks = find_blocks_secondary
-
-        blocks = self.block.find_blocks("something")
-        for block in blocks:
-            try:
-                assertions.assertEqual(block.primary, block.secondary)
-            except AssertionError: # pragma: no cover
-                self.fail("Assertion should have passed")
-            self.assertTrue(assertions.isPassDirective(block.primary))
-            self.assertFalse(assertions.isPassDirective(block.secondary))
-
-    def test_find_blocks_second_passing(self):
-        youshallnotpass = [augeasparser.AugeasBlockNode(name="notpassing",
-                                                        ancestor=self.block,
-                                                        filepath="/path/to/whatever",
-                                                        metadata=self.metadata)]
-        youshallpass = [augeasparser.AugeasBlockNode(name=assertions.PASS,
-                                                     ancestor=self.block,
-                                                     filepath=assertions.PASS,
-                                                     metadata=self.metadata)]
-        find_blocks_primary = mock.MagicMock(return_value=youshallnotpass)
-        find_blocks_secondary = mock.MagicMock(return_value=youshallpass)
-        self.block.primary.find_blocks = find_blocks_primary
-        self.block.secondary.find_blocks = find_blocks_secondary
-
-        blocks = self.block.find_blocks("something")
-        for block in blocks:
-            try:
-                assertions.assertEqual(block.primary, block.secondary)
-            except AssertionError: # pragma: no cover
-                self.fail("Assertion should have passed")
-            self.assertFalse(assertions.isPassDirective(block.primary))
-            self.assertTrue(assertions.isPassDirective(block.secondary))
-
-    def test_find_dirs_first_passing(self):
-        notpassing = [augeasparser.AugeasDirectiveNode(name="notpassing",
-                                                       ancestor=self.block,
-                                                       filepath="/path/to/whatever",
-                                                       metadata=self.metadata)]
-        passing = [augeasparser.AugeasDirectiveNode(name=assertions.PASS,
-                                                    ancestor=self.block,
-                                                    filepath=assertions.PASS,
-                                                    metadata=self.metadata)]
-        find_dirs_primary = mock.MagicMock(return_value=passing)
-        find_dirs_secondary = mock.MagicMock(return_value=notpassing)
-        self.block.primary.find_directives = find_dirs_primary
-        self.block.secondary.find_directives = find_dirs_secondary
-
-        directives = self.block.find_directives("something")
-        for directive in directives:
-            try:
-                assertions.assertEqual(directive.primary, directive.secondary)
-            except AssertionError: # pragma: no cover
-                self.fail("Assertion should have passed")
-            self.assertTrue(assertions.isPassDirective(directive.primary))
-            self.assertFalse(assertions.isPassDirective(directive.secondary))
-
-    def test_find_dirs_second_passing(self):
-        notpassing = [augeasparser.AugeasDirectiveNode(name="notpassing",
-                                                       ancestor=self.block,
-                                                       filepath="/path/to/whatever",
-                                                       metadata=self.metadata)]
-        passing = [augeasparser.AugeasDirectiveNode(name=assertions.PASS,
-                                                    ancestor=self.block,
-                                                    filepath=assertions.PASS,
-                                                    metadata=self.metadata)]
-        find_dirs_primary = mock.MagicMock(return_value=notpassing)
-        find_dirs_secondary = mock.MagicMock(return_value=passing)
-        self.block.primary.find_directives = find_dirs_primary
-        self.block.secondary.find_directives = find_dirs_secondary
-
-        directives = self.block.find_directives("something")
-        for directive in directives:
-            try:
-                assertions.assertEqual(directive.primary, directive.secondary)
-            except AssertionError: # pragma: no cover
-                self.fail("Assertion should have passed")
-            self.assertFalse(assertions.isPassDirective(directive.primary))
-            self.assertTrue(assertions.isPassDirective(directive.secondary))
-
-    def test_find_coms_first_passing(self):
-        notpassing = [augeasparser.AugeasCommentNode(comment="notpassing",
-                                                     ancestor=self.block,
-                                                     filepath="/path/to/whatever",
-                                                     metadata=self.metadata)]
-        passing = [augeasparser.AugeasCommentNode(comment=assertions.PASS,
-                                                  ancestor=self.block,
-                                                  filepath=assertions.PASS,
-                                                  metadata=self.metadata)]
-        find_coms_primary = mock.MagicMock(return_value=passing)
-        find_coms_secondary = mock.MagicMock(return_value=notpassing)
-        self.block.primary.find_comments = find_coms_primary
-        self.block.secondary.find_comments = find_coms_secondary
-
-        comments = self.block.find_comments("something")
-        for comment in comments:
-            try:
-                assertions.assertEqual(comment.primary, comment.secondary)
-            except AssertionError: # pragma: no cover
-                self.fail("Assertion should have passed")
-            self.assertTrue(assertions.isPassComment(comment.primary))
-            self.assertFalse(assertions.isPassComment(comment.secondary))
-
-    def test_find_coms_second_passing(self):
-        notpassing = [augeasparser.AugeasCommentNode(comment="notpassing",
-                                                     ancestor=self.block,
-                                                     filepath="/path/to/whatever",
-                                                     metadata=self.metadata)]
-        passing = [augeasparser.AugeasCommentNode(comment=assertions.PASS,
-                                                  ancestor=self.block,
-                                                  filepath=assertions.PASS,
-                                                  metadata=self.metadata)]
-        find_coms_primary = mock.MagicMock(return_value=notpassing)
-        find_coms_secondary = mock.MagicMock(return_value=passing)
-        self.block.primary.find_comments = find_coms_primary
-        self.block.secondary.find_comments = find_coms_secondary
-
-        comments = self.block.find_comments("something")
-        for comment in comments:
-            try:
-                assertions.assertEqual(comment.primary, comment.secondary)
-            except AssertionError: # pragma: no cover
-                self.fail("Assertion should have passed")
-            self.assertFalse(assertions.isPassComment(comment.primary))
-            self.assertTrue(assertions.isPassComment(comment.secondary))
-
-    def test_find_blocks_no_pass_equal(self):
-        notpassing1 = [augeasparser.AugeasBlockNode(name="notpassing",
-                                                    ancestor=self.block,
-                                                    filepath="/path/to/whatever",
-                                                    metadata=self.metadata)]
-        notpassing2 = [augeasparser.AugeasBlockNode(name="notpassing",
-                                                    ancestor=self.block,
-                                                    filepath="/path/to/whatever",
-                                                    metadata=self.metadata)]
-        find_blocks_primary = mock.MagicMock(return_value=notpassing1)
-        find_blocks_secondary = mock.MagicMock(return_value=notpassing2)
-        self.block.primary.find_blocks = find_blocks_primary
-        self.block.secondary.find_blocks = find_blocks_secondary
-
-        blocks = self.block.find_blocks("anything")
-        for block in blocks:
-            self.assertEqual(block.primary, block.secondary)
-            self.assertTrue(block.primary is not block.secondary)
-
-    def test_find_dirs_no_pass_equal(self):
-        notpassing1 = [augeasparser.AugeasDirectiveNode(name="notpassing",
-                                                        ancestor=self.block,
-                                                        filepath="/path/to/whatever",
-                                                        metadata=self.metadata)]
-        notpassing2 = [augeasparser.AugeasDirectiveNode(name="notpassing",
-                                                        ancestor=self.block,
-                                                        filepath="/path/to/whatever",
-                                                        metadata=self.metadata)]
-        find_dirs_primary = mock.MagicMock(return_value=notpassing1)
-        find_dirs_secondary = mock.MagicMock(return_value=notpassing2)
-        self.block.primary.find_directives = find_dirs_primary
-        self.block.secondary.find_directives = find_dirs_secondary
-
-        directives = self.block.find_directives("anything")
-        for directive in directives:
-            self.assertEqual(directive.primary, directive.secondary)
-            self.assertTrue(directive.primary is not directive.secondary)
-
-    def test_find_comments_no_pass_equal(self):
-        notpassing1 = [augeasparser.AugeasCommentNode(comment="notpassing",
-                                                      ancestor=self.block,
-                                                      filepath="/path/to/whatever",
-                                                      metadata=self.metadata)]
-        notpassing2 = [augeasparser.AugeasCommentNode(comment="notpassing",
-                                                      ancestor=self.block,
-                                                      filepath="/path/to/whatever",
-                                                      metadata=self.metadata)]
-        find_coms_primary = mock.MagicMock(return_value=notpassing1)
-        find_coms_secondary = mock.MagicMock(return_value=notpassing2)
-        self.block.primary.find_comments = find_coms_primary
-        self.block.secondary.find_comments = find_coms_secondary
-
-        comments = self.block.find_comments("anything")
-        for comment in comments:
-            self.assertEqual(comment.primary, comment.secondary)
-            self.assertTrue(comment.primary is not comment.secondary)
-
-    def test_find_blocks_no_pass_notequal(self):
-        notpassing1 = [augeasparser.AugeasBlockNode(name="notpassing",
-                                                    ancestor=self.block,
-                                                    filepath="/path/to/whatever",
-                                                    metadata=self.metadata)]
-        notpassing2 = [augeasparser.AugeasBlockNode(name="different",
-                                                    ancestor=self.block,
-                                                    filepath="/path/to/whatever",
-                                                    metadata=self.metadata)]
-        find_blocks_primary = mock.MagicMock(return_value=notpassing1)
-        find_blocks_secondary = mock.MagicMock(return_value=notpassing2)
-        self.block.primary.find_blocks = find_blocks_primary
-        self.block.secondary.find_blocks = find_blocks_secondary
-
-        with self.assertRaises(AssertionError):
-            _ = self.block.find_blocks("anything")
-
-    def test_parsernode_notequal(self):
-        ne_block = augeasparser.AugeasBlockNode(name="different",
-                                                ancestor=self.block,
-                                                filepath="/path/to/whatever",
-                                                metadata=self.metadata)
-        ne_directive = augeasparser.AugeasDirectiveNode(name="different",
-                                                        ancestor=self.block,
-                                                        filepath="/path/to/whatever",
-                                                        metadata=self.metadata)
-        ne_comment = augeasparser.AugeasCommentNode(comment="different",
-                                                    ancestor=self.block,
-                                                    filepath="/path/to/whatever",
-                                                    metadata=self.metadata)
-        self.assertFalse(self.block == ne_block)
-        self.assertFalse(self.directive == ne_directive)
-        self.assertFalse(self.comment == ne_comment)
-
-    def test_parsed_paths(self):
-        mock_p = mock.MagicMock(return_value=['/path/file.conf',
-                                              '/another/path',
-                                              '/path/other.conf'])
-        mock_s = mock.MagicMock(return_value=['/path/*.conf', '/another/path'])
-        self.block.primary.parsed_paths = mock_p
-        self.block.secondary.parsed_paths = mock_s
-        self.block.parsed_paths()
-        self.assertTrue(mock_p.called)
-        self.assertTrue(mock_s.called)
-
-    def test_parsed_paths_error(self):
-        mock_p = mock.MagicMock(return_value=['/path/file.conf'])
-        mock_s = mock.MagicMock(return_value=['/path/*.conf', '/another/path'])
-        self.block.primary.parsed_paths = mock_p
-        self.block.secondary.parsed_paths = mock_s
-        with self.assertRaises(AssertionError):
-            self.block.parsed_paths()
-
-    def test_find_ancestors(self):
-        primarymock = mock.MagicMock(return_value=[])
-        secondarymock = mock.MagicMock(return_value=[])
-        self.block.primary.find_ancestors = primarymock
-        self.block.secondary.find_ancestors = secondarymock
-        self.block.find_ancestors("anything")
-        self.assertTrue(primarymock.called)
-        self.assertTrue(secondarymock.called)

certbot-apache/tests/entrypoint_test.py

@@ -1,10 +1,7 @@
 """Test for certbot_apache._internal.entrypoint for override class resolution"""
 import unittest
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 
 from certbot_apache._internal import configurator
 from certbot_apache._internal import entrypoint

certbot-apache/tests/fedora_test.py

@@ -1,10 +1,7 @@
 """Test for certbot_apache._internal.configurator for Fedora 29+ overrides"""
 import unittest
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 
 from certbot import errors
 from certbot.compat import filesystem
@@ -103,7 +100,7 @@ class MultipleVhostsTestFedora(util.ApacheTest):
     def test_get_parser(self):
         self.assertIsInstance(self.config.parser, override_fedora.FedoraParser)
 
-    @mock.patch("certbot_apache._internal.apache_util._get_runtime_cfg")
+    @mock.patch("certbot_apache._internal.parser.ApacheParser._get_runtime_cfg")
     def test_opportunistic_httpd_runtime_parsing(self, mock_get):
         define_val = (
             'Define: TEST1\n'
@@ -123,7 +120,7 @@ class MultipleVhostsTestFedora(util.ApacheTest):
                 return mod_val
             return ""
         mock_get.side_effect = mock_get_cfg
-        self.config.parser.modules = {}
+        self.config.parser.modules = set()
         self.config.parser.variables = {}
 
         with mock.patch("certbot.util.get_os_info") as mock_osi:
@@ -158,7 +155,7 @@ class MultipleVhostsTestFedora(util.ApacheTest):
                 raise Exception("Missed: %s" % vhost)  # pragma: no cover
         self.assertEqual(found, 2)
 
-    @mock.patch("certbot_apache._internal.apache_util._get_runtime_cfg")
+    @mock.patch("certbot_apache._internal.parser.ApacheParser._get_runtime_cfg")
     def test_get_sysconfig_vars(self, mock_cfg):
         """Make sure we read the sysconfig OPTIONS variable correctly"""
         # Return nothing for the process calls

certbot-apache/tests/gentoo_test.py

@@ -1,10 +1,7 @@
 """Test for certbot_apache._internal.configurator for Gentoo overrides"""
 import unittest
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 
 from certbot import errors
 from certbot.compat import filesystem
@@ -24,19 +21,19 @@ def get_vh_truth(temp_dir, config_name):
         obj.VirtualHost(
             os.path.join(prefix, "gentoo.example.com.conf"),
             os.path.join(aug_pre, "gentoo.example.com.conf/VirtualHost"),
-            {obj.Addr.fromstring("*:80")},
+            set([obj.Addr.fromstring("*:80")]),
             False, True, "gentoo.example.com"),
         obj.VirtualHost(
             os.path.join(prefix, "00_default_vhost.conf"),
             os.path.join(aug_pre, "00_default_vhost.conf/IfDefine/VirtualHost"),
-            {obj.Addr.fromstring("*:80")},
+            set([obj.Addr.fromstring("*:80")]),
             False, True, "localhost"),
         obj.VirtualHost(
             os.path.join(prefix, "00_default_ssl_vhost.conf"),
             os.path.join(aug_pre,
                          "00_default_ssl_vhost.conf" +
                          "/IfDefine/IfDefine/IfModule/VirtualHost"),
-            {obj.Addr.fromstring("_default_:443")},
+            set([obj.Addr.fromstring("_default_:443")]),
             True, True, "localhost")
     ]
     return vh_truth
@@ -93,7 +90,7 @@ class MultipleVhostsTestGentoo(util.ApacheTest):
         for define in defines:
             self.assertTrue(define in self.config.parser.variables.keys())
 
-    @mock.patch("certbot_apache._internal.apache_util.parse_from_subprocess")
+    @mock.patch("certbot_apache._internal.parser.ApacheParser.parse_from_subprocess")
     def test_no_binary_configdump(self, mock_subprocess):
         """Make sure we don't call binary dumps other than modules from Apache
         as this is not supported in Gentoo currently"""
@@ -107,7 +104,7 @@ class MultipleVhostsTestGentoo(util.ApacheTest):
         self.config.parser.reset_modules()
         self.assertTrue(mock_subprocess.called)
 
-    @mock.patch("certbot_apache._internal.apache_util._get_runtime_cfg")
+    @mock.patch("certbot_apache._internal.parser.ApacheParser._get_runtime_cfg")
     def test_opportunistic_httpd_runtime_parsing(self, mock_get):
         mod_val = (
             'Loaded Modules:\n'
@@ -120,7 +117,7 @@ class MultipleVhostsTestGentoo(util.ApacheTest):
                 return mod_val
             return None  # pragma: no cover
         mock_get.side_effect = mock_get_cfg
-        self.config.parser.modules = {}
+        self.config.parser.modules = set()
 
         with mock.patch("certbot.util.get_os_info") as mock_osi:
             # Make sure we have the have the Gentoo httpd constants

certbot-apache/tests/http_01_test.py

@@ -1,13 +1,10 @@
 """Test for certbot_apache._internal.http_01."""
 import unittest
-import errno
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 
 from acme import challenges
+from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
 from certbot import achallenges
 from certbot import errors
 from certbot.compat import filesystem
@@ -43,8 +40,8 @@ class ApacheHttp01Test(util.ApacheTest):
 
         modules = ["ssl", "rewrite", "authz_core", "authz_host"]
         for mod in modules:
-            self.config.parser.modules["mod_{0}.c".format(mod)] = None
-            self.config.parser.modules[mod + "_module"] = None
+            self.config.parser.modules.add("mod_{0}.c".format(mod))
+            self.config.parser.modules.add(mod + "_module")
 
         from certbot_apache._internal.http_01 import ApacheHttp01
         self.http = ApacheHttp01(self.config)
@@ -55,24 +52,24 @@ class ApacheHttp01Test(util.ApacheTest):
     @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.enable_mod")
     def test_enable_modules_apache_2_2(self, mock_enmod):
         self.config.version = (2, 2)
-        del self.config.parser.modules["authz_host_module"]
-        del self.config.parser.modules["mod_authz_host.c"]
+        self.config.parser.modules.remove("authz_host_module")
+        self.config.parser.modules.remove("mod_authz_host.c")
 
         enmod_calls = self.common_enable_modules_test(mock_enmod)
         self.assertEqual(enmod_calls[0][0][0], "authz_host")
 
     @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.enable_mod")
     def test_enable_modules_apache_2_4(self, mock_enmod):
-        del self.config.parser.modules["authz_core_module"]
-        del self.config.parser.modules["mod_authz_host.c"]
+        self.config.parser.modules.remove("authz_core_module")
+        self.config.parser.modules.remove("mod_authz_core.c")
 
         enmod_calls = self.common_enable_modules_test(mock_enmod)
         self.assertEqual(enmod_calls[0][0][0], "authz_core")
 
     def common_enable_modules_test(self, mock_enmod):
         """Tests enabling mod_rewrite and other modules."""
-        del self.config.parser.modules["rewrite_module"]
-        del self.config.parser.modules["mod_rewrite.c"]
+        self.config.parser.modules.remove("rewrite_module")
+        self.config.parser.modules.remove("mod_rewrite.c")
 
         self.http.prepare_http01_modules()
 
@@ -200,12 +197,6 @@ class ApacheHttp01Test(util.ApacheTest):
 
         self.assertTrue(os.path.exists(challenge_dir))
 
-    @mock.patch("certbot_apache._internal.http_01.filesystem.makedirs")
-    def test_failed_makedirs(self, mock_makedirs):
-        mock_makedirs.side_effect = OSError(errno.EACCES, "msg")
-        self.http.add_chall(self.achalls[0])
-        self.assertRaises(errors.PluginError, self.http.perform)
-
     def _test_challenge_conf(self):
         with open(self.http.challenge_conf_pre) as f:
             pre_conf_contents = f.read()

certbot-apache/tests/obj_test.py

@@ -14,13 +14,13 @@ class VirtualHostTest(unittest.TestCase):
         self.addr_default = Addr.fromstring("_default_:443")
 
         self.vhost1 = VirtualHost(
-            "filep", "vh_path", {self.addr1}, False, False, "localhost")
+            "filep", "vh_path", set([self.addr1]), False, False, "localhost")
 
         self.vhost1b = VirtualHost(
-            "filep", "vh_path", {self.addr1}, False, False, "localhost")
+            "filep", "vh_path", set([self.addr1]), False, False, "localhost")
 
         self.vhost2 = VirtualHost(
-            "fp", "vhp", {self.addr2}, False, False, "localhost")
+            "fp", "vhp", set([self.addr2]), False, False, "localhost")
 
     def test_repr(self):
         self.assertEqual(repr(self.addr2),
@@ -42,7 +42,7 @@ class VirtualHostTest(unittest.TestCase):
 
         complex_vh = VirtualHost(
             "fp", "vhp",
-            {Addr.fromstring("*:443"), Addr.fromstring("1.2.3.4:443")},
+            set([Addr.fromstring("*:443"), Addr.fromstring("1.2.3.4:443")]),
             False, False)
         self.assertTrue(complex_vh.conflicts([self.addr1]))
         self.assertTrue(complex_vh.conflicts([self.addr2]))
@@ -57,14 +57,14 @@ class VirtualHostTest(unittest.TestCase):
     def test_same_server(self):
         from certbot_apache._internal.obj import VirtualHost
         no_name1 = VirtualHost(
-            "fp", "vhp", {self.addr1}, False, False, None)
+            "fp", "vhp", set([self.addr1]), False, False, None)
         no_name2 = VirtualHost(
-            "fp", "vhp", {self.addr2}, False, False, None)
+            "fp", "vhp", set([self.addr2]), False, False, None)
         no_name3 = VirtualHost(
-            "fp", "vhp", {self.addr_default},
+            "fp", "vhp", set([self.addr_default]),
             False, False, None)
         no_name4 = VirtualHost(
-            "fp", "vhp", {self.addr2, self.addr_default},
+            "fp", "vhp", set([self.addr2, self.addr_default]),
             False, False, None)
 
         self.assertTrue(self.vhost1.same_server(self.vhost2))

certbot-apache/tests/ocsp_prefetch_test.py

@@ -0,0 +1,415 @@
+"""Test for certbot_apache._internal.configurator OCSP Prefetching functionality"""
+import base64
+from datetime import datetime
+from datetime import timedelta
+import json
+import sys
+import unittest
+
+import mock
+# six is used in mock.patch()
+import six  # pylint: disable=unused-import
+
+from acme.magic_typing import Dict, List, Set, Union  # pylint: disable=unused-import, no-name-in-module
+
+from certbot import errors
+
+from certbot.compat import os
+import util
+
+
+class MockDBM(object):
+    # pylint: disable=missing-docstring
+    """Main mock DBM class for Py3 dbm module"""
+    def __init__(self, library='Berkeley DB'):
+        self.ndbm = Mockdbm_impl(library)
+
+
+class Mockdbm_impl(object):
+    """Mock dbm implementation that satisfies both bsddb and dbm interfaces"""
+    # pylint: disable=missing-docstring
+
+    def __init__(self, library='Berkeley DB'):
+        self.library = library
+        self.name = 'ndbm'
+
+    def open(self, path, mode):
+        return Mockdb(path, mode)
+
+    def hashopen(self, path, mode):
+        return Mockdb(path, mode)
+
+
+class Mockdb(object):
+    """Mock dbm.db for both bsddb and dbm databases"""
+    # pylint: disable=missing-docstring
+    def __init__(self, path, mode):
+        self._data = dict()  # type: Dict[str, str]
+        if mode == "r" or mode == "w":
+            if not path.endswith(".db"):
+                path = path+".db"
+            with open(path, 'r') as fh:
+                try:
+                    self._data = json.loads(fh.read())
+                except Exception:  # pylint: disable=broad-except
+                    self._data = dict()
+        self.path = path
+        self.mode = mode
+
+    def __setitem__(self, key, item):
+        bkey = base64.b64encode(key)
+        bitem = base64.b64encode(item)
+        self._data[bkey.decode()] = bitem.decode()
+
+    def __getitem__(self, key):
+        bkey = base64.b64encode(key)
+        return base64.b64decode(self._data[bkey.decode()])
+
+    def keys(self):
+        return [base64.b64decode(k) for k in self._data.keys()]
+
+    def sync(self):
+        return
+
+    def close(self):
+        with open(self.path, 'w') as fh:
+            fh.write(json.dumps(self._data))
+
+
+class OCSPPrefetchTest(util.ApacheTest):
+    """Tests for OCSP Prefetch feature"""
+    # pylint: disable=protected-access
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        super(OCSPPrefetchTest, self).setUp()
+
+        self.config = util.get_apache_configurator(
+            self.config_path, self.vhost_path, self.config_dir, self.work_dir,
+            os_info="debian")
+
+        self.lineage = mock.MagicMock(cert_path="cert", chain_path="chain")
+        self.config.parser.modules.add("headers_module")
+        self.config.parser.modules.add("mod_headers.c")
+        self.config.parser.modules.add("ssl_module")
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("socache_dbm_module")
+        self.config.parser.modules.add("mod_socache_dbm.c")
+
+        self.vh_truth = util.get_vh_truth(
+            self.temp_dir, "debian_apache_2_4/multiple_vhosts")
+        self.config._ensure_ocsp_dirs()
+        self.db_path = os.path.join(self.work_dir, "ocsp", "ocsp_cache") + ".db"
+
+    def _call_mocked(self, func, *args, **kwargs):
+        """Helper method to call functins with mock stack"""
+
+        def mock_restart():
+            """Mock ApacheConfigurator.restart that creates the dbm file"""
+            # Mock the Apache dbm file creation
+            open(self.db_path, 'a').close()
+
+        ver_path = "certbot_apache._internal.configurator.ApacheConfigurator.get_version"
+        res_path = "certbot_apache._internal.prefetch_ocsp.OCSPPrefetchMixin.restart"
+        cry_path = "certbot.crypto_util.cert_sha1_fingerprint"
+
+        with mock.patch(ver_path) as mock_ver:
+            mock_ver.return_value = (2, 4, 10)
+            with mock.patch(cry_path) as mock_cry:
+                mock_cry.return_value = b'j\x056\x1f\xfa\x08B\xe8D\xa1Bn\xeb*A\xebWx\xdd\xfe'
+                with mock.patch(res_path, side_effect=mock_restart):
+                    return func(*args, **kwargs)
+
+    def call_mocked_py2(self, func, *args, **kwargs):
+        """Calls methods with imports mocked to suit Py2 environment"""
+        if 'dbm' in sys.modules.keys():  # pragma: no cover
+            sys.modules['dbm'] = None
+        sys.modules['bsddb'] = Mockdbm_impl()
+        return self._call_mocked(func, *args, **kwargs)
+
+    def call_mocked_py3(self, func, *args, **kwargs):
+        """Calls methods with imports mocked to suit Py3 environment"""
+        real_import = six.moves.builtins.__import__
+
+        def mock_import(*args, **kwargs):
+            """Mock import to raise ImportError for Py2 specific module to make
+            ApacheConfigurator pick the correct one for Python3 regardless of the
+            python version the tests are running under."""
+            if args[0] == "bsddb":
+                raise ImportError
+            return real_import(*args, **kwargs)
+
+        with mock.patch('six.moves.builtins.__import__', side_effect=mock_import):
+            sys.modules['dbm'] = MockDBM()
+            return self._call_mocked(func, *args, **kwargs)
+
+    @mock.patch("certbot_apache._internal.override_debian.DebianConfigurator.enable_mod")
+    def test_ocsp_prefetch_enable_mods(self, mock_enable):
+        self.config.parser.modules.discard("socache_dbm_module")
+        self.config.parser.modules.discard("mod_socache_dbm.c")
+        self.config.parser.modules.discard("headers_module")
+        self.config.parser.modules.discard("mod_header.c")
+
+        ref_path = "certbot_apache._internal.override_debian.DebianConfigurator._ocsp_refresh"
+        with mock.patch(ref_path):
+            self.call_mocked_py2(self.config.enable_ocsp_prefetch,
+                                 self.lineage, ["ocspvhost.com"])
+        self.assertTrue(mock_enable.called)
+        self.assertEqual(len(self.config._ocsp_prefetch), 1)
+
+    @mock.patch("certbot_apache._internal.override_debian.DebianConfigurator.enable_mod")
+    def test_ocsp_prefetch_enable_error(self, _mock_enable):
+        ref_path = "certbot_apache._internal.override_debian.DebianConfigurator._ocsp_refresh"
+        self.config.recovery_routine = mock.MagicMock()
+        with mock.patch(ref_path, side_effect=errors.PluginError("failed")):
+            self.assertRaises(errors.PluginError,
+                              self.call_mocked_py2,
+                              self.config.enable_ocsp_prefetch,
+                              self.lineage,
+                              ["ocspvhost.com"])
+        self.assertTrue(self.config.recovery_routine.called)
+
+    def test_ocsp_prefetch_vhost_not_found_error(self):
+        choose_path = "certbot_apache._internal.override_debian.DebianConfigurator.choose_vhosts"
+        with mock.patch(choose_path) as mock_choose:
+            mock_choose.return_value = []
+            self.assertRaises(errors.MisconfigurationError,
+                              self.call_mocked_py2,
+                              self.config.enable_ocsp_prefetch,
+                              self.lineage,
+                              ["domainnotfound"])
+
+    @mock.patch("certbot_apache._internal.constants.OCSP_INTERNAL_TTL", 0)
+    def test_ocsp_prefetch_refresh(self):
+        def ocsp_req_mock(_cert, _chain, workfile):
+            """Method to mock the OCSP request and write response to file"""
+            with open(workfile, 'w') as fh:
+                fh.write("MOCKRESPONSE")
+            return False
+
+        ocsp_path = "certbot._internal.ocsp.RevocationChecker.ocsp_revoked_by_paths"
+        with mock.patch(ocsp_path, side_effect=ocsp_req_mock):
+            with mock.patch('certbot._internal.ocsp.RevocationChecker.ocsp_times') as mock_times:
+                produced_at = datetime.today() - timedelta(days=1)
+                this_update = datetime.today() - timedelta(days=2)
+                next_update = datetime.today() + timedelta(days=2)
+                mock_times.return_value = produced_at, this_update, next_update
+                self.call_mocked_py2(self.config.enable_ocsp_prefetch,
+                                     self.lineage, ["ocspvhost.com"])
+        odbm = self.config._read_dbm(self.db_path)
+        self.assertEqual(len(odbm.keys()), 1)
+        # The actual response data is prepended by Apache timestamp
+        self.assertTrue(odbm[list(odbm.keys())[0]].endswith(b'MOCKRESPONSE'))
+
+        with mock.patch(ocsp_path, side_effect=ocsp_req_mock) as mock_ocsp:
+            with mock.patch('certbot._internal.ocsp.RevocationChecker.ocsp_times') as mock_times:
+                produced_at = datetime.today() - timedelta(days=1)
+                this_update = datetime.today() - timedelta(days=2)
+                next_update = datetime.today() + timedelta(days=2)
+                mock_times.return_value = produced_at, this_update, next_update
+                self.call_mocked_py2(self.config.update_ocsp_prefetch, None)
+                self.assertTrue(mock_ocsp.called)
+
+    def test_ocsp_prefetch_refresh_noop(self):
+        def ocsp_req_mock(_cert, _chain, workfile):
+            """Method to mock the OCSP request and write response to file"""
+            with open(workfile, 'w') as fh:
+                fh.write("MOCKRESPONSE")
+            return True
+
+        ocsp_path = "certbot._internal.ocsp.RevocationChecker.ocsp_revoked_by_paths"
+        with mock.patch(ocsp_path, side_effect=ocsp_req_mock):
+            self.call_mocked_py2(self.config.enable_ocsp_prefetch,
+                                 self.lineage, ["ocspvhost.com"])
+        self.assertEqual(len(self.config._ocsp_prefetch), 1)
+        refresh_path = "certbot_apache._internal.override_debian.DebianConfigurator._ocsp_refresh"
+        with mock.patch(refresh_path) as mock_refresh:
+            self.call_mocked_py2(self.config.update_ocsp_prefetch, None)
+            self.assertFalse(mock_refresh.called)
+
+    @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.config_test")
+    def test_ocsp_prefetch_backup_db(self, _mock_test):
+        def ocsp_del_db():
+            """Side effect of _reload() that deletes the DBM file, like Apache
+            does when restarting"""
+            os.remove(self.db_path)
+            self.assertFalse(os.path.isfile(self.db_path))
+
+        # Make sure that the db file exists
+        open(self.db_path, 'a').close()
+        self.call_mocked_py2(self.config._write_to_dbm, self.db_path, b'mock_key', b'mock_value')
+
+        # Mock OCSP prefetch dict to signify that there should be a db
+        self.config._ocsp_prefetch = {"mock": "value"}
+        rel_path = "certbot_apache._internal.configurator.ApacheConfigurator._reload"
+        with mock.patch(rel_path, side_effect=ocsp_del_db):
+            self.config.restart()
+
+        odbm = self.config._read_dbm(self.db_path)
+        self.assertEqual(odbm[b'mock_key'], b'mock_value')
+
+    @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.config_test")
+    @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator._reload")
+    def test_ocsp_prefetch_backup_db_error(self, _mock_reload, _mock_test):
+        log_path = "certbot_apache._internal.prefetch_ocsp.logger.debug"
+        log_string = "Encountered an issue while trying to backup OCSP dbm file"
+        log_string2 = "Encountered an issue when trying to restore OCSP dbm file"
+        self.config._ocsp_prefetch = {"mock": "value"}
+        with mock.patch("shutil.copy2", side_effect=IOError):
+            with mock.patch(log_path) as mock_log:
+                self.config.restart()
+                self.assertTrue(mock_log.called)
+                self.assertEqual(mock_log.call_count, 2)
+                self.assertTrue(log_string in mock_log.call_args_list[0][0][0])
+                self.assertTrue(log_string2 in mock_log.call_args_list[1][0][0])
+
+    @mock.patch("certbot_apache._internal.prefetch_ocsp.OCSPPrefetchMixin.restart")
+    def test_ocsp_prefetch_refresh_fail(self, _mock_restart):
+        ocsp_path = "certbot._internal.ocsp.RevocationChecker.ocsp_revoked_by_paths"
+        log_path = "certbot_apache._internal.prefetch_ocsp.logger.warning"
+        with mock.patch(ocsp_path) as mock_ocsp:
+            mock_ocsp.return_value = True
+            with mock.patch(log_path) as mock_log:
+                self.call_mocked_py2(self.config.enable_ocsp_prefetch,
+                                     self.lineage, ["ocspvhost.com"])
+                self.assertTrue(mock_log.called)
+                self.assertTrue(
+                    "trying to prefetch OCSP" in mock_log.call_args[0][0])
+
+    @mock.patch("certbot_apache._internal.override_debian.DebianConfigurator._ocsp_refresh_needed")
+    def test_ocsp_prefetch_update_noop(self, mock_refresh):
+        self.config.update_ocsp_prefetch(None)
+        self.assertFalse(mock_refresh.called)
+
+    def test_ocsp_prefetch_preflight_check_noerror(self):
+        self.call_mocked_py2(self.config._ensure_ocsp_prefetch_compatibility)
+        self.call_mocked_py3(self.config._ensure_ocsp_prefetch_compatibility)
+
+        real_import = six.moves.builtins.__import__
+
+        def mock_import(*args, **kwargs):
+            if args[0] == "bsddb":
+                raise ImportError
+            return real_import(*args, **kwargs)
+        with mock.patch('six.moves.builtins.__import__', side_effect=mock_import):
+            sys.modules['dbm'] = MockDBM(library='Not Berkeley DB')
+            self.assertRaises(errors.NotSupportedError,
+                              self._call_mocked,
+                              self.config._ensure_ocsp_prefetch_compatibility)
+
+    def test_ocsp_prefetch_open_dbm_no_file(self):
+        open(self.db_path, 'a').close()
+        db_not_exists = self.db_path+"nonsense"
+        self.call_mocked_py2(self.config._write_to_dbm, self.db_path, b'k', b'v')
+        self.assertRaises(errors.PluginError,
+                          self.call_mocked_py2,
+                          self.config._write_to_dbm,
+                          db_not_exists,
+                          b'k', b'v')
+
+    def test_ocsp_prefetch_py2_open_file_error(self):
+        open(self.db_path, 'a').close()
+        mock_db = mock.MagicMock()
+        mock_db.hashopen.side_effect = Exception("error")
+        sys.modules["bsddb"] = mock_db
+        self.assertRaises(errors.PluginError,
+                          self.config._write_to_dbm,
+                          self.db_path,
+                          b'k', b'v')
+
+    def test_ocsp_prefetch_py3_open_file_error(self):
+        open(self.db_path, 'a').close()
+        mock_db = mock.MagicMock()
+        mock_db.ndbm.open.side_effect = Exception("error")
+        sys.modules["dbm"] = mock_db
+        sys.modules["bsddb"] = None
+        self.assertRaises(errors.PluginError,
+                          self.config._write_to_dbm,
+                          self.db_path,
+                          b'k', b'v')
+
+    def test_ocsp_prefetch_open_close_py2_noerror(self):
+        expected_val = b'whatever_value'
+        open(self.db_path, 'a').close()
+        self.call_mocked_py2(
+            self.config._write_to_dbm, self.db_path,
+            b'key', expected_val
+        )
+        db2 = self.call_mocked_py2(self.config._read_dbm, self.db_path)
+        self.assertEqual(db2[b'key'], expected_val)
+
+    def test_ocsp_prefetch_open_close_py3_noerror(self):
+        expected_val = b'whatever_value'
+        open(self.db_path, 'a').close()
+        self.call_mocked_py3(
+            self.config._write_to_dbm, self.db_path,
+            b'key', expected_val
+        )
+        db2 = self.call_mocked_py3(self.config._read_dbm, self.db_path)
+        self.assertEqual(db2[b'key'], expected_val)
+
+    def test_ocsp_prefetch_safe_open_hash_mismatch(self):
+        import random
+        def mock_hash(_filepath):
+            # shound not be the same value twice
+            return str(random.getrandbits(1024))
+
+        open(self.db_path, 'a').close()
+        with mock.patch("certbot_apache._internal.apache_util._file_hash", side_effect=mock_hash):
+            self.assertRaises(
+                errors.PluginError,
+                self.call_mocked_py3,
+                self.config._write_to_dbm, self.db_path,
+                b'anything', b'irrelevant'
+            )
+
+    def test_ocsp_prefetch_safe_open_hash_fail_open(self):
+        open(self.db_path, 'a').close()
+        with mock.patch("certbot_apache._internal.apache_util._file_hash", side_effect=IOError):
+            self.assertRaises(
+                errors.PluginError,
+                self.call_mocked_py3,
+                self.config._write_to_dbm, self.db_path,
+                b'anything', b'irrelevant'
+            )
+
+    @mock.patch("certbot_apache._internal.constants.OCSP_APACHE_TTL", 1234)
+    def test_ttl(self):
+        self.assertEqual(self.config._ocsp_ttl(None), 1234)
+        next_update = datetime.today() + timedelta(days=6)
+        ttl = self.config._ocsp_ttl(next_update)
+        # ttl should be roughly 3 days
+        self.assertTrue(ttl > 86400*2)
+        self.assertTrue(ttl < 86400*4)
+
+    @mock.patch("certbot_apache._internal.prefetch_ocsp.OCSPPrefetchMixin._ocsp_prefetch_fetch_state")
+    @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.config_test")
+    @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator._reload")
+    def test_restart_load_state_call(self, _rl, _ct, mock_fch):
+        self.assertFalse(self.config._ocsp_prefetch)
+        self.config.restart()
+        self.assertTrue(mock_fch.called)
+
+    @mock.patch("certbot_apache._internal.prefetch_ocsp.OCSPPrefetchMixin._ocsp_prefetch_backup_db")
+    @mock.patch("certbot_apache._internal.prefetch_ocsp.OCSPPrefetchMixin._ocsp_prefetch_restore_db")
+    @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.config_test")
+    @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator._reload")
+    def test_restart_backupdb_call(self, _rl, _ctest, mock_rest, mock_bck):
+        self.config._ocsp_prefetch = True
+        self.config.restart()
+        self.assertTrue(mock_rest.called)
+        self.assertTrue(mock_bck.called)
+
+    @mock.patch("certbot_apache._internal.prefetch_ocsp.OCSPPrefetchMixin._ocsp_prefetch_backup_db")
+    @mock.patch("certbot_apache._internal.prefetch_ocsp.OCSPPrefetchMixin._ocsp_prefetch_restore_db")
+    @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.config_test")
+    @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator._reload")
+    def test_restart_recover_error(self, mock_reload, _ctest, mock_rest, mock_bck):
+        self.config._ocsp_prefetch = True
+        mock_reload.side_effect = errors.MisconfigurationError
+        self.assertRaises(errors.MisconfigurationError, self.config.restart)
+        self.assertTrue(mock_bck.called)
+        self.assertTrue(mock_rest.called)
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot-apache/tests/parser_test.py

@@ -2,10 +2,7 @@
 import shutil
 import unittest
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 
 from certbot import errors
 from certbot.compat import os
@@ -117,7 +114,7 @@ class BasicParserTest(util.ParserTest):
         """
         from certbot_apache._internal.parser import get_aug_path
         # This makes sure that find_dir will work
-        self.parser.modules["mod_ssl.c"] = "/fake/path"
+        self.parser.modules.add("mod_ssl.c")
 
         self.parser.add_dir_to_ifmodssl(
             get_aug_path(self.parser.loc["default"]),
@@ -131,7 +128,7 @@ class BasicParserTest(util.ParserTest):
     def test_add_dir_to_ifmodssl_multiple(self):
         from certbot_apache._internal.parser import get_aug_path
         # This makes sure that find_dir will work
-        self.parser.modules["mod_ssl.c"] = "/fake/path"
+        self.parser.modules.add("mod_ssl.c")
 
         self.parser.add_dir_to_ifmodssl(
             get_aug_path(self.parser.loc["default"]),
@@ -168,7 +165,7 @@ class BasicParserTest(util.ParserTest):
             self.assertTrue(mock_logger.debug.called)
 
     @mock.patch("certbot_apache._internal.parser.ApacheParser.find_dir")
-    @mock.patch("certbot_apache._internal.apache_util._get_runtime_cfg")
+    @mock.patch("certbot_apache._internal.parser.ApacheParser._get_runtime_cfg")
     def test_update_runtime_variables(self, mock_cfg, _):
         define_val = (
             'ServerRoot: "/etc/apache2"\n'
@@ -263,7 +260,7 @@ class BasicParserTest(util.ParserTest):
         expected_vars = {"TEST": "", "U_MICH": "", "TLS": "443",
                          "example_path": "Documents/path"}
 
-        self.parser.modules = {}
+        self.parser.modules = set()
         with mock.patch(
             "certbot_apache._internal.parser.ApacheParser.parse_file") as mock_parse:
             self.parser.update_runtime_variables()
@@ -274,7 +271,7 @@ class BasicParserTest(util.ParserTest):
             self.assertEqual(mock_parse.call_count, 25)
 
     @mock.patch("certbot_apache._internal.parser.ApacheParser.find_dir")
-    @mock.patch("certbot_apache._internal.apache_util._get_runtime_cfg")
+    @mock.patch("certbot_apache._internal.parser.ApacheParser._get_runtime_cfg")
     def test_update_runtime_variables_alt_values(self, mock_cfg, _):
         inc_val = (
             'Included configuration files:\n'
@@ -285,7 +282,7 @@ class BasicParserTest(util.ParserTest):
                  os.path.dirname(self.parser.loc["root"]))
 
         mock_cfg.return_value = inc_val
-        self.parser.modules = {}
+        self.parser.modules = set()
 
         with mock.patch(
             "certbot_apache._internal.parser.ApacheParser.parse_file") as mock_parse:
@@ -296,7 +293,7 @@ class BasicParserTest(util.ParserTest):
             # path derived from root configuration Include statements
             self.assertEqual(mock_parse.call_count, 1)
 
-    @mock.patch("certbot_apache._internal.apache_util._get_runtime_cfg")
+    @mock.patch("certbot_apache._internal.parser.ApacheParser._get_runtime_cfg")
     def test_update_runtime_vars_bad_output(self, mock_cfg):
         mock_cfg.return_value = "Define: TLS=443=24"
         self.parser.update_runtime_variables()
@@ -306,7 +303,7 @@ class BasicParserTest(util.ParserTest):
             errors.PluginError, self.parser.update_runtime_variables)
 
     @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.option")
-    @mock.patch("certbot_apache._internal.apache_util.subprocess.Popen")
+    @mock.patch("certbot_apache._internal.parser.subprocess.Popen")
     def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_opt):
         mock_popen.side_effect = OSError
         mock_opt.return_value = "nonexistent"
@@ -314,7 +311,7 @@ class BasicParserTest(util.ParserTest):
             errors.MisconfigurationError,
             self.parser.update_runtime_variables)
 
-    @mock.patch("certbot_apache._internal.apache_util.subprocess.Popen")
+    @mock.patch("certbot_apache._internal.parser.subprocess.Popen")
     def test_update_runtime_vars_bad_exit(self, mock_popen):
         mock_popen().communicate.return_value = ("", "")
         mock_popen.returncode = -1
@@ -358,7 +355,7 @@ class ParserInitTest(util.ApacheTest):
                 ApacheParser, os.path.relpath(self.config_path),
                 "/dummy/vhostpath", version=(2, 4, 22), configurator=self.config)
 
-    @mock.patch("certbot_apache._internal.apache_util._get_runtime_cfg")
+    @mock.patch("certbot_apache._internal.parser.ApacheParser._get_runtime_cfg")
     def test_unparseable(self, mock_cfg):
         from certbot_apache._internal.parser import ApacheParser
         mock_cfg.return_value = ('Define: TEST')

certbot-apache/tests/parsernode_configurator_test.py

@@ -1,47 +0,0 @@
-"""Tests for ApacheConfigurator for AugeasParserNode classes"""
-import unittest
-
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
-
-import util
-
-try:
-    import apacheconfig
-    HAS_APACHECONFIG = True
-except ImportError:  # pragma: no cover
-    HAS_APACHECONFIG = False
-
-
-@unittest.skipIf(not HAS_APACHECONFIG, reason='Tests require apacheconfig dependency')
-class ConfiguratorParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-methods
-    """Test AugeasParserNode using available test configurations"""
-
-    def setUp(self):  # pylint: disable=arguments-differ
-        super(ConfiguratorParserNodeTest, self).setUp()
-
-        self.config = util.get_apache_configurator(
-            self.config_path, self.vhost_path, self.config_dir,
-            self.work_dir, use_parsernode=True)
-        self.vh_truth = util.get_vh_truth(
-            self.temp_dir, "debian_apache_2_4/multiple_vhosts")
-
-    def test_parsernode_get_vhosts(self):
-        self.config.USE_PARSERNODE = True
-        vhosts = self.config.get_virtual_hosts()
-        # Legacy get_virtual_hosts() do not set the node
-        self.assertTrue(vhosts[0].node is not None)
-
-    def test_parsernode_get_vhosts_mismatch(self):
-        vhosts = self.config.get_virtual_hosts_v2()
-        # One of the returned VirtualHost objects differs
-        vhosts[0].name = "IdidntExpectThat"
-        self.config.get_virtual_hosts_v2 = mock.MagicMock(return_value=vhosts)
-        with self.assertRaises(AssertionError):
-            _ = self.config.get_virtual_hosts()
-
-
-if __name__ == "__main__":
-    unittest.main()  # pragma: no cover

certbot-apache/tests/parsernode_test.py

@@ -1,128 +0,0 @@
-""" Tests for ParserNode interface """
-
-import unittest
-
-from certbot_apache._internal import interfaces
-from certbot_apache._internal import parsernode_util as util
-
-
-class DummyParserNode(interfaces.ParserNode):
-    """ A dummy class implementing ParserNode interface """
-
-    def __init__(self, **kwargs):
-        """
-        Initializes the ParserNode instance.
-        """
-        ancestor, dirty, filepath, metadata = util.parsernode_kwargs(kwargs)
-        self.ancestor = ancestor
-        self.dirty = dirty
-        self.filepath = filepath
-        self.metadata = metadata
-        super(DummyParserNode, self).__init__(**kwargs)
-
-    def save(self, msg):  # pragma: no cover
-        """Save"""
-        pass
-
-    def find_ancestors(self, name):  # pragma: no cover
-        """ Find ancestors """
-        return []
-
-
-class DummyCommentNode(DummyParserNode):
-    """ A dummy class implementing CommentNode interface """
-
-    def __init__(self, **kwargs):
-        """
-        Initializes the CommentNode instance and sets its instance variables.
-        """
-        comment, kwargs = util.commentnode_kwargs(kwargs)
-        self.comment = comment
-        super(DummyCommentNode, self).__init__(**kwargs)
-
-
-class DummyDirectiveNode(DummyParserNode):
-    """ A dummy class implementing DirectiveNode interface """
-
-    # pylint: disable=too-many-arguments
-    def __init__(self, **kwargs):
-        """
-        Initializes the DirectiveNode instance and sets its instance variables.
-        """
-        name, parameters, enabled, kwargs = util.directivenode_kwargs(kwargs)
-        self.name = name
-        self.parameters = parameters
-        self.enabled = enabled
-
-        super(DummyDirectiveNode, self).__init__(**kwargs)
-
-    def set_parameters(self, parameters):  # pragma: no cover
-        """Set parameters"""
-        pass
-
-
-class DummyBlockNode(DummyDirectiveNode):
-    """ A dummy class implementing BlockNode interface """
-
-    def add_child_block(self, name, parameters=None, position=None):  # pragma: no cover
-        """Add child block"""
-        pass
-
-    def add_child_directive(self, name, parameters=None, position=None):  # pragma: no cover
-        """Add child directive"""
-        pass
-
-    def add_child_comment(self, comment="", position=None):  # pragma: no cover
-        """Add child comment"""
-        pass
-
-    def find_blocks(self, name, exclude=True):  # pragma: no cover
-        """Find blocks"""
-        pass
-
-    def find_directives(self, name, exclude=True):  # pragma: no cover
-        """Find directives"""
-        pass
-
-    def find_comments(self, comment, exact=False):  # pragma: no cover
-        """Find comments"""
-        pass
-
-    def delete_child(self, child):  # pragma: no cover
-        """Delete child"""
-        pass
-
-    def unsaved_files(self):  # pragma: no cover
-        """Unsaved files"""
-        pass
-
-
-interfaces.CommentNode.register(DummyCommentNode)
-interfaces.DirectiveNode.register(DummyDirectiveNode)
-interfaces.BlockNode.register(DummyBlockNode)
-
-class ParserNodeTest(unittest.TestCase):
-    """Dummy placeholder test case for ParserNode interfaces"""
-
-    def test_dummy(self):
-        dummyblock = DummyBlockNode(
-            name="None",
-            parameters=(),
-            ancestor=None,
-            dirty=False,
-            filepath="/some/random/path"
-        )
-        dummydirective = DummyDirectiveNode(
-            name="Name",
-            ancestor=None,
-            filepath="/another/path"
-        )
-        dummycomment = DummyCommentNode(
-            comment="Comment",
-            ancestor=dummyblock,
-            filepath="/some/file"
-        )
-
-
-if __name__ == "__main__":
-    unittest.main()  # pragma: no cover

certbot-apache/tests/parsernode_util_test.py

@@ -1,115 +0,0 @@
-""" Tests for ParserNode utils """
-import unittest
-
-from certbot_apache._internal import parsernode_util as util
-
-
-class ParserNodeUtilTest(unittest.TestCase):
-    """Tests for ParserNode utils"""
-
-    def _setup_parsernode(self):
-        """ Sets up kwargs dict for ParserNode """
-        return {
-            "ancestor": None,
-            "dirty": False,
-            "filepath": "/tmp",
-        }
-
-    def _setup_commentnode(self):
-        """ Sets up kwargs dict for CommentNode """
-
-        pn = self._setup_parsernode()
-        pn["comment"] = "x"
-        return pn
-
-    def _setup_directivenode(self):
-        """ Sets up kwargs dict for DirectiveNode """
-
-        pn = self._setup_parsernode()
-        pn["name"] = "Name"
-        pn["parameters"] = ("first",)
-        pn["enabled"] = True
-        return pn
-
-    def test_unknown_parameter(self):
-        params = self._setup_parsernode()
-        params["unknown"] = "unknown"
-        self.assertRaises(TypeError, util.parsernode_kwargs, params)
-
-        params = self._setup_commentnode()
-        params["unknown"] = "unknown"
-        self.assertRaises(TypeError, util.commentnode_kwargs, params)
-
-        params = self._setup_directivenode()
-        params["unknown"] = "unknown"
-        self.assertRaises(TypeError, util.directivenode_kwargs, params)
-
-    def test_parsernode(self):
-        params = self._setup_parsernode()
-        ctrl = self._setup_parsernode()
-
-        ancestor, dirty, filepath, metadata = util.parsernode_kwargs(params)
-        self.assertEqual(ancestor, ctrl["ancestor"])
-        self.assertEqual(dirty, ctrl["dirty"])
-        self.assertEqual(filepath, ctrl["filepath"])
-        self.assertEqual(metadata, {})
-
-    def test_parsernode_from_metadata(self):
-        params = self._setup_parsernode()
-        params.pop("filepath")
-        md = {"some": "value"}
-        params["metadata"] = md
-
-        # Just testing that error from missing required parameters is not raised
-        _, _, _, metadata = util.parsernode_kwargs(params)
-        self.assertEqual(metadata, md)
-
-    def test_commentnode(self):
-        params = self._setup_commentnode()
-        ctrl = self._setup_commentnode()
-
-        comment, _ = util.commentnode_kwargs(params)
-        self.assertEqual(comment, ctrl["comment"])
-
-    def test_commentnode_from_metadata(self):
-        params = self._setup_commentnode()
-        params.pop("comment")
-        params["metadata"] = {}
-
-        # Just testing that error from missing required parameters is not raised
-        util.commentnode_kwargs(params)
-
-    def test_directivenode(self):
-        params = self._setup_directivenode()
-        ctrl = self._setup_directivenode()
-
-        name, parameters, enabled, _ = util.directivenode_kwargs(params)
-        self.assertEqual(name, ctrl["name"])
-        self.assertEqual(parameters, ctrl["parameters"])
-        self.assertEqual(enabled, ctrl["enabled"])
-
-    def test_directivenode_from_metadata(self):
-        params = self._setup_directivenode()
-        params.pop("filepath")
-        params.pop("name")
-        params["metadata"] = {"irrelevant": "value"}
-
-        # Just testing that error from missing required parameters is not raised
-        util.directivenode_kwargs(params)
-
-    def test_missing_required(self):
-        c_params = self._setup_commentnode()
-        c_params.pop("comment")
-        self.assertRaises(TypeError, util.commentnode_kwargs, c_params)
-
-        d_params = self._setup_directivenode()
-        d_params.pop("ancestor")
-        self.assertRaises(TypeError, util.directivenode_kwargs, d_params)
-
-        p_params = self._setup_parsernode()
-        p_params.pop("filepath")
-        self.assertRaises(TypeError, util.parsernode_kwargs, p_params)
-
-
-if __name__ == "__main__":
-    unittest.main()  # pragma: no cover

certbot-apache/tests/util.py

@@ -5,10 +5,7 @@ import unittest
 
 import augeas
 import josepy as jose
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 import zope.component
 
 from certbot.compat import os
@@ -87,9 +84,7 @@ def get_apache_configurator(
         config_path, vhost_path,
         config_dir, work_dir, version=(2, 4, 7),
         os_info="generic",
-        conf_vhost_path=None,
-        use_parsernode=False,
-        openssl_version="1.1.1a"):
+        conf_vhost_path=None):
     """Create an Apache Configurator with the specified options.
 
     :param conf: Function that returns binary paths. self.conf in Configurator
@@ -115,22 +110,19 @@ def get_apache_configurator(
             mock_exe_exists.return_value = True
             with mock.patch("certbot_apache._internal.parser.ApacheParser."
                             "update_runtime_variables"):
-                with mock.patch("certbot_apache._internal.apache_util.parse_from_subprocess") as mock_sp:
-                    mock_sp.return_value = []
-                    try:
-                        config_class = entrypoint.OVERRIDE_CLASSES[os_info]
-                    except KeyError:
-                        config_class = configurator.ApacheConfigurator
-                    config = config_class(config=mock_le_config, name="apache",
-                                          version=version, use_parsernode=use_parsernode,
-                                          openssl_version=openssl_version)
-                    if not conf_vhost_path:
-                        config_class.OS_DEFAULTS["vhost_root"] = vhost_path
-                    else:
-                        # Custom virtualhost path was requested
-                        config.config.apache_vhost_root = conf_vhost_path
-                    config.config.apache_ctl = config_class.OS_DEFAULTS["ctl"]
-                    config.prepare()
+                try:
+                    config_class = entrypoint.OVERRIDE_CLASSES[os_info]
+                except KeyError:
+                    config_class = configurator.ApacheConfigurator
+                config = config_class(config=mock_le_config, name="apache",
+                    version=version)
+                if not conf_vhost_path:
+                    config_class.OS_DEFAULTS["vhost_root"] = vhost_path
+                else:
+                    # Custom virtualhost path was requested
+                    config.config.apache_vhost_root = conf_vhost_path
+                config.config.apache_ctl = config_class.OS_DEFAULTS["ctl"]
+                config.prepare()
     return config
 
 
@@ -145,71 +137,71 @@ def get_vh_truth(temp_dir, config_name):
             obj.VirtualHost(
                 os.path.join(prefix, "encryption-example.conf"),
                 os.path.join(aug_pre, "encryption-example.conf/Virtualhost"),
-                {obj.Addr.fromstring("*:80")},
+                set([obj.Addr.fromstring("*:80")]),
                 False, True, "encryption-example.demo"),
             obj.VirtualHost(
                 os.path.join(prefix, "default-ssl.conf"),
                 os.path.join(aug_pre,
                              "default-ssl.conf/IfModule/VirtualHost"),
-                {obj.Addr.fromstring("_default_:443")}, True, True),
+                set([obj.Addr.fromstring("_default_:443")]), True, True),
             obj.VirtualHost(
                 os.path.join(prefix, "000-default.conf"),
                 os.path.join(aug_pre, "000-default.conf/VirtualHost"),
-                {obj.Addr.fromstring("*:80"),
-                     obj.Addr.fromstring("[::]:80")},
+                set([obj.Addr.fromstring("*:80"),
+                     obj.Addr.fromstring("[::]:80")]),
                 False, True, "ip-172-30-0-17"),
             obj.VirtualHost(
                 os.path.join(prefix, "certbot.conf"),
                 os.path.join(aug_pre, "certbot.conf/VirtualHost"),
-                {obj.Addr.fromstring("*:80")}, False, True,
+                set([obj.Addr.fromstring("*:80")]), False, True,
                 "certbot.demo", aliases=["www.certbot.demo"]),
             obj.VirtualHost(
                 os.path.join(prefix, "mod_macro-example.conf"),
                 os.path.join(aug_pre,
                              "mod_macro-example.conf/Macro/VirtualHost"),
-                {obj.Addr.fromstring("*:80")}, False, True,
+                set([obj.Addr.fromstring("*:80")]), False, True,
                 modmacro=True),
             obj.VirtualHost(
                 os.path.join(prefix, "default-ssl-port-only.conf"),
                 os.path.join(aug_pre, ("default-ssl-port-only.conf/"
                                        "IfModule/VirtualHost")),
-                {obj.Addr.fromstring("_default_:443")}, True, True),
+                set([obj.Addr.fromstring("_default_:443")]), True, True),
             obj.VirtualHost(
                 os.path.join(prefix, "wildcard.conf"),
                 os.path.join(aug_pre, "wildcard.conf/VirtualHost"),
-                {obj.Addr.fromstring("*:80")}, False, True,
+                set([obj.Addr.fromstring("*:80")]), False, True,
                 "ip-172-30-0-17", aliases=["*.blue.purple.com"]),
             obj.VirtualHost(
                 os.path.join(prefix, "ocsp-ssl.conf"),
                 os.path.join(aug_pre, "ocsp-ssl.conf/IfModule/VirtualHost"),
-                {obj.Addr.fromstring("10.2.3.4:443")}, True, True,
+                set([obj.Addr.fromstring("10.2.3.4:443")]), True, True,
                 "ocspvhost.com"),
             obj.VirtualHost(
                 os.path.join(prefix, "non-symlink.conf"),
                 os.path.join(aug_pre, "non-symlink.conf/VirtualHost"),
-                {obj.Addr.fromstring("*:80")}, False, True,
+                set([obj.Addr.fromstring("*:80")]), False, True,
                 "nonsym.link"),
             obj.VirtualHost(
                 os.path.join(prefix, "default-ssl-port-only.conf"),
                 os.path.join(aug_pre,
                              "default-ssl-port-only.conf/VirtualHost"),
-                {obj.Addr.fromstring("*:80")}, True, True, ""),
+                set([obj.Addr.fromstring("*:80")]), True, True, ""),
             obj.VirtualHost(
                 os.path.join(temp_dir, config_name,
                              "apache2/apache2.conf"),
                 "/files" + os.path.join(temp_dir, config_name,
                                         "apache2/apache2.conf/VirtualHost"),
-                {obj.Addr.fromstring("*:80")}, False, True,
+                set([obj.Addr.fromstring("*:80")]), False, True,
                 "vhost.in.rootconf"),
             obj.VirtualHost(
                 os.path.join(prefix, "duplicatehttp.conf"),
                 os.path.join(aug_pre, "duplicatehttp.conf/VirtualHost"),
-                {obj.Addr.fromstring("10.2.3.4:80")}, False, True,
+                set([obj.Addr.fromstring("10.2.3.4:80")]), False, True,
                 "duplicate.example.com"),
             obj.VirtualHost(
                 os.path.join(prefix, "duplicatehttps.conf"),
                 os.path.join(aug_pre, "duplicatehttps.conf/IfModule/VirtualHost"),
-                {obj.Addr.fromstring("10.2.3.4:443")}, True, True,
+                set([obj.Addr.fromstring("10.2.3.4:443")]), True, True,
                 "duplicate.example.com")]
         return vh_truth
     if config_name == "debian_apache_2_4/multi_vhosts":
@@ -220,27 +212,27 @@ def get_vh_truth(temp_dir, config_name):
             obj.VirtualHost(
                 os.path.join(prefix, "default.conf"),
                 os.path.join(aug_pre, "default.conf/VirtualHost[1]"),
-                {obj.Addr.fromstring("*:80")},
+                set([obj.Addr.fromstring("*:80")]),
                 False, True, "ip-172-30-0-17"),
             obj.VirtualHost(
                 os.path.join(prefix, "default.conf"),
                 os.path.join(aug_pre, "default.conf/VirtualHost[2]"),
-                {obj.Addr.fromstring("*:80")},
+                set([obj.Addr.fromstring("*:80")]),
                 False, True, "banana.vomit.com"),
             obj.VirtualHost(
                 os.path.join(prefix, "multi-vhost.conf"),
                 os.path.join(aug_pre, "multi-vhost.conf/VirtualHost[1]"),
-                {obj.Addr.fromstring("*:80")},
+                set([obj.Addr.fromstring("*:80")]),
                 False, True, "1.multi.vhost.tld"),
             obj.VirtualHost(
                 os.path.join(prefix, "multi-vhost.conf"),
                 os.path.join(aug_pre, "multi-vhost.conf/IfModule/VirtualHost"),
-                {obj.Addr.fromstring("*:80")},
+                set([obj.Addr.fromstring("*:80")]),
                 False, True, "2.multi.vhost.tld"),
             obj.VirtualHost(
                 os.path.join(prefix, "multi-vhost.conf"),
                 os.path.join(aug_pre, "multi-vhost.conf/VirtualHost[2]"),
-                {obj.Addr.fromstring("*:80")},
+                set([obj.Addr.fromstring("*:80")]),
                 False, True, "3.multi.vhost.tld")]
         return vh_truth
     return None  # pragma: no cover

certbot-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="1.5.0"
+LE_AUTO_VERSION="1.1.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -910,11 +910,20 @@ elif [ -f /etc/manjaro-release ]; then
   }
   BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION"
 elif [ -f /etc/gentoo-release ]; then
-  DEPRECATED_OS=1
+  Bootstrap() {
+    DeprecationBootstrap "Gentoo" BootstrapGentooCommon
+  }
+  BOOTSTRAP_VERSION="BootstrapGentooCommon $BOOTSTRAP_GENTOO_COMMON_VERSION"
 elif uname | grep -iq FreeBSD ; then
-  DEPRECATED_OS=1
+  Bootstrap() {
+    DeprecationBootstrap "FreeBSD" BootstrapFreeBsd
+  }
+  BOOTSTRAP_VERSION="BootstrapFreeBsd $BOOTSTRAP_FREEBSD_VERSION"
 elif uname | grep -iq Darwin ; then
-  DEPRECATED_OS=1
+  Bootstrap() {
+    DeprecationBootstrap "macOS" BootstrapMac
+  }
+  BOOTSTRAP_VERSION="BootstrapMac $BOOTSTRAP_MAC_VERSION"
 elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then
   Bootstrap() {
     ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon
@@ -1265,11 +1274,11 @@ if [ "$1" = "--le-auto-phase2" ]; then
 # pip install hashin
 # hashin -r dependency-requirements.txt cryptography==1.5.2
 # ```
-ConfigArgParse==1.0 \
-    --hash=sha256:bf378245bc9cdc403a527e5b7406b991680c2a530e7e81af747880b54eb57133
-certifi==2019.11.28 \
-    --hash=sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3 \
-    --hash=sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f
+ConfigArgParse==0.14.0 \
+    --hash=sha256:2e2efe2be3f90577aca9415e32cb629aa2ecd92078adbe27b53a03e53ff12e91
+certifi==2019.9.11 \
+    --hash=sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50 \
+    --hash=sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef
 cffi==1.13.2 \
     --hash=sha256:0b49274afc941c626b605fb59b59c3485c17dc776dc3cc7cc14aca74cc19cc42 \
     --hash=sha256:0e3ea92942cb1168e38c05c1d56b0527ce31f1a370f6117f1d490b8dcd6b3a04 \
@@ -1334,9 +1343,7 @@ cryptography==2.8 \
 distro==1.4.0 \
     --hash=sha256:362dde65d846d23baee4b5c058c8586f219b5a54be1cf5fc6ff55c4578392f57 \
     --hash=sha256:eedf82a470ebe7d010f1872c17237c79ab04097948800029994fa458e52fb4b4
-# Package enum34 needs to be explicitly limited to Python2.x, in order to avoid
-# certbot-auto failures on Python 3.6+ which enum34 doesn't support. See #5456.
-enum34==1.1.6 ; python_version < '3.4' \
+enum34==1.1.6 \
     --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850 \
     --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \
     --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \
@@ -1344,6 +1351,8 @@ enum34==1.1.6 ; python_version < '3.4' \
 funcsigs==1.0.2 \
     --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
     --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50
+future==0.18.2 \
+    --hash=sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d
 idna==2.8 \
     --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \
     --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c
@@ -1356,40 +1365,40 @@ josepy==1.2.0 \
 mock==1.3.0 \
     --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 \
     --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb
-parsedatetime==2.5 \
-    --hash=sha256:3b835fc54e472c17ef447be37458b400e3fefdf14bb1ffdedb5d2c853acf4ba1 \
-    --hash=sha256:d2e9ddb1e463de871d32088a3f3cea3dc8282b1b2800e081bd0ef86900451667
-pbr==5.4.4 \
-    --hash=sha256:139d2625547dbfa5fb0b81daebb39601c478c21956dc57e2e07b74450a8c506b \
-    --hash=sha256:61aa52a0f18b71c5cc58232d2cf8f8d09cd67fcad60b742a60124cb8d6951488
-pyOpenSSL==19.1.0 \
-    --hash=sha256:621880965a720b8ece2f1b2f54ea2071966ab00e2970ad2ce11d596102063504 \
-    --hash=sha256:9a24494b2602aaf402be5c9e30a0b82d4a5c67528fe8fb475e3f3bc00dd69507
+parsedatetime==2.4 \
+    --hash=sha256:3d817c58fb9570d1eec1dd46fa9448cd644eeed4fb612684b02dfda3a79cb84b \
+    --hash=sha256:9ee3529454bf35c40a77115f5a596771e59e1aee8c53306f346c461b8e913094
+pbr==5.4.3 \
+    --hash=sha256:2c8e420cd4ed4cec4e7999ee47409e876af575d4c35a45840d59e8b5f3155ab8 \
+    --hash=sha256:b32c8ccaac7b1a20c0ce00ce317642e6cf231cf038f9875e0280e28af5bf7ac9
+pyOpenSSL==19.0.0 \
+    --hash=sha256:aeca66338f6de19d1aa46ed634c3b9ae519a64b458f8468aec688e7e3c20f200 \
+    --hash=sha256:c727930ad54b10fc157015014b666f2d8b41f70c0d03e83ab67624fd3dd5d1e6
 pyRFC3339==1.1 \
     --hash=sha256:67196cb83b470709c580bb4738b83165e67c6cc60e1f2e4f286cfcb402a926f4 \
     --hash=sha256:81b8cbe1519cdb79bed04910dd6fa4e181faf8c88dff1e1b987b5f7ab23a5b1a
 pycparser==2.19 \
     --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3
-pyparsing==2.4.6 \
-    --hash=sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f \
-    --hash=sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec
+pyparsing==2.4.5 \
+    --hash=sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f \
+    --hash=sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a
 python-augeas==0.5.0 \
     --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2
 pytz==2019.3 \
     --hash=sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d \
     --hash=sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be
-requests==2.22.0 \
-    --hash=sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4 \
-    --hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31
+requests==2.21.0 \
+    --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \
+    --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b
 requests-toolbelt==0.9.1 \
     --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \
     --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0
-six==1.14.0 \
-    --hash=sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a \
-    --hash=sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c
-urllib3==1.25.8 \
-    --hash=sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc \
-    --hash=sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc
+six==1.13.0 \
+    --hash=sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd \
+    --hash=sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66
+urllib3==1.24.3 \
+    --hash=sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4 \
+    --hash=sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb
 zope.component==4.6 \
     --hash=sha256:ec2afc5bbe611dcace98bb39822c122d44743d635dafc7315b9aef25097db9e6
 zope.deferredimport==4.3.1 \
@@ -1401,86 +1410,47 @@ zope.deprecation==4.4.0 \
 zope.event==4.4 \
     --hash=sha256:69c27debad9bdacd9ce9b735dad382142281ac770c4a432b533d6d65c4614bcf \
     --hash=sha256:d8e97d165fd5a0997b45f5303ae11ea3338becfe68c401dd88ffd2113fe5cae7
-zope.hookable==5.0.0 \
-    --hash=sha256:0992a0dd692003c09fb958e1480cebd1a28f2ef32faa4857d864f3ca8e9d6952 \
-    --hash=sha256:0f325838dbac827a1e2ed5d482c1f2656b6844dc96aa098f7727e76395fcd694 \
-    --hash=sha256:22a317ba00f61bac99eac1a5e330be7cb8c316275a21269ec58aa396b602af0c \
-    --hash=sha256:25531cb5e7b35e8a6d1d6eddef624b9a22ce5dcf8f4448ef0f165acfa8c3fc21 \
-    --hash=sha256:30890892652766fc80d11f078aca9a5b8150bef6b88aba23799581a53515c404 \
-    --hash=sha256:342d682d93937e5b8c232baffb32a87d5eee605d44f74566657c64a239b7f342 \
-    --hash=sha256:46b2fddf1f5aeb526e02b91f7e62afbb9fff4ffd7aafc97cdb00a0d717641567 \
-    --hash=sha256:523318ff96df9b8d378d997c00c5d4cbfbff68dc48ff5ee5addabdb697d27528 \
-    --hash=sha256:53aa02eb8921d4e667c69d76adeed8fe426e43870c101cb08dcd2f3468aff742 \
-    --hash=sha256:62e79e8fdde087cb20822d7874758f5acbedbffaf3c0fbe06309eb8a41ee4e06 \
-    --hash=sha256:74bf2f757f7385b56dc3548adae508d8b3ef952d600b4b12b88f7d1706b05dcc \
-    --hash=sha256:751ee9d89eb96e00c1d7048da9725ce392a708ed43406416dc5ed61e4d199764 \
-    --hash=sha256:7b83bc341e682771fe810b360cd5d9c886a948976aea4b979ff214e10b8b523b \
-    --hash=sha256:81eeeb27dbb0ddaed8070daee529f0d1bfe4f74c7351cce2aaca3ea287c4cc32 \
-    --hash=sha256:856509191e16930335af4d773c0fc31a17bae8991eb6f167a09d5eddf25b56cc \
-    --hash=sha256:8853e81fd07b18fa9193b19e070dc0557848d9945b1d2dac3b7782543458c87d \
-    --hash=sha256:94506a732da2832029aecdfe6ea07eb1b70ee06d802fff34e1b3618fe7cdf026 \
-    --hash=sha256:95ad874a8cc94e786969215d660143817f745225579bfe318c4676e218d3147c \
-    --hash=sha256:9758ec9174966ffe5c499b6c3d149f80aa0a9238020006a2b87c6af5963fcf48 \
-    --hash=sha256:a169823e331da939aa7178fc152e65699aeb78957e46c6f80ccb50ee4c3616c2 \
-    --hash=sha256:a67878a798f6ca292729a28c2226592b3d000dc6ee7825d31887b553686c7ac7 \
-    --hash=sha256:a9a6d9eb2319a09905670810e2de971d6c49013843700b4975e2fc0afe96c8db \
-    --hash=sha256:b3e118b58a3d2301960e6f5f25736d92f6b9f861728d3b8c26d69f54d8a157d2 \
-    --hash=sha256:ca6705c2a1fb5059a4efbe9f5426be4cdf71b3c9564816916fc7aa7902f19ede \
-    --hash=sha256:cf711527c9d4ae72085f137caffb4be74fc007ffb17cd103628c7d5ba17e205f \
-    --hash=sha256:d087602a6845ebe9d5a1c5a949fedde2c45f372d77fbce4f7fe44b68b28a1d03 \
-    --hash=sha256:d1080e1074ddf75ad6662a9b34626650759c19a9093e1a32a503d37e48da135b \
-    --hash=sha256:db9c60368aff2b7e6c47115f3ad9bd6e96aa298b12ed5f8cb13f5673b30be565 \
-    --hash=sha256:dbeb127a04473f5a989169eb400b67beb921c749599b77650941c21fe39cb8d9 \
-    --hash=sha256:dca336ca3682d869d291d7cd18284f6ff6876e4244eb1821430323056b000e2c \
-    --hash=sha256:dd69a9be95346d10c853b6233fcafe3c0315b89424b378f2ad45170d8e161568 \
-    --hash=sha256:dd79f8fae5894f1ee0a0042214685f2d039341250c994b825c10a4cd075d80f6 \
-    --hash=sha256:e647d850aa1286d98910133cee12bd87c354f7b7bb3f3cd816a62ba7fa2f7007 \
-    --hash=sha256:f37a210b5c04b2d4e4bac494ab15b70196f219a1e1649ddca78560757d4278fb \
-    --hash=sha256:f67820b6d33a705dc3c1c457156e51686f7b350ff57f2112e1a9a4dad38ec268 \
-    --hash=sha256:f68969978ccf0e6123902f7365aae5b7a9e99169d4b9105c47cf28e788116894 \
-    --hash=sha256:f717a0b34460ae1ac0064e91b267c0588ac2c098ffd695992e72cd5462d97a67 \
-    --hash=sha256:f9d58ccec8684ca276d5a4e7b0dfacca028336300a8f715d616d9f0ce9ae8096 \
-    --hash=sha256:fcc3513a54e656067cbf7b98bab0d6b9534b9eabc666d1f78aad6acdf0962736
-zope.interface==4.7.1 \
-    --hash=sha256:048b16ac882a05bc7ef534e8b9f15c9d7a6c190e24e8938a19b7617af4ed854a \
-    --hash=sha256:05816cf8e7407cf62f2ec95c0a5d69ec4fa5741d9ccd10db9f21691916a9a098 \
-    --hash=sha256:065d6a1ac89d35445168813bed45048ed4e67a4cdfc5a68fdb626a770378869f \
-    --hash=sha256:14157421f4121a57625002cc4f48ac7521ea238d697c4a4459a884b62132b977 \
-    --hash=sha256:18dc895945694f397a0be86be760ff664b790f95d8e7752d5bab80284ff9105d \
-    --hash=sha256:1962c9f838bd6ae4075d0014f72697510daefc7e1c7e48b2607df0b6e157989c \
-    --hash=sha256:1a67408cacd198c7e6274a19920bb4568d56459e659e23c4915528686ac1763a \
-    --hash=sha256:21bf781076dd616bd07cf0223f79d61ab4f45176076f90bc2890e18c48195da4 \
-    --hash=sha256:21c0a5d98650aebb84efa16ce2c8df1a46bdc4fe8a9e33237d0ca0b23f416ead \
-    --hash=sha256:23cfeea25d1e42ff3bf4f9a0c31e9d5950aa9e7c4b12f0c4bd086f378f7b7a71 \
-    --hash=sha256:24b6fce1fb71abf9f4093e3259084efcc0ef479f89356757780685bd2b06ef37 \
-    --hash=sha256:24f84ce24eb6b5fcdcb38ad9761524f1ae96f7126abb5e597f8a3973d9921409 \
-    --hash=sha256:25e0ef4a824017809d6d8b0ce4ab3288594ba283e4d4f94d8cfb81d73ed65114 \
-    --hash=sha256:2e8fdd625e9aba31228e7ddbc36bad5c38dc3ee99a86aa420f89a290bd987ce9 \
-    --hash=sha256:2f3bc2f49b67b1bea82b942d25bc958d4f4ea6709b411cb2b6b9718adf7914ce \
-    --hash=sha256:35d24be9d04d50da3a6f4d61de028c1dd087045385a0ff374d93ef85af61b584 \
-    --hash=sha256:35dbe4e8c73003dff40dfaeb15902910a4360699375e7b47d3c909a83ff27cd0 \
-    --hash=sha256:3dfce831b824ab5cf446ed0c350b793ac6fa5fe33b984305cb4c966a86a8fb79 \
-    --hash=sha256:3f7866365df5a36a7b8de8056cd1c605648f56f9a226d918ed84c85d25e8d55f \
-    --hash=sha256:455cc8c01de3bac6f9c223967cea41f4449f58b4c2e724ec8177382ddd183ab4 \
-    --hash=sha256:4bb937e998be9d5e345f486693e477ba79e4344674484001a0b646be1d530487 \
-    --hash=sha256:52303a20902ca0888dfb83230ca3ee6fbe63c0ad1dd60aa0bba7958ccff454d8 \
-    --hash=sha256:6e0a897d4e09859cc80c6a16a29697406ead752292ace17f1805126a4f63c838 \
-    --hash=sha256:6e1816e7c10966330d77af45f77501f9a68818c065dec0ad11d22b50a0e212e7 \
-    --hash=sha256:73b5921c5c6ce3358c836461b5470bf675601c96d5e5d8f2a446951470614f67 \
-    --hash=sha256:8093cd45cdb5f6c8591cfd1af03d32b32965b0f79b94684cd0c9afdf841982bb \
-    --hash=sha256:864b4a94b60db301899cf373579fd9ef92edddbf0fb2cd5ae99f53ef423ccc56 \
-    --hash=sha256:8a27b4d3ea9c6d086ce8e7cdb3e8d319b6752e2a03238a388ccc83ccbe165f50 \
-    --hash=sha256:91b847969d4784abd855165a2d163f72ac1e58e6dce09a5e46c20e58f19cc96d \
-    --hash=sha256:b47b1028be4758c3167e474884ccc079b94835f058984b15c145966c4df64d27 \
-    --hash=sha256:b68814a322835d8ad671b7acc23a3b2acecba527bb14f4b53fc925f8a27e44d8 \
-    --hash=sha256:bcb50a032c3b6ec7fb281b3a83d2b31ab5246c5b119588725b1350d3a1d9f6a3 \
-    --hash=sha256:c56db7d10b25ce8918b6aec6b08ac401842b47e6c136773bfb3b590753f7fb67 \
-    --hash=sha256:c94b77a13d4f47883e4f97f9fa00f5feadd38af3e6b3c7be45cfdb0a14c7149b \
-    --hash=sha256:db381f6fdaef483ad435f778086ccc4890120aff8df2ba5cfeeac24d280b3145 \
-    --hash=sha256:e6487d01c8b7ed86af30ea141fcc4f93f8a7dde26f94177c1ad637c353bd5c07 \
-    --hash=sha256:e86923fa728dfba39c5bb6046a450bd4eec8ad949ac404eca728cfce320d1732 \
-    --hash=sha256:f6ca36dc1e9eeb46d779869c60001b3065fb670b5775c51421c099ea2a77c3c9 \
-    --hash=sha256:fb62f2cbe790a50d95593fb40e8cca261c31a2f5637455ea39440d6457c2ba25
+zope.hookable==4.2.0 \
+    --hash=sha256:22886e421234e7e8cedc21202e1d0ab59960e40a47dd7240e9659a2d82c51370 \
+    --hash=sha256:39912f446e45b4e1f1951b5ffa2d5c8b074d25727ec51855ae9eab5408f105ab \
+    --hash=sha256:3adb7ea0871dbc56b78f62c4f5c024851fc74299f4f2a95f913025b076cde220 \
+    --hash=sha256:3d7c4b96341c02553d8b8d71065a9366ef67e6c6feca714f269894646bb8268b \
+    --hash=sha256:4e826a11a529ed0464ffcecf34b0b7bd1b4928dd5848c5c61bedd7833e8f4801 \
+    --hash=sha256:700d68cc30728de1c4c62088a981c6daeaefdf20a0d81995d2c0b7f442c5f88c \
+    --hash=sha256:77c82a430cedfbf508d1aa406b2f437363c24fa90c73f577ead0fb5295749b83 \
+    --hash=sha256:c1df3929a3666fc5a0c80d60a0c1e6f6ef97c7f6ed2f1b7cf49f3e6f3d4dde15 \
+    --hash=sha256:dba8b2dd2cd41cb5f37bfa3f3d82721b8ae10e492944e48ddd90a439227f2893 \
+    --hash=sha256:f492540305b15b5591bd7195d61f28946bb071de071cee5d68b6b8414da90fd2
+zope.interface==4.6.0 \
+    --hash=sha256:086707e0f413ff8800d9c4bc26e174f7ee4c9c8b0302fbad68d083071822316c \
+    --hash=sha256:1157b1ec2a1f5bf45668421e3955c60c610e31913cc695b407a574efdbae1f7b \
+    --hash=sha256:11ebddf765bff3bbe8dbce10c86884d87f90ed66ee410a7e6c392086e2c63d02 \
+    --hash=sha256:14b242d53f6f35c2d07aa2c0e13ccb710392bcd203e1b82a1828d216f6f6b11f \
+    --hash=sha256:1b3d0dcabc7c90b470e59e38a9acaa361be43b3a6ea644c0063951964717f0e5 \
+    --hash=sha256:20a12ab46a7e72b89ce0671e7d7a6c3c1ca2c2766ac98112f78c5bddaa6e4375 \
+    --hash=sha256:298f82c0ab1b182bd1f34f347ea97dde0fffb9ecf850ecf7f8904b8442a07487 \
+    --hash=sha256:2f6175722da6f23dbfc76c26c241b67b020e1e83ec7fe93c9e5d3dd18667ada2 \
+    --hash=sha256:3b877de633a0f6d81b600624ff9137312d8b1d0f517064dfc39999352ab659f0 \
+    --hash=sha256:4265681e77f5ac5bac0905812b828c9fe1ce80c6f3e3f8574acfb5643aeabc5b \
+    --hash=sha256:550695c4e7313555549aa1cdb978dc9413d61307531f123558e438871a883d63 \
+    --hash=sha256:5f4d42baed3a14c290a078e2696c5f565501abde1b2f3f1a1c0a94fbf6fbcc39 \
+    --hash=sha256:62dd71dbed8cc6a18379700701d959307823b3b2451bdc018594c48956ace745 \
+    --hash=sha256:7040547e5b882349c0a2cc9b50674b1745db551f330746af434aad4f09fba2cc \
+    --hash=sha256:7e099fde2cce8b29434684f82977db4e24f0efa8b0508179fce1602d103296a2 \
+    --hash=sha256:7e5c9a5012b2b33e87980cee7d1c82412b2ebabcb5862d53413ba1a2cfde23aa \
+    --hash=sha256:81295629128f929e73be4ccfdd943a0906e5fe3cdb0d43ff1e5144d16fbb52b1 \
+    --hash=sha256:95cc574b0b83b85be9917d37cd2fad0ce5a0d21b024e1a5804d044aabea636fc \
+    --hash=sha256:968d5c5702da15c5bf8e4a6e4b67a4d92164e334e9c0b6acf080106678230b98 \
+    --hash=sha256:9e998ba87df77a85c7bed53240a7257afe51a07ee6bc3445a0bf841886da0b97 \
+    --hash=sha256:a0c39e2535a7e9c195af956610dba5a1073071d2d85e9d2e5d789463f63e52ab \
+    --hash=sha256:a15e75d284178afe529a536b0e8b28b7e107ef39626a7809b4ee64ff3abc9127 \
+    --hash=sha256:a6a6ff82f5f9b9702478035d8f6fb6903885653bff7ec3a1e011edc9b1a7168d \
+    --hash=sha256:b639f72b95389620c1f881d94739c614d385406ab1d6926a9ffe1c8abbea23fe \
+    --hash=sha256:bad44274b151d46619a7567010f7cde23a908c6faa84b97598fd2f474a0c6891 \
+    --hash=sha256:bbcef00d09a30948756c5968863316c949d9cedbc7aabac5e8f0ffbdb632e5f1 \
+    --hash=sha256:d788a3999014ddf416f2dc454efa4a5dbeda657c6aba031cf363741273804c6b \
+    --hash=sha256:eed88ae03e1ef3a75a0e96a55a99d7937ed03e53d0cffc2451c208db445a2966 \
+    --hash=sha256:f99451f3a579e73b5dd58b1b08d1179791d49084371d9a47baad3b22417f0317
 zope.proxy==4.3.3 \
     --hash=sha256:04646ac04ffa9c8e32fb2b5c3cd42995b2548ea14251f3c21ca704afae88e42c \
     --hash=sha256:07b6bceea232559d24358832f1cd2ed344bbf05ca83855a5b9698b5f23c5ed60 \
@@ -1533,18 +1503,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==1.5.0 \
-    --hash=sha256:ec1f01af06b52a6f079f5b02cb70e88f0671a7b13ecb3e45b040563e32c6e53a \
-    --hash=sha256:c52017a4f84137e1312c898d6ae69c5f7977d79d2bd4c2df013cbbf39b6539bf
-acme==1.5.0 \
-    --hash=sha256:66de67b394bb7606f97f2c21507e6eb6a88936db2a940f5c4893025f87e3852a \
-    --hash=sha256:b051ff7dd3935b2032c2f8c8386e905d9b658eba9f3455e352650d85bea9c8f0
-certbot-apache==1.5.0 \
-    --hash=sha256:d2c28be6dcd6c56a8040c8c733e72c1341238b1b47fb59f544eb832b9d5c81ba \
-    --hash=sha256:3eec5a49ae4fcf86213f962eb1e11d8a725b65e7dcee18f9b92c7aa73f821764
-certbot-nginx==1.5.0 \
-    --hash=sha256:3d27fd02ebe15b07ce5fa9525ceeda82aa5fdc45aa064729434faff0442d1f91 \
-    --hash=sha256:b38f101588af6d2b8ea7c2e3334f249afbe14461a85add2f1420091d860df983
+certbot==1.1.0 \
+    --hash=sha256:66a5cab9267349941604c2c98082bfef85877653c023fc324b1c3869fb16add6 \
+    --hash=sha256:46e93661a0db53f416c0f5476d8d2e62bc7259b7660dd983453b85df9ef6e8b8
+acme==1.1.0 \
+    --hash=sha256:11b9beba706fb8f652c8910d46dd1939d670cac8169f3c66c18c080ed3353e71 \
+    --hash=sha256:c305a20eeb9cb02240347703d497891c13d43a47c794fa100d4dbb479a5370d9
+certbot-apache==1.1.0 \
+    --hash=sha256:9c847ff223c2e465e241c78d22f97cee77d5e551df608bed06c55f8627f4cbd2 \
+    --hash=sha256:05e84dfe96b72582cde97c490977d8e2d33d440c927a320debb4cf287f6fadcc
+certbot-nginx==1.1.0 \
+    --hash=sha256:bf06fa2f5059f0fdb7d352c8739e1ed0830db4f0d89e812dab4f081bda6ec7d6 \
+    --hash=sha256:0a80ecbd2a30f3757c7652cabfff854ca07873b1cf02ebbe1892786c3b3a5874
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------

certbot-ci/certbot_integration_tests/assets/hook.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import print_function
 import os
 import sys
 
@@ -8,4 +7,5 @@ if hook_script_type == 'deploy' and ('RENEWED_DOMAINS' not in os.environ or 'REN
     sys.stderr.write('Environment variables not properly set!\n')
     sys.exit(1)
 
-print(hook_script_type)
+with open(sys.argv[2], 'a') as file_h:
+    file_h.write(hook_script_type + '\n')

certbot-ci/certbot_integration_tests/certbot_tests/assertions.py

@@ -1,5 +1,4 @@
 """This module contains advanced assertions for the certbot integration tests."""
-import io
 import os
 
 try:
@@ -22,8 +21,7 @@ def assert_hook_execution(probe_path, probe_content):
     :param probe_path: path to the file that received the hook output
     :param probe_content: content expected when the hook is executed
     """
-    encoding = 'utf-8' if POSIX_MODE else 'utf-16'
-    with io.open(probe_path, 'rt', encoding=encoding) as file:
+    with open(probe_path, 'r') as file:
         data = file.read()
 
     lines = [line.strip() for line in data.splitlines()]

certbot-ci/certbot_integration_tests/certbot_tests/context.py

@@ -1,4 +1,5 @@
 """Module to handle the context of integration tests."""
+import logging
 import os
 import shutil
 import sys

certbot-ci/certbot_integration_tests/certbot_tests/test_main.py

@@ -595,23 +595,6 @@ def test_ocsp_status_live(context):
     assert output.count('REVOKED') == 1, 'Expected {0} to be REVOKED'.format(cert)
 
 
-def test_ocsp_renew(context):
-    """Test that revoked certificates are renewed."""
-    # Obtain a certificate
-    certname = context.get_domain('ocsp-renew')
-    context.certbot(['--domains', certname])
-
-    # Test that "certbot renew" does not renew the certificate
-    assert_cert_count_for_lineage(context.config_dir, certname, 1)
-    context.certbot(['renew'], force_renew=False)
-    assert_cert_count_for_lineage(context.config_dir, certname, 1)
-
-    # Revoke the certificate and test that it does renew the certificate
-    context.certbot(['revoke', '--cert-name', certname, '--no-delete-after-revoke'])
-    context.certbot(['renew'], force_renew=False)
-    assert_cert_count_for_lineage(context.config_dir, certname, 2)
-
-
 def test_dry_run_deactivate_authzs(context):
     """Test that Certbot deactivates authorizations when performing a dry run"""
 

certbot-ci/certbot_integration_tests/utils/acme_server.py

@@ -38,7 +38,7 @@ class ACMEServer(object):
         :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble)
         :param list nodes: list of node names that will be setup by pytest xdist
         :param bool http_proxy: if False do not start the HTTP proxy
-        :param bool stdout: if True stream all subprocesses stdout to standard stdout
+        :param bool stdout: if True stream subprocesses stdout to standard stdout
         """
         self._construct_acme_xdist(acme_server, nodes)
 
@@ -86,8 +86,7 @@ class ACMEServer(object):
                                                 'alpine', 'rm', '-rf', '/workspace/boulder'])
                 process.wait()
         finally:
-            if os.path.exists(self._workspace):
-                shutil.rmtree(self._workspace)
+            shutil.rmtree(self._workspace)
         if self._stdout != sys.stdout:
             self._stdout.close()
         print('=> Test infrastructure stopped and cleaned up.')
@@ -132,7 +131,7 @@ class ACMEServer(object):
         environ['PEBBLE_AUTHZREUSE'] = '100'
 
         self._launch_process(
-            [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053', '-strict'],
+            [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053'],
             env=environ)
 
         self._launch_process(
@@ -166,24 +165,17 @@ class ACMEServer(object):
         os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'),
                   join(instance_path, 'test/rate-limit-policies.yml'))
 
-        try:
-            # Launch the Boulder server
-            self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path)
+        # Launch the Boulder server
+        self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path)
 
-            # Wait for the ACME CA server to be up.
-            print('=> Waiting for boulder instance to respond...')
-            misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=300)
+        # Wait for the ACME CA server to be up.
+        print('=> Waiting for boulder instance to respond...')
+        misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=240)
 
-            # Configure challtestsrv to answer any A record request with ip of the docker host.
-            response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT),
-                                     json={'ip': '10.77.77.1'})
-            response.raise_for_status()
-        except BaseException:
-            # If we failed to set up boulder, print its logs.
-            print('=> Boulder setup failed. Boulder logs are:')
-            process = self._launch_process(['docker-compose', 'logs'], cwd=instance_path, force_stderr=True)
-            process.wait()
-            raise
+        # Configure challtestsrv to answer any A record request with ip of the docker host.
+        response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT),
+                                 json={'ip': '10.77.77.1'})
+        response.raise_for_status()
 
         print('=> Finished boulder instance deployment.')
 
@@ -196,12 +188,11 @@ class ACMEServer(object):
         self._launch_process(command)
         print('=> Finished configuring the HTTP proxy.')
 
-    def _launch_process(self, command, cwd=os.getcwd(), env=None, force_stderr=False):
+    def _launch_process(self, command, cwd=os.getcwd(), env=None):
         """Launch silently a subprocess OS command"""
         if not env:
             env = os.environ
-        stdout = sys.stderr if force_stderr else self._stdout
-        process = subprocess.Popen(command, stdout=stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env)
+        process = subprocess.Popen(command, stdout=self._stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env)
         self._processes.append(process)
         return process
 

certbot-ci/certbot_integration_tests/utils/misc.py

@@ -140,12 +140,13 @@ def generate_test_file_hooks(config_dir, hook_probe):
             entrypoint_script = '''\
 #!/usr/bin/env bash
 set -e
-"{0}" "{1}" "{2}" >> "{3}"
+"{0}" "{1}" "{2}" "{3}"
 '''.format(sys.executable, hook_path, entrypoint_script_path, hook_probe)
         else:
-            entrypoint_script_path = os.path.join(hook_dir, 'entrypoint.ps1')
+            entrypoint_script_path = os.path.join(hook_dir, 'entrypoint.bat')
             entrypoint_script = '''\
-& "{0}" "{1}" "{2}" >> "{3}"
+@echo off
+"{0}" "{1}" "{2}" "{3}"
             '''.format(sys.executable, hook_path, entrypoint_script_path, hook_probe)
 
         with open(entrypoint_script_path, 'w') as file_h:
@@ -235,7 +236,7 @@ def generate_csr(domains, key_path, csr_path, key_type=RSA_KEY_TYPE):
         file_h.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, key))
 
     req = crypto.X509Req()
-    san = ', '.join('DNS:{0}'.format(item) for item in domains)
+    san = ', '.join(['DNS:{0}'.format(item) for item in domains])
     san_constraint = crypto.X509Extension(b'subjectAltName', False, san.encode('utf-8'))
     req.add_extensions([san_constraint])
 

certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py

@@ -7,7 +7,7 @@ import requests
 
 from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT
 
-PEBBLE_VERSION = 'v2.3.0'
+PEBBLE_VERSION = 'v2.2.1'
 ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets')
 
 

certbot-ci/setup.py

@@ -40,7 +40,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
-    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Intended Audience :: Developers',
@@ -49,6 +49,7 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',

certbot-ci/windows_installer_integration_tests/conftest.py

@@ -1,38 +0,0 @@
-"""
-General conftest for pytest execution of all integration tests lying
-in the window_installer_integration tests package.
-As stated by pytest documentation, conftest module is used to set on
-for a directory a specific configuration using built-in pytest hooks.
-
-See https://docs.pytest.org/en/latest/reference.html#hook-reference
-"""
-from __future__ import print_function
-import os
-
-import pytest
-
-ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
-
-
-def pytest_addoption(parser):
-    """
-    Standard pytest hook to add options to the pytest parser.
-    :param parser: current pytest parser that will be used on the CLI
-    """
-    parser.addoption('--installer-path',
-                     default=os.path.join(ROOT_PATH, 'windows-installer', 'build',
-                                          'nsis', 'certbot-beta-installer-win32.exe'),
-                     help='set the path of the windows installer to use, default to '
-                          'CERTBOT_ROOT_PATH\\windows-installer\\build\\nsis\\certbot-beta-installer-win32.exe')
-    parser.addoption('--allow-persistent-changes', action='store_true',
-                     help='needs to be set, and confirm that the test will make persistent changes on this machine')
-
-
-def pytest_configure(config):
-    """
-    Standard pytest hook used to add a configuration logic for each node of a pytest run.
-    :param config: the current pytest configuration
-    """
-    if not config.option.allow_persistent_changes:
-        raise RuntimeError('This integration test would install Certbot on your machine. '
-                           'Please run it again with the `--allow-persistent-changes` flag set to acknowledge.')

certbot-ci/windows_installer_integration_tests/test_main.py

@@ -1,61 +0,0 @@
-import os
-import time
-import unittest
-import subprocess
-import re
-
-
-@unittest.skipIf(os.name != 'nt', reason='Windows installer tests must be run on Windows.')
-def test_it(request):
-    try:
-        subprocess.check_call(['certbot', '--version'])
-    except (subprocess.CalledProcessError, OSError):
-        pass
-    else:
-        raise AssertionError('Expect certbot to not be available in the PATH.')
-
-    try:
-        # Install certbot
-        subprocess.check_call([request.config.option.installer_path, '/S'])
-
-        # Assert certbot is installed and runnable
-        output = subprocess.check_output(['certbot', '--version'], universal_newlines=True)
-        assert re.match(r'^certbot \d+\.\d+\.\d+.*$', output), 'Flag --version does not output a version.'
-
-        # Assert renew task is installed and ready
-        output = _ps('(Get-ScheduledTask -TaskName "Certbot Renew Task").State', capture_stdout=True)
-        assert output.strip() == 'Ready'
-
-        # Assert renew task is working
-        now = time.time()
-        _ps('Start-ScheduledTask -TaskName "Certbot Renew Task"')
-
-        status = 'Running'
-        while status != 'Ready':
-            status = _ps('(Get-ScheduledTask -TaskName "Certbot Renew Task").State', capture_stdout=True).strip()
-            time.sleep(1)
-
-        log_path = os.path.join('C:\\', 'Certbot', 'log', 'letsencrypt.log')
-
-        modification_time = os.path.getmtime(log_path)
-        assert now < modification_time, 'Certbot log file has not been modified by the renew task.'
-
-        with open(log_path) as file_h:
-            data = file_h.read()
-        assert 'no renewal failures' in data, 'Renew task did not execute properly.'
-
-    finally:
-        # Sadly this command cannot work in non interactive mode: uninstaller will ask explicitly permission in an UAC prompt
-        # print('Uninstalling Certbot ...')
-        # uninstall_path = _ps('(gci "HKLM:\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall"'
-        #                      ' | foreach { gp $_.PSPath }'
-        #                      ' | ? { $_ -match "Certbot" }'
-        #                      ' | select UninstallString)'
-        #                      '.UninstallString', capture_stdout=True)
-        # subprocess.check_call([uninstall_path, '/S'])
-        pass
-
-
-def _ps(powershell_str, capture_stdout=False):
-    fn = subprocess.check_output if capture_stdout else subprocess.check_call
-    return fn(['powershell.exe', '-c', powershell_str], universal_newlines=True)

certbot-compatibility-test/Dockerfile

@@ -31,7 +31,7 @@ COPY certbot-nginx /opt/certbot/src/certbot-nginx/
 COPY certbot-compatibility-test /opt/certbot/src/certbot-compatibility-test/
 COPY tools /opt/certbot/src/tools
 
-RUN VIRTUALENV_NO_DOWNLOAD=1 virtualenv -p python2 /opt/certbot/venv && \
+RUN VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages -p python2 /opt/certbot/venv && \
     /opt/certbot/venv/bin/pip install -U setuptools && \
     /opt/certbot/venv/bin/pip install -U pip
 ENV PATH /opt/certbot/venv/bin:$PATH

certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py

@@ -3,10 +3,7 @@ import os
 import shutil
 import subprocess
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 import zope.interface
 
 from certbot import errors as le_errors

certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py

@@ -5,7 +5,7 @@ import subprocess
 
 import zope.interface
 
-from acme.magic_typing import Set
+from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
 from certbot._internal import configuration
 from certbot_compatibility_test import errors
 from certbot_compatibility_test import interfaces

certbot-compatibility-test/certbot_compatibility_test/test_driver.py

@@ -15,8 +15,8 @@ from urllib3.util import connection
 from acme import challenges
 from acme import crypto_util
 from acme import messages
-from acme.magic_typing import List
-from acme.magic_typing import Tuple
+from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Tuple  # pylint: disable=unused-import, no-name-in-module
 from certbot import achallenges
 from certbot import errors as le_errors
 from certbot.tests import acme_util
@@ -96,7 +96,7 @@ def test_authenticator(plugin, config, temp_dir):
 
 def _create_achalls(plugin):
     """Returns a list of annotated challenges to test on plugin"""
-    achalls = []
+    achalls = list()
     names = plugin.get_testable_domain_names()
     for domain in names:
         prefs = plugin.get_chall_pref(domain)

certbot-compatibility-test/certbot_compatibility_test/validator.py

@@ -4,7 +4,7 @@ import socket
 
 import requests
 import six
-from six.moves import xrange
+from six.moves import xrange  # pylint: disable=import-error, redefined-builtin
 
 from acme import crypto_util
 from acme import errors as acme_errors
@@ -13,6 +13,7 @@ logger = logging.getLogger(__name__)
 
 
 class Validator(object):
+    # pylint: disable=no-self-use
     """Collection of functions to test a live webserver's configuration"""
 
     def certificate(self, cert, name, alt_host=None, port=443):

certbot-compatibility-test/certbot_compatibility_test/validator_test.py

@@ -1,10 +1,7 @@
 """Tests for certbot_compatibility_test.validator."""
 import unittest
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import mock
 import OpenSSL
 import requests
 

certbot-compatibility-test/setup.py

@@ -1,29 +1,19 @@
-from distutils.version import StrictVersion
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.6.0.dev0'
+version = '1.2.0.dev0'
 
 install_requires = [
     'certbot',
     'certbot-apache',
+    'mock',
     'six',
     'requests',
     'zope.interface',
 ]
 
-setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 if sys.version_info < (2, 7, 9):
     # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning)
     install_requires.append('ndg-httpsclient')
@@ -38,7 +28,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
-    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Intended Audience :: Developers',
@@ -47,6 +37,7 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',

certbot-dns-cloudflare/certbot_dns_cloudflare/__init__.py

@@ -22,40 +22,17 @@ Credentials
 
 Use of this plugin requires a configuration file containing Cloudflare API
 credentials, obtained from your Cloudflare
-`account page <https://dash.cloudflare.com/profile/api-tokens>`_.
-
-Previously, Cloudflare's "Global API Key" was used for authentication, however
-this key can access the entire Cloudflare API for all domains in your account,
-meaning it could cause a lot of damage if leaked.
-
-Cloudflare's newer API Tokens can be restricted to specific domains and
-operations, and are therefore now the recommended authentication option.
-
-However, due to some shortcomings in Cloudflare's implementation of Tokens,
-Tokens created for Certbot currently require ``Zone:Zone:Read`` and ``Zone:DNS:Edit``
-permissions for **all** zones in your account. While this is not ideal, your Token
-will still have fewer permission than the Global key, so it's still worth doing.
-Hopefully Cloudflare will improve this in the future.
-
-Using Cloudflare Tokens also requires at least version 2.3.1 of the ``cloudflare``
-python module. If the version that automatically installed with this plugin is
-older than that, and you can't upgrade it on your system, you'll have to stick to
-the Global key.
+`account page <https://www.cloudflare.com/a/account/my-account>`_. This plugin
+does not currently support Cloudflare's "API Tokens", so please ensure you use
+the "Global API Key" for authentication.
 
 .. code-block:: ini
-   :name: certbot_cloudflare_token.ini
-   :caption: Example credentials file using restricted API Token (recommended):
-
-   # Cloudflare API token used by Certbot
-   dns_cloudflare_api_token = 0123456789abcdef0123456789abcdef01234567
-
-.. code-block:: ini
-   :name: certbot_cloudflare_key.ini
-   :caption: Example credentials file using Global API Key (not recommended):
+   :name: credentials.ini
+   :caption: Example credentials file:
 
    # Cloudflare API credentials used by Certbot
    dns_cloudflare_email = cloudflare@example.com
-   dns_cloudflare_api_key = 0123456789abcdef0123456789abcdef01234
+   dns_cloudflare_api_key = 0123456789abcdef0123456789abcdef01234567
 
 The path to this file can be provided interactively or using the
 ``--dns-cloudflare-credentials`` command-line argument. Certbot records the path

certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py

@@ -4,10 +4,6 @@ import logging
 import CloudFlare
 import zope.interface
 
-from acme.magic_typing import Any
-from acme.magic_typing import Dict
-from acme.magic_typing import List
-
 from certbot import errors
 from certbot import interfaces
 from certbot.plugins import dns_common
@@ -38,39 +34,18 @@ class Authenticator(dns_common.DNSAuthenticator):
         super(Authenticator, cls).add_parser_arguments(add)
         add('credentials', help='Cloudflare credentials INI file.')
 
-    def more_info(self):  # pylint: disable=missing-function-docstring
+    def more_info(self):  # pylint: disable=missing-docstring,no-self-use
         return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \
                'the Cloudflare API.'
 
-    def _validate_credentials(self, credentials):
-        token = credentials.conf('api-token')
-        email = credentials.conf('email')
-        key = credentials.conf('api-key')
-        if token:
-            if email or key:
-                raise errors.PluginError('{}: dns_cloudflare_email and dns_cloudflare_api_key are '
-                                         'not needed when using an API Token'
-                                         .format(credentials.confobj.filename))
-        elif email or key:
-            if not email:
-                raise errors.PluginError('{}: dns_cloudflare_email is required when using a Global '
-                                         'API Key. (should be email address associated with '
-                                         'Cloudflare account)'.format(credentials.confobj.filename))
-            if not key:
-                raise errors.PluginError('{}: dns_cloudflare_api_key is required when using a '
-                                         'Global API Key. (see {})'
-                                         .format(credentials.confobj.filename, ACCOUNT_URL))
-        else:
-            raise errors.PluginError('{}: Either dns_cloudflare_api_token (recommended), or '
-                                     'dns_cloudflare_email and dns_cloudflare_api_key are required.'
-                                     ' (see {})'.format(credentials.confobj.filename, ACCOUNT_URL))
-
     def _setup_credentials(self):
         self.credentials = self._configure_credentials(
             'credentials',
             'Cloudflare credentials INI file',
-            None,
-            self._validate_credentials
+            {
+                'email': 'email address associated with Cloudflare account',
+                'api-key': 'API key for Cloudflare account, obtained from {0}'.format(ACCOUNT_URL)
+            }
         )
 
     def _perform(self, domain, validation_name, validation):
@@ -80,8 +55,6 @@ class Authenticator(dns_common.DNSAuthenticator):
         self._get_cloudflare_client().del_txt_record(domain, validation_name, validation)
 
     def _get_cloudflare_client(self):
-        if self.credentials.conf('api-token'):
-            return _CloudflareClient(None, self.credentials.conf('api-token'))
         return _CloudflareClient(self.credentials.conf('email'), self.credentials.conf('api-key'))
 
 
@@ -115,15 +88,8 @@ class _CloudflareClient(object):
             logger.debug('Attempting to add record to zone %s: %s', zone_id, data)
             self.cf.zones.dns_records.post(zone_id, data=data)  # zones | pylint: disable=no-member
         except CloudFlare.exceptions.CloudFlareAPIError as e:
-            code = int(e)
-            hint = None
-
-            if code == 9109:
-                hint = 'Does your API token have "Zone:DNS:Edit" permissions?'
-
             logger.error('Encountered CloudFlareAPIError adding TXT record: %d %s', e, e)
-            raise errors.PluginError('Error communicating with the Cloudflare API: {0}{1}'
-                                     .format(e, ' ({0})'.format(hint) if hint else ''))
+            raise errors.PluginError('Error communicating with the Cloudflare API: {0}'.format(e))
 
         record_id = self._find_txt_record_id(zone_id, record_name, record_content)
         logger.debug('Successfully added TXT record with record_id: %s', record_id)
@@ -173,8 +139,6 @@ class _CloudflareClient(object):
         """
 
         zone_name_guesses = dns_common.base_domain_name_guesses(domain)
-        zones = []  # type: List[Dict[str, Any]]
-        code = msg = None
 
         for zone_name in zone_name_guesses:
             params = {'name': zone_name,
@@ -184,26 +148,16 @@ class _CloudflareClient(object):
                 zones = self.cf.zones.get(params=params)  # zones | pylint: disable=no-member
             except CloudFlare.exceptions.CloudFlareAPIError as e:
                 code = int(e)
-                msg = str(e)
                 hint = None
 
                 if code == 6003:
-                    hint = ('Did you copy your entire API token/key? To use Cloudflare tokens, '
-                            'you\'ll need the python package cloudflare>=2.3.1.{}'
-                    .format(' This certbot is running cloudflare ' + str(CloudFlare.__version__)
-                    if hasattr(CloudFlare, '__version__') else ''))
+                    hint = 'Did you copy your entire API key?'
                 elif code == 9103:
-                    hint = 'Did you enter the correct email address and Global key?'
-                elif code == 9109:
-                    hint = 'Did you enter a valid Cloudflare Token?'
+                    hint = 'Did you enter the correct email address?'
 
-                if hint:
-                    raise errors.PluginError('Error determining zone_id: {0} {1}. Please confirm '
-                                  'that you have supplied valid Cloudflare API credentials. ({2})'
-                                                                         .format(code, msg, hint))
-                else:
-                    logger.debug('Unrecognised CloudFlareAPIError while finding zone_id: %d %s. '
-                                 'Continuing with next zone guess...', e, e)
+                raise errors.PluginError('Error determining zone_id: {0} {1}. Please confirm that '
+                                         'you have supplied valid Cloudflare API credentials.{2}'
+                                         .format(code, e, ' ({0})'.format(hint) if hint else ''))
 
             if zones:
                 zone_id = zones[0]['id']
@@ -211,10 +165,9 @@ class _CloudflareClient(object):
                 return zone_id
 
         raise errors.PluginError('Unable to determine zone_id for {0} using zone names: {1}. '
-                                'Please confirm that the domain name has been entered correctly '
-                                'and is already associated with the supplied Cloudflare account.{2}'
-                                .format(domain, zone_name_guesses, ' The error from Cloudflare was:'
-                                ' {0} {1}'.format(code, msg) if code is not None else ''))
+                                 'Please confirm that the domain name has been entered correctly '
+                                 'and is already associated with the supplied Cloudflare account.'
+                                 .format(domain, zone_name_guesses))
 
     def _find_txt_record_id(self, zone_id, record_name, record_content):
         """