WIP

Fix AppVeyor on the apache-parser-v2 branch

.codecov.yml

@@ -0,0 +1,18 @@
+coverage:
+  status:
+    project:
+      default: off
+      linux:
+        flags: linux
+        # Fixed target instead of auto set by #7173, can
+        # be removed when flags in Codecov are added back.
+        target: 97.5
+        threshold: 0.1
+        base: auto
+      windows:
+        flags: windows
+        # Fixed target instead of auto set by #7173, can
+        # be removed when flags in Codecov are added back.
+        target: 97.6
+        threshold: 0.1
+        base: auto

.github/stale.yml

@@ -0,0 +1,35 @@
+# Configuration for https://github.com/marketplace/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 365
+
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+# When changing this value, be sure to also update markComment below.
+daysUntilClose: 30
+
+# Ignore issues with an assignee (defaults to false)
+exemptAssignees: true
+
+# Label to use when marking as stale
+staleLabel: needs-update
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+  We've made a lot of changes to Certbot since this issue was opened. If you
+  still have this issue with an up-to-date version of Certbot, can you please
+  add a comment letting us know? This helps us to better see what issues are
+  still affecting our users. If there is no activity in the next 30 days, this
+  issue will be automatically closed.
+
+# Comment to post when closing a stale Issue or Pull Request.
+closeComment: >
+  This issue has been closed due to lack of activity, but if you think it
+  should be reopened, please open a new issue with a link to this one and we'll
+  take a look.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 1
+
+# Don't mark pull requests as stale.
+only: issues

.gitignore

@@ -44,3 +44,8 @@ tests/letstest/venv/
 
 # docker files
 .docker
+
+# certbot tests
+.certbot_test_workspace
+**/assets/pebble*
+**/assets/challtestsrv*

.pylintrc

@@ -251,7 +251,7 @@ ignored-modules=pkg_resources,confargparse,argparse,six.moves,six.moves.urllib
 
 # List of classes names for which member attributes should not be checked
 # (useful for classes with attributes dynamically set).
-ignored-classes=SQLObject
+ignored-classes=Field,Header,JWS,closing
 
 # When zope mode is activated, add a predefined set of Zope acquired attributes
 # to generated-members.
@@ -304,7 +304,7 @@ max-args=6
 
 # Argument names that match this expression will be ignored. Default to name
 # with leading underscore
-ignored-argument-names=_.*
+ignored-argument-names=(unused)?_.*|dummy
 
 # Maximum number of locals for function / method body
 max-locals=15

.travis.yml

@@ -4,64 +4,11 @@ cache:
     directories:
         - $HOME/.cache/pip
 
-before_install:
-  - '([ $TRAVIS_OS_NAME == linux ] && dpkg -s libaugeas0) || (brew update && brew install augeas python3 && brew upgrade python && brew link python)'
-
 before_script:
-  - 'if [ $TRAVIS_OS_NAME = osx ] ; then ulimit -n 1024 ; fi'
-
-matrix:
-  include:
-    - python: "2.7"
-      env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=all TOXENV=py27_install
-      sudo: required
-      services: docker
-    - python: "2.7"
-      env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=all TOXENV=py27_install
-      sudo: required
-      services: docker
-    - python: "2.7"
-      env: TOXENV=py27-cover FYI="py27 tests + code coverage"
-    - sudo: required
-      env: TOXENV=nginx_compat
-      services: docker
-      before_install:
-      addons:
-    - python: "2.7"
-      env: TOXENV=lint
-    - python: "3.4"
-      env: TOXENV=mypy
-    - python: "3.5"
-      env: TOXENV=mypy
-    - python: "2.7"
-      env: TOXENV='py27-{acme,apache,certbot,dns,nginx,postfix}-oldest'
-      sudo: required
-      services: docker
-    - python: "3.4"
-      env: TOXENV=py34
-      sudo: required
-      services: docker
-    - python: "3.7"
-      dist: xenial
-      env: TOXENV=py37
-      sudo: required
-      services: docker
-    - sudo: required
-      env: TOXENV=apache_compat
-      services: docker
-      before_install:
-      addons:
-    - sudo: required
-      env: TOXENV=le_auto_trusty
-      services: docker
-      before_install:
-      addons:
-    - python: "2.7"
-      env: TOXENV=apacheconftest
-      sudo: required
-    - python: "2.7"
-      env: TOXENV=nginxroundtrip
-
+  - 'if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ulimit -n 1024 ; fi'
+  # On Travis, the fastest parallelization for integration tests has proved to be 4.
+  - 'if [[ "$TOXENV" == *"integration"* ]]; then export PYTEST_ADDOPTS="--numprocesses 4"; fi'
+  - export TOX_TESTENV_PASSENV=TRAVIS
 
 # Only build pushes to the master branch, PRs, and branches beginning with
 # `test-` or of the form `digit(s).digit(s).x`. This reduces the number of
@@ -69,10 +16,242 @@ matrix:
 # is a cap of on the number of simultaneous runs.
 branches:
   only:
+    # apache-parser-v2 is a temporary branch for doing work related to
+    # rewriting the parser in the Apache plugin.
+    - apache-parser-v2
     - master
     - /^\d+\.\d+\.x$/
     - /^test-.*$/
 
+# Jobs for the main test suite are always executed (including on PRs) except for pushes on master.
+not-on-master: &not-on-master
+  if: NOT (type = push AND branch = master)
+
+# Jobs for the extended test suite are executed for cron jobs and pushes to
+# non-development branches. See the explanation for apache-parser-v2 above.
+extended-test-suite: &extended-test-suite
+  if: type = cron OR (type = push AND branch NOT IN (apache-parser-v2, master))
+
+matrix:
+  include:
+    # Main test suite
+    - python: "2.7"
+      env: ACME_SERVER=pebble TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *not-on-master
+
+    # This job is always executed, including on master
+    - python: "2.7"
+      env: TOXENV=py27-cover FYI="py27 tests + code coverage"
+
+    - python: "2.7"
+      env: TOXENV=lint
+      <<: *not-on-master
+    - python: "3.4"
+      env: TOXENV=mypy
+      <<: *not-on-master
+    - python: "3.5"
+      env: TOXENV=mypy
+      <<: *not-on-master
+    - python: "2.7"
+      # Ubuntu Trusty or older must be used because the oldest version of
+      # cryptography we support cannot be compiled against the version of
+      # OpenSSL in Xenial or newer.
+      dist: trusty
+      env: TOXENV='py27-{acme,apache,certbot,dns,nginx}-oldest'
+      sudo: required
+      services: docker
+      <<: *not-on-master
+    - python: "3.4"
+      env: TOXENV=py34
+      sudo: required
+      services: docker
+      <<: *not-on-master
+    - python: "3.7"
+      dist: xenial
+      env: TOXENV=py37
+      sudo: required
+      services: docker
+      <<: *not-on-master
+    - sudo: required
+      env: TOXENV=apache_compat
+      services: docker
+      before_install:
+      addons:
+      <<: *not-on-master
+    - sudo: required
+      env: TOXENV=le_auto_xenial
+      services: docker
+      <<: *not-on-master
+    - python: "2.7"
+      env: TOXENV=apacheconftest-with-pebble
+      sudo: required
+      services: docker
+      <<: *not-on-master
+    - python: "2.7"
+      env: TOXENV=nginxroundtrip
+      <<: *not-on-master
+
+    # Extended test suite on cron jobs and pushes to tested branches other than master
+    - sudo: required
+      env: TOXENV=nginx_compat
+      services: docker
+      before_install:
+      addons:
+      <<: *extended-test-suite
+    - python: "2.7"
+      env:
+        - TOXENV=travis-test-farm-apache2
+        - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
+      <<: *extended-test-suite
+    - python: "2.7"
+      env:
+        - TOXENV=travis-test-farm-leauto-upgrades
+        - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
+      git:
+        depth: false  # This is needed to have the history to checkout old versions of certbot-auto.
+      <<: *extended-test-suite
+    - python: "2.7"
+      env:
+        - TOXENV=travis-test-farm-certonly-standalone
+        - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
+      <<: *extended-test-suite
+    - python: "2.7"
+      env:
+        - TOXENV=travis-test-farm-sdists
+        - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
+      <<: *extended-test-suite
+    - python: "3.7"
+      dist: xenial
+      env: TOXENV=py37 CERTBOT_NO_PIN=1
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration-certbot-oldest
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration-certbot-oldest
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration-nginx-oldest
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration-nginx-oldest
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "3.4"
+      env: TOXENV=py34
+      <<: *extended-test-suite
+    - python: "3.5"
+      env: TOXENV=py35
+      <<: *extended-test-suite
+    - python: "3.6"
+      env: TOXENV=py36
+      <<: *extended-test-suite
+    - python: "3.7"
+      dist: xenial
+      env: TOXENV=py37
+      <<: *extended-test-suite
+    - python: "3.4"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "3.4"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "3.5"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "3.5"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "3.6"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "3.6"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "3.7"
+      dist: xenial
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "3.7"
+      dist: xenial
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - sudo: required
+      env: TOXENV=le_auto_jessie
+      services: docker
+      <<: *extended-test-suite
+    - sudo: required
+      env: TOXENV=le_auto_centos6
+      services: docker
+      <<: *extended-test-suite
+    - sudo: required
+      env: TOXENV=docker_dev
+      services: docker
+      addons:
+        apt:
+          packages:  # don't install nginx and apache
+            - libaugeas0
+      <<: *extended-test-suite
+    - language: generic
+      env: TOXENV=py27
+      os: osx
+      # Using this osx_image is a workaround for
+      # https://travis-ci.community/t/xcode-8-3-homebrew-outdated-error/3798.
+      osx_image: xcode10.2
+      addons:
+        homebrew:
+          packages:
+            - augeas
+            - python2
+      <<: *extended-test-suite
+    - language: generic
+      env: TOXENV=py3
+      os: osx
+      # Using this osx_image is a workaround for
+      # https://travis-ci.community/t/xcode-8-3-homebrew-outdated-error/3798.
+      osx_image: xcode10.2
+      addons:
+        homebrew:
+          packages:
+            - augeas
+            - python3
+      <<: *extended-test-suite
+
 # container-based infrastructure
 sudo: false
 
@@ -80,7 +259,6 @@ addons:
   apt:
     packages:  # Keep in sync with letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh and Boulder.
     - python-dev
-    - python-virtualenv
     - gcc
     - libaugeas0
     - libssl-dev
@@ -90,19 +268,25 @@ addons:
     - nginx-light
     - openssl
 
-install: "travis_retry $(command -v pip || command -v pip3) install codecov tox"
-script:
-    - travis_retry tox
-    - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)'
+# tools/pip_install.py is used to pin packages to a known working version
+# except in tests where the environment variable CERTBOT_NO_PIN is set.
+# virtualenv is listed here explicitly to make sure it is upgraded when
+# CERTBOT_NO_PIN is set to work around failures we've seen when using an older
+# version of virtualenv.
+install: "tools/pip_install.py -U codecov tox virtualenv"
+script: tox
 
-after_success: '[ "$TOXENV" == "py27-cover" ] && codecov'
+after_success: '[ "$TOXENV" == "py27-cover" ] && codecov -F linux'
 
 notifications:
   email: false
   irc:
     channels:
-      - secure: "SGWZl3ownKx9xKVV2VnGt7DqkTmutJ89oJV9tjKhSs84kLijU6EYdPnllqISpfHMTxXflNZuxtGo0wTDYHXBuZL47w1O32W6nzuXdra5zC+i4sYQwYULUsyfOv9gJX8zWAULiK0Z3r0oho45U+FR5ZN6TPCidi8/eGU+EEPwaAw="
+      # This is set to a secure variable to prevent forks from sending
+      # notifications. This value was created by installing
+      # https://github.com/travis-ci/travis.rb and running
+      # `travis encrypt "chat.freenode.net#certbot-devel"`.
+      - secure: "EWW66E2+KVPZyIPR8ViENZwfcup4Gx3/dlimmAZE0WuLwxDCshBBOd3O8Rf6pBokEoZlXM5eDT6XdyJj8n0DLslgjO62pExdunXpbcMwdY7l1ELxX2/UbnDTE6UnPYa09qVBHNG7156Z6yE0x2lH4M9Ykvp0G0cubjPQHylAwo0="
     on_cancel: never
     on_success: never
     on_failure: always
-    use_notice: true

AUTHORS.md

@@ -5,6 +5,7 @@ Authors
 * Aaron Zuehlke
 * Ada Lovelace
 * [Adam Woodbeck](https://github.com/awoodbeck)
+* [Adrien Ferrand](https://github.com/adferrand)
 * [Aidin Gharibnavaz](https://github.com/aidin36)
 * [AJ ONeal](https://github.com/coolaj86)
 * [Alcaro](https://github.com/Alcaro)
@@ -14,6 +15,7 @@ Authors
 * [Alex Gaynor](https://github.com/alex)
 * [Alex Halderman](https://github.com/jhalderm)
 * [Alex Jordan](https://github.com/strugee)
+* [Alex Zorin](https://github.com/alexzorin)
 * [Amjad Mashaal](https://github.com/TheNavigat)
 * [Andrew Murray](https://github.com/radarhere)
 * [Anselm Levskaya](https://github.com/levskaya)
@@ -75,6 +77,7 @@ Authors
 * [Fabian](https://github.com/faerbit)
 * [Faidon Liambotis](https://github.com/paravoid)
 * [Fan Jiang](https://github.com/tcz001)
+* [Felix Lechner](https://github.com/lechner)
 * [Felix Schwarz](https://github.com/FelixSchwarz)
 * [Felix Yan](https://github.com/felixonmars)
 * [Filip Ochnik](https://github.com/filipochnik)

CHANGELOG.md

@@ -1,12 +1,13 @@
 # Certbot change log
 
-Certbot adheres to [Semantic Versioning](http://semver.org/).
+Certbot adheres to [Semantic Versioning](https://semver.org/).
 
-## 0.30.0 - master
+## 0.37.0 - master
 
 ### Added
 
-*
+* Turn off session tickets for apache plugin by default
+* acme: Authz deactivation added to `acme` module.
 
 ### Changed
 
@@ -16,12 +17,362 @@ Certbot adheres to [Semantic Versioning](http://semver.org/).
 
 *
 
+More details about these changes can be found on our GitHub repo.
+
+## 0.36.0 - 2019-07-11
+
+### Added
+
+* Turn off session tickets for nginx plugin by default
+* Added missing error types from RFC8555 to acme
+
+### Changed
+
+* Support for Ubuntu 14.04 Trusty has been removed.
+* Update the 'manage your account' help to be more generic.
+* The error message when Certbot's Apache plugin is unable to modify your
+  Apache configuration has been improved.
+* Certbot's config_changes subcommand has been deprecated and will be
+  removed in a future release.
+* `certbot config_changes` no longer accepts a --num parameter.
+* The functions `certbot.plugins.common.Installer.view_config_changes` and
+  `certbot.reverter.Reverter.view_config_changes` have been deprecated and will
+  be removed in a future release.
+
+### Fixed
+
+* Replace some unnecessary platform-specific line separation.
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.35.1 - 2019-06-10
+
+### Fixed
+
+* Support for specifying an authoritative base domain in our dns-rfc2136 plugin
+  has been removed. This feature was added in our last release but had a bug
+  which caused the plugin to fail so the feature has been removed until it can
+  be added properly.
+
 Despite us having broken lockstep, we are continuing to release new versions of
 all Certbot components during releases for the time being, however, the only
 package with changes other than its version number was:
 
+* certbot-dns-rfc2136
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.35.0 - 2019-06-05
+
+### Added
+
+* dns_rfc2136 plugin now supports explicitly specifing an authorative
+  base domain for cases when the automatic method does not work (e.g.
+  Split horizon DNS)
+
+### Changed
+
 *
 
+### Fixed
+
+* Renewal parameter `webroot_path` is always saved, avoiding some regressions
+  when `webroot` authenticator plugin is invoked with no challenge to perform.
+* Certbot now accepts OCSP responses when an explicit authorized
+  responder, different from the issuer, is used to sign OCSP
+  responses.
+* Scripts in Certbot hook directories are no longer executed when their
+  filenames end in a tilde.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+package with changes other than its version number was:
+
+* certbot
+* certbot-dns-rfc2136
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.34.2 - 2019-05-07
+
+### Fixed
+
+* certbot-auto no longer writes a check_permissions.py script at the root
+  of the filesystem.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+changes in this release were to certbot-auto.
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.34.1 - 2019-05-06
+
+### Fixed
+
+* certbot-auto no longer prints a blank line when there are no permissions
+  problems.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+changes in this release were to certbot-auto.
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.34.0 - 2019-05-01
+
+### Changed
+
+* Apache plugin now tries to restart httpd on Fedora using systemctl if a
+  configuration test error is detected. This has to be done due to the way
+  Fedora now generates the self signed certificate files upon first
+  restart.
+* Updated Certbot and its plugins to improve the handling of file system permissions
+  on Windows as a step towards adding proper Windows support to Certbot.
+* Updated urllib3 to 1.24.2 in certbot-auto.
+* Removed the fallback introduced with 0.32.0 in `acme` to retry a challenge response
+  with a `keyAuthorization` if sending the response without this field caused a
+  `malformed` error to be received from the ACME server.
+* Linode DNS plugin now supports api keys created from their new panel
+  at [cloud.linode.com](https://cloud.linode.com)
+
+### Fixed
+
+* Fixed Google DNS Challenge issues when private zones exist
+* Adding a warning noting that future versions of Certbot will automatically configure the
+  webserver so that all requests redirect to secure HTTPS access. You can control this
+  behavior and disable this warning with the --redirect and --no-redirect flags.
+* certbot-auto now prints warnings when run as root with insecure file system
+  permissions. If you see these messages, you should fix the problem by
+  following the instructions at
+  https://community.letsencrypt.org/t/certbot-auto-deployment-best-practices/91979/,
+  however, these warnings can be disabled as necessary with the flag
+  --no-permissions-check.
+* `acme` module uses now a POST-as-GET request to retrieve the registration
+  from an ACME v2 server
+* Convert the tsig algorithm specified in the certbot_dns_rfc2136 configuration file to
+  all uppercase letters before validating. This makes the value in the config case
+  insensitive.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+package with changes other than its version number was:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-dns-cloudflare
+* certbot-dns-cloudxns
+* certbot-dns-digitalocean
+* certbot-dns-dnsimple
+* certbot-dns-dnsmadeeasy
+* certbot-dns-gehirn
+* certbot-dns-google
+* certbot-dns-linode
+* certbot-dns-luadns
+* certbot-dns-nsone
+* certbot-dns-ovh
+* certbot-dns-rfc2136
+* certbot-dns-route53
+* certbot-dns-sakuracloud
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.33.1 - 2019-04-04
+
+### Fixed
+
+* A bug causing certbot-auto to print warnings or crash on some RHEL based
+  systems has been resolved.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+changes in this release were to certbot-auto.
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.33.0 - 2019-04-03
+
+### Added
+
+* Fedora 29+ is now supported by certbot-auto. Since Python 2.x is on a deprecation
+  path in Fedora, certbot-auto will install and use Python 3.x on Fedora 29+.
+* CLI flag `--https-port` has been added for Nginx plugin exclusively, and replaces
+  `--tls-sni-01-port`. It defines the HTTPS port the Nginx plugin will use while
+  setting up a new SSL vhost. By default the HTTPS port is 443.
+
+### Changed
+
+* Support for TLS-SNI-01 has been removed from all official Certbot plugins.
+* Attributes related to the TLS-SNI-01 challenge in `acme.challenges` and `acme.standalone`
+  modules are deprecated and will be removed soon.
+* CLI flags `--tls-sni-01-port` and `--tls-sni-01-address` are now no-op, will
+  generate a deprecation warning if used, and will be removed soon.
+* Options `tls-sni` and `tls-sni-01` in `--preferred-challenges` flag are now no-op,
+  will generate a deprecation warning if used, and will be removed soon.
+* CLI flag `--standalone-supported-challenges` has been removed.
+
+### Fixed
+
+* Certbot uses the Python library cryptography for OCSP when cryptography>=2.5
+  is installed. We fixed a bug in Certbot causing it to interpret timestamps in
+  the OCSP response as being in the local timezone rather than UTC.
+* Issue causing the default CentOS 6 TLS configuration to ignore some of the
+  HTTPS VirtualHosts created by Certbot. mod_ssl loading is now moved to main
+  http.conf for this environment where possible.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+package with changes other than its version number was:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.32.0 - 2019-03-06
+
+### Added
+
+* If possible, Certbot uses built-in support for OCSP from recent cryptography
+  versions instead of the OpenSSL binary: as a consequence Certbot does not need
+  the OpenSSL binary to be installed anymore if cryptography>=2.5 is installed.
+
+### Changed
+
+* Certbot and its acme module now depend on josepy>=1.1.0 to avoid printing the
+  warnings described at https://github.com/certbot/josepy/issues/13.
+* Apache plugin now respects CERTBOT_DOCS environment variable when adding
+  command line defaults.
+* The running of manual plugin hooks is now always included in Certbot's log
+  output.
+* Tests execution for certbot, certbot-apache and certbot-nginx packages now relies on pytest.
+* An ACME CA server may return a "Retry-After" HTTP header on authorization polling, as
+  specified in the ACME protocol, to indicate when the next polling should occur. Certbot now
+  reads this header if set and respect its value.
+* The `acme` module avoids sending the `keyAuthorization` field in the JWS
+  payload when responding to a challenge as the field is not included in the
+  current ACME protocol. To ease the migration path for ACME CA servers,
+  Certbot and its `acme` module will first try the request without the
+  `keyAuthorization` field but will temporarily retry the request with the
+  field included if a `malformed` error is received. This fallback will be
+  removed in version 0.34.0.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+package with changes other than its version number was:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.31.0 - 2019-02-07
+
+### Added
+
+* Avoid reprocessing challenges that are already validated
+  when a certificate is issued.
+* Support for initiating (but not solving end-to-end) TLS-ALPN-01 challenges
+  with the `acme` module.
+
+### Changed
+
+* Certbot's official Docker images are now based on Alpine Linux 3.9 rather
+  than 3.7. The new version comes with OpenSSL 1.1.1.
+* Lexicon-based DNS plugins are now fully compatible with Lexicon 3.x (support
+  on 2.x branch is maintained).
+* Apache plugin now attempts to configure all VirtualHosts matching requested
+  domain name instead of only a single one when answering the HTTP-01 challenge.
+
+### Fixed
+
+* Fixed accessing josepy contents through acme.jose when the full acme.jose
+  path is used.
+* Clarify behavior for deleting certs as part of revocation.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+package with changes other than its version number was:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-dns-cloudxns
+* certbot-dns-dnsimple
+* certbot-dns-dnsmadeeasy
+* certbot-dns-gehirn
+* certbot-dns-linode
+* certbot-dns-luadns
+* certbot-dns-nsone
+* certbot-dns-ovh
+* certbot-dns-sakuracloud
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.30.2 - 2019-01-25
+
+### Fixed
+
+* Update the version of setuptools pinned in certbot-auto to 40.6.3 to
+  solve installation problems on newer OSes.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, this
+release only affects certbot-auto.
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.30.1 - 2019-01-24
+
+### Fixed
+
+* Always download the pinned version of pip in pipstrap to address breakages
+* Rename old,default.conf to old-and-default.conf to address commas in filenames
+  breaking recent versions of pip.
+* Add VIRTUALENV_NO_DOWNLOAD=1 to all calls to virtualenv to address breakages
+  from venv downloading the latest pip
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+package with changes other than its version number was:
+
+* certbot-apache
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.30.0 - 2019-01-02
+
+### Added
+
+* Added the `update_account` subcommand for account management commands.
+
+### Changed
+
+* Copied account management functionality from the `register` subcommand
+  to the `update_account` subcommand.
+* Marked usage `register --update-registration` for deprecation and
+  removal in a future release.
+
+### Fixed
+
+* Older modules in the josepy library can now be accessed through acme.jose
+  like it could in previous versions of acme. This is only done to preserve
+  backwards compatibility and support for doing this with new modules in josepy
+  will not be added. Users of the acme library should switch to using josepy
+  directly if they haven't done so already.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+package with changes other than its version number was:
+
+* acme
+
 More details about these changes can be found on our GitHub repo.
 
 ## 0.29.1 - 2018-12-05

CODE_OF_CONDUCT.md

@@ -0,0 +1 @@
+This project is governed by [EFF's Public Projects Code of Conduct](https://www.eff.org/pages/eppcode).

CONTRIBUTING.md

@@ -33,3 +33,5 @@ started. In particular, we recommend you read these sections
  - [Finding issues to work on](https://certbot.eff.org/docs/contributing.html#find-issues-to-work-on)
  - [Coding style](https://certbot.eff.org/docs/contributing.html#coding-style)
  - [Submitting a pull request](https://certbot.eff.org/docs/contributing.html#submitting-a-pull-request)
+ - [EFF's Public Projects Code of Conduct](https://www.eff.org/pages/eppcode)
+

Dockerfile

@@ -1,27 +0,0 @@
-FROM python:2-alpine3.7
-
-ENTRYPOINT [ "certbot" ]
-EXPOSE 80 443
-VOLUME /etc/letsencrypt /var/lib/letsencrypt
-WORKDIR /opt/certbot
-
-COPY CHANGELOG.md README.rst setup.py src/
-COPY acme src/acme
-COPY certbot src/certbot
-
-RUN apk add --no-cache --virtual .certbot-deps \
-        libffi \
-        libssl1.0 \
-        openssl \
-        ca-certificates \
-        binutils
-RUN apk add --no-cache --virtual .build-deps \
-        gcc \
-        linux-headers \
-        openssl-dev \
-        musl-dev \
-        libffi-dev \
-    && pip install --no-cache-dir \
-        --editable /opt/certbot/src/acme \
-        --editable /opt/certbot/src \
-    && apk del .build-deps

Dockerfile-dev

@@ -1,5 +1,5 @@
 # This Dockerfile builds an image for development.
-FROM ubuntu:xenial
+FROM debian:buster
 
 # Note: this only exposes the port to other docker containers.
 EXPOSE 80 443

Dockerfile-old

@@ -1,75 +0,0 @@
-# https://github.com/letsencrypt/letsencrypt/pull/431#issuecomment-103659297
-# it is more likely developers will already have ubuntu:trusty rather
-# than e.g. debian:jessie and image size differences are negligible
-FROM ubuntu:trusty
-MAINTAINER Jakub Warmuz <jakub@warmuz.org>
-MAINTAINER William Budington <bill@eff.org>
-
-# Note: this only exposes the port to other docker containers. You
-# still have to bind to 443@host at runtime, as per the ACME spec.
-EXPOSE 443
-
-# TODO: make sure --config-dir and --work-dir cannot be changed
-# through the CLI (certbot-docker wrapper that uses standalone
-# authenticator and text mode only?)
-VOLUME /etc/letsencrypt /var/lib/letsencrypt
-
-WORKDIR /opt/certbot
-
-# no need to mkdir anything:
-# https://docs.docker.com/reference/builder/#copy
-# If <dest> doesn't exist, it is created along with all missing
-# directories in its path.
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-COPY letsencrypt-auto-source/letsencrypt-auto /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto
-RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/* \
-           /tmp/* \
-           /var/tmp/*
-
-# the above is not likely to change, so by putting it further up the
-# Dockerfile we make sure we cache as much as possible
-
-
-COPY setup.py README.rst CHANGELOG.md MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/
-
-# all above files are necessary for setup.py and venv setup, however,
-# package source code directory has to be copied separately to a
-# subdirectory...
-# https://docs.docker.com/reference/builder/#copy: "If <src> is a
-# directory, the entire contents of the directory are copied,
-# including filesystem metadata. Note: The directory itself is not
-# copied, just its contents." Order again matters, three files are far
-# more likely to be cached than the whole project directory
-
-COPY certbot /opt/certbot/src/certbot/
-COPY acme /opt/certbot/src/acme/
-COPY certbot-apache /opt/certbot/src/certbot-apache/
-COPY certbot-nginx /opt/certbot/src/certbot-nginx/
-
-
-RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv
-
-# PATH is set now so pipstrap upgrades the correct (v)env
-ENV PATH /opt/certbot/venv/bin:$PATH
-RUN /opt/certbot/venv/bin/python /opt/certbot/src/pipstrap.py && \
-    /opt/certbot/venv/bin/pip install \
-    -e /opt/certbot/src/acme \
-    -e /opt/certbot/src \
-    -e /opt/certbot/src/certbot-apache \
-    -e /opt/certbot/src/certbot-nginx
-
-# install in editable mode (-e) to save space: it's not possible to
-# "rm -rf /opt/certbot/src" (it's stays in the underlaying image);
-# this might also help in debugging: you can "docker run --entrypoint
-# bash" and investigate, apply patches, etc.
-
-# set up certbot/letsencrypt wrapper to warn people about Dockerfile changes
-COPY tools/docker-warning.sh /opt/certbot/bin/certbot
-RUN ln -s /opt/certbot/bin/certbot /opt/certbot/bin/letsencrypt
-ENV PATH /opt/certbot/bin:$PATH
-
-ENTRYPOINT [ "certbot" ]

README.rst

@@ -28,45 +28,19 @@ Contributing
 If you'd like to contribute to this project please read `Developer Guide
 <https://certbot.eff.org/docs/contributing.html>`_.
 
+This project is governed by `EFF's Public Projects Code of Conduct <https://www.eff.org/pages/eppcode>`_.
+
 .. _installation:
 
-Installation
-------------
-
-The easiest way to install Certbot is by visiting `certbot.eff.org`_, where you can
-find the correct installation instructions for many web server and OS combinations.
-For more information, see `Get Certbot <https://certbot.eff.org/docs/install.html>`_.
-
-.. _certbot.eff.org: https://certbot.eff.org/
-
 How to run the client
 ---------------------
 
-In many cases, you can just run ``certbot-auto`` or ``certbot``, and the
-client will guide you through the process of obtaining and installing certs
-interactively.
-
-For full command line help, you can type::
-
-  ./certbot-auto --help all
-
-
-You can also tell it exactly what you want it to do from the command line.
-For instance, if you want to obtain a cert for ``example.com``,
-``www.example.com``, and ``other.example.net``, using the Apache plugin to both
-obtain and install the certs, you could do this::
-
-  ./certbot-auto --apache -d example.com -d www.example.com -d other.example.net
-
-(The first time you run the command, it will make an account, and ask for an
-email and agreement to the Let's Encrypt Subscriber Agreement; you can
-automate those with ``--email`` and ``--agree-tos``)
-
-If you want to use a webserver that doesn't have full plugin support yet, you
-can still use "standalone" or "webroot" plugins to obtain a certificate::
-
-  ./certbot-auto certonly --standalone --email admin@example.com -d example.com -d www.example.com -d other.example.net
+The easiest way to install and run Certbot is by visiting `certbot.eff.org`_,
+where you can find the correct instructions for many web server and OS
+combinations.  For more information, see `Get Certbot
+<https://certbot.eff.org/docs/install.html>`_.
 
+.. _certbot.eff.org: https://certbot.eff.org/
 
 Understanding the client in more depth
 --------------------------------------
@@ -99,8 +73,8 @@ ACME working area in github: https://github.com/ietf-wg-acme/acme
 
 |build-status| |coverage| |docs| |container|
 
-.. |build-status| image:: https://travis-ci.org/certbot/certbot.svg?branch=master
-   :target: https://travis-ci.org/certbot/certbot
+.. |build-status| image:: https://travis-ci.com/certbot/certbot.svg?branch=master
+   :target: https://travis-ci.com/certbot/certbot
    :alt: Travis CI status
 
 .. |coverage| image:: https://codecov.io/gh/certbot/certbot/branch/master/graph/badge.svg

acme/acme/__init__.py

@@ -1,12 +1,50 @@
 """ACME protocol implementation.
 
-This module is an implementation of the `ACME protocol`_. Latest
-supported version: `draft-ietf-acme-01`_.
-
+This module is an implementation of the `ACME protocol`_.
 
 .. _`ACME protocol`: https://ietf-wg-acme.github.io/acme
 
-.. _`draft-ietf-acme-01`:
-  https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01
-
 """
+import sys
+import warnings
+
+# This code exists to keep backwards compatibility with people using acme.jose
+# before it became the standalone josepy package.
+#
+# It is based on
+# https://github.com/requests/requests/blob/1278ecdf71a312dc2268f3bfc0aabfab3c006dcf/requests/packages.py
+
+import josepy as jose
+
+for mod in list(sys.modules):
+    # This traversal is apparently necessary such that the identities are
+    # preserved (acme.jose.* is josepy.*)
+    if mod == 'josepy' or mod.startswith('josepy.'):
+        sys.modules['acme.' + mod.replace('josepy', 'jose', 1)] = sys.modules[mod]
+
+
+# This class takes a similar approach to the cryptography project to deprecate attributes
+# in public modules. See the _ModuleWithDeprecation class here:
+# https://github.com/pyca/cryptography/blob/91105952739442a74582d3e62b3d2111365b0dc7/src/cryptography/utils.py#L129
+class _TLSSNI01DeprecationModule(object):
+    """
+    Internal class delegating to a module, and displaying warnings when
+    attributes related to TLS-SNI-01 are accessed.
+    """
+    def __init__(self, module):
+        self.__dict__['_module'] = module
+
+    def __getattr__(self, attr):
+        if 'TLSSNI01' in attr:
+            warnings.warn('{0} attribute is deprecated, and will be removed soon.'.format(attr),
+                          DeprecationWarning, stacklevel=2)
+        return getattr(self._module, attr)
+
+    def __setattr__(self, attr, value):  # pragma: no cover
+        setattr(self._module, attr, value)
+
+    def __delattr__(self, attr):  # pragma: no cover
+        delattr(self._module, attr)
+
+    def __dir__(self):  # pragma: no cover
+        return ['_module'] + dir(self._module)

acme/acme/challenges.py

@@ -4,7 +4,7 @@ import functools
 import hashlib
 import logging
 import socket
-import warnings
+import sys
 
 from cryptography.hazmat.primitives import hashes  # type: ignore
 import josepy as jose
@@ -15,15 +15,13 @@ import six
 from acme import errors
 from acme import crypto_util
 from acme import fields
+from acme import _TLSSNI01DeprecationModule
 
 logger = logging.getLogger(__name__)
 
 
-# pylint: disable=too-few-public-methods
-
-
 class Challenge(jose.TypedJSONObjectWithFields):
-    # _fields_to_partial_json | pylint: disable=abstract-method
+    # _fields_to_partial_json
     """ACME challenge."""
     TYPES = {}  # type: dict
 
@@ -37,7 +35,7 @@ class Challenge(jose.TypedJSONObjectWithFields):
 
 
 class ChallengeResponse(jose.TypedJSONObjectWithFields):
-    # _fields_to_partial_json | pylint: disable=abstract-method
+    # _fields_to_partial_json
     """ACME challenge response."""
     TYPES = {}  # type: dict
     resource_type = 'challenge'
@@ -96,6 +94,7 @@ class _TokenChallenge(Challenge):
         """
         # TODO: check that path combined with uri does not go above
         # URI_ROOT_PATH!
+        # pylint: disable=unsupported-membership-test
         return b'..' not in self.token and b'/' not in self.token
 
 
@@ -140,10 +139,14 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse):
 
         return True
 
+    def to_partial_json(self):
+        jobj = super(KeyAuthorizationChallengeResponse, self).to_partial_json()
+        jobj.pop('keyAuthorization', None)
+        return jobj
+
 
 @six.add_metaclass(abc.ABCMeta)
 class KeyAuthorizationChallenge(_TokenChallenge):
-    # pylint: disable=abstract-class-little-used,too-many-ancestors
     """Challenge based on Key Authorization.
 
     :param response_cls: Subclass of `KeyAuthorizationChallengeResponse`
@@ -175,7 +178,7 @@ class KeyAuthorizationChallenge(_TokenChallenge):
         :rtype: KeyAuthorizationChallengeResponse
 
         """
-        return self.response_cls(
+        return self.response_cls(  # pylint: disable=not-callable
             key_authorization=self.key_authorization(account_key))
 
     @abc.abstractmethod
@@ -212,7 +215,7 @@ class DNS01Response(KeyAuthorizationChallengeResponse):
     """ACME dns-01 challenge response."""
     typ = "dns-01"
 
-    def simple_verify(self, chall, domain, account_public_key):
+    def simple_verify(self, chall, domain, account_public_key):  # pylint: disable=unused-argument
         """Simple verify.
 
         This method no longer checks DNS records and is a simple wrapper
@@ -228,7 +231,6 @@ class DNS01Response(KeyAuthorizationChallengeResponse):
         :rtype: bool
 
         """
-        # pylint: disable=unused-argument
         verified = self.verify(chall, account_public_key)
         if not verified:
             logger.debug("Verification of key authorization in response failed")
@@ -433,7 +435,6 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse):
         kwargs.setdefault("port", self.PORT)
         kwargs["name"] = self.z_domain
         # TODO: try different methods?
-        # pylint: disable=protected-access
         return crypto_util.probe_sni(**kwargs)
 
     def verify_cert(self, cert):
@@ -494,11 +495,6 @@ class TLSSNI01(KeyAuthorizationChallenge):
     # boulder#962, ietf-wg-acme#22
     #n = jose.Field("n", encoder=int, decoder=int)
 
-    def __init__(self, *args, **kwargs):
-        warnings.warn("TLS-SNI-01 is deprecated, and will stop working soon.",
-            DeprecationWarning, stacklevel=2)
-        super(TLSSNI01, self).__init__(*args, **kwargs)
-
     def validation(self, account_key, **kwargs):
         """Generate validation.
 
@@ -513,6 +509,17 @@ class TLSSNI01(KeyAuthorizationChallenge):
         return self.response(account_key).gen_cert(key=kwargs.get('cert_key'))
 
 
+@ChallengeResponse.register
+class TLSALPN01Response(KeyAuthorizationChallengeResponse):
+    """ACME TLS-ALPN-01 challenge response.
+
+    This class only allows initiating a TLS-ALPN-01 challenge returned from the
+    CA. Full support for responding to TLS-ALPN-01 challenges by generating and
+    serving the expected response certificate is not currently provided.
+    """
+    typ = "tls-alpn-01"
+
+
 @Challenge.register  # pylint: disable=too-many-ancestors
 class TLSALPN01(KeyAuthorizationChallenge):
     """ACME tls-alpn-01 challenge.
@@ -522,13 +529,14 @@ class TLSALPN01(KeyAuthorizationChallenge):
 
     """
     typ = "tls-alpn-01"
+    response_cls = TLSALPN01Response
 
     def validation(self, account_key, **kwargs):
         """Generate validation for the challenge."""
         raise NotImplementedError()
 
 
-@Challenge.register  # pylint: disable=too-many-ancestors
+@Challenge.register
 class DNS(_TokenChallenge):
     """ACME "dns" challenge."""
     typ = "dns"
@@ -609,3 +617,7 @@ class DNSResponse(ChallengeResponse):
 
         """
         return chall.check_validation(self.validation, account_public_key)
+
+
+# Patching ourselves to warn about TLS-SNI challenge deprecation and removal.
+sys.modules[__name__] = _TLSSNI01DeprecationModule(sys.modules[__name__])

acme/acme/challenges_test.py

@@ -1,13 +1,12 @@
 """Tests for acme.challenges."""
 import unittest
-import warnings
 
 import josepy as jose
 import mock
 import OpenSSL
 import requests
 
-from six.moves.urllib import parse as urllib_parse  # pylint: disable=import-error
+from six.moves.urllib import parse as urllib_parse  # pylint: disable=relative-import
 
 from acme import errors
 from acme import test_util
@@ -94,7 +93,8 @@ class DNS01ResponseTest(unittest.TestCase):
         self.response = self.chall.response(KEY)
 
     def test_to_partial_json(self):
-        self.assertEqual(self.jmsg, self.msg.to_partial_json())
+        self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'},
+                         self.msg.to_partial_json())
 
     def test_from_json(self):
         from acme.challenges import DNS01Response
@@ -165,7 +165,8 @@ class HTTP01ResponseTest(unittest.TestCase):
         self.response = self.chall.response(KEY)
 
     def test_to_partial_json(self):
-        self.assertEqual(self.jmsg, self.msg.to_partial_json())
+        self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'},
+                         self.msg.to_partial_json())
 
     def test_from_json(self):
         from acme.challenges import HTTP01Response
@@ -285,7 +286,8 @@ class TLSSNI01ResponseTest(unittest.TestCase):
         self.assertEqual(self.z_domain, self.response.z_domain)
 
     def test_to_partial_json(self):
-        self.assertEqual(self.jmsg, self.response.to_partial_json())
+        self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'},
+                         self.response.to_partial_json())
 
     def test_from_json(self):
         from acme.challenges import TLSSNI01Response
@@ -365,25 +367,16 @@ class TLSSNI01Test(unittest.TestCase):
             'type': 'tls-sni-01',
             'token': 'a82d5ff8ef740d12881f6d3c2277ab2e',
         }
-
-    def _msg(self):
         from acme.challenges import TLSSNI01
-        with warnings.catch_warnings(record=True) as warn:
-            warnings.simplefilter("always")
-            msg = TLSSNI01(
-                token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e'))
-            assert warn is not None # using a raw assert for mypy
-            self.assertTrue(len(warn) == 1)
-            self.assertTrue(issubclass(warn[-1].category, DeprecationWarning))
-            self.assertTrue('deprecated' in str(warn[-1].message))
-        return msg
+        self.msg = TLSSNI01(
+            token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e'))
 
     def test_to_partial_json(self):
-        self.assertEqual(self.jmsg, self._msg().to_partial_json())
+        self.assertEqual(self.jmsg, self.msg.to_partial_json())
 
     def test_from_json(self):
         from acme.challenges import TLSSNI01
-        self.assertEqual(self._msg(), TLSSNI01.from_json(self.jmsg))
+        self.assertEqual(self.msg, TLSSNI01.from_json(self.jmsg))
 
     def test_from_json_hashable(self):
         from acme.challenges import TLSSNI01
@@ -398,10 +391,46 @@ class TLSSNI01Test(unittest.TestCase):
     @mock.patch('acme.challenges.TLSSNI01Response.gen_cert')
     def test_validation(self, mock_gen_cert):
         mock_gen_cert.return_value = ('cert', 'key')
-        self.assertEqual(('cert', 'key'), self._msg().validation(
+        self.assertEqual(('cert', 'key'), self.msg.validation(
             KEY, cert_key=mock.sentinel.cert_key))
         mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key)
 
+    def test_deprecation_message(self):
+        with mock.patch('acme.warnings.warn') as mock_warn:
+            from acme.challenges import TLSSNI01
+            assert TLSSNI01
+        self.assertEqual(mock_warn.call_count, 1)
+        self.assertTrue('deprecated' in mock_warn.call_args[0][0])
+
+
+class TLSALPN01ResponseTest(unittest.TestCase):
+    # pylint: disable=too-many-instance-attributes
+
+    def setUp(self):
+        from acme.challenges import TLSALPN01Response
+        self.msg = TLSALPN01Response(key_authorization=u'foo')
+        self.jmsg = {
+            'resource': 'challenge',
+            'type': 'tls-alpn-01',
+            'keyAuthorization': u'foo',
+        }
+
+        from acme.challenges import TLSALPN01
+        self.chall = TLSALPN01(token=(b'x' * 16))
+        self.response = self.chall.response(KEY)
+
+    def test_to_partial_json(self):
+        self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'},
+                         self.msg.to_partial_json())
+
+    def test_from_json(self):
+        from acme.challenges import TLSALPN01Response
+        self.assertEqual(self.msg, TLSALPN01Response.from_json(self.jmsg))
+
+    def test_from_json_hashable(self):
+        from acme.challenges import TLSALPN01Response
+        hash(TLSALPN01Response.from_json(self.jmsg))
+
 
 class TLSALPN01Test(unittest.TestCase):
 

acme/acme/client.py

@@ -6,16 +6,16 @@ from email.utils import parsedate_tz
 import heapq
 import logging
 import time
+import re
+import sys
 
 import six
 from six.moves import http_client  # pylint: disable=import-error
 import josepy as jose
 import OpenSSL
-import re
-from requests_toolbelt.adapters.source import SourceAddressAdapter
 import requests
 from requests.adapters import HTTPAdapter
-import sys
+from requests_toolbelt.adapters.source import SourceAddressAdapter
 
 from acme import crypto_util
 from acme import errors
@@ -123,14 +123,20 @@ class ClientBase(object):  # pylint: disable=too-many-instance-attributes
         """
         return self.update_registration(regr, update={'status': 'deactivated'})
 
-    def query_registration(self, regr):
-        """Query server about registration.
+    def deactivate_authorization(self, authzr):
+        # type: (messages.AuthorizationResource) -> messages.AuthorizationResource
+        """Deactivate authorization.
 
-        :param messages.RegistrationResource: Existing Registration
-            Resource.
+        :param messages.AuthorizationResource authzr: The Authorization resource
+            to be deactivated.
+
+        :returns: The Authorization resource that was deactivated.
+        :rtype: `.AuthorizationResource`
 
         """
-        return self._send_recv_regr(regr, messages.UpdateRegistration())
+        body = messages.UpdateAuthorization(status='deactivated')
+        response = self._post(authzr.uri, body)
+        return self._authzr_from_response(response)
 
     def _authzr_from_response(self, response, identifier=None, uri=None):
         authzr = messages.AuthorizationResource(
@@ -276,6 +282,15 @@ class Client(ClientBase):
         # pylint: disable=no-member
         return self._regr_from_response(response)
 
+    def query_registration(self, regr):
+        """Query server about registration.
+
+        :param messages.RegistrationResource: Existing Registration
+            Resource.
+
+        """
+        return self._send_recv_regr(regr, messages.UpdateRegistration())
+
     def agree_to_tos(self, regr):
         """Agree to the terms-of-service.
 
@@ -603,10 +618,13 @@ class ClientV2(ClientBase):
             Resource.
 
         """
-        self.net.account = regr
-        updated_regr = super(ClientV2, self).query_registration(regr)
-        self.net.account = updated_regr
-        return updated_regr
+        self.net.account = regr  # See certbot/certbot#6258
+        # ACME v2 requires to use a POST-as-GET request (POST an empty JWS) here.
+        # This is done by passing None instead of an empty UpdateRegistration to _post().
+        response = self._post(regr.uri, None)
+        self.net.account = self._regr_from_response(response, uri=regr.uri,
+                                                    terms_of_service=regr.terms_of_service)
+        return self.net.account
 
     def update_registration(self, regr, update=None):
         """Update registration.
@@ -652,7 +670,7 @@ class ClientV2(ClientBase):
         response = self._post(self.directory['newOrder'], order)
         body = messages.Order.from_json(response.json())
         authorizations = []
-        for url in body.authorizations:
+        for url in body.authorizations:  # pylint: disable=not-an-iterable
             authorizations.append(self._authzr_from_response(self._post_as_get(url), uri=url))
         return messages.OrderResource(
             body=body,
@@ -714,7 +732,7 @@ class ClientV2(ClientBase):
                 for chall in authzr.body.challenges:
                     if chall.error != None:
                         failed.append(authzr)
-        if len(failed) > 0:
+        if failed:
             raise errors.ValidationError(failed)
         return orderr.update(authorizations=responses)
 
@@ -739,8 +757,7 @@ class ClientV2(ClientBase):
             if body.error is not None:
                 raise errors.IssuanceError(body.error)
             if body.certificate is not None:
-                certificate_response = self._post_as_get(body.certificate,
-                                                    content_type=DER_CONTENT_TYPE).text
+                certificate_response = self._post_as_get(body.certificate).text
                 return orderr.update(body=body, fullchain_pem=certificate_response)
         raise errors.TimeoutError()
 
@@ -759,10 +776,7 @@ class ClientV2(ClientBase):
 
     def external_account_required(self):
         """Checks if ACME server requires External Account Binding authentication."""
-        if hasattr(self.directory, 'meta') and self.directory.meta.external_account_required:
-            return True
-        else:
-            return False
+        return hasattr(self.directory, 'meta') and self.directory.meta.external_account_required
 
     def _post_as_get(self, *args, **kwargs):
         """
@@ -778,11 +792,11 @@ class ClientV2(ClientBase):
             # We add an empty payload for POST-as-GET requests
             new_args = args[:1] + (None,) + args[1:]
             try:
-                return self._post(*new_args, **kwargs)  # pylint: disable=star-args
+                return self._post(*new_args, **kwargs)
             except messages.Error as error:
                 if error.code == 'malformed':
                     logger.debug('Error during a POST-as-GET request, '
-                                 'your ACME CA may not support it:\n%s', error)
+                                 'your ACME CA server may not support it:\n%s', error)
                     logger.debug('Retrying request with GET.')
                 else:  # pragma: no cover
                     raise
@@ -866,8 +880,7 @@ class BackwardsCompatibleClientV2(object):
             for domain in dnsNames:
                 authorizations.append(self.client.request_domain_challenges(domain))
             return messages.OrderResource(authorizations=authorizations, csr_pem=csr_pem)
-        else:
-            return self.client.new_order(csr_pem)
+        return self.client.new_order(csr_pem)
 
     def finalize_order(self, orderr, deadline):
         """Finalize an order and obtain a certificate.
@@ -904,8 +917,7 @@ class BackwardsCompatibleClientV2(object):
             chain = crypto_util.dump_pyopenssl_chain(chain).decode()
 
             return orderr.update(fullchain_pem=(cert + chain))
-        else:
-            return self.client.finalize_order(orderr, deadline)
+        return self.client.finalize_order(orderr, deadline)
 
     def revoke(self, cert, rsn):
         """Revoke certificate.
@@ -923,8 +935,7 @@ class BackwardsCompatibleClientV2(object):
     def _acme_version_from_directory(self, directory):
         if hasattr(directory, 'newNonce'):
             return 2
-        else:
-            return 1
+        return 1
 
     def external_account_required(self):
         """Checks if the server requires an external account for ACMEv2 servers.
@@ -932,8 +943,7 @@ class BackwardsCompatibleClientV2(object):
         Always return False for ACMEv1 servers, as it doesn't use External Account Binding."""
         if self.acme_version == 1:
             return False
-        else:
-            return self.client.external_account_required()
+        return self.client.external_account_required()
 
 
 class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
@@ -1011,7 +1021,6 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
             if self.account is not None:
                 kwargs["kid"] = self.account["uri"]
         kwargs["key"] = self.key
-        # pylint: disable=star-args
         return jws.JWS.sign(jobj, **kwargs).json_dumps(indent=2)
 
     @classmethod
@@ -1192,10 +1201,7 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
 
     def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE,
             acme_version=1, **kwargs):
-        try:
-            new_nonce_url = kwargs.pop('new_nonce_url')
-        except KeyError:
-            new_nonce_url = None
+        new_nonce_url = kwargs.pop('new_nonce_url', None)
         data = self._wrap_in_jws(obj, self._get_nonce(url, new_nonce_url), url, acme_version)
         kwargs.setdefault('headers', {'Content-Type': content_type})
         response = self._send_request('POST', url, data=data, **kwargs)

acme/acme/client_test.py

@@ -64,7 +64,7 @@ class ClientTestBase(unittest.TestCase):
         reg = messages.Registration(
             contact=self.contact, key=KEY.public_key())
         the_arg = dict(reg) # type: Dict
-        self.new_reg = messages.NewRegistration(**the_arg) # pylint: disable=star-args
+        self.new_reg = messages.NewRegistration(**the_arg)
         self.regr = messages.RegistrationResource(
             body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1')
 
@@ -358,7 +358,6 @@ class ClientTest(ClientTestBase):
 
     def test_register(self):
         # "Instance of 'Field' has no to_json/update member" bug:
-        # pylint: disable=no-member
         self.response.status_code = http_client.CREATED
         self.response.json.return_value = self.regr.body.to_json()
         self.response.headers['Location'] = self.regr.uri
@@ -371,7 +370,6 @@ class ClientTest(ClientTestBase):
 
     def test_update_registration(self):
         # "Instance of 'Field' has no to_json/update member" bug:
-        # pylint: disable=no-member
         self.response.headers['Location'] = self.regr.uri
         self.response.json.return_value = self.regr.body.to_json()
         self.assertEqual(self.regr, self.client.update_registration(self.regr))
@@ -639,6 +637,14 @@ class ClientTest(ClientTestBase):
             errors.PollError, self.client.poll_and_request_issuance,
             csr, authzrs, mintime=mintime, max_attempts=2)
 
+    def test_deactivate_authorization(self):
+        authzb = self.authzr.body.update(status=messages.STATUS_DEACTIVATED)
+        self.response.json.return_value = authzb.to_json()
+        authzr = self.client.deactivate_authorization(self.authzr)
+        self.assertEqual(authzb, authzr.body)
+        self.assertEqual(self.client.net.post.call_count, 1)
+        self.assertTrue(self.authzr.uri in self.net.post.call_args_list[0][0])
+
     def test_check_cert(self):
         self.response.headers['Location'] = self.certr.uri
         self.response.content = CERT_DER
@@ -707,6 +713,7 @@ class ClientTest(ClientTestBase):
             self.certr,
             self.rsn)
 
+
 class ClientV2Test(ClientTestBase):
     """Tests for acme.client.ClientV2."""
 
@@ -843,7 +850,6 @@ class ClientV2Test(ClientTestBase):
 
     def test_update_registration(self):
         # "Instance of 'Field' has no to_json/update member" bug:
-        # pylint: disable=no-member
         self.response.headers['Location'] = self.regr.uri
         self.response.json.return_value = self.regr.body.to_json()
         self.assertEqual(self.regr, self.client.update_registration(self.regr))
@@ -905,7 +911,7 @@ class MockJSONDeSerializable(jose.JSONDeSerializable):
         return {'foo': self.value}
 
     @classmethod
-    def from_json(cls, value):
+    def from_json(cls, jobj):
         pass  # pragma: no cover
 
 
@@ -950,7 +956,6 @@ class ClientNetworkTest(unittest.TestCase):
         self.assertEqual(jws.signature.combined.kid, u'acct-uri')
         self.assertEqual(jws.signature.combined.url, u'url')
 
-
     def test_check_response_not_ok_jobj_no_error(self):
         self.response.ok = False
         self.response.json.return_value = {}
@@ -1113,8 +1118,8 @@ class ClientNetworkTest(unittest.TestCase):
 
         # Requests Library Exceptions
         except requests.exceptions.ConnectionError as z: #pragma: no cover
-            self.assertTrue("('Connection aborted.', error(111, 'Connection refused'))"
-                              == str(z) or "[WinError 10061]" in str(z))
+            self.assertTrue("'Connection aborted.'" in str(z) or "[WinError 10061]" in str(z))
+
 
 class ClientNetworkWithMockedResponseTest(unittest.TestCase):
     """Tests for acme.client.ClientNetwork which mock out response."""

acme/acme/crypto_util.py

@@ -18,17 +18,14 @@ from acme.magic_typing import Callable, Union, Tuple, Optional
 
 logger = logging.getLogger(__name__)
 
-# TLSSNI01 certificate serving and probing is not affected by SSL
-# vulnerabilities: prober needs to check certificate for expected
-# contents anyway. Working SNI is the only thing that's necessary for
-# the challenge and thus scoping down SSL/TLS method (version) would
-# cause interoperability issues: TLSv1_METHOD is only compatible with
+# Default SSL method selected here is the most compatible, while secure
+# SSL method: TLSv1_METHOD is only compatible with
 # TLSv1_METHOD, while SSLv23_METHOD is compatible with all other
 # methods, including TLSv2_METHOD (read more at
 # https://www.openssl.org/docs/ssl/SSLv23_method.html). _serve_sni
 # should be changed to use "set_options" to disable SSLv2 and SSLv3,
 # in case it's used for things other than probing/serving!
-_DEFAULT_TLSSNI01_SSL_METHOD = SSL.SSLv23_METHOD  # type: ignore
+_DEFAULT_SSL_METHOD = SSL.SSLv23_METHOD  # type: ignore
 
 
 class SSLSocket(object):  # pylint: disable=too-few-public-methods
@@ -40,7 +37,7 @@ class SSLSocket(object):  # pylint: disable=too-few-public-methods
     :ivar method: See `OpenSSL.SSL.Context` for allowed values.
 
     """
-    def __init__(self, sock, certs, method=_DEFAULT_TLSSNI01_SSL_METHOD):
+    def __init__(self, sock, certs, method=_DEFAULT_SSL_METHOD):
         self.sock = sock
         self.certs = certs
         self.method = method
@@ -112,7 +109,7 @@ class SSLSocket(object):  # pylint: disable=too-few-public-methods
 
 
 def probe_sni(name, host, port=443, timeout=300,
-              method=_DEFAULT_TLSSNI01_SSL_METHOD, source_address=('', 0)):
+              method=_DEFAULT_SSL_METHOD, source_address=('', 0)):
     """Probe SNI server for SSL certificate.
 
     :param bytes name: Byte string to send as the server name in the
@@ -137,7 +134,6 @@ def probe_sni(name, host, port=443, timeout=300,
     socket_kwargs = {'source_address': source_address}
 
     try:
-        # pylint: disable=star-args
         logger.debug(
             "Attempting to connect to %s:%d%s.", host, port,
             " from {0}:{1}".format(
@@ -198,8 +194,7 @@ def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req):
 
     if common_name is None:
         return sans
-    else:
-        return [common_name] + [d for d in sans if d != common_name]
+    return [common_name] + [d for d in sans if d != common_name]
 
 def _pyopenssl_cert_or_req_san(cert_or_req):
     """Get Subject Alternative Names from certificate or CSR using pyOpenSSL.

acme/acme/jose_test.py

@@ -0,0 +1,53 @@
+"""Tests for acme.jose shim."""
+import importlib
+import unittest
+
+class JoseTest(unittest.TestCase):
+    """Tests for acme.jose shim."""
+
+    def _test_it(self, submodule, attribute):
+        if submodule:
+            acme_jose_path = 'acme.jose.' + submodule
+            josepy_path = 'josepy.' + submodule
+        else:
+            acme_jose_path = 'acme.jose'
+            josepy_path = 'josepy'
+        acme_jose_mod = importlib.import_module(acme_jose_path)
+        josepy_mod = importlib.import_module(josepy_path)
+
+        self.assertIs(acme_jose_mod, josepy_mod)
+        self.assertIs(getattr(acme_jose_mod, attribute), getattr(josepy_mod, attribute))
+
+        # We use the imports below with eval, but pylint doesn't
+        # understand that.
+        # pylint: disable=eval-used,unused-variable
+        import acme
+        import josepy
+        acme_jose_mod = eval(acme_jose_path)
+        josepy_mod = eval(josepy_path)
+        self.assertIs(acme_jose_mod, josepy_mod)
+        self.assertIs(getattr(acme_jose_mod, attribute), getattr(josepy_mod, attribute))
+
+    def test_top_level(self):
+        self._test_it('', 'RS512')
+
+    def test_submodules(self):
+        # This test ensures that the modules in josepy that were
+        # available at the time it was moved into its own package are
+        # available under acme.jose. Backwards compatibility with new
+        # modules or testing code is not maintained.
+        mods_and_attrs = [('b64', 'b64decode',),
+                          ('errors', 'Error',),
+                          ('interfaces', 'JSONDeSerializable',),
+                          ('json_util', 'Field',),
+                          ('jwa', 'HS256',),
+                          ('jwk', 'JWK',),
+                          ('jws', 'JWS',),
+                          ('util', 'ImmutableMap',),]
+
+        for mod, attr in mods_and_attrs:
+            self._test_it(mod, attr)
+
+
+if __name__ == '__main__':
+    unittest.main()  # pragma: no cover

acme/acme/messages.py

@@ -1,7 +1,10 @@
 """ACME protocol messages."""
-import collections
-import six
 import json
+import six
+try:
+    from collections.abc import Hashable  # pylint: disable=no-name-in-module
+except ImportError:  # pragma: no cover
+    from collections import Hashable
 
 import josepy as jose
 
@@ -15,20 +18,35 @@ OLD_ERROR_PREFIX = "urn:acme:error:"
 ERROR_PREFIX = "urn:ietf:params:acme:error:"
 
 ERROR_CODES = {
+    'accountDoesNotExist': 'The request specified an account that does not exist',
+    'alreadyRevoked': 'The request specified a certificate to be revoked that has' \
+    ' already been revoked',
     'badCSR': 'The CSR is unacceptable (e.g., due to a short key)',
     'badNonce': 'The client sent an unacceptable anti-replay nonce',
+    'badPublicKey': 'The JWS was signed by a public key the server does not support',
+    'badRevocationReason': 'The revocation reason provided is not allowed by the server',
+    'badSignatureAlgorithm': 'The JWS was signed with an algorithm the server does not support',
+    'caa': 'Certification Authority Authorization (CAA) records forbid the CA from issuing' \
+    ' a certificate',
+    'compound': 'Specific error conditions are indicated in the "subproblems" array',
     'connection': ('The server could not connect to the client to verify the'
                    ' domain'),
+    'dns': 'There was a problem with a DNS query during identifier validation',
     'dnssec': 'The server could not validate a DNSSEC signed domain',
+    'incorrectResponse': 'Response recieved didn\'t match the challenge\'s requirements',
     # deprecate invalidEmail
     'invalidEmail': 'The provided email for a registration was invalid',
     'invalidContact': 'The provided contact URI was invalid',
     'malformed': 'The request message was malformed',
+    'rejectedIdentifier': 'The server will not issue certificates for the identifier',
+    'orderNotReady': 'The request attempted to finalize an order that is not ready to be finalized',
     'rateLimited': 'There were too many requests of a given type',
     'serverInternal': 'The server experienced an internal error',
     'tls': 'The server experienced a TLS error during domain verification',
     'unauthorized': 'The client lacks sufficient authorization',
+    'unsupportedContact': 'A contact URL for an account used an unsupported protocol scheme',
     'unknownHost': 'The server could not resolve a domain name',
+    'unsupportedIdentifier': 'An identifier is of an unsupported type',
     'externalAccountRequired': 'The server requires external account binding',
 }
 
@@ -43,8 +61,7 @@ def is_acme_error(err):
     """Check if argument is an ACME error."""
     if isinstance(err, Error) and (err.typ is not None):
         return (ERROR_PREFIX in err.typ) or (OLD_ERROR_PREFIX in err.typ)
-    else:
-        return False
+    return False
 
 
 @six.python_2_unicode_compatible
@@ -99,6 +116,7 @@ class Error(jose.JSONObjectWithFields, errors.Error):
         code = str(self.typ).split(':')[-1]
         if code in ERROR_CODES:
             return code
+        return None
 
     def __str__(self):
         return b' :: '.join(
@@ -107,24 +125,25 @@ class Error(jose.JSONObjectWithFields, errors.Error):
             if part is not None).decode()
 
 
-class _Constant(jose.JSONDeSerializable, collections.Hashable):  # type: ignore
+class _Constant(jose.JSONDeSerializable, Hashable):  # type: ignore
     """ACME constant."""
     __slots__ = ('name',)
     POSSIBLE_NAMES = NotImplemented
 
     def __init__(self, name):
-        self.POSSIBLE_NAMES[name] = self
+        super(_Constant, self).__init__()
+        self.POSSIBLE_NAMES[name] = self  # pylint: disable=unsupported-assignment-operation
         self.name = name
 
     def to_partial_json(self):
         return self.name
 
     @classmethod
-    def from_json(cls, value):
-        if value not in cls.POSSIBLE_NAMES:
+    def from_json(cls, jobj):
+        if jobj not in cls.POSSIBLE_NAMES:  # pylint: disable=unsupported-membership-test
             raise jose.DeserializationError(
                 '{0} not recognized'.format(cls.__name__))
-        return cls.POSSIBLE_NAMES[value]
+        return cls.POSSIBLE_NAMES[jobj]  # pylint: disable=unsubscriptable-object
 
     def __repr__(self):
         return '{0}({1})'.format(self.__class__.__name__, self.name)
@@ -149,6 +168,7 @@ STATUS_VALID = Status('valid')
 STATUS_INVALID = Status('invalid')
 STATUS_REVOKED = Status('revoked')
 STATUS_READY = Status('ready')
+STATUS_DEACTIVATED = Status('deactivated')
 
 
 class IdentifierType(_Constant):
@@ -183,7 +203,6 @@ class Directory(jose.JSONDeSerializable):
 
         def __init__(self, **kwargs):
             kwargs = dict((self._internal_name(k), v) for k, v in kwargs.items())
-            # pylint: disable=star-args
             super(Directory.Meta, self).__init__(**kwargs)
 
         @property
@@ -319,7 +338,7 @@ class Registration(ResourceBody):
 
     def _filter_contact(self, prefix):
         return tuple(
-            detail[len(prefix):] for detail in self.contact
+            detail[len(prefix):] for detail in self.contact  # pylint: disable=not-an-iterable
             if detail.startswith(prefix))
 
     @property
@@ -391,7 +410,6 @@ class ChallengeBody(ResourceBody):
 
     def __init__(self, **kwargs):
         kwargs = dict((self._internal_name(k), v) for k, v in kwargs.items())
-        # pylint: disable=star-args
         super(ChallengeBody, self).__init__(**kwargs)
 
     def encode(self, name):
@@ -454,7 +472,7 @@ class Authorization(ResourceBody):
     :ivar datetime.datetime expires:
 
     """
-    identifier = jose.Field('identifier', decoder=Identifier.from_json)
+    identifier = jose.Field('identifier', decoder=Identifier.from_json, omitempty=True)
     challenges = jose.Field('challenges', omitempty=True)
     combinations = jose.Field('combinations', omitempty=True)
 
@@ -474,7 +492,7 @@ class Authorization(ResourceBody):
     def resolved_combinations(self):
         """Combinations with challenges instead of indices."""
         return tuple(tuple(self.challenges[idx] for idx in combo)
-                     for combo in self.combinations)
+                     for combo in self.combinations)  # pylint: disable=not-an-iterable
 
 
 @Directory.register
@@ -484,6 +502,12 @@ class NewAuthorization(Authorization):
     resource = fields.Resource(resource_type)
 
 
+class UpdateAuthorization(Authorization):
+    """Update authorization."""
+    resource_type = 'authz'
+    resource = fields.Resource(resource_type)
+
+
 class AuthorizationResource(ResourceWithURI):
     """Authorization Resource.
 

acme/acme/standalone.py

@@ -17,6 +17,7 @@ import OpenSSL
 from acme import challenges
 from acme import crypto_util
 from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module
+from acme import _TLSSNI01DeprecationModule
 
 
 logger = logging.getLogger(__name__)
@@ -37,7 +38,7 @@ class TLSServer(socketserver.TCPServer):
         self.certs = kwargs.pop("certs", {})
         self.method = kwargs.pop(
             # pylint: disable=protected-access
-            "method", crypto_util._DEFAULT_TLSSNI01_SSL_METHOD)
+            "method", crypto_util._DEFAULT_SSL_METHOD)
         self.allow_reuse_address = kwargs.pop("allow_reuse_address", True)
         socketserver.TCPServer.__init__(self, *args, **kwargs)
 
@@ -82,7 +83,7 @@ class BaseDualNetworkedServers(object):
                 kwargs["ipv6"] = ip_version
                 new_address = (server_address[0],) + (port,) + server_address[2:]
                 new_args = (new_address,) + remaining_args
-                server = ServerClass(*new_args, **kwargs) # pylint: disable=star-args
+                server = ServerClass(*new_args, **kwargs)
                 logger.debug(
                     "Successfully bound to %s:%s using %s", new_address[0],
                     new_address[1], "IPv6" if ip_version else "IPv4")
@@ -90,8 +91,8 @@ class BaseDualNetworkedServers(object):
                 if self.servers:
                     # Already bound using IPv6.
                     logger.debug(
-                        "Certbot wasn't able to bind to %s:%s using %s, this " +
-                        "is often expected due to the dual stack nature of " +
+                        "Certbot wasn't able to bind to %s:%s using %s, this "
+                        "is often expected due to the dual stack nature of "
                         "IPv6 socket implementations.",
                         new_address[0], new_address[1],
                         "IPv6" if ip_version else "IPv4")
@@ -104,7 +105,7 @@ class BaseDualNetworkedServers(object):
                 # If two servers are set up and port 0 was passed in, ensure we always
                 # bind to the same port for both servers.
                 port = server.socket.getsockname()[1]
-        if len(self.servers) == 0:
+        if not self.servers:
             raise socket.error("Could not bind to IPv4 or IPv6.")
 
     def serve_forever(self):
@@ -296,5 +297,9 @@ def simple_tls_sni_01_server(cli_args, forever=True):
         server.handle_request()
 
 
+# Patching ourselves to warn about TLS-SNI challenge deprecation and removal.
+sys.modules[__name__] = _TLSSNI01DeprecationModule(sys.modules[__name__])
+
+
 if __name__ == "__main__":
     sys.exit(simple_tls_sni_01_server(sys.argv))  # pragma: no cover

acme/acme/standalone_test.py

@@ -1,13 +1,15 @@
 """Tests for acme.standalone."""
+import multiprocessing
 import os
 import shutil
 import socket
 import threading
 import tempfile
 import unittest
+import time
+from contextlib import closing
 
 from six.moves import http_client  # pylint: disable=import-error
-from six.moves import queue  # pylint: disable=import-error
 from six.moves import socketserver  # type: ignore  # pylint: disable=import-error
 
 import josepy as jose
@@ -16,6 +18,7 @@ import requests
 
 from acme import challenges
 from acme import crypto_util
+from acme import errors
 from acme import test_util
 from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module
 
@@ -28,14 +31,14 @@ class TLSServerTest(unittest.TestCase):
         from acme.standalone import TLSServer
         server = TLSServer(
             ('', 0), socketserver.BaseRequestHandler, bind_and_activate=True)
-        server.server_close()  # pylint: disable=no-member
+        server.server_close()
 
     def test_ipv6(self):
         if socket.has_ipv6:
             from acme.standalone import TLSServer
             server = TLSServer(
                 ('', 0), socketserver.BaseRequestHandler, bind_and_activate=True, ipv6=True)
-            server.server_close()  # pylint: disable=no-member
+            server.server_close()
 
 
 class TLSSNI01ServerTest(unittest.TestCase):
@@ -49,12 +52,11 @@ class TLSSNI01ServerTest(unittest.TestCase):
         )}
         from acme.standalone import TLSSNI01Server
         self.server = TLSSNI01Server(('localhost', 0), certs=self.certs)
-        # pylint: disable=no-member
         self.thread = threading.Thread(target=self.server.serve_forever)
         self.thread.start()
 
     def tearDown(self):
-        self.server.shutdown()  # pylint: disable=no-member
+        self.server.shutdown()
         self.thread.join()
 
     def test_it(self):
@@ -77,13 +79,12 @@ class HTTP01ServerTest(unittest.TestCase):
         from acme.standalone import HTTP01Server
         self.server = HTTP01Server(('', 0), resources=self.resources)
 
-        # pylint: disable=no-member
         self.port = self.server.socket.getsockname()[1]
         self.thread = threading.Thread(target=self.server.serve_forever)
         self.thread.start()
 
     def tearDown(self):
-        self.server.shutdown()  # pylint: disable=no-member
+        self.server.shutdown()
         self.thread.join()
 
     def test_index(self):
@@ -136,7 +137,6 @@ class BaseDualNetworkedServersTest(unittest.TestCase):
                 # NB: On Windows, socket.IPPROTO_IPV6 constant may be missing.
                 # We use the corresponding value (41) instead.
                 level = getattr(socket, "IPPROTO_IPV6", 41)
-                # pylint: disable=no-member
                 self.socket.setsockopt(level, socket.IPV6_V6ONLY, 1)
                 try:
                     self.server_bind()
@@ -209,7 +209,6 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase):
         from acme.standalone import HTTP01DualNetworkedServers
         self.servers = HTTP01DualNetworkedServers(('', 0), resources=self.resources)
 
-        # pylint: disable=no-member
         self.port = self.servers.getsocknames()[0][1]
         self.servers.serve_forever()
 
@@ -248,7 +247,6 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase):
         self.assertFalse(self._test_http01(add=False))
 
 
-@test_util.broken_on_windows
 class TestSimpleTLSSNI01Server(unittest.TestCase):
     """Tests for acme.standalone.simple_tls_sni_01_server."""
 
@@ -263,35 +261,45 @@ class TestSimpleTLSSNI01Server(unittest.TestCase):
         shutil.copy(test_util.vector_path('rsa2048_key.pem'),
                     os.path.join(localhost_dir, 'key.pem'))
 
+        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
+            sock.bind(('', 0))
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            self.port = sock.getsockname()[1]
+
         from acme.standalone import simple_tls_sni_01_server
-        self.thread = threading.Thread(
-            target=simple_tls_sni_01_server, kwargs={
-                'cli_args': ('filename',),
-                'forever': False,
-            },
-        )
+        self.process = multiprocessing.Process(target=simple_tls_sni_01_server,
+                                               args=(['path', '-p', str(self.port)],))
         self.old_cwd = os.getcwd()
         os.chdir(self.test_cwd)
 
     def tearDown(self):
         os.chdir(self.old_cwd)
-        self.thread.join()
+        if self.process.is_alive():
+            self.process.terminate()
+            self.process.join(timeout=5)
+            # Check that we didn't timeout waiting for the process to
+            # terminate.
+            self.assertNotEqual(self.process.exitcode, None)
         shutil.rmtree(self.test_cwd)
 
-    @mock.patch('acme.standalone.logger')
-    def test_it(self, mock_logger):
-        # Use a Queue because mock objects aren't thread safe.
-        q = queue.Queue()  # type: queue.Queue[int]
-        # Add port number to the queue.
-        mock_logger.info.side_effect = lambda *args: q.put(args[-1])
-        self.thread.start()
+    @mock.patch('acme.standalone.TLSSNI01Server.handle_request')
+    def test_mock(self, handle):
+        from acme.standalone import simple_tls_sni_01_server
+        simple_tls_sni_01_server(cli_args=['path', '-p', str(self.port)], forever=False)
+        self.assertEqual(handle.call_count, 1)
 
-        # After the timeout, an exception is raised if the queue is empty.
-        port = q.get(timeout=5)
-        cert = crypto_util.probe_sni(b'localhost', b'0.0.0.0', port)
+    def test_live(self):
+        self.process.start()
+        cert = None
+        for _ in range(50):
+            time.sleep(0.1)
+            try:
+                cert = crypto_util.probe_sni(b'localhost', b'127.0.0.1', self.port)
+                break
+            except errors.Error:  # pragma: no cover
+                pass
         self.assertEqual(jose.ComparableX509(cert),
-                         test_util.load_comparable_cert(
-                             'rsa2048_cert.pem'))
+                         test_util.load_comparable_cert('rsa2048_cert.pem'))
 
 
 if __name__ == "__main__":

acme/acme/test_util.py

@@ -4,9 +4,8 @@
 
 """
 import os
-import sys
-import pkg_resources
 import unittest
+import pkg_resources
 
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
@@ -93,13 +92,4 @@ def skip_unless(condition, reason):  # pragma: no cover
         return unittest.skipUnless(condition, reason)
     elif condition:
         return lambda cls: cls
-    else:
-        return lambda cls: None
-
-def broken_on_windows(function):
-    """Decorator to skip temporarily a broken test on Windows."""
-    reason = 'Test is broken and ignored on windows but should be fixed.'
-    return unittest.skipIf(
-        sys.platform == 'win32'
-        and os.environ.get('SKIP_BROKEN_TESTS_ON_WINDOWS', 'true') == 'true',
-        reason)(function)
+    return lambda cls: None

acme/docs/index.rst

@@ -16,13 +16,6 @@ Contents:
 .. automodule:: acme
    :members:
 
-
-Example client:
-
-.. include:: ../examples/example_client.py
-   :code: python
-
-
 Indices and tables
 ==================
 

acme/examples/example_client.py

@@ -1,47 +0,0 @@
-"""Example script showing how to use acme client API."""
-import logging
-import os
-import pkg_resources
-
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives.asymmetric import rsa
-import josepy as jose
-import OpenSSL
-
-from acme import client
-from acme import messages
-
-
-logging.basicConfig(level=logging.DEBUG)
-
-
-DIRECTORY_URL = 'https://acme-staging.api.letsencrypt.org/directory'
-BITS = 2048  # minimum for Boulder
-DOMAIN = 'example1.com'  # example.com is ignored by Boulder
-
-# generate_private_key requires cryptography>=0.5
-key = jose.JWKRSA(key=rsa.generate_private_key(
-    public_exponent=65537,
-    key_size=BITS,
-    backend=default_backend()))
-acme = client.Client(DIRECTORY_URL, key)
-
-regr = acme.register()
-logging.info('Auto-accepting TOS: %s', regr.terms_of_service)
-acme.agree_to_tos(regr)
-logging.debug(regr)
-
-authzr = acme.request_challenges(
-    identifier=messages.Identifier(typ=messages.IDENTIFIER_FQDN, value=DOMAIN))
-logging.debug(authzr)
-
-authzr, authzr_response = acme.poll(authzr)
-
-csr = OpenSSL.crypto.load_certificate_request(
-    OpenSSL.crypto.FILETYPE_ASN1, pkg_resources.resource_string(
-        'acme', os.path.join('testdata', 'csr.der')))
-try:
-    acme.request_issuance(jose.util.ComparableX509(csr), (authzr,))
-except messages.Error as error:
-    print ("This script is doomed to fail as no authorization "
-           "challenges are ever solved. Error from server: {0}".format(error))

acme/examples/http01_example.py

@@ -0,0 +1,240 @@
+"""Example ACME-V2 API for HTTP-01 challenge.
+
+Brief:
+
+This a complete usage example of the python-acme API.
+
+Limitations of this example:
+    - Works for only one Domain name
+    - Performs only HTTP-01 challenge
+    - Uses ACME-v2
+
+Workflow:
+    (Account creation)
+    - Create account key
+    - Register account and accept TOS
+    (Certificate actions)
+    - Select HTTP-01 within offered challenges by the CA server
+    - Set up http challenge resource
+    - Set up standalone web server
+    - Create domain private key and CSR
+    - Issue certificate
+    - Renew certificate
+    - Revoke certificate
+    (Account update actions)
+    - Change contact information
+    - Deactivate Account
+"""
+from contextlib import contextmanager
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.asymmetric import rsa
+import OpenSSL
+
+from acme import challenges
+from acme import client
+from acme import crypto_util
+from acme import errors
+from acme import messages
+from acme import standalone
+import josepy as jose
+
+# Constants:
+
+# This is the staging point for ACME-V2 within Let's Encrypt.
+DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory'
+
+USER_AGENT = 'python-acme-example'
+
+# Account key size
+ACC_KEY_BITS = 2048
+
+# Certificate private key size
+CERT_PKEY_BITS = 2048
+
+# Domain name for the certificate.
+DOMAIN = 'client.example.com'
+
+# If you are running Boulder locally, it is possible to configure any port
+# number to execute the challenge, but real CA servers will always use port
+# 80, as described in the ACME specification.
+PORT = 80
+
+
+# Useful methods and classes:
+
+
+def new_csr_comp(domain_name, pkey_pem=None):
+    """Create certificate signing request."""
+    if pkey_pem is None:
+        # Create private key.
+        pkey = OpenSSL.crypto.PKey()
+        pkey.generate_key(OpenSSL.crypto.TYPE_RSA, CERT_PKEY_BITS)
+        pkey_pem = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM,
+                                                  pkey)
+    csr_pem = crypto_util.make_csr(pkey_pem, [domain_name])
+    return pkey_pem, csr_pem
+
+
+def select_http01_chall(orderr):
+    """Extract authorization resource from within order resource."""
+    # Authorization Resource: authz.
+    # This object holds the offered challenges by the server and their status.
+    authz_list = orderr.authorizations
+
+    for authz in authz_list:
+        # Choosing challenge.
+        # authz.body.challenges is a set of ChallengeBody objects.
+        for i in authz.body.challenges:
+            # Find the supported challenge.
+            if isinstance(i.chall, challenges.HTTP01):
+                return i
+
+    raise Exception('HTTP-01 challenge was not offered by the CA server.')
+
+
+@contextmanager
+def challenge_server(http_01_resources):
+    """Manage standalone server set up and shutdown."""
+
+    # Setting up a fake server that binds at PORT and any address.
+    address = ('', PORT)
+    try:
+        servers = standalone.HTTP01DualNetworkedServers(address,
+                                                        http_01_resources)
+        # Start client standalone web server.
+        servers.serve_forever()
+        yield servers
+    finally:
+        # Shutdown client web server and unbind from PORT
+        servers.shutdown_and_server_close()
+
+
+def perform_http01(client_acme, challb, orderr):
+    """Set up standalone webserver and perform HTTP-01 challenge."""
+
+    response, validation = challb.response_and_validation(client_acme.net.key)
+
+    resource = standalone.HTTP01RequestHandler.HTTP01Resource(
+        chall=challb.chall, response=response, validation=validation)
+
+    with challenge_server({resource}):
+        # Let the CA server know that we are ready for the challenge.
+        client_acme.answer_challenge(challb, response)
+
+        # Wait for challenge status and then issue a certificate.
+        # It is possible to set a deadline time.
+        finalized_orderr = client_acme.poll_and_finalize(orderr)
+
+    return finalized_orderr.fullchain_pem
+
+
+# Main examples:
+
+
+def example_http():
+    """This example executes the whole process of fulfilling a HTTP-01
+    challenge for one specific domain.
+
+    The workflow consists of:
+    (Account creation)
+    - Create account key
+    - Register account and accept TOS
+    (Certificate actions)
+    - Select HTTP-01 within offered challenges by the CA server
+    - Set up http challenge resource
+    - Set up standalone web server
+    - Create domain private key and CSR
+    - Issue certificate
+    - Renew certificate
+    - Revoke certificate
+    (Account update actions)
+    - Change contact information
+    - Deactivate Account
+
+    """
+    # Create account key
+
+    acc_key = jose.JWKRSA(
+        key=rsa.generate_private_key(public_exponent=65537,
+                                     key_size=ACC_KEY_BITS,
+                                     backend=default_backend()))
+
+    # Register account and accept TOS
+
+    net = client.ClientNetwork(acc_key, user_agent=USER_AGENT)
+    directory = messages.Directory.from_json(net.get(DIRECTORY_URL).json())
+    client_acme = client.ClientV2(directory, net=net)
+
+    # Terms of Service URL is in client_acme.directory.meta.terms_of_service
+    # Registration Resource: regr
+    # Creates account with contact information.
+    email = ('fake@example.com')
+    regr = client_acme.new_account(
+        messages.NewRegistration.from_data(
+            email=email, terms_of_service_agreed=True))
+
+    # Create domain private key and CSR
+    pkey_pem, csr_pem = new_csr_comp(DOMAIN)
+
+    # Issue certificate
+
+    orderr = client_acme.new_order(csr_pem)
+
+    # Select HTTP-01 within offered challenges by the CA server
+    challb = select_http01_chall(orderr)
+
+    # The certificate is ready to be used in the variable "fullchain_pem".
+    fullchain_pem = perform_http01(client_acme, challb, orderr)
+
+    # Renew certificate
+
+    _, csr_pem = new_csr_comp(DOMAIN, pkey_pem)
+
+    orderr = client_acme.new_order(csr_pem)
+
+    challb = select_http01_chall(orderr)
+
+    # Performing challenge
+    fullchain_pem = perform_http01(client_acme, challb, orderr)
+
+    # Revoke certificate
+
+    fullchain_com = jose.ComparableX509(
+        OpenSSL.crypto.load_certificate(
+            OpenSSL.crypto.FILETYPE_PEM, fullchain_pem))
+
+    try:
+        client_acme.revoke(fullchain_com, 0)  # revocation reason = 0
+    except errors.ConflictError:
+        # Certificate already revoked.
+        pass
+
+    # Query registration status.
+    client_acme.net.account = regr
+    try:
+        regr = client_acme.query_registration(regr)
+    except errors.Error as err:
+        if err.typ == messages.OLD_ERROR_PREFIX + 'unauthorized' \
+                or err.typ == messages.ERROR_PREFIX + 'unauthorized':
+            # Status is deactivated.
+            pass
+        raise
+
+    # Change contact information
+
+    email = 'newfake@example.com'
+    regr = client_acme.update_registration(
+        regr.update(
+            body=regr.body.update(
+                contact=('mailto:' + email,)
+            )
+        )
+    )
+
+    # Deactivate account/registration
+
+    regr = client_acme.deactivate_registration(regr)
+
+
+if __name__ == "__main__":
+    example_http()

acme/setup.py

@@ -3,21 +3,23 @@ from setuptools import find_packages
 from setuptools.command.test import test as TestCommand
 import sys
 
-version = '0.30.0.dev0'
+version = '0.37.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
     # load_pem_private/public_key (>=0.6)
     # rsa_recover_prime_factors (>=0.8)
-    'cryptography>=0.8',
+    'cryptography>=1.2.3',
     # formerly known as acme.jose:
-    'josepy>=1.0.0',
+    # 1.1.0+ is required to avoid the warnings described at
+    # https://github.com/certbot/josepy/issues/13.
+    'josepy>=1.1.0',
     # Connection.set_tlsext_host_name (>=0.13)
     'mock',
-    'PyOpenSSL>=0.13',
+    'PyOpenSSL>=0.13.1',
     'pyrfc3339',
     'pytz',
-    'requests[security]>=2.4.1',  # security extras added in 2.4.1
+    'requests[security]>=2.6.0',  # security extras added in 2.4.1
     'requests-toolbelt>=0.3.0',
     'setuptools',
     'six>=1.9.0',  # needed for python_2_unicode_compatible
@@ -34,6 +36,7 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+
 class PyTest(TestCommand):
     user_options = []
 
@@ -48,6 +51,7 @@ class PyTest(TestCommand):
         errno = pytest.main(shlex.split(self.pytest_args))
         sys.exit(errno)
 
+
 setup(
     name='acme',
     version=version,
@@ -80,7 +84,7 @@ setup(
         'dev': dev_extras,
         'docs': docs_extras,
     },
-    tests_require=["pytest"],
     test_suite='acme',
+    tests_require=["pytest"],
     cmdclass={"test": PyTest},
 )

appveyor.yml

@@ -7,10 +7,21 @@ environment:
 
 branches:
   only:
+    # apache-parser-v2 is a temporary branch for doing work related to
+    # rewriting the parser in the Apache plugin.
+    - apache-parser-v2
     - master
     - /^\d+\.\d+\.x$/ # Version branches like X.X.X
     - /^test-.*$/
 
+init:
+  # Since master can receive only commits from PR that have already been tested, following
+  # condition avoid to launch all jobs except the coverage one for commits pushed to master.
+  - ps: |
+      if (-Not $Env:APPVEYOR_PULL_REQUEST_NUMBER -And $Env:APPVEYOR_REPO_BRANCH -Eq 'master' `
+          -And -Not ($Env:TOXENV -Like '*-cover'))
+      { $Env:APPVEYOR_SKIP_FINALIZE_ON_EXIT = 'true'; Exit-AppVeyorBuild }
+
 install:
   # Use Python 3.7 by default
   - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%"
@@ -19,13 +30,15 @@ install:
   # Upgrade pip to avoid warnings
   - "python -m pip install --upgrade pip"
   # Ready to install tox and coverage
-  - "pip install tox codecov"
+  # tools/pip_install.py is used to pin packages to a known working version.
+  - "python tools\\pip_install.py tox codecov"
 
 build: off
 
 test_script:
+  - set TOX_TESTENV_PASSENV=APPVEYOR
   # Test env is set by TOXENV env variable
   - tox
 
 on_success:
-  - if exist .coverage codecov
+  - if exist .coverage codecov -F windows

certbot-apache/MANIFEST.in

@@ -5,3 +5,4 @@ recursive-include certbot_apache/tests/testdata *
 include certbot_apache/centos-options-ssl-apache.conf
 include certbot_apache/options-ssl-apache.conf
 recursive-include certbot_apache/augeas_lens *.aug
+recursive-include certbot_apache/tls_configs *.conf

certbot-apache/certbot_apache/apache_util.py

@@ -1,8 +1,11 @@
 """ Utility functions for certbot-apache plugin """
 import binascii
-import os
+
+import pkg_resources
 
 from certbot import util
+from certbot.compat import os
+
 
 def get_mod_deps(mod_name):
     """Get known module dependencies.
@@ -104,3 +107,15 @@ def parse_define_file(filepath, varname):
 def unique_id():
     """ Returns an unique id to be used as a VirtualHost identifier"""
     return binascii.hexlify(os.urandom(16)).decode("utf-8")
+
+
+def find_ssl_apache_conf(prefix):
+    """
+    Find a TLS Apache config file in the dedicated storage.
+    :param str prefix: prefix of the TLS Apache config file to find
+    :return: the path the TLS Apache config file
+    :rtype: str
+    """
+    return pkg_resources.resource_filename(
+        "certbot_apache",
+        os.path.join("tls_configs", "{0}-options-ssl-apache.conf".format(prefix)))

certbot-apache/certbot_apache/augeas_configurator.py

@@ -1,207 +0,0 @@
-"""Class of Augeas Configurators."""
-import logging
-
-
-from certbot import errors
-from certbot.plugins import common
-
-from certbot_apache import constants
-
-logger = logging.getLogger(__name__)
-
-
-class AugeasConfigurator(common.Installer):
-    """Base Augeas Configurator class.
-
-    :ivar config: Configuration.
-    :type config: :class:`~certbot.interfaces.IConfig`
-
-    :ivar aug: Augeas object
-    :type aug: :class:`augeas.Augeas`
-
-    :ivar str save_notes: Human-readable configuration change notes
-    :ivar reverter: saves and reverts checkpoints
-    :type reverter: :class:`certbot.reverter.Reverter`
-
-    """
-    def __init__(self, *args, **kwargs):
-        super(AugeasConfigurator, self).__init__(*args, **kwargs)
-
-        # Placeholder for augeas
-        self.aug = None
-
-        self.save_notes = ""
-
-
-    def init_augeas(self):
-        """ Initialize the actual Augeas instance """
-        import augeas
-        self.aug = augeas.Augeas(
-            # specify a directory to load our preferred lens from
-            loadpath=constants.AUGEAS_LENS_DIR,
-            # Do not save backup (we do it ourselves), do not load
-            # anything by default
-            flags=(augeas.Augeas.NONE |
-                   augeas.Augeas.NO_MODL_AUTOLOAD |
-                   augeas.Augeas.ENABLE_SPAN))
-        # See if any temporary changes need to be recovered
-        # This needs to occur before VirtualHost objects are setup...
-        # because this will change the underlying configuration and potential
-        # vhosts
-        self.recovery_routine()
-
-    def check_parsing_errors(self, lens):
-        """Verify Augeas can parse all of the lens files.
-
-        :param str lens: lens to check for errors
-
-        :raises .errors.PluginError: If there has been an error in parsing with
-            the specified lens.
-
-        """
-        error_files = self.aug.match("/augeas//error")
-
-        for path in error_files:
-            # Check to see if it was an error resulting from the use of
-            # the httpd lens
-            lens_path = self.aug.get(path + "/lens")
-            # As aug.get may return null
-            if lens_path and lens in lens_path:
-                msg = (
-                    "There has been an error in parsing the file {0} on line {1}: "
-                    "{2}".format(
-                    # Strip off /augeas/files and /error
-                    path[13:len(path) - 6],
-                    self.aug.get(path + "/line"),
-                    self.aug.get(path + "/message")))
-                raise errors.PluginError(msg)
-
-    def ensure_augeas_state(self):
-        """Makes sure that all Augeas dom changes are written to files to avoid
-        loss of configuration directives when doing additional augeas parsing,
-        causing a possible augeas.load() resulting dom reset
-        """
-
-        if self.unsaved_files():
-            self.save_notes += "(autosave)"
-            self.save()
-
-    def unsaved_files(self):
-        """Lists files that have modified Augeas DOM but the changes have not
-        been written to the filesystem yet, used by `self.save()` and
-        ApacheConfigurator to check the file state.
-
-        :raises .errors.PluginError: If there was an error in Augeas, in
-            an attempt to save the configuration, or an error creating a
-            checkpoint
-
-        :returns: `set` of unsaved files
-        """
-        save_state = self.aug.get("/augeas/save")
-        self.aug.set("/augeas/save", "noop")
-        # Existing Errors
-        ex_errs = self.aug.match("/augeas//error")
-        try:
-            # This is a noop save
-            self.aug.save()
-        except (RuntimeError, IOError):
-            self._log_save_errors(ex_errs)
-            # Erase Save Notes
-            self.save_notes = ""
-            raise errors.PluginError(
-                "Error saving files, check logs for more info.")
-
-        # Return the original save method
-        self.aug.set("/augeas/save", save_state)
-
-        # Retrieve list of modified files
-        # Note: Noop saves can cause the file to be listed twice, I used a
-        # set to remove this possibility. This is a known augeas 0.10 error.
-        save_paths = self.aug.match("/augeas/events/saved")
-
-        save_files = set()
-        if save_paths:
-            for path in save_paths:
-                save_files.add(self.aug.get(path)[6:])
-        return save_files
-
-    def save(self, title=None, temporary=False):
-        """Saves all changes to the configuration files.
-
-        This function first checks for save errors, if none are found,
-        all configuration changes made will be saved. According to the
-        function parameters. If an exception is raised, a new checkpoint
-        was not created.
-
-        :param str title: The title of the save. If a title is given, the
-            configuration will be saved as a new checkpoint and put in a
-            timestamped directory.
-
-        :param bool temporary: Indicates whether the changes made will
-            be quickly reversed in the future (ie. challenges)
-
-        """
-        save_files = self.unsaved_files()
-        if save_files:
-            self.add_to_checkpoint(save_files,
-                                   self.save_notes, temporary=temporary)
-
-        self.save_notes = ""
-        self.aug.save()
-
-        # Force reload if files were modified
-        # This is needed to recalculate augeas directive span
-        if save_files:
-            for sf in save_files:
-                self.aug.remove("/files/"+sf)
-            self.aug.load()
-        if title and not temporary:
-            self.finalize_checkpoint(title)
-
-    def _log_save_errors(self, ex_errs):
-        """Log errors due to bad Augeas save.
-
-        :param list ex_errs: Existing errors before save
-
-        """
-        # Check for the root of save problems
-        new_errs = self.aug.match("/augeas//error")
-        # logger.error("During Save - %s", mod_conf)
-        logger.error("Unable to save files: %s. Attempted Save Notes: %s",
-                     ", ".join(err[13:len(err) - 6] for err in new_errs
-                               # Only new errors caused by recent save
-                               if err not in ex_errs), self.save_notes)
-
-    # Wrapper functions for Reverter class
-    def recovery_routine(self):
-        """Revert all previously modified files.
-
-        Reverts all modified files that have not been saved as a checkpoint
-
-        :raises .errors.PluginError: If unable to recover the configuration
-
-        """
-        super(AugeasConfigurator, self).recovery_routine()
-        # Need to reload configuration after these changes take effect
-        self.aug.load()
-
-    def revert_challenge_config(self):
-        """Used to cleanup challenge configurations.
-
-        :raises .errors.PluginError: If unable to revert the challenge config.
-
-        """
-        self.revert_temporary_config()
-        self.aug.load()
-
-    def rollback_checkpoints(self, rollback=1):
-        """Rollback saved checkpoints.
-
-        :param int rollback: Number of checkpoints to revert
-
-        :raises .errors.PluginError: If there is a problem with the input or
-            the function is unable to correctly revert the configuration
-
-        """
-        super(AugeasConfigurator, self).rollback_checkpoints(rollback)
-        self.aug.load()

certbot-apache/certbot_apache/augeasparser.py

@@ -0,0 +1,87 @@
+""" Tests for ParserNode interface """
+from certbot_apache import interfaces
+
+
+class AugeasCommentNode(interfaces.CommentNode):
+    """ Augeas implementation of CommentNode interface """
+    ancestor = None
+    comment = ""
+    dirty = False
+
+    def __init__(self, comment, ancestor=None):
+        self.comment = comment
+        self.ancestor = ancestor
+
+    def save(self, msg):  # pragma: no cover
+        pass
+
+
+class AugeasDirectiveNode(interfaces.DirectiveNode):
+    """ Augeas implementation of DirectiveNode interface """
+    ancestor = None
+    parameters = tuple()  # type: Tuple[str, ...]
+    dirty = False
+    enabled = True
+    name = ""
+
+    def __init__(self, name, parameters=tuple(), ancestor=None):
+        self.name = name
+        self.parameters = parameters
+        self.ancestor = ancestor
+
+    def save(self, msg):  # pragma: no cover
+        pass
+
+    def set_parameters(self, parameters):  # pragma: no cover
+        self.parameters = tuple("CERTBOT_PASS_ASSERT")
+
+
+class AugeasBlockNode(interfaces.BlockNode):
+    """ Augeas implementation of BlockNode interface """
+    ancestor = None
+    parameters = tuple()  # type: Tuple[str, ...]
+    children = tuple()  # type: Tuple[interfaces.ParserNode, ...]
+    dirty = False
+    enabled = True
+    name = ""
+
+    def __init__(self, name, parameters=tuple(), ancestor=None):
+        self.name = name
+        self.parameters = parameters
+        self.ancestor = ancestor
+
+    def save(self, msg):  # pragma: no cover
+        pass
+
+    def add_child_block(self, name, parameters=None, position=None):  # pragma: no cover
+        new_block = AugeasBlockNode("CERTBOT_PASS_ASSERT", ancestor=self)
+        self.children += (new_block,)
+        return new_block
+
+    def add_child_directive(self, name, parameters=None, position=None):  # pragma: no cover
+        new_dir = AugeasDirectiveNode("CERTBOT_PASS_ASSERT", ancestor=self)
+        self.children += (new_dir,)
+        return new_dir
+
+    def add_child_comment(self, comment="", position=None):  # pragma: no cover
+        new_comment = AugeasCommentNode("CERTBOT_PASS_ASSERT", ancestor=self)
+        self.children += (new_comment,)
+        return new_comment
+
+    def find_blocks(self, name, exclude=True):  # pragma: no cover
+        return [AugeasBlockNode("CERTBOT_PASS_ASSERT", ancestor=self)]
+
+    def find_directives(self, name, exclude=True):  # pragma: no cover
+        return [AugeasDirectiveNode("CERTBOT_PASS_ASSERT", ancestor=self)]
+
+    def find_comments(self, comment, exact=False):  # pragma: no cover
+        return [AugeasCommentNode("CERTBOT_PASS_ASSERT", ancestor=self)]
+
+    def delete_child(self, child):  # pragma: no cover
+        pass
+
+    def set_parameters(self, parameters):  # pragma: no cover
+        self.parameters = tuple("CERTBOT_PASS_ASSERT")
+
+    def unsaved_files(self):  # pragma: no cover
+        return ["CERTBOT_PASS_ASSERT"]

certbot-apache/certbot_apache/configurator.py

@@ -1,40 +1,40 @@
-"""Apache Configuration based off of Augeas Configurator."""
+"""Apache Configurator."""
 # pylint: disable=too-many-lines
 import copy
 import fnmatch
 import logging
-import os
-import pkg_resources
 import re
-import six
 import socket
 import time
 
+from collections import defaultdict
+
+import six
+
 import zope.component
 import zope.interface
 
 from acme import challenges
-from acme.magic_typing import Any, DefaultDict, Dict, List, Set, Union  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import DefaultDict, Dict, List, Set, Union  # pylint: disable=unused-import, no-name-in-module
 
 from certbot import errors
 from certbot import interfaces
 from certbot import util
 
 from certbot.achallenges import KeyAuthorizationAnnotatedChallenge  # pylint: disable=unused-import
+from certbot.compat import filesystem
+from certbot.compat import os
 from certbot.plugins import common
 from certbot.plugins.util import path_surgery
 from certbot.plugins.enhancements import AutoHSTSEnhancement
 
 from certbot_apache import apache_util
-from certbot_apache import augeas_configurator
+from certbot_apache import augeasparser
 from certbot_apache import constants
 from certbot_apache import display_ops
 from certbot_apache import http_01
 from certbot_apache import obj
 from certbot_apache import parser
-from certbot_apache import tls_sni_01
-
-from collections import defaultdict
 
 logger = logging.getLogger(__name__)
 
@@ -70,13 +70,10 @@ logger = logging.getLogger(__name__)
 
 @zope.interface.implementer(interfaces.IAuthenticator, interfaces.IInstaller)
 @zope.interface.provider(interfaces.IPluginFactory)
-class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
+class ApacheConfigurator(common.Installer):
     # pylint: disable=too-many-instance-attributes,too-many-public-methods
     """Apache configurator.
 
-    State of Configurator: This code has been been tested and built for Ubuntu
-    14.04 Apache 2.4 and it works for Ubuntu 12.04 Apache 2.2
-
     :ivar config: Configuration.
     :type config: :class:`~certbot.interfaces.IConfig`
 
@@ -92,6 +89,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
     """
 
     description = "Apache Web Server plugin"
+    if os.environ.get("CERTBOT_DOCS") == "1":
+        description += (  # pragma: no cover
+            " (Please note that the default values of the Apache plugin options"
+            " change depending on the operating system Certbot is run on.)"
+        )
 
     OS_DEFAULTS = dict(
         server_root="/etc/apache2",
@@ -108,14 +110,24 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2",
-        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
-            "certbot_apache", "options-ssl-apache.conf")
     )
 
     def option(self, key):
         """Get a value from options"""
         return self.options.get(key)
 
+    def pick_apache_config(self):
+        """
+        Pick the appropriate TLS Apache configuration file for current version of Apache and OS.
+        :return: the path to the TLS Apache configuration file to use
+        :rtype: str
+        """
+        # Disabling TLS session tickets is supported by Apache 2.4.11+.
+        # So for old versions of Apache we pick a configuration without this option.
+        if self.version < (2, 4, 11):
+            return apache_util.find_ssl_apache_conf("old")
+        return apache_util.find_ssl_apache_conf("current")
+
     def _prepare_options(self):
         """
         Set the values possibly changed by command line parameters to
@@ -141,28 +153,36 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         # When adding, modifying or deleting command line arguments, be sure to
         # include the changes in the list used in method _prepare_options() to
         # ensure consistent behavior.
-        add("enmod", default=cls.OS_DEFAULTS["enmod"],
+
+        # Respect CERTBOT_DOCS environment variable and use default values from
+        # base class regardless of the underlying distribution (overrides).
+        if os.environ.get("CERTBOT_DOCS") == "1":
+            DEFAULTS = ApacheConfigurator.OS_DEFAULTS
+        else:
+            # cls.OS_DEFAULTS can be distribution specific, see override classes
+            DEFAULTS = cls.OS_DEFAULTS
+        add("enmod", default=DEFAULTS["enmod"],
             help="Path to the Apache 'a2enmod' binary")
-        add("dismod", default=cls.OS_DEFAULTS["dismod"],
+        add("dismod", default=DEFAULTS["dismod"],
             help="Path to the Apache 'a2dismod' binary")
-        add("le-vhost-ext", default=cls.OS_DEFAULTS["le_vhost_ext"],
+        add("le-vhost-ext", default=DEFAULTS["le_vhost_ext"],
             help="SSL vhost configuration extension")
-        add("server-root", default=cls.OS_DEFAULTS["server_root"],
+        add("server-root", default=DEFAULTS["server_root"],
             help="Apache server root directory")
         add("vhost-root", default=None,
             help="Apache server VirtualHost configuration root")
-        add("logs-root", default=cls.OS_DEFAULTS["logs_root"],
+        add("logs-root", default=DEFAULTS["logs_root"],
             help="Apache server logs directory")
         add("challenge-location",
-            default=cls.OS_DEFAULTS["challenge_location"],
+            default=DEFAULTS["challenge_location"],
             help="Directory path for challenge configuration")
-        add("handle-modules", default=cls.OS_DEFAULTS["handle_modules"],
+        add("handle-modules", default=DEFAULTS["handle_modules"],
             help="Let installer handle enabling required modules for you " +
                  "(Only Ubuntu/Debian currently)")
-        add("handle-sites", default=cls.OS_DEFAULTS["handle_sites"],
+        add("handle-sites", default=DEFAULTS["handle_sites"],
             help="Let installer handle enabling sites for you " +
                  "(Only Ubuntu/Debian currently)")
-        add("ctl", default=cls.OS_DEFAULTS["ctl"],
+        add("ctl", default=DEFAULTS["ctl"],
             help="Full path to Apache control script")
         util.add_deprecated_argument(
             add, argument_name="init-script", nargs=1)
@@ -188,6 +208,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         self._enhanced_vhosts = defaultdict(set)  # type: DefaultDict[str, Set[obj.VirtualHost]]
         # Temporary state for AutoHSTS enhancement
         self._autohsts = {}  # type: Dict[str, Dict[str, Union[int, float]]]
+        # Reverter save notes
+        self.save_notes = ""
 
         # These will be set in the prepare function
         self._prepared = False
@@ -202,15 +224,13 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
     @property
     def mod_ssl_conf(self):
         """Full absolute path to SSL configuration file."""
-        return os.path.join(self.config.config_dir,
-                            constants.MOD_SSL_CONF_DEST)
+        return os.path.join(self.config.config_dir, constants.MOD_SSL_CONF_DEST)
 
     @property
     def updated_mod_ssl_conf_digest(self):
         """Full absolute path to digest of updated SSL configuration file."""
         return os.path.join(self.config.config_dir, constants.UPDATED_MOD_SSL_CONF_DIGEST)
 
-
     def prepare(self):
         """Prepare the authenticator/installer.
 
@@ -220,12 +240,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         :raises .errors.PluginError: If there is any other error
 
         """
-        # Perform the actual Augeas initialization to be able to react
-        try:
-            self.init_augeas()
-        except ImportError:
-            raise errors.NoInstallationError("Problem in Augeas installation")
-
         self._prepare_options()
 
         # Verify Apache is installed
@@ -241,18 +255,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                          '.'.join(str(i) for i in self.version))
         if self.version < (2, 2):
             raise errors.NotSupportedError(
-                "Apache Version %s not supported.", str(self.version))
-
-        if not self._check_aug_version():
-            raise errors.NotSupportedError(
-                "Apache plugin support requires libaugeas0 and augeas-lenses "
-                "version 1.2.0 or higher, please make sure you have you have "
-                "those installed.")
+                "Apache Version {0} not supported.".format(str(self.version)))
 
+        # Recover from previous crash before Augeas initialization to have the
+        # correct parse tree from the get go.
+        self.recovery_routine()
+        # Perform the actual Augeas initialization to be able to react
         self.parser = self.get_parser()
 
         # Check for errors in parsing files with Augeas
-        self.check_parsing_errors("httpd.aug")
+        self.parser.check_parsing_errors("httpd.aug")
 
         # Get all of the available vhosts
         self.vhosts = self.get_virtual_hosts()
@@ -266,9 +278,76 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         except (OSError, errors.LockError):
             logger.debug("Encountered error:", exc_info=True)
             raise errors.PluginError(
-                "Unable to lock %s", self.option("server_root"))
+                "Unable to create a lock file in {0}. Are you running"
+                " Certbot with sufficient privileges to modify your"
+                " Apache configuration?".format(self.option("server_root")))
+
+        # TODO: apache-parser-v2
+        self.parser_root = augeasparser.AugeasBlockNode(None, ancestor=None)
+
         self._prepared = True
 
+    def save(self, title=None, temporary=False):
+        """Saves all changes to the configuration files.
+
+        This function first checks for save errors, if none are found,
+        all configuration changes made will be saved. According to the
+        function parameters. If an exception is raised, a new checkpoint
+        was not created.
+
+        :param str title: The title of the save. If a title is given, the
+            configuration will be saved as a new checkpoint and put in a
+            timestamped directory.
+
+        :param bool temporary: Indicates whether the changes made will
+            be quickly reversed in the future (ie. challenges)
+
+        """
+        save_files = self.parser.unsaved_files()
+        if save_files:
+            self.add_to_checkpoint(save_files,
+                                   self.save_notes, temporary=temporary)
+        # Handle the parser specific tasks
+        self.parser.save(save_files)
+        if title and not temporary:
+            self.finalize_checkpoint(title)
+
+    def recovery_routine(self):
+        """Revert all previously modified files.
+
+        Reverts all modified files that have not been saved as a checkpoint
+
+        :raises .errors.PluginError: If unable to recover the configuration
+
+        """
+        super(ApacheConfigurator, self).recovery_routine()
+        # Reload configuration after these changes take effect if needed
+        # ie. ApacheParser has been initialized.
+        if self.parser:
+            # TODO: wrap into non-implementation specific  parser interface
+            self.parser.aug.load()
+
+    def revert_challenge_config(self):
+        """Used to cleanup challenge configurations.
+
+        :raises .errors.PluginError: If unable to revert the challenge config.
+
+        """
+        self.revert_temporary_config()
+        self.parser.aug.load()
+
+    def rollback_checkpoints(self, rollback=1):
+        """Rollback saved checkpoints.
+
+        :param int rollback: Number of checkpoints to revert
+
+        :raises .errors.PluginError: If there is a problem with the input or
+            the function is unable to correctly revert the configuration
+
+        """
+        super(ApacheConfigurator, self).rollback_checkpoints(rollback)
+        self.parser.aug.load()
+
     def _verify_exe_availability(self, exe):
         """Checks availability of Apache executable"""
         if not util.exe_exists(exe):
@@ -276,26 +355,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 raise errors.NoInstallationError(
                     'Cannot find Apache executable {0}'.format(exe))
 
-    def _check_aug_version(self):
-        """ Checks that we have recent enough version of libaugeas.
-        If augeas version is recent enough, it will support case insensitive
-        regexp matching"""
-
-        self.aug.set("/test/path/testing/arg", "aRgUMeNT")
-        try:
-            matches = self.aug.match(
-                "/test//*[self::arg=~regexp('argument', 'i')]")
-        except RuntimeError:
-            self.aug.remove("/test/path")
-            return False
-        self.aug.remove("/test/path")
-        return matches
-
     def get_parser(self):
         """Initializes the ApacheParser"""
         # If user provided vhost_root value in command line, use it
         return parser.ApacheParser(
-            self.aug, self.option("server_root"), self.conf("vhost-root"),
+            self.option("server_root"), self.conf("vhost-root"),
             self.version, configurator=self)
 
     def _wildcard_domain(self, domain):
@@ -382,7 +446,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         """
         if len(name.split(".")) == len(domain.split(".")):
             return fnmatch.fnmatch(name, domain)
-
+        return None
 
     def _choose_vhosts_wildcard(self, domain, create_ssl=True):
         """Prompts user to choose vhosts to install a wildcard certificate for"""
@@ -428,7 +492,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         self._wildcard_vhosts[domain] = return_vhosts
         return return_vhosts
 
-
     def _deploy_cert(self, vhost, cert_path, key_path, chain_path, fullchain_path):
         """
         Helper function for deploy_cert() that handles the actual deployment
@@ -436,8 +499,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         domain originally passed for deploy_cert(). This is especially true
         with wildcard certificates
         """
-
-
         # This is done first so that ssl module is enabled and cert_path,
         # cert_key... can all be parsed appropriately
         self.prepare_server_https("443")
@@ -451,10 +512,25 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 "cert_key": self.parser.find_dir("SSLCertificateKeyFile",
                                                  None, vhost.path)}
 
+        # TODO: apache-parser-v2
+        # We need to add a BlockNode reference to obj.VirtualHost
+        # v2_path = {"cert_path": vhost.node.find_directives(
+        #    "SSLCertificateFile"),
+        #            "cert_key": vhost.node.find_directives(
+        #    "SSLCertificateKeyFile")}
+        # parserassert.legacy_assert_dir(path["cert_path"], v2_path["cert_path"])
+        # parserassert.legacy_assert_dir(path["cert_key"], v2_path["cert_key"])
+
         # Only include if a certificate chain is specified
         if chain_path is not None:
             path["chain_path"] = self.parser.find_dir(
                 "SSLCertificateChainFile", None, vhost.path)
+            # TODO: apache-parser-v2
+            # v2_path["chain_path"] = vhost.node.find_directives(
+            #   "SSLCertificateChainFile")
+            # parserassert.legacy_assert_dir(path["chain_path"],
+            # v2_path["chain_path])
+
 
         # Handle errors when certificate/key directives cannot be found
         if not path["cert_path"]:
@@ -477,11 +553,17 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             # install SSLCertificateFile, SSLCertificateKeyFile,
             # and SSLCertificateChainFile directives
             set_cert_path = cert_path
-            self.aug.set(path["cert_path"][-1], cert_path)
-            self.aug.set(path["cert_key"][-1], key_path)
+            self.parser.aug.set(path["cert_path"][-1], cert_path)
+            self.parser.aug.set(path["cert_key"][-1], key_path)
+            # TODO: apache-parser-v2
+            # path["cert_path"][-1].set_parameters(cert_path,)
+            # path["cert_key"][-1].set_parameters(key_path,)
             if chain_path is not None:
                 self.parser.add_dir(vhost.path,
                                     "SSLCertificateChainFile", chain_path)
+                # TODO: apache-parser-v2
+                # vhost.node.add_child_directive("SSLCertificateChainFile",
+                #                                (chain_path,))
             else:
                 raise errors.PluginError("--chain-path is required for your "
                                          "version of Apache")
@@ -490,8 +572,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 raise errors.PluginError("Please provide the --fullchain-path "
                                          "option pointing to your full chain file")
             set_cert_path = fullchain_path
-            self.aug.set(path["cert_path"][-1], fullchain_path)
-            self.aug.set(path["cert_key"][-1], key_path)
+            self.parser.aug.set(path["cert_path"][-1], fullchain_path)
+            self.parser.aug.set(path["cert_key"][-1], key_path)
+            # TODO: apache-parser-v2
+            # path["cert_path"][-1].set_parameters(fullchain_path,)
+            # path["cert_key"][-1].set_parameters(key_path,)
 
         # Enable the new vhost if needed
         if not vhost.enabled:
@@ -577,8 +662,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         self.assoc[target_name] = vhost
         return vhost
 
-    def included_in_wildcard(self, names, target_name):
-        """Is target_name covered by a wildcard?
+    def domain_in_names(self, names, target_name):
+        """Checks if target domain is covered by one or more of the provided
+        names. The target name is matched by wildcard as well as exact match.
 
         :param names: server aliases
         :type names: `collections.Iterable` of `str`
@@ -649,7 +735,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             names = vhost.get_names()
             if target_name in names:
                 points = 3
-            elif self.included_in_wildcard(names, target_name):
+            elif self.domain_in_names(names, target_name):
                 points = 2
             elif any(addr.get_addr() == target_name for addr in vhost.addrs):
                 points = 1
@@ -708,7 +794,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                     if name:
                         all_names.add(name)
 
-        if len(vhost_macro) > 0:
+        if vhost_macro:
             zope.component.getUtility(interfaces.IDisplay).notification(
                 "Apache mod_macro seems to be in use in file(s):\n{0}"
                 "\n\nUnfortunately mod_macro is not yet supported".format(
@@ -736,29 +822,48 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         return ""
 
-    def _get_vhost_names(self, path):
+    def _get_vhost_names(self, vhost):
         """Helper method for getting the ServerName and
-        ServerAlias values from vhost in path
+        ServerAlias values from vhost
 
-        :param path: Path to read ServerName and ServerAliases from
+        :param vhost: VirtualHost object to read ServerName and ServerAliases from
 
         :returns: Tuple including ServerName and `list` of ServerAlias strings
         """
 
         servername_match = self.parser.find_dir(
-            "ServerName", None, start=path, exclude=False)
+            "ServerName", None, start=vhost.path, exclude=False)
         serveralias_match = self.parser.find_dir(
-            "ServerAlias", None, start=path, exclude=False)
+            "ServerAlias", None, start=vhost.path, exclude=False)
+        # TODO: apache-parser-v2
+        # v2_servername_match = vhost.node.find_directives("ServerName",
+        #                                                  exclude=False)
+        # v2_serveralias_match = vhost.node.find_directives("ServerAlias",
+        #                                                   exclude=False)
+        # parserassert.legacy_assert_dir(servername_match, v2_servername_match)
+        # parserassert.legacy_assert_dir(serveralias_match, v2_serveralias_match)
+
 
         serveraliases = []
         for alias in serveralias_match:
             serveralias = self.parser.get_arg(alias)
             serveraliases.append(serveralias)
+        # TODO: apache-parser-v2
+        # v2_serveraliases = []
+        # for alias in v2_serveralias_match:
+        #    v2_serveraliases += list(alias.parameters)
+        # parserassert.legacy_assert_list(serveraliases, v2_serveraliases)
+
 
         servername = None
         if servername_match:
             # Get last ServerName as each overwrites the previous
             servername = self.parser.get_arg(servername_match[-1])
+        # TODO: apache-parser-v2
+        # v2_servername = None
+        # if v2_servername_match:
+        #     v2_servername = v2_servername_match[-1].parameters[-1]
+        # parserassert.simpleassert(servername, v2_servername)
 
         return (servername, serveraliases)
 
@@ -770,7 +875,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         """
 
-        servername, serveraliases = self._get_vhost_names(host.path)
+        servername, serveraliases = self._get_vhost_names(host)
 
         for alias in serveraliases:
             if not host.modmacro:
@@ -790,7 +895,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         """
         addrs = set()
         try:
-            args = self.aug.match(path + "/arg")
+            args = self.parser.aug.match(path + "/arg")
         except RuntimeError:
             logger.warning("Encountered a problem while parsing file: %s, skipping", path)
             return None
@@ -808,7 +913,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 is_ssl = True
 
         filename = apache_util.get_file_path(
-            self.aug.get("/augeas/files%s/path" % apache_util.get_file_path(path)))
+            self.parser.aug.get("/augeas/files%s/path" % apache_util.get_file_path(path)))
         if filename is None:
             return None
 
@@ -838,7 +943,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         # Make a list of parser paths because the parser_paths
         # dictionary may be modified during the loop.
         for vhost_path in list(self.parser.parser_paths):
-            paths = self.aug.match(
+            paths = self.parser.aug.match(
                 ("/files%s//*[label()=~regexp('%s')]" %
                     (vhost_path, parser.case_i("VirtualHost"))))
             paths = [path for path in paths if
@@ -848,7 +953,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 if not new_vhost:
                     continue
                 internal_path = apache_util.get_internal_aug_path(new_vhost.path)
-                realpath = os.path.realpath(new_vhost.filep)
+                realpath = filesystem.realpath(new_vhost.filep)
                 if realpath not in file_paths:
                     file_paths[realpath] = new_vhost.filep
                     internal_paths[realpath].add(internal_path)
@@ -1054,6 +1159,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 # Ugly but takes care of protocol def, eg: 1.1.1.1:443 https
                 if listen.split(":")[-1].split(" ")[0] == port:
                     return True
+        return None
 
     def prepare_https_modules(self, temp):
         """Helper method for prepare_server_https, taking care of enabling
@@ -1069,23 +1175,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             if "ssl_module" not in self.parser.modules:
                 self.enable_mod("ssl", temp=temp)
 
-    def make_addrs_sni_ready(self, addrs):
-        """Checks to see if the server is ready for SNI challenges.
-
-        :param addrs: Addresses to check SNI compatibility
-        :type addrs: :class:`~certbot_apache.obj.Addr`
-
-        """
-        # Version 2.4 and later are automatically SNI ready.
-        if self.version >= (2, 4):
-            return
-
-        for addr in addrs:
-            if not self.is_name_vhost(addr):
-                logger.debug("Setting VirtualHost at %s to be a name "
-                             "based virtual host", addr)
-                self.add_name_vhost(addr)
-
     def make_vhost_ssl(self, nonssl_vhost):  # pylint: disable=too-many-locals
         """Makes an ssl_vhost version of a nonssl_vhost.
 
@@ -1108,16 +1197,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         avail_fp = nonssl_vhost.filep
         ssl_fp = self._get_ssl_vhost_path(avail_fp)
 
-        orig_matches = self.aug.match("/files%s//* [label()=~regexp('%s')]" %
+        orig_matches = self.parser.aug.match("/files%s//* [label()=~regexp('%s')]" %
                                       (self._escape(ssl_fp),
                                        parser.case_i("VirtualHost")))
 
         self._copy_create_ssl_vhost_skeleton(nonssl_vhost, ssl_fp)
 
         # Reload augeas to take into account the new vhost
-        self.aug.load()
+        self.parser.aug.load()
         # Get Vhost augeas path for new vhost
-        new_matches = self.aug.match("/files%s//* [label()=~regexp('%s')]" %
+        new_matches = self.parser.aug.match("/files%s//* [label()=~regexp('%s')]" %
                                      (self._escape(ssl_fp),
                                       parser.case_i("VirtualHost")))
 
@@ -1128,7 +1217,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             # Make Augeas aware of the new vhost
             self.parser.parse_file(ssl_fp)
             # Try to search again
-            new_matches = self.aug.match(
+            new_matches = self.parser.aug.match(
                 "/files%s//* [label()=~regexp('%s')]" %
                 (self._escape(ssl_fp),
                  parser.case_i("VirtualHost")))
@@ -1190,16 +1279,15 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         """
 
         if self.conf("vhost-root") and os.path.exists(self.conf("vhost-root")):
-            fp = os.path.join(os.path.realpath(self.option("vhost_root")),
+            fp = os.path.join(filesystem.realpath(self.option("vhost_root")),
                               os.path.basename(non_ssl_vh_fp))
         else:
             # Use non-ssl filepath
-            fp = os.path.realpath(non_ssl_vh_fp)
+            fp = filesystem.realpath(non_ssl_vh_fp)
 
         if fp.endswith(".conf"):
             return fp[:-(len(".conf"))] + self.option("le_vhost_ext")
-        else:
-            return fp + self.option("le_vhost_ext")
+        return fp + self.option("le_vhost_ext")
 
     def _sift_rewrite_rule(self, line):
         """Decides whether a line should be copied to a SSL vhost.
@@ -1279,8 +1367,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 "vhost for your HTTPS site located at {1} because they have "
                 "the potential to create redirection loops.".format(
                     vhost.filep, ssl_fp), reporter.MEDIUM_PRIORITY)
-        self.aug.set("/augeas/files%s/mtime" % (self._escape(ssl_fp)), "0")
-        self.aug.set("/augeas/files%s/mtime" % (self._escape(vhost.filep)), "0")
+        self.parser.aug.set("/augeas/files%s/mtime" % (self._escape(ssl_fp)), "0")
+        self.parser.aug.set("/augeas/files%s/mtime" % (self._escape(vhost.filep)), "0")
 
     def _sift_rewrite_rules(self, contents):
         """ Helper function for _copy_create_ssl_vhost_skeleton to prepare the
@@ -1355,7 +1443,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         """
 
         try:
-            span_val = self.aug.span(vhost.path)
+            span_val = self.parser.aug.span(vhost.path)
         except ValueError:
             logger.critical("Error while reading the VirtualHost %s from "
                          "file %s", vhost.name, vhost.filep, exc_info=True)
@@ -1390,13 +1478,13 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
     def _update_ssl_vhosts_addrs(self, vh_path):
         ssl_addrs = set()
-        ssl_addr_p = self.aug.match(vh_path + "/arg")
+        ssl_addr_p = self.parser.aug.match(vh_path + "/arg")
 
         for addr in ssl_addr_p:
             old_addr = obj.Addr.fromstring(
                 str(self.parser.get_arg(addr)))
             ssl_addr = old_addr.get_addr_obj("443")
-            self.aug.set(addr, str(ssl_addr))
+            self.parser.aug.set(addr, str(ssl_addr))
             ssl_addrs.add(ssl_addr)
 
         return ssl_addrs
@@ -1415,15 +1503,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                                            vh_path, False)) > 1:
                 directive_path = self.parser.find_dir(directive, None,
                                                       vh_path, False)
-                self.aug.remove(re.sub(r"/\w*$", "", directive_path[0]))
+                self.parser.aug.remove(re.sub(r"/\w*$", "", directive_path[0]))
 
     def _remove_directives(self, vh_path, directives):
         for directive in directives:
-            while len(self.parser.find_dir(directive, None,
-                                           vh_path, False)) > 0:
+            while self.parser.find_dir(directive, None, vh_path, False):
                 directive_path = self.parser.find_dir(directive, None,
                                                       vh_path, False)
-                self.aug.remove(re.sub(r"/\w*$", "", directive_path[0]))
+                self.parser.aug.remove(re.sub(r"/\w*$", "", directive_path[0]))
 
     def _add_dummy_ssl_directives(self, vh_path):
         self.parser.add_dir(vh_path, "SSLCertificateFile",
@@ -1437,7 +1524,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
     def _add_servername_alias(self, target_name, vhost):
         vh_path = vhost.path
-        sname, saliases = self._get_vhost_names(vh_path)
+        sname, saliases = self._get_vhost_names(vhost)
         if target_name == sname or target_name in saliases:
             return
         if self._has_matching_wildcard(vh_path, target_name):
@@ -1462,8 +1549,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         """
         matches = self.parser.find_dir(
             "ServerAlias", start=vh_path, exclude=False)
-        aliases = (self.aug.get(match) for match in matches)
-        return self.included_in_wildcard(aliases, target_name)
+        aliases = (self.parser.aug.get(match) for match in matches)
+        return self.domain_in_names(aliases, target_name)
 
     def _add_name_vhost_if_necessary(self, vhost):
         """Add NameVirtualHost Directives if necessary for new vhost.
@@ -1645,7 +1732,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         if header_path:
             pat = '(?:[ "]|^)(strict-transport-security)(?:[ "]|$)'
             for match in header_path:
-                if re.search(pat, self.aug.get(match).lower()):
+                if re.search(pat, self.parser.aug.get(match).lower()):
                     hsts_dirpath = match
         if not hsts_dirpath:
             err_msg = ("Certbot was unable to find the existing HSTS header "
@@ -1659,7 +1746,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         # Our match statement was for string strict-transport-security, but
         # we need to update the value instead. The next index is for the value
         hsts_dirpath = hsts_dirpath.replace("arg[3]", "arg[4]")
-        self.aug.set(hsts_dirpath, hsts_maxage)
+        self.parser.aug.set(hsts_dirpath, hsts_maxage)
         note_msg = ("Increasing HSTS max-age value to {0} for VirtualHost "
                     "in {1}\n".format(nextstep_value, vhost.filep))
         logger.debug(note_msg)
@@ -1741,7 +1828,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         # We'll simply delete the directive, so that we'll have a
         # consistent OCSP cache path.
         if stapling_cache_aug_path:
-            self.aug.remove(
+            self.parser.aug.remove(
                     re.sub(r"/\w*$", "", stapling_cache_aug_path[0]))
 
         self.parser.add_dir_to_ifmodssl(ssl_vhost_aug_path,
@@ -1818,7 +1905,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             # "Existing Header directive for virtualhost"
             pat = '(?:[ "]|^)(%s)(?:[ "]|$)' % (header_substring.lower())
             for match in header_path:
-                if re.search(pat, self.aug.get(match).lower()):
+                if re.search(pat, self.parser.aug.get(match).lower()):
                     raise errors.PluginEnhancementAlreadyPresent(
                         "Existing %s header" % (header_substring))
 
@@ -1911,7 +1998,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             self.parser.add_dir(vhost.path, "RewriteRule",
                     constants.REWRITE_HTTPS_ARGS)
 
-
     def _verify_no_certbot_redirect(self, vhost):
         """Checks to see if a redirect was already installed by certbot.
 
@@ -1946,11 +2032,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                              constants.REWRITE_HTTPS_ARGS_WITH_END]
 
             for dir_path, args_paths in rewrite_args_dict.items():
-                arg_vals = [self.aug.get(x) for x in args_paths]
+                arg_vals = [self.parser.aug.get(x) for x in args_paths]
 
                 # Search for past redirection rule, delete it, set the new one
                 if arg_vals in constants.OLD_REWRITE_HTTPS_ARGS:
-                    self.aug.remove(dir_path)
+                    self.parser.aug.remove(dir_path)
                     self._set_https_redirection_rewrite_rule(vhost)
                     self.save()
                     raise errors.PluginEnhancementAlreadyPresent(
@@ -2006,7 +2092,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         redirect_filepath = self._write_out_redirect(ssl_vhost, text)
 
-        self.aug.load()
+        self.parser.aug.load()
         # Make a new vhost data structure and add it to the lists
         new_vhost = self._create_vhost(parser.get_aug_path(self._escape(redirect_filepath)))
         self.vhosts.append(new_vhost)
@@ -2142,7 +2228,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             vhost.enabled = True
         return
 
-    def enable_mod(self, mod_name, temp=False): # pylint: disable=unused-argument
+    def enable_mod(self, mod_name, temp=False):  # pylint: disable=unused-argument
         """Enables module in Apache.
 
         Both enables and reloads Apache so module is active.
@@ -2179,7 +2265,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         :raises .errors.MisconfigurationError: If reload fails
 
         """
-        error = ""
         try:
             util.run_script(self.option("restart_cmd"))
         except errors.SubprocessError as err:
@@ -2253,7 +2338,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
     ###########################################################################
     def get_chall_pref(self, unused_domain):  # pylint: disable=no-self-use
         """Return list of challenge preferences."""
-        return [challenges.HTTP01, challenges.TLSSNI01]
+        return [challenges.HTTP01]
 
     def perform(self, achalls):
         """Perform the configuration related challenge.
@@ -2266,20 +2351,15 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         self._chall_out.update(achalls)
         responses = [None] * len(achalls)
         http_doer = http_01.ApacheHttp01(self)
-        sni_doer = tls_sni_01.ApacheTlsSni01(self)
 
         for i, achall in enumerate(achalls):
             # Currently also have chall_doer hold associated index of the
             # challenge. This helps to put all of the responses back together
             # when they are all complete.
-            if isinstance(achall.chall, challenges.HTTP01):
-                http_doer.add_chall(achall, i)
-            else:  # tls-sni-01
-                sni_doer.add_chall(achall, i)
+            http_doer.add_chall(achall, i)
 
         http_response = http_doer.perform()
-        sni_response = sni_doer.perform()
-        if http_response or sni_response:
+        if http_response:
             # Must reload in order to activate the challenges.
             # Handled here because we may be able to load up other challenge
             # types
@@ -2290,7 +2370,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             time.sleep(3)
 
             self._update_responses(responses, http_response, http_doer)
-            self._update_responses(responses, sni_response, sni_doer)
 
         return responses
 
@@ -2317,8 +2396,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         # XXX if we ever try to enforce a local privilege boundary (eg, running
         # certbot for unprivileged users via setuid), this function will need
         # to be modified.
-        return common.install_version_controlled_file(options_ssl, options_ssl_digest,
-            self.option("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES)
+        apache_config_path = self.pick_apache_config()
+        return common.install_version_controlled_file(
+            options_ssl, options_ssl_digest, apache_config_path, constants.ALL_SSL_OPTIONS_HASHES)
 
     def enable_autohsts(self, _unused_lineage, domains):
         """

certbot-apache/certbot_apache/constants.py

@@ -9,6 +9,7 @@ MOD_SSL_CONF_DEST = "options-ssl-apache.conf"
 UPDATED_MOD_SSL_CONF_DIGEST = ".updated-options-ssl-apache-conf-digest.txt"
 """Name of the hash of the updated or informed mod_ssl_conf as saved in `IConfig.config_dir`."""
 
+# NEVER REMOVE A SINGLE HASH FROM THIS LIST UNLESS YOU KNOW EXACTLY WHAT YOU ARE DOING!
 ALL_SSL_OPTIONS_HASHES = [
     '2086bca02db48daf93468332543c60ac6acdb6f0b58c7bfdf578a5d47092f82a',
     '4844d36c9a0f587172d9fa10f4f1c9518e3bcfa1947379f155e16a70a728c21a',
@@ -18,6 +19,10 @@ ALL_SSL_OPTIONS_HASHES = [
     'cfdd7c18d2025836ea3307399f509cfb1ebf2612c87dd600a65da2a8e2f2797b',
     '80720bd171ccdc2e6b917ded340defae66919e4624962396b992b7218a561791',
     'c0c022ea6b8a51ecc8f1003d0a04af6c3f2bc1c3ce506b3c2dfc1f11ef931082',
+    '717b0a89f5e4c39b09a42813ac6e747cfbdeb93439499e73f4f70a1fe1473f20',
+    '0fcdc81280cd179a07ec4d29d3595068b9326b455c488de4b09f585d5dafc137',
+    '86cc09ad5415cd6d5f09a947fe2501a9344328b1e8a8b458107ea903e80baa6c',
+    '06675349e457eae856120cdebb564efe546f0b87399f2264baeb41e442c724c7',
 ]
 """SHA256 hashes of the contents of previous versions of all versions of MOD_SSL_CONF_SRC"""
 

certbot-apache/certbot_apache/display_ops.py

@@ -1,14 +1,12 @@
 """Contains UI methods for Apache operations."""
 import logging
-import os
 
 import zope.component
 
+import certbot.display.util as display_util
 from certbot import errors
 from certbot import interfaces
-
-import certbot.display.util as display_util
-
+from certbot.compat import os
 
 logger = logging.getLogger(__name__)
 
@@ -26,7 +24,7 @@ def select_vhost_multiple(vhosts):
         return list()
     tags_list = [vhost.display_repr()+"\n" for vhost in vhosts]
     # Remove the extra newline from the last entry
-    if len(tags_list):
+    if tags_list:
         tags_list[-1] = tags_list[-1][:-1]
     code, names = zope.component.getUtility(interfaces.IDisplay).checklist(
         "Which VirtualHosts would you like to install the wildcard certificate for?",
@@ -62,8 +60,7 @@ def select_vhost(domain, vhosts):
     code, tag = _vhost_menu(domain, vhosts)
     if code == display_util.OK:
         return vhosts[tag]
-    else:
-        return None
+    return None
 
 def _vhost_menu(domain, vhosts):
     """Select an appropriate Apache Vhost.
@@ -93,7 +90,7 @@ def _vhost_menu(domain, vhosts):
     for vhost in vhosts:
         if len(vhost.get_names()) == 1:
             disp_name = next(iter(vhost.get_names()))
-        elif len(vhost.get_names()) == 0:
+        elif not vhost.get_names():
             disp_name = ""
         else:
             disp_name = "Multiple Names"

certbot-apache/certbot_apache/entrypoint.py

@@ -1,8 +1,13 @@
 """ Entry point for Apache Plugin """
+# Pylint does not like disutils.version when running inside a venv.
+# See: https://github.com/PyCQA/pylint/issues/73
+from distutils.version import LooseVersion  # pylint: disable=no-name-in-module,import-error
+
 from certbot import util
 
 from certbot_apache import configurator
 from certbot_apache import override_arch
+from certbot_apache import override_fedora
 from certbot_apache import override_darwin
 from certbot_apache import override_debian
 from certbot_apache import override_centos
@@ -16,7 +21,8 @@ OVERRIDE_CLASSES = {
     "ubuntu": override_debian.DebianConfigurator,
     "centos": override_centos.CentOSConfigurator,
     "centos linux": override_centos.CentOSConfigurator,
-    "fedora": override_centos.CentOSConfigurator,
+    "fedora_old": override_centos.CentOSConfigurator,
+    "fedora": override_fedora.FedoraConfigurator,
     "ol": override_centos.CentOSConfigurator,
     "red hat enterprise linux server": override_centos.CentOSConfigurator,
     "rhel": override_centos.CentOSConfigurator,
@@ -27,12 +33,19 @@ OVERRIDE_CLASSES = {
     "suse": override_suse.OpenSUSEConfigurator,
 }
 
+
 def get_configurator():
     """ Get correct configurator class based on the OS fingerprint """
-    os_info = util.get_os_info()
+    os_name, os_version = util.get_os_info()
+    os_name = os_name.lower()
     override_class = None
+
+    # Special case for older Fedora versions
+    if os_name == 'fedora' and LooseVersion(os_version) < LooseVersion('29'):
+        os_name = 'fedora_old'
+
     try:
-        override_class = OVERRIDE_CLASSES[os_info[0].lower()]
+        override_class = OVERRIDE_CLASSES[os_name]
     except KeyError:
         # OS not found in the list
         os_like = util.get_systemd_os_like()
@@ -45,4 +58,5 @@ def get_configurator():
             override_class = configurator.ApacheConfigurator
     return override_class
 
+
 ENTRYPOINT = get_configurator()

certbot-apache/certbot_apache/http_01.py

@@ -1,15 +1,19 @@
 """A class that performs HTTP-01 challenges for Apache"""
 import logging
-import os
 
-from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import List, Set  # pylint: disable=unused-import, no-name-in-module
+
 from certbot import errors
+from certbot.compat import os
+from certbot.compat import filesystem
 from certbot.plugins import common
+
 from certbot_apache.obj import VirtualHost  # pylint: disable=unused-import
 from certbot_apache.parser import get_aug_path
 
 logger = logging.getLogger(__name__)
 
+
 class ApacheHttp01(common.TLSSNI01):
     """Class that performs HTTP-01 challenges within the Apache configurator."""
 
@@ -89,15 +93,27 @@ class ApacheHttp01(common.TLSSNI01):
                     self.configurator.enable_mod(mod, temp=True)
 
     def _mod_config(self):
+        selected_vhosts = []  # type: List[VirtualHost]
+        http_port = str(self.configurator.config.http01_port)
         for chall in self.achalls:
-            vh = self.configurator.find_best_http_vhost(
-                chall.domain, filter_defaults=False,
-                port=str(self.configurator.config.http01_port))
-            if vh:
-                self._set_up_include_directives(vh)
-            else:
-                for vh in self._relevant_vhosts():
-                    self._set_up_include_directives(vh)
+            # Search for matching VirtualHosts
+            for vh in self._matching_vhosts(chall.domain):
+                selected_vhosts.append(vh)
+
+        # Ensure that we have one or more VirtualHosts that we can continue
+        # with. (one that listens to port configured with --http-01-port)
+        found = False
+        for vhost in selected_vhosts:
+            if any(a.is_wildcard() or a.get_port() == http_port for a in vhost.addrs):
+                found = True
+
+        if not found:
+            for vh in self._relevant_vhosts():
+                selected_vhosts.append(vh)
+
+        # Add the challenge configuration
+        for vh in selected_vhosts:
+            self._set_up_include_directives(vh)
 
         self.configurator.reverter.register_file_creation(
             True, self.challenge_conf_pre)
@@ -121,6 +137,20 @@ class ApacheHttp01(common.TLSSNI01):
         with open(self.challenge_conf_post, "w") as new_conf:
             new_conf.write(config_text_post)
 
+    def _matching_vhosts(self, domain):
+        """Return all VirtualHost objects that have the requested domain name or
+        a wildcard name that would match the domain in ServerName or ServerAlias
+        directive.
+        """
+        matching_vhosts = []
+        for vhost in self.configurator.vhosts:
+            if self.configurator.domain_in_names(vhost.get_names(), domain):
+                # domain_in_names also matches the exact names, so no need
+                # to check "domain in vhost.get_names()" explicitly here
+                matching_vhosts.append(vhost)
+
+        return matching_vhosts
+
     def _relevant_vhosts(self):
         http01_port = str(self.configurator.config.http01_port)
         relevant_vhosts = []
@@ -139,8 +169,7 @@ class ApacheHttp01(common.TLSSNI01):
 
     def _set_up_challenges(self):
         if not os.path.isdir(self.challenge_dir):
-            os.makedirs(self.challenge_dir)
-            os.chmod(self.challenge_dir, 0o755)
+            filesystem.makedirs(self.challenge_dir, 0o755)
 
         responses = []
         for achall in self.achalls:
@@ -156,7 +185,7 @@ class ApacheHttp01(common.TLSSNI01):
         self.configurator.reverter.register_file_creation(True, name)
         with open(name, 'wb') as f:
             f.write(validation.encode())
-        os.chmod(name, 0o644)
+        filesystem.chmod(name, 0o644)
 
         return response
 

certbot-apache/certbot_apache/interfaces.py

@@ -0,0 +1,375 @@
+"""Parser interfaces."""
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class ParserNode(object):
+    """
+    ParserNode is the basic building block of the tree of such nodes,
+    representing the structure of the configuration. It is largely meant to keep
+    the structure information intact and idiomatically accessible.
+
+    The root node as well as the child nodes of it should be instances of ParserNode.
+    Nodes keep track of their differences to on-disk representation of configuration
+    by marking modified ParserNodes as dirty to enable partial write-to-disk for
+    different files in the configuration structure.
+
+    While for the most parts the usage and the child types are obvious, "include"-
+    and similar directives are an exception to this rule. This is because of the
+    nature of include directives - which unroll the contents of another file or
+    configuration block to their place. While we could unroll the included nodes
+    to the parent tree, it remains important to keep the context of include nodes
+    separate in order to write back the original configuration as it was.
+
+    For parsers that require the implementation to keep track of the whitespacing,
+    it's responsibility of each ParserNode object itself to store its prepending
+    whitespaces in order to be able to reconstruct the complete configuration file i
+    as it was when originally read from the disk.
+    """
+
+    @property
+    @abc.abstractmethod
+    def ancestor(self):  # pragma: no cover
+        """
+        This property contains a reference to ancestor node, or None if the node
+        is the root node of the configuration tree.
+
+        :returns: The ancestor BlockNode object, or None for root node.
+        :rtype: ParserNode
+        """
+
+        raise NotImplementedError
+
+    @property
+    @abc.abstractmethod
+    def dirty(self):  # pragma: no cover
+        """
+        This property contains a boolean value of the information if this node has
+        been modified since last save (or after the initial parse).
+
+        :returns: True if this node has had changes that have not yet been written
+            to disk.
+        :rtype: bool
+        """
+
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def save(self, msg):
+        """
+        Save traverses the children, and attempts to write the AST to disk for
+        all the objects that are marked dirty. The actual operation of course
+        depends on the underlying implementation. save() shouldn't be called
+        from the Configurator outside of its designated save() method in order
+        to ensure that the Reverter checkpoints are created properly.
+
+        Note: this approach of keeping internal structure of the configuration
+        within the ParserNode tree does not represent the file inclusion structure
+        of actual configuration files that reside in the filesystem. To handle
+        file writes properly, the file specific temporary trees should be extracted
+        from the full ParserNode tree where necessary when writing to disk.
+
+        """
+
+
+@six.add_metaclass(abc.ABCMeta)
+class CommentNode(ParserNode):
+    """
+    CommentNode class is used for representation of comments within the parsed
+    configuration structure. Because of the nature of comments, it is not able
+    to have child nodes and hence it is always treated as a leaf node.
+
+    CommentNode stores its contents in class variable 'comment' and does not
+    have a specific name.
+
+    """
+
+    @property
+    @abc.abstractmethod
+    def comment(self):  # pragma: no cover
+        """
+        Comment property contains the contents of the comment without the comment
+        directives (typically # or /* ... */).
+
+        :returns: A string containing the comment
+        :rtype: str
+        """
+
+        raise NotImplementedError
+
+@six.add_metaclass(abc.ABCMeta)
+class DirectiveNode(ParserNode):
+    """
+    DirectiveNode class represents a configuration directive within the configuration.
+    It can have zero or more arguments attached to it. Because of the nature of
+    single directives, it is not able to have child nodes and hence it is always
+    treated as a leaf node.
+    """
+
+    @property
+    @abc.abstractmethod
+    def enabled(self):  # pragma: no cover
+        """
+        Configuration blocks may have conditional statements enabling or disabling
+        their contents. This property returns the state of this DirectiveNode.
+
+        :returns: True if the DirectiveNode is parsed and enabled in the configuration.
+        :rtype: bool
+        """
+
+        raise NotImplementedError
+
+    @property
+    @abc.abstractmethod
+    def name(self):  # pragma: no cover
+        """
+        Name property contains the name of the directive.
+
+        :returns: Name of this node
+        :rtype: str
+        """
+
+        raise NotImplementedError
+
+    @property
+    @abc.abstractmethod
+    def parameters(self):  # pragma: no cover
+        """
+        This property contains a tuple of parameters of this ParserNode object
+        excluding whitespaces.
+
+        :returns: A tuple of parameters for this node
+        :rtype: tuple
+        """
+
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def set_parameters(self, parameters):
+        """
+        Sets the sequence of parameters for this ParserNode object without
+        whitespaces, and marks this object dirty.
+
+        :param list parameters: sequence of parameters
+        """
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BlockNode(ParserNode):
+    """
+    BlockNode class represents a block of nested configuration directives, comments
+    and other blocks as its children. A BlockNode can have zero or more arguments
+    attached to it.
+
+    Configuration blocks typically consist of one or more child nodes of all possible
+    types. Because of this, the BlockNode class has various discovery and structure
+    management methods.
+
+    Lists of arguments used as an optional argument for some of the methods should
+    be lists of strings that are applicable arguments for each specific BlockNode
+    or DirectiveNode types. As an example, for a following configuration example:
+
+        <VirtualHost *:80>
+           ...
+        </VirtualHost>
+
+    The node type would be BlockNode, name would be 'VirtualHost' and arguments
+    would be: ['*:80'].
+
+    While for the following example:
+
+        LoadModule alias_module /usr/lib/apache2/modules/mod_alias.so
+
+    The node type would be DirectiveNode, name would be 'LoadModule' and arguments
+    would be: ['alias_module', '/usr/lib/apache2/modules/mod_alias.so']
+
+    The applicable arguments are dependent on the underlying configuration language
+    and its grammar.
+    """
+
+    @abc.abstractmethod
+    def add_child_block(self, name, arguments=None, position=None):
+        """
+        Adds a new BlockNode child node with provided values and marks the callee
+        BlockNode dirty. This is used to add new children to the AST.
+
+        :param str name: The name of the child node to add
+        :param list arguments: list of arguments for the node
+        :param int position: Position in the list of children to add the new child
+            node to. Defaults to None, which appends the newly created node to the list.
+            If an integer is given, the child is inserted before that index in the
+            list similar to list().insert.
+
+        :returns: BlockNode instance of the created child block
+
+        """
+
+    @abc.abstractmethod
+    def add_child_directive(self, name, arguments=None, position=None):
+        """
+        Adds a new DirectiveNode child node with provided values and marks the
+        callee BlockNode dirty. This is used to add new children to the AST.
+
+        :param str name: The name of the child node to add
+        :param list arguments: list of arguments for the node
+        :param int position: Position in the list of children to add the new child
+            node to. Defaults to None, which appends the newly created node to the list.
+            If an integer is given, the child is inserted before that index in the
+            list similar to list().insert.
+
+        :returns: DirectiveNode instance of the created child directive
+
+        """
+
+    @abc.abstractmethod
+    def add_child_comment(self, comment="", position=None):
+        """
+        Adds a new CommentNode child node with provided value and marks the
+        callee BlockNode dirty. This is used to add new children to the AST.
+
+        :param str comment: Comment contents
+        :param int position: Position in the list of children to add the new child
+            node to. Defaults to None, which appends the newly created node to the list.
+            If an integer is given, the child is inserted before that index in the
+            list similar to list().insert.
+
+        :returns: CommentNode instance of the created child comment
+
+        """
+
+    @property
+    @abc.abstractmethod
+    def children(self):  # pragma: no cover
+        """
+        This property contains a list ParserNode objects that are the children
+        for this node. The order of children is the same as that of the parsed
+        configuration block.
+
+        :returns: A tuple of this block's children
+        :rtype: tuple
+        """
+
+        raise NotImplementedError
+
+    @property
+    @abc.abstractmethod
+    def enabled(self):
+        """
+        Configuration blocks may have conditional statements enabling or disabling
+        their contents. This property returns the state of this configuration block.
+        In case of unmatched conditional statement in block, this block itself should
+        be set enabled while its children should be set disabled.
+
+        :returns: True if the BlockNode is parsed and enabled in the configuration.
+        :rtype: bool
+        """
+
+    @abc.abstractmethod
+    def find_blocks(self, name, exclude=True):
+        """
+        Find a configuration block by name. This method walks the child tree of
+        ParserNodes under the instance it was called from. This way it is possible
+        to search for the whole configuration tree, when starting from root node or
+        to do a partial search when starting from a specified branch.
+
+        :param str name: The name of the directive to search for
+        :param bool exclude: If the search results should exclude the contents of
+            ParserNode objects that reside within conditional blocks and because
+            of current state are not enabled.
+
+        :returns: A list of found BlockNode objects.
+        """
+
+    @abc.abstractmethod
+    def find_directives(self, name, exclude=True):
+        """
+        Find a directive by name. This method walks the child tree of ParserNodes
+        under the instance it was called from. This way it is possible to search
+        for the whole configuration tree, when starting from root node, or to do
+        a partial search when starting from a specified branch.
+
+        :param str name: The name of the directive to search for
+        :param bool exclude: If the search results should exclude the contents of
+            ParserNode objects that reside within conditional blocks and because
+            of current state are not enabled.
+
+        :returns: A list of found DirectiveNode objects.
+
+        """
+
+    @abc.abstractmethod
+    def find_comments(self, comment, exact=False):
+        """
+        Find comments with value containing or being exactly the same as search term.
+
+        This method walks the child tree of ParserNodes under the instance it was
+        called from. This way it is possible to search for the whole configuration
+        tree, when starting from root node, or to do a partial search when starting
+        from a specified branch.
+
+        :param str comment: The content of comment to search for
+        :param bool exact: If the comment needs to exactly match the search term
+
+        :returns: A list of found CommentNode objects.
+
+        """
+
+    @abc.abstractmethod
+    def delete_child(self, child):
+        """
+        Remove a specified child node from the list of children of the called
+        BlockNode object.
+
+        :param ParserNode child: Child ParserNode object to remove from the list
+            of children of the callee.
+        """
+
+    @property
+    @abc.abstractmethod
+    def name(self):  # pragma: no cover
+        """
+        Name property contains the name of the block. As an example for config:
+            <VirtualHost *:80> ... </VirtualHost>
+        the name would be "VirtualHost".
+
+        :returns: Name of this node
+        :rtype: str
+        """
+
+        raise NotImplementedError
+
+    @property
+    @abc.abstractmethod
+    def parameters(self):  # pragma: no cover
+        """
+        This property contains a tuple of parameters of this ParserNode object
+        excluding whitespaces.
+
+        :returns: A tuple of parameters for this node
+        :rtype: tuple
+        """
+
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def set_parameters(self, parameters):
+        """
+        Sets the sequence of parameters for this ParserNode object without
+        whitespaces, and marks this object dirty.
+
+        :param list parameters: sequence of parameters
+        """
+
+    @abc.abstractmethod
+    def unsaved_files(self):
+        """
+        Returns a list of file paths that have been changed since the last save
+        (or the initial configuration parse). The intended use for this method
+        is to tell the Reverter which files need to be included in a checkpoint.
+
+        This is typically called for the root of the ParserNode tree.
+
+        :returns: list of file paths of files that have been changed but not yet
+            saved to disk.
+        """

certbot-apache/certbot_apache/obj.py

@@ -26,7 +26,7 @@ class Addr(common.Addr):
     def __repr__(self):
         return "certbot_apache.obj.Addr(" + repr(self.tup) + ")"
 
-    def __hash__(self):
+    def __hash__(self):  # pylint: disable=useless-super-delegation
         # Python 3 requires explicit overridden for __hash__ if __eq__ or
         # __cmp__ is overridden. See https://bugs.python.org/issue2235
         return super(Addr, self).__hash__()
@@ -47,8 +47,7 @@ class Addr(common.Addr):
             return 0
         elif self.get_addr() == "*":
             return 1
-        else:
-            return 2
+        return 2
 
     def conflicts(self, addr):
         r"""Returns if address could conflict with correct function of self.

certbot-apache/certbot_apache/override_arch.py

@@ -1,6 +1,4 @@
 """ Distribution specific override class for Arch Linux """
-import pkg_resources
-
 import zope.interface
 
 from certbot import interfaces
@@ -26,6 +24,4 @@ class ArchConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/httpd/conf",
-        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
-            "certbot_apache", "options-ssl-apache.conf")
     )

certbot-apache/certbot_apache/override_centos.py

@@ -1,14 +1,23 @@
 """ Distribution specific override class for CentOS family (RHEL, Fedora) """
-import pkg_resources
+import logging
 
 import zope.interface
 
+from certbot import errors
 from certbot import interfaces
+from certbot import util
+from certbot.errors import MisconfigurationError
+
+from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
 
 from certbot_apache import apache_util
 from certbot_apache import configurator
 from certbot_apache import parser
 
+
+logger = logging.getLogger(__name__)
+
+
 @zope.interface.provider(interfaces.IPluginFactory)
 class CentOSConfigurator(configurator.ApacheConfigurator):
     """CentOS specific ApacheConfigurator override class"""
@@ -29,10 +38,52 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/httpd/conf.d",
-        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
-            "certbot_apache", "centos-options-ssl-apache.conf")
     )
 
+    def config_test(self):
+        """
+        Override config_test to mitigate configtest error in vanilla installation
+        of mod_ssl in Fedora. The error is caused by non-existent self-signed
+        certificates referenced by the configuration, that would be autogenerated
+        during the first (re)start of httpd.
+        """
+
+        os_info = util.get_os_info()
+        fedora = os_info[0].lower() == "fedora"
+
+        try:
+            super(CentOSConfigurator, self).config_test()
+        except errors.MisconfigurationError:
+            if fedora:
+                self._try_restart_fedora()
+            else:
+                raise
+
+    def _try_restart_fedora(self):
+        """
+        Tries to restart httpd using systemctl to generate the self signed keypair.
+        """
+
+        try:
+            util.run_script(['systemctl', 'restart', 'httpd'])
+        except errors.SubprocessError as err:
+            raise errors.MisconfigurationError(str(err))
+
+        # Finish with actual config check to see if systemctl restart helped
+        super(CentOSConfigurator, self).config_test()
+
+    def pick_apache_config(self):
+        """
+        Pick the appropriate TLS Apache configuration file for current version of Apache and OS.
+        :return: the path to the TLS Apache configuration file to use
+        :rtype: str
+        """
+        # Disabling TLS session tickets is supported by Apache 2.4.11+.
+        # So for old versions of Apache we pick a configuration without this option.
+        if self.version < (2, 4, 11):
+            return apache_util.find_ssl_apache_conf("centos-old")
+        return apache_util.find_ssl_apache_conf("centos-current")
+
     def _prepare_options(self):
         """
         Override the options dictionary initialization in order to support
@@ -44,9 +95,86 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
     def get_parser(self):
         """Initializes the ApacheParser"""
         return CentOSParser(
-            self.aug, self.option("server_root"), self.option("vhost_root"),
+            self.option("server_root"), self.option("vhost_root"),
             self.version, configurator=self)
 
+    def _deploy_cert(self, *args, **kwargs):  # pylint: disable=arguments-differ
+        """
+        Override _deploy_cert in order to ensure that the Apache configuration
+        has "LoadModule ssl_module..." before parsing the VirtualHost configuration
+        that was created by Certbot
+        """
+        super(CentOSConfigurator, self)._deploy_cert(*args, **kwargs)
+        if self.version < (2, 4, 0):
+            self._deploy_loadmodule_ssl_if_needed()
+
+    def _deploy_loadmodule_ssl_if_needed(self):
+        """
+        Add "LoadModule ssl_module <pre-existing path>" to main httpd.conf if
+        it doesn't exist there already.
+        """
+
+        loadmods = self.parser.find_dir("LoadModule", "ssl_module", exclude=False)
+
+        correct_ifmods = []  # type: List[str]
+        loadmod_args = []  # type: List[str]
+        loadmod_paths = []  # type: List[str]
+        for m in loadmods:
+            noarg_path = m.rpartition("/")[0]
+            path_args = self.parser.get_all_args(noarg_path)
+            if loadmod_args:
+                if loadmod_args != path_args:
+                    msg = ("Certbot encountered multiple LoadModule directives "
+                           "for LoadModule ssl_module with differing library paths. "
+                           "Please remove or comment out the one(s) that are not in "
+                           "use, and run Certbot again.")
+                    raise MisconfigurationError(msg)
+            else:
+                loadmod_args = path_args
+
+            if self.parser.not_modssl_ifmodule(noarg_path):  # pylint: disable=no-member
+                if self.parser.loc["default"] in noarg_path:
+                    # LoadModule already in the main configuration file
+                    if ("ifmodule/" in noarg_path.lower() or
+                        "ifmodule[1]" in noarg_path.lower()):
+                        # It's the first or only IfModule in the file
+                        return
+                # Populate the list of known !mod_ssl.c IfModules
+                nodir_path = noarg_path.rpartition("/directive")[0]
+                correct_ifmods.append(nodir_path)
+            else:
+                loadmod_paths.append(noarg_path)
+
+        if not loadmod_args:
+            # Do not try to enable mod_ssl
+            return
+
+        # Force creation as the directive wasn't found from the beginning of
+        # httpd.conf
+        rootconf_ifmod = self.parser.create_ifmod(
+            parser.get_aug_path(self.parser.loc["default"]),
+            "!mod_ssl.c", beginning=True)
+        # parser.get_ifmod returns a path postfixed with "/", remove that
+        self.parser.add_dir(rootconf_ifmod[:-1], "LoadModule", loadmod_args)
+        correct_ifmods.append(rootconf_ifmod[:-1])
+        self.save_notes += "Added LoadModule ssl_module to main configuration.\n"
+
+        # Wrap LoadModule mod_ssl inside of <IfModule !mod_ssl.c> if it's not
+        # configured like this already.
+        for loadmod_path in loadmod_paths:
+            nodir_path = loadmod_path.split("/directive")[0]
+            # Remove the old LoadModule directive
+            self.parser.aug.remove(loadmod_path)
+
+            # Create a new IfModule !mod_ssl.c if not already found on path
+            ssl_ifmod = self.parser.get_ifmod(nodir_path, "!mod_ssl.c",
+                                            beginning=True)[:-1]
+            if ssl_ifmod not in correct_ifmods:
+                self.parser.add_dir(ssl_ifmod, "LoadModule", loadmod_args)
+                correct_ifmods.append(ssl_ifmod)
+                self.save_notes += ("Wrapped pre-existing LoadModule ssl_module "
+                                    "inside of <IfModule !mod_ssl> block.\n")
+
 
 class CentOSParser(parser.ApacheParser):
     """CentOS specific ApacheParser override class"""
@@ -64,5 +192,35 @@ class CentOSParser(parser.ApacheParser):
     def parse_sysconfig_var(self):
         """ Parses Apache CLI options from CentOS configuration file """
         defines = apache_util.parse_define_file(self.sysconfig_filep, "OPTIONS")
-        for k in defines.keys():
+        for k in defines:
             self.variables[k] = defines[k]
+
+    def not_modssl_ifmodule(self, path):
+        """Checks if the provided Augeas path has argument !mod_ssl"""
+
+        if "ifmodule" not in path.lower():
+            return False
+
+        # Trim the path to the last ifmodule
+        workpath = path.lower()
+        while workpath:
+            # Get path to the last IfModule (ignore the tail)
+            parts = workpath.rpartition("ifmodule")
+
+            if not parts[0]:
+                # IfModule not found
+                break
+            ifmod_path = parts[0] + parts[1]
+            # Check if ifmodule had an index
+            if parts[2].startswith("["):
+                # Append the index from tail
+                ifmod_path += parts[2].partition("/")[0]
+            # Get the original path trimmed to correct length
+            # This is required to preserve cases
+            ifmod_real_path = path[0:len(ifmod_path)]
+            if "!mod_ssl.c" in self.get_all_args(ifmod_real_path):
+                return True
+            # Set the workpath to the heading part
+            workpath = parts[0]
+
+        return False

certbot-apache/certbot_apache/override_darwin.py

@@ -1,6 +1,4 @@
 """ Distribution specific override class for macOS """
-import pkg_resources
-
 import zope.interface
 
 from certbot import interfaces
@@ -26,6 +24,4 @@ class DarwinConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/other",
-        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
-            "certbot_apache", "options-ssl-apache.conf")
     )

certbot-apache/certbot_apache/override_debian.py

@@ -1,19 +1,20 @@
 """ Distribution specific override class for Debian family (Ubuntu/Debian) """
 import logging
-import os
-import pkg_resources
 
 import zope.interface
 
 from certbot import errors
 from certbot import interfaces
 from certbot import util
+from certbot.compat import filesystem
+from certbot.compat import os
 
 from certbot_apache import apache_util
 from certbot_apache import configurator
 
 logger = logging.getLogger(__name__)
 
+
 @zope.interface.provider(interfaces.IPluginFactory)
 class DebianConfigurator(configurator.ApacheConfigurator):
     """Debian specific ApacheConfigurator override class"""
@@ -33,8 +34,6 @@ class DebianConfigurator(configurator.ApacheConfigurator):
         handle_modules=True,
         handle_sites=True,
         challenge_location="/etc/apache2",
-        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
-            "certbot_apache", "options-ssl-apache.conf")
     )
 
     def enable_site(self, vhost):
@@ -51,7 +50,7 @@ class DebianConfigurator(configurator.ApacheConfigurator):
 
         """
         if vhost.enabled:
-            return
+            return None
 
         enabled_path = ("%s/sites-enabled/%s" %
                         (self.parser.root,
@@ -64,11 +63,11 @@ class DebianConfigurator(configurator.ApacheConfigurator):
         try:
             os.symlink(vhost.filep, enabled_path)
         except OSError as err:
-            if os.path.islink(enabled_path) and os.path.realpath(
+            if os.path.islink(enabled_path) and filesystem.realpath(
                enabled_path) == vhost.filep:
                 # Already in shape
                 vhost.enabled = True
-                return
+                return None
             else:
                 logger.warning(
                     "Could not symlink %s to %s, got error: %s", enabled_path,
@@ -81,9 +80,9 @@ class DebianConfigurator(configurator.ApacheConfigurator):
         vhost.enabled = True
         logger.info("Enabling available site: %s", vhost.filep)
         self.save_notes += "Enabled site %s\n" % vhost.filep
+        return None
 
     def enable_mod(self, mod_name, temp=False):
-        # pylint: disable=unused-argument
         """Enables module in Apache.
 
         Both enables and reloads Apache so module is active.

certbot-apache/certbot_apache/override_fedora.py

@@ -0,0 +1,94 @@
+""" Distribution specific override class for Fedora 29+ """
+import zope.interface
+
+from certbot import errors
+from certbot import interfaces
+from certbot import util
+
+from certbot_apache import apache_util
+from certbot_apache import configurator
+from certbot_apache import parser
+
+
+@zope.interface.provider(interfaces.IPluginFactory)
+class FedoraConfigurator(configurator.ApacheConfigurator):
+    """Fedora 29+ specific ApacheConfigurator override class"""
+
+    OS_DEFAULTS = dict(
+        server_root="/etc/httpd",
+        vhost_root="/etc/httpd/conf.d",
+        vhost_files="*.conf",
+        logs_root="/var/log/httpd",
+        ctl="httpd",
+        version_cmd=['httpd', '-v'],
+        restart_cmd=['apachectl', 'graceful'],
+        restart_cmd_alt=['apachectl', 'restart'],
+        conftest_cmd=['apachectl', 'configtest'],
+        enmod=None,
+        dismod=None,
+        le_vhost_ext="-le-ssl.conf",
+        handle_modules=False,
+        handle_sites=False,
+        challenge_location="/etc/httpd/conf.d",
+    )
+
+    def config_test(self):
+        """
+        Override config_test to mitigate configtest error in vanilla installation
+        of mod_ssl in Fedora. The error is caused by non-existent self-signed
+        certificates referenced by the configuration, that would be autogenerated
+        during the first (re)start of httpd.
+        """
+        try:
+            super(FedoraConfigurator, self).config_test()
+        except errors.MisconfigurationError:
+            self._try_restart_fedora()
+
+    def get_parser(self):
+        """Initializes the ApacheParser"""
+        return FedoraParser(
+            self.option("server_root"), self.option("vhost_root"),
+            self.version, configurator=self)
+
+    def _try_restart_fedora(self):
+        """
+        Tries to restart httpd using systemctl to generate the self signed keypair.
+        """
+        try:
+            util.run_script(['systemctl', 'restart', 'httpd'])
+        except errors.SubprocessError as err:
+            raise errors.MisconfigurationError(str(err))
+
+        # Finish with actual config check to see if systemctl restart helped
+        super(FedoraConfigurator, self).config_test()
+
+    def _prepare_options(self):
+        """
+        Override the options dictionary initialization to keep using apachectl
+        instead of httpd and so take advantages of this new bash script in newer versions
+        of Fedora to restart httpd.
+        """
+        super(FedoraConfigurator, self)._prepare_options()
+        self.options["restart_cmd"][0] = 'apachectl'
+        self.options["restart_cmd_alt"][0] = 'apachectl'
+        self.options["conftest_cmd"][0] = 'apachectl'
+
+
+class FedoraParser(parser.ApacheParser):
+    """Fedora 29+ specific ApacheParser override class"""
+    def __init__(self, *args, **kwargs):
+        # Fedora 29+ specific configuration file for Apache
+        self.sysconfig_filep = "/etc/sysconfig/httpd"
+        super(FedoraParser, self).__init__(*args, **kwargs)
+
+    def update_runtime_variables(self):
+        """ Override for update_runtime_variables for custom parsing """
+        # Opportunistic, works if SELinux not enforced
+        super(FedoraParser, self).update_runtime_variables()
+        self._parse_sysconfig_var()
+
+    def _parse_sysconfig_var(self):
+        """ Parses Apache CLI options from Fedora configuration file """
+        defines = apache_util.parse_define_file(self.sysconfig_filep, "OPTIONS")
+        for k in defines:
+            self.variables[k] = defines[k]

certbot-apache/certbot_apache/override_gentoo.py

@@ -1,6 +1,4 @@
 """ Distribution specific override class for Gentoo Linux """
-import pkg_resources
-
 import zope.interface
 
 from certbot import interfaces
@@ -29,8 +27,6 @@ class GentooConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/vhosts.d",
-        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
-            "certbot_apache", "options-ssl-apache.conf")
     )
 
     def _prepare_options(self):
@@ -44,7 +40,7 @@ class GentooConfigurator(configurator.ApacheConfigurator):
     def get_parser(self):
         """Initializes the ApacheParser"""
         return GentooParser(
-            self.aug, self.option("server_root"), self.option("vhost_root"),
+            self.option("server_root"), self.option("vhost_root"),
             self.version, configurator=self)
 
 
@@ -64,7 +60,7 @@ class GentooParser(parser.ApacheParser):
         """ Parses Apache CLI options from Gentoo configuration file """
         defines = apache_util.parse_define_file(self.apacheconfig_filep,
                                                 "APACHE2_OPTS")
-        for k in defines.keys():
+        for k in defines:
             self.variables[k] = defines[k]
 
     def update_modules(self):

certbot-apache/certbot_apache/override_suse.py

@@ -1,6 +1,4 @@
 """ Distribution specific override class for OpenSUSE """
-import pkg_resources
-
 import zope.interface
 
 from certbot import interfaces
@@ -26,6 +24,4 @@ class OpenSUSEConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/vhosts.d",
-        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
-            "certbot_apache", "options-ssl-apache.conf")
     )

certbot-apache/certbot_apache/parser.py

@@ -2,7 +2,6 @@
 import copy
 import fnmatch
 import logging
-import os
 import re
 import subprocess
 import sys
@@ -10,7 +9,11 @@ import sys
 import six
 
 from acme.magic_typing import Dict, List, Set  # pylint: disable=unused-import, no-name-in-module
+
 from certbot import errors
+from certbot.compat import os
+
+from certbot_apache import constants
 
 logger = logging.getLogger(__name__)
 
@@ -31,7 +34,7 @@ class ApacheParser(object):
     arg_var_interpreter = re.compile(r"\$\{[^ \}]*}")
     fnmatch_chars = set(["*", "?", "\\", "[", "]"])
 
-    def __init__(self, aug, root, vhostroot=None, version=(2, 4),
+    def __init__(self, root, vhostroot=None, version=(2, 4),
                  configurator=None):
         # Note: Order is important here.
 
@@ -40,15 +43,25 @@ class ApacheParser(object):
         # issues with aug.load() after adding new files / defines to parse tree
         self.configurator = configurator
 
+        # Initialize augeas
+        self.aug = None
+        self.init_augeas()
+
+        if not self.check_aug_version():
+            raise errors.NotSupportedError(
+                "Apache plugin support requires libaugeas0 and augeas-lenses "
+                "version 1.2.0 or higher, please make sure you have you have "
+                "those installed.")
+
         self.modules = set()  # type: Set[str]
         self.parser_paths = {}  # type: Dict[str, List[str]]
         self.variables = {}  # type: Dict[str, str]
 
-        self.aug = aug
         # Find configuration root and make sure augeas can parse it.
         self.root = os.path.abspath(root)
         self.loc = {"root": self._find_config_root()}
         self.parse_file(self.loc["root"])
+        self.handle_includes()
 
         if version >= (2, 4):
             # Look up variables from httpd and add to DOM if not already parsed
@@ -76,6 +89,146 @@ class ApacheParser(object):
             if self.find_dir("Define", exclude=False):
                 raise errors.PluginError("Error parsing runtime variables")
 
+    def init_augeas(self):
+        """ Initialize the actual Augeas instance """
+
+        try:
+            import augeas
+        except ImportError:  # pragma: no cover
+            raise errors.NoInstallationError("Problem in Augeas installation")
+
+        self.aug = augeas.Augeas(
+            # specify a directory to load our preferred lens from
+            loadpath=constants.AUGEAS_LENS_DIR,
+            # Do not save backup (we do it ourselves), do not load
+            # anything by default
+            flags=(augeas.Augeas.NONE |
+                   augeas.Augeas.NO_MODL_AUTOLOAD |
+                   augeas.Augeas.ENABLE_SPAN))
+
+    def check_parsing_errors(self, lens):
+        """Verify Augeas can parse all of the lens files.
+
+        :param str lens: lens to check for errors
+
+        :raises .errors.PluginError: If there has been an error in parsing with
+            the specified lens.
+
+        """
+        error_files = self.aug.match("/augeas//error")
+
+        for path in error_files:
+            # Check to see if it was an error resulting from the use of
+            # the httpd lens
+            lens_path = self.aug.get(path + "/lens")
+            # As aug.get may return null
+            if lens_path and lens in lens_path:
+                msg = (
+                    "There has been an error in parsing the file {0} on line {1}: "
+                    "{2}".format(
+                    # Strip off /augeas/files and /error
+                    path[13:len(path) - 6],
+                    self.aug.get(path + "/line"),
+                    self.aug.get(path + "/message")))
+                raise errors.PluginError(msg)
+
+    def check_aug_version(self):
+        """ Checks that we have recent enough version of libaugeas.
+        If augeas version is recent enough, it will support case insensitive
+        regexp matching"""
+
+        self.aug.set("/test/path/testing/arg", "aRgUMeNT")
+        try:
+            matches = self.aug.match(
+                "/test//*[self::arg=~regexp('argument', 'i')]")
+        except RuntimeError:
+            self.aug.remove("/test/path")
+            return False
+        self.aug.remove("/test/path")
+        return matches
+
+    def unsaved_files(self):
+        """Lists files that have modified Augeas DOM but the changes have not
+        been written to the filesystem yet, used by `self.save()` and
+        ApacheConfigurator to check the file state.
+
+        :raises .errors.PluginError: If there was an error in Augeas, in
+            an attempt to save the configuration, or an error creating a
+            checkpoint
+
+        :returns: `set` of unsaved files
+        """
+        save_state = self.aug.get("/augeas/save")
+        self.aug.set("/augeas/save", "noop")
+        # Existing Errors
+        ex_errs = self.aug.match("/augeas//error")
+        try:
+            # This is a noop save
+            self.aug.save()
+        except (RuntimeError, IOError):
+            self._log_save_errors(ex_errs)
+            # Erase Save Notes
+            self.configurator.save_notes = ""
+            raise errors.PluginError(
+                "Error saving files, check logs for more info.")
+
+        # Return the original save method
+        self.aug.set("/augeas/save", save_state)
+
+        # Retrieve list of modified files
+        # Note: Noop saves can cause the file to be listed twice, I used a
+        # set to remove this possibility. This is a known augeas 0.10 error.
+        save_paths = self.aug.match("/augeas/events/saved")
+
+        save_files = set()
+        if save_paths:
+            for path in save_paths:
+                save_files.add(self.aug.get(path)[6:])
+        return save_files
+
+    def ensure_augeas_state(self):
+        """Makes sure that all Augeas dom changes are written to files to avoid
+        loss of configuration directives when doing additional augeas parsing,
+        causing a possible augeas.load() resulting dom reset
+        """
+
+        if self.unsaved_files():
+            self.configurator.save_notes += "(autosave)"
+            self.configurator.save()
+
+    def save(self, save_files):
+        """Saves all changes to the configuration files.
+
+        save() is called from ApacheConfigurator to handle the parser specific
+        tasks of saving.
+
+        :param list save_files: list of strings of file paths that we need to save.
+
+        """
+        self.configurator.save_notes = ""
+        self.aug.save()
+
+        # Force reload if files were modified
+        # This is needed to recalculate augeas directive span
+        if save_files:
+            for sf in save_files:
+                self.aug.remove("/files/"+sf)
+            self.aug.load()
+
+    def _log_save_errors(self, ex_errs):
+        """Log errors due to bad Augeas save.
+
+        :param list ex_errs: Existing errors before save
+
+        """
+        # Check for the root of save problems
+        new_errs = self.aug.match("/augeas//error")
+        # logger.error("During Save - %s", mod_conf)
+        logger.error("Unable to save files: %s. Attempted Save Notes: %s",
+                     ", ".join(err[13:len(err) - 6] for err in new_errs
+                               # Only new errors caused by recent save
+                               if err not in ex_errs), self.configurator.save_notes)
+
     def add_include(self, main_config, inc_path):
         """Add Include for a new configuration file if one does not exist
 
@@ -83,7 +236,7 @@ class ApacheParser(object):
         :param str inc_path: path of file to include
 
         """
-        if len(self.find_dir(case_i("Include"), inc_path)) == 0:
+        if not self.find_dir(case_i("Include"), inc_path):
             logger.debug("Adding Include %s to %s",
                          inc_path, get_aug_path(main_config))
             self.add_dir(
@@ -93,12 +246,7 @@ class ApacheParser(object):
             # Add new path to parser paths
             new_dir = os.path.dirname(inc_path)
             new_file = os.path.basename(inc_path)
-            if new_dir in self.existing_paths.keys():
-                # Add to existing path
-                self.existing_paths[new_dir].append(new_file)
-            else:
-                # Create a new path
-                self.existing_paths[new_dir] = [new_file]
+            self.existing_paths.setdefault(new_dir, []).append(new_file)
 
     def add_mod(self, mod_name):
         """Shortcut for updating parser modules."""
@@ -122,6 +270,7 @@ class ApacheParser(object):
 
         """
         mods = set()  # type: Set[str]
+        self.handle_includes()
         matches = self.find_dir("LoadModule")
         iterator = iter(matches)
         # Make sure prev_size != cur_size for do: while: iteration
@@ -139,7 +288,7 @@ class ApacheParser(object):
                     mods.add(os.path.basename(mod_filename)[:-2] + "c")
                 else:
                     logger.debug("Could not read LoadModule directive from " +
-                                 "Augeas path: {0}".format(match_name[6:]))
+                                 "Augeas path: %s", match_name[6:])
         self.modules.update(mods)
 
     def update_runtime_variables(self):
@@ -174,11 +323,7 @@ class ApacheParser(object):
     def update_includes(self):
         """Get includes from httpd process, and add them to DOM if needed"""
 
-        # Find_dir iterates over configuration for Include and IncludeOptional
-        # directives to make sure we see the full include tree present in the
-        # configuration files
-        _ = self.find_dir("Include")
-
+        self.handle_includes()
         inc_cmd = [self.configurator.option("ctl"), "-t", "-D",
                    "DUMP_INCLUDES"]
         matches = self.parse_from_subprocess(inc_cmd, r"\(.*\) (.*)")
@@ -229,8 +374,8 @@ class ApacheParser(object):
                 "Error running command %s for runtime parameters!%s",
                 command, os.linesep)
             raise errors.MisconfigurationError(
-                "Error accessing loaded Apache parameters: %s",
-                command)
+                "Error accessing loaded Apache parameters: {0}".format(
+                command))
         # Small errors that do not impede
         if proc.returncode != 0:
             logger.warning("Error in checking parameter list: %s", stderr)
@@ -256,12 +401,12 @@ class ApacheParser(object):
         """
         filtered = []
         if args == 1:
-            for i in range(len(matches)):
-                if matches[i].endswith("/arg"):
+            for i, match in enumerate(matches):
+                if match.endswith("/arg"):
                     filtered.append(matches[i][:-4])
         else:
-            for i in range(len(matches)):
-                if matches[i].endswith("/arg[%d]" % args):
+            for i, match in enumerate(matches):
+                if match.endswith("/arg[%d]" % args):
                     # Make sure we don't cause an IndexError (end of list)
                     # Check to make sure arg + 1 doesn't exist
                     if (i == (len(matches) - 1) or
@@ -286,7 +431,7 @@ class ApacheParser(object):
         """
         # TODO: Add error checking code... does the path given even exist?
         #       Does it throw exceptions?
-        if_mod_path = self._get_ifmod(aug_conf_path, "mod_ssl.c")
+        if_mod_path = self.get_ifmod(aug_conf_path, "mod_ssl.c")
         # IfModule can have only one valid argument, so append after
         self.aug.insert(if_mod_path + "arg", "directive", False)
         nvh_path = if_mod_path + "directive[1]"
@@ -297,22 +442,54 @@ class ApacheParser(object):
             for i, arg in enumerate(args):
                 self.aug.set("%s/arg[%d]" % (nvh_path, i + 1), arg)
 
-    def _get_ifmod(self, aug_conf_path, mod):
+    def get_ifmod(self, aug_conf_path, mod, beginning=False):
         """Returns the path to <IfMod mod> and creates one if it doesn't exist.
 
         :param str aug_conf_path: Augeas configuration path
         :param str mod: module ie. mod_ssl.c
+        :param bool beginning: If the IfModule should be created to the beginning
+            of augeas path DOM tree.
+
+        :returns: Augeas path of the requested IfModule directive that pre-existed
+            or was created during the process. The path may be dynamic,
+            i.e. .../IfModule[last()]
+        :rtype: str
 
         """
         if_mods = self.aug.match(("%s/IfModule/*[self::arg='%s']" %
                                   (aug_conf_path, mod)))
-        if len(if_mods) == 0:
-            self.aug.set("%s/IfModule[last() + 1]" % aug_conf_path, "")
-            self.aug.set("%s/IfModule[last()]/arg" % aug_conf_path, mod)
-            if_mods = self.aug.match(("%s/IfModule/*[self::arg='%s']" %
-                                      (aug_conf_path, mod)))
+        if not if_mods:
+            return self.create_ifmod(aug_conf_path, mod, beginning)
+
         # Strip off "arg" at end of first ifmod path
-        return if_mods[0][:len(if_mods[0]) - 3]
+        return if_mods[0].rpartition("arg")[0]
+
+    def create_ifmod(self, aug_conf_path, mod, beginning=False):
+        """Creates a new <IfMod mod> and returns its path.
+
+        :param str aug_conf_path: Augeas configuration path
+        :param str mod: module ie. mod_ssl.c
+        :param bool beginning: If the IfModule should be created to the beginning
+            of augeas path DOM tree.
+
+        :returns: Augeas path of the newly created IfModule directive.
+            The path may be dynamic, i.e. .../IfModule[last()]
+        :rtype: str
+
+        """
+        if beginning:
+            c_path_arg = "{}/IfModule[1]/arg".format(aug_conf_path)
+            # Insert IfModule before the first directive
+            self.aug.insert("{}/directive[1]".format(aug_conf_path),
+                            "IfModule", True)
+            retpath = "{}/IfModule[1]/".format(aug_conf_path)
+        else:
+            c_path = "{}/IfModule[last() + 1]".format(aug_conf_path)
+            c_path_arg = "{}/IfModule[last()]/arg".format(aug_conf_path)
+            self.aug.set(c_path, "")
+            retpath = "{}/IfModule[last()]/".format(aug_conf_path)
+        self.aug.set(c_path_arg, mod)
+        return retpath
 
     def add_dir(self, aug_conf_path, directive, args):
         """Appends directive to the end fo the file given by aug_conf_path.
@@ -427,6 +604,8 @@ class ApacheParser(object):
         # includes = self.aug.match(start +
         # "//* [self::directive='Include']/* [label()='arg']")
 
+        regex = "%s" % (case_i(directive))
+
         regex = "(%s)|(%s)|(%s)" % (case_i(directive),
                                     case_i("Include"),
                                     case_i("IncludeOptional"))
@@ -458,6 +637,63 @@ class ApacheParser(object):
 
         return ordered_matches
 
+    def handle_includes(self, known_matches=None):
+        """
+        This method searches through the configuration tree for Include and
+        IncludeOptional directives and includes them in the Augeas DOM tree
+        recursively.
+
+        :param list known_matches: List of already known Augeas DOM paths of found
+            include directives.
+        """
+
+        start = get_aug_path(self.loc["root"])
+        regex = "(%s)|(%s)" % (case_i("Include"), case_i("IncludeOptional"))
+
+        all_matches = self.aug.match(
+            "%s//*[self::directive=~regexp('%s')]" % (start, regex))
+
+        if known_matches and all([True for m in all_matches if m in known_matches]):
+            # We already found everything previously
+            return
+
+        # Always use the exclusion as we want to have the correct state
+        matches = self._exclude_dirs(all_matches)
+
+        for match in matches:
+            incpath = self.get_arg(match + "/arg")
+            # Attempts to add a transform to the file if one does not already exist
+
+            # Standardize the include argument based on server root
+            if not incpath.startswith("/"):
+                # Normpath will condense ../
+                incpath = os.path.normpath(os.path.join(self.root, incpath))
+            else:
+                incpath = os.path.normpath(incpath)
+
+            if os.path.isdir(incpath):
+                self.parse_file(os.path.join(incpath, "*"))
+            else:
+                self.parse_file(incpath)
+
+        # Iterate. The results will be checked and the recursion broken when
+        # everything has been found.
+        self.handle_includes(all_matches)
+
+    def get_all_args(self, match):
+        """
+        Tries to fetch all arguments for a directive. See get_arg.
+
+        Note that if match is an ancestor node, it returns all names of
+        child directives as well as the list of arguments.
+
+        """
+
+        if match[-1] != "/":
+            match = match+"/"
+        allargs = self.aug.match(match + '*')
+        return [self.get_arg(arg) for arg in allargs]
+
     def get_arg(self, match):
         """Uses augeas.get to get argument value and interprets result.
 
@@ -562,10 +798,10 @@ class ApacheParser(object):
             arg = os.path.normpath(arg)
 
         # Attempts to add a transform to the file if one does not already exist
-        if os.path.isdir(arg):
-            self.parse_file(os.path.join(arg, "*"))
-        else:
-            self.parse_file(arg)
+        #if os.path.isdir(arg):
+        #    self.parse_file(os.path.join(arg, "*"))
+        #else:
+        #    self.parse_file(arg)
 
         # Argument represents an fnmatch regular expression, convert it
         # Split up the path and convert each into an Augeas accepted regex
@@ -601,9 +837,8 @@ class ApacheParser(object):
         if sys.version_info < (3, 6):
             # This strips off final /Z(?ms)
             return fnmatch.translate(clean_fn_match)[:-7]
-        else:  # pragma: no cover
-            # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z
-            return fnmatch.translate(clean_fn_match)[4:-3]
+        # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z
+        return fnmatch.translate(clean_fn_match)[4:-3]  # pragma: no cover
 
     def parse_file(self, filepath):
         """Parse file with Augeas
@@ -617,8 +852,7 @@ class ApacheParser(object):
         use_new, remove_old = self._check_path_actions(filepath)
         # Ensure that we have the latest Augeas DOM state on disk before
         # calling aug.load() which reloads the state from disk
-        if self.configurator:
-            self.configurator.ensure_augeas_state()
+        self.ensure_augeas_state()
         # Test if augeas included file for Httpd.lens
         # Note: This works for augeas globs, ie. *.conf
         if use_new:
@@ -685,10 +919,7 @@ class ApacheParser(object):
                 use_new = False
             else:
                 use_new = True
-            if new_file_match == "*":
-                remove_old = True
-            else:
-                remove_old = False
+            remove_old = new_file_match == "*"
         except KeyError:
             use_new = True
             remove_old = False

certbot-apache/certbot_apache/parserassert.py

@@ -0,0 +1,84 @@
+"""Runtime assertions for two different parser implementations"""
+
+def simpleassert(old_value, new_value):
+    """
+    Simple assertion
+    """
+    assert old_value == new_value
+
+def legacy_assert_list(old_list, new_list):
+    """
+    Used to assert that both of the lists contain the same values.
+    """
+
+    assert len(old_list) == len(new_list)
+    assert all([True for ol in old_list if ol in new_list])
+
+def legacy_assert_dir(old_result, new_result):
+    """
+    Used to test and ensure that the new implementation search results matches
+    the old implementation results. This test is intended to be used only to test
+    the old Augeas implementation results against the ParserNode Augeas implementation.
+
+    This test expects the DirectiveNode to have Augeas path in its attribute dict
+    "_metadata" with key "augpath".
+
+    """
+
+    if new_result != "CERTBOT_PASS_ASSERT":
+        return
+
+    assert len(old_result) == len(new_result)
+
+    matching = []
+
+    for oldres in old_result:
+        match = [res for res in new_result if res._metadata["augpath"] == oldres]
+        assert len(match) == 1
+
+
+def legacy_assert_args(old_result, new_result, parser):
+    """
+    Used to test and ensure that returned parameter values are the same for
+    the old and the new implementation.
+
+    Uses ApacheParser to actually fetch the parameter for the old result.
+
+    This assertion is structured this way because of how parser.get_arg() is
+    currently used in the ApacheConfigurator, making it easier to test the
+    results.
+    """
+
+    if isinstance(old_result, list):
+        for old in old_result:
+            oldarg = parser.get_arg(old_result)
+            assert oldarg in new_result.parameters
+    else:
+        oldarg = parser.get_arg(old_result)
+        assert oldarg in new_result.parameters
+
+def assert_dir(first, second):
+    """
+    Used to test that DirectiveNode results match for both implementations.
+    """
+
+    if "CERTBOT_PASS_ASSERT" in [first, second]:
+        return
+
+    assert first.name == second.name
+    assert first.parameters == second.parameters
+    assert first.dirty == second.dirty
+
+
+def assert_block(first, second):
+    """
+    Used to test that BlockNode results match for both implementations.
+    """
+
+    if "CERTBOT_PASS_ASSERT" in [first, second]:
+        return
+
+    assert first.name == second.name
+    assert first.parameters == second.parameters
+    assert len(first.children) == len(second.children)
+    assert first.dirty == second.dirty

certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test

@@ -3,6 +3,11 @@
 # A hackish script to see if the client is behaving as expected 
 # with each of the "passing" conf files.
 
+if [ -z "$SERVER" ]; then
+    echo "Please set SERVER to the ACME server's directory URL."
+    exit 1
+fi
+
 export EA=/etc/apache2/ 
 TESTDIR="`dirname $0`"
 cd $TESTDIR/passing
@@ -56,13 +61,16 @@ if [ "$1" = --debian-modules ] ; then
     done
 fi
 
+CERTBOT_CMD="sudo $(command -v certbot) --server $SERVER -vvvv"
+CERTBOT_CMD="$CERTBOT_CMD --debug --apache --register-unsafely-without-email"
+CERTBOT_CMD="$CERTBOT_CMD --agree-tos certonly -t --no-verify-ssl"
 
 FAILS=0
 trap CleanupExit INT
 for f in *.conf ; do 
     echo -n  testing "$f"...
     Setup
-    RESULT=`echo c | sudo $(command -v certbot) -vvvv --debug --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1`
+    RESULT=`echo c | $CERTBOT_CMD 2>&1`
     if echo $RESULT | grep -Eq \("Which names would you like"\|"mod_macro is not yet"\) ; then
         echo passed
     else

certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test-pebble.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+"""
+This executable script wraps the apache-conf-test bash script, in order to setup a pebble instance
+before its execution. Directory URL is passed through the SERVER environment variable.
+"""
+import os
+import subprocess
+import sys
+
+from certbot_integration_tests.utils import acme_server
+
+SCRIPT_DIRNAME = os.path.dirname(__file__)
+
+
+def main(args=None):
+    if not args:
+        args = sys.argv[1:]
+    with acme_server.ACMEServer('pebble', [], False) as acme_xdist:
+        environ = os.environ.copy()
+        environ['SERVER'] = acme_xdist['directory_url']
+        command = [os.path.join(SCRIPT_DIRNAME, 'apache-conf-test')]
+        command.extend(args)
+        return subprocess.call(command, env=environ)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

certbot-apache/certbot_apache/tests/apache-conf-files/passing/finalize-1243.conf

@@ -1,7 +1,7 @@
 #LoadModule ssl_module modules/mod_ssl.so
 
-Listen 443
-<VirtualHost *:443>
+Listen 4443
+<VirtualHost *:4443>
 	# The ServerName directive sets the request scheme, hostname and port that
 	# the server uses to identify itself. This is used when creating
 	# redirection URLs. In the context of virtual hosts, the ServerName

certbot-apache/certbot_apache/tests/autohsts_test.py

@@ -35,8 +35,9 @@ class AutoHSTSTest(util.ApacheTest):
             pat = '(?:[ "]|^)(strict-transport-security)(?:[ "]|$)'
             for head in header_path:
                 if re.search(pat, self.config.parser.aug.get(head).lower()):
-                    return self.config.parser.aug.get(head.replace("arg[3]",
-                                                                   "arg[4]"))
+                    return self.config.parser.aug.get(
+                        head.replace("arg[3]", "arg[4]"))
+        return None  # pragma: no cover
 
     @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
     @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod")

certbot-apache/certbot_apache/tests/centos6_test.py

@@ -0,0 +1,222 @@
+"""Test for certbot_apache.configurator for CentOS 6 overrides"""
+import unittest
+
+from certbot.compat import os
+from certbot.errors import MisconfigurationError
+
+from certbot_apache import obj
+from certbot_apache import override_centos
+from certbot_apache import parser
+from certbot_apache.tests import util
+
+
+def get_vh_truth(temp_dir, config_name):
+    """Return the ground truth for the specified directory."""
+    prefix = os.path.join(
+        temp_dir, config_name, "httpd/conf.d")
+
+    aug_pre = "/files" + prefix
+    vh_truth = [
+        obj.VirtualHost(
+            os.path.join(prefix, "test.example.com.conf"),
+            os.path.join(aug_pre, "test.example.com.conf/VirtualHost"),
+            set([obj.Addr.fromstring("*:80")]),
+            False, True, "test.example.com"),
+        obj.VirtualHost(
+            os.path.join(prefix, "ssl.conf"),
+            os.path.join(aug_pre, "ssl.conf/VirtualHost"),
+            set([obj.Addr.fromstring("_default_:443")]),
+            True, True, None)
+    ]
+    return vh_truth
+
+class CentOS6Tests(util.ApacheTest):
+    """Tests for CentOS 6"""
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        test_dir = "centos6_apache/apache"
+        config_root = "centos6_apache/apache/httpd"
+        vhost_root = "centos6_apache/apache/httpd/conf.d"
+        super(CentOS6Tests, self).setUp(test_dir=test_dir,
+                                        config_root=config_root,
+                                        vhost_root=vhost_root)
+
+        self.config = util.get_apache_configurator(
+            self.config_path, self.vhost_path, self.config_dir, self.work_dir,
+            version=(2, 2, 15), os_info="centos")
+        self.vh_truth = get_vh_truth(
+            self.temp_dir, "centos6_apache/apache")
+
+    def test_get_parser(self):
+        self.assertTrue(isinstance(self.config.parser,
+                                   override_centos.CentOSParser))
+
+    def test_get_virtual_hosts(self):
+        """Make sure all vhosts are being properly found."""
+        vhs = self.config.get_virtual_hosts()
+        self.assertEqual(len(vhs), 2)
+        found = 0
+
+        for vhost in vhs:
+            for centos_truth in self.vh_truth:
+                if vhost == centos_truth:
+                    found += 1
+                    break
+            else:
+                raise Exception("Missed: %s" % vhost)  # pragma: no cover
+        self.assertEqual(found, 2)
+
+    def test_loadmod_default(self):
+        ssl_loadmods = self.config.parser.find_dir(
+            "LoadModule", "ssl_module", exclude=False)
+        self.assertEqual(len(ssl_loadmods), 1)
+        # Make sure the LoadModule ssl_module is in ssl.conf (default)
+        self.assertTrue("ssl.conf" in ssl_loadmods[0])
+        # ...and that it's not inside of <IfModule>
+        self.assertFalse("IfModule" in ssl_loadmods[0])
+
+        # Get the example vhost
+        self.config.assoc["test.example.com"] = self.vh_truth[0]
+        self.config.deploy_cert(
+            "random.demo", "example/cert.pem", "example/key.pem",
+            "example/cert_chain.pem", "example/fullchain.pem")
+        self.config.save()
+
+        post_loadmods = self.config.parser.find_dir(
+            "LoadModule", "ssl_module", exclude=False)
+
+        # We should now have LoadModule ssl_module in root conf and ssl.conf
+        self.assertEqual(len(post_loadmods), 2)
+        for lm in post_loadmods:
+            # lm[:-7] removes "/arg[#]" from the path
+            arguments = self.config.parser.get_all_args(lm[:-7])
+            self.assertEqual(arguments, ["ssl_module", "modules/mod_ssl.so"])
+            # ...and both of them should be wrapped in <IfModule !mod_ssl.c>
+            # lm[:-17] strips off /directive/arg[1] from the path.
+            ifmod_args = self.config.parser.get_all_args(lm[:-17])
+            self.assertTrue("!mod_ssl.c" in ifmod_args)
+
+    def test_loadmod_multiple(self):
+        sslmod_args = ["ssl_module", "modules/mod_ssl.so"]
+        # Adds another LoadModule to main httpd.conf in addtition to ssl.conf
+        self.config.parser.add_dir(self.config.parser.loc["default"], "LoadModule",
+                                   sslmod_args)
+        self.config.save()
+        pre_loadmods = self.config.parser.find_dir(
+            "LoadModule", "ssl_module", exclude=False)
+        # LoadModules are not within IfModule blocks
+        self.assertFalse(any(["ifmodule" in m.lower() for m in pre_loadmods]))
+        self.config.assoc["test.example.com"] = self.vh_truth[0]
+        self.config.deploy_cert(
+            "random.demo", "example/cert.pem", "example/key.pem",
+            "example/cert_chain.pem", "example/fullchain.pem")
+        post_loadmods = self.config.parser.find_dir(
+            "LoadModule", "ssl_module", exclude=False)
+
+        for mod in post_loadmods:
+            self.assertTrue(self.config.parser.not_modssl_ifmodule(mod))  #pylint: disable=no-member
+
+    def test_loadmod_rootconf_exists(self):
+        sslmod_args = ["ssl_module", "modules/mod_ssl.so"]
+        rootconf_ifmod = self.config.parser.get_ifmod(
+            parser.get_aug_path(self.config.parser.loc["default"]),
+            "!mod_ssl.c", beginning=True)
+        self.config.parser.add_dir(rootconf_ifmod[:-1], "LoadModule", sslmod_args)
+        self.config.save()
+        # Get the example vhost
+        self.config.assoc["test.example.com"] = self.vh_truth[0]
+        self.config.deploy_cert(
+            "random.demo", "example/cert.pem", "example/key.pem",
+            "example/cert_chain.pem", "example/fullchain.pem")
+        self.config.save()
+
+        root_loadmods = self.config.parser.find_dir(
+            "LoadModule", "ssl_module",
+            start=parser.get_aug_path(self.config.parser.loc["default"]),
+            exclude=False)
+
+        mods = [lm for lm in root_loadmods if self.config.parser.loc["default"] in lm]
+
+        self.assertEqual(len(mods), 1)
+        # [:-7] removes "/arg[#]" from the path
+        self.assertEqual(
+            self.config.parser.get_all_args(mods[0][:-7]),
+            sslmod_args)
+
+    def test_neg_loadmod_already_on_path(self):
+        loadmod_args = ["ssl_module", "modules/mod_ssl.so"]
+        ifmod = self.config.parser.get_ifmod(
+            self.vh_truth[1].path, "!mod_ssl.c", beginning=True)
+        self.config.parser.add_dir(ifmod[:-1], "LoadModule", loadmod_args)
+        self.config.parser.add_dir(self.vh_truth[1].path, "LoadModule", loadmod_args)
+        self.config.save()
+        pre_loadmods = self.config.parser.find_dir(
+            "LoadModule", "ssl_module", start=self.vh_truth[1].path, exclude=False)
+        self.assertEqual(len(pre_loadmods), 2)
+        # The ssl.conf now has two LoadModule directives, one inside of
+        # !mod_ssl.c IfModule
+        self.config.assoc["test.example.com"] = self.vh_truth[0]
+        self.config.deploy_cert(
+            "random.demo", "example/cert.pem", "example/key.pem",
+            "example/cert_chain.pem", "example/fullchain.pem")
+        self.config.save()
+        # Ensure that the additional LoadModule wasn't written into the IfModule
+        post_loadmods = self.config.parser.find_dir(
+            "LoadModule", "ssl_module", start=self.vh_truth[1].path, exclude=False)
+        self.assertEqual(len(post_loadmods), 1)
+
+    def test_loadmod_non_duplicate(self):
+        # the modules/mod_ssl.so exists in ssl.conf
+        sslmod_args = ["ssl_module", "modules/mod_somethingelse.so"]
+        rootconf_ifmod = self.config.parser.get_ifmod(
+            parser.get_aug_path(self.config.parser.loc["default"]),
+            "!mod_ssl.c", beginning=True)
+        self.config.parser.add_dir(rootconf_ifmod[:-1], "LoadModule", sslmod_args)
+        self.config.save()
+        self.config.assoc["test.example.com"] = self.vh_truth[0]
+        pre_matches = self.config.parser.find_dir("LoadModule",
+                                                  "ssl_module", exclude=False)
+
+        self.assertRaises(MisconfigurationError, self.config.deploy_cert,
+                "random.demo", "example/cert.pem", "example/key.pem",
+                "example/cert_chain.pem", "example/fullchain.pem")
+
+        post_matches = self.config.parser.find_dir("LoadModule",
+                                                   "ssl_module", exclude=False)
+        # Make sure that none was changed
+        self.assertEqual(pre_matches, post_matches)
+
+    def test_loadmod_not_found(self):
+        # Remove all existing LoadModule ssl_module... directives
+        orig_loadmods = self.config.parser.find_dir("LoadModule",
+                                                    "ssl_module",
+                                                    exclude=False)
+        for mod in orig_loadmods:
+            noarg_path = mod.rpartition("/")[0]
+            self.config.parser.aug.remove(noarg_path)
+        self.config.save()
+        self.config.deploy_cert(
+            "random.demo", "example/cert.pem", "example/key.pem",
+            "example/cert_chain.pem", "example/fullchain.pem")
+
+        post_loadmods = self.config.parser.find_dir("LoadModule",
+                                                    "ssl_module",
+                                                    exclude=False)
+        self.assertFalse(post_loadmods)
+
+    def test_no_ifmod_search_false(self):
+        #pylint: disable=no-member
+
+        self.assertFalse(self.config.parser.not_modssl_ifmodule(
+            "/path/does/not/include/ifmod"
+        ))
+        self.assertFalse(self.config.parser.not_modssl_ifmodule(
+            ""
+        ))
+        self.assertFalse(self.config.parser.not_modssl_ifmodule(
+            "/path/includes/IfModule/but/no/arguments"
+        ))
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/centos_test.py

@@ -1,15 +1,17 @@
 """Test for certbot_apache.configurator for Centos overrides"""
-import os
 import unittest
 
 import mock
 
 from certbot import errors
+from certbot.compat import filesystem
+from certbot.compat import os
 
 from certbot_apache import obj
 from certbot_apache import override_centos
 from certbot_apache.tests import util
 
+
 def get_vh_truth(temp_dir, config_name):
     """Return the ground truth for the specified directory."""
     prefix = os.path.join(
@@ -30,6 +32,59 @@ def get_vh_truth(temp_dir, config_name):
     ]
     return vh_truth
 
+class FedoraRestartTest(util.ApacheTest):
+    """Tests for Fedora specific self-signed certificate override"""
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        test_dir = "centos7_apache/apache"
+        config_root = "centos7_apache/apache/httpd"
+        vhost_root = "centos7_apache/apache/httpd/conf.d"
+        super(FedoraRestartTest, self).setUp(test_dir=test_dir,
+                                             config_root=config_root,
+                                             vhost_root=vhost_root)
+        self.config = util.get_apache_configurator(
+            self.config_path, self.vhost_path, self.config_dir, self.work_dir,
+            os_info="fedora_old")
+        self.vh_truth = get_vh_truth(
+            self.temp_dir, "centos7_apache/apache")
+
+    def _run_fedora_test(self):
+        self.assertIsInstance(self.config, override_centos.CentOSConfigurator)
+        with mock.patch("certbot.util.get_os_info") as mock_info:
+            mock_info.return_value = ["fedora", "28"]
+            self.config.config_test()
+
+    def test_non_fedora_error(self):
+        c_test = "certbot_apache.configurator.ApacheConfigurator.config_test"
+        with mock.patch(c_test) as mock_test:
+            mock_test.side_effect = errors.MisconfigurationError
+            with mock.patch("certbot.util.get_os_info") as mock_info:
+                mock_info.return_value = ["not_fedora"]
+                self.assertRaises(errors.MisconfigurationError,
+                                  self.config.config_test)
+
+    def test_fedora_restart_error(self):
+        c_test = "certbot_apache.configurator.ApacheConfigurator.config_test"
+        with mock.patch(c_test) as mock_test:
+            # First call raises error, second doesn't
+            mock_test.side_effect = [errors.MisconfigurationError, '']
+            with mock.patch("certbot.util.run_script") as mock_run:
+                mock_run.side_effect = errors.SubprocessError
+                self.assertRaises(errors.MisconfigurationError,
+                                  self._run_fedora_test)
+
+    def test_fedora_restart(self):
+        c_test = "certbot_apache.configurator.ApacheConfigurator.config_test"
+        with mock.patch(c_test) as mock_test:
+            with mock.patch("certbot.util.run_script") as mock_run:
+                # First call raises error, second doesn't
+                mock_test.side_effect = [errors.MisconfigurationError, '']
+                self._run_fedora_test()
+                self.assertEqual(mock_test.call_count, 2)
+                self.assertEqual(mock_run.call_args[0][0],
+                                ['systemctl', 'restart', 'httpd'])
+
+
 class MultipleVhostsTestCentOS(util.ApacheTest):
     """Multiple vhost tests for CentOS / RHEL family of distros"""
 
@@ -50,8 +105,7 @@ class MultipleVhostsTestCentOS(util.ApacheTest):
             self.temp_dir, "centos7_apache/apache")
 
     def test_get_parser(self):
-        self.assertTrue(isinstance(self.config.parser,
-                                   override_centos.CentOSParser))
+        self.assertIsInstance(self.config.parser, override_centos.CentOSParser)
 
     @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg")
     def test_opportunistic_httpd_runtime_parsing(self, mock_get):
@@ -107,7 +161,7 @@ class MultipleVhostsTestCentOS(util.ApacheTest):
         """Make sure we read the sysconfig OPTIONS variable correctly"""
         # Return nothing for the process calls
         mock_cfg.return_value = ""
-        self.config.parser.sysconfig_filep = os.path.realpath(
+        self.config.parser.sysconfig_filep = filesystem.realpath(
             os.path.join(self.config.parser.root, "../sysconfig/httpd"))
         self.config.parser.variables = {}
 
@@ -136,6 +190,13 @@ class MultipleVhostsTestCentOS(util.ApacheTest):
                                        errors.SubprocessError]
         self.assertRaises(errors.MisconfigurationError, self.config.restart)
 
+    def test_pick_correct_tls_config(self):
+        self.config.version = (2, 4, 10)
+        self.assertTrue('centos-old' in self.config.pick_apache_config())
+
+        self.config.version = (2, 4, 11)
+        self.assertTrue('centos-current' in self.config.pick_apache_config())
+
 
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/complex_parsing_test.py

@@ -1,9 +1,9 @@
 """Tests for certbot_apache.parser."""
-import os
 import shutil
 import unittest
 
 from certbot import errors
+from certbot.compat import os
 
 from certbot_apache.tests import util
 
@@ -91,6 +91,7 @@ class ComplexParserTest(util.ParserTest):
         from certbot_apache import parser
         self.parser.add_dir(parser.get_aug_path(self.parser.loc["default"]),
                             "Include", [arg])
+        self.parser.handle_includes()
         if hit:
             self.assertTrue(self.parser.find_dir("FNMATCH_DIRECTIVE"))
         else:

certbot-apache/certbot_apache/tests/configurator_reverter_test.py

@@ -1,5 +1,4 @@
-"""Test for certbot_apache.augeas_configurator."""
-import os
+"""Test for certbot_apache.configurator implementations of reverter"""
 import shutil
 import unittest
 
@@ -10,12 +9,12 @@ from certbot import errors
 from certbot_apache.tests import util
 
 
-class AugeasConfiguratorTest(util.ApacheTest):
-    """Test for Augeas Configurator base class."""
+class ConfiguratorReverterTest(util.ApacheTest):
+    """Test for ApacheConfigurator reverter methods"""
 
 
     def setUp(self):  # pylint: disable=arguments-differ
-        super(AugeasConfiguratorTest, self).setUp()
+        super(ConfiguratorReverterTest, self).setUp()
 
         self.config = util.get_apache_configurator(
             self.config_path, self.vhost_path, self.config_dir, self.work_dir)
@@ -28,20 +27,6 @@ class AugeasConfiguratorTest(util.ApacheTest):
         shutil.rmtree(self.work_dir)
         shutil.rmtree(self.temp_dir)
 
-    def test_bad_parse(self):
-        # pylint: disable=protected-access
-        self.config.parser.parse_file(os.path.join(
-            self.config.parser.root, "conf-available", "bad_conf_file.conf"))
-        self.assertRaises(
-            errors.PluginError, self.config.check_parsing_errors, "httpd.aug")
-
-    def test_bad_save(self):
-        mock_save = mock.Mock()
-        mock_save.side_effect = IOError
-        self.config.aug.save = mock_save
-
-        self.assertRaises(errors.PluginError, self.config.save)
-
     def test_bad_save_checkpoint(self):
         self.config.reverter.add_to_checkpoint = mock.Mock(
             side_effect=errors.ReverterError)
@@ -63,23 +48,9 @@ class AugeasConfiguratorTest(util.ApacheTest):
 
         self.assertTrue(mock_finalize.is_called)
 
-    def test_recovery_routine(self):
-        mock_load = mock.Mock()
-        self.config.aug.load = mock_load
-
-        self.config.recovery_routine()
-        self.assertEqual(mock_load.call_count, 1)
-
-    def test_recovery_routine_error(self):
-        self.config.reverter.recovery_routine = mock.Mock(
-            side_effect=errors.ReverterError)
-
-        self.assertRaises(
-            errors.PluginError, self.config.recovery_routine)
-
     def test_revert_challenge_config(self):
         mock_load = mock.Mock()
-        self.config.aug.load = mock_load
+        self.config.parser.aug.load = mock_load
 
         self.config.revert_challenge_config()
         self.assertEqual(mock_load.call_count, 1)
@@ -93,7 +64,7 @@ class AugeasConfiguratorTest(util.ApacheTest):
 
     def test_rollback_checkpoints(self):
         mock_load = mock.Mock()
-        self.config.aug.load = mock_load
+        self.config.parser.aug.load = mock_load
 
         self.config.rollback_checkpoints()
         self.assertEqual(mock_load.call_count, 1)
@@ -103,13 +74,11 @@ class AugeasConfiguratorTest(util.ApacheTest):
             side_effect=errors.ReverterError)
         self.assertRaises(errors.PluginError, self.config.rollback_checkpoints)
 
-    def test_view_config_changes(self):
-        self.config.view_config_changes()
-
-    def test_view_config_changes_error(self):
-        self.config.reverter.view_config_changes = mock.Mock(
-            side_effect=errors.ReverterError)
-        self.assertRaises(errors.PluginError, self.config.view_config_changes)
+    def test_recovery_routine_reload(self):
+        mock_load = mock.Mock()
+        self.config.parser.aug.load = mock_load
+        self.config.recovery_routine()
+        self.assertEqual(mock_load.call_count, 1)
 
 
 if __name__ == "__main__":

certbot-apache/certbot_apache/tests/configurator_test.py

@@ -1,6 +1,6 @@
 # pylint: disable=too-many-public-methods,too-many-lines
 """Test for certbot_apache.configurator."""
-import os
+import copy
 import shutil
 import socket
 import tempfile
@@ -15,22 +15,21 @@ from acme import challenges
 from certbot import achallenges
 from certbot import crypto_util
 from certbot import errors
-
+from certbot.compat import os
+from certbot.compat import filesystem
 from certbot.tests import acme_util
 from certbot.tests import util as certbot_util
 
 from certbot_apache import apache_util
 from certbot_apache import constants
-from certbot_apache import parser
 from certbot_apache import obj
-
+from certbot_apache import parser
 from certbot_apache.tests import util
 
 
 class MultipleVhostsTest(util.ApacheTest):
     """Test two standard well-configured HTTP vhosts."""
 
-
     def setUp(self):  # pylint: disable=arguments-differ
         super(MultipleVhostsTest, self).setUp()
 
@@ -52,25 +51,14 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.deploy_cert = mocked_deploy_cert
         return self.config
 
-    @mock.patch("certbot_apache.configurator.ApacheConfigurator.init_augeas")
     @mock.patch("certbot_apache.configurator.path_surgery")
-    def test_prepare_no_install(self, mock_surgery, _init_augeas):
+    def test_prepare_no_install(self, mock_surgery):
         silly_path = {"PATH": "/tmp/nothingness2342"}
         mock_surgery.return_value = False
         with mock.patch.dict('os.environ', silly_path):
             self.assertRaises(errors.NoInstallationError, self.config.prepare)
             self.assertEqual(mock_surgery.call_count, 1)
 
-    @mock.patch("certbot_apache.augeas_configurator.AugeasConfigurator.init_augeas")
-    def test_prepare_no_augeas(self, mock_init_augeas):
-        """ Test augeas initialization ImportError """
-        def side_effect_error():
-            """ Side effect error for the test """
-            raise ImportError
-        mock_init_augeas.side_effect = side_effect_error
-        self.assertRaises(
-            errors.NoInstallationError, self.config.prepare)
-
     @mock.patch("certbot_apache.parser.ApacheParser")
     @mock.patch("certbot_apache.configurator.util.exe_exists")
     def test_prepare_version(self, mock_exe_exists, _):
@@ -82,16 +70,6 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertRaises(
             errors.NotSupportedError, self.config.prepare)
 
-    @mock.patch("certbot_apache.parser.ApacheParser")
-    @mock.patch("certbot_apache.configurator.util.exe_exists")
-    def test_prepare_old_aug(self, mock_exe_exists, _):
-        mock_exe_exists.return_value = True
-        self.config.config_test = mock.Mock()
-        # pylint: disable=protected-access
-        self.config._check_aug_version = mock.Mock(return_value=False)
-        self.assertRaises(
-            errors.NotSupportedError, self.config.prepare)
-
     def test_prepare_locked(self):
         server_root = self.config.conf("server-root")
         self.config.config_test = mock.Mock()
@@ -115,6 +93,37 @@ class MultipleVhostsTest(util.ApacheTest):
         # Weak test..
         ApacheConfigurator.add_parser_arguments(mock.MagicMock())
 
+    def test_docs_parser_arguments(self):
+        os.environ["CERTBOT_DOCS"] = "1"
+        from certbot_apache.configurator import ApacheConfigurator
+        mock_add = mock.MagicMock()
+        ApacheConfigurator.add_parser_arguments(mock_add)
+        parserargs = ["server_root", "enmod", "dismod", "le_vhost_ext",
+                      "vhost_root", "logs_root", "challenge_location",
+                      "handle_modules", "handle_sites", "ctl"]
+        exp = dict()
+
+        for k in ApacheConfigurator.OS_DEFAULTS:
+            if k in parserargs:
+                exp[k.replace("_", "-")] = ApacheConfigurator.OS_DEFAULTS[k]
+        # Special cases
+        exp["vhost-root"] = None
+        exp["init-script"] = None
+
+        found = set()
+        for call in mock_add.call_args_list:
+            # init-script is a special case: deprecated argument
+            if call[0][0] != "init-script":
+                self.assertEqual(exp[call[0][0]], call[1]['default'])
+            found.add(call[0][0])
+
+        # Make sure that all (and only) the expected values exist
+        self.assertEqual(len(mock_add.call_args_list), len(found))
+        for e in exp:
+            self.assertTrue(e in found)
+
+        del os.environ["CERTBOT_DOCS"]
+
     def test_add_parser_arguments_all_configurators(self):  # pylint: disable=no-self-use
         from certbot_apache.entrypoint import OVERRIDE_CLASSES
         for cls in OVERRIDE_CLASSES.values():
@@ -139,7 +148,8 @@ class MultipleVhostsTest(util.ApacheTest):
         names = self.config.get_all_names()
         self.assertEqual(names, set(
             ["certbot.demo", "ocspvhost.com", "encryption-example.demo",
-             "nonsym.link", "vhost.in.rootconf", "www.certbot.demo"]
+             "nonsym.link", "vhost.in.rootconf", "www.certbot.demo",
+             "duplicate.example.com"]
         ))
 
     @certbot_util.patch_get_utility()
@@ -158,8 +168,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.vhosts.append(vhost)
 
         names = self.config.get_all_names()
-        # Names get filtered, only 5 are returned
-        self.assertEqual(len(names), 8)
+        self.assertEqual(len(names), 9)
         self.assertTrue("zombo.com" in names)
         self.assertTrue("google.com" in names)
         self.assertTrue("certbot.demo" in names)
@@ -200,7 +209,7 @@ class MultipleVhostsTest(util.ApacheTest):
     def test_get_virtual_hosts(self):
         """Make sure all vhosts are being properly found."""
         vhs = self.config.get_virtual_hosts()
-        self.assertEqual(len(vhs), 10)
+        self.assertEqual(len(vhs), 12)
         found = 0
 
         for vhost in vhs:
@@ -211,7 +220,7 @@ class MultipleVhostsTest(util.ApacheTest):
             else:
                 raise Exception("Missed: %s" % vhost)  # pragma: no cover
 
-        self.assertEqual(found, 10)
+        self.assertEqual(found, 12)
 
         # Handle case of non-debian layout get_virtual_hosts
         with mock.patch(
@@ -219,7 +228,7 @@ class MultipleVhostsTest(util.ApacheTest):
         ) as mock_conf:
             mock_conf.return_value = False
             vhs = self.config.get_virtual_hosts()
-            self.assertEqual(len(vhs), 10)
+            self.assertEqual(len(vhs), 12)
 
     @mock.patch("certbot_apache.display_ops.select_vhost")
     def test_choose_vhost_none_avail(self, mock_select):
@@ -322,7 +331,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.vhosts = [
             vh for vh in self.config.vhosts
             if vh.name not in ["certbot.demo", "nonsym.link",
-                "encryption-example.demo",
+                "encryption-example.demo", "duplicate.example.com",
                 "ocspvhost.com", "vhost.in.rootconf"]
             and "*.blue.purple.com" not in vh.aliases
         ]
@@ -333,7 +342,7 @@ class MultipleVhostsTest(util.ApacheTest):
     def test_non_default_vhosts(self):
         # pylint: disable=protected-access
         vhosts = self.config._non_default_vhosts(self.config.vhosts)
-        self.assertEqual(len(vhosts), 8)
+        self.assertEqual(len(vhosts), 10)
 
     def test_deploy_cert_enable_new_vhost(self):
         # Create
@@ -353,6 +362,7 @@ class MultipleVhostsTest(util.ApacheTest):
             """Mock method for parser.find_dir"""
             if directive == "Include" and argument.endswith("options-ssl-apache.conf"):
                 return ["/path/to/whatever"]
+            return None  # pragma: no cover
 
         mock_add = mock.MagicMock()
         self.config.parser.add_dir = mock_add
@@ -464,8 +474,7 @@ class MultipleVhostsTest(util.ApacheTest):
             but an SSLCertificateKeyFile directive is missing."""
             if "SSLCertificateFile" in args:
                 return ["example/cert.pem"]
-            else:
-                return []
+            return []
 
         mock_find_dir = mock.MagicMock(return_value=[])
         mock_find_dir.side_effect = side_effect
@@ -645,7 +654,7 @@ class MultipleVhostsTest(util.ApacheTest):
         # span excludes the closing </VirtualHost> tag in older versions
         # of Augeas
         return_value = [self.vh_truth[0].filep, 1, 12, 0, 0, 0, 1142]
-        with mock.patch.object(self.config.aug, 'span') as mock_span:
+        with mock.patch.object(self.config.parser.aug, 'span') as mock_span:
             mock_span.return_value = return_value
             self.test_make_vhost_ssl()
 
@@ -653,7 +662,7 @@ class MultipleVhostsTest(util.ApacheTest):
         # span includes the closing </VirtualHost> tag in newer versions
         # of Augeas
         return_value = [self.vh_truth[0].filep, 1, 12, 0, 0, 0, 1157]
-        with mock.patch.object(self.config.aug, 'span') as mock_span:
+        with mock.patch.object(self.config.parser.aug, 'span') as mock_span:
             mock_span.return_value = return_value
             self.test_make_vhost_ssl()
 
@@ -666,8 +675,7 @@ class MultipleVhostsTest(util.ApacheTest):
     def test_make_vhost_ssl_nonexistent_vhost_path(self):
         ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[1])
         self.assertEqual(os.path.dirname(ssl_vhost.filep),
-                            os.path.dirname(os.path.realpath(
-                                self.vh_truth[1].filep)))
+                         os.path.dirname(filesystem.realpath(self.vh_truth[1].filep)))
 
     def test_make_vhost_ssl(self):
         ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])
@@ -688,7 +696,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertEqual(self.config.is_name_vhost(self.vh_truth[0]),
                          self.config.is_name_vhost(ssl_vhost))
 
-        self.assertEqual(len(self.config.vhosts), 11)
+        self.assertEqual(len(self.config.vhosts), 13)
 
     def test_clean_vhost_ssl(self):
         # pylint: disable=protected-access
@@ -781,32 +789,19 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertEqual(self.config.add_name_vhost.call_count, 2)
 
     @mock.patch("certbot_apache.configurator.http_01.ApacheHttp01.perform")
-    @mock.patch("certbot_apache.configurator.tls_sni_01.ApacheTlsSni01.perform")
     @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
-    def test_perform(self, mock_restart, mock_tls_perform, mock_http_perform):
+    def test_perform(self, mock_restart, mock_http_perform):
         # Only tests functionality specific to configurator.perform
         # Note: As more challenges are offered this will have to be expanded
         account_key, achalls = self.get_key_and_achalls()
 
-        all_expected = []
-        http_expected = []
-        tls_expected = []
-        for achall in achalls:
-            response = achall.response(account_key)
-            if isinstance(achall.chall, challenges.HTTP01):
-                http_expected.append(response)
-            else:
-                tls_expected.append(response)
-            all_expected.append(response)
-
-        mock_http_perform.return_value = http_expected
-        mock_tls_perform.return_value = tls_expected
+        expected = [achall.response(account_key) for achall in achalls]
+        mock_http_perform.return_value = expected
 
         responses = self.config.perform(achalls)
 
         self.assertEqual(mock_http_perform.call_count, 1)
-        self.assertEqual(mock_tls_perform.call_count, 1)
-        self.assertEqual(responses, all_expected)
+        self.assertEqual(responses, expected)
 
         self.assertEqual(mock_restart.call_count, 1)
 
@@ -1026,7 +1021,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
         # pylint: disable=protected-access
         http_vh = self.config._get_http_vhost(ssl_vh)
-        self.assertTrue(http_vh.ssl == False)
+        self.assertFalse(http_vh.ssl)
 
     @mock.patch("certbot.util.run_script")
     @mock.patch("certbot.util.exe_exists")
@@ -1215,7 +1210,7 @@ class MultipleVhostsTest(util.ApacheTest):
         except errors.PluginEnhancementAlreadyPresent:
             args_paths = self.config.parser.find_dir(
                 "RewriteRule", None, http_vhost.path, False)
-            arg_vals = [self.config.aug.get(x) for x in args_paths]
+            arg_vals = [self.config.parser.aug.get(x) for x in args_paths]
             self.assertEqual(arg_vals, constants.REWRITE_HTTPS_ARGS)
 
 
@@ -1269,7 +1264,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
         # pylint: disable=protected-access
         self.config._enable_redirect(self.vh_truth[1], "")
-        self.assertEqual(len(self.config.vhosts), 11)
+        self.assertEqual(len(self.config.vhosts), 13)
 
     def test_create_own_redirect_for_old_apache_version(self):
         self.config.parser.modules.add("rewrite_module")
@@ -1280,7 +1275,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
         # pylint: disable=protected-access
         self.config._enable_redirect(self.vh_truth[1], "")
-        self.assertEqual(len(self.config.vhosts), 11)
+        self.assertEqual(len(self.config.vhosts), 13)
 
     def test_sift_rewrite_rule(self):
         # pylint: disable=protected-access
@@ -1318,24 +1313,6 @@ class MultipleVhostsTest(util.ApacheTest):
 
         return account_key, (achall1, achall2, achall3)
 
-    def test_make_addrs_sni_ready(self):
-        self.config.version = (2, 2)
-        self.config.make_addrs_sni_ready(
-            set([obj.Addr.fromstring("*:443"), obj.Addr.fromstring("*:80")]))
-        self.assertTrue(self.config.parser.find_dir(
-            "NameVirtualHost", "*:80", exclude=False))
-        self.assertTrue(self.config.parser.find_dir(
-            "NameVirtualHost", "*:443", exclude=False))
-
-    def test_aug_version(self):
-        mock_match = mock.Mock(return_value=["something"])
-        self.config.aug.match = mock_match
-        # pylint: disable=protected-access
-        self.assertEqual(self.config._check_aug_version(),
-                         ["something"])
-        self.config.aug.match.side_effect = RuntimeError
-        self.assertFalse(self.config._check_aug_version())
-
     def test_enable_site_nondebian(self):
         inc_path = "/path/to/wherever"
         vhost = self.vh_truth[0]
@@ -1358,8 +1335,8 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.parser.modules.add("ssl_module")
         self.config.parser.modules.add("mod_ssl.c")
         self.config.parser.modules.add("socache_shmcb_module")
-        tmp_path = os.path.realpath(tempfile.mkdtemp("vhostroot"))
-        os.chmod(tmp_path, 0o755)
+        tmp_path = filesystem.realpath(tempfile.mkdtemp("vhostroot"))
+        filesystem.chmod(tmp_path, 0o755)
         mock_p = "certbot_apache.configurator.ApacheConfigurator._get_ssl_vhost_path"
         mock_a = "certbot_apache.parser.ApacheParser.add_include"
 
@@ -1391,7 +1368,7 @@ class MultipleVhostsTest(util.ApacheTest):
         # pylint: disable=protected-access
         cases = {u"*.example.org": True, b"*.x.example.org": True,
                  u"a.example.org": False, b"a.x.example.org": False}
-        for key in cases.keys():
+        for key in cases:
             self.assertEqual(self.config._wildcard_domain(key), cases[key])
 
     def test_choose_vhosts_wildcard(self):
@@ -1503,6 +1480,29 @@ class MultipleVhostsTest(util.ApacheTest):
         second_id = self.config.add_vhost_id(self.vh_truth[0])
         self.assertEqual(first_id, second_id)
 
+    def test_realpath_replaces_symlink(self):
+        orig_match = self.config.parser.aug.match
+        mock_vhost = copy.deepcopy(self.vh_truth[0])
+        mock_vhost.filep = mock_vhost.filep.replace('sites-enabled', u'sites-available')
+        mock_vhost.path = mock_vhost.path.replace('sites-enabled', 'sites-available')
+        mock_vhost.enabled = False
+        self.config.parser.parse_file(mock_vhost.filep)
+
+        def mock_match(aug_expr):
+            """Return a mocked match list of VirtualHosts"""
+            if "/mocked/path" in aug_expr:
+                return [self.vh_truth[1].path, self.vh_truth[0].path, mock_vhost.path]
+            return orig_match(aug_expr)
+
+        self.config.parser.parser_paths = ["/mocked/path"]
+        self.config.parser.aug.match = mock_match
+        vhs = self.config.get_virtual_hosts()
+        self.assertEqual(len(vhs), 2)
+        self.assertTrue(vhs[0] == self.vh_truth[1])
+        # mock_vhost should have replaced the vh_truth[0], because its filepath
+        # isn't a symlink
+        self.assertTrue(vhs[1] == mock_vhost)
+
 
 class AugeasVhostsTest(util.ApacheTest):
     """Test vhosts with illegal names dependent on augeas version."""
@@ -1521,16 +1521,16 @@ class AugeasVhostsTest(util.ApacheTest):
             self.work_dir)
 
     def test_choosevhost_with_illegal_name(self):
-        self.config.aug = mock.MagicMock()
-        self.config.aug.match.side_effect = RuntimeError
-        path = "debian_apache_2_4/augeas_vhosts/apache2/sites-available/old,default.conf"
+        self.config.parser.aug = mock.MagicMock()
+        self.config.parser.aug.match.side_effect = RuntimeError
+        path = "debian_apache_2_4/augeas_vhosts/apache2/sites-available/old-and-default.conf"
         chosen_vhost = self.config._create_vhost(path)
         self.assertEqual(None, chosen_vhost)
 
     def test_choosevhost_works(self):
-        path = "debian_apache_2_4/augeas_vhosts/apache2/sites-available/old,default.conf"
+        path = "debian_apache_2_4/augeas_vhosts/apache2/sites-available/old-and-default.conf"
         chosen_vhost = self.config._create_vhost(path)
-        self.assertTrue(chosen_vhost == None or chosen_vhost.path == path)
+        self.assertTrue(chosen_vhost is None or chosen_vhost.path == path)
 
     @mock.patch("certbot_apache.configurator.ApacheConfigurator._create_vhost")
     def test_get_vhost_continue(self, mock_vhost):
@@ -1706,7 +1706,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
                                              self.config.updated_mod_ssl_conf_digest)
 
     def _current_ssl_options_hash(self):
-        return crypto_util.sha256sum(self.config.option("MOD_SSL_CONF_SRC"))
+        return crypto_util.sha256sum(self.config.pick_apache_config())
 
     def _assert_current_file(self):
         self.assertTrue(os.path.isfile(self.config.mod_ssl_conf))
@@ -1742,7 +1742,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
             self.assertFalse(mock_logger.warning.called)
         self.assertTrue(os.path.isfile(self.config.mod_ssl_conf))
         self.assertEqual(crypto_util.sha256sum(
-            self.config.option("MOD_SSL_CONF_SRC")),
+            self.config.pick_apache_config()),
             self._current_ssl_options_hash())
         self.assertNotEqual(crypto_util.sha256sum(self.config.mod_ssl_conf),
             self._current_ssl_options_hash())
@@ -1758,18 +1758,31 @@ class InstallSslOptionsConfTest(util.ApacheTest):
                 "%s has been manually modified; updated file "
                 "saved to %s. We recommend updating %s for security purposes.")
         self.assertEqual(crypto_util.sha256sum(
-            self.config.option("MOD_SSL_CONF_SRC")),
+            self.config.pick_apache_config()),
             self._current_ssl_options_hash())
         # only print warning once
         with mock.patch("certbot.plugins.common.logger") as mock_logger:
             self._call()
             self.assertFalse(mock_logger.warning.called)
 
-    def test_current_file_hash_in_all_hashes(self):
+    def test_ssl_config_files_hash_in_all_hashes(self):
+        """
+        It is really critical that all TLS Apache config files have their SHA256 hash registered in
+        constants.ALL_SSL_OPTIONS_HASHES. Otherwise Certbot will mistakenly assume that the config
+        file has been manually edited by the user, and will refuse to update it.
+        This test ensures that all necessary hashes are present.
+        """
         from certbot_apache.constants import ALL_SSL_OPTIONS_HASHES
-        self.assertTrue(self._current_ssl_options_hash() in ALL_SSL_OPTIONS_HASHES,
-            "Constants.ALL_SSL_OPTIONS_HASHES must be appended"
-            " with the sha256 hash of self.config.mod_ssl_conf when it is updated.")
+        import pkg_resources
+        tls_configs_dir = pkg_resources.resource_filename("certbot_apache", "tls_configs")
+        all_files = [os.path.join(tls_configs_dir, name) for name in os.listdir(tls_configs_dir)
+                     if name.endswith('options-ssl-apache.conf')]
+        self.assertTrue(all_files)
+        for one_file in all_files:
+            file_hash = crypto_util.sha256sum(one_file)
+            self.assertTrue(file_hash in ALL_SSL_OPTIONS_HASHES,
+                            "Constants.ALL_SSL_OPTIONS_HASHES must be appended with the sha256 "
+                            "hash of {0} when it is updated.".format(one_file))
 
 
 if __name__ == "__main__":

certbot-apache/certbot_apache/tests/debian_test.py

@@ -1,11 +1,11 @@
 """Test for certbot_apache.configurator for Debian overrides"""
-import os
 import shutil
 import unittest
 
 import mock
 
 from certbot import errors
+from certbot.compat import os
 
 from certbot_apache import apache_util
 from certbot_apache import obj
@@ -79,9 +79,9 @@ class MultipleVhostsTestDebian(util.ApacheTest):
 
     def test_enable_site_failure(self):
         self.config.parser.root = "/tmp/nonexistent"
-        with mock.patch("os.path.isdir") as mock_dir:
+        with mock.patch("certbot.compat.os.path.isdir") as mock_dir:
             mock_dir.return_value = True
-            with mock.patch("os.path.islink") as mock_link:
+            with mock.patch("certbot.compat.os.path.islink") as mock_link:
                 mock_link.return_value = False
                 self.assertRaises(
                     errors.NotSupportedError,

certbot-apache/certbot_apache/tests/entrypoint_test.py

@@ -6,6 +6,7 @@ import mock
 from certbot_apache import configurator
 from certbot_apache import entrypoint
 
+
 class EntryPointTest(unittest.TestCase):
     """Entrypoint tests"""
 
@@ -14,8 +15,13 @@ class EntryPointTest(unittest.TestCase):
     def test_get_configurator(self):
 
         with mock.patch("certbot.util.get_os_info") as mock_info:
-            for distro in entrypoint.OVERRIDE_CLASSES.keys():
-                mock_info.return_value = (distro, "whatever")
+            for distro in entrypoint.OVERRIDE_CLASSES:
+                return_value = (distro, "whatever")
+                if distro == 'fedora_old':
+                    return_value = ('fedora', '28')
+                elif distro == 'fedora':
+                    return_value = ('fedora', '29')
+                mock_info.return_value = return_value
                 self.assertEqual(entrypoint.get_configurator(),
                                  entrypoint.OVERRIDE_CLASSES[distro])
 
@@ -23,7 +29,7 @@ class EntryPointTest(unittest.TestCase):
         with mock.patch("certbot.util.get_os_info") as mock_info:
             mock_info.return_value = ("nonexistent", "irrelevant")
             with mock.patch("certbot.util.get_systemd_os_like") as mock_like:
-                for like in entrypoint.OVERRIDE_CLASSES.keys():
+                for like in entrypoint.OVERRIDE_CLASSES:
                     mock_like.return_value = [like]
                     self.assertEqual(entrypoint.get_configurator(),
                                      entrypoint.OVERRIDE_CLASSES[like])

certbot-apache/certbot_apache/tests/fedora_test.py

@@ -0,0 +1,195 @@
+"""Test for certbot_apache.configurator for Fedora 29+ overrides"""
+import unittest
+
+import mock
+
+from certbot import errors
+from certbot.compat import filesystem
+from certbot.compat import os
+
+from certbot_apache import obj
+from certbot_apache import override_fedora
+from certbot_apache.tests import util
+
+
+def get_vh_truth(temp_dir, config_name):
+    """Return the ground truth for the specified directory."""
+    prefix = os.path.join(
+        temp_dir, config_name, "httpd/conf.d")
+
+    aug_pre = "/files" + prefix
+    # TODO: eventually, these tests should have a dedicated configuration instead
+    #  of reusing the ones from centos_test
+    vh_truth = [
+        obj.VirtualHost(
+            os.path.join(prefix, "centos.example.com.conf"),
+            os.path.join(aug_pre, "centos.example.com.conf/VirtualHost"),
+            {obj.Addr.fromstring("*:80")},
+            False, True, "centos.example.com"),
+        obj.VirtualHost(
+            os.path.join(prefix, "ssl.conf"),
+            os.path.join(aug_pre, "ssl.conf/VirtualHost"),
+            {obj.Addr.fromstring("_default_:443")},
+            True, True, None)
+    ]
+    return vh_truth
+
+
+class FedoraRestartTest(util.ApacheTest):
+    """Tests for Fedora specific self-signed certificate override"""
+
+    # TODO: eventually, these tests should have a dedicated configuration instead
+    #  of reusing the ones from centos_test
+    def setUp(self):  # pylint: disable=arguments-differ
+        test_dir = "centos7_apache/apache"
+        config_root = "centos7_apache/apache/httpd"
+        vhost_root = "centos7_apache/apache/httpd/conf.d"
+        super(FedoraRestartTest, self).setUp(test_dir=test_dir,
+                                             config_root=config_root,
+                                             vhost_root=vhost_root)
+        self.config = util.get_apache_configurator(
+            self.config_path, self.vhost_path, self.config_dir, self.work_dir,
+            os_info="fedora")
+        self.vh_truth = get_vh_truth(
+            self.temp_dir, "centos7_apache/apache")
+
+    def _run_fedora_test(self):
+        self.assertIsInstance(self.config, override_fedora.FedoraConfigurator)
+        self.config.config_test()
+
+    def test_fedora_restart_error(self):
+        c_test = "certbot_apache.configurator.ApacheConfigurator.config_test"
+        with mock.patch(c_test) as mock_test:
+            # First call raises error, second doesn't
+            mock_test.side_effect = [errors.MisconfigurationError, '']
+            with mock.patch("certbot.util.run_script") as mock_run:
+                mock_run.side_effect = errors.SubprocessError
+                self.assertRaises(errors.MisconfigurationError,
+                                  self._run_fedora_test)
+
+    def test_fedora_restart(self):
+        c_test = "certbot_apache.configurator.ApacheConfigurator.config_test"
+        with mock.patch(c_test) as mock_test:
+            with mock.patch("certbot.util.run_script") as mock_run:
+                # First call raises error, second doesn't
+                mock_test.side_effect = [errors.MisconfigurationError, '']
+                self._run_fedora_test()
+                self.assertEqual(mock_test.call_count, 2)
+                self.assertEqual(mock_run.call_args[0][0],
+                                ['systemctl', 'restart', 'httpd'])
+
+
+class MultipleVhostsTestFedora(util.ApacheTest):
+    """Multiple vhost tests for CentOS / RHEL family of distros"""
+
+    _multiprocess_can_split_ = True
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        test_dir = "centos7_apache/apache"
+        config_root = "centos7_apache/apache/httpd"
+        vhost_root = "centos7_apache/apache/httpd/conf.d"
+        super(MultipleVhostsTestFedora, self).setUp(test_dir=test_dir,
+                                                    config_root=config_root,
+                                                    vhost_root=vhost_root)
+
+        self.config = util.get_apache_configurator(
+            self.config_path, self.vhost_path, self.config_dir, self.work_dir,
+            os_info="fedora")
+        self.vh_truth = get_vh_truth(
+            self.temp_dir, "centos7_apache/apache")
+
+    def test_get_parser(self):
+        self.assertIsInstance(self.config.parser, override_fedora.FedoraParser)
+
+    @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg")
+    def test_opportunistic_httpd_runtime_parsing(self, mock_get):
+        define_val = (
+            'Define: TEST1\n'
+            'Define: TEST2\n'
+            'Define: DUMP_RUN_CFG\n'
+        )
+        mod_val = (
+            'Loaded Modules:\n'
+            ' mock_module (static)\n'
+            ' another_module (static)\n'
+        )
+        def mock_get_cfg(command):
+            """Mock httpd process stdout"""
+            if command == ['httpd', '-t', '-D', 'DUMP_RUN_CFG']:
+                return define_val
+            elif command == ['httpd', '-t', '-D', 'DUMP_MODULES']:
+                return mod_val
+            return ""
+        mock_get.side_effect = mock_get_cfg
+        self.config.parser.modules = set()
+        self.config.parser.variables = {}
+
+        with mock.patch("certbot.util.get_os_info") as mock_osi:
+            # Make sure we have the have the CentOS httpd constants
+            mock_osi.return_value = ("fedora", "29")
+            self.config.parser.update_runtime_variables()
+
+        self.assertEqual(mock_get.call_count, 3)
+        self.assertEqual(len(self.config.parser.modules), 4)
+        self.assertEqual(len(self.config.parser.variables), 2)
+        self.assertTrue("TEST2" in self.config.parser.variables.keys())
+        self.assertTrue("mod_another.c" in self.config.parser.modules)
+
+    @mock.patch("certbot_apache.configurator.util.run_script")
+    def test_get_version(self, mock_run_script):
+        mock_run_script.return_value = ('', None)
+        self.assertRaises(errors.PluginError, self.config.get_version)
+        self.assertEqual(mock_run_script.call_args[0][0][0], 'httpd')
+
+    def test_get_virtual_hosts(self):
+        """Make sure all vhosts are being properly found."""
+        vhs = self.config.get_virtual_hosts()
+        self.assertEqual(len(vhs), 2)
+        found = 0
+
+        for vhost in vhs:
+            for centos_truth in self.vh_truth:
+                if vhost == centos_truth:
+                    found += 1
+                    break
+            else:
+                raise Exception("Missed: %s" % vhost)  # pragma: no cover
+        self.assertEqual(found, 2)
+
+    @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg")
+    def test_get_sysconfig_vars(self, mock_cfg):
+        """Make sure we read the sysconfig OPTIONS variable correctly"""
+        # Return nothing for the process calls
+        mock_cfg.return_value = ""
+        self.config.parser.sysconfig_filep = filesystem.realpath(
+            os.path.join(self.config.parser.root, "../sysconfig/httpd"))
+        self.config.parser.variables = {}
+
+        with mock.patch("certbot.util.get_os_info") as mock_osi:
+            # Make sure we have the have the CentOS httpd constants
+            mock_osi.return_value = ("fedora", "29")
+            self.config.parser.update_runtime_variables()
+
+        self.assertTrue("mock_define" in self.config.parser.variables.keys())
+        self.assertTrue("mock_define_too" in self.config.parser.variables.keys())
+        self.assertTrue("mock_value" in self.config.parser.variables.keys())
+        self.assertEqual("TRUE", self.config.parser.variables["mock_value"])
+        self.assertTrue("MOCK_NOSEP" in self.config.parser.variables.keys())
+        self.assertEqual("NOSEP_VAL", self.config.parser.variables["NOSEP_TWO"])
+
+    @mock.patch("certbot_apache.configurator.util.run_script")
+    def test_alt_restart_works(self, mock_run_script):
+        mock_run_script.side_effect = [None, errors.SubprocessError, None]
+        self.config.restart()
+        self.assertEqual(mock_run_script.call_count, 3)
+
+    @mock.patch("certbot_apache.configurator.util.run_script")
+    def test_alt_restart_errors(self, mock_run_script):
+        mock_run_script.side_effect = [None,
+                                       errors.SubprocessError,
+                                       errors.SubprocessError]
+        self.assertRaises(errors.MisconfigurationError, self.config.restart)
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/gentoo_test.py

@@ -1,15 +1,17 @@
 """Test for certbot_apache.configurator for Gentoo overrides"""
-import os
 import unittest
 
 import mock
 
 from certbot import errors
+from certbot.compat import filesystem
+from certbot.compat import os
 
-from certbot_apache import override_gentoo
 from certbot_apache import obj
+from certbot_apache import override_gentoo
 from certbot_apache.tests import util
 
+
 def get_vh_truth(temp_dir, config_name):
     """Return the ground truth for the specified directory."""
     prefix = os.path.join(
@@ -80,7 +82,7 @@ class MultipleVhostsTestGentoo(util.ApacheTest):
         """Make sure we read the Gentoo APACHE2_OPTS variable correctly"""
         defines = ['DEFAULT_VHOST', 'INFO',
                    'SSL', 'SSL_DEFAULT_VHOST', 'LANGUAGE']
-        self.config.parser.apacheconfig_filep = os.path.realpath(
+        self.config.parser.apacheconfig_filep = filesystem.realpath(
             os.path.join(self.config.parser.root, "../conf.d/apache2"))
         self.config.parser.variables = {}
         with mock.patch("certbot_apache.override_gentoo.GentooParser.update_modules"):
@@ -113,6 +115,7 @@ class MultipleVhostsTestGentoo(util.ApacheTest):
             """Mock httpd process stdout"""
             if command == ['apache2ctl', 'modules']:
                 return mod_val
+            return None  # pragma: no cover
         mock_get.side_effect = mock_get_cfg
         self.config.parser.modules = set()
 

certbot-apache/certbot_apache/tests/http_01_test.py

@@ -1,15 +1,15 @@
 """Test for certbot_apache.http_01."""
-import mock
-import os
 import unittest
+import mock
 
 from acme import challenges
 from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
 
 from certbot import achallenges
 from certbot import errors
-
+from certbot.compat import os
 from certbot.tests import acme_util
+
 from certbot_apache.parser import get_aug_path
 from certbot_apache.tests import util
 
@@ -20,15 +20,15 @@ NUM_ACHALLS = 3
 class ApacheHttp01Test(util.ApacheTest):
     """Test for certbot_apache.http_01.ApacheHttp01."""
 
-    def setUp(self, *args, **kwargs):
+    def setUp(self, *args, **kwargs):  # pylint: disable=arguments-differ
         super(ApacheHttp01Test, self).setUp(*args, **kwargs)
 
         self.account_key = self.rsa512jwk
         self.achalls = []  # type: List[achallenges.KeyAuthorizationAnnotatedChallenge]
         vh_truth = util.get_vh_truth(
             self.temp_dir, "debian_apache_2_4/multiple_vhosts")
-        # Takes the vhosts for encryption-example.demo, certbot.demo, and
-        # vhost.in.rootconf
+        # Takes the vhosts for encryption-example.demo, certbot.demo
+        # and vhost.in.rootconf
         self.vhosts = [vh_truth[0], vh_truth[3], vh_truth[10]]
 
         for i in range(NUM_ACHALLS):
@@ -39,7 +39,7 @@ class ApacheHttp01Test(util.ApacheTest):
                         "pending"),
                     domain=self.vhosts[i].name, account_key=self.account_key))
 
-        modules = ["rewrite", "authz_core", "authz_host"]
+        modules = ["ssl", "rewrite", "authz_core", "authz_host"]
         for mod in modules:
             self.config.parser.modules.add("mod_{0}.c".format(mod))
             self.config.parser.modules.add(mod + "_module")
@@ -78,7 +78,7 @@ class ApacheHttp01Test(util.ApacheTest):
         calls = mock_enmod.call_args_list
         other_calls = []
         for call in calls:
-            if "rewrite" != call[0][0]:
+            if call[0][0] != "rewrite":
                 other_calls.append(call)
 
         # If these lists are equal, we never enabled mod_rewrite
@@ -111,6 +111,17 @@ class ApacheHttp01Test(util.ApacheTest):
                 domain="something.nonexistent", account_key=self.account_key)]
         self.common_perform_test(achalls, vhosts)
 
+    def test_configure_multiple_vhosts(self):
+        vhosts = [v for v in self.config.vhosts if "duplicate.example.com" in v.get_names()]
+        self.assertEqual(len(vhosts), 2)
+        achalls = [
+            achallenges.KeyAuthorizationAnnotatedChallenge(
+                challb=acme_util.chall_to_challb(
+                    challenges.HTTP01(token=((b'a' * 16))),
+                    "pending"),
+                domain="duplicate.example.com", account_key=self.account_key)]
+        self.common_perform_test(achalls, vhosts)
+
     def test_no_vhost(self):
         for achall in self.achalls:
             self.http.add_chall(achall)
@@ -176,15 +187,14 @@ class ApacheHttp01Test(util.ApacheTest):
             self._test_challenge_file(achall)
 
         for vhost in vhosts:
-            if not vhost.ssl:
-                matches = self.config.parser.find_dir("Include",
-                                                      self.http.challenge_conf_pre,
-                                                      vhost.path)
-                self.assertEqual(len(matches), 1)
-                matches = self.config.parser.find_dir("Include",
-                                                      self.http.challenge_conf_post,
-                                                      vhost.path)
-                self.assertEqual(len(matches), 1)
+            matches = self.config.parser.find_dir("Include",
+                                                self.http.challenge_conf_pre,
+                                                vhost.path)
+            self.assertEqual(len(matches), 1)
+            matches = self.config.parser.find_dir("Include",
+                                                self.http.challenge_conf_post,
+                                                vhost.path)
+            self.assertEqual(len(matches), 1)
 
         self.assertTrue(os.path.exists(challenge_dir))
 

certbot-apache/certbot_apache/tests/parser_test.py

@@ -1,12 +1,12 @@
+# pylint: disable=too-many-public-methods
 """Tests for certbot_apache.parser."""
-import os
 import shutil
 import unittest
 
-import augeas
 import mock
 
 from certbot import errors
+from certbot.compat import os
 
 from certbot_apache.tests import util
 
@@ -22,6 +22,27 @@ class BasicParserTest(util.ParserTest):
         shutil.rmtree(self.config_dir)
         shutil.rmtree(self.work_dir)
 
+    def test_bad_parse(self):
+        self.parser.parse_file(os.path.join(self.parser.root,
+                                            "conf-available", "bad_conf_file.conf"))
+        self.assertRaises(
+            errors.PluginError, self.parser.check_parsing_errors, "httpd.aug")
+
+    def test_bad_save(self):
+        mock_save = mock.Mock()
+        mock_save.side_effect = IOError
+        self.parser.aug.save = mock_save
+        self.assertRaises(errors.PluginError, self.parser.unsaved_files)
+
+    def test_aug_version(self):
+        mock_match = mock.Mock(return_value=["something"])
+        self.parser.aug.match = mock_match
+        # pylint: disable=protected-access
+        self.assertEqual(self.parser.check_aug_version(),
+                         ["something"])
+        self.parser.aug.match.side_effect = RuntimeError
+        self.assertFalse(self.parser.check_aug_version())
+
     def test_find_config_root_no_root(self):
         # pylint: disable=protected-access
         os.remove(self.parser.loc["root"])
@@ -52,7 +73,7 @@ class BasicParserTest(util.ParserTest):
         test2 = self.parser.find_dir("documentroot")
 
         self.assertEqual(len(test), 1)
-        self.assertEqual(len(test2), 7)
+        self.assertEqual(len(test2), 8)
 
     def test_add_dir(self):
         aug_default = "/files" + self.parser.loc["default"]
@@ -234,6 +255,7 @@ class BasicParserTest(util.ParserTest):
                 return inc_val
             elif cmd[-1] == "DUMP_MODULES":
                 return mod_val
+            return None  # pragma: no cover
 
         mock_cfg.side_effect = mock_get_vars
 
@@ -310,21 +332,38 @@ class BasicParserTest(util.ParserTest):
 class ParserInitTest(util.ApacheTest):
     def setUp(self):  # pylint: disable=arguments-differ
         super(ParserInitTest, self).setUp()
-        self.aug = augeas.Augeas(
-            flags=augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD)
 
     def tearDown(self):
         shutil.rmtree(self.temp_dir)
         shutil.rmtree(self.config_dir)
         shutil.rmtree(self.work_dir)
 
+    @mock.patch("certbot_apache.parser.ApacheParser.init_augeas")
+    def test_prepare_no_augeas(self, mock_init_augeas):
+        from certbot_apache.parser import ApacheParser
+        mock_init_augeas.side_effect = errors.NoInstallationError
+        self.config.config_test = mock.Mock()
+        self.assertRaises(
+            errors.NoInstallationError, ApacheParser,
+            os.path.relpath(self.config_path), "/dummy/vhostpath",
+            version=(2, 4, 22), configurator=self.config)
+
+    def test_init_old_aug(self):
+        from certbot_apache.parser import ApacheParser
+        with mock.patch("certbot_apache.parser.ApacheParser.check_aug_version") as mock_c:
+            mock_c.return_value = False
+            self.assertRaises(
+                errors.NotSupportedError,
+                ApacheParser, os.path.relpath(self.config_path),
+                "/dummy/vhostpath", version=(2, 4, 22), configurator=self.config)
+
     @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg")
     def test_unparseable(self, mock_cfg):
         from certbot_apache.parser import ApacheParser
         mock_cfg.return_value = ('Define: TEST')
         self.assertRaises(
             errors.PluginError,
-            ApacheParser, self.aug, os.path.relpath(self.config_path),
+            ApacheParser, os.path.relpath(self.config_path),
             "/dummy/vhostpath", version=(2, 2, 22), configurator=self.config)
 
     def test_root_normalized(self):
@@ -336,8 +375,7 @@ class ParserInitTest(util.ApacheTest):
                 self.temp_dir,
                 "debian_apache_2_4/////multiple_vhosts/../multiple_vhosts/apache2")
 
-            parser = ApacheParser(self.aug, path,
-                                  "/dummy/vhostpath", configurator=self.config)
+            parser = ApacheParser(path, "/dummy/vhostpath", configurator=self.config)
 
         self.assertEqual(parser.root, self.config_path)
 
@@ -346,7 +384,7 @@ class ParserInitTest(util.ApacheTest):
         with mock.patch("certbot_apache.parser.ApacheParser."
                         "update_runtime_variables"):
             parser = ApacheParser(
-                self.aug, os.path.relpath(self.config_path),
+                os.path.relpath(self.config_path),
                 "/dummy/vhostpath", configurator=self.config)
 
         self.assertEqual(parser.root, self.config_path)
@@ -356,7 +394,7 @@ class ParserInitTest(util.ApacheTest):
         with mock.patch("certbot_apache.parser.ApacheParser."
                         "update_runtime_variables"):
             parser = ApacheParser(
-                self.aug, self.config_path + os.path.sep,
+                self.config_path + os.path.sep,
                 "/dummy/vhostpath", configurator=self.config)
         self.assertEqual(parser.root, self.config_path)
 

certbot-apache/certbot_apache/tests/parsernode_test.py

@@ -0,0 +1,86 @@
+""" Tests for ParserNode interface """
+
+import unittest
+
+from acme.magic_typing import Optional, Tuple  # pylint: disable=unused-import, no-name-in-module
+
+from certbot_apache import interfaces
+
+
+
+class DummyCommentNode(interfaces.CommentNode):
+    """ A dummy class implementing CommentNode interface """
+    ancestor = None
+    comment = ""
+    dirty = False
+
+    def save(self, msg):  # pragma: no cover
+        pass
+
+
+class DummyDirectiveNode(interfaces.DirectiveNode):
+    """ A dummy class implementing DirectiveNode interface """
+    ancestor = None
+    parameters = tuple()  # type: Tuple[str, ...]
+    dirty = False
+    enabled = True
+    name = ""
+
+    def save(self, msg):  # pragma: no cover
+        pass
+
+    def set_parameters(self, parameters):  # pragma: no cover
+        pass
+
+
+class DummyBlockNode(interfaces.BlockNode):
+    """ A dummy class implementing BlockNode interface """
+    ancestor = None
+    parameters = tuple()  # type: Tuple[str, ...]
+    children = tuple()  # type: Tuple[interfaces.ParserNode, ...]
+    dirty = False
+    enabled = True
+    name = ""
+
+    def save(self, msg):  # pragma: no cover
+        pass
+
+    def add_child_block(self, name, arguments=None, position=None):  # pragma: no cover
+        pass
+
+    def add_child_directive(self, name, arguments=None, position=None):  # pragma: no cover
+        pass
+
+    def add_child_comment(self, comment="", position=None):  # pragma: no cover
+        pass
+
+    def find_blocks(self, name, exclude=True):  # pragma: no cover
+        pass
+
+    def find_directives(self, name, exclude=True):  # pragma: no cover
+        pass
+
+    def find_comments(self, comment, exact=False):  # pragma: no cover
+        pass
+
+    def delete_child(self, child):  # pragma: no cover
+        pass
+
+    def set_parameters(self, parameters):  # pragma: no cover
+        pass
+
+    def unsaved_files(self):  # pragma: no cover
+        pass
+
+
+class ParserNodeTest(unittest.TestCase):
+    """Dummy placeholder test case for ParserNode interfaces"""
+
+    def test_dummy(self):
+        dummyblock = DummyBlockNode()
+        dummydirective = DummyDirectiveNode()
+        dummycomment = DummyCommentNode()
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/README

@@ -0,0 +1,9 @@
+
+This directory holds Apache 2.0 module-specific configuration files;
+any files in this directory which have the ".conf" extension will be
+processed as Apache configuration files.
+
+Files are processed in alphabetical order, so if using configuration
+directives which depend on, say, mod_perl being loaded, ensure that
+these are placed in a filename later in the sort order than "perl.conf".
+

certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/ssl.conf

@@ -0,0 +1,222 @@
+#
+# This is the Apache server configuration file providing SSL support.
+# It contains the configuration directives to instruct the server how to
+# serve pages over an https connection. For detailing information about these 
+# directives see <URL:http://httpd.apache.org/docs/2.2/mod/mod_ssl.html>
+# 
+# Do NOT simply read the instructions in here without understanding
+# what they do.  They're here only as hints or reminders.  If you are unsure
+# consult the online docs. You have been warned.  
+#
+
+LoadModule ssl_module modules/mod_ssl.so
+
+#
+# When we also provide SSL we have to listen to the 
+# the HTTPS port in addition.
+#
+Listen 443
+
+##
+##  SSL Global Context
+##
+##  All SSL configuration in this context applies both to
+##  the main server and all SSL-enabled virtual hosts.
+##
+
+#   Pass Phrase Dialog:
+#   Configure the pass phrase gathering process.
+#   The filtering dialog program (`builtin' is a internal
+#   terminal dialog) has to provide the pass phrase on stdout.
+SSLPassPhraseDialog  builtin
+
+#   Inter-Process Session Cache:
+#   Configure the SSL Session Cache: First the mechanism 
+#   to use and second the expiring timeout (in seconds).
+SSLSessionCache         shmcb:/var/cache/mod_ssl/scache(512000)
+SSLSessionCacheTimeout  300
+
+#   Semaphore:
+#   Configure the path to the mutual exclusion semaphore the
+#   SSL engine uses internally for inter-process synchronization. 
+SSLMutex default
+
+#   Pseudo Random Number Generator (PRNG):
+#   Configure one or more sources to seed the PRNG of the 
+#   SSL library. The seed data should be of good random quality.
+#   WARNING! On some platforms /dev/random blocks if not enough entropy
+#   is available. This means you then cannot use the /dev/random device
+#   because it would lead to very long connection times (as long as
+#   it requires to make more entropy available). But usually those
+#   platforms additionally provide a /dev/urandom device which doesn't
+#   block. So, if available, use this one instead. Read the mod_ssl User
+#   Manual for more details.
+SSLRandomSeed startup file:/dev/urandom  256
+SSLRandomSeed connect builtin
+#SSLRandomSeed startup file:/dev/random  512
+#SSLRandomSeed connect file:/dev/random  512
+#SSLRandomSeed connect file:/dev/urandom 512
+
+#
+# Use "SSLCryptoDevice" to enable any supported hardware
+# accelerators. Use "openssl engine -v" to list supported
+# engine names.  NOTE: If you enable an accelerator and the
+# server does not start, consult the error logs and ensure
+# your accelerator is functioning properly. 
+#
+SSLCryptoDevice builtin
+#SSLCryptoDevice ubsec
+
+##
+## SSL Virtual Host Context
+##
+
+<VirtualHost _default_:443>
+
+# General setup for the virtual host, inherited from global configuration
+#DocumentRoot "/var/www/html"
+#ServerName www.example.com:443
+
+# Use separate log files for the SSL virtual host; note that LogLevel
+# is not inherited from httpd.conf.
+ErrorLog logs/ssl_error_log
+TransferLog logs/ssl_access_log
+LogLevel warn
+
+#   SSL Engine Switch:
+#   Enable/Disable SSL for this virtual host.
+SSLEngine on
+
+#   SSL Protocol support:
+# List the enable protocol levels with which clients will be able to
+# connect.  Disable SSLv2 access by default:
+SSLProtocol all -SSLv2
+
+#   SSL Cipher Suite:
+# List the ciphers that the client is permitted to negotiate.
+# See the mod_ssl documentation for a complete list.
+SSLCipherSuite DEFAULT:!EXP:!SSLv2:!DES:!IDEA:!SEED:+3DES
+
+#   Server Certificate:
+# Point SSLCertificateFile at a PEM encoded certificate.  If
+# the certificate is encrypted, then you will be prompted for a
+# pass phrase.  Note that a kill -HUP will prompt again.  A new
+# certificate can be generated using the genkey(1) command.
+SSLCertificateFile /etc/pki/tls/certs/localhost.crt
+
+#   Server Private Key:
+#   If the key is not combined with the certificate, use this
+#   directive to point at the key file.  Keep in mind that if
+#   you've both a RSA and a DSA private key you can configure
+#   both in parallel (to also allow the use of DSA ciphers, etc.)
+SSLCertificateKeyFile /etc/pki/tls/private/localhost.key
+
+#   Server Certificate Chain:
+#   Point SSLCertificateChainFile at a file containing the
+#   concatenation of PEM encoded CA certificates which form the
+#   certificate chain for the server certificate. Alternatively
+#   the referenced file can be the same as SSLCertificateFile
+#   when the CA certificates are directly appended to the server
+#   certificate for convinience.
+#SSLCertificateChainFile /etc/pki/tls/certs/server-chain.crt
+
+#   Certificate Authority (CA):
+#   Set the CA certificate verification path where to find CA
+#   certificates for client authentication or alternatively one
+#   huge file containing all of them (file must be PEM encoded)
+#SSLCACertificateFile /etc/pki/tls/certs/ca-bundle.crt
+
+#   Client Authentication (Type):
+#   Client certificate verification type and depth.  Types are
+#   none, optional, require and optional_no_ca.  Depth is a
+#   number which specifies how deeply to verify the certificate
+#   issuer chain before deciding the certificate is not valid.
+#SSLVerifyClient require
+#SSLVerifyDepth  10
+
+#   Access Control:
+#   With SSLRequire you can do per-directory access control based
+#   on arbitrary complex boolean expressions containing server
+#   variable checks and other lookup directives.  The syntax is a
+#   mixture between C and Perl.  See the mod_ssl documentation
+#   for more details.
+#<Location />
+#SSLRequire (    %{SSL_CIPHER} !~ m/^(EXP|NULL)/ \
+#            and %{SSL_CLIENT_S_DN_O} eq "Snake Oil, Ltd." \
+#            and %{SSL_CLIENT_S_DN_OU} in {"Staff", "CA", "Dev"} \
+#            and %{TIME_WDAY} >= 1 and %{TIME_WDAY} <= 5 \
+#            and %{TIME_HOUR} >= 8 and %{TIME_HOUR} <= 20       ) \
+#           or %{REMOTE_ADDR} =~ m/^192\.76\.162\.[0-9]+$/
+#</Location>
+
+#   SSL Engine Options:
+#   Set various options for the SSL engine.
+#   o FakeBasicAuth:
+#     Translate the client X.509 into a Basic Authorisation.  This means that
+#     the standard Auth/DBMAuth methods can be used for access control.  The
+#     user name is the `one line' version of the client's X.509 certificate.
+#     Note that no password is obtained from the user. Every entry in the user
+#     file needs this password: `xxj31ZMTZzkVA'.
+#   o ExportCertData:
+#     This exports two additional environment variables: SSL_CLIENT_CERT and
+#     SSL_SERVER_CERT. These contain the PEM-encoded certificates of the
+#     server (always existing) and the client (only existing when client
+#     authentication is used). This can be used to import the certificates
+#     into CGI scripts.
+#   o StdEnvVars:
+#     This exports the standard SSL/TLS related `SSL_*' environment variables.
+#     Per default this exportation is switched off for performance reasons,
+#     because the extraction step is an expensive operation and is usually
+#     useless for serving static content. So one usually enables the
+#     exportation for CGI and SSI requests only.
+#   o StrictRequire:
+#     This denies access when "SSLRequireSSL" or "SSLRequire" applied even
+#     under a "Satisfy any" situation, i.e. when it applies access is denied
+#     and no other module can change it.
+#   o OptRenegotiate:
+#     This enables optimized SSL connection renegotiation handling when SSL
+#     directives are used in per-directory context. 
+#SSLOptions +FakeBasicAuth +ExportCertData +StrictRequire
+<Files ~ "\.(cgi|shtml|phtml|php3?)$">
+    SSLOptions +StdEnvVars
+</Files>
+<Directory "/var/www/cgi-bin">
+    SSLOptions +StdEnvVars
+</Directory>
+
+#   SSL Protocol Adjustments:
+#   The safe and default but still SSL/TLS standard compliant shutdown
+#   approach is that mod_ssl sends the close notify alert but doesn't wait for
+#   the close notify alert from client. When you need a different shutdown
+#   approach you can use one of the following variables:
+#   o ssl-unclean-shutdown:
+#     This forces an unclean shutdown when the connection is closed, i.e. no
+#     SSL close notify alert is send or allowed to received.  This violates
+#     the SSL/TLS standard but is needed for some brain-dead browsers. Use
+#     this when you receive I/O errors because of the standard approach where
+#     mod_ssl sends the close notify alert.
+#   o ssl-accurate-shutdown:
+#     This forces an accurate shutdown when the connection is closed, i.e. a
+#     SSL close notify alert is send and mod_ssl waits for the close notify
+#     alert of the client. This is 100% SSL/TLS standard compliant, but in
+#     practice often causes hanging connections with brain-dead browsers. Use
+#     this only for browsers where you know that their SSL implementation
+#     works correctly. 
+#   Notice: Most problems of broken clients are also related to the HTTP
+#   keep-alive facility, so you usually additionally want to disable
+#   keep-alive for those clients, too. Use variable "nokeepalive" for this.
+#   Similarly, one has to force some clients to use HTTP/1.0 to workaround
+#   their broken HTTP/1.1 implementation. Use variables "downgrade-1.0" and
+#   "force-response-1.0" for this.
+SetEnvIf User-Agent ".*MSIE.*" \
+         nokeepalive ssl-unclean-shutdown \
+         downgrade-1.0 force-response-1.0
+
+#   Per-Server Logging:
+#   The home of a custom SSL log file. Use this when you want a
+#   compact non-error SSL logfile on a virtual host basis.
+CustomLog logs/ssl_request_log \
+          "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b"
+
+</VirtualHost>                                  
+

certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/test.example.com.conf

@@ -0,0 +1,7 @@
+<VirtualHost *:80>
+    ServerName test.example.com
+    ServerAdmin webmaster@dummy-host.example.com
+    DocumentRoot /var/www/htdocs
+    ErrorLog logs/dummy-host.example.com-error_log
+    CustomLog logs/dummy-host.example.com-access_log common
+</VirtualHost>

certbot-apache/certbot_apache/tests/testdata/centos6_apache/apache/httpd/conf.d/welcome.conf

@@ -0,0 +1,11 @@
+# 
+# This configuration file enables the default "Welcome"
+# page if there is no default index page present for
+# the root URL.  To disable the Welcome page, comment
+# out all the lines below.
+#
+<LocationMatch "^/+$">
+    Options -Indexes
+    ErrorDocument 403 /error/noindex.html
+</LocationMatch>
+

certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old,default.conf

@@ -1 +0,0 @@
-../sites-available/old,default.conf

certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/augeas_vhosts/apache2/sites-enabled/old-and-default.conf

@@ -0,0 +1 @@
+../sites-available/old-and-default.conf

certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/duplicatehttp.conf

@@ -0,0 +1,9 @@
+<VirtualHost 10.2.3.4:80>
+	ServerName duplicate.example.com
+
+	ServerAdmin webmaster@certbot.demo
+	DocumentRoot /var/www/html
+
+	ErrorLog ${APACHE_LOG_DIR}/error.log
+	CustomLog ${APACHE_LOG_DIR}/access.log combined
+</VirtualHost>

certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/duplicatehttps.conf

@@ -0,0 +1,14 @@
+<IfModule mod_ssl.c>
+<VirtualHost 10.2.3.4:443>
+	ServerName duplicate.example.com
+
+	ServerAdmin webmaster@certbot.demo
+	DocumentRoot /var/www/html
+
+	ErrorLog ${APACHE_LOG_DIR}/error.log
+	CustomLog ${APACHE_LOG_DIR}/access.log combined
+
+SSLCertificateFile	/etc/apache2/certs/certbot-cert_5.pem
+SSLCertificateKeyFile /etc/apache2/ssl/key-certbot_15.pem
+</VirtualHost>
+</IfModule>

certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-enabled/duplicatehttp.conf

@@ -0,0 +1 @@
+../sites-available/duplicatehttp.conf

certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-enabled/duplicatehttps.conf

@@ -0,0 +1 @@
+../sites-available/duplicatehttps.conf

certbot-apache/certbot_apache/tests/tls_sni_01_test.py

@@ -1,151 +0,0 @@
-"""Test for certbot_apache.tls_sni_01."""
-import shutil
-import unittest
-
-import mock
-
-from certbot import errors
-from certbot.plugins import common_test
-
-from certbot_apache import obj
-from certbot_apache.tests import util
-
-from six.moves import xrange  # pylint: disable=redefined-builtin, import-error
-
-
-class TlsSniPerformTest(util.ApacheTest):
-    """Test the ApacheTlsSni01 challenge."""
-
-    auth_key = common_test.AUTH_KEY
-    achalls = common_test.ACHALLS
-
-    def setUp(self):  # pylint: disable=arguments-differ
-        super(TlsSniPerformTest, self).setUp()
-
-        config = util.get_apache_configurator(
-            self.config_path, self.vhost_path, self.config_dir,
-            self.work_dir)
-        config.config.tls_sni_01_port = 443
-
-        from certbot_apache import tls_sni_01
-        self.sni = tls_sni_01.ApacheTlsSni01(config)
-
-    def tearDown(self):
-        shutil.rmtree(self.temp_dir)
-        shutil.rmtree(self.config_dir)
-        shutil.rmtree(self.work_dir)
-
-    def test_perform0(self):
-        resp = self.sni.perform()
-        self.assertEqual(len(resp), 0)
-
-    @mock.patch("certbot.util.exe_exists")
-    @mock.patch("certbot.util.run_script")
-    def test_perform1(self, _, mock_exists):
-        self.sni.configurator.parser.modules.add("socache_shmcb_module")
-        self.sni.configurator.parser.modules.add("ssl_module")
-
-        mock_exists.return_value = True
-        self.sni.configurator.parser.update_runtime_variables = mock.Mock()
-
-        achall = self.achalls[0]
-        self.sni.add_chall(achall)
-        response = self.achalls[0].response(self.auth_key)
-        mock_setup_cert = mock.MagicMock(return_value=response)
-        # pylint: disable=protected-access
-        self.sni._setup_challenge_cert = mock_setup_cert
-
-        responses = self.sni.perform()
-        mock_setup_cert.assert_called_once_with(achall)
-
-        # Check to make sure challenge config path is included in apache config
-        self.assertEqual(
-            len(self.sni.configurator.parser.find_dir(
-                "Include", self.sni.challenge_conf)), 1)
-        self.assertEqual(len(responses), 1)
-        self.assertEqual(responses[0], response)
-
-    def test_perform2(self):
-        # Avoid load module
-        self.sni.configurator.parser.modules.add("ssl_module")
-        self.sni.configurator.parser.modules.add("socache_shmcb_module")
-        acme_responses = []
-        for achall in self.achalls:
-            self.sni.add_chall(achall)
-            acme_responses.append(achall.response(self.auth_key))
-
-        mock_setup_cert = mock.MagicMock(side_effect=acme_responses)
-        # pylint: disable=protected-access
-        self.sni._setup_challenge_cert = mock_setup_cert
-
-        with mock.patch(
-            "certbot_apache.override_debian.DebianConfigurator.enable_mod"):
-            sni_responses = self.sni.perform()
-
-        self.assertEqual(mock_setup_cert.call_count, 2)
-
-        # Make sure calls made to mocked function were correct
-        self.assertEqual(
-            mock_setup_cert.call_args_list[0], mock.call(self.achalls[0]))
-        self.assertEqual(
-            mock_setup_cert.call_args_list[1], mock.call(self.achalls[1]))
-
-        self.assertEqual(
-            len(self.sni.configurator.parser.find_dir(
-                "Include", self.sni.challenge_conf)),
-            1)
-        self.assertEqual(len(sni_responses), 2)
-        for i in xrange(2):
-            self.assertEqual(sni_responses[i], acme_responses[i])
-
-    def test_mod_config(self):
-        z_domains = []
-        for achall in self.achalls:
-            self.sni.add_chall(achall)
-            z_domain = achall.response(self.auth_key).z_domain
-            z_domains.append(set([z_domain.decode('ascii')]))
-
-        self.sni._mod_config()  # pylint: disable=protected-access
-        self.sni.configurator.save()
-
-        self.sni.configurator.parser.find_dir(
-            "Include", self.sni.challenge_conf)
-        vh_match = self.sni.configurator.aug.match(
-            "/files" + self.sni.challenge_conf + "//VirtualHost")
-
-        vhs = []
-        for match in vh_match:
-            # pylint: disable=protected-access
-            vhs.append(self.sni.configurator._create_vhost(match))
-        self.assertEqual(len(vhs), 2)
-        for vhost in vhs:
-            self.assertEqual(vhost.addrs, set([obj.Addr.fromstring("*:443")]))
-            names = vhost.get_names()
-            self.assertTrue(names in z_domains)
-
-    def test_get_addrs_default(self):
-        self.sni.configurator.choose_vhost = mock.Mock(
-            return_value=obj.VirtualHost(
-                "path", "aug_path",
-                set([obj.Addr.fromstring("_default_:443")]),
-                False, False)
-        )
-
-        # pylint: disable=protected-access
-        self.assertEqual(
-            set([obj.Addr.fromstring("*:443")]),
-            self.sni._get_addrs(self.achalls[0]))
-
-    def test_get_addrs_no_vhost_found(self):
-        self.sni.configurator.choose_vhost = mock.Mock(
-            side_effect=errors.MissingCommandlineFlag(
-                "Failed to run Apache plugin non-interactively"))
-
-        # pylint: disable=protected-access
-        self.assertEqual(
-            set([obj.Addr.fromstring("*:443")]),
-            self.sni._get_addrs(self.achalls[0]))
-
-
-if __name__ == "__main__":
-    unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/util.py

@@ -1,5 +1,4 @@
 """Common utilities for certbot_apache."""
-import os
 import shutil
 import sys
 import unittest
@@ -9,10 +8,9 @@ import josepy as jose
 import mock
 import zope.component
 
+from certbot.compat import os
 from certbot.display import util as display_util
-
 from certbot.plugins import common
-
 from certbot.tests import util as test_util
 
 from certbot_apache import configurator
@@ -80,8 +78,7 @@ class ParserTest(ApacheTest):
         with mock.patch("certbot_apache.parser.ApacheParser."
                         "update_runtime_variables"):
             self.parser = ApacheParser(
-                self.aug, self.config_path, self.vhost_path,
-                configurator=self.config)
+                self.config_path, self.vhost_path, configurator=self.config)
 
 
 def get_apache_configurator(  # pylint: disable=too-many-arguments, too-many-locals
@@ -196,7 +193,17 @@ def get_vh_truth(temp_dir, config_name):
                 "/files" + os.path.join(temp_dir, config_name,
                                         "apache2/apache2.conf/VirtualHost"),
                 set([obj.Addr.fromstring("*:80")]), False, True,
-                "vhost.in.rootconf")]
+                "vhost.in.rootconf"),
+            obj.VirtualHost(
+                os.path.join(prefix, "duplicatehttp.conf"),
+                os.path.join(aug_pre, "duplicatehttp.conf/VirtualHost"),
+                set([obj.Addr.fromstring("10.2.3.4:80")]), False, True,
+                "duplicate.example.com"),
+            obj.VirtualHost(
+                os.path.join(prefix, "duplicatehttps.conf"),
+                os.path.join(aug_pre, "duplicatehttps.conf/IfModule/VirtualHost"),
+                set([obj.Addr.fromstring("10.2.3.4:443")]), True, True,
+                "duplicate.example.com")]
         return vh_truth
     if config_name == "debian_apache_2_4/multi_vhosts":
         prefix = os.path.join(

certbot-apache/certbot_apache/tls_configs/centos-current-options-ssl-apache.conf

@@ -10,16 +10,10 @@ SSLEngine on
 SSLProtocol             all -SSLv2 -SSLv3
 SSLCipherSuite          ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
 SSLHonorCipherOrder     on
+SSLSessionTickets       off
 
 SSLOptions +StrictRequire
 
 # Add vhost name to log entries:
 LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" vhost_combined
 LogFormat "%v %h %l %u %t \"%r\" %>s %b" vhost_common
-
-#CustomLog /var/log/apache2/access.log vhost_combined
-#LogLevel warn
-#ErrorLog /var/log/apache2/error.log
-
-# Always ensure Cookies have "Secure" set (JAH 2012/1)
-#Header edit Set-Cookie (?i)^(.*)(;\s*secure)??((\s*;)?(.*)) "$1; Secure$3$4"

certbot-apache/certbot_apache/tls_configs/centos-old-options-ssl-apache.conf

@@ -0,0 +1,18 @@
+# This file contains important security parameters. If you modify this file
+# manually, Certbot will be unable to automatically provide future security
+# updates. Instead, Certbot will print and log an error message with a path to
+# the up-to-date file that you will need to refer to when manually updating
+# this file.
+
+SSLEngine on
+
+# Intermediate configuration, tweak to your needs
+SSLProtocol             all -SSLv2 -SSLv3
+SSLCipherSuite          ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
+SSLHonorCipherOrder     on
+
+SSLOptions +StrictRequire
+
+# Add vhost name to log entries:
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" vhost_combined
+LogFormat "%v %h %l %u %t \"%r\" %>s %b" vhost_common

certbot-apache/certbot_apache/tls_configs/current-options-ssl-apache.conf

@@ -11,16 +11,10 @@ SSLProtocol             all -SSLv2 -SSLv3
 SSLCipherSuite          ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
 SSLHonorCipherOrder     on
 SSLCompression          off
+SSLSessionTickets       off
 
 SSLOptions +StrictRequire
 
 # Add vhost name to log entries:
 LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" vhost_combined
 LogFormat "%v %h %l %u %t \"%r\" %>s %b" vhost_common
-
-#CustomLog /var/log/apache2/access.log vhost_combined
-#LogLevel warn
-#ErrorLog /var/log/apache2/error.log
-
-# Always ensure Cookies have "Secure" set (JAH 2012/1)
-#Header edit Set-Cookie (?i)^(.*)(;\s*secure)??((\s*;)?(.*)) "$1; Secure$3$4"

certbot-apache/certbot_apache/tls_configs/old-options-ssl-apache.conf

@@ -0,0 +1,19 @@
+# This file contains important security parameters. If you modify this file
+# manually, Certbot will be unable to automatically provide future security
+# updates. Instead, Certbot will print and log an error message with a path to
+# the up-to-date file that you will need to refer to when manually updating
+# this file.
+
+SSLEngine on
+
+# Intermediate configuration, tweak to your needs
+SSLProtocol             all -SSLv2 -SSLv3
+SSLCipherSuite          ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
+SSLHonorCipherOrder     on
+SSLCompression          off
+
+SSLOptions +StrictRequire
+
+# Add vhost name to log entries:
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" vhost_combined
+LogFormat "%v %h %l %u %t \"%r\" %>s %b" vhost_common

certbot-apache/certbot_apache/tls_sni_01.py

@@ -1,174 +0,0 @@
-"""A class that performs TLS-SNI-01 challenges for Apache"""
-
-import os
-import logging
-
-from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
-from certbot.plugins import common
-from certbot.errors import PluginError, MissingCommandlineFlag
-
-from certbot_apache import obj
-
-logger = logging.getLogger(__name__)
-
-
-class ApacheTlsSni01(common.TLSSNI01):
-    """Class that performs TLS-SNI-01 challenges within the Apache configurator
-
-    :ivar configurator: ApacheConfigurator object
-    :type configurator: :class:`~apache.configurator.ApacheConfigurator`
-
-    :ivar list achalls: Annotated TLS-SNI-01
-        (`.KeyAuthorizationAnnotatedChallenge`) challenges.
-
-    :param list indices: Meant to hold indices of challenges in a
-        larger array. ApacheTlsSni01 is capable of solving many challenges
-        at once which causes an indexing issue within ApacheConfigurator
-        who must return all responses in order.  Imagine ApacheConfigurator
-        maintaining state about where all of the http-01 Challenges,
-        TLS-SNI-01 Challenges belong in the response array.  This is an
-        optional utility.
-
-    :param str challenge_conf: location of the challenge config file
-
-    """
-
-    VHOST_TEMPLATE = """\
-<VirtualHost {vhost}>
-    ServerName {server_name}
-    UseCanonicalName on
-    SSLStrictSNIVHostCheck on
-
-    LimitRequestBody 1048576
-
-    Include {ssl_options_conf_path}
-    SSLCertificateFile {cert_path}
-    SSLCertificateKeyFile {key_path}
-
-    DocumentRoot {document_root}
-</VirtualHost>
-
-"""
-
-    def __init__(self, *args, **kwargs):
-        super(ApacheTlsSni01, self).__init__(*args, **kwargs)
-
-        self.challenge_conf = os.path.join(
-            self.configurator.conf("challenge-location"),
-            "le_tls_sni_01_cert_challenge.conf")
-
-    def perform(self):
-        """Perform a TLS-SNI-01 challenge."""
-        if not self.achalls:
-            return []
-        # Save any changes to the configuration as a precaution
-        # About to make temporary changes to the config
-        self.configurator.save("Changes before challenge setup", True)
-
-        # Prepare the server for HTTPS
-        self.configurator.prepare_server_https(
-            str(self.configurator.config.tls_sni_01_port), True)
-
-        responses = []
-
-        # Create all of the challenge certs
-        for achall in self.achalls:
-            responses.append(self._setup_challenge_cert(achall))
-
-        # Setup the configuration
-        addrs = self._mod_config()
-        self.configurator.save("Don't lose mod_config changes", True)
-        self.configurator.make_addrs_sni_ready(addrs)
-
-        # Save reversible changes
-        self.configurator.save("SNI Challenge", True)
-
-        return responses
-
-    def _mod_config(self):
-        """Modifies Apache config files to include challenge vhosts.
-
-        Result: Apache config includes virtual servers for issued challs
-
-        :returns: All TLS-SNI-01 addresses used
-        :rtype: set
-
-        """
-        addrs = set()  # type: Set[obj.Addr]
-        config_text = "<IfModule mod_ssl.c>\n"
-
-        for achall in self.achalls:
-            achall_addrs = self._get_addrs(achall)
-            addrs.update(achall_addrs)
-
-            config_text += self._get_config_text(achall, achall_addrs)
-
-        config_text += "</IfModule>\n"
-
-        self.configurator.parser.add_include(
-            self.configurator.parser.loc["default"], self.challenge_conf)
-        self.configurator.reverter.register_file_creation(
-            True, self.challenge_conf)
-
-        logger.debug("writing a config file with text:\n %s", config_text)
-        with open(self.challenge_conf, "w") as new_conf:
-            new_conf.write(config_text)
-
-        return addrs
-
-    def _get_addrs(self, achall):
-        """Return the Apache addresses needed for TLS-SNI-01."""
-        # TODO: Checkout _default_ rules.
-        addrs = set()
-        default_addr = obj.Addr(("*", str(
-            self.configurator.config.tls_sni_01_port)))
-
-        try:
-            vhost = self.configurator.choose_vhost(achall.domain,
-                                                   create_if_no_ssl=False)
-        except (PluginError, MissingCommandlineFlag):
-            # We couldn't find the virtualhost for this domain, possibly
-            # because it's a new vhost that's not configured yet
-            # (GH #677). See also GH #2600.
-            logger.warning("Falling back to default vhost %s...", default_addr)
-            addrs.add(default_addr)
-            return addrs
-
-        for addr in vhost.addrs:
-            if "_default_" == addr.get_addr():
-                addrs.add(default_addr)
-            else:
-                addrs.add(
-                    addr.get_sni_addr(
-                        self.configurator.config.tls_sni_01_port))
-
-        return addrs
-
-    def _get_config_text(self, achall, ip_addrs):
-        """Chocolate virtual server configuration text
-
-        :param .KeyAuthorizationAnnotatedChallenge achall: Annotated
-            TLS-SNI-01 challenge.
-
-        :param list ip_addrs: addresses of challenged domain
-            :class:`list` of type `~.obj.Addr`
-
-        :returns: virtual host configuration text
-        :rtype: str
-
-        """
-        ips = " ".join(str(i) for i in ip_addrs)
-        document_root = os.path.join(
-            self.configurator.config.work_dir, "tls_sni_01_page/")
-        # TODO: Python docs is not clear how multiline string literal
-        # newlines are parsed on different platforms. At least on
-        # Linux (Debian sid), when source file uses CRLF, Python still
-        # parses it as "\n"... c.f.:
-        # https://docs.python.org/2.7/reference/lexical_analysis.html
-        return self.VHOST_TEMPLATE.format(
-            vhost=ips,
-            server_name=achall.response(achall.account_key).z_domain.decode('ascii'),
-            ssl_options_conf_path=self.configurator.mod_ssl_conf,
-            cert_path=self.get_cert_path(achall),
-            key_path=self.get_key_path(achall),
-            document_root=document_root).replace("\n", os.linesep)

certbot-apache/docs/conf.py

@@ -13,7 +13,7 @@
 # serve to show the default.
 
 import sys
-import os
+from certbot.compat import os
 import shlex
 
 import mock

certbot-apache/local-oldest-requirements.txt

@@ -1,2 +1,3 @@
-acme[dev]==0.25.0
-certbot[dev]==0.26.0
+# Remember to update setup.py to match the package versions below.
+acme[dev]==0.29.0
+-e .[dev]

certbot-apache/setup.py

@@ -1,14 +1,16 @@
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
 
-version = '0.30.0.dev0'
+version = '0.37.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    'acme>=0.25.0',
-    'certbot>=0.26.0',
+    'acme>=0.29.0',
+    'certbot>=0.37.0.dev0',
     'mock',
     'python-augeas',
     'setuptools',
@@ -21,6 +23,22 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
+
+
 setup(
     name='certbot-apache',
     version=version,
@@ -64,4 +82,6 @@ setup(
         ],
     },
     test_suite='certbot_apache',
+    tests_require=["pytest"],
+    cmdclass={"test": PyTest},
 )

certbot-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.29.1"
+LE_AUTO_VERSION="0.36.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -45,6 +45,7 @@ Help for certbot itself cannot be provided until it is installed.
   -h, --help                                print this help
   -n, --non-interactive, --noninteractive   run without asking for user input
   --no-bootstrap                            do not install OS dependencies
+  --no-permissions-check                    do not warn about file system permissions
   --no-self-upgrade                         do not download updates
   --os-packages-only                        install OS dependencies and exit
   --install-only                            install certbot, upgrade if needed, and exit
@@ -67,6 +68,8 @@ for arg in "$@" ; do
       # Do not upgrade this script (also prevents client upgrades, because each
       # copy of the script pins a hash of the python client)
       NO_SELF_UPGRADE=1;;
+    --no-permissions-check)
+      NO_PERMISSIONS_CHECK=1;;
     --no-bootstrap)
       NO_BOOTSTRAP=1;;
     --help)
@@ -172,7 +175,11 @@ SetRootAuthMechanism() {
       sudo)
         SUDO="sudo -E"
         ;;
-      '') ;; # Nothing to do for plain root method.
+      '')
+        # If we're not running with root, don't check that this script can only
+        # be modified by system users and groups.
+        NO_PERMISSIONS_CHECK=1
+        ;;
       *)
         error "Error: unknown root authorization mechanism '$LE_AUTO_SUDO'."
         exit 1
@@ -333,63 +340,11 @@ BootstrapDebCommon() {
   fi
 
   augeas_pkg="libaugeas0 augeas-lenses"
-  AUGVERSION=`LC_ALL=C apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2`
 
   if [ "$ASSUME_YES" = 1 ]; then
     YES_FLAG="-y"
   fi
 
-  AddBackportRepo() {
-    # ARGS:
-    BACKPORT_NAME="$1"
-    BACKPORT_SOURCELINE="$2"
-    say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME."
-    if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then
-      # This can theoretically error if sources.list.d is empty, but in that case we don't care.
-      if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then
-        if [ "$ASSUME_YES" = 1 ]; then
-          /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..."
-          sleep 1s
-          /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..."
-          sleep 1s
-          /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..."
-          sleep 1s
-          add_backports=1
-        else
-          read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response
-          case $response in
-            [yY][eE][sS]|[yY]|"")
-              add_backports=1;;
-            *)
-              add_backports=0;;
-          esac
-        fi
-        if [ "$add_backports" = 1 ]; then
-          sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list"
-          apt-get $QUIET_FLAG update
-        fi
-      fi
-    fi
-    if [ "$add_backports" != 0 ]; then
-      apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg
-      augeas_pkg=
-    fi
-  }
-
-
-  if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then
-    if lsb_release -a | grep -q wheezy ; then
-      AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main"
-    elif lsb_release -a | grep -q precise ; then
-      # XXX add ARM case
-      AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse"
-    else
-      echo "No libaugeas0 version is available that's new enough to run the"
-      echo "Certbot apache plugin..."
-    fi
-    # XXX add a case for ubuntu PPAs
-  fi
-
   apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends \
     python \
     python-dev \
@@ -540,11 +495,18 @@ BOOTSTRAP_RPM_PYTHON3_VERSION=1
 BootstrapRpmPython3() {
   # Tested with:
   #   - CentOS 6
+  #   - Fedora 29
 
   InitializeRPMCommonBase
 
+  # Fedora 29 must use python3-virtualenv
+  if $TOOL list python3-virtualenv >/dev/null 2>&1; then
+    python_pkgs="python3
+      python3-virtualenv
+      python3-devel
+    "
   # EPEL uses python34
-  if $TOOL list python34 >/dev/null 2>&1; then
+  elif $TOOL list python34 >/dev/null 2>&1; then
     python_pkgs="python34
       python34-devel
       python34-tools
@@ -573,10 +535,20 @@ BootstrapSuseCommon() {
     QUIET_FLAG='-qq'
   fi
 
+  if zypper search -x python-virtualenv >/dev/null 2>&1; then
+    OPENSUSE_VIRTUALENV_PACKAGES="python-virtualenv"
+  else
+    # Since Leap 15.0 (and associated Tumbleweed version), python-virtualenv
+    # is a source package, and python2-virtualenv must be used instead.
+    # Also currently python2-setuptools is not a dependency of python2-virtualenv,
+    # while it should be. Installing it explicitly until upstream fix.
+    OPENSUSE_VIRTUALENV_PACKAGES="python2-virtualenv python2-setuptools"
+  fi
+
   zypper $QUIET_FLAG $zypper_flags in $install_flags \
     python \
     python-devel \
-    python-virtualenv \
+    $OPENSUSE_VIRTUALENV_PACKAGES \
     gcc \
     augeas-lenses \
     libopenssl-devel \
@@ -783,7 +755,13 @@ elif [ -f /etc/redhat-release ]; then
   prev_le_python="$LE_PYTHON"
   unset LE_PYTHON
   DeterminePythonVersion "NOCRASH"
-  if [ "$PYVER" -eq 26 ]; then
+  # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then.
+  RPM_DIST_NAME=`(. /etc/os-release 2> /dev/null && echo $ID) || echo "unknown"`
+  RPM_DIST_VERSION=0
+  if [ "$RPM_DIST_NAME" = "fedora" ]; then
+    RPM_DIST_VERSION=`(. /etc/os-release 2> /dev/null && echo $VERSION_ID) || echo "0"`
+  fi
+  if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then
     Bootstrap() {
       BootstrapMessage "RedHat-based OSes that will use Python3"
       BootstrapRpmPython3
@@ -940,6 +918,130 @@ else:
 UNLIKELY_EOF
 }
 
+# Create a new virtual environment for Certbot. It will overwrite any existing one.
+# Parameters: LE_PYTHON, VENV_PATH, PYVER, VERBOSE
+CreateVenv() {
+    "$1" - "$2" "$3" "$4" << "UNLIKELY_EOF"
+#!/usr/bin/env python
+import os
+import shutil
+import subprocess
+import sys
+
+
+def create_venv(venv_path, pyver, verbose):
+    if os.path.exists(venv_path):
+        shutil.rmtree(venv_path)
+
+    stdout = sys.stdout if verbose == '1' else open(os.devnull, 'w')
+
+    if int(pyver) <= 27:
+        # Use virtualenv binary
+        environ = os.environ.copy()
+        environ['VIRTUALENV_NO_DOWNLOAD'] = '1'
+        command = ['virtualenv', '--no-site-packages', '--python', sys.executable, venv_path]
+        subprocess.check_call(command, stdout=stdout, env=environ)
+    else:
+        # Use embedded venv module in Python 3
+        command = [sys.executable, '-m', 'venv', venv_path]
+        subprocess.check_call(command, stdout=stdout)
+
+
+if __name__ == '__main__':
+    create_venv(*sys.argv[1:])
+
+UNLIKELY_EOF
+}
+
+# Check that the given PATH_TO_CHECK has secured permissions.
+# Parameters: LE_PYTHON, PATH_TO_CHECK
+CheckPathPermissions() {
+    "$1" - "$2" << "UNLIKELY_EOF"
+"""Verifies certbot-auto cannot be modified by unprivileged users.
+
+This script takes the path to certbot-auto as its only command line
+argument.  It then checks that the file can only be modified by uid/gid
+< 1000 and if other users can modify the file, it prints a warning with
+a suggestion on how to solve the problem.
+
+Permissions on symlinks in the absolute path of certbot-auto are ignored
+and only the canonical path to certbot-auto is checked. There could be
+permissions problems due to the symlinks that are unreported by this
+script, however, issues like this were not caused by our documentation
+and are ignored for the sake of simplicity.
+
+All warnings are printed to stdout rather than stderr so all stderr
+output from this script can be suppressed to avoid printing messages if
+this script fails for some reason.
+
+"""
+from __future__ import print_function
+
+import os
+import stat
+import sys
+
+
+FORUM_POST_URL = 'https://community.letsencrypt.org/t/certbot-auto-deployment-best-practices/91979/'
+
+
+def has_safe_permissions(path):
+    """Returns True if the given path has secure permissions.
+
+    The permissions are considered safe if the file is only writable by
+    uid/gid < 1000.
+
+    The reason we allow more IDs than 0 is because on some systems such
+    as Debian, system users/groups other than uid/gid 0 are used for the
+    path we recommend in our instructions which is /usr/local/bin.  1000
+    was chosen because on Debian 0-999 is reserved for system IDs[1] and
+    on RHEL either 0-499 or 0-999 is reserved depending on the
+    version[2][3]. Due to these differences across different OSes, this
+    detection isn't perfect so we only determine permissions are
+    insecure when we can be reasonably confident there is a problem
+    regardless of the underlying OS.
+
+    [1] https://www.debian.org/doc/debian-policy/ch-opersys.html#uid-and-gid-classes
+    [2] https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/ch-managing_users_and_groups
+    [3] https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/system_administrators_guide/ch-managing_users_and_groups
+
+    :param str path: filesystem path to check
+    :returns: True if the path has secure permissions, otherwise, False
+    :rtype: bool
+
+    """
+    # os.stat follows symlinks before obtaining information about a file.
+    stat_result = os.stat(path)
+    if stat_result.st_mode & stat.S_IWOTH:
+        return False
+    if stat_result.st_mode & stat.S_IWGRP and stat_result.st_gid >= 1000:
+        return False
+    if stat_result.st_mode & stat.S_IWUSR and stat_result.st_uid >= 1000:
+        return False
+    return True
+
+
+def main(certbot_auto_path):
+    current_path = os.path.realpath(certbot_auto_path)
+    last_path = None
+    permissions_ok = True
+    # This loop makes use of the fact that os.path.dirname('/') == '/'.
+    while current_path != last_path and permissions_ok:
+        permissions_ok = has_safe_permissions(current_path)
+        last_path = current_path
+        current_path = os.path.dirname(current_path)
+
+    if not permissions_ok:
+        print('{0} has insecure permissions!'.format(certbot_auto_path))
+        print('To learn how to fix them, visit {0}'.format(FORUM_POST_URL))
+
+
+if __name__ == '__main__':
+    main(sys.argv[1])
+
+UNLIKELY_EOF
+}
+
 if [ "$1" = "--le-auto-phase2" ]; then
   # Phase 2: Create venv, install LE, and run.
 
@@ -995,20 +1097,7 @@ if [ "$1" = "--le-auto-phase2" ]; then
   if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then
     say "Creating virtual environment..."
     DeterminePythonVersion
-    rm -rf "$VENV_PATH"
-    if [ "$PYVER" -le 27 ]; then
-      if [ "$VERBOSE" = 1 ]; then
-        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
-      else
-        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
-      fi
-    else
-      if [ "$VERBOSE" = 1 ]; then
-        "$LE_PYTHON" -m venv "$VENV_PATH"
-      else
-        "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null
-      fi
-    fi
+    CreateVenv "$LE_PYTHON" "$VENV_PATH" "$PYVER" "$VERBOSE"
 
     if [ -n "$BOOTSTRAP_VERSION" ]; then
       echo "$BOOTSTRAP_VERSION" > "$BOOTSTRAP_VERSION_PATH"
@@ -1022,202 +1111,197 @@ if [ "$1" = "--le-auto-phase2" ]; then
     # There is no $ interpolation due to quotes on starting heredoc delimiter.
     # -------------------------------------------------------------------------
     cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt"
-# This is the flattened list of packages certbot-auto installs. To generate
-# this, do
-# `pip install --no-cache-dir -e acme -e . -e certbot-apache -e certbot-nginx`,
-# and then use `hashin` or a more secure method to gather the hashes.
-
-# Hashin example:
-# pip install hashin
-# hashin -r dependency-requirements.txt cryptography==1.5.2
-# sets the new certbot-auto pinned version of cryptography to 1.5.2
-
-argparse==1.4.0 \
-    --hash=sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314 \
-    --hash=sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4
-
-# This comes before cffi because cffi will otherwise install an unchecked
-# version via setup_requires.
-pycparser==2.14 \
-    --hash=sha256:7959b4a74abdc27b312fed1c21e6caf9309ce0b29ea86b591fd2e99ecdf27f73 \
-    --no-binary pycparser
-
-asn1crypto==0.22.0 \
-    --hash=sha256:d232509fefcfcdb9a331f37e9c9dc20441019ad927c7d2176cf18ed5da0ba097 \
-    --hash=sha256:cbbadd640d3165ab24b06ef25d1dca09a3441611ac15f6a6b452474fdf0aed1a
-cffi==1.11.5 \
-    --hash=sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50 \
-    --hash=sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596 \
-    --hash=sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef \
-    --hash=sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743 \
-    --hash=sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f \
-    --hash=sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31 \
-    --hash=sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04 \
-    --hash=sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6 \
-    --hash=sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3 \
-    --hash=sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6 \
-    --hash=sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b \
-    --hash=sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca \
-    --hash=sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e \
-    --hash=sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb \
-    --hash=sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd \
-    --hash=sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1 \
-    --hash=sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917 \
-    --hash=sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359 \
-    --hash=sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f \
-    --hash=sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95 \
-    --hash=sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801 \
-    --hash=sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257 \
-    --hash=sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184 \
-    --hash=sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc \
-    --hash=sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085 \
-    --hash=sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93 \
-    --hash=sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2 \
-    --hash=sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30 \
-    --hash=sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5 \
-    --hash=sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e \
-    --hash=sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b \
-    --hash=sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4
-ConfigArgParse==0.12.0 \
-    --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \
-    --no-binary ConfigArgParse
+# This is the flattened list of packages certbot-auto installs.
+# To generate this, do (with docker and package hashin installed):
+# ```
+# letsencrypt-auto-source/rebuild_dependencies.py \
+#   letsencrypt-auto-sources/pieces/dependency-requirements.txt
+# ```
+ConfigArgParse==0.14.0 \
+    --hash=sha256:2e2efe2be3f90577aca9415e32cb629aa2ecd92078adbe27b53a03e53ff12e91
+asn1crypto==0.24.0 \
+    --hash=sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87 \
+    --hash=sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49
+certifi==2019.3.9 \
+    --hash=sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5 \
+    --hash=sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae
+cffi==1.12.2 \
+    --hash=sha256:00b97afa72c233495560a0793cdc86c2571721b4271c0667addc83c417f3d90f \
+    --hash=sha256:0ba1b0c90f2124459f6966a10c03794082a2f3985cd699d7d63c4a8dae113e11 \
+    --hash=sha256:0bffb69da295a4fc3349f2ec7cbe16b8ba057b0a593a92cbe8396e535244ee9d \
+    --hash=sha256:21469a2b1082088d11ccd79dd84157ba42d940064abbfa59cf5f024c19cf4891 \
+    --hash=sha256:2e4812f7fa984bf1ab253a40f1f4391b604f7fc424a3e21f7de542a7f8f7aedf \
+    --hash=sha256:2eac2cdd07b9049dd4e68449b90d3ef1adc7c759463af5beb53a84f1db62e36c \
+    --hash=sha256:2f9089979d7456c74d21303c7851f158833d48fb265876923edcb2d0194104ed \
+    --hash=sha256:3dd13feff00bddb0bd2d650cdb7338f815c1789a91a6f68fdc00e5c5ed40329b \
+    --hash=sha256:4065c32b52f4b142f417af6f33a5024edc1336aa845b9d5a8d86071f6fcaac5a \
+    --hash=sha256:51a4ba1256e9003a3acf508e3b4f4661bebd015b8180cc31849da222426ef585 \
+    --hash=sha256:59888faac06403767c0cf8cfb3f4a777b2939b1fbd9f729299b5384f097f05ea \
+    --hash=sha256:59c87886640574d8b14910840327f5cd15954e26ed0bbd4e7cef95fa5aef218f \
+    --hash=sha256:610fc7d6db6c56a244c2701575f6851461753c60f73f2de89c79bbf1cc807f33 \
+    --hash=sha256:70aeadeecb281ea901bf4230c6222af0248c41044d6f57401a614ea59d96d145 \
+    --hash=sha256:71e1296d5e66c59cd2c0f2d72dc476d42afe02aeddc833d8e05630a0551dad7a \
+    --hash=sha256:8fc7a49b440ea752cfdf1d51a586fd08d395ff7a5d555dc69e84b1939f7ddee3 \
+    --hash=sha256:9b5c2afd2d6e3771d516045a6cfa11a8da9a60e3d128746a7fe9ab36dfe7221f \
+    --hash=sha256:9c759051ebcb244d9d55ee791259ddd158188d15adee3c152502d3b69005e6bd \
+    --hash=sha256:b4d1011fec5ec12aa7cc10c05a2f2f12dfa0adfe958e56ae38dc140614035804 \
+    --hash=sha256:b4f1d6332339ecc61275bebd1f7b674098a66fea11a00c84d1c58851e618dc0d \
+    --hash=sha256:c030cda3dc8e62b814831faa4eb93dd9a46498af8cd1d5c178c2de856972fd92 \
+    --hash=sha256:c2e1f2012e56d61390c0e668c20c4fb0ae667c44d6f6a2eeea5d7148dcd3df9f \
+    --hash=sha256:c37c77d6562074452120fc6c02ad86ec928f5710fbc435a181d69334b4de1d84 \
+    --hash=sha256:c8149780c60f8fd02752d0429246088c6c04e234b895c4a42e1ea9b4de8d27fb \
+    --hash=sha256:cbeeef1dc3c4299bd746b774f019de9e4672f7cc666c777cd5b409f0b746dac7 \
+    --hash=sha256:e113878a446c6228669144ae8a56e268c91b7f1fafae927adc4879d9849e0ea7 \
+    --hash=sha256:e21162bf941b85c0cda08224dade5def9360f53b09f9f259adb85fc7dd0e7b35 \
+    --hash=sha256:fb6934ef4744becbda3143d30c6604718871495a5e36c408431bf33d9c146889
+chardet==3.0.4 \
+    --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
+    --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691
 configobj==5.0.6 \
-    --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \
-    --no-binary configobj
-cryptography==2.2.2 \
-    --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \
-    --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \
-    --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \
-    --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \
-    --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \
-    --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \
-    --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \
-    --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \
-    --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \
-    --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \
-    --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \
-    --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \
-    --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \
-    --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \
-    --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \
-    --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \
-    --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \
-    --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \
-    --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887
-enum34==1.1.2 ; python_version < '3.4' \
-    --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \
-    --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501
+    --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902
+cryptography==2.6.1 \
+    --hash=sha256:066f815f1fe46020877c5983a7e747ae140f517f1b09030ec098503575265ce1 \
+    --hash=sha256:210210d9df0afba9e000636e97810117dc55b7157c903a55716bb73e3ae07705 \
+    --hash=sha256:26c821cbeb683facb966045e2064303029d572a87ee69ca5a1bf54bf55f93ca6 \
+    --hash=sha256:2afb83308dc5c5255149ff7d3fb9964f7c9ee3d59b603ec18ccf5b0a8852e2b1 \
+    --hash=sha256:2db34e5c45988f36f7a08a7ab2b69638994a8923853dec2d4af121f689c66dc8 \
+    --hash=sha256:409c4653e0f719fa78febcb71ac417076ae5e20160aec7270c91d009837b9151 \
+    --hash=sha256:45a4f4cf4f4e6a55c8128f8b76b4c057027b27d4c67e3fe157fa02f27e37830d \
+    --hash=sha256:48eab46ef38faf1031e58dfcc9c3e71756a1108f4c9c966150b605d4a1a7f659 \
+    --hash=sha256:6b9e0ae298ab20d371fc26e2129fd683cfc0cfde4d157c6341722de645146537 \
+    --hash=sha256:6c4778afe50f413707f604828c1ad1ff81fadf6c110cb669579dea7e2e98a75e \
+    --hash=sha256:8c33fb99025d353c9520141f8bc989c2134a1f76bac6369cea060812f5b5c2bb \
+    --hash=sha256:9873a1760a274b620a135054b756f9f218fa61ca030e42df31b409f0fb738b6c \
+    --hash=sha256:9b069768c627f3f5623b1cbd3248c5e7e92aec62f4c98827059eed7053138cc9 \
+    --hash=sha256:9e4ce27a507e4886efbd3c32d120db5089b906979a4debf1d5939ec01b9dd6c5 \
+    --hash=sha256:acb424eaca214cb08735f1a744eceb97d014de6530c1ea23beb86d9c6f13c2ad \
+    --hash=sha256:c8181c7d77388fe26ab8418bb088b1a1ef5fde058c6926790c8a0a3d94075a4a \
+    --hash=sha256:d4afbb0840f489b60f5a580a41a1b9c3622e08ecb5eec8614d4fb4cd914c4460 \
+    --hash=sha256:d9ed28030797c00f4bc43c86bf819266c76a5ea61d006cd4078a93ebf7da6bfd \
+    --hash=sha256:e603aa7bb52e4e8ed4119a58a03b60323918467ef209e6ff9db3ac382e5cf2c6
+# Package enum34 needs to be explicitly limited to Python2.x, in order to avoid
+# certbot-auto failures on Python 3.6+ which enum34 doesn't support. See #5456.
+enum34==1.1.6 ; python_version < '3.4' \
+    --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850 \
+    --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \
+    --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \
+    --hash=sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1
 funcsigs==1.0.2 \
     --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
     --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50
-idna==2.5 \
-    --hash=sha256:cc19709fd6d0cbfed39ea875d29ba6d4e22c0cebc510a76d6302a28385e8bb70 \
-    --hash=sha256:3cb5ce08046c4e3a560fc02f138d0ac63e00f8ce5901a56b32ec8b7994082aab
-ipaddress==1.0.16 \
-    --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \
-    --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0
+future==0.17.1 \
+    --hash=sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8
+idna==2.8 \
+    --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \
+    --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c
+ipaddress==1.0.22 \
+    --hash=sha256:64b28eec5e78e7510698f6d4da08800a5c575caa4a286c93d651c5d3ff7b6794 \
+    --hash=sha256:b146c751ea45cad6188dd6cf2d9b757f6f4f8d6ffb96a023e6f2e26eea02a72c
 josepy==1.1.0 \
     --hash=sha256:1309a25aac3caeff5239729c58ff9b583f7d022ffdb1553406ddfc8e5b52b76e \
     --hash=sha256:fb5c62c77d26e04df29cb5ecd01b9ce69b6fcc9e521eb1ca193b7faa2afa7086
-linecache2==1.0.0 \
-    --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \
-    --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c
-# Using an older version of mock here prevents regressions of #5276.
 mock==1.3.0 \
-    --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \
-    --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6
-ordereddict==1.1 \
-    --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \
-    --no-binary ordereddict
-packaging==16.8 \
-    --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \
-    --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e
-parsedatetime==2.1 \
-    --hash=sha256:ce9d422165cf6e963905cd5f74f274ebf7cc98c941916169178ef93f0e557838 \
-    --hash=sha256:17c578775520c99131634e09cfca5a05ea9e1bd2a05cd06967ebece10df7af2d
-pbr==1.8.1 \
-    --hash=sha256:46c8db75ae75a056bd1cc07fa21734fe2e603d11a07833ecc1eeb74c35c72e0c \
-    --hash=sha256:e2127626a91e6c885db89668976db31020f0af2da728924b56480fc7ccf09649
-pyOpenSSL==16.2.0 \
-    --hash=sha256:26ca380ddf272f7556e48064bbcd5bd71f83dfc144f3583501c7ddbd9434ee17 \
-    --hash=sha256:7779a3bbb74e79db234af6a08775568c6769b5821faecf6e2f4143edb227516e
-pyparsing==2.1.8 \
-    --hash=sha256:2f0f5ceb14eccd5aef809d6382e87df22ca1da583c79f6db01675ce7d7f49c18 \
-    --hash=sha256:03a4869b9f3493807ee1f1cb405e6d576a1a2ca4d81a982677c0c1ad6177c56b \
-    --hash=sha256:ab09aee814c0241ff0c503cff30018219fe1fc14501d89f406f4664a0ec9fbcd \
-    --hash=sha256:6e9a7f052f8e26bcf749e4033e3115b6dc7e3c85aafcb794b9a88c9d9ef13c97 \
-    --hash=sha256:9f463a6bcc4eeb6c08f1ed84439b17818e2085937c0dee0d7674ac127c67c12b \
-    --hash=sha256:3626b4d81cfb300dad57f52f2f791caaf7b06c09b368c0aa7b868e53a5775424 \
-    --hash=sha256:367b90cc877b46af56d4580cd0ae278062903f02b8204ab631f5a2c0f50adfd0 \
-    --hash=sha256:9f1ea360086cd68681e7f4ca8f1f38df47bf81942a0d76a9673c2d23eff35b13
-pyRFC3339==1.0 \
-    --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \
-    --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535
+    --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 \
+    --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb
+parsedatetime==2.4 \
+    --hash=sha256:3d817c58fb9570d1eec1dd46fa9448cd644eeed4fb612684b02dfda3a79cb84b \
+    --hash=sha256:9ee3529454bf35c40a77115f5a596771e59e1aee8c53306f346c461b8e913094
+pbr==5.1.3 \
+    --hash=sha256:8257baf496c8522437e8a6cfe0f15e00aedc6c0e0e7c9d55eeeeab31e0853843 \
+    --hash=sha256:8c361cc353d988e4f5b998555c88098b9d5964c2e11acf7b0d21925a66bb5824
+pyOpenSSL==19.0.0 \
+    --hash=sha256:aeca66338f6de19d1aa46ed634c3b9ae519a64b458f8468aec688e7e3c20f200 \
+    --hash=sha256:c727930ad54b10fc157015014b666f2d8b41f70c0d03e83ab67624fd3dd5d1e6
+pyRFC3339==1.1 \
+    --hash=sha256:67196cb83b470709c580bb4738b83165e67c6cc60e1f2e4f286cfcb402a926f4 \
+    --hash=sha256:81b8cbe1519cdb79bed04910dd6fa4e181faf8c88dff1e1b987b5f7ab23a5b1a
+pycparser==2.19 \
+    --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3
+pyparsing==2.3.1 \
+    --hash=sha256:66c9268862641abcac4a96ba74506e594c884e3f57690a696d21ad8210ed667a \
+    --hash=sha256:f6c5ef0d7480ad048c054c37632c67fca55299990fff127850181659eea33fc3
 python-augeas==0.5.0 \
-    --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \
-    --no-binary python-augeas
-pytz==2015.7 \
-    --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \
-    --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \
-    --hash=sha256:ead4aefa7007249e05e51b01095719d5a8dd95760089f5730aac5698b1932918 \
-    --hash=sha256:3cca0df08bd0ed98432390494ce3ded003f5e661aa460be7a734bffe35983605 \
-    --hash=sha256:3ede470d3d17ba3c07638dfa0d10452bc1b6e5ad326127a65ba77e6aaeb11bec \
-    --hash=sha256:68c47964f7186eec306b13629627722b9079cd4447ed9e5ecaecd4eac84ca734 \
-    --hash=sha256:dd5d3991950aae40a6c81de1578942e73d629808cefc51d12cd157980e6cfc18 \
-    --hash=sha256:a77c52062c07eb7c7b30545dbc73e32995b7e117eea750317b5cb5c7a4618f14 \
-    --hash=sha256:81af9aec4bc960a9a0127c488f18772dae4634689233f06f65443e7b11ebeb51 \
-    --hash=sha256:e079b1dadc5c06246cc1bb6fe1b23a50b1d1173f2edd5104efd40bb73a28f406 \
-    --hash=sha256:fbd26746772c24cb93c8b97cbdad5cb9e46c86bbdb1b9d8a743ee00e2fb1fc5d \
-    --hash=sha256:99266ef30a37e43932deec2b7ca73e83c8dbc3b9ff703ec73eca6b1dae6befea \
-    --hash=sha256:8b6ce1c993909783bc96e0b4f34ea223bff7a4df2c90bdb9c4e0f1ac928689e3
-requests==2.20.0 \
-    --hash=sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c \
-    --hash=sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279
-six==1.10.0 \
-    --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \
-    --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a
-traceback2==1.4.0 \
-    --hash=sha256:8253cebec4b19094d67cc5ed5af99bf1dba1285292226e98a31929f87a5d6b23 \
-    --hash=sha256:05acc67a09980c2ecfedd3423f7ae0104839eccb55fc645773e1caa0951c3030
-unittest2==1.1.0 \
-    --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \
-    --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579
-zope.component==4.2.2 \
-    --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \
-    --no-binary zope.component
-zope.event==4.1.0 \
-    --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \
-    --no-binary zope.event
-zope.interface==4.1.3 \
-    --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \
-    --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \
-    --hash=sha256:6788416f7ea7f5b8a97be94825377aa25e8bdc73463e07baaf9858b29e737077 \
-    --hash=sha256:6f3230f7254518201e5a3708cbb2de98c848304f06e3ded8bfb39e5825cba2e1 \
-    --hash=sha256:5fa575a5240f04200c3088427d0d4b7b737f6e9018818a51d8d0f927a6a2517a \
-    --hash=sha256:522194ad6a545735edd75c8a83f48d65d1af064e432a7d320d64f56bafc12e99 \
-    --hash=sha256:e8c7b2d40943f71c99148c97f66caa7f5134147f57423f8db5b4825099ce9a09 \
-    --hash=sha256:279024f0208601c3caa907c53876e37ad88625f7eaf1cb3842dbe360b2287017 \
-    --hash=sha256:2e221a9eec7ccc58889a278ea13dcfed5ef939d80b07819a9a8b3cb1c681484f \
-    --hash=sha256:69118965410ec86d44dc6b9017ee3ddbd582e0c0abeef62b3a19dbf6c8ad132b \
-    --hash=sha256:d04df8686ec864d0cade8cf199f7f83aecd416109a20834d568f8310ded12dea \
-    --hash=sha256:e75a947e15ee97e7e71e02ea302feb2fc62d3a2bb4668bf9dfbed43a506ac7e7 \
-    --hash=sha256:4e45d22fb883222a5ab9f282a116fec5ee2e8d1a568ccff6a2d75bbd0eb6bcfc \
-    --hash=sha256:bce9339bb3c7a55e0803b63d21c5839e8e479bc85c4adf42ae415b72f94facb2 \
-    --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \
-    --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \
-    --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392
-requests-toolbelt==0.8.0 \
-    --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \
-    --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5
-chardet==3.0.2 \
-    --hash=sha256:4f7832e7c583348a9eddd927ee8514b3bf717c061f57b21dbe7697211454d9bb \
-    --hash=sha256:6ebf56457934fdce01fb5ada5582762a84eed94cad43ed877964aebbdd8174c0
-urllib3==1.21.1 \
-    --hash=sha256:8ed6d5c1ff9d6ba84677310060d6a3a78ca3072ce0684cb3c645023009c114b1 \
-    --hash=sha256:b14486978518ca0901a76ba973d7821047409d7f726f22156b24e83fd71382a5
-certifi==2017.4.17 \
-    --hash=sha256:f4318671072f030a33c7ca6acaef720ddd50ff124d1388e50c1bda4cbd6d7010 \
-    --hash=sha256:f7527ebf7461582ce95f7a9e03dd141ce810d40590834f4ec20cddd54234c10a
+    --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2
+pytz==2018.9 \
+    --hash=sha256:32b0891edff07e28efe91284ed9c31e123d84bea3fd98e1f72be2508f43ef8d9 \
+    --hash=sha256:d5f05e487007e29e03409f9398d074e158d920d36eb82eaf66fb1136b0c5374c
+requests==2.21.0 \
+    --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \
+    --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b
+requests-toolbelt==0.9.1 \
+    --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \
+    --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0
+six==1.12.0 \
+    --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
+    --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73
+urllib3==1.24.2 \
+    --hash=sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0 \
+    --hash=sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3
+zope.component==4.5 \
+    --hash=sha256:6edfd626c3b593b72895a8cfcf79bff41f4619194ce996a85bce31ac02b94e55 \
+    --hash=sha256:984a06ba3def0b02b1117fa4c45b56e772e8c29c0340820fbf367e440a93a3a4
+zope.deferredimport==4.3 \
+    --hash=sha256:2ddef5a7ecfff132a2dd796253366ecf9748a446e30f1a0b3a636aec9d9c05c5 \
+    --hash=sha256:4aae9cbacb2146cca58e62be0a914f0cec034d3b2d41135ea212ca8a96f4b5ec
+zope.deprecation==4.4.0 \
+    --hash=sha256:0d453338f04bacf91bbfba545d8bcdf529aa829e67b705eac8c1a7fdce66e2df \
+    --hash=sha256:f1480b74995958b24ce37b0ef04d3663d2683e5d6debc96726eff18acf4ea113
+zope.event==4.4 \
+    --hash=sha256:69c27debad9bdacd9ce9b735dad382142281ac770c4a432b533d6d65c4614bcf \
+    --hash=sha256:d8e97d165fd5a0997b45f5303ae11ea3338becfe68c401dd88ffd2113fe5cae7
+zope.hookable==4.2.0 \
+    --hash=sha256:22886e421234e7e8cedc21202e1d0ab59960e40a47dd7240e9659a2d82c51370 \
+    --hash=sha256:39912f446e45b4e1f1951b5ffa2d5c8b074d25727ec51855ae9eab5408f105ab \
+    --hash=sha256:3adb7ea0871dbc56b78f62c4f5c024851fc74299f4f2a95f913025b076cde220 \
+    --hash=sha256:3d7c4b96341c02553d8b8d71065a9366ef67e6c6feca714f269894646bb8268b \
+    --hash=sha256:4e826a11a529ed0464ffcecf34b0b7bd1b4928dd5848c5c61bedd7833e8f4801 \
+    --hash=sha256:700d68cc30728de1c4c62088a981c6daeaefdf20a0d81995d2c0b7f442c5f88c \
+    --hash=sha256:77c82a430cedfbf508d1aa406b2f437363c24fa90c73f577ead0fb5295749b83 \
+    --hash=sha256:c1df3929a3666fc5a0c80d60a0c1e6f6ef97c7f6ed2f1b7cf49f3e6f3d4dde15 \
+    --hash=sha256:dba8b2dd2cd41cb5f37bfa3f3d82721b8ae10e492944e48ddd90a439227f2893 \
+    --hash=sha256:f492540305b15b5591bd7195d61f28946bb071de071cee5d68b6b8414da90fd2
+zope.interface==4.6.0 \
+    --hash=sha256:086707e0f413ff8800d9c4bc26e174f7ee4c9c8b0302fbad68d083071822316c \
+    --hash=sha256:1157b1ec2a1f5bf45668421e3955c60c610e31913cc695b407a574efdbae1f7b \
+    --hash=sha256:11ebddf765bff3bbe8dbce10c86884d87f90ed66ee410a7e6c392086e2c63d02 \
+    --hash=sha256:14b242d53f6f35c2d07aa2c0e13ccb710392bcd203e1b82a1828d216f6f6b11f \
+    --hash=sha256:1b3d0dcabc7c90b470e59e38a9acaa361be43b3a6ea644c0063951964717f0e5 \
+    --hash=sha256:20a12ab46a7e72b89ce0671e7d7a6c3c1ca2c2766ac98112f78c5bddaa6e4375 \
+    --hash=sha256:298f82c0ab1b182bd1f34f347ea97dde0fffb9ecf850ecf7f8904b8442a07487 \
+    --hash=sha256:2f6175722da6f23dbfc76c26c241b67b020e1e83ec7fe93c9e5d3dd18667ada2 \
+    --hash=sha256:3b877de633a0f6d81b600624ff9137312d8b1d0f517064dfc39999352ab659f0 \
+    --hash=sha256:4265681e77f5ac5bac0905812b828c9fe1ce80c6f3e3f8574acfb5643aeabc5b \
+    --hash=sha256:550695c4e7313555549aa1cdb978dc9413d61307531f123558e438871a883d63 \
+    --hash=sha256:5f4d42baed3a14c290a078e2696c5f565501abde1b2f3f1a1c0a94fbf6fbcc39 \
+    --hash=sha256:62dd71dbed8cc6a18379700701d959307823b3b2451bdc018594c48956ace745 \
+    --hash=sha256:7040547e5b882349c0a2cc9b50674b1745db551f330746af434aad4f09fba2cc \
+    --hash=sha256:7e099fde2cce8b29434684f82977db4e24f0efa8b0508179fce1602d103296a2 \
+    --hash=sha256:7e5c9a5012b2b33e87980cee7d1c82412b2ebabcb5862d53413ba1a2cfde23aa \
+    --hash=sha256:81295629128f929e73be4ccfdd943a0906e5fe3cdb0d43ff1e5144d16fbb52b1 \
+    --hash=sha256:95cc574b0b83b85be9917d37cd2fad0ce5a0d21b024e1a5804d044aabea636fc \
+    --hash=sha256:968d5c5702da15c5bf8e4a6e4b67a4d92164e334e9c0b6acf080106678230b98 \
+    --hash=sha256:9e998ba87df77a85c7bed53240a7257afe51a07ee6bc3445a0bf841886da0b97 \
+    --hash=sha256:a0c39e2535a7e9c195af956610dba5a1073071d2d85e9d2e5d789463f63e52ab \
+    --hash=sha256:a15e75d284178afe529a536b0e8b28b7e107ef39626a7809b4ee64ff3abc9127 \
+    --hash=sha256:a6a6ff82f5f9b9702478035d8f6fb6903885653bff7ec3a1e011edc9b1a7168d \
+    --hash=sha256:b639f72b95389620c1f881d94739c614d385406ab1d6926a9ffe1c8abbea23fe \
+    --hash=sha256:bad44274b151d46619a7567010f7cde23a908c6faa84b97598fd2f474a0c6891 \
+    --hash=sha256:bbcef00d09a30948756c5968863316c949d9cedbc7aabac5e8f0ffbdb632e5f1 \
+    --hash=sha256:d788a3999014ddf416f2dc454efa4a5dbeda657c6aba031cf363741273804c6b \
+    --hash=sha256:eed88ae03e1ef3a75a0e96a55a99d7937ed03e53d0cffc2451c208db445a2966 \
+    --hash=sha256:f99451f3a579e73b5dd58b1b08d1179791d49084371d9a47baad3b22417f0317
+zope.proxy==4.3.1 \
+    --hash=sha256:0cbcfcafaa3b5fde7ba7a7b9a2b5f09af25c9b90087ad65f9e61359fed0ca63b \
+    --hash=sha256:3de631dd5054a3a20b9ebff0e375f39c0565f1fb9131200d589a6a8f379214cd \
+    --hash=sha256:5429134d04d42262f4dac25f6dea907f6334e9a751ffc62cb1d40226fb52bdeb \
+    --hash=sha256:563c2454b2d0f23bca54d2e0e4d781149b7b06cb5df67e253ca3620f37202dd2 \
+    --hash=sha256:5bcf773345016b1461bb07f70c635b9386e5eaaa08e37d3939dcdf12d3fdbec5 \
+    --hash=sha256:8d84b7aef38c693874e2f2084514522bf73fd720fde0ce2a9352a51315ffa475 \
+    --hash=sha256:90de9473c05819b36816b6cb957097f809691836ed3142648bf62da84b4502fe \
+    --hash=sha256:dd592a69fe872445542a6e1acbefb8e28cbe6b4007b8f5146da917e49b155cc3 \
+    --hash=sha256:e7399ab865399fce322f9cefc6f2f3e4099d087ba581888a9fea1bbe1db42a08 \
+    --hash=sha256:e7d1c280d86d72735a420610df592aac72332194e531a8beff43a592c3a1b8eb \
+    --hash=sha256:e90243fee902adb0c39eceb3c69995c0f2004bc3fdb482fbf629efc656d124ed
 
 # Contains the requirements for the letsencrypt package.
 #
@@ -1230,18 +1314,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.29.1 \
-    --hash=sha256:2ba2c60fd1969e75d3e5048d3f7d95afd0949670b39a6a0037ba4a594e9f26a5 \
-    --hash=sha256:6fc604d207c48b95dea3458bb33a11b17aa625628eb197927ffee8b458f62692
-acme==0.29.1 \
-    --hash=sha256:4be3848f8813c455021f13519642d8ec2746b78d4d0bc2ae04c3dcb1d8862f60 \
-    --hash=sha256:a2e203ade83cd1eaf19112004a63073830211cf7759d437f634babb08c49b47c
-certbot-apache==0.29.1 \
-    --hash=sha256:8d8b6b7c5f333cf5297153c6a1eacc09b4a5c73e8f93544800b3ad016d5e34d0 \
-    --hash=sha256:c3af1c66c86cfeef7dac4fe9b16c7c755ebd12bc526408c27781bd34b9de8128
-certbot-nginx==0.29.1 \
-    --hash=sha256:5ba3a7d93d3ce317fb8b3d0222c708fb79e96c7a9b1ba56e12e46892c2d12869 \
-    --hash=sha256:0c1205ebb91eef4b7d15293c6778ffc962d09563b315120b2d226348d751e38d
+certbot==0.36.0 \
+    --hash=sha256:486cee6c4861762fe4a94b4f44f7d227034d026d1a8d7ba2911ef4e86a737613 \
+    --hash=sha256:bf6745b823644cdca8461150455aeb67d417f87f80b9ec774c716e9587ef20a2
+acme==0.36.0 \
+    --hash=sha256:5570c8e87383fbc733224fd0f7d164313b67dd9c21deafe9ddc8e769441f0c86 \
+    --hash=sha256:0461ee3c882d865e98e624561843dc135fa1a1412b15603d7ebfbb392de6a668
+certbot-apache==0.36.0 \
+    --hash=sha256:2537f7fb67a38b6d1ed5ee79f6a799090ca609695ac3799bb840b2fb677ac98d \
+    --hash=sha256:458d20a3e9e8a88563d3deb0bbe38752bd2b80100f0e5854e4069390c1b4e5cd
+certbot-nginx==0.36.0 \
+    --hash=sha256:4303b54adf2030671c54bb3964c1f43aec0f677045e0cdb6d4fb931268d08310 \
+    --hash=sha256:4c34e6114dd8204b6667f101579dd9ab2b38fef0dd5a15702585edcb2aefb322
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
@@ -1271,7 +1355,6 @@ from distutils.version import StrictVersion
 from hashlib import sha256
 from os import environ
 from os.path import join
-from pipes import quote
 from shutil import rmtree
 try:
     from subprocess import check_output
@@ -1291,7 +1374,7 @@ except ImportError:
                 cmd = popenargs[0]
             raise CalledProcessError(retcode, cmd)
         return output
-from sys import exit, version_info
+import sys
 from tempfile import mkdtemp
 try:
     from urllib2 import build_opener, HTTPHandler, HTTPSHandler
@@ -1313,7 +1396,7 @@ maybe_argparse = (
     [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/'
       'argparse-1.4.0.tar.gz',
       '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')]
-    if version_info < (2, 7, 0) else [])
+    if sys.version_info < (2, 7, 0) else [])
 
 
 PACKAGES = maybe_argparse + [
@@ -1322,9 +1405,9 @@ PACKAGES = maybe_argparse + [
      'pip-{0}.tar.gz'.format(PIP_VERSION),
      '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'),
     # This version of setuptools has only optional dependencies:
-    ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/'
-     'setuptools-29.0.1.tar.gz',
-     'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'),
+    ('37/1b/b25507861991beeade31473868463dad0e58b1978c209de27384ae541b0b/'
+     'setuptools-40.6.3.zip',
+     '3b474dad69c49f0d2d86696b68105f3a6f195f7ab655af12ef9a9c326d2b08f8'),
     ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/'
      'wheel-0.29.0.tar.gz',
      '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648')
@@ -1394,11 +1477,9 @@ def get_index_base():
 
 
 def main():
-    pip_version = StrictVersion(check_output(['pip', '--version'])
+    python = sys.executable or 'python'
+    pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version'])
                                 .decode('utf-8').split()[1])
-    min_pip_version = StrictVersion(PIP_VERSION)
-    if pip_version >= min_pip_version:
-        return 0
     has_pip_cache = pip_version >= StrictVersion('6.0')
     index_base = get_index_base()
     temp = mkdtemp(prefix='pipstrap-')
@@ -1407,12 +1488,12 @@ def main():
                                      temp,
                                      digest)
                      for path, digest in PACKAGES]
-        check_output('pip install --no-index --no-deps -U ' +
-                     # Disable cache since we're not using it and it otherwise
-                     # sometimes throws permission warnings:
-                     ('--no-cache-dir ' if has_pip_cache else '') +
-                     ' '.join(quote(d) for d in downloads),
-                     shell=True)
+        # Calling pip as a module is the preferred way to avoid problems about pip self-upgrade.
+        command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U']
+        # Disable cache since it is not used and it otherwise sometimes throws permission warnings:
+        command.extend(['--no-cache-dir'] if has_pip_cache else [])
+        command.extend(downloads)
+        check_output(command)
     except HashError as exc:
         print(exc)
     except Exception:
@@ -1425,7 +1506,7 @@ def main():
 
 
 if __name__ == '__main__':
-    exit(main())
+    sys.exit(main())
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
@@ -1509,6 +1590,24 @@ else
     exit 0
   fi
 
+  DeterminePythonVersion "NOCRASH"
+  # Don't warn about file permissions if the user disabled the check or we
+  # can't find an up-to-date Python.
+  if [ "$PYVER" -ge "$MIN_PYVER" -a "$NO_PERMISSIONS_CHECK" != 1 ]; then
+    # If the script fails for some reason, don't break certbot-auto.
+    set +e
+    # Suppress unexpected error output.
+    CHECK_PERM_OUT=$(CheckPathPermissions "$LE_PYTHON" "$0" 2>/dev/null)
+    CHECK_PERM_STATUS="$?"
+    set -e
+    # Only print output if the script ran successfully and it actually produced
+    # output. The latter check resolves
+    # https://github.com/certbot/certbot/issues/7012.
+    if [ "$CHECK_PERM_STATUS" = 0 -a -n "$CHECK_PERM_OUT" ]; then
+      error "$CHECK_PERM_OUT"
+    fi
+  fi
+
   if [ "$NO_SELF_UPGRADE" != 1 ]; then
     TEMP_DIR=$(TempDir)
     trap 'rm -rf "$TEMP_DIR"' EXIT
@@ -1665,7 +1764,6 @@ if __name__ == '__main__':
 
 UNLIKELY_EOF
     # ---------------------------------------------------------------------------
-    DeterminePythonVersion "NOCRASH"
     if [ "$PYVER" -lt "$MIN_PYVER" ]; then
       error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates."
     elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then

certbot-ci/MANIFEST.in

@@ -0,0 +1 @@
+recursive-include certbot_integration_tests/assets *

certbot-ci/certbot_integration_tests/.coveragerc

@@ -0,0 +1,8 @@
+[run]
+# Avoid false warnings because certbot packages are not installed in the thread that executes
+# the coverage: indeed, certbot is launched as a CLI from a subprocess.
+disable_warnings = module-not-imported,no-data-collected
+
+[report]
+# Exclude unit tests in coverage during integration tests.
+omit = **/*_test.py,**/tests/*,**/dns_common*,**/certbot_nginx/parser_obj.py

certbot-ci/certbot_integration_tests/__init__.py

@@ -0,0 +1 @@
+"""Package certbot_integration_test is for tests that require a live acme ca server instance"""

certbot-ci/certbot_integration_tests/assets/cert.pem

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFlTCCA32gAwIBAgIUR3wbM8qFE68f8NxfciHhUjR1GeUwDQYJKoZIhvcNAQEL
+BQAwWTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAwwJbmdpbngud3RmMCAX
+DTE5MDQxODIwMDUwM1oYDzIyOTMwMTMwMjAwNTAzWjBZMQswCQYDVQQGEwJBVTET
+MBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQ
+dHkgTHRkMRIwEAYDVQQDDAluZ2lueC53dGYwggIiMA0GCSqGSIb3DQEBAQUAA4IC
+DwAwggIKAoICAQC/W+yxYE0PWJOS4df71Yx596fDjW03I9JZuu9kfP7mneMgy+OC
+HyRm0TEhl6FPUp9tD9YeEHloUZNjHEOg/qrnbEOspv3Ha3RFinzrzkMwbzEPR3Xf
+0go+aVsWelDhapFl8fccw4tWwijVZQquhBsWOUnPenS3Txe96kEv2NNJlJ0qFUa+
+rOTruzRzOzlbgKv5WRb4+BxxWonHLkAQ5IT87GBlsCerVIyPD+BnZveZGl6e9oMH
+ZlZvUT6aWRnzFWjAnQGiJpVIw7l9r4EW0jq1z7wqb37FrqrFbtWrOfUZVE7AlqXH
+aKIR82/xwkcZfFk3sCAM0IcZc8B2SDLi4zNZtDivW6qQgTC/3z5yf1hnJ+j00dtE
+X5qYlgXRaM2raOn31lxcerk5pjgagQ7Zj+v3YZS0QnenrgyXJcdnXLDj+cIARzx4
+QHtoO0nyP0RJqxvwX/H98513JTkeqFBc/Bx11UWYsUv20Qoo9IAuz0VDARu6rquu
+k9anv56yvxo77qZ8r80l3z8eMyDA+UjuSD2p1Za09RAHfva7o8rMUqULHNQ4pfFH
+JIUozHoinAg/9lBC/W80fcbILks+Sdi6E9WQ0n8PLl7oFLx9prEDCycKuC0z76J/
+Shb6R6sWr1YtzUFUc5EH2g9pMriaqT8uGO4CMOeRemXahrdT/H+Xg5m4TQIDAQAB
+o1MwUTAdBgNVHQ4EFgQU46gJeu9ZOfTQ6c4vfbWbSLUpEMowHwYDVR0jBBgwFoAU
+46gJeu9ZOfTQ6c4vfbWbSLUpEMowDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B
+AQsFAAOCAgEAcnfkXDUTsEGs0MleegkGbRCVy72a3U7tv1KVTLB8qLPc3tpPJJoT
+D4PbOuw9+yIE+HetZTZooOpaZoorLQdiwAEjlQ44RVuXSHSARQ8KW9ZZeiWN/Qvl
+Ip4xJ/cHxcKTFKSc/99o8M+kmPKEXF9SUMfKPc5jXarNxCsnA3VriYqJ+CnYEox2
+duNUEe3A9Y2d8ZxjmscBqlcXpk1kFwsCRT5UYVoUYwyjYznLkO5A+GJ0ZnMyRMQp
+obUiB34hUrNgyOaBvizk+pNh9EV4rEBPRQwhy4vDMco4AjQcwLWQAQ9G4GSt/E+Q
+62XdVDa6CAuOvBCudDPki7kEqNLbj1tMY1K/gsbgb6TYA/xTOVulAnqm4OEZ2svJ
+0Jqw3BzMfRTaxsNU6jxm8WehVL15GjoJUzfs7Te+l7Vm/QNc1Dv2pmEhVfBibwMa
+YxUZ8ClQtQ1lsOpne97Og0p/Cm93kKELNBLTjzXtpXGGPPYisAyNwe0Hadq8SiOd
+pXeNwXa5vHOXHv8xBENzBvFJ3TRN2GmMlHBp/eOfVUx/huNSpcnh2gO3fn5EbMj7
+43IaR133JW5yWbneYAMJOEAMdEB5EthRmEDtLVA7kLqLc/ywFTQ4VbS2b+PsOr5O
+501nzt0OTMMEz+UafvGXj5OPJBhe26RtnYXzVwwLfto/F5udM5zglWo=
+-----END CERTIFICATE-----

certbot-ci/certbot_integration_tests/assets/key.pem

@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQC/W+yxYE0PWJOS
+4df71Yx596fDjW03I9JZuu9kfP7mneMgy+OCHyRm0TEhl6FPUp9tD9YeEHloUZNj
+HEOg/qrnbEOspv3Ha3RFinzrzkMwbzEPR3Xf0go+aVsWelDhapFl8fccw4tWwijV
+ZQquhBsWOUnPenS3Txe96kEv2NNJlJ0qFUa+rOTruzRzOzlbgKv5WRb4+BxxWonH
+LkAQ5IT87GBlsCerVIyPD+BnZveZGl6e9oMHZlZvUT6aWRnzFWjAnQGiJpVIw7l9
+r4EW0jq1z7wqb37FrqrFbtWrOfUZVE7AlqXHaKIR82/xwkcZfFk3sCAM0IcZc8B2
+SDLi4zNZtDivW6qQgTC/3z5yf1hnJ+j00dtEX5qYlgXRaM2raOn31lxcerk5pjga
+gQ7Zj+v3YZS0QnenrgyXJcdnXLDj+cIARzx4QHtoO0nyP0RJqxvwX/H98513JTke
+qFBc/Bx11UWYsUv20Qoo9IAuz0VDARu6rquuk9anv56yvxo77qZ8r80l3z8eMyDA
++UjuSD2p1Za09RAHfva7o8rMUqULHNQ4pfFHJIUozHoinAg/9lBC/W80fcbILks+
+Sdi6E9WQ0n8PLl7oFLx9prEDCycKuC0z76J/Shb6R6sWr1YtzUFUc5EH2g9pMria
+qT8uGO4CMOeRemXahrdT/H+Xg5m4TQIDAQABAoICAAGGL+pxw+tdXz+KQPgmiUnn
+aRSrqbUIugIw9Pst67HWjBqUxSkiKl4PSH7mAEjrdY2e1KvEodLs42mkrf04ShAx
+0pArfFX8Sx7KrZgLOonGOPPQM+YmfCJnIGybaM2C1cmkFb3K6O81+LFKbr1ZHAYf
+SrE2XnufS6cdmItTBMvPPTk6lieqpOAjy5UnYZuS+Muxo/czsrZMbFCD08rOpyiE
+kXf94TMCJ2R0UetA7LPxe9N0TzLd485bLU55azV+dCkklwC9oe7EcFPJ9BNEdWdB
+UlRcMvxMGdwct+L3QTaEb2QlTwi5kqDl+XxJeduAHA3Pf1Haz1iqjVvj01PvT1di
+Cs0+ZeFBsa+BfiGDe9ONwuSQljda1CuI+vDv5bGUExulOSG1dHJ7RK9PBaXFaR/b
+/9tRBwAw1Erm7s1JIkjda5Oc46jFb3HzDaZYB1n5hUmEIrYM8HhUOGITyVT3hxDO
+AWlaV3aveQ0MmMXLptVXDgbjPGbWDGMLD9d5vUE9R7IyOLeXOmjthYlCH2rj378M
+r2PkgX2tD0A/yoEZ8XCFdtBWSVajLdL0/gkm7sKosMABBy3yrSCxbHeq5TFuTAXA
+hOdypX4NOZkA6WJU+hn3GkQyIScLqSrvGRA9kzHGoEWVZDKkB9DXg+dmTARZDWXD
+mCnHkJo6+FcbhUpXniuZAoIBAQDmE94vvdstB+HEtXxN1uNDY7H8gPc/BUonU6a9
+G5YOIbjByCfEDcXF8AUWekc6lc8DNG3ydx0dnb2ZAkxmdlsaD8GLqHGILzlSsOwR
+sez8nR4+4n9vYMfx9Qal8Ren5xEP9Z9sJcNqbKVGta1WFtQzrgYbpVXXf/Luv0xS
+YoVK8KaEACciD6XX4wmajrAXPPQgThvqQtXuTn/AxWsUDg1DK0tw1VRUuOJuJwpw
+f6qocM9AyqUNvdeVyjFx8Slag34ZI7fmxPtHX/e6opTg3zVXab1Ow8AMICOHMRL6
+m5/+wnWa9xMoKI4kYfk/QFqeTccnLDlwi6kQM8WRfbwr9AyPAoIBAQDU60wrX6Lm
+0vIfngv1/4j/w+AGAwjvxiuJ7Q7LwQ2fGsZGOIfMK/lpBxCn543kGbQT+KQKNOjO
++EywObftnJ6Y2+om2NoLkCnCiptsfr5WlN8pxtIPQu2iu5xXA67WpQv4Nc4769PM
+wJGVW3pmPKi6H0QjjqYAZd1NAXdN9Au14zZVh3KBWoz82kTHWKSL6Ld1UClG728W
+k/moyCFFMMGTXX/LVliQzDVLM6L5jbAOaG317qAuxZIqFJ9NLwHFW9uH/i1S6Qfp
++lOmOfVYKu1O/qh1DUBQfuJkR1XIn2ifZEjxOsxeTmWu1LXpyoZy526JRu49pk8Z
+DdEu+w7hsdNjAoIBAD1YWsub8Y6GJXpPcX9HpnzXXiOXN1VEUcs+kJyneFD4SMzS
+U1gA3BS0tIaTv94tB28xUYdunwLAhkb/x+Mh95RxUwert+m5va0Ao1DsgeWw9tmJ
+hrTptyYaUNV5/Pa1s2Tv9rvdLcd4hHDgDAGCQL4uzk4cvVCiOuHRe8YTorqig6N6
+bvSz+2IelPbyyJzJkcXzTZoei+/oWkPJ340PWhXou0qwdrXIPgdkvXHVeGlE+t2p
+qmyJi6vSp3Bb/sy1dq+5SFVtfBpBykmnA88ZdJ2EAge4RcJ150MqoIbVa8l/i9/v
+tNnmRlAJF233+LFwx4L4VbBebIt3YlwyjDOj9J0CggEAIknKOGnsV/O8ni7bikAe
+leG7X/x5IfPt6wZMDbAHO4oaSBCufcjPH4TNv9xgU014XIb8E9C1dS8zWmXRIujH
++aHgsWTWqGoM75FWukAm8taCob2s8lw63KwN301uiI6HwO8ZSTkPILgaOc1DhtdZ
+7K9AT+GXBhVhcBc+WUVl5WKzy05GuGIWtlmIHfo+dXGCqdfA7fV9FEu8NtwTz4qs
+gcja3aoIFTltk7C7HCkfIxLaMnK9RQr4IOK1TL63MEs8rUfXkLSKW7m+YtSOmCZB
+lSkZg9AgfVYRq0h5nhddx91kicSISN+jLGaA7Sd6Q2LVwDG2CCOSNVyuRTyVBu+W
+NQKCAQAWN6vB6oToNIoBLdOThm0HD07cNHcrnBjtaKsYsQDgqbr2m8LRCRzNRML4
+jG0IAOWpuCiEGsgUPxywiI1Ufvyq7ZSNT1QQNzCR47NM3Ve6S2abrQkMIk9VJ+za
+CB9c1BH92GokoRxqswb/BiMttG2EIP8L8/pSRYEcVnaaxAkf9QOhEwj4LJPGX0mS
+t7kWIUVHPdFJ67F25dYr3mUHgyV+QJupQICkkkgY3nBOU1fS42vAugaxqH0wAP3T
+53FlpY3NuE7+kYC3FjfcBer99F1pOac3X9jxhk26w9dr2/QNA33xhDXHKYvoLUCG
+RPQylahJByU7IrtQzSCf/RE7q4v0
+-----END PRIVATE KEY-----