Merge branch 'master-uptsream' into apache-integration-tests

# Conflicts:
#	.travis.yml
#	tox.ini

.codecov.yml

@@ -4,11 +4,15 @@ coverage:
       default: off
       linux:
         flags: linux
-        target: auto
+        # Fixed target instead of auto set by #7173, can
+        # be removed when flags in Codecov are added back.
+        target: 97.4
         threshold: 0.1
         base: auto
       windows:
         flags: windows
-        target: auto
+        # Fixed target instead of auto set by #7173, can
+        # be removed when flags in Codecov are added back.
+        target: 97.7
         threshold: 0.1
         base: auto

.coveragerc

@@ -1,2 +1,5 @@
+[run]
+omit = */setup.py
+
 [report]
 omit = */setup.py

.github/stale.yml

@@ -0,0 +1,35 @@
+# Configuration for https://github.com/marketplace/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 365
+
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+# When changing this value, be sure to also update markComment below.
+daysUntilClose: 30
+
+# Ignore issues with an assignee (defaults to false)
+exemptAssignees: true
+
+# Label to use when marking as stale
+staleLabel: needs-update
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+  We've made a lot of changes to Certbot since this issue was opened. If you
+  still have this issue with an up-to-date version of Certbot, can you please
+  add a comment letting us know? This helps us to better see what issues are
+  still affecting our users. If there is no activity in the next 30 days, this
+  issue will be automatically closed.
+
+# Comment to post when closing a stale Issue or Pull Request.
+closeComment: >
+  This issue has been closed due to lack of activity, but if you think it
+  should be reopened, please open a new issue with a link to this one and we'll
+  take a look.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 1
+
+# Don't mark pull requests as stale.
+only: issues

.gitignore

@@ -44,3 +44,8 @@ tests/letstest/venv/
 
 # docker files
 .docker
+
+# certbot tests
+.certbot_test_workspace
+**/assets/pebble*
+**/assets/challtestsrv*

.travis.yml

@@ -1,11 +1,26 @@
 language: python
+dist: xenial
 
 cache:
     directories:
         - $HOME/.cache/pip
 
 before_script:
-  - 'if [ $TRAVIS_OS_NAME = osx ] ; then ulimit -n 1024 ; fi'
+  # Install required apt packages
+  - |
+    if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then
+      ./certbot-auto --non-interactive --os-packages-only
+      sudo -E apt-get -yq --no-install-suggests --no-install-recommends install nginx-light
+      sudo -E /etc/init.d/nginx stop
+      sudo -E apt-get -yq --no-install-suggests --no-install-recommends install apache2
+      sudo -E /etc/init.d/apache2 stop
+      sudo -E chmod 777 -R /var/lib/apache2/module
+    fi
+  - 'if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ulimit -n 1024 ; fi'
+  # On Travis, the fastest parallelization for integration tests has proved to be 4.
+  - 'if [[ "$TOXENV" == *"integration"* ]]; then export PYTEST_ADDOPTS="--numprocesses 4"; fi'
+  # Use Travis retry feature for farm tests since they are flaky
+  - 'if [[ "$TOXENV" == "travis-test-farm"* ]]; then export TRAVIS_RETRY=travis_retry; fi'
   - export TOX_TESTENV_PASSENV=TRAVIS
 
 # Only build pushes to the master branch, PRs, and branches beginning with
@@ -14,6 +29,9 @@ before_script:
 # is a cap of on the number of simultaneous runs.
 branches:
   only:
+    # apache-parser-v2 is a temporary branch for doing work related to
+    # rewriting the parser in the Apache plugin.
+    - apache-parser-v2
     - master
     - /^\d+\.\d+\.x$/
     - /^test-.*$/
@@ -22,34 +40,22 @@ branches:
 not-on-master: &not-on-master
   if: NOT (type = push AND branch = master)
 
-# Jobs for the extended test suite are executed for cron jobs and pushes on non-master branches.
+# Jobs for the extended test suite are executed for cron jobs and pushes to
+# non-development branches. See the explanation for apache-parser-v2 above.
 extended-test-suite: &extended-test-suite
-  if: type = cron OR (type = push AND branch != master)
+  if: type = cron OR (type = push AND branch NOT IN (apache-parser-v2, master))
 
 matrix:
   include:
     # Main test suite
     - python: "2.7"
-      env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=all TOXENV=py27_install
-      sudo: required
-      services: docker
-      <<: *not-on-master
-    - python: "2.7"
-      env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=all TOXENV=py27_install
-      sudo: required
-      services: docker
+      env: ACME_SERVER=pebble TOXENV=integration
       <<: *not-on-master
 
     # This job is always executed, including on master
     - python: "2.7"
       env: TOXENV=py27-cover FYI="py27 tests + code coverage"
 
-    - sudo: required
-      env: TOXENV=nginx_compat
-      services: docker
-      before_install:
-      addons:
-      <<: *not-on-master
     - python: "2.7"
       env: TOXENV=lint
       <<: *not-on-master
@@ -60,132 +66,151 @@ matrix:
       env: TOXENV=mypy
       <<: *not-on-master
     - python: "2.7"
-      env: TOXENV='py27-{acme,apache,certbot,dns,nginx,postfix}-oldest'
-      sudo: required
-      services: docker
+      # Ubuntu Trusty or older must be used because the oldest version of
+      # cryptography we support cannot be compiled against the version of
+      # OpenSSL in Xenial or newer.
+      dist: trusty
+      env: TOXENV='py27-{acme,apache,certbot,dns,nginx}-oldest'
       <<: *not-on-master
     - python: "3.4"
       env: TOXENV=py34
-      sudo: required
-      services: docker
       <<: *not-on-master
     - python: "3.7"
-      dist: xenial
       env: TOXENV=py37
-      sudo: required
-      services: docker
       <<: *not-on-master
-    - sudo: required
-      env: TOXENV=apache_compat
-      services: docker
-      before_install:
-      addons:
+    - python: "3.8-dev"
+      env: TOXENV=py38
       <<: *not-on-master
-    - sudo: required
-      env: TOXENV=le_auto_trusty
-      services: docker
-      before_install:
-      addons:
+    - env: TOXENV=apache_compat
+      <<: *not-on-master
+    - env: TOXENV=le_auto_xenial
       <<: *not-on-master
     - python: "2.7"
       env: TOXENV=apacheconftest-with-pebble
-      sudo: required
-      services: docker
       <<: *not-on-master
     - python: "2.7"
       env: TOXENV=nginxroundtrip
       <<: *not-on-master
 
     # Extended test suite on cron jobs and pushes to tested branches other than master
+    - env: TOXENV=nginx_compat
+      <<: *extended-test-suite
+    - python: "2.7"
+      env:
+        - TOXENV=travis-test-farm-apache2
+        - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
+      <<: *extended-test-suite
+    - python: "2.7"
+      env:
+        - TOXENV=travis-test-farm-leauto-upgrades
+        - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
+      git:
+        depth: false  # This is needed to have the history to checkout old versions of certbot-auto.
+      <<: *extended-test-suite
+    - python: "2.7"
+      env:
+        - TOXENV=travis-test-farm-certonly-standalone
+        - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
+      <<: *extended-test-suite
+    - python: "2.7"
+      env:
+        - TOXENV=travis-test-farm-sdists
+        - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
+      <<: *extended-test-suite
     - python: "3.7"
-      dist: xenial
       env: TOXENV=py37 CERTBOT_NO_PIN=1
       <<: *extended-test-suite
     - python: "2.7"
-      env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest
-      sudo: required
-      services: docker
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
       <<: *extended-test-suite
     - python: "2.7"
-      env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest
-      sudo: required
-      services: docker
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
       <<: *extended-test-suite
     - python: "2.7"
-      env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest
-      sudo: required
-      services: docker
+      env: ACME_SERVER=boulder-v1 TOXENV=integration-certbot-oldest
+      # Ubuntu Trusty or older must be used because the oldest version of
+      # cryptography we support cannot be compiled against the version of
+      # OpenSSL in Xenial or newer.
+      dist: trusty
       <<: *extended-test-suite
     - python: "2.7"
-      env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest
-      sudo: required
-      services: docker
+      env: ACME_SERVER=boulder-v2 TOXENV=integration-certbot-oldest
+      # Ubuntu Trusty or older must be used because the oldest version of
+      # cryptography we support cannot be compiled against the version of
+      # OpenSSL in Xenial or newer.
+      dist: trusty
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration-nginx-oldest
+      # Ubuntu Trusty or older must be used because the oldest version of
+      # cryptography we support cannot be compiled against the version of
+      # OpenSSL in Xenial or newer.
+      dist: trusty
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration-nginx-oldest
+      # Ubuntu Trusty or older must be used because the oldest version of
+      # cryptography we support cannot be compiled against the version of
+      # OpenSSL in Xenial or newer.
+      dist: trusty
       <<: *extended-test-suite
     - python: "3.4"
-      env: TOXENV=py34 BOULDER_INTEGRATION=v1
-      sudo: required
-      services: docker
+      env: TOXENV=py34
+      <<: *extended-test-suite
+    - python: "3.5"
+      env: TOXENV=py35
+      <<: *extended-test-suite
+    - python: "3.6"
+      env: TOXENV=py36
+      <<: *extended-test-suite
+    - python: "3.7"
+      env: TOXENV=py37
+      <<: *extended-test-suite
+    - python: "3.8-dev"
+      env: TOXENV=py38
       <<: *extended-test-suite
     - python: "3.4"
-      env: TOXENV=py34 BOULDER_INTEGRATION=v2
-      sudo: required
-      services: docker
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
+      <<: *extended-test-suite
+    - python: "3.4"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
       <<: *extended-test-suite
     - python: "3.5"
-      env: TOXENV=py35 BOULDER_INTEGRATION=v1
-      sudo: required
-      services: docker
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
       <<: *extended-test-suite
     - python: "3.5"
-      env: TOXENV=py35 BOULDER_INTEGRATION=v2
-      sudo: required
-      services: docker
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
       <<: *extended-test-suite
     - python: "3.6"
-      env: TOXENV=py36 BOULDER_INTEGRATION=v1
-      sudo: required
-      services: docker
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
       <<: *extended-test-suite
     - python: "3.6"
-      env: TOXENV=py36 BOULDER_INTEGRATION=v2
-      sudo: required
-      services: docker
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
       <<: *extended-test-suite
     - python: "3.7"
-      dist: xenial
-      env: TOXENV=py37 BOULDER_INTEGRATION=v1
-      sudo: required
-      services: docker
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
       <<: *extended-test-suite
     - python: "3.7"
-      dist: xenial
-      env: TOXENV=py37 BOULDER_INTEGRATION=v2
-      sudo: required
-      services: docker
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
       <<: *extended-test-suite
-    - sudo: required
-      env: TOXENV=le_auto_xenial
-      services: docker
+    - python: "3.8-dev"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
       <<: *extended-test-suite
-    - sudo: required
-      env: TOXENV=le_auto_jessie
-      services: docker
+    - python: "3.8-dev"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
       <<: *extended-test-suite
-    - sudo: required
-      env: TOXENV=le_auto_centos6
-      services: docker
+    - env: TOXENV=le_auto_jessie
       <<: *extended-test-suite
-    - sudo: required
-      env: TOXENV=docker_dev
-      services: docker
-      addons:
-        apt:
-          packages:  # don't install nginx and apache
-            - libaugeas0
+    - env: TOXENV=le_auto_centos6
+      <<: *extended-test-suite
+    - env: TOXENV=docker_dev
       <<: *extended-test-suite
     - language: generic
       env: TOXENV=py27
       os: osx
+      # Using this osx_image is a workaround for
+      # https://travis-ci.community/t/xcode-8-3-homebrew-outdated-error/3798.
+      osx_image: xcode10.2
       addons:
         homebrew:
           packages:
@@ -195,6 +220,9 @@ matrix:
     - language: generic
       env: TOXENV=py3
       os: osx
+      # Using this osx_image is a workaround for
+      # https://travis-ci.community/t/xcode-8-3-homebrew-outdated-error/3798.
+      osx_image: xcode10.2
       addons:
         homebrew:
           packages:
@@ -202,27 +230,17 @@ matrix:
             - python3
       <<: *extended-test-suite
 
-# container-based infrastructure
-sudo: false
-
-addons:
-  apt:
-    packages:  # Keep in sync with letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh and Boulder.
-    - python-dev
-    - python-virtualenv
-    - gcc
-    - libaugeas0
-    - libssl-dev
-    - libffi-dev
-    - ca-certificates
-    # For certbot-nginx integration testing
-    - nginx-light
-    - openssl
-
-install: "$(command -v pip || command -v pip3) install codecov tox"
-script:
-    - tox
-    - '[ -z "${BOULDER_INTEGRATION+x}" ] || (tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)'
+# tools/pip_install.py is used to pin packages to a known working version
+# except in tests where the environment variable CERTBOT_NO_PIN is set.
+# virtualenv is listed here explicitly to make sure it is upgraded when
+# CERTBOT_NO_PIN is set to work around failures we've seen when using an older
+# version of virtualenv.
+install: 'tools/pip_install.py -U codecov tox virtualenv'
+# Most of the time TRAVIS_RETRY is an empty string, and has no effect on the
+# script command. It is set only to `travis_retry` during farm tests, in
+# order to trigger the Travis retry feature, and compensate the inherent
+# flakiness of these specific tests.
+script: '$TRAVIS_RETRY tox'
 
 after_success: '[ "$TOXENV" == "py27-cover" ] && codecov -F linux'
 
@@ -230,8 +248,11 @@ notifications:
   email: false
   irc:
     channels:
-      - secure: "SGWZl3ownKx9xKVV2VnGt7DqkTmutJ89oJV9tjKhSs84kLijU6EYdPnllqISpfHMTxXflNZuxtGo0wTDYHXBuZL47w1O32W6nzuXdra5zC+i4sYQwYULUsyfOv9gJX8zWAULiK0Z3r0oho45U+FR5ZN6TPCidi8/eGU+EEPwaAw="
+      # This is set to a secure variable to prevent forks from sending
+      # notifications. This value was created by installing
+      # https://github.com/travis-ci/travis.rb and running
+      # `travis encrypt "chat.freenode.net#certbot-devel"`.
+      - secure: "EWW66E2+KVPZyIPR8ViENZwfcup4Gx3/dlimmAZE0WuLwxDCshBBOd3O8Rf6pBokEoZlXM5eDT6XdyJj8n0DLslgjO62pExdunXpbcMwdY7l1ELxX2/UbnDTE6UnPYa09qVBHNG7156Z6yE0x2lH4M9Ykvp0G0cubjPQHylAwo0="
     on_cancel: never
     on_success: never
     on_failure: always
-    use_notice: true

AUTHORS.md

@@ -5,6 +5,7 @@ Authors
 * Aaron Zuehlke
 * Ada Lovelace
 * [Adam Woodbeck](https://github.com/awoodbeck)
+* [Adrien Ferrand](https://github.com/adferrand)
 * [Aidin Gharibnavaz](https://github.com/aidin36)
 * [AJ ONeal](https://github.com/coolaj86)
 * [Alcaro](https://github.com/Alcaro)
@@ -14,6 +15,7 @@ Authors
 * [Alex Gaynor](https://github.com/alex)
 * [Alex Halderman](https://github.com/jhalderm)
 * [Alex Jordan](https://github.com/strugee)
+* [Alex Zorin](https://github.com/alexzorin)
 * [Amjad Mashaal](https://github.com/TheNavigat)
 * [Andrew Murray](https://github.com/radarhere)
 * [Anselm Levskaya](https://github.com/levskaya)
@@ -75,6 +77,7 @@ Authors
 * [Fabian](https://github.com/faerbit)
 * [Faidon Liambotis](https://github.com/paravoid)
 * [Fan Jiang](https://github.com/tcz001)
+* [Felix Lechner](https://github.com/lechner)
 * [Felix Schwarz](https://github.com/FelixSchwarz)
 * [Felix Yan](https://github.com/felixonmars)
 * [Filip Ochnik](https://github.com/filipochnik)
@@ -159,6 +162,7 @@ Authors
 * [Michael Schumacher](https://github.com/schumaml)
 * [Michael Strache](https://github.com/Jarodiv)
 * [Michael Sverdlin](https://github.com/sveder)
+* [Michael Watters](https://github.com/blackknight36)
 * [Michal Moravec](https://github.com/https://github.com/Majkl578)
 * [Michal Papis](https://github.com/mpapis)
 * [Minn Soe](https://github.com/MinnSoe)

CHANGELOG.md

@@ -2,28 +2,173 @@
 
 Certbot adheres to [Semantic Versioning](https://semver.org/).
 
-## 0.35.0 - master
+## 0.39.0 - master
 
 ### Added
 
+* Support for Python 3.8 was added to Certbot and all of its components.
+
+### Changed
+
+* Don't send OCSP requests for expired certificates
+
+### Fixed
+
 *
 
+More details about these changes can be found on our GitHub repo.
+
+## 0.38.0 - 2019-09-03
+
+### Added
+
+* Disable session tickets for Nginx users when appropriate.
+
+### Changed
+
+* If Certbot fails to rollback your server configuration, the error message
+  links to the Let's Encrypt forum. Change the link to the Help category now
+  that the Server category has been closed.
+* Replace platform.linux_distribution with distro.linux_distribution as a step
+  towards Python 3.8 support in Certbot.
+
+### Fixed
+
+* Fixed OS detection in the Apache plugin on Scientific Linux.
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.37.2 - 2019-08-21
+
+* Stop disabling TLS session tickets in Nginx as it caused TLS failures on
+  some systems.
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.37.1 - 2019-08-08
+
+### Fixed
+
+* Stop disabling TLS session tickets in Apache as it caused TLS failures on
+  some systems.
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.37.0 - 2019-08-07
+
+### Added
+
+* Turn off session tickets for apache plugin by default
+* acme: Authz deactivation added to `acme` module.
+
+### Changed
+
+* Follow updated Mozilla recommendations for Nginx ssl_protocols, ssl_ciphers,
+  and ssl_prefer_server_ciphers
+
+### Fixed
+
+* Fix certbot-auto failures on RHEL 8.
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.36.0 - 2019-07-11
+
+### Added
+
+* Turn off session tickets for nginx plugin by default
+* Added missing error types from RFC8555 to acme
+
+### Changed
+
+* Support for Ubuntu 14.04 Trusty has been removed.
+* Update the 'manage your account' help to be more generic.
+* The error message when Certbot's Apache plugin is unable to modify your
+  Apache configuration has been improved.
+* Certbot's config_changes subcommand has been deprecated and will be
+  removed in a future release.
+* `certbot config_changes` no longer accepts a --num parameter.
+* The functions `certbot.plugins.common.Installer.view_config_changes` and
+  `certbot.reverter.Reverter.view_config_changes` have been deprecated and will
+  be removed in a future release.
+
+### Fixed
+
+* Replace some unnecessary platform-specific line separation.
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.35.1 - 2019-06-10
+
+### Fixed
+
+* Support for specifying an authoritative base domain in our dns-rfc2136 plugin
+  has been removed. This feature was added in our last release but had a bug
+  which caused the plugin to fail so the feature has been removed until it can
+  be added properly.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+package with changes other than its version number was:
+
+* certbot-dns-rfc2136
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.35.0 - 2019-06-05
+
+### Added
+
+* dns_rfc2136 plugin now supports explicitly specifing an authorative
+  base domain for cases when the automatic method does not work (e.g.
+  Split horizon DNS)
+
 ### Changed
 
 *
 
 ### Fixed
 
-* certbot-auto no longer prints a blank line when there are no permissions
-  problems.
-* certbot-auto no longer writes a check_permissions.py script at the root
-  of the filesystem.
+* Renewal parameter `webroot_path` is always saved, avoiding some regressions
+  when `webroot` authenticator plugin is invoked with no challenge to perform.
+* Certbot now accepts OCSP responses when an explicit authorized
+  responder, different from the issuer, is used to sign OCSP
+  responses.
+* Scripts in Certbot hook directories are no longer executed when their
+  filenames end in a tilde.
 
 Despite us having broken lockstep, we are continuing to release new versions of
 all Certbot components during releases for the time being, however, the only
 package with changes other than its version number was:
 
-*
+* certbot
+* certbot-dns-rfc2136
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.34.2 - 2019-05-07
+
+### Fixed
+
+* certbot-auto no longer writes a check_permissions.py script at the root
+  of the filesystem.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+changes in this release were to certbot-auto.
+
+More details about these changes can be found on our GitHub repo.
+
+## 0.34.1 - 2019-05-06
+
+### Fixed
+
+* certbot-auto no longer prints a blank line when there are no permissions
+  problems.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+changes in this release were to certbot-auto.
 
 More details about these changes can be found on our GitHub repo.
 
@@ -43,6 +188,10 @@ More details about these changes can be found on our GitHub repo.
   `malformed` error to be received from the ACME server.
 * Linode DNS plugin now supports api keys created from their new panel
   at [cloud.linode.com](https://cloud.linode.com)
+
+### Fixed
+
+* Fixed Google DNS Challenge issues when private zones exist
 * Adding a warning noting that future versions of Certbot will automatically configure the
   webserver so that all requests redirect to secure HTTPS access. You can control this
   behavior and disable this warning with the --redirect and --no-redirect flags.

Dockerfile

@@ -1,35 +0,0 @@
-FROM python:2-alpine3.9
-
-ENTRYPOINT [ "certbot" ]
-EXPOSE 80 443
-VOLUME /etc/letsencrypt /var/lib/letsencrypt
-WORKDIR /opt/certbot
-
-COPY CHANGELOG.md README.rst setup.py src/
-
-# Generate constraints file to pin dependency versions
-COPY letsencrypt-auto-source/pieces/dependency-requirements.txt .
-COPY tools /opt/certbot/tools
-RUN sh -c 'cat dependency-requirements.txt | /opt/certbot/tools/strip_hashes.py > unhashed_requirements.txt'
-RUN sh -c 'cat tools/dev_constraints.txt unhashed_requirements.txt | /opt/certbot/tools/merge_requirements.py > docker_constraints.txt'
-
-COPY acme src/acme
-COPY certbot src/certbot
-
-RUN apk add --no-cache --virtual .certbot-deps \
-        libffi \
-        libssl1.1 \
-        openssl \
-        ca-certificates \
-        binutils
-RUN apk add --no-cache --virtual .build-deps \
-        gcc \
-        linux-headers \
-        openssl-dev \
-        musl-dev \
-        libffi-dev \
-    && pip install -r /opt/certbot/dependency-requirements.txt \
-    && pip install --no-cache-dir --no-deps \
-        --editable /opt/certbot/src/acme \
-        --editable /opt/certbot/src \
-    && apk del .build-deps

Dockerfile-dev

@@ -1,5 +1,5 @@
 # This Dockerfile builds an image for development.
-FROM ubuntu:xenial
+FROM debian:buster
 
 # Note: this only exposes the port to other docker containers.
 EXPOSE 80 443

Dockerfile-old

@@ -1,75 +0,0 @@
-# https://github.com/letsencrypt/letsencrypt/pull/431#issuecomment-103659297
-# it is more likely developers will already have ubuntu:trusty rather
-# than e.g. debian:jessie and image size differences are negligible
-FROM ubuntu:trusty
-MAINTAINER Jakub Warmuz <jakub@warmuz.org>
-MAINTAINER William Budington <bill@eff.org>
-
-# Note: this only exposes the port to other docker containers. You
-# still have to bind to 443@host at runtime, as per the ACME spec.
-EXPOSE 443
-
-# TODO: make sure --config-dir and --work-dir cannot be changed
-# through the CLI (certbot-docker wrapper that uses standalone
-# authenticator and text mode only?)
-VOLUME /etc/letsencrypt /var/lib/letsencrypt
-
-WORKDIR /opt/certbot
-
-# no need to mkdir anything:
-# https://docs.docker.com/reference/builder/#copy
-# If <dest> doesn't exist, it is created along with all missing
-# directories in its path.
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-COPY letsencrypt-auto-source/letsencrypt-auto /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto
-RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/* \
-           /tmp/* \
-           /var/tmp/*
-
-# the above is not likely to change, so by putting it further up the
-# Dockerfile we make sure we cache as much as possible
-
-
-COPY setup.py README.rst CHANGELOG.md MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/
-
-# all above files are necessary for setup.py and venv setup, however,
-# package source code directory has to be copied separately to a
-# subdirectory...
-# https://docs.docker.com/reference/builder/#copy: "If <src> is a
-# directory, the entire contents of the directory are copied,
-# including filesystem metadata. Note: The directory itself is not
-# copied, just its contents." Order again matters, three files are far
-# more likely to be cached than the whole project directory
-
-COPY certbot /opt/certbot/src/certbot/
-COPY acme /opt/certbot/src/acme/
-COPY certbot-apache /opt/certbot/src/certbot-apache/
-COPY certbot-nginx /opt/certbot/src/certbot-nginx/
-
-
-RUN VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages -p python2 /opt/certbot/venv
-
-# PATH is set now so pipstrap upgrades the correct (v)env
-ENV PATH /opt/certbot/venv/bin:$PATH
-RUN /opt/certbot/venv/bin/python /opt/certbot/src/pipstrap.py && \
-    /opt/certbot/venv/bin/pip install \
-    -e /opt/certbot/src/acme \
-    -e /opt/certbot/src \
-    -e /opt/certbot/src/certbot-apache \
-    -e /opt/certbot/src/certbot-nginx
-
-# install in editable mode (-e) to save space: it's not possible to
-# "rm -rf /opt/certbot/src" (it's stays in the underlaying image);
-# this might also help in debugging: you can "docker run --entrypoint
-# bash" and investigate, apply patches, etc.
-
-# set up certbot/letsencrypt wrapper to warn people about Dockerfile changes
-COPY tools/docker-warning.sh /opt/certbot/bin/certbot
-RUN ln -s /opt/certbot/bin/certbot /opt/certbot/bin/letsencrypt
-ENV PATH /opt/certbot/bin:$PATH
-
-ENTRYPOINT [ "certbot" ]

acme/acme/client.py

@@ -123,6 +123,21 @@ class ClientBase(object):  # pylint: disable=too-many-instance-attributes
         """
         return self.update_registration(regr, update={'status': 'deactivated'})
 
+    def deactivate_authorization(self, authzr):
+        # type: (messages.AuthorizationResource) -> messages.AuthorizationResource
+        """Deactivate authorization.
+
+        :param messages.AuthorizationResource authzr: The Authorization resource
+            to be deactivated.
+
+        :returns: The Authorization resource that was deactivated.
+        :rtype: `.AuthorizationResource`
+
+        """
+        body = messages.UpdateAuthorization(status='deactivated')
+        response = self._post(authzr.uri, body)
+        return self._authzr_from_response(response)
+
     def _authzr_from_response(self, response, identifier=None, uri=None):
         authzr = messages.AuthorizationResource(
             body=messages.Authorization.from_json(response.json()),

acme/acme/client_test.py

@@ -637,6 +637,14 @@ class ClientTest(ClientTestBase):
             errors.PollError, self.client.poll_and_request_issuance,
             csr, authzrs, mintime=mintime, max_attempts=2)
 
+    def test_deactivate_authorization(self):
+        authzb = self.authzr.body.update(status=messages.STATUS_DEACTIVATED)
+        self.response.json.return_value = authzb.to_json()
+        authzr = self.client.deactivate_authorization(self.authzr)
+        self.assertEqual(authzb, authzr.body)
+        self.assertEqual(self.client.net.post.call_count, 1)
+        self.assertTrue(self.authzr.uri in self.net.post.call_args_list[0][0])
+
     def test_check_cert(self):
         self.response.headers['Location'] = self.certr.uri
         self.response.content = CERT_DER

acme/acme/messages.py

@@ -18,20 +18,35 @@ OLD_ERROR_PREFIX = "urn:acme:error:"
 ERROR_PREFIX = "urn:ietf:params:acme:error:"
 
 ERROR_CODES = {
+    'accountDoesNotExist': 'The request specified an account that does not exist',
+    'alreadyRevoked': 'The request specified a certificate to be revoked that has' \
+    ' already been revoked',
     'badCSR': 'The CSR is unacceptable (e.g., due to a short key)',
     'badNonce': 'The client sent an unacceptable anti-replay nonce',
+    'badPublicKey': 'The JWS was signed by a public key the server does not support',
+    'badRevocationReason': 'The revocation reason provided is not allowed by the server',
+    'badSignatureAlgorithm': 'The JWS was signed with an algorithm the server does not support',
+    'caa': 'Certification Authority Authorization (CAA) records forbid the CA from issuing' \
+    ' a certificate',
+    'compound': 'Specific error conditions are indicated in the "subproblems" array',
     'connection': ('The server could not connect to the client to verify the'
                    ' domain'),
+    'dns': 'There was a problem with a DNS query during identifier validation',
     'dnssec': 'The server could not validate a DNSSEC signed domain',
+    'incorrectResponse': 'Response recieved didn\'t match the challenge\'s requirements',
     # deprecate invalidEmail
     'invalidEmail': 'The provided email for a registration was invalid',
     'invalidContact': 'The provided contact URI was invalid',
     'malformed': 'The request message was malformed',
+    'rejectedIdentifier': 'The server will not issue certificates for the identifier',
+    'orderNotReady': 'The request attempted to finalize an order that is not ready to be finalized',
     'rateLimited': 'There were too many requests of a given type',
     'serverInternal': 'The server experienced an internal error',
     'tls': 'The server experienced a TLS error during domain verification',
     'unauthorized': 'The client lacks sufficient authorization',
+    'unsupportedContact': 'A contact URL for an account used an unsupported protocol scheme',
     'unknownHost': 'The server could not resolve a domain name',
+    'unsupportedIdentifier': 'An identifier is of an unsupported type',
     'externalAccountRequired': 'The server requires external account binding',
 }
 
@@ -153,6 +168,7 @@ STATUS_VALID = Status('valid')
 STATUS_INVALID = Status('invalid')
 STATUS_REVOKED = Status('revoked')
 STATUS_READY = Status('ready')
+STATUS_DEACTIVATED = Status('deactivated')
 
 
 class IdentifierType(_Constant):
@@ -456,7 +472,7 @@ class Authorization(ResourceBody):
     :ivar datetime.datetime expires:
 
     """
-    identifier = jose.Field('identifier', decoder=Identifier.from_json)
+    identifier = jose.Field('identifier', decoder=Identifier.from_json, omitempty=True)
     challenges = jose.Field('challenges', omitempty=True)
     combinations = jose.Field('combinations', omitempty=True)
 
@@ -486,6 +502,12 @@ class NewAuthorization(Authorization):
     resource = fields.Resource(resource_type)
 
 
+class UpdateAuthorization(Authorization):
+    """Update authorization."""
+    resource_type = 'authz'
+    resource = fields.Resource(resource_type)
+
+
 class AuthorizationResource(ResourceWithURI):
     """Authorization Resource.
 

acme/setup.py

@@ -3,7 +3,7 @@ from setuptools import find_packages
 from setuptools.command.test import test as TestCommand
 import sys
 
-version = '0.35.0.dev0'
+version = '0.39.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
@@ -73,6 +73,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
     ],

appveyor.yml

@@ -4,9 +4,13 @@ environment:
   matrix:
     - TOXENV: py35
     - TOXENV: py37-cover
+    - TOXENV: integration-certbot
 
 branches:
   only:
+    # apache-parser-v2 is a temporary branch for doing work related to
+    # rewriting the parser in the Apache plugin.
+    - apache-parser-v2
     - master
     - /^\d+\.\d+\.x$/ # Version branches like X.X.X
     - /^test-.*$/
@@ -21,13 +25,16 @@ init:
 
 install:
   # Use Python 3.7 by default
-  - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%"
+  - SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%
+  # Using 4 processes is proven to be the most efficient integration tests config for AppVeyor
+  - IF %TOXENV%==integration-certbot SET PYTEST_ADDOPTS=--numprocesses=4
   # Check env
-  - "python --version"
+  - python --version
   # Upgrade pip to avoid warnings
-  - "python -m pip install --upgrade pip"
+  - python -m pip install --upgrade pip
   # Ready to install tox and coverage
-  - "pip install tox codecov"
+  # tools/pip_install.py is used to pin packages to a known working version.
+  - python tools\\pip_install.py tox codecov
 
 build: off
 

certbot-apache/certbot_apache/augeas_configurator.py

@@ -1,207 +0,0 @@
-"""Class of Augeas Configurators."""
-import logging
-
-
-from certbot import errors
-from certbot.plugins import common
-
-from certbot_apache import constants
-
-logger = logging.getLogger(__name__)
-
-
-class AugeasConfigurator(common.Installer):
-    """Base Augeas Configurator class.
-
-    :ivar config: Configuration.
-    :type config: :class:`~certbot.interfaces.IConfig`
-
-    :ivar aug: Augeas object
-    :type aug: :class:`augeas.Augeas`
-
-    :ivar str save_notes: Human-readable configuration change notes
-    :ivar reverter: saves and reverts checkpoints
-    :type reverter: :class:`certbot.reverter.Reverter`
-
-    """
-    def __init__(self, *args, **kwargs):
-        super(AugeasConfigurator, self).__init__(*args, **kwargs)
-
-        # Placeholder for augeas
-        self.aug = None
-
-        self.save_notes = ""
-
-
-    def init_augeas(self):
-        """ Initialize the actual Augeas instance """
-        import augeas
-        self.aug = augeas.Augeas(
-            # specify a directory to load our preferred lens from
-            loadpath=constants.AUGEAS_LENS_DIR,
-            # Do not save backup (we do it ourselves), do not load
-            # anything by default
-            flags=(augeas.Augeas.NONE |
-                   augeas.Augeas.NO_MODL_AUTOLOAD |
-                   augeas.Augeas.ENABLE_SPAN))
-        # See if any temporary changes need to be recovered
-        # This needs to occur before VirtualHost objects are setup...
-        # because this will change the underlying configuration and potential
-        # vhosts
-        self.recovery_routine()
-
-    def check_parsing_errors(self, lens):
-        """Verify Augeas can parse all of the lens files.
-
-        :param str lens: lens to check for errors
-
-        :raises .errors.PluginError: If there has been an error in parsing with
-            the specified lens.
-
-        """
-        error_files = self.aug.match("/augeas//error")
-
-        for path in error_files:
-            # Check to see if it was an error resulting from the use of
-            # the httpd lens
-            lens_path = self.aug.get(path + "/lens")
-            # As aug.get may return null
-            if lens_path and lens in lens_path:
-                msg = (
-                    "There has been an error in parsing the file {0} on line {1}: "
-                    "{2}".format(
-                    # Strip off /augeas/files and /error
-                    path[13:len(path) - 6],
-                    self.aug.get(path + "/line"),
-                    self.aug.get(path + "/message")))
-                raise errors.PluginError(msg)
-
-    def ensure_augeas_state(self):
-        """Makes sure that all Augeas dom changes are written to files to avoid
-        loss of configuration directives when doing additional augeas parsing,
-        causing a possible augeas.load() resulting dom reset
-        """
-
-        if self.unsaved_files():
-            self.save_notes += "(autosave)"
-            self.save()
-
-    def unsaved_files(self):
-        """Lists files that have modified Augeas DOM but the changes have not
-        been written to the filesystem yet, used by `self.save()` and
-        ApacheConfigurator to check the file state.
-
-        :raises .errors.PluginError: If there was an error in Augeas, in
-            an attempt to save the configuration, or an error creating a
-            checkpoint
-
-        :returns: `set` of unsaved files
-        """
-        save_state = self.aug.get("/augeas/save")
-        self.aug.set("/augeas/save", "noop")
-        # Existing Errors
-        ex_errs = self.aug.match("/augeas//error")
-        try:
-            # This is a noop save
-            self.aug.save()
-        except (RuntimeError, IOError):
-            self._log_save_errors(ex_errs)
-            # Erase Save Notes
-            self.save_notes = ""
-            raise errors.PluginError(
-                "Error saving files, check logs for more info.")
-
-        # Return the original save method
-        self.aug.set("/augeas/save", save_state)
-
-        # Retrieve list of modified files
-        # Note: Noop saves can cause the file to be listed twice, I used a
-        # set to remove this possibility. This is a known augeas 0.10 error.
-        save_paths = self.aug.match("/augeas/events/saved")
-
-        save_files = set()
-        if save_paths:
-            for path in save_paths:
-                save_files.add(self.aug.get(path)[6:])
-        return save_files
-
-    def save(self, title=None, temporary=False):
-        """Saves all changes to the configuration files.
-
-        This function first checks for save errors, if none are found,
-        all configuration changes made will be saved. According to the
-        function parameters. If an exception is raised, a new checkpoint
-        was not created.
-
-        :param str title: The title of the save. If a title is given, the
-            configuration will be saved as a new checkpoint and put in a
-            timestamped directory.
-
-        :param bool temporary: Indicates whether the changes made will
-            be quickly reversed in the future (ie. challenges)
-
-        """
-        save_files = self.unsaved_files()
-        if save_files:
-            self.add_to_checkpoint(save_files,
-                                   self.save_notes, temporary=temporary)
-
-        self.save_notes = ""
-        self.aug.save()
-
-        # Force reload if files were modified
-        # This is needed to recalculate augeas directive span
-        if save_files:
-            for sf in save_files:
-                self.aug.remove("/files/"+sf)
-            self.aug.load()
-        if title and not temporary:
-            self.finalize_checkpoint(title)
-
-    def _log_save_errors(self, ex_errs):
-        """Log errors due to bad Augeas save.
-
-        :param list ex_errs: Existing errors before save
-
-        """
-        # Check for the root of save problems
-        new_errs = self.aug.match("/augeas//error")
-        # logger.error("During Save - %s", mod_conf)
-        logger.error("Unable to save files: %s. Attempted Save Notes: %s",
-                     ", ".join(err[13:len(err) - 6] for err in new_errs
-                               # Only new errors caused by recent save
-                               if err not in ex_errs), self.save_notes)
-
-    # Wrapper functions for Reverter class
-    def recovery_routine(self):
-        """Revert all previously modified files.
-
-        Reverts all modified files that have not been saved as a checkpoint
-
-        :raises .errors.PluginError: If unable to recover the configuration
-
-        """
-        super(AugeasConfigurator, self).recovery_routine()
-        # Need to reload configuration after these changes take effect
-        self.aug.load()
-
-    def revert_challenge_config(self):
-        """Used to cleanup challenge configurations.
-
-        :raises .errors.PluginError: If unable to revert the challenge config.
-
-        """
-        self.revert_temporary_config()
-        self.aug.load()
-
-    def rollback_checkpoints(self, rollback=1):
-        """Rollback saved checkpoints.
-
-        :param int rollback: Number of checkpoints to revert
-
-        :raises .errors.PluginError: If there is a problem with the input or
-            the function is unable to correctly revert the configuration
-
-        """
-        super(AugeasConfigurator, self).rollback_checkpoints(rollback)
-        self.aug.load()

certbot-apache/certbot_apache/configurator.py

@@ -1,4 +1,4 @@
-"""Apache Configuration based off of Augeas Configurator."""
+"""Apache Configurator."""
 # pylint: disable=too-many-lines
 import copy
 import fnmatch
@@ -23,13 +23,13 @@ from certbot import interfaces
 from certbot import util
 
 from certbot.achallenges import KeyAuthorizationAnnotatedChallenge  # pylint: disable=unused-import
+from certbot.compat import filesystem
 from certbot.compat import os
 from certbot.plugins import common
 from certbot.plugins.util import path_surgery
 from certbot.plugins.enhancements import AutoHSTSEnhancement
 
 from certbot_apache import apache_util
-from certbot_apache import augeas_configurator
 from certbot_apache import constants
 from certbot_apache import display_ops
 from certbot_apache import http_01
@@ -70,13 +70,10 @@ logger = logging.getLogger(__name__)
 
 @zope.interface.implementer(interfaces.IAuthenticator, interfaces.IInstaller)
 @zope.interface.provider(interfaces.IPluginFactory)
-class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
+class ApacheConfigurator(common.Installer):
     # pylint: disable=too-many-instance-attributes,too-many-public-methods
     """Apache configurator.
 
-    State of Configurator: This code has been been tested and built for Ubuntu
-    14.04 Apache 2.4 and it works for Ubuntu 12.04 Apache 2.2
-
     :ivar config: Configuration.
     :type config: :class:`~certbot.interfaces.IConfig`
 
@@ -201,6 +198,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         self._enhanced_vhosts = defaultdict(set)  # type: DefaultDict[str, Set[obj.VirtualHost]]
         # Temporary state for AutoHSTS enhancement
         self._autohsts = {}  # type: Dict[str, Dict[str, Union[int, float]]]
+        # Reverter save notes
+        self.save_notes = ""
 
         # These will be set in the prepare function
         self._prepared = False
@@ -231,12 +230,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         :raises .errors.PluginError: If there is any other error
 
         """
-        # Perform the actual Augeas initialization to be able to react
-        try:
-            self.init_augeas()
-        except ImportError:
-            raise errors.NoInstallationError("Problem in Augeas installation")
-
         self._prepare_options()
 
         # Verify Apache is installed
@@ -254,16 +247,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             raise errors.NotSupportedError(
                 "Apache Version {0} not supported.".format(str(self.version)))
 
-        if not self._check_aug_version():
-            raise errors.NotSupportedError(
-                "Apache plugin support requires libaugeas0 and augeas-lenses "
-                "version 1.2.0 or higher, please make sure you have you have "
-                "those installed.")
-
+        # Recover from previous crash before Augeas initialization to have the
+        # correct parse tree from the get go.
+        self.recovery_routine()
+        # Perform the actual Augeas initialization to be able to react
         self.parser = self.get_parser()
 
         # Check for errors in parsing files with Augeas
-        self.check_parsing_errors("httpd.aug")
+        self.parser.check_parsing_errors("httpd.aug")
 
         # Get all of the available vhosts
         self.vhosts = self.get_virtual_hosts()
@@ -276,9 +267,73 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             util.lock_dir_until_exit(self.option("server_root"))
         except (OSError, errors.LockError):
             logger.debug("Encountered error:", exc_info=True)
-            raise errors.PluginError("Unable to lock {0}".format(self.option("server_root")))
+            raise errors.PluginError(
+                "Unable to create a lock file in {0}. Are you running"
+                " Certbot with sufficient privileges to modify your"
+                " Apache configuration?".format(self.option("server_root")))
         self._prepared = True
 
+    def save(self, title=None, temporary=False):
+        """Saves all changes to the configuration files.
+
+        This function first checks for save errors, if none are found,
+        all configuration changes made will be saved. According to the
+        function parameters. If an exception is raised, a new checkpoint
+        was not created.
+
+        :param str title: The title of the save. If a title is given, the
+            configuration will be saved as a new checkpoint and put in a
+            timestamped directory.
+
+        :param bool temporary: Indicates whether the changes made will
+            be quickly reversed in the future (ie. challenges)
+
+        """
+        save_files = self.parser.unsaved_files()
+        if save_files:
+            self.add_to_checkpoint(save_files,
+                                   self.save_notes, temporary=temporary)
+        # Handle the parser specific tasks
+        self.parser.save(save_files)
+        if title and not temporary:
+            self.finalize_checkpoint(title)
+
+    def recovery_routine(self):
+        """Revert all previously modified files.
+
+        Reverts all modified files that have not been saved as a checkpoint
+
+        :raises .errors.PluginError: If unable to recover the configuration
+
+        """
+        super(ApacheConfigurator, self).recovery_routine()
+        # Reload configuration after these changes take effect if needed
+        # ie. ApacheParser has been initialized.
+        if self.parser:
+            # TODO: wrap into non-implementation specific  parser interface
+            self.parser.aug.load()
+
+    def revert_challenge_config(self):
+        """Used to cleanup challenge configurations.
+
+        :raises .errors.PluginError: If unable to revert the challenge config.
+
+        """
+        self.revert_temporary_config()
+        self.parser.aug.load()
+
+    def rollback_checkpoints(self, rollback=1):
+        """Rollback saved checkpoints.
+
+        :param int rollback: Number of checkpoints to revert
+
+        :raises .errors.PluginError: If there is a problem with the input or
+            the function is unable to correctly revert the configuration
+
+        """
+        super(ApacheConfigurator, self).rollback_checkpoints(rollback)
+        self.parser.aug.load()
+
     def _verify_exe_availability(self, exe):
         """Checks availability of Apache executable"""
         if not util.exe_exists(exe):
@@ -286,26 +341,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 raise errors.NoInstallationError(
                     'Cannot find Apache executable {0}'.format(exe))
 
-    def _check_aug_version(self):
-        """ Checks that we have recent enough version of libaugeas.
-        If augeas version is recent enough, it will support case insensitive
-        regexp matching"""
-
-        self.aug.set("/test/path/testing/arg", "aRgUMeNT")
-        try:
-            matches = self.aug.match(
-                "/test//*[self::arg=~regexp('argument', 'i')]")
-        except RuntimeError:
-            self.aug.remove("/test/path")
-            return False
-        self.aug.remove("/test/path")
-        return matches
-
     def get_parser(self):
         """Initializes the ApacheParser"""
         # If user provided vhost_root value in command line, use it
         return parser.ApacheParser(
-            self.aug, self.option("server_root"), self.conf("vhost-root"),
+            self.option("server_root"), self.conf("vhost-root"),
             self.version, configurator=self)
 
     def _wildcard_domain(self, domain):
@@ -484,8 +524,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             # install SSLCertificateFile, SSLCertificateKeyFile,
             # and SSLCertificateChainFile directives
             set_cert_path = cert_path
-            self.aug.set(path["cert_path"][-1], cert_path)
-            self.aug.set(path["cert_key"][-1], key_path)
+            self.parser.aug.set(path["cert_path"][-1], cert_path)
+            self.parser.aug.set(path["cert_key"][-1], key_path)
             if chain_path is not None:
                 self.parser.add_dir(vhost.path,
                                     "SSLCertificateChainFile", chain_path)
@@ -497,8 +537,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 raise errors.PluginError("Please provide the --fullchain-path "
                                          "option pointing to your full chain file")
             set_cert_path = fullchain_path
-            self.aug.set(path["cert_path"][-1], fullchain_path)
-            self.aug.set(path["cert_key"][-1], key_path)
+            self.parser.aug.set(path["cert_path"][-1], fullchain_path)
+            self.parser.aug.set(path["cert_key"][-1], key_path)
 
         # Enable the new vhost if needed
         if not vhost.enabled:
@@ -798,7 +838,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         """
         addrs = set()
         try:
-            args = self.aug.match(path + "/arg")
+            args = self.parser.aug.match(path + "/arg")
         except RuntimeError:
             logger.warning("Encountered a problem while parsing file: %s, skipping", path)
             return None
@@ -816,7 +856,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 is_ssl = True
 
         filename = apache_util.get_file_path(
-            self.aug.get("/augeas/files%s/path" % apache_util.get_file_path(path)))
+            self.parser.aug.get("/augeas/files%s/path" % apache_util.get_file_path(path)))
         if filename is None:
             return None
 
@@ -846,7 +886,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         # Make a list of parser paths because the parser_paths
         # dictionary may be modified during the loop.
         for vhost_path in list(self.parser.parser_paths):
-            paths = self.aug.match(
+            paths = self.parser.aug.match(
                 ("/files%s//*[label()=~regexp('%s')]" %
                     (vhost_path, parser.case_i("VirtualHost"))))
             paths = [path for path in paths if
@@ -856,7 +896,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 if not new_vhost:
                     continue
                 internal_path = apache_util.get_internal_aug_path(new_vhost.path)
-                realpath = os.path.realpath(new_vhost.filep)
+                realpath = filesystem.realpath(new_vhost.filep)
                 if realpath not in file_paths:
                     file_paths[realpath] = new_vhost.filep
                     internal_paths[realpath].add(internal_path)
@@ -1100,16 +1140,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         avail_fp = nonssl_vhost.filep
         ssl_fp = self._get_ssl_vhost_path(avail_fp)
 
-        orig_matches = self.aug.match("/files%s//* [label()=~regexp('%s')]" %
+        orig_matches = self.parser.aug.match("/files%s//* [label()=~regexp('%s')]" %
                                       (self._escape(ssl_fp),
                                        parser.case_i("VirtualHost")))
 
         self._copy_create_ssl_vhost_skeleton(nonssl_vhost, ssl_fp)
 
         # Reload augeas to take into account the new vhost
-        self.aug.load()
+        self.parser.aug.load()
         # Get Vhost augeas path for new vhost
-        new_matches = self.aug.match("/files%s//* [label()=~regexp('%s')]" %
+        new_matches = self.parser.aug.match("/files%s//* [label()=~regexp('%s')]" %
                                      (self._escape(ssl_fp),
                                       parser.case_i("VirtualHost")))
 
@@ -1120,7 +1160,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             # Make Augeas aware of the new vhost
             self.parser.parse_file(ssl_fp)
             # Try to search again
-            new_matches = self.aug.match(
+            new_matches = self.parser.aug.match(
                 "/files%s//* [label()=~regexp('%s')]" %
                 (self._escape(ssl_fp),
                  parser.case_i("VirtualHost")))
@@ -1182,11 +1222,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         """
 
         if self.conf("vhost-root") and os.path.exists(self.conf("vhost-root")):
-            fp = os.path.join(os.path.realpath(self.option("vhost_root")),
+            fp = os.path.join(filesystem.realpath(self.option("vhost_root")),
                               os.path.basename(non_ssl_vh_fp))
         else:
             # Use non-ssl filepath
-            fp = os.path.realpath(non_ssl_vh_fp)
+            fp = filesystem.realpath(non_ssl_vh_fp)
 
         if fp.endswith(".conf"):
             return fp[:-(len(".conf"))] + self.option("le_vhost_ext")
@@ -1270,8 +1310,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 "vhost for your HTTPS site located at {1} because they have "
                 "the potential to create redirection loops.".format(
                     vhost.filep, ssl_fp), reporter.MEDIUM_PRIORITY)
-        self.aug.set("/augeas/files%s/mtime" % (self._escape(ssl_fp)), "0")
-        self.aug.set("/augeas/files%s/mtime" % (self._escape(vhost.filep)), "0")
+        self.parser.aug.set("/augeas/files%s/mtime" % (self._escape(ssl_fp)), "0")
+        self.parser.aug.set("/augeas/files%s/mtime" % (self._escape(vhost.filep)), "0")
 
     def _sift_rewrite_rules(self, contents):
         """ Helper function for _copy_create_ssl_vhost_skeleton to prepare the
@@ -1346,7 +1386,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         """
 
         try:
-            span_val = self.aug.span(vhost.path)
+            span_val = self.parser.aug.span(vhost.path)
         except ValueError:
             logger.critical("Error while reading the VirtualHost %s from "
                          "file %s", vhost.name, vhost.filep, exc_info=True)
@@ -1381,13 +1421,13 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
     def _update_ssl_vhosts_addrs(self, vh_path):
         ssl_addrs = set()
-        ssl_addr_p = self.aug.match(vh_path + "/arg")
+        ssl_addr_p = self.parser.aug.match(vh_path + "/arg")
 
         for addr in ssl_addr_p:
             old_addr = obj.Addr.fromstring(
                 str(self.parser.get_arg(addr)))
             ssl_addr = old_addr.get_addr_obj("443")
-            self.aug.set(addr, str(ssl_addr))
+            self.parser.aug.set(addr, str(ssl_addr))
             ssl_addrs.add(ssl_addr)
 
         return ssl_addrs
@@ -1406,14 +1446,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                                            vh_path, False)) > 1:
                 directive_path = self.parser.find_dir(directive, None,
                                                       vh_path, False)
-                self.aug.remove(re.sub(r"/\w*$", "", directive_path[0]))
+                self.parser.aug.remove(re.sub(r"/\w*$", "", directive_path[0]))
 
     def _remove_directives(self, vh_path, directives):
         for directive in directives:
             while self.parser.find_dir(directive, None, vh_path, False):
                 directive_path = self.parser.find_dir(directive, None,
                                                       vh_path, False)
-                self.aug.remove(re.sub(r"/\w*$", "", directive_path[0]))
+                self.parser.aug.remove(re.sub(r"/\w*$", "", directive_path[0]))
 
     def _add_dummy_ssl_directives(self, vh_path):
         self.parser.add_dir(vh_path, "SSLCertificateFile",
@@ -1452,7 +1492,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         """
         matches = self.parser.find_dir(
             "ServerAlias", start=vh_path, exclude=False)
-        aliases = (self.aug.get(match) for match in matches)
+        aliases = (self.parser.aug.get(match) for match in matches)
         return self.domain_in_names(aliases, target_name)
 
     def _add_name_vhost_if_necessary(self, vhost):
@@ -1635,7 +1675,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         if header_path:
             pat = '(?:[ "]|^)(strict-transport-security)(?:[ "]|$)'
             for match in header_path:
-                if re.search(pat, self.aug.get(match).lower()):
+                if re.search(pat, self.parser.aug.get(match).lower()):
                     hsts_dirpath = match
         if not hsts_dirpath:
             err_msg = ("Certbot was unable to find the existing HSTS header "
@@ -1649,7 +1689,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         # Our match statement was for string strict-transport-security, but
         # we need to update the value instead. The next index is for the value
         hsts_dirpath = hsts_dirpath.replace("arg[3]", "arg[4]")
-        self.aug.set(hsts_dirpath, hsts_maxage)
+        self.parser.aug.set(hsts_dirpath, hsts_maxage)
         note_msg = ("Increasing HSTS max-age value to {0} for VirtualHost "
                     "in {1}\n".format(nextstep_value, vhost.filep))
         logger.debug(note_msg)
@@ -1731,7 +1771,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         # We'll simply delete the directive, so that we'll have a
         # consistent OCSP cache path.
         if stapling_cache_aug_path:
-            self.aug.remove(
+            self.parser.aug.remove(
                     re.sub(r"/\w*$", "", stapling_cache_aug_path[0]))
 
         self.parser.add_dir_to_ifmodssl(ssl_vhost_aug_path,
@@ -1808,7 +1848,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             # "Existing Header directive for virtualhost"
             pat = '(?:[ "]|^)(%s)(?:[ "]|$)' % (header_substring.lower())
             for match in header_path:
-                if re.search(pat, self.aug.get(match).lower()):
+                if re.search(pat, self.parser.aug.get(match).lower()):
                     raise errors.PluginEnhancementAlreadyPresent(
                         "Existing %s header" % (header_substring))
 
@@ -1935,11 +1975,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                              constants.REWRITE_HTTPS_ARGS_WITH_END]
 
             for dir_path, args_paths in rewrite_args_dict.items():
-                arg_vals = [self.aug.get(x) for x in args_paths]
+                arg_vals = [self.parser.aug.get(x) for x in args_paths]
 
                 # Search for past redirection rule, delete it, set the new one
                 if arg_vals in constants.OLD_REWRITE_HTTPS_ARGS:
-                    self.aug.remove(dir_path)
+                    self.parser.aug.remove(dir_path)
                     self._set_https_redirection_rewrite_rule(vhost)
                     self.save()
                     raise errors.PluginEnhancementAlreadyPresent(
@@ -1995,7 +2035,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         redirect_filepath = self._write_out_redirect(ssl_vhost, text)
 
-        self.aug.load()
+        self.parser.aug.load()
         # Make a new vhost data structure and add it to the lists
         new_vhost = self._create_vhost(parser.get_aug_path(self._escape(redirect_filepath)))
         self.vhosts.append(new_vhost)

certbot-apache/certbot_apache/constants.py

@@ -9,6 +9,7 @@ MOD_SSL_CONF_DEST = "options-ssl-apache.conf"
 UPDATED_MOD_SSL_CONF_DIGEST = ".updated-options-ssl-apache-conf-digest.txt"
 """Name of the hash of the updated or informed mod_ssl_conf as saved in `IConfig.config_dir`."""
 
+# NEVER REMOVE A SINGLE HASH FROM THIS LIST UNLESS YOU KNOW EXACTLY WHAT YOU ARE DOING!
 ALL_SSL_OPTIONS_HASHES = [
     '2086bca02db48daf93468332543c60ac6acdb6f0b58c7bfdf578a5d47092f82a',
     '4844d36c9a0f587172d9fa10f4f1c9518e3bcfa1947379f155e16a70a728c21a',
@@ -18,6 +19,10 @@ ALL_SSL_OPTIONS_HASHES = [
     'cfdd7c18d2025836ea3307399f509cfb1ebf2612c87dd600a65da2a8e2f2797b',
     '80720bd171ccdc2e6b917ded340defae66919e4624962396b992b7218a561791',
     'c0c022ea6b8a51ecc8f1003d0a04af6c3f2bc1c3ce506b3c2dfc1f11ef931082',
+    '717b0a89f5e4c39b09a42813ac6e747cfbdeb93439499e73f4f70a1fe1473f20',
+    '0fcdc81280cd179a07ec4d29d3595068b9326b455c488de4b09f585d5dafc137',
+    '86cc09ad5415cd6d5f09a947fe2501a9344328b1e8a8b458107ea903e80baa6c',
+    '06675349e457eae856120cdebb564efe546f0b87399f2264baeb41e442c724c7',
 ]
 """SHA256 hashes of the contents of previous versions of all versions of MOD_SSL_CONF_SRC"""
 

certbot-apache/certbot_apache/entrypoint.py

@@ -31,6 +31,8 @@ OVERRIDE_CLASSES = {
     "gentoo base system": override_gentoo.GentooConfigurator,
     "opensuse": override_suse.OpenSUSEConfigurator,
     "suse": override_suse.OpenSUSEConfigurator,
+    "scientific": override_centos.CentOSConfigurator,
+    "scientific linux": override_centos.CentOSConfigurator,
 }
 
 

certbot-apache/certbot_apache/http_01.py

@@ -5,6 +5,7 @@ from acme.magic_typing import List, Set  # pylint: disable=unused-import, no-nam
 
 from certbot import errors
 from certbot.compat import os
+from certbot.compat import filesystem
 from certbot.plugins import common
 
 from certbot_apache.obj import VirtualHost  # pylint: disable=unused-import
@@ -168,8 +169,7 @@ class ApacheHttp01(common.TLSSNI01):
 
     def _set_up_challenges(self):
         if not os.path.isdir(self.challenge_dir):
-            os.makedirs(self.challenge_dir)
-            os.chmod(self.challenge_dir, 0o755)
+            filesystem.makedirs(self.challenge_dir, 0o755)
 
         responses = []
         for achall in self.achalls:
@@ -185,7 +185,7 @@ class ApacheHttp01(common.TLSSNI01):
         self.configurator.reverter.register_file_creation(True, name)
         with open(name, 'wb') as f:
             f.write(validation.encode())
-        os.chmod(name, 0o644)
+        filesystem.chmod(name, 0o644)
 
         return response
 

certbot-apache/certbot_apache/override_centos.py

@@ -86,7 +86,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
     def get_parser(self):
         """Initializes the ApacheParser"""
         return CentOSParser(
-            self.aug, self.option("server_root"), self.option("vhost_root"),
+            self.option("server_root"), self.option("vhost_root"),
             self.version, configurator=self)
 
     def _deploy_cert(self, *args, **kwargs):  # pylint: disable=arguments-differ
@@ -155,7 +155,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
         for loadmod_path in loadmod_paths:
             nodir_path = loadmod_path.split("/directive")[0]
             # Remove the old LoadModule directive
-            self.aug.remove(loadmod_path)
+            self.parser.aug.remove(loadmod_path)
 
             # Create a new IfModule !mod_ssl.c if not already found on path
             ssl_ifmod = self.parser.get_ifmod(nodir_path, "!mod_ssl.c",

certbot-apache/certbot_apache/override_debian.py

@@ -7,6 +7,7 @@ import zope.interface
 from certbot import errors
 from certbot import interfaces
 from certbot import util
+from certbot.compat import filesystem
 from certbot.compat import os
 
 from certbot_apache import apache_util
@@ -65,7 +66,7 @@ class DebianConfigurator(configurator.ApacheConfigurator):
         try:
             os.symlink(vhost.filep, enabled_path)
         except OSError as err:
-            if os.path.islink(enabled_path) and os.path.realpath(
+            if os.path.islink(enabled_path) and filesystem.realpath(
                enabled_path) == vhost.filep:
                 # Already in shape
                 vhost.enabled = True

certbot-apache/certbot_apache/override_fedora.py

@@ -51,7 +51,7 @@ class FedoraConfigurator(configurator.ApacheConfigurator):
     def get_parser(self):
         """Initializes the ApacheParser"""
         return FedoraParser(
-            self.aug, self.option("server_root"), self.option("vhost_root"),
+            self.option("server_root"), self.option("vhost_root"),
             self.version, configurator=self)
 
     def _try_restart_fedora(self):

certbot-apache/certbot_apache/override_gentoo.py

@@ -44,7 +44,7 @@ class GentooConfigurator(configurator.ApacheConfigurator):
     def get_parser(self):
         """Initializes the ApacheParser"""
         return GentooParser(
-            self.aug, self.option("server_root"), self.option("vhost_root"),
+            self.option("server_root"), self.option("vhost_root"),
             self.version, configurator=self)
 
 

certbot-apache/certbot_apache/parser.py

@@ -13,6 +13,8 @@ from acme.magic_typing import Dict, List, Set  # pylint: disable=unused-import,
 from certbot import errors
 from certbot.compat import os
 
+from certbot_apache import constants
+
 logger = logging.getLogger(__name__)
 
 
@@ -32,7 +34,7 @@ class ApacheParser(object):
     arg_var_interpreter = re.compile(r"\$\{[^ \}]*}")
     fnmatch_chars = set(["*", "?", "\\", "[", "]"])
 
-    def __init__(self, aug, root, vhostroot=None, version=(2, 4),
+    def __init__(self, root, vhostroot=None, version=(2, 4),
                  configurator=None):
         # Note: Order is important here.
 
@@ -41,11 +43,20 @@ class ApacheParser(object):
         # issues with aug.load() after adding new files / defines to parse tree
         self.configurator = configurator
 
+        # Initialize augeas
+        self.aug = None
+        self.init_augeas()
+
+        if not self.check_aug_version():
+            raise errors.NotSupportedError(
+                "Apache plugin support requires libaugeas0 and augeas-lenses "
+                "version 1.2.0 or higher, please make sure you have you have "
+                "those installed.")
+
         self.modules = set()  # type: Set[str]
         self.parser_paths = {}  # type: Dict[str, List[str]]
         self.variables = {}  # type: Dict[str, str]
 
-        self.aug = aug
         # Find configuration root and make sure augeas can parse it.
         self.root = os.path.abspath(root)
         self.loc = {"root": self._find_config_root()}
@@ -77,6 +88,146 @@ class ApacheParser(object):
             if self.find_dir("Define", exclude=False):
                 raise errors.PluginError("Error parsing runtime variables")
 
+    def init_augeas(self):
+        """ Initialize the actual Augeas instance """
+
+        try:
+            import augeas
+        except ImportError:  # pragma: no cover
+            raise errors.NoInstallationError("Problem in Augeas installation")
+
+        self.aug = augeas.Augeas(
+            # specify a directory to load our preferred lens from
+            loadpath=constants.AUGEAS_LENS_DIR,
+            # Do not save backup (we do it ourselves), do not load
+            # anything by default
+            flags=(augeas.Augeas.NONE |
+                   augeas.Augeas.NO_MODL_AUTOLOAD |
+                   augeas.Augeas.ENABLE_SPAN))
+
+    def check_parsing_errors(self, lens):
+        """Verify Augeas can parse all of the lens files.
+
+        :param str lens: lens to check for errors
+
+        :raises .errors.PluginError: If there has been an error in parsing with
+            the specified lens.
+
+        """
+        error_files = self.aug.match("/augeas//error")
+
+        for path in error_files:
+            # Check to see if it was an error resulting from the use of
+            # the httpd lens
+            lens_path = self.aug.get(path + "/lens")
+            # As aug.get may return null
+            if lens_path and lens in lens_path:
+                msg = (
+                    "There has been an error in parsing the file {0} on line {1}: "
+                    "{2}".format(
+                    # Strip off /augeas/files and /error
+                    path[13:len(path) - 6],
+                    self.aug.get(path + "/line"),
+                    self.aug.get(path + "/message")))
+                raise errors.PluginError(msg)
+
+    def check_aug_version(self):
+        """ Checks that we have recent enough version of libaugeas.
+        If augeas version is recent enough, it will support case insensitive
+        regexp matching"""
+
+        self.aug.set("/test/path/testing/arg", "aRgUMeNT")
+        try:
+            matches = self.aug.match(
+                "/test//*[self::arg=~regexp('argument', 'i')]")
+        except RuntimeError:
+            self.aug.remove("/test/path")
+            return False
+        self.aug.remove("/test/path")
+        return matches
+
+    def unsaved_files(self):
+        """Lists files that have modified Augeas DOM but the changes have not
+        been written to the filesystem yet, used by `self.save()` and
+        ApacheConfigurator to check the file state.
+
+        :raises .errors.PluginError: If there was an error in Augeas, in
+            an attempt to save the configuration, or an error creating a
+            checkpoint
+
+        :returns: `set` of unsaved files
+        """
+        save_state = self.aug.get("/augeas/save")
+        self.aug.set("/augeas/save", "noop")
+        # Existing Errors
+        ex_errs = self.aug.match("/augeas//error")
+        try:
+            # This is a noop save
+            self.aug.save()
+        except (RuntimeError, IOError):
+            self._log_save_errors(ex_errs)
+            # Erase Save Notes
+            self.configurator.save_notes = ""
+            raise errors.PluginError(
+                "Error saving files, check logs for more info.")
+
+        # Return the original save method
+        self.aug.set("/augeas/save", save_state)
+
+        # Retrieve list of modified files
+        # Note: Noop saves can cause the file to be listed twice, I used a
+        # set to remove this possibility. This is a known augeas 0.10 error.
+        save_paths = self.aug.match("/augeas/events/saved")
+
+        save_files = set()
+        if save_paths:
+            for path in save_paths:
+                save_files.add(self.aug.get(path)[6:])
+        return save_files
+
+    def ensure_augeas_state(self):
+        """Makes sure that all Augeas dom changes are written to files to avoid
+        loss of configuration directives when doing additional augeas parsing,
+        causing a possible augeas.load() resulting dom reset
+        """
+
+        if self.unsaved_files():
+            self.configurator.save_notes += "(autosave)"
+            self.configurator.save()
+
+    def save(self, save_files):
+        """Saves all changes to the configuration files.
+
+        save() is called from ApacheConfigurator to handle the parser specific
+        tasks of saving.
+
+        :param list save_files: list of strings of file paths that we need to save.
+
+        """
+        self.configurator.save_notes = ""
+        self.aug.save()
+
+        # Force reload if files were modified
+        # This is needed to recalculate augeas directive span
+        if save_files:
+            for sf in save_files:
+                self.aug.remove("/files/"+sf)
+            self.aug.load()
+
+    def _log_save_errors(self, ex_errs):
+        """Log errors due to bad Augeas save.
+
+        :param list ex_errs: Existing errors before save
+
+        """
+        # Check for the root of save problems
+        new_errs = self.aug.match("/augeas//error")
+        # logger.error("During Save - %s", mod_conf)
+        logger.error("Unable to save files: %s. Attempted Save Notes: %s",
+                     ", ".join(err[13:len(err) - 6] for err in new_errs
+                               # Only new errors caused by recent save
+                               if err not in ex_errs), self.configurator.save_notes)
+
     def add_include(self, main_config, inc_path):
         """Add Include for a new configuration file if one does not exist
 
@@ -658,8 +809,7 @@ class ApacheParser(object):
         use_new, remove_old = self._check_path_actions(filepath)
         # Ensure that we have the latest Augeas DOM state on disk before
         # calling aug.load() which reloads the state from disk
-        if self.configurator:
-            self.configurator.ensure_augeas_state()
+        self.ensure_augeas_state()
         # Test if augeas included file for Httpd.lens
         # Note: This works for augeas globs, ie. *.conf
         if use_new:

certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test-pebble.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+"""
+This executable script wraps the apache-conf-test bash script, in order to setup a pebble instance
+before its execution. Directory URL is passed through the SERVER environment variable.
+"""
+import os
+import subprocess
+import sys
+
+from certbot_integration_tests.utils import acme_server
+
+SCRIPT_DIRNAME = os.path.dirname(__file__)
+
+
+def main(args=None):
+    if not args:
+        args = sys.argv[1:]
+    with acme_server.ACMEServer('pebble', [], False) as acme_xdist:
+        environ = os.environ.copy()
+        environ['SERVER'] = acme_xdist['directory_url']
+        command = [os.path.join(SCRIPT_DIRNAME, 'apache-conf-test')]
+        command.extend(args)
+        return subprocess.call(command, env=environ)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

certbot-apache/certbot_apache/tests/centos6_test.py

@@ -165,10 +165,6 @@ class CentOS6Tests(util.ApacheTest):
             "LoadModule", "ssl_module", start=self.vh_truth[1].path, exclude=False)
         self.assertEqual(len(post_loadmods), 1)
 
-
-
-
-
     def test_loadmod_non_duplicate(self):
         # the modules/mod_ssl.so exists in ssl.conf
         sslmod_args = ["ssl_module", "modules/mod_somethingelse.so"]
@@ -197,7 +193,7 @@ class CentOS6Tests(util.ApacheTest):
                                                     exclude=False)
         for mod in orig_loadmods:
             noarg_path = mod.rpartition("/")[0]
-            self.config.aug.remove(noarg_path)
+            self.config.parser.aug.remove(noarg_path)
         self.config.save()
         self.config.deploy_cert(
             "random.demo", "example/cert.pem", "example/key.pem",

certbot-apache/certbot_apache/tests/centos_test.py

@@ -4,6 +4,7 @@ import unittest
 import mock
 
 from certbot import errors
+from certbot.compat import filesystem
 from certbot.compat import os
 
 from certbot_apache import obj
@@ -160,7 +161,7 @@ class MultipleVhostsTestCentOS(util.ApacheTest):
         """Make sure we read the sysconfig OPTIONS variable correctly"""
         # Return nothing for the process calls
         mock_cfg.return_value = ""
-        self.config.parser.sysconfig_filep = os.path.realpath(
+        self.config.parser.sysconfig_filep = filesystem.realpath(
             os.path.join(self.config.parser.root, "../sysconfig/httpd"))
         self.config.parser.variables = {}
 

certbot-apache/certbot_apache/tests/configurator_reverter_test.py

@@ -1,21 +1,20 @@
-"""Test for certbot_apache.augeas_configurator."""
+"""Test for certbot_apache.configurator implementations of reverter"""
 import shutil
 import unittest
 
 import mock
 
 from certbot import errors
-from certbot.compat import os
 
 from certbot_apache.tests import util
 
 
-class AugeasConfiguratorTest(util.ApacheTest):
-    """Test for Augeas Configurator base class."""
+class ConfiguratorReverterTest(util.ApacheTest):
+    """Test for ApacheConfigurator reverter methods"""
 
 
     def setUp(self):  # pylint: disable=arguments-differ
-        super(AugeasConfiguratorTest, self).setUp()
+        super(ConfiguratorReverterTest, self).setUp()
 
         self.config = util.get_apache_configurator(
             self.config_path, self.vhost_path, self.config_dir, self.work_dir)
@@ -28,20 +27,6 @@ class AugeasConfiguratorTest(util.ApacheTest):
         shutil.rmtree(self.work_dir)
         shutil.rmtree(self.temp_dir)
 
-    def test_bad_parse(self):
-        # pylint: disable=protected-access
-        self.config.parser.parse_file(os.path.join(
-            self.config.parser.root, "conf-available", "bad_conf_file.conf"))
-        self.assertRaises(
-            errors.PluginError, self.config.check_parsing_errors, "httpd.aug")
-
-    def test_bad_save(self):
-        mock_save = mock.Mock()
-        mock_save.side_effect = IOError
-        self.config.aug.save = mock_save
-
-        self.assertRaises(errors.PluginError, self.config.save)
-
     def test_bad_save_checkpoint(self):
         self.config.reverter.add_to_checkpoint = mock.Mock(
             side_effect=errors.ReverterError)
@@ -63,23 +48,9 @@ class AugeasConfiguratorTest(util.ApacheTest):
 
         self.assertTrue(mock_finalize.is_called)
 
-    def test_recovery_routine(self):
-        mock_load = mock.Mock()
-        self.config.aug.load = mock_load
-
-        self.config.recovery_routine()
-        self.assertEqual(mock_load.call_count, 1)
-
-    def test_recovery_routine_error(self):
-        self.config.reverter.recovery_routine = mock.Mock(
-            side_effect=errors.ReverterError)
-
-        self.assertRaises(
-            errors.PluginError, self.config.recovery_routine)
-
     def test_revert_challenge_config(self):
         mock_load = mock.Mock()
-        self.config.aug.load = mock_load
+        self.config.parser.aug.load = mock_load
 
         self.config.revert_challenge_config()
         self.assertEqual(mock_load.call_count, 1)
@@ -93,7 +64,7 @@ class AugeasConfiguratorTest(util.ApacheTest):
 
     def test_rollback_checkpoints(self):
         mock_load = mock.Mock()
-        self.config.aug.load = mock_load
+        self.config.parser.aug.load = mock_load
 
         self.config.rollback_checkpoints()
         self.assertEqual(mock_load.call_count, 1)
@@ -103,13 +74,11 @@ class AugeasConfiguratorTest(util.ApacheTest):
             side_effect=errors.ReverterError)
         self.assertRaises(errors.PluginError, self.config.rollback_checkpoints)
 
-    def test_view_config_changes(self):
-        self.config.view_config_changes()
-
-    def test_view_config_changes_error(self):
-        self.config.reverter.view_config_changes = mock.Mock(
-            side_effect=errors.ReverterError)
-        self.assertRaises(errors.PluginError, self.config.view_config_changes)
+    def test_recovery_routine_reload(self):
+        mock_load = mock.Mock()
+        self.config.parser.aug.load = mock_load
+        self.config.recovery_routine()
+        self.assertEqual(mock_load.call_count, 1)
 
 
 if __name__ == "__main__":

certbot-apache/certbot_apache/tests/configurator_test.py

@@ -16,6 +16,7 @@ from certbot import achallenges
 from certbot import crypto_util
 from certbot import errors
 from certbot.compat import os
+from certbot.compat import filesystem
 from certbot.tests import acme_util
 from certbot.tests import util as certbot_util
 
@@ -50,25 +51,14 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.deploy_cert = mocked_deploy_cert
         return self.config
 
-    @mock.patch("certbot_apache.configurator.ApacheConfigurator.init_augeas")
     @mock.patch("certbot_apache.configurator.path_surgery")
-    def test_prepare_no_install(self, mock_surgery, _init_augeas):
+    def test_prepare_no_install(self, mock_surgery):
         silly_path = {"PATH": "/tmp/nothingness2342"}
         mock_surgery.return_value = False
         with mock.patch.dict('os.environ', silly_path):
             self.assertRaises(errors.NoInstallationError, self.config.prepare)
             self.assertEqual(mock_surgery.call_count, 1)
 
-    @mock.patch("certbot_apache.augeas_configurator.AugeasConfigurator.init_augeas")
-    def test_prepare_no_augeas(self, mock_init_augeas):
-        """ Test augeas initialization ImportError """
-        def side_effect_error():
-            """ Side effect error for the test """
-            raise ImportError
-        mock_init_augeas.side_effect = side_effect_error
-        self.assertRaises(
-            errors.NoInstallationError, self.config.prepare)
-
     @mock.patch("certbot_apache.parser.ApacheParser")
     @mock.patch("certbot_apache.configurator.util.exe_exists")
     def test_prepare_version(self, mock_exe_exists, _):
@@ -80,16 +70,6 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertRaises(
             errors.NotSupportedError, self.config.prepare)
 
-    @mock.patch("certbot_apache.parser.ApacheParser")
-    @mock.patch("certbot_apache.configurator.util.exe_exists")
-    def test_prepare_old_aug(self, mock_exe_exists, _):
-        mock_exe_exists.return_value = True
-        self.config.config_test = mock.Mock()
-        # pylint: disable=protected-access
-        self.config._check_aug_version = mock.Mock(return_value=False)
-        self.assertRaises(
-            errors.NotSupportedError, self.config.prepare)
-
     def test_prepare_locked(self):
         server_root = self.config.conf("server-root")
         self.config.config_test = mock.Mock()
@@ -674,7 +654,7 @@ class MultipleVhostsTest(util.ApacheTest):
         # span excludes the closing </VirtualHost> tag in older versions
         # of Augeas
         return_value = [self.vh_truth[0].filep, 1, 12, 0, 0, 0, 1142]
-        with mock.patch.object(self.config.aug, 'span') as mock_span:
+        with mock.patch.object(self.config.parser.aug, 'span') as mock_span:
             mock_span.return_value = return_value
             self.test_make_vhost_ssl()
 
@@ -682,7 +662,7 @@ class MultipleVhostsTest(util.ApacheTest):
         # span includes the closing </VirtualHost> tag in newer versions
         # of Augeas
         return_value = [self.vh_truth[0].filep, 1, 12, 0, 0, 0, 1157]
-        with mock.patch.object(self.config.aug, 'span') as mock_span:
+        with mock.patch.object(self.config.parser.aug, 'span') as mock_span:
             mock_span.return_value = return_value
             self.test_make_vhost_ssl()
 
@@ -695,8 +675,7 @@ class MultipleVhostsTest(util.ApacheTest):
     def test_make_vhost_ssl_nonexistent_vhost_path(self):
         ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[1])
         self.assertEqual(os.path.dirname(ssl_vhost.filep),
-                            os.path.dirname(os.path.realpath(
-                                self.vh_truth[1].filep)))
+                         os.path.dirname(filesystem.realpath(self.vh_truth[1].filep)))
 
     def test_make_vhost_ssl(self):
         ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])
@@ -1231,7 +1210,7 @@ class MultipleVhostsTest(util.ApacheTest):
         except errors.PluginEnhancementAlreadyPresent:
             args_paths = self.config.parser.find_dir(
                 "RewriteRule", None, http_vhost.path, False)
-            arg_vals = [self.config.aug.get(x) for x in args_paths]
+            arg_vals = [self.config.parser.aug.get(x) for x in args_paths]
             self.assertEqual(arg_vals, constants.REWRITE_HTTPS_ARGS)
 
 
@@ -1334,15 +1313,6 @@ class MultipleVhostsTest(util.ApacheTest):
 
         return account_key, (achall1, achall2, achall3)
 
-    def test_aug_version(self):
-        mock_match = mock.Mock(return_value=["something"])
-        self.config.aug.match = mock_match
-        # pylint: disable=protected-access
-        self.assertEqual(self.config._check_aug_version(),
-                         ["something"])
-        self.config.aug.match.side_effect = RuntimeError
-        self.assertFalse(self.config._check_aug_version())
-
     def test_enable_site_nondebian(self):
         inc_path = "/path/to/wherever"
         vhost = self.vh_truth[0]
@@ -1365,8 +1335,8 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.parser.modules.add("ssl_module")
         self.config.parser.modules.add("mod_ssl.c")
         self.config.parser.modules.add("socache_shmcb_module")
-        tmp_path = os.path.realpath(tempfile.mkdtemp("vhostroot"))
-        os.chmod(tmp_path, 0o755)
+        tmp_path = filesystem.realpath(tempfile.mkdtemp("vhostroot"))
+        filesystem.chmod(tmp_path, 0o755)
         mock_p = "certbot_apache.configurator.ApacheConfigurator._get_ssl_vhost_path"
         mock_a = "certbot_apache.parser.ApacheParser.add_include"
 
@@ -1511,7 +1481,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertEqual(first_id, second_id)
 
     def test_realpath_replaces_symlink(self):
-        orig_match = self.config.aug.match
+        orig_match = self.config.parser.aug.match
         mock_vhost = copy.deepcopy(self.vh_truth[0])
         mock_vhost.filep = mock_vhost.filep.replace('sites-enabled', u'sites-available')
         mock_vhost.path = mock_vhost.path.replace('sites-enabled', 'sites-available')
@@ -1525,7 +1495,7 @@ class MultipleVhostsTest(util.ApacheTest):
             return orig_match(aug_expr)
 
         self.config.parser.parser_paths = ["/mocked/path"]
-        self.config.aug.match = mock_match
+        self.config.parser.aug.match = mock_match
         vhs = self.config.get_virtual_hosts()
         self.assertEqual(len(vhs), 2)
         self.assertTrue(vhs[0] == self.vh_truth[1])
@@ -1551,8 +1521,8 @@ class AugeasVhostsTest(util.ApacheTest):
             self.work_dir)
 
     def test_choosevhost_with_illegal_name(self):
-        self.config.aug = mock.MagicMock()
-        self.config.aug.match.side_effect = RuntimeError
+        self.config.parser.aug = mock.MagicMock()
+        self.config.parser.aug.match.side_effect = RuntimeError
         path = "debian_apache_2_4/augeas_vhosts/apache2/sites-available/old-and-default.conf"
         chosen_vhost = self.config._create_vhost(path)
         self.assertEqual(None, chosen_vhost)

certbot-apache/certbot_apache/tests/debian_test.py

@@ -79,9 +79,9 @@ class MultipleVhostsTestDebian(util.ApacheTest):
 
     def test_enable_site_failure(self):
         self.config.parser.root = "/tmp/nonexistent"
-        with mock.patch("os.path.isdir") as mock_dir:
+        with mock.patch("certbot.compat.os.path.isdir") as mock_dir:
             mock_dir.return_value = True
-            with mock.patch("os.path.islink") as mock_link:
+            with mock.patch("certbot.compat.os.path.islink") as mock_link:
                 mock_link.return_value = False
                 self.assertRaises(
                     errors.NotSupportedError,

certbot-apache/certbot_apache/tests/fedora_test.py

@@ -4,6 +4,7 @@ import unittest
 import mock
 
 from certbot import errors
+from certbot.compat import filesystem
 from certbot.compat import os
 
 from certbot_apache import obj
@@ -160,7 +161,7 @@ class MultipleVhostsTestFedora(util.ApacheTest):
         """Make sure we read the sysconfig OPTIONS variable correctly"""
         # Return nothing for the process calls
         mock_cfg.return_value = ""
-        self.config.parser.sysconfig_filep = os.path.realpath(
+        self.config.parser.sysconfig_filep = filesystem.realpath(
             os.path.join(self.config.parser.root, "../sysconfig/httpd"))
         self.config.parser.variables = {}
 

certbot-apache/certbot_apache/tests/gentoo_test.py

@@ -4,6 +4,7 @@ import unittest
 import mock
 
 from certbot import errors
+from certbot.compat import filesystem
 from certbot.compat import os
 
 from certbot_apache import obj
@@ -81,7 +82,7 @@ class MultipleVhostsTestGentoo(util.ApacheTest):
         """Make sure we read the Gentoo APACHE2_OPTS variable correctly"""
         defines = ['DEFAULT_VHOST', 'INFO',
                    'SSL', 'SSL_DEFAULT_VHOST', 'LANGUAGE']
-        self.config.parser.apacheconfig_filep = os.path.realpath(
+        self.config.parser.apacheconfig_filep = filesystem.realpath(
             os.path.join(self.config.parser.root, "../conf.d/apache2"))
         self.config.parser.variables = {}
         with mock.patch("certbot_apache.override_gentoo.GentooParser.update_modules"):

certbot-apache/certbot_apache/tests/http_01_test.py

@@ -7,6 +7,7 @@ from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-
 
 from certbot import achallenges
 from certbot import errors
+from certbot.compat import filesystem
 from certbot.compat import os
 from certbot.tests import acme_util
 
@@ -180,7 +181,7 @@ class ApacheHttp01Test(util.ApacheTest):
         self.assertEqual(self.http.perform(), expected_response)
 
         self.assertTrue(os.path.isdir(self.http.challenge_dir))
-        self._has_min_permissions(self.http.challenge_dir, 0o755)
+        self.assertTrue(filesystem.has_min_permissions(self.http.challenge_dir, 0o755))
         self._test_challenge_conf()
 
         for achall in achalls:
@@ -218,15 +219,10 @@ class ApacheHttp01Test(util.ApacheTest):
         name = os.path.join(self.http.challenge_dir, achall.chall.encode("token"))
         validation = achall.validation(self.account_key)
 
-        self._has_min_permissions(name, 0o644)
+        self.assertTrue(filesystem.has_min_permissions(name, 0o644))
         with open(name, 'rb') as f:
             self.assertEqual(f.read(), validation.encode())
 
-    def _has_min_permissions(self, path, min_mode):
-        """Tests the given file has at least the permissions in mode."""
-        st_mode = os.stat(path).st_mode
-        self.assertEqual(st_mode, st_mode | min_mode)
-
 
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/parser_test.py

@@ -1,8 +1,8 @@
+# pylint: disable=too-many-public-methods
 """Tests for certbot_apache.parser."""
 import shutil
 import unittest
 
-import augeas
 import mock
 
 from certbot import errors
@@ -22,6 +22,27 @@ class BasicParserTest(util.ParserTest):
         shutil.rmtree(self.config_dir)
         shutil.rmtree(self.work_dir)
 
+    def test_bad_parse(self):
+        self.parser.parse_file(os.path.join(self.parser.root,
+                                            "conf-available", "bad_conf_file.conf"))
+        self.assertRaises(
+            errors.PluginError, self.parser.check_parsing_errors, "httpd.aug")
+
+    def test_bad_save(self):
+        mock_save = mock.Mock()
+        mock_save.side_effect = IOError
+        self.parser.aug.save = mock_save
+        self.assertRaises(errors.PluginError, self.parser.unsaved_files)
+
+    def test_aug_version(self):
+        mock_match = mock.Mock(return_value=["something"])
+        self.parser.aug.match = mock_match
+        # pylint: disable=protected-access
+        self.assertEqual(self.parser.check_aug_version(),
+                         ["something"])
+        self.parser.aug.match.side_effect = RuntimeError
+        self.assertFalse(self.parser.check_aug_version())
+
     def test_find_config_root_no_root(self):
         # pylint: disable=protected-access
         os.remove(self.parser.loc["root"])
@@ -311,21 +332,38 @@ class BasicParserTest(util.ParserTest):
 class ParserInitTest(util.ApacheTest):
     def setUp(self):  # pylint: disable=arguments-differ
         super(ParserInitTest, self).setUp()
-        self.aug = augeas.Augeas(
-            flags=augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD)
 
     def tearDown(self):
         shutil.rmtree(self.temp_dir)
         shutil.rmtree(self.config_dir)
         shutil.rmtree(self.work_dir)
 
+    @mock.patch("certbot_apache.parser.ApacheParser.init_augeas")
+    def test_prepare_no_augeas(self, mock_init_augeas):
+        from certbot_apache.parser import ApacheParser
+        mock_init_augeas.side_effect = errors.NoInstallationError
+        self.config.config_test = mock.Mock()
+        self.assertRaises(
+            errors.NoInstallationError, ApacheParser,
+            os.path.relpath(self.config_path), "/dummy/vhostpath",
+            version=(2, 4, 22), configurator=self.config)
+
+    def test_init_old_aug(self):
+        from certbot_apache.parser import ApacheParser
+        with mock.patch("certbot_apache.parser.ApacheParser.check_aug_version") as mock_c:
+            mock_c.return_value = False
+            self.assertRaises(
+                errors.NotSupportedError,
+                ApacheParser, os.path.relpath(self.config_path),
+                "/dummy/vhostpath", version=(2, 4, 22), configurator=self.config)
+
     @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg")
     def test_unparseable(self, mock_cfg):
         from certbot_apache.parser import ApacheParser
         mock_cfg.return_value = ('Define: TEST')
         self.assertRaises(
             errors.PluginError,
-            ApacheParser, self.aug, os.path.relpath(self.config_path),
+            ApacheParser, os.path.relpath(self.config_path),
             "/dummy/vhostpath", version=(2, 2, 22), configurator=self.config)
 
     def test_root_normalized(self):
@@ -337,8 +375,7 @@ class ParserInitTest(util.ApacheTest):
                 self.temp_dir,
                 "debian_apache_2_4/////multiple_vhosts/../multiple_vhosts/apache2")
 
-            parser = ApacheParser(self.aug, path,
-                                  "/dummy/vhostpath", configurator=self.config)
+            parser = ApacheParser(path, "/dummy/vhostpath", configurator=self.config)
 
         self.assertEqual(parser.root, self.config_path)
 
@@ -347,7 +384,7 @@ class ParserInitTest(util.ApacheTest):
         with mock.patch("certbot_apache.parser.ApacheParser."
                         "update_runtime_variables"):
             parser = ApacheParser(
-                self.aug, os.path.relpath(self.config_path),
+                os.path.relpath(self.config_path),
                 "/dummy/vhostpath", configurator=self.config)
 
         self.assertEqual(parser.root, self.config_path)
@@ -357,7 +394,7 @@ class ParserInitTest(util.ApacheTest):
         with mock.patch("certbot_apache.parser.ApacheParser."
                         "update_runtime_variables"):
             parser = ApacheParser(
-                self.aug, self.config_path + os.path.sep,
+                self.config_path + os.path.sep,
                 "/dummy/vhostpath", configurator=self.config)
         self.assertEqual(parser.root, self.config_path)
 

certbot-apache/certbot_apache/tests/util.py

@@ -78,8 +78,7 @@ class ParserTest(ApacheTest):
         with mock.patch("certbot_apache.parser.ApacheParser."
                         "update_runtime_variables"):
             self.parser = ApacheParser(
-                self.aug, self.config_path, self.vhost_path,
-                configurator=self.config)
+                self.config_path, self.vhost_path, configurator=self.config)
 
 
 def get_apache_configurator(  # pylint: disable=too-many-arguments, too-many-locals

certbot-apache/local-oldest-requirements.txt

@@ -1,3 +1,3 @@
 # Remember to update setup.py to match the package versions below.
 acme[dev]==0.29.0
-certbot[dev]==0.34.0
+-e .[dev]

certbot-apache/setup.py

@@ -4,13 +4,13 @@ from setuptools.command.test import test as TestCommand
 import sys
 
 
-version = '0.35.0.dev0'
+version = '0.39.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
     'acme>=0.29.0',
-    'certbot>=0.34.0',
+    'certbot>=0.39.0.dev0',
     'mock',
     'python-augeas',
     'setuptools',
@@ -62,6 +62,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.34.0"
+LE_AUTO_VERSION="0.38.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -755,13 +755,31 @@ elif [ -f /etc/redhat-release ]; then
   prev_le_python="$LE_PYTHON"
   unset LE_PYTHON
   DeterminePythonVersion "NOCRASH"
-  # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then.
+
   RPM_DIST_NAME=`(. /etc/os-release 2> /dev/null && echo $ID) || echo "unknown"`
-  RPM_DIST_VERSION=0
-  if [ "$RPM_DIST_NAME" = "fedora" ]; then
-    RPM_DIST_VERSION=`(. /etc/os-release 2> /dev/null && echo $VERSION_ID) || echo "0"`
+
+  # Set RPM_DIST_VERSION to VERSION_ID from /etc/os-release after splitting on
+  # '.' characters (e.g. "8.0" becomes "8"). If the command exits with an
+  # error, RPM_DIST_VERSION is set to "unknown".
+  RPM_DIST_VERSION=$( (. /etc/os-release 2> /dev/null && echo "$VERSION_ID") | cut -d '.' -f1 || echo "unknown")
+
+  # If RPM_DIST_VERSION is an empty string or it contains any nonnumeric
+  # characters, the value is unexpected so we set RPM_DIST_VERSION to 0.
+  if [ -z "$RPM_DIST_VERSION" ] || [ -n "$(echo "$RPM_DIST_VERSION" | tr -d '[0-9]')" ]; then
+     RPM_DIST_VERSION=0
   fi
+
+  # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then.
+  # RHEL 8 also uses python3 by default.
   if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then
+    RPM_USE_PYTHON_3=1
+  elif [ "$RPM_DIST_NAME" = "rhel" -a "$RPM_DIST_VERSION" -ge 8 ]; then
+    RPM_USE_PYTHON_3=1
+  else
+    RPM_USE_PYTHON_3=0
+  fi
+
+  if [ "$RPM_USE_PYTHON_3" = 1 ]; then
     Bootstrap() {
       BootstrapMessage "RedHat-based OSes that will use Python3"
       BootstrapRpmPython3
@@ -775,6 +793,7 @@ elif [ -f /etc/redhat-release ]; then
     }
     BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
   fi
+
   LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
   Bootstrap() {
@@ -953,6 +972,95 @@ if __name__ == '__main__':
 UNLIKELY_EOF
 }
 
+# Check that the given PATH_TO_CHECK has secured permissions.
+# Parameters: LE_PYTHON, PATH_TO_CHECK
+CheckPathPermissions() {
+    "$1" - "$2" << "UNLIKELY_EOF"
+"""Verifies certbot-auto cannot be modified by unprivileged users.
+
+This script takes the path to certbot-auto as its only command line
+argument.  It then checks that the file can only be modified by uid/gid
+< 1000 and if other users can modify the file, it prints a warning with
+a suggestion on how to solve the problem.
+
+Permissions on symlinks in the absolute path of certbot-auto are ignored
+and only the canonical path to certbot-auto is checked. There could be
+permissions problems due to the symlinks that are unreported by this
+script, however, issues like this were not caused by our documentation
+and are ignored for the sake of simplicity.
+
+All warnings are printed to stdout rather than stderr so all stderr
+output from this script can be suppressed to avoid printing messages if
+this script fails for some reason.
+
+"""
+from __future__ import print_function
+
+import os
+import stat
+import sys
+
+
+FORUM_POST_URL = 'https://community.letsencrypt.org/t/certbot-auto-deployment-best-practices/91979/'
+
+
+def has_safe_permissions(path):
+    """Returns True if the given path has secure permissions.
+
+    The permissions are considered safe if the file is only writable by
+    uid/gid < 1000.
+
+    The reason we allow more IDs than 0 is because on some systems such
+    as Debian, system users/groups other than uid/gid 0 are used for the
+    path we recommend in our instructions which is /usr/local/bin.  1000
+    was chosen because on Debian 0-999 is reserved for system IDs[1] and
+    on RHEL either 0-499 or 0-999 is reserved depending on the
+    version[2][3]. Due to these differences across different OSes, this
+    detection isn't perfect so we only determine permissions are
+    insecure when we can be reasonably confident there is a problem
+    regardless of the underlying OS.
+
+    [1] https://www.debian.org/doc/debian-policy/ch-opersys.html#uid-and-gid-classes
+    [2] https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/ch-managing_users_and_groups
+    [3] https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/system_administrators_guide/ch-managing_users_and_groups
+
+    :param str path: filesystem path to check
+    :returns: True if the path has secure permissions, otherwise, False
+    :rtype: bool
+
+    """
+    # os.stat follows symlinks before obtaining information about a file.
+    stat_result = os.stat(path)
+    if stat_result.st_mode & stat.S_IWOTH:
+        return False
+    if stat_result.st_mode & stat.S_IWGRP and stat_result.st_gid >= 1000:
+        return False
+    if stat_result.st_mode & stat.S_IWUSR and stat_result.st_uid >= 1000:
+        return False
+    return True
+
+
+def main(certbot_auto_path):
+    current_path = os.path.realpath(certbot_auto_path)
+    last_path = None
+    permissions_ok = True
+    # This loop makes use of the fact that os.path.dirname('/') == '/'.
+    while current_path != last_path and permissions_ok:
+        permissions_ok = has_safe_permissions(current_path)
+        last_path = current_path
+        current_path = os.path.dirname(current_path)
+
+    if not permissions_ok:
+        print('{0} has insecure permissions!'.format(certbot_auto_path))
+        print('To learn how to fix them, visit {0}'.format(FORUM_POST_URL))
+
+
+if __name__ == '__main__':
+    main(sys.argv[1])
+
+UNLIKELY_EOF
+}
+
 if [ "$1" = "--le-auto-phase2" ]; then
   # Phase 2: Create venv, install LE, and run.
 
@@ -1026,73 +1134,76 @@ if [ "$1" = "--le-auto-phase2" ]; then
 # To generate this, do (with docker and package hashin installed):
 # ```
 # letsencrypt-auto-source/rebuild_dependencies.py \
-#   letsencrypt-auto-sources/pieces/dependency-requirements.txt
+#   letsencrypt-auto-source/pieces/dependency-requirements.txt
+# ```
+# If you want to update a single dependency, run commands similar to these:
+# ```
+# pip install hashin
+# hashin -r dependency-requirements.txt cryptography==1.5.2
 # ```
 ConfigArgParse==0.14.0 \
     --hash=sha256:2e2efe2be3f90577aca9415e32cb629aa2ecd92078adbe27b53a03e53ff12e91
 asn1crypto==0.24.0 \
     --hash=sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87 \
     --hash=sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49
-certifi==2019.3.9 \
-    --hash=sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5 \
-    --hash=sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae
-cffi==1.12.2 \
-    --hash=sha256:00b97afa72c233495560a0793cdc86c2571721b4271c0667addc83c417f3d90f \
-    --hash=sha256:0ba1b0c90f2124459f6966a10c03794082a2f3985cd699d7d63c4a8dae113e11 \
-    --hash=sha256:0bffb69da295a4fc3349f2ec7cbe16b8ba057b0a593a92cbe8396e535244ee9d \
-    --hash=sha256:21469a2b1082088d11ccd79dd84157ba42d940064abbfa59cf5f024c19cf4891 \
-    --hash=sha256:2e4812f7fa984bf1ab253a40f1f4391b604f7fc424a3e21f7de542a7f8f7aedf \
-    --hash=sha256:2eac2cdd07b9049dd4e68449b90d3ef1adc7c759463af5beb53a84f1db62e36c \
-    --hash=sha256:2f9089979d7456c74d21303c7851f158833d48fb265876923edcb2d0194104ed \
-    --hash=sha256:3dd13feff00bddb0bd2d650cdb7338f815c1789a91a6f68fdc00e5c5ed40329b \
-    --hash=sha256:4065c32b52f4b142f417af6f33a5024edc1336aa845b9d5a8d86071f6fcaac5a \
-    --hash=sha256:51a4ba1256e9003a3acf508e3b4f4661bebd015b8180cc31849da222426ef585 \
-    --hash=sha256:59888faac06403767c0cf8cfb3f4a777b2939b1fbd9f729299b5384f097f05ea \
-    --hash=sha256:59c87886640574d8b14910840327f5cd15954e26ed0bbd4e7cef95fa5aef218f \
-    --hash=sha256:610fc7d6db6c56a244c2701575f6851461753c60f73f2de89c79bbf1cc807f33 \
-    --hash=sha256:70aeadeecb281ea901bf4230c6222af0248c41044d6f57401a614ea59d96d145 \
-    --hash=sha256:71e1296d5e66c59cd2c0f2d72dc476d42afe02aeddc833d8e05630a0551dad7a \
-    --hash=sha256:8fc7a49b440ea752cfdf1d51a586fd08d395ff7a5d555dc69e84b1939f7ddee3 \
-    --hash=sha256:9b5c2afd2d6e3771d516045a6cfa11a8da9a60e3d128746a7fe9ab36dfe7221f \
-    --hash=sha256:9c759051ebcb244d9d55ee791259ddd158188d15adee3c152502d3b69005e6bd \
-    --hash=sha256:b4d1011fec5ec12aa7cc10c05a2f2f12dfa0adfe958e56ae38dc140614035804 \
-    --hash=sha256:b4f1d6332339ecc61275bebd1f7b674098a66fea11a00c84d1c58851e618dc0d \
-    --hash=sha256:c030cda3dc8e62b814831faa4eb93dd9a46498af8cd1d5c178c2de856972fd92 \
-    --hash=sha256:c2e1f2012e56d61390c0e668c20c4fb0ae667c44d6f6a2eeea5d7148dcd3df9f \
-    --hash=sha256:c37c77d6562074452120fc6c02ad86ec928f5710fbc435a181d69334b4de1d84 \
-    --hash=sha256:c8149780c60f8fd02752d0429246088c6c04e234b895c4a42e1ea9b4de8d27fb \
-    --hash=sha256:cbeeef1dc3c4299bd746b774f019de9e4672f7cc666c777cd5b409f0b746dac7 \
-    --hash=sha256:e113878a446c6228669144ae8a56e268c91b7f1fafae927adc4879d9849e0ea7 \
-    --hash=sha256:e21162bf941b85c0cda08224dade5def9360f53b09f9f259adb85fc7dd0e7b35 \
-    --hash=sha256:fb6934ef4744becbda3143d30c6604718871495a5e36c408431bf33d9c146889
+certifi==2019.6.16 \
+    --hash=sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939 \
+    --hash=sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695
+cffi==1.12.3 \
+    --hash=sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774 \
+    --hash=sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d \
+    --hash=sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90 \
+    --hash=sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b \
+    --hash=sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63 \
+    --hash=sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45 \
+    --hash=sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25 \
+    --hash=sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3 \
+    --hash=sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b \
+    --hash=sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647 \
+    --hash=sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016 \
+    --hash=sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4 \
+    --hash=sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb \
+    --hash=sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753 \
+    --hash=sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7 \
+    --hash=sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9 \
+    --hash=sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f \
+    --hash=sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8 \
+    --hash=sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f \
+    --hash=sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc \
+    --hash=sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42 \
+    --hash=sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3 \
+    --hash=sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909 \
+    --hash=sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45 \
+    --hash=sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d \
+    --hash=sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512 \
+    --hash=sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff \
+    --hash=sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201
 chardet==3.0.4 \
     --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
     --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691
 configobj==5.0.6 \
     --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902
-cryptography==2.6.1 \
-    --hash=sha256:066f815f1fe46020877c5983a7e747ae140f517f1b09030ec098503575265ce1 \
-    --hash=sha256:210210d9df0afba9e000636e97810117dc55b7157c903a55716bb73e3ae07705 \
-    --hash=sha256:26c821cbeb683facb966045e2064303029d572a87ee69ca5a1bf54bf55f93ca6 \
-    --hash=sha256:2afb83308dc5c5255149ff7d3fb9964f7c9ee3d59b603ec18ccf5b0a8852e2b1 \
-    --hash=sha256:2db34e5c45988f36f7a08a7ab2b69638994a8923853dec2d4af121f689c66dc8 \
-    --hash=sha256:409c4653e0f719fa78febcb71ac417076ae5e20160aec7270c91d009837b9151 \
-    --hash=sha256:45a4f4cf4f4e6a55c8128f8b76b4c057027b27d4c67e3fe157fa02f27e37830d \
-    --hash=sha256:48eab46ef38faf1031e58dfcc9c3e71756a1108f4c9c966150b605d4a1a7f659 \
-    --hash=sha256:6b9e0ae298ab20d371fc26e2129fd683cfc0cfde4d157c6341722de645146537 \
-    --hash=sha256:6c4778afe50f413707f604828c1ad1ff81fadf6c110cb669579dea7e2e98a75e \
-    --hash=sha256:8c33fb99025d353c9520141f8bc989c2134a1f76bac6369cea060812f5b5c2bb \
-    --hash=sha256:9873a1760a274b620a135054b756f9f218fa61ca030e42df31b409f0fb738b6c \
-    --hash=sha256:9b069768c627f3f5623b1cbd3248c5e7e92aec62f4c98827059eed7053138cc9 \
-    --hash=sha256:9e4ce27a507e4886efbd3c32d120db5089b906979a4debf1d5939ec01b9dd6c5 \
-    --hash=sha256:acb424eaca214cb08735f1a744eceb97d014de6530c1ea23beb86d9c6f13c2ad \
-    --hash=sha256:c8181c7d77388fe26ab8418bb088b1a1ef5fde058c6926790c8a0a3d94075a4a \
-    --hash=sha256:d4afbb0840f489b60f5a580a41a1b9c3622e08ecb5eec8614d4fb4cd914c4460 \
-    --hash=sha256:d9ed28030797c00f4bc43c86bf819266c76a5ea61d006cd4078a93ebf7da6bfd \
-    --hash=sha256:e603aa7bb52e4e8ed4119a58a03b60323918467ef209e6ff9db3ac382e5cf2c6
-# Package enum34 needs to be explicitly limited to Python2.x, in order to avoid
-# certbot-auto failures on Python 3.6+ which enum34 doesn't support. See #5456.
-enum34==1.1.6 ; python_version < '3.4' \
+cryptography==2.7 \
+    --hash=sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c \
+    --hash=sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643 \
+    --hash=sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216 \
+    --hash=sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799 \
+    --hash=sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a \
+    --hash=sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9 \
+    --hash=sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc \
+    --hash=sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8 \
+    --hash=sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53 \
+    --hash=sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1 \
+    --hash=sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609 \
+    --hash=sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292 \
+    --hash=sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e \
+    --hash=sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6 \
+    --hash=sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed \
+    --hash=sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d
+distro==1.4.0 \
+    --hash=sha256:362dde65d846d23baee4b5c058c8586f219b5a54be1cf5fc6ff55c4578392f57 \
+    --hash=sha256:eedf82a470ebe7d010f1872c17237c79ab04097948800029994fa458e52fb4b4
+enum34==1.1.6 \
     --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850 \
     --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \
     --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \
@@ -1108,18 +1219,18 @@ idna==2.8 \
 ipaddress==1.0.22 \
     --hash=sha256:64b28eec5e78e7510698f6d4da08800a5c575caa4a286c93d651c5d3ff7b6794 \
     --hash=sha256:b146c751ea45cad6188dd6cf2d9b757f6f4f8d6ffb96a023e6f2e26eea02a72c
-josepy==1.1.0 \
-    --hash=sha256:1309a25aac3caeff5239729c58ff9b583f7d022ffdb1553406ddfc8e5b52b76e \
-    --hash=sha256:fb5c62c77d26e04df29cb5ecd01b9ce69b6fcc9e521eb1ca193b7faa2afa7086
+josepy==1.2.0 \
+    --hash=sha256:8ea15573203f28653c00f4ac0142520777b1c59d9eddd8da3f256c6ba3cac916 \
+    --hash=sha256:9cec9a839fe9520f0420e4f38e7219525daccce4813296627436fe444cd002d3
 mock==1.3.0 \
     --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 \
     --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb
 parsedatetime==2.4 \
     --hash=sha256:3d817c58fb9570d1eec1dd46fa9448cd644eeed4fb612684b02dfda3a79cb84b \
     --hash=sha256:9ee3529454bf35c40a77115f5a596771e59e1aee8c53306f346c461b8e913094
-pbr==5.1.3 \
-    --hash=sha256:8257baf496c8522437e8a6cfe0f15e00aedc6c0e0e7c9d55eeeeab31e0853843 \
-    --hash=sha256:8c361cc353d988e4f5b998555c88098b9d5964c2e11acf7b0d21925a66bb5824
+pbr==5.4.2 \
+    --hash=sha256:56e52299170b9492513c64be44736d27a512fa7e606f21942160b68ce510b4bc \
+    --hash=sha256:9b321c204a88d8ab5082699469f52cc94c5da45c51f114113d01b3d993c24cdf
 pyOpenSSL==19.0.0 \
     --hash=sha256:aeca66338f6de19d1aa46ed634c3b9ae519a64b458f8468aec688e7e3c20f200 \
     --hash=sha256:c727930ad54b10fc157015014b666f2d8b41f70c0d03e83ab67624fd3dd5d1e6
@@ -1128,14 +1239,14 @@ pyRFC3339==1.1 \
     --hash=sha256:81b8cbe1519cdb79bed04910dd6fa4e181faf8c88dff1e1b987b5f7ab23a5b1a
 pycparser==2.19 \
     --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3
-pyparsing==2.3.1 \
-    --hash=sha256:66c9268862641abcac4a96ba74506e594c884e3f57690a696d21ad8210ed667a \
-    --hash=sha256:f6c5ef0d7480ad048c054c37632c67fca55299990fff127850181659eea33fc3
+pyparsing==2.4.2 \
+    --hash=sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80 \
+    --hash=sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4
 python-augeas==0.5.0 \
     --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2
-pytz==2018.9 \
-    --hash=sha256:32b0891edff07e28efe91284ed9c31e123d84bea3fd98e1f72be2508f43ef8d9 \
-    --hash=sha256:d5f05e487007e29e03409f9398d074e158d920d36eb82eaf66fb1136b0c5374c
+pytz==2019.2 \
+    --hash=sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32 \
+    --hash=sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7
 requests==2.21.0 \
     --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \
     --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b
@@ -1145,15 +1256,15 @@ requests-toolbelt==0.9.1 \
 six==1.12.0 \
     --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
     --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73
-urllib3==1.24.2 \
-    --hash=sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0 \
-    --hash=sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3
+urllib3==1.24.3 \
+    --hash=sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4 \
+    --hash=sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb
 zope.component==4.5 \
     --hash=sha256:6edfd626c3b593b72895a8cfcf79bff41f4619194ce996a85bce31ac02b94e55 \
     --hash=sha256:984a06ba3def0b02b1117fa4c45b56e772e8c29c0340820fbf367e440a93a3a4
-zope.deferredimport==4.3 \
-    --hash=sha256:2ddef5a7ecfff132a2dd796253366ecf9748a446e30f1a0b3a636aec9d9c05c5 \
-    --hash=sha256:4aae9cbacb2146cca58e62be0a914f0cec034d3b2d41135ea212ca8a96f4b5ec
+zope.deferredimport==4.3.1 \
+    --hash=sha256:57b2345e7b5eef47efcd4f634ff16c93e4265de3dcf325afc7315ade48d909e1 \
+    --hash=sha256:9a0c211df44aa95f1c4e6d2626f90b400f56989180d3ef96032d708da3d23e0a
 zope.deprecation==4.4.0 \
     --hash=sha256:0d453338f04bacf91bbfba545d8bcdf529aa829e67b705eac8c1a7fdce66e2df \
     --hash=sha256:f1480b74995958b24ce37b0ef04d3663d2683e5d6debc96726eff18acf4ea113
@@ -1201,18 +1312,18 @@ zope.interface==4.6.0 \
     --hash=sha256:d788a3999014ddf416f2dc454efa4a5dbeda657c6aba031cf363741273804c6b \
     --hash=sha256:eed88ae03e1ef3a75a0e96a55a99d7937ed03e53d0cffc2451c208db445a2966 \
     --hash=sha256:f99451f3a579e73b5dd58b1b08d1179791d49084371d9a47baad3b22417f0317
-zope.proxy==4.3.1 \
-    --hash=sha256:0cbcfcafaa3b5fde7ba7a7b9a2b5f09af25c9b90087ad65f9e61359fed0ca63b \
-    --hash=sha256:3de631dd5054a3a20b9ebff0e375f39c0565f1fb9131200d589a6a8f379214cd \
-    --hash=sha256:5429134d04d42262f4dac25f6dea907f6334e9a751ffc62cb1d40226fb52bdeb \
-    --hash=sha256:563c2454b2d0f23bca54d2e0e4d781149b7b06cb5df67e253ca3620f37202dd2 \
-    --hash=sha256:5bcf773345016b1461bb07f70c635b9386e5eaaa08e37d3939dcdf12d3fdbec5 \
-    --hash=sha256:8d84b7aef38c693874e2f2084514522bf73fd720fde0ce2a9352a51315ffa475 \
-    --hash=sha256:90de9473c05819b36816b6cb957097f809691836ed3142648bf62da84b4502fe \
-    --hash=sha256:dd592a69fe872445542a6e1acbefb8e28cbe6b4007b8f5146da917e49b155cc3 \
-    --hash=sha256:e7399ab865399fce322f9cefc6f2f3e4099d087ba581888a9fea1bbe1db42a08 \
-    --hash=sha256:e7d1c280d86d72735a420610df592aac72332194e531a8beff43a592c3a1b8eb \
-    --hash=sha256:e90243fee902adb0c39eceb3c69995c0f2004bc3fdb482fbf629efc656d124ed
+zope.proxy==4.3.2 \
+    --hash=sha256:320a7619992e42142549ebf61e14ce27683b4d14b0cbc45f7c037ba64edb560c \
+    --hash=sha256:824d4dbabbb7deb84f25fdb96ea1eeca436a1802c3c8d323b3eb4ac9d527d41c \
+    --hash=sha256:8a32eb9c94908f3544da2dae3f4a9e6961d78819b88ac6b6f4a51cee2d65f4a0 \
+    --hash=sha256:96265fd3bc3ea646f98482e16307a69de21402eeaaaaf4b841c1161ac2f71bb0 \
+    --hash=sha256:ab6d6975d9c51c13cac828ff03168de21fb562b0664c59bcdc4a4b10f39a5b17 \
+    --hash=sha256:af10cb772391772463f65a58348e2de5ecc06693c16d2078be276dc068bcbb54 \
+    --hash=sha256:b8fd3a3de3f7b6452775e92af22af5977b17b69ac86a38a3ddfe870e40a0d05f \
+    --hash=sha256:bb7088f1bed3b8214284a5e425dc23da56f2f28e8815b7580bfed9e245b6c0b6 \
+    --hash=sha256:bc29b3665eac34f14c4aef5224bef045efcfb1a7d12d78c8685858de5fbf21c0 \
+    --hash=sha256:c39fa6a159affeae5fe31b49d9f5b12bd674fe77271a9a324408b271440c50a7 \
+    --hash=sha256:e946a036ac5b9f897e986ac9dc950a34cffc857d88eae6727b8434fbc4752366
 
 # Contains the requirements for the letsencrypt package.
 #
@@ -1225,18 +1336,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.34.0 \
-    --hash=sha256:51dddf2cb1c50a9f8b993090890bf4858d8fadffce38bafcdf6bf585a2040317 \
-    --hash=sha256:e75bdabfd9183bd9842ada42a51070f120d15982e81c490df59dde62e4df2c8b
-acme==0.34.0 \
-    --hash=sha256:3448024d2c274aebfb9b31b53862576d167626ce2fd1997a78d450c32a292fa3 \
-    --hash=sha256:92478e58f541c5c7c527427a50650005cdede799b78f0a0a65b8093d6368bcfd
-certbot-apache==0.34.0 \
-    --hash=sha256:79e686f25b63dac17d771d71f791f252774da22125f3f6e0665f4cf791d516fe \
-    --hash=sha256:d5ae09b4801fbac23d5acf64a5ee265108199d2852fbe743e7b6ab06fa08edf6
-certbot-nginx==0.34.0 \
-    --hash=sha256:868d7dcb59bb2548cb4a2ae187db5da1bfe33aac306b1b844b96ee00a39cac52 \
-    --hash=sha256:d6c728b85c523711ec0dc800f8d4ebbef192fb0ca1ec7914c173207e4aba5194
+certbot==0.38.0 \
+    --hash=sha256:618abf3ae17c2fc3cb99baa4bf000dd5e2d7875b7811f5ef1edf6ebd7a33945f \
+    --hash=sha256:c27712101794e3adf54f3a3067c63be5caa507a930a79865bc654b6864121c6b
+acme==0.38.0 \
+    --hash=sha256:6231571b4a94d6d621b28bef6f6d4846b3c2ebca840f9718d3212036c3bd2af8 \
+    --hash=sha256:1c1e9c0826a8f72d670b0ca28b7e6392ce4781eb33222f35133705b6551885d8
+certbot-apache==0.38.0 \
+    --hash=sha256:0b5a2c2bcc430470b5131941ebdfde0a13e28dec38918c1a4ebea5dd35ad38bc \
+    --hash=sha256:2d335543e0ae9292303238736907ce6b321ac49eb49fe4e0b775abdc0ba57c62
+certbot-nginx==0.38.0 \
+    --hash=sha256:af82944e171d2e93c81438b185f8051e742c6f47f7382cb1a647b1c7ca2b53f2 \
+    --hash=sha256:cecd1fa3de6e19980fdb9c3b3269b15b7da71b5748ee7ae5caddcc18dbb208ac
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
@@ -1505,100 +1616,16 @@ else
   # Don't warn about file permissions if the user disabled the check or we
   # can't find an up-to-date Python.
   if [ "$PYVER" -ge "$MIN_PYVER" -a "$NO_PERMISSIONS_CHECK" != 1 ]; then
-    # ---------------------------------------------------------------------------
-    cat << "UNLIKELY_EOF" > "$TEMP_DIR/check_permissions.py"
-"""Verifies certbot-auto cannot be modified by unprivileged users.
-
-This script takes the path to certbot-auto as its only command line
-argument.  It then checks that the file can only be modified by uid/gid
-< 1000 and if other users can modify the file, it prints a warning with
-a suggestion on how to solve the problem.
-
-Permissions on symlinks in the absolute path of certbot-auto are ignored
-and only the canonical path to certbot-auto is checked. There could be
-permissions problems due to the symlinks that are unreported by this
-script, however, issues like this were not caused by our documentation
-and are ignored for the sake of simplicity.
-
-All warnings are printed to stdout rather than stderr so all stderr
-output from this script can be suppressed to avoid printing messages if
-this script fails for some reason.
-
-"""
-from __future__ import print_function
-
-import os
-import stat
-import sys
-
-
-FORUM_POST_URL = 'https://community.letsencrypt.org/t/certbot-auto-deployment-best-practices/91979/'
-
-
-def has_safe_permissions(path):
-    """Returns True if the given path has secure permissions.
-
-    The permissions are considered safe if the file is only writable by
-    uid/gid < 1000.
-
-    The reason we allow more IDs than 0 is because on some systems such
-    as Debian, system users/groups other than uid/gid 0 are used for the
-    path we recommend in our instructions which is /usr/local/bin.  1000
-    was chosen because on Debian 0-999 is reserved for system IDs[1] and
-    on RHEL either 0-499 or 0-999 is reserved depending on the
-    version[2][3]. Due to these differences across different OSes, this
-    detection isn't perfect so we only determine permissions are
-    insecure when we can be reasonably confident there is a problem
-    regardless of the underlying OS.
-
-    [1] https://www.debian.org/doc/debian-policy/ch-opersys.html#uid-and-gid-classes
-    [2] https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/ch-managing_users_and_groups
-    [3] https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/system_administrators_guide/ch-managing_users_and_groups
-
-    :param str path: filesystem path to check
-    :returns: True if the path has secure permissions, otherwise, False
-    :rtype: bool
-
-    """
-    # os.stat follows symlinks before obtaining information about a file.
-    stat_result = os.stat(path)
-    if stat_result.st_mode & stat.S_IWOTH:
-        return False
-    if stat_result.st_mode & stat.S_IWGRP and stat_result.st_gid >= 1000:
-        return False
-    if stat_result.st_mode & stat.S_IWUSR and stat_result.st_uid >= 1000:
-        return False
-    return True
-
-
-def main(certbot_auto_path):
-    current_path = os.path.realpath(certbot_auto_path)
-    last_path = None
-    permissions_ok = True
-    # This loop makes use of the fact that os.path.dirname('/') == '/'.
-    while current_path != last_path and permissions_ok:
-        permissions_ok = has_safe_permissions(current_path)
-        last_path = current_path
-        current_path = os.path.dirname(current_path)
-
-    if not permissions_ok:
-        print('{0} has insecure permissions!'.format(certbot_auto_path))
-        print('To learn how to fix them, visit {0}'.format(FORUM_POST_URL))
-
-
-if __name__ == '__main__':
-    main(sys.argv[1])
-
-UNLIKELY_EOF
-    # ---------------------------------------------------------------------------
     # If the script fails for some reason, don't break certbot-auto.
     set +e
-    # Suppress unexpected error output and only print the script's output if it
-    # ran successfully.
-    CHECK_PERM_OUT=$("$LE_PYTHON" "$TEMP_DIR/check_permissions.py" "$0" 2>/dev/null)
+    # Suppress unexpected error output.
+    CHECK_PERM_OUT=$(CheckPathPermissions "$LE_PYTHON" "$0" 2>/dev/null)
     CHECK_PERM_STATUS="$?"
     set -e
-    if [ "$CHECK_PERM_STATUS" = 0 ]; then
+    # Only print output if the script ran successfully and it actually produced
+    # output. The latter check resolves
+    # https://github.com/certbot/certbot/issues/7012.
+    if [ "$CHECK_PERM_STATUS" = 0 -a -n "$CHECK_PERM_OUT" ]; then
       error "$CHECK_PERM_OUT"
     fi
   fi

certbot-ci/certbot_integration_tests/.coveragerc

@@ -2,6 +2,7 @@
 # Avoid false warnings because certbot packages are not installed in the thread that executes
 # the coverage: indeed, certbot is launched as a CLI from a subprocess.
 disable_warnings = module-not-imported,no-data-collected
+omit = **/*_test.py,**/tests/*,**/dns_common*,**/certbot_nginx/parser_obj.py
 
 [report]
 # Exclude unit tests in coverage during integration tests.

certbot-ci/certbot_integration_tests/apache_tests/apache_config.py

@@ -0,0 +1,171 @@
+import shutil
+import subprocess
+import os
+
+import pkg_resources
+import getpass
+
+
+def construct_apache_config_dir(apache_root, http_port, https_port, key_path=None,
+                                cert_path=None, wtf_prefix='le'):
+    config_path = os.path.join(apache_root, 'config')
+    shutil.copytree('/etc/apache2', config_path, symlinks=True)
+
+    webroot_path = os.path.join(apache_root, 'www')
+    os.mkdir(webroot_path)
+    with open(os.path.join(webroot_path, 'index.html'), 'w') as file_h:
+        file_h.write('Hello World!')
+
+    main_config_path = os.path.join(config_path, 'apache2.conf')
+    with open(main_config_path, 'w') as file_h:
+        file_h.write('''\
+ServerRoot "{config}"
+DefaultRuntimeDir ${{APACHE_RUN_DIR}}
+PidFile ${{APACHE_PID_FILE}}
+Timeout 300
+KeepAlive On
+MaxKeepAliveRequests 100
+KeepAliveTimeout 5
+User ${{APACHE_RUN_USER}}
+Group ${{APACHE_RUN_GROUP}}
+HostnameLookups Off
+ErrorLog ${{APACHE_LOG_DIR}}/error.log
+LogLevel warn
+
+IncludeOptional mods-enabled/*.load
+IncludeOptional mods-enabled/*.conf
+
+Include ports.conf
+
+<Directory />
+    Options FollowSymLinks
+    AllowOverride None
+    Require all denied
+</Directory>
+
+<Directory /usr/share>
+    AllowOverride None
+    Require all granted
+</Directory>
+
+<Directory {webroot}/>
+    Options Indexes FollowSymLinks
+    AllowOverride None
+    Require all granted
+</Directory>
+
+AccessFileName .htaccess
+
+<FilesMatch "^\.ht">
+    Require all denied
+</FilesMatch>
+
+LogFormat "%v:%p %h %l %u %t \\"%r\\" %>s %O \\"%{{Referer}}i\\" \\"%{{User-Agent}}i\\"" vhost_combined
+LogFormat "%h %l %u %t \\"%r\\" %>s %O \\"%{{Referer}}i\\" \\"%{{User-Agent}}i\\"" combined
+LogFormat "%h %l %u %t \\"%r\\" %>s %O" common
+LogFormat "%{{Referer}}i -> %U" referer
+LogFormat "%{{User-agent}}i" agent
+
+IncludeOptional conf-enabled/*.conf
+IncludeOptional sites-enabled/*.conf
+'''.format(config=config_path, webroot=webroot_path))
+
+    with open(os.path.join(config_path, 'ports.conf'), 'w') as file_h:
+        file_h.write('''\
+Listen {http}
+<IfModule ssl_module>
+    Listen {https}
+</IfModule>
+<IfModule mod_gnutls.c>
+    Listen {https}
+</IfModule>
+'''.format(http=http_port, https=https_port))
+
+    new_environ = os.environ.copy()
+    new_environ['APACHE_CONFDIR'] = config_path
+
+    run_path = os.path.join(apache_root, 'run')
+    lock_path = os.path.join(apache_root, 'lock')
+    logs_path = os.path.join(apache_root, 'logs')
+    os.mkdir(run_path)
+    os.mkdir(lock_path)
+    os.mkdir(logs_path)
+
+    user = getpass.getuser()
+    user = user if user != 'root' else 'www-data'
+    group = user
+
+    pid_file = os.path.join(run_path, 'apache.pid')
+
+    with open(os.path.join(config_path, 'envvars'), 'w') as file_h:
+        file_h.write('''\
+unset HOME
+export APACHE_RUN_USER={user}
+export APACHE_RUN_GROUP={group}
+export APACHE_PID_FILE={pid_file}
+export APACHE_RUN_DIR={run_path}
+export APACHE_LOCK_DIR={lock_path}
+export APACHE_LOG_DIR={logs_path}
+export LANG=C
+'''.format(user=user, group=group, pid_file=pid_file,
+           run_path=run_path, lock_path=lock_path, logs_path=logs_path))
+
+    new_environ['APACHE_RUN_USER'] = user
+    new_environ['APACHE_RUN_GROUP'] = group
+    new_environ['APACHE_PID_FILE'] = pid_file
+    new_environ['APACHE_RUN_DIR'] = run_path
+    new_environ['APACHE_LOCK_DIR'] = lock_path
+    new_environ['APACHE_LOG_DIR'] = logs_path
+
+    le_host = 'apache.{0}.wtf'.format(wtf_prefix)
+
+    with open(os.path.join(config_path, 'sites-available', '000-default.conf'), 'w') as file_h:
+        file_h.write('''\
+<VirtualHost *:{http}>
+    ServerAdmin webmaster@localhost
+    ServerName {le_host}
+    DocumentRoot {webroot}
+    
+    ErrorLog ${{APACHE_LOG_DIR}}/error.log
+    CustomLog ${{APACHE_LOG_DIR}}/access.log combined
+</VirtualHost>
+'''.format(http=http_port, le_host=le_host, webroot=webroot_path))
+
+    key_path = key_path if key_path \
+        else pkg_resources.resource_filename('certbot_integration_tests', 'assets/key.pem')
+    cert_path = cert_path if cert_path \
+        else pkg_resources.resource_filename('certbot_integration_tests', 'assets/cert.pem')
+
+    with open(os.path.join(config_path, 'sites-available', 'default-ssl.conf'), 'w') as file_h:
+        file_h.write('''\
+<IfModule mod_ssl.c>
+    <VirtualHost _default_:{https}>
+        ServerAdmin webmaster@localhost
+        ServerName {le_host}
+        DocumentRoot {webroot}
+        
+        ErrorLog ${{APACHE_LOG_DIR}}/error.log
+        CustomLog ${{APACHE_LOG_DIR}}/access.log combined
+        
+        SSLEngine on
+        SSLCertificateFile {cert_path}
+        SSLCertificateKeyFile {key_path}
+        
+        <FilesMatch "\.(cgi|shtml|phtml|php)$">
+            SSLOptions +StdEnvVars
+        </FilesMatch>
+        
+        <Directory /usr/lib/cgi-bin>
+            SSLOptions +StdEnvVars
+        </Directory>
+    </VirtualHost>
+</IfModule>
+'''.format(https=https_port, le_host=le_host, webroot=webroot_path,
+           cert_path=cert_path, key_path=key_path))
+
+    return new_environ, config_path, pid_file
+
+
+def test():
+    env = construct_apache_config_dir('/tmp/test1', 5001, 5002)
+    subprocess.call(['apache2ctl', '-DFOREGROUND'], env=env)

certbot-ci/certbot_integration_tests/apache_tests/context.py

@@ -0,0 +1,51 @@
+import errno
+import os
+import signal
+import subprocess
+
+from certbot_integration_tests.certbot_tests import context as certbot_context
+from certbot_integration_tests.apache_tests import apache_config
+from certbot_integration_tests.utils import certbot_call
+
+
+class IntegrationTestsContext(certbot_context.IntegrationTestsContext):
+    def __init__(self, request):
+        super(IntegrationTestsContext, self).__init__(request)
+
+        subprocess.check_output(['chmod', '+x', self.workspace])
+
+        self.apache_root = os.path.join(self.workspace, 'apache')
+        os.mkdir(self.apache_root)
+
+        self.env, self.config_dir, self.pid_file = apache_config.construct_apache_config_dir(
+            self.apache_root, self.http_01_port, self.tls_alpn_01_port,
+            wtf_prefix=self.worker_id)
+
+    def cleanup(self):
+        self._stop_apache()
+        super(IntegrationTestsContext, self).cleanup()
+
+    def certbot_test_apache(self, args):
+        command = ['--authenticator', 'apache', '--installer', 'apache',
+                   '--apache-server-root', self.config_dir,
+                   '--apache-challenge-location', self.apache_root]
+        command.extend(args)
+
+        return certbot_call.certbot_test(
+            command, self.directory_url, self.http_01_port, self.tls_alpn_01_port,
+            self.config_dir, self.workspace, env=self.env, force_renew=True)
+
+    def _stop_apache(self):
+        try:
+            with open(self.pid_file) as file_h:
+                pid = int(file_h.read().strip())
+        except BaseException:
+            pid = None
+
+        if pid:
+            try:
+                os.kill(pid, signal.SIGTERM)
+            except OSError as err:
+                # Ignore "No such process" error, Apache may already be stopped.
+                if err.errno != errno.ESRCH:
+                    raise

certbot-ci/certbot_integration_tests/apache_tests/test_main.py

@@ -0,0 +1,18 @@
+import pytest
+
+from certbot_integration_tests.apache_tests import context as apache_context
+
+
+@pytest.fixture()
+def context(request):
+    # Fixture request is a built-in pytest fixture describing current test request.
+    integration_test_context = apache_context.IntegrationTestsContext(request)
+    try:
+        yield integration_test_context
+    finally:
+        integration_test_context.cleanup()
+
+
+def test_it(context):
+    command = ['-d', 'apache.{0}.wtf'.format(context.worker_id)]
+    context.certbot_test_apache(command)

certbot-ci/certbot_integration_tests/assets/hook.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+import sys
+import os
+
+hook_script_type = os.path.basename(os.path.dirname(sys.argv[1]))
+if hook_script_type == 'deploy' and ('RENEWED_DOMAINS' not in os.environ or 'RENEWED_LINEAGE' not in os.environ):
+    sys.stderr.write('Environment variables not properly set!\n')
+    sys.exit(1)
+
+with open(sys.argv[2], 'a') as file_h:
+    file_h.write(hook_script_type + '\n')

certbot-ci/certbot_integration_tests/certbot_tests/assertions.py

@@ -1,6 +1,17 @@
 """This module contains advanced assertions for the certbot integration tests."""
 import os
-import grp
+try:
+    import grp
+    POSIX_MODE = True
+except ImportError:
+    import win32api
+    import win32security
+    import ntsecuritycon
+    POSIX_MODE = False
+
+EVERYBODY_SID = 'S-1-1-0'
+SYSTEM_SID = 'S-1-5-18'
+ADMINS_SID = 'S-1-5-32-544'
 
 
 def assert_hook_execution(probe_path, probe_content):
@@ -10,9 +21,10 @@ def assert_hook_execution(probe_path, probe_content):
     :param probe_content: content expected when the hook is executed
     """
     with open(probe_path, 'r') as file:
-        lines = file.readlines()
+        data = file.read()
 
-    assert '{0}{1}'.format(probe_content, os.linesep) in lines
+    lines = [line.strip() for line in data.splitlines()]
+    assert probe_content in lines
 
 
 def assert_saved_renew_hook(config_dir, lineage):
@@ -38,16 +50,51 @@ def assert_cert_count_for_lineage(config_dir, lineage, count):
     assert len(certs) == count
 
 
-def assert_equals_permissions(file1, file2, mask):
+def assert_equals_group_permissions(file1, file2):
     """
-    Assert that permissions on two files are identical in respect to a given umask.
+    Assert that two files have the same permissions for group owner.
     :param file1: first file path to compare
     :param file2: second file path to compare
-    :param mask: 3-octal representation of a POSIX umask under which the two files mode
-                 should match (eg. 0o074 will test RWX on group and R on world)
     """
-    mode_file1 = os.stat(file1).st_mode & mask
-    mode_file2 = os.stat(file2).st_mode & mask
+    # On Windows there is no group, so this assertion does nothing on this platform
+    if POSIX_MODE:
+        mode_file1 = os.stat(file1).st_mode & 0o070
+        mode_file2 = os.stat(file2).st_mode & 0o070
+
+        assert mode_file1 == mode_file2
+
+
+def assert_equals_world_read_permissions(file1, file2):
+    """
+    Assert that two files have the same read permissions for everyone.
+    :param file1: first file path to compare
+    :param file2: second file path to compare
+    """
+    if POSIX_MODE:
+        mode_file1 = os.stat(file1).st_mode & 0o004
+        mode_file2 = os.stat(file2).st_mode & 0o004
+    else:
+        everybody = win32security.ConvertStringSidToSid(EVERYBODY_SID)
+
+        security1 = win32security.GetFileSecurity(file1, win32security.DACL_SECURITY_INFORMATION)
+        dacl1 = security1.GetSecurityDescriptorDacl()
+
+        mode_file1 = dacl1.GetEffectiveRightsFromAcl({
+            'TrusteeForm': win32security.TRUSTEE_IS_SID,
+            'TrusteeType': win32security.TRUSTEE_IS_USER,
+            'Identifier': everybody,
+        })
+        mode_file1 = mode_file1 & ntsecuritycon.FILE_GENERIC_READ
+
+        security2 = win32security.GetFileSecurity(file2, win32security.DACL_SECURITY_INFORMATION)
+        dacl2 = security2.GetSecurityDescriptorDacl()
+
+        mode_file2 = dacl2.GetEffectiveRightsFromAcl({
+            'TrusteeForm': win32security.TRUSTEE_IS_SID,
+            'TrusteeType': win32security.TRUSTEE_IS_USER,
+            'Identifier': everybody,
+        })
+        mode_file2 = mode_file2 & ntsecuritycon.FILE_GENERIC_READ
 
     assert mode_file1 == mode_file2
 
@@ -57,20 +104,57 @@ def assert_equals_group_owner(file1, file2):
     Assert that two files have the same group owner.
     :param file1: first file path to compare
     :param file2: second file path to compare
-    :return:
     """
-    group_owner_file1 = grp.getgrgid(os.stat(file1).st_gid)[0]
-    group_owner_file2 = grp.getgrgid(os.stat(file2).st_gid)[0]
+    # On Windows there is no group, so this assertion does nothing on this platform
+    if POSIX_MODE:
+        group_owner_file1 = grp.getgrgid(os.stat(file1).st_gid)[0]
+        group_owner_file2 = grp.getgrgid(os.stat(file2).st_gid)[0]
 
-    assert group_owner_file1 == group_owner_file2
+        assert group_owner_file1 == group_owner_file2
 
 
-def assert_world_permissions(file, mode):
+def assert_world_no_permissions(file):
     """
-    Assert that a file has the expected world permission.
-    :param file: file path to check
-    :param mode: world permissions mode expected
+    Assert that the given file is not world-readable.
+    :param file: path of the file to check
     """
-    mode_file_all = os.stat(file).st_mode & 0o007
+    if POSIX_MODE:
+        mode_file_all = os.stat(file).st_mode & 0o007
+        assert mode_file_all == 0
+    else:
+        security = win32security.GetFileSecurity(file, win32security.DACL_SECURITY_INFORMATION)
+        dacl = security.GetSecurityDescriptorDacl()
+        mode = dacl.GetEffectiveRightsFromAcl({
+            'TrusteeForm': win32security.TRUSTEE_IS_SID,
+            'TrusteeType': win32security.TRUSTEE_IS_USER,
+            'Identifier': win32security.ConvertStringSidToSid(EVERYBODY_SID),
+        })
 
-    assert mode_file_all == mode
+        assert not mode
+
+
+def assert_world_read_permissions(file):
+    """
+    Assert that the given file is world-readable, but not world-writable or world-executable.
+    :param file: path of the file to check
+    """
+    if POSIX_MODE:
+        mode_file_all = os.stat(file).st_mode & 0o007
+        assert mode_file_all == 4
+    else:
+        security = win32security.GetFileSecurity(file, win32security.DACL_SECURITY_INFORMATION)
+        dacl = security.GetSecurityDescriptorDacl()
+        mode = dacl.GetEffectiveRightsFromAcl({
+            'TrusteeForm': win32security.TRUSTEE_IS_SID,
+            'TrusteeType': win32security.TRUSTEE_IS_USER,
+            'Identifier': win32security.ConvertStringSidToSid(EVERYBODY_SID),
+        })
+
+        assert not mode & ntsecuritycon.FILE_GENERIC_WRITE
+        assert not mode & ntsecuritycon.FILE_GENERIC_EXECUTE
+        assert mode & ntsecuritycon.FILE_GENERIC_READ == ntsecuritycon.FILE_GENERIC_READ
+
+
+def _get_current_user():
+    account_name = win32api.GetUserNameEx(win32api.NameSamCompatible)
+    return win32security.LookupAccountName(None, account_name)[0]

certbot-ci/certbot_integration_tests/certbot_tests/context.py

@@ -1,12 +1,11 @@
 """Module to handle the context of integration tests."""
+import logging
 import os
 import shutil
-import subprocess
 import sys
 import tempfile
-from distutils.version import LooseVersion
 
-from certbot_integration_tests.utils import misc
+from certbot_integration_tests.utils import certbot_call
 
 
 class IntegrationTestsContext(object):
@@ -21,7 +20,7 @@ class IntegrationTestsContext(object):
             self.worker_id = 'primary'
             acme_xdist = request.config.acme_xdist
 
-        self.acme_server =acme_xdist['acme_server']
+        self.acme_server = acme_xdist['acme_server']
         self.directory_url = acme_xdist['directory_url']
         self.tls_alpn_01_port = acme_xdist['https_port'][self.worker_id]
         self.http_01_port = acme_xdist['http_port'][self.worker_id]
@@ -30,14 +29,12 @@ class IntegrationTestsContext(object):
         # is listening on challtestsrv_port.
         self.challtestsrv_port = acme_xdist['challtestsrv_port']
 
-        # Certbot version does not depend on the test context. But getting its value requires
-        # calling certbot from a subprocess. Since it will be called a lot of times through
-        # _common_test_no_force_renew, we cache its value as a member of the fixture context.
-        self.certbot_version = misc.get_certbot_version()
-
         self.workspace = tempfile.mkdtemp()
         self.config_dir = os.path.join(self.workspace, 'conf')
-        self.hook_probe = tempfile.mkstemp(dir=self.workspace)[1]
+
+        probe = tempfile.mkstemp(dir=self.workspace)
+        os.close(probe[0])
+        self.hook_probe = probe[1]
 
         self.manual_dns_auth_hook = (
             '{0} -c "import os; import requests; import json; '
@@ -60,71 +57,18 @@ class IntegrationTestsContext(object):
         """Cleanup the integration test context."""
         shutil.rmtree(self.workspace)
 
-    def _common_test_no_force_renew(self, args):
-        """
-        Base command to execute certbot in a distributed integration test context,
-        not renewing certificates by default.
-        """
-        new_environ = os.environ.copy()
-        new_environ['TMPDIR'] = self.workspace
-
-        additional_args = []
-        if self.certbot_version >= LooseVersion('0.30.0'):
-            additional_args.append('--no-random-sleep-on-renew')
-
-        command = [
-            'certbot',
-            '--server', self.directory_url,
-            '--no-verify-ssl',
-            '--http-01-port', str(self.http_01_port),
-            '--https-port', str(self.tls_alpn_01_port),
-            '--manual-public-ip-logging-ok',
-            '--config-dir', self.config_dir,
-            '--work-dir', os.path.join(self.workspace, 'work'),
-            '--logs-dir', os.path.join(self.workspace, 'logs'),
-            '--non-interactive',
-            '--no-redirect',
-            '--agree-tos',
-            '--register-unsafely-without-email',
-            '--debug',
-            '-vv'
-        ]
-
-        command.extend(args)
-        command.extend(additional_args)
-
-        print('Invoke command:\n{0}'.format(subprocess.list2cmdline(command)))
-        return subprocess.check_output(command, universal_newlines=True,
-                                       cwd=self.workspace, env=new_environ)
-
-    def _common_test(self, args):
-        """
-        Base command to execute certbot in a distributed integration test context,
-        renewing certificates by default.
-        """
-        command = ['--renew-by-default']
-        command.extend(args)
-        return self._common_test_no_force_renew(command)
-
-    def certbot_no_force_renew(self, args):
+    def certbot(self, args, force_renew=True):
         """
         Execute certbot with given args, not renewing certificates by default.
         :param args: args to pass to certbot
+        :param force_renew: set to False to not renew by default
         :return: output of certbot execution
         """
         command = ['--authenticator', 'standalone', '--installer', 'null']
         command.extend(args)
-        return self._common_test_no_force_renew(command)
-
-    def certbot(self, args):
-        """
-        Execute certbot with given args, renewing certificates by default.
-        :param args: args to pass to certbot
-        :return: output of certbot execution
-        """
-        command = ['--renew-by-default']
-        command.extend(args)
-        return self.certbot_no_force_renew(command)
+        return certbot_call.certbot_test(
+            command, self.directory_url, self.http_01_port, self.tls_alpn_01_port,
+            self.config_dir, self.workspace, force_renew=force_renew)
 
     def get_domain(self, subdomain='le'):
         """

certbot-ci/certbot_integration_tests/certbot_tests/test_main.py

@@ -5,13 +5,17 @@ import os
 import re
 import shutil
 import subprocess
+import time
 from os.path import join, exists
 
 import pytest
 from certbot_integration_tests.certbot_tests import context as certbot_context
 from certbot_integration_tests.certbot_tests.assertions import (
-    assert_hook_execution, assert_saved_renew_hook, assert_cert_count_for_lineage,
-    assert_world_permissions, assert_equals_group_owner, assert_equals_permissions,
+    assert_hook_execution, assert_saved_renew_hook,
+    assert_cert_count_for_lineage,
+    assert_world_no_permissions, assert_world_read_permissions,
+    assert_equals_group_owner, assert_equals_group_permissions, assert_equals_world_read_permissions,
+    EVERYBODY_SID
 )
 from certbot_integration_tests.utils import misc
 
@@ -83,9 +87,9 @@ def test_http_01(context):
         context.certbot([
             '--domains', certname, '--preferred-challenges', 'http-01', 'run',
             '--cert-name', certname,
-            '--pre-hook', 'echo wtf.pre >> "{0}"'.format(context.hook_probe),
-            '--post-hook', 'echo wtf.post >> "{0}"'.format(context.hook_probe),
-            '--deploy-hook', 'echo deploy >> "{0}"'.format(context.hook_probe)
+            '--pre-hook', misc.echo('wtf_pre', context.hook_probe),
+            '--post-hook', misc.echo('wtf_post', context.hook_probe),
+            '--deploy-hook', misc.echo('deploy', context.hook_probe),
         ])
 
     assert_hook_execution(context.hook_probe, 'deploy')
@@ -103,9 +107,9 @@ def test_manual_http_auth(context):
             '--cert-name', certname,
             '--manual-auth-hook', scripts[0],
             '--manual-cleanup-hook', scripts[1],
-            '--pre-hook', 'echo wtf.pre >> "{0}"'.format(context.hook_probe),
-            '--post-hook', 'echo wtf.post >> "{0}"'.format(context.hook_probe),
-            '--renew-hook', 'echo renew >> "{0}"'.format(context.hook_probe)
+            '--pre-hook', misc.echo('wtf_pre', context.hook_probe),
+            '--post-hook', misc.echo('wtf_post', context.hook_probe),
+            '--renew-hook', misc.echo('renew', context.hook_probe),
         ])
 
     with pytest.raises(AssertionError):
@@ -121,9 +125,9 @@ def test_manual_dns_auth(context):
         'run', '--cert-name', certname,
         '--manual-auth-hook', context.manual_dns_auth_hook,
         '--manual-cleanup-hook', context.manual_dns_cleanup_hook,
-        '--pre-hook', 'echo wtf.pre >> "{0}"'.format(context.hook_probe),
-        '--post-hook', 'echo wtf.post >> "{0}"'.format(context.hook_probe),
-        '--renew-hook', 'echo renew >> "{0}"'.format(context.hook_probe)
+        '--pre-hook', misc.echo('wtf_pre', context.hook_probe),
+        '--post-hook', misc.echo('wtf_post', context.hook_probe),
+        '--renew-hook', misc.echo('renew', context.hook_probe),
     ])
 
     with pytest.raises(AssertionError):
@@ -172,21 +176,19 @@ def test_renew_files_permissions(context):
     certname = context.get_domain('renew')
     context.certbot(['-d', certname])
 
+    privkey1 = join(context.config_dir, 'archive', certname, 'privkey1.pem')
+    privkey2 = join(context.config_dir, 'archive', certname, 'privkey2.pem')
+
     assert_cert_count_for_lineage(context.config_dir, certname, 1)
-    assert_world_permissions(
-        join(context.config_dir, 'archive', certname, 'privkey1.pem'), 0)
+    assert_world_no_permissions(privkey1)
 
     context.certbot(['renew'])
 
     assert_cert_count_for_lineage(context.config_dir, certname, 2)
-    assert_world_permissions(
-        join(context.config_dir, 'archive', certname, 'privkey2.pem'), 0)
-    assert_equals_group_owner(
-        join(context.config_dir, 'archive', certname, 'privkey1.pem'),
-        join(context.config_dir, 'archive', certname, 'privkey2.pem'))
-    assert_equals_permissions(
-        join(context.config_dir, 'archive', certname, 'privkey1.pem'),
-        join(context.config_dir, 'archive', certname, 'privkey2.pem'), 0o074)
+    assert_world_no_permissions(privkey2)
+    assert_equals_group_owner(privkey1, privkey2)
+    assert_equals_world_read_permissions(privkey1, privkey2)
+    assert_equals_group_permissions(privkey1, privkey2)
 
 
 def test_renew_with_hook_scripts(context):
@@ -210,15 +212,35 @@ def test_renew_files_propagate_permissions(context):
 
     assert_cert_count_for_lineage(context.config_dir, certname, 1)
 
-    os.chmod(join(context.config_dir, 'archive', certname, 'privkey1.pem'), 0o444)
+    privkey1 = join(context.config_dir, 'archive', certname, 'privkey1.pem')
+    privkey2 = join(context.config_dir, 'archive', certname, 'privkey2.pem')
+
+    if os.name != 'nt':
+        os.chmod(privkey1, 0o444)
+    else:
+        import win32security
+        import ntsecuritycon
+        # Get the current DACL of the private key
+        security = win32security.GetFileSecurity(privkey1, win32security.DACL_SECURITY_INFORMATION)
+        dacl = security.GetSecurityDescriptorDacl()
+        # Create a read permission for Everybody group
+        everybody = win32security.ConvertStringSidToSid(EVERYBODY_SID)
+        dacl.AddAccessAllowedAce(win32security.ACL_REVISION, ntsecuritycon.FILE_GENERIC_READ, everybody)
+        # Apply the updated DACL to the private key
+        security.SetSecurityDescriptorDacl(1, dacl, 0)
+        win32security.SetFileSecurity(privkey1, win32security.DACL_SECURITY_INFORMATION, security)
+
     context.certbot(['renew'])
 
     assert_cert_count_for_lineage(context.config_dir, certname, 2)
-    assert_world_permissions(
-        join(context.config_dir, 'archive', certname, 'privkey2.pem'), 4)
-    assert_equals_permissions(
-        join(context.config_dir, 'archive', certname, 'privkey1.pem'),
-        join(context.config_dir, 'archive', certname, 'privkey2.pem'), 0o074)
+    if os.name != 'nt':
+        # On Linux, read world permissions + all group permissions will be copied from the previous private key
+        assert_world_read_permissions(privkey2)
+        assert_equals_world_read_permissions(privkey1, privkey2)
+        assert_equals_group_permissions(privkey1, privkey2)
+    else:
+        # On Windows, world will never have any permissions, and group permission is irrelevant for this platform
+        assert_world_no_permissions(privkey2)
 
 
 def test_graceful_renew_it_is_not_time(context):
@@ -228,8 +250,8 @@ def test_graceful_renew_it_is_not_time(context):
 
     assert_cert_count_for_lineage(context.config_dir, certname, 1)
 
-    context.certbot_no_force_renew([
-        'renew', '--deploy-hook', 'echo deploy >> "{0}"'.format(context.hook_probe)])
+    context.certbot(['renew', '--deploy-hook', misc.echo('deploy', context.hook_probe)],
+                    force_renew=False)
 
     assert_cert_count_for_lineage(context.config_dir, certname, 1)
     with pytest.raises(AssertionError):
@@ -249,8 +271,8 @@ def test_graceful_renew_it_is_time(context):
     with open(join(context.config_dir, 'renewal', '{0}.conf'.format(certname)), 'w') as file:
         file.writelines(lines)
 
-    context.certbot_no_force_renew([
-        'renew', '--deploy-hook', 'echo deploy >> "{0}"'.format(context.hook_probe)])
+    context.certbot(['renew', '--deploy-hook', misc.echo('deploy', context.hook_probe)],
+                    force_renew=False)
 
     assert_cert_count_for_lineage(context.config_dir, certname, 2)
     assert_hook_execution(context.hook_probe, 'deploy')
@@ -316,9 +338,9 @@ def test_renew_hook_override(context):
     context.certbot([
         'certonly', '-d', certname,
         '--preferred-challenges', 'http-01',
-        '--pre-hook', 'echo pre >> "{0}"'.format(context.hook_probe),
-        '--post-hook', 'echo post >> "{0}"'.format(context.hook_probe),
-        '--deploy-hook', 'echo deploy >> "{0}"'.format(context.hook_probe)
+        '--pre-hook', misc.echo('pre', context.hook_probe),
+        '--post-hook', misc.echo('post', context.hook_probe),
+        '--deploy-hook', misc.echo('deploy', context.hook_probe),
     ])
 
     assert_hook_execution(context.hook_probe, 'pre')
@@ -329,14 +351,14 @@ def test_renew_hook_override(context):
     open(context.hook_probe, 'w').close()
     context.certbot([
         'renew', '--cert-name', certname,
-        '--pre-hook', 'echo pre-override >> "{0}"'.format(context.hook_probe),
-        '--post-hook', 'echo post-override >> "{0}"'.format(context.hook_probe),
-        '--deploy-hook', 'echo deploy-override >> "{0}"'.format(context.hook_probe)
+        '--pre-hook', misc.echo('pre_override', context.hook_probe),
+        '--post-hook', misc.echo('post_override', context.hook_probe),
+        '--deploy-hook', misc.echo('deploy_override', context.hook_probe),
     ])
 
-    assert_hook_execution(context.hook_probe, 'pre-override')
-    assert_hook_execution(context.hook_probe, 'post-override')
-    assert_hook_execution(context.hook_probe, 'deploy-override')
+    assert_hook_execution(context.hook_probe, 'pre_override')
+    assert_hook_execution(context.hook_probe, 'post_override')
+    assert_hook_execution(context.hook_probe, 'deploy_override')
     with pytest.raises(AssertionError):
         assert_hook_execution(context.hook_probe, 'pre')
     with pytest.raises(AssertionError):
@@ -348,11 +370,11 @@ def test_renew_hook_override(context):
     open(context.hook_probe, 'w').close()
     context.certbot(['renew', '--cert-name', certname])
 
-    assert_hook_execution(context.hook_probe, 'pre-override')
-    assert_hook_execution(context.hook_probe, 'post-override')
-    assert_hook_execution(context.hook_probe, 'deploy-override')
+    assert_hook_execution(context.hook_probe, 'pre_override')
+    assert_hook_execution(context.hook_probe, 'post_override')
+    assert_hook_execution(context.hook_probe, 'deploy_override')
+
 
-    
 def test_invalid_domain_with_dns_challenge(context):
     """Test certificate issuance failure with DNS-01 challenge."""
     # Manual dns auth hooks from misc are designed to fail if the domain contains 'fail-*'.
@@ -511,7 +533,7 @@ def test_revoke_multiple_lineages(context):
         data = file.read()
 
     data = re.sub('archive_dir = .*\n',
-                  'archive_dir = {0}\n'.format(join(context.config_dir, 'archive', cert1)),
+                  'archive_dir = {0}\n'.format(join(context.config_dir, 'archive', cert1).replace('\\', '\\\\')),
                   data)
 
     with open(join(context.config_dir, 'renewal', '{0}.conf'.format(cert2)), 'w') as file:
@@ -554,11 +576,9 @@ def test_ocsp_status_stale(context):
 
 def test_ocsp_status_live(context):
     """Test retrieval of OCSP statuses for live config"""
-    if context.acme_server == 'pebble':
-        pytest.skip('Pebble does not support OCSP status requests.')
+    cert = context.get_domain('ocsp-check')
 
     # OSCP 1: Check live certificate OCSP status (VALID)
-    cert = context.get_domain('ocsp-check')
     context.certbot(['--domains', cert])
     output = context.certbot(['certificates'])
 
@@ -567,6 +587,9 @@ def test_ocsp_status_live(context):
 
     # OSCP 2: Check live certificate OCSP status (REVOKED)
     context.certbot(['revoke', '--cert-name', cert, '--no-delete-after-revoke'])
+    # Sometimes in oldest tests (using openssl binary and not cryptography), the OCSP status is
+    # not seen immediately by Certbot as invalid. Waiting few seconds solves this transient issue.
+    time.sleep(5)
     output = context.certbot(['certificates'])
 
     assert output.count('INVALID') == 1, 'Expected {0} to be INVALID'.format(cert)

certbot-ci/certbot_integration_tests/conftest.py

@@ -68,17 +68,18 @@ def _setup_primary_node(config):
     :param config: Configuration of the pytest primary node
     """
     # Check for runtime compatibility: some tools are required to be available in PATH
-    try:
-        subprocess.check_output(['docker', '-v'], stderr=subprocess.STDOUT)
-    except (subprocess.CalledProcessError, OSError):
-        raise ValueError('Error: docker is required in PATH to launch the integration tests, '
-                         'but is not installed or not available for current user.')
+    if 'boulder' in config.option.acme_server:
+        try:
+            subprocess.check_output(['docker', '-v'], stderr=subprocess.STDOUT)
+        except (subprocess.CalledProcessError, OSError):
+            raise ValueError('Error: docker is required in PATH to launch the integration tests on'
+                             'boulder, but is not installed or not available for current user.')
 
-    try:
-        subprocess.check_output(['docker-compose', '-v'], stderr=subprocess.STDOUT)
-    except (subprocess.CalledProcessError, OSError):
-        raise ValueError('Error: docker-compose is required in PATH to launch the integration tests, '
-                         'but is not installed or not available for current user.')
+        try:
+            subprocess.check_output(['docker-compose', '-v'], stderr=subprocess.STDOUT)
+        except (subprocess.CalledProcessError, OSError):
+            raise ValueError('Error: docker-compose is required in PATH to launch the integration tests, '
+                             'but is not installed or not available for current user.')
 
     # Parameter numprocesses is added to option by pytest-xdist
     workers = ['primary'] if not config.option.numprocesses\
@@ -86,7 +87,9 @@ def _setup_primary_node(config):
 
     # By calling setup_acme_server we ensure that all necessary acme server instances will be
     # fully started. This runtime is reflected by the acme_xdist returned.
-    acme_xdist = acme_lib.setup_acme_server(config.option.acme_server, workers)
-    print('ACME xdist config:\n{0}'.format(acme_xdist))
+    acme_server = acme_lib.ACMEServer(config.option.acme_server, workers)
+    config.add_cleanup(acme_server.stop)
+    print('ACME xdist config:\n{0}'.format(acme_server.acme_xdist))
+    acme_server.start()
 
-    return acme_xdist
+    return acme_server.acme_xdist

certbot-ci/certbot_integration_tests/nginx_tests/context.py

@@ -2,7 +2,7 @@ import os
 import subprocess
 
 from certbot_integration_tests.certbot_tests import context as certbot_context
-from certbot_integration_tests.utils import misc
+from certbot_integration_tests.utils import misc, certbot_call
 from certbot_integration_tests.nginx_tests import nginx_config as config
 
 
@@ -33,11 +33,14 @@ class IntegrationTestsContext(certbot_context.IntegrationTestsContext):
         """
         Main command to execute certbot using the nginx plugin.
         :param list args: list of arguments to pass to nginx
+        :param bool force_renew: set to False to not renew by default
         """
         command = ['--authenticator', 'nginx', '--installer', 'nginx',
                    '--nginx-server-root', self.nginx_root]
         command.extend(args)
-        return self._common_test(command)
+        return certbot_call.certbot_test(
+            command, self.directory_url, self.http_01_port, self.tls_alpn_01_port,
+            self.config_dir, self.workspace, force_renew=True)
 
     def _start_nginx(self, default_server):
         self.nginx_config = config.construct_nginx_config(

certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py

@@ -21,9 +21,9 @@ def construct_nginx_config(nginx_root, nginx_webroot, http_port, https_port, oth
     :rtype: str
     """
     key_path = key_path if key_path \
-        else pkg_resources.resource_filename('certbot_integration_tests', 'assets/nginx_key.pem')
+        else pkg_resources.resource_filename('certbot_integration_tests', 'assets/key.pem')
     cert_path = cert_path if cert_path \
-        else pkg_resources.resource_filename('certbot_integration_tests', 'assets/nginx_cert.pem')
+        else pkg_resources.resource_filename('certbot_integration_tests', 'assets/cert.pem')
     return '''\
 # This error log will be written regardless of server scope error_log
 # definitions, so we have to set this here in the main scope.

certbot-ci/certbot_integration_tests/nginx_tests/test_main.py

@@ -30,6 +30,7 @@ def context(request):
     ('nginx6.{0}.wtf,nginx7.{0}.wtf', ['--preferred-challenges', 'http'], {'default_server': False}),
 ], indirect=['context'])
 def test_certificate_deployment(certname_pattern, params, context):
+    # type: (str, list, nginx_context.IntegrationTestsContext) -> None
     """
     Test various scenarios to deploy a certificate to nginx using certbot.
     """
@@ -45,10 +46,7 @@ def test_certificate_deployment(certname_pattern, params, context):
 
     assert server_cert == certbot_cert
 
-    command = ['--authenticator', 'nginx', '--installer', 'nginx',
-               '--nginx-server-root', context.nginx_root,
-               'rollback', '--checkpoints', '1']
-    context._common_test_no_force_renew(command)
+    context.certbot_test_nginx(['rollback', '--checkpoints', '1'])
 
     with open(context.nginx_config_path, 'r') as file_h:
         current_nginx_config = file_h.read()

certbot-ci/certbot_integration_tests/utils/acme_server.py

@@ -1,7 +1,10 @@
+#!/usr/bin/env python
 """Module to setup an ACME CA server environment able to run multiple tests in parallel"""
 from __future__ import print_function
+import errno
+import json
 import tempfile
-import atexit
+import time
 import os
 import subprocess
 import shutil
@@ -9,188 +12,208 @@ import sys
 from os.path import join
 
 import requests
-import json
-import yaml
 
-from certbot_integration_tests.utils import misc
-
-# These ports are set implicitly in the docker-compose.yml files of Boulder/Pebble.
-CHALLTESTSRV_PORT = 8055
-HTTP_01_PORT = 5002
+from certbot_integration_tests.utils import misc, proxy, pebble_artifacts
+from certbot_integration_tests.utils.constants import *
 
 
-def setup_acme_server(acme_server, nodes):
+class ACMEServer(object):
     """
-    This method will setup an ACME CA server and an HTTP reverse proxy instance, to allow parallel
-    execution of integration tests against the unique http-01 port expected by the ACME CA server.
-    Instances are properly closed and cleaned when the Python process exits using atexit.
+    ACMEServer configures and handles the lifecycle of an ACME CA server and an HTTP reverse proxy
+    instance, to allow parallel execution of integration tests against the unique http-01 port
+    expected by the ACME CA server.
     Typically all pytest integration tests will be executed in this context.
-    This method returns an object describing ports and directory url to use for each pytest node
-    with the relevant pytest xdist node.
-    :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble)
-    :param str[] nodes: list of node names that will be setup by pytest xdist
-    :return: a dict describing the challenge ports that have been setup for the nodes
-    :rtype: dict
+    ACMEServer gives access the acme_xdist parameter, listing the ports and directory url to use
+    for each pytest node. It exposes also start and stop methods in order to start the stack, and
+    stop it with proper resources cleanup.
+    ACMEServer is also a context manager, and so can be used to ensure ACME server is started/stopped
+    upon context enter/exit.
     """
-    acme_type = 'pebble' if acme_server == 'pebble' else 'boulder'
-    acme_xdist = _construct_acme_xdist(acme_server, nodes)
-    workspace = _construct_workspace(acme_type)
+    def __init__(self, acme_server, nodes, http_proxy=True, stdout=False):
+        """
+        Create an ACMEServer instance.
+        :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble)
+        :param list nodes: list of node names that will be setup by pytest xdist
+        :param bool http_proxy: if False do not start the HTTP proxy
+        :param bool stdout: if True stream subprocesses stdout to standard stdout
+        """
+        self._construct_acme_xdist(acme_server, nodes)
 
-    _prepare_traefik_proxy(workspace, acme_xdist)
-    _prepare_acme_server(workspace, acme_type, acme_xdist)
+        self._acme_type = 'pebble' if acme_server == 'pebble' else 'boulder'
+        self._proxy = http_proxy
+        self._workspace = tempfile.mkdtemp()
+        self._processes = []
+        self._stdout = sys.stdout if stdout else open(os.devnull, 'w')
 
-    return acme_xdist
+    def start(self):
+        """Start the test stack"""
+        try:
+            if self._proxy:
+                self._prepare_http_proxy()
+            if self._acme_type == 'pebble':
+                self._prepare_pebble_server()
+            if self._acme_type == 'boulder':
+                self._prepare_boulder_server()
+        except BaseException as e:
+            self.stop()
+            raise e
 
+    def stop(self):
+        """Stop the test stack, and clean its resources"""
+        print('=> Tear down the test infrastructure...')
+        try:
+            for process in self._processes:
+                try:
+                    process.terminate()
+                except OSError as e:
+                    # Process may be not started yet, so no PID and terminate fails.
+                    # Then the process never started, and the situation is acceptable.
+                    if e.errno != errno.ESRCH:
+                        raise
+            for process in self._processes:
+                process.wait()
 
-def _construct_acme_xdist(acme_server, nodes):
-    """Generate and return the acme_xdist dict"""
-    acme_xdist = {'acme_server': acme_server, 'challtestsrv_port': CHALLTESTSRV_PORT}
+            if os.path.exists(os.path.join(self._workspace, 'boulder')):
+                # Boulder docker generates build artifacts owned by root with 0o744 permissions.
+                # If we started the acme server from a normal user that has access to the Docker
+                # daemon, this user will not be able to delete these artifacts from the host.
+                # We need to do it through a docker.
+                process = self._launch_process(['docker', 'run', '--rm', '-v',
+                                                '{0}:/workspace'.format(self._workspace),
+                                                'alpine', 'rm', '-rf', '/workspace/boulder'])
+                process.wait()
+        finally:
+            shutil.rmtree(self._workspace)
+        if self._stdout != sys.stdout:
+            self._stdout.close()
+        print('=> Test infrastructure stopped and cleaned up.')
 
-    # Directory and ACME port are set implicitly in the docker-compose.yml files of Boulder/Pebble.
-    if acme_server == 'pebble':
-        acme_xdist['directory_url'] = 'https://localhost:14000/dir'
-    else:  # boulder
-        port = 4001 if acme_server == 'boulder-v2' else 4000
-        acme_xdist['directory_url'] = 'http://localhost:{0}/directory'.format(port)
+    def __enter__(self):
+        self.start()
+        return self.acme_xdist
 
-    acme_xdist['http_port'] = {node: port for (node, port)
-                               in zip(nodes, range(5200, 5200 + len(nodes)))}
-    acme_xdist['https_port'] = {node: port for (node, port)
-                                in zip(nodes, range(5100, 5100 + len(nodes)))}
-    acme_xdist['other_port'] = {node: port for (node, port)
-                                in zip(nodes, range(5300, 5300 + len(nodes)))}
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.stop()
 
-    return acme_xdist
+    def _construct_acme_xdist(self, acme_server, nodes):
+        """Generate and return the acme_xdist dict"""
+        acme_xdist = {'acme_server': acme_server, 'challtestsrv_port': CHALLTESTSRV_PORT}
 
+        # Directory and ACME port are set implicitly in the docker-compose.yml files of Boulder/Pebble.
+        if acme_server == 'pebble':
+            acme_xdist['directory_url'] = PEBBLE_DIRECTORY_URL
+        else:  # boulder
+            acme_xdist['directory_url'] = BOULDER_V2_DIRECTORY_URL \
+                if acme_server == 'boulder-v2' else BOULDER_V1_DIRECTORY_URL
 
-def _construct_workspace(acme_type):
-    """Create a temporary workspace for integration tests stack"""
-    workspace = tempfile.mkdtemp()
+        acme_xdist['http_port'] = {node: port for (node, port)
+                                   in zip(nodes, range(5200, 5200 + len(nodes)))}
+        acme_xdist['https_port'] = {node: port for (node, port)
+                                    in zip(nodes, range(5100, 5100 + len(nodes)))}
+        acme_xdist['other_port'] = {node: port for (node, port)
+                                    in zip(nodes, range(5300, 5300 + len(nodes)))}
 
-    def cleanup():
-        """Cleanup function to call that will teardown relevant dockers and their configuration."""
-        for instance in [acme_type, 'traefik']:
-            print('=> Tear down the {0} instance...'.format(instance))
-            instance_path = join(workspace, instance)
-            try:
-                if os.path.isfile(join(instance_path, 'docker-compose.yml')):
-                    _launch_command(['docker-compose', 'down'], cwd=instance_path)
-            except subprocess.CalledProcessError:
-                pass
-            print('=> Finished tear down of {0} instance.'.format(acme_type))
+        self.acme_xdist = acme_xdist
 
-        shutil.rmtree(workspace)
+    def _prepare_pebble_server(self):
+        """Configure and launch the Pebble server"""
+        print('=> Starting pebble instance deployment...')
+        pebble_path, challtestsrv_path, pebble_config_path = pebble_artifacts.fetch(self._workspace)
 
-    # Here with atexit we ensure that clean function is called no matter what.
-    atexit.register(cleanup)
+        # Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid
+        # nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment.
+        environ = os.environ.copy()
+        environ['PEBBLE_VA_NOSLEEP'] = '1'
+        environ['PEBBLE_WFE_NONCEREJECT'] = '0'
 
-    return workspace
+        self._launch_process(
+            [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053'],
+            env=environ)
 
+        self._launch_process(
+            [challtestsrv_path, '-management', ':{0}'.format(CHALLTESTSRV_PORT), '-defaultIPv6', '""',
+             '-defaultIPv4', '127.0.0.1', '-http01', '""', '-tlsalpn01', '""', '-https01', '""'])
 
-def _prepare_acme_server(workspace, acme_type, acme_xdist):
-    """Configure and launch the ACME server, Boulder or Pebble"""
-    print('=> Starting {0} instance deployment...'.format(acme_type))
-    instance_path = join(workspace, acme_type)
-    try:
-        # Load Boulder/Pebble from git, that includes a docker-compose.yml ready for production.
-        _launch_command(['git', 'clone', 'https://github.com/letsencrypt/{0}'.format(acme_type),
-                         '--single-branch', '--depth=1', instance_path])
-        if acme_type == 'boulder':
-            # Allow Boulder to ignore usual limit rate policies, useful for tests.
-            os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'),
-                      join(instance_path, 'test/rate-limit-policies.yml'))
-        if acme_type == 'pebble':
-            # Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid
-            # nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment.
-            with open(os.path.join(instance_path, 'docker-compose.yml'), 'r') as file_handler:
-                config = yaml.load(file_handler.read())
-
-            config['services']['pebble'].setdefault('environment', [])\
-                .extend(['PEBBLE_VA_NOSLEEP=1', 'PEBBLE_WFE_NONCEREJECT=0'])
-            with open(os.path.join(instance_path, 'docker-compose.yml'), 'w') as file_handler:
-                file_handler.write(yaml.dump(config))
-
-        # Launch the ACME CA server.
-        _launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path)
+        # pebble_ocsp_server is imported here and not at the top of module in order to avoid a useless
+        # ImportError, in the case where cryptography dependency is too old to support ocsp, but
+        # Boulder is used instead of Pebble, so pebble_ocsp_server is not used. This is the typical
+        # situation of integration-certbot-oldest tox testenv.
+        from certbot_integration_tests.utils import pebble_ocsp_server
+        self._launch_process([sys.executable, pebble_ocsp_server.__file__])
 
         # Wait for the ACME CA server to be up.
-        print('=> Waiting for {0} instance to respond...'.format(acme_type))
-        misc.check_until_timeout(acme_xdist['directory_url'])
+        print('=> Waiting for pebble instance to respond...')
+        misc.check_until_timeout(self.acme_xdist['directory_url'])
+
+        print('=> Finished pebble instance deployment.')
+
+    def _prepare_boulder_server(self):
+        """Configure and launch the Boulder server"""
+        print('=> Starting boulder instance deployment...')
+        instance_path = join(self._workspace, 'boulder')
+
+        # Load Boulder from git, that includes a docker-compose.yml ready for production.
+        process = self._launch_process(['git', 'clone', 'https://github.com/letsencrypt/boulder',
+                                        '--single-branch', '--depth=1', instance_path])
+        process.wait()
+
+        # Allow Boulder to ignore usual limit rate policies, useful for tests.
+        os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'),
+                  join(instance_path, 'test/rate-limit-policies.yml'))
+
+        # Launch the Boulder server
+        self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path)
+
+        # Wait for the ACME CA server to be up.
+        print('=> Waiting for boulder instance to respond...')
+        misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=240)
 
         # Configure challtestsrv to answer any A record request with ip of the docker host.
-        acme_subnet = '10.77.77' if acme_type == 'boulder' else '10.30.50'
-        response = requests.post('http://localhost:{0}/set-default-ipv4'
-                                 .format(acme_xdist['challtestsrv_port']),
-                                 json={'ip': '{0}.1'.format(acme_subnet)})
+        response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT),
+                                 json={'ip': '10.77.77.1'})
         response.raise_for_status()
 
-        print('=> Finished {0} instance deployment.'.format(acme_type))
-    except BaseException:
-        print('Error while setting up {0} instance.'.format(acme_type))
-        raise
+        print('=> Finished boulder instance deployment.')
+
+    def _prepare_http_proxy(self):
+        """Configure and launch an HTTP proxy"""
+        print('=> Configuring the HTTP proxy...')
+        mapping = {r'.+\.{0}\.wtf'.format(node): 'http://127.0.0.1:{0}'.format(port)
+                   for node, port in self.acme_xdist['http_port'].items()}
+        command = [sys.executable, proxy.__file__, str(HTTP_01_PORT), json.dumps(mapping)]
+        self._launch_process(command)
+        print('=> Finished configuring the HTTP proxy.')
+
+    def _launch_process(self, command, cwd=os.getcwd(), env=None):
+        """Launch silently an subprocess OS command"""
+        if not env:
+            env = os.environ
+        process = subprocess.Popen(command, stdout=self._stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env)
+        self._processes.append(process)
+        return process
 
 
-def _prepare_traefik_proxy(workspace, acme_xdist):
-    """Configure and launch Traefik, the HTTP reverse proxy"""
-    print('=> Starting traefik instance deployment...')
-    instance_path = join(workspace, 'traefik')
-    traefik_subnet = '10.33.33'
-    traefik_api_port = 8056
+def main():
+    args = sys.argv[1:]
+    server_type = args[0] if args else 'pebble'
+    possible_values = ('pebble', 'boulder-v1', 'boulder-v2')
+    if server_type not in possible_values:
+        raise ValueError('Invalid server value {0}, should be one of {1}'
+                         .format(server_type, possible_values))
+
+    acme_server = ACMEServer(server_type, [], http_proxy=False, stdout=True)
+
     try:
-        os.mkdir(instance_path)
+        with acme_server as acme_xdist:
+            print('--> Instance of {0} is running, directory URL is {0}'
+                  .format(acme_xdist['directory_url']))
+            print('--> Press CTRL+C to stop the ACME server.')
 
-        with open(join(instance_path, 'docker-compose.yml'), 'w') as file_h:
-            file_h.write('''\
-version: '3'
-services:
-  traefik:
-    image: traefik
-    command: --api --rest
-    ports:
-      - {http_01_port}:80
-      - {traefik_api_port}:8080
-    networks:
-      traefiknet:
-        ipv4_address: {traefik_subnet}.2
-networks:
-  traefiknet:
-    ipam:
-      config:
-        - subnet: {traefik_subnet}.0/24
-'''.format(traefik_subnet=traefik_subnet,
-           traefik_api_port=traefik_api_port,
-           http_01_port=HTTP_01_PORT))
-
-        _launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path)
-
-        misc.check_until_timeout('http://localhost:{0}/api'.format(traefik_api_port))
-        config = {
-            'backends': {
-                node: {
-                    'servers': {node: {'url': 'http://{0}.1:{1}'.format(traefik_subnet, port)}}
-                } for node, port in acme_xdist['http_port'].items()
-            },
-            'frontends': {
-                node: {
-                    'backend': node, 'passHostHeader': True,
-                    'routes': {node: {'rule': 'HostRegexp: {{subdomain:.+}}.{0}.wtf'.format(node)}}
-                } for node in acme_xdist['http_port'].keys()
-            }
-        }
-        response = requests.put('http://localhost:{0}/api/providers/rest'.format(traefik_api_port),
-                                data=json.dumps(config))
-        response.raise_for_status()
-
-        print('=> Finished traefik instance deployment.')
-    except BaseException:
-        print('Error while setting up traefik instance.')
-        raise
+            while True:
+                time.sleep(3600)
+    except KeyboardInterrupt:
+        pass
 
 
-def _launch_command(command, cwd=os.getcwd()):
-    """Launch silently an OS command, output will be displayed in case of failure"""
-    try:
-        subprocess.check_output(command, stderr=subprocess.STDOUT, cwd=cwd, universal_newlines=True)
-    except subprocess.CalledProcessError as e:
-        sys.stderr.write(e.output)
-        raise
+if __name__ == '__main__':
+    main()

certbot-ci/certbot_integration_tests/utils/certbot_call.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python
+"""Module to call certbot in test mode"""
+from __future__ import absolute_import
+from distutils.version import LooseVersion
+import subprocess
+import sys
+import os
+
+import certbot_integration_tests
+from certbot_integration_tests.utils.constants import *
+
+
+def certbot_test(certbot_args, directory_url, http_01_port, tls_alpn_01_port,
+                 config_dir, workspace, env=None, force_renew=True):
+    """
+    Invoke the certbot executable available in PATH in a test context for the given args.
+    The test context consists in running certbot in debug mode, with various flags suitable
+    for tests (eg. no ssl check, customizable ACME challenge ports and config directory ...).
+    This command captures stdout and returns it to the caller.
+    :param list certbot_args: the arguments to pass to the certbot executable
+    :param str directory_url: URL of the ACME directory server to use
+    :param int http_01_port: port for the HTTP-01 challenges
+    :param int tls_alpn_01_port: port for the TLS-ALPN-01 challenges
+    :param str config_dir: certbot configuration directory to use
+    :param str workspace: certbot current directory to use
+    :param obj env: the environment variables to use (default: None, then current env will be used)
+    :param bool force_renew: set False to not force renew existing certificates (default: True)
+    :return: stdout as string
+    :rtype: str
+    """
+    new_environ = env if env else os.environ.copy()
+    command, env = _prepare_args_env(certbot_args, directory_url, http_01_port, tls_alpn_01_port,
+                                     config_dir, workspace, force_renew, new_environ)
+
+    return subprocess.check_output(command, universal_newlines=True, cwd=workspace, env=env)
+
+
+def _prepare_environ(workspace, new_environ):
+    new_environ = new_environ.copy()
+    new_environ['TMPDIR'] = workspace
+
+    # So, pytest is nice, and a little too nice for our usage.
+    # In order to help user to call seamlessly any piece of python code without requiring to
+    # install it as a full-fledged setuptools distribution for instance, it may inject the path
+    # to the test files into the PYTHONPATH. This allows the python interpreter to import
+    # as modules any python file available at this path.
+    # See https://docs.pytest.org/en/3.2.5/pythonpath.html for the explanation and description.
+    # However this behavior is not good in integration tests, in particular the nginx oldest ones.
+    # Indeed during these kind of tests certbot is installed as a transitive dependency to
+    # certbot-nginx. Here is the trick: this certbot version is not necessarily the same as
+    # the certbot codebase lying in current working directory. For instance in oldest tests
+    # certbot==0.36.0 may be installed while the codebase corresponds to certbot==0.37.0.dev0.
+    # Then during a pytest run, PYTHONPATH contains the path to the Certbot codebase, so invoking
+    # certbot will import the modules from the codebase (0.37.0.dev0), not from the
+    # required/installed version (0.36.0).
+    # This will lead to funny and totally incomprehensible errors. To avoid that, we ensure that
+    # if PYTHONPATH is set, it does not contain the path to the root of the codebase.
+    if new_environ.get('PYTHONPATH'):
+        # certbot_integration_tests.__file__ is:
+        # '/path/to/certbot/certbot-ci/certbot_integration_tests/__init__.pyc'
+        # ... and we want '/path/to/certbot'
+        certbot_root = os.path.dirname(os.path.dirname(os.path.dirname(certbot_integration_tests.__file__)))
+        python_paths = [path for path in new_environ['PYTHONPATH'].split(':') if path != certbot_root]
+        new_environ['PYTHONPATH'] = ':'.join(python_paths)
+
+    return new_environ
+
+
+def _compute_additional_args(workspace, environ, force_renew):
+    additional_args = []
+    output = subprocess.check_output(['certbot', '--version'],
+                                     universal_newlines=True, stderr=subprocess.STDOUT,
+                                     cwd=workspace, env=environ)
+    version_str = output.split(' ')[1].strip()  # Typical response is: output = 'certbot 0.31.0.dev0'
+    if LooseVersion(version_str) >= LooseVersion('0.30.0'):
+        additional_args.append('--no-random-sleep-on-renew')
+
+    if force_renew:
+        additional_args.append('--renew-by-default')
+
+    return additional_args
+
+
+def _prepare_args_env(certbot_args, directory_url, http_01_port, tls_alpn_01_port,
+                      config_dir, workspace, force_renew, new_environ):
+
+    new_environ = _prepare_environ(workspace, new_environ)
+    additional_args = _compute_additional_args(workspace, new_environ, force_renew)
+
+    command = [
+        'certbot',
+        '--server', directory_url,
+        '--no-verify-ssl',
+        '--http-01-port', str(http_01_port),
+        '--https-port', str(tls_alpn_01_port),
+        '--manual-public-ip-logging-ok',
+        '--config-dir', config_dir,
+        '--work-dir', os.path.join(workspace, 'work'),
+        '--logs-dir', os.path.join(workspace, 'logs'),
+        '--non-interactive',
+        '--no-redirect',
+        '--agree-tos',
+        '--register-unsafely-without-email',
+        '--debug',
+        '-vv'
+    ]
+
+    command.extend(certbot_args)
+    command.extend(additional_args)
+
+    print('--> Invoke command:\n=====\n{0}\n====='.format(subprocess.list2cmdline(command)))
+
+    return command, new_environ
+
+
+def main():
+    args = sys.argv[1:]
+
+    # Default config is pebble
+    directory_url = os.environ.get('SERVER', PEBBLE_DIRECTORY_URL)
+    http_01_port = int(os.environ.get('HTTP_01_PORT', HTTP_01_PORT))
+    tls_alpn_01_port = int(os.environ.get('TLS_ALPN_01_PORT', TLS_ALPN_01_PORT))
+
+    # Execution of certbot in a self-contained workspace
+    workspace = os.environ.get('WORKSPACE', os.path.join(os.getcwd(), '.certbot_test_workspace'))
+    if not os.path.exists(workspace):
+        print('--> Creating a workspace for certbot_test: {0}'.format(workspace))
+        os.mkdir(workspace)
+    else:
+        print('--> Using an existing workspace for certbot_test: {0}'.format(workspace))
+    config_dir = os.path.join(workspace, 'conf')
+
+    # Invoke certbot in test mode, without capturing output so users see directly the outcome.
+    command, env = _prepare_args_env(args, directory_url, http_01_port, tls_alpn_01_port,
+                                     config_dir, workspace, True)
+    subprocess.check_call(command, universal_newlines=True, cwd=workspace, env=env)
+
+
+if __name__ == '__main__':
+    main()

certbot-ci/certbot_integration_tests/utils/constants.py

@@ -0,0 +1,9 @@
+"""Some useful constants to use throughout certbot-ci integration tests"""
+HTTP_01_PORT = 5002
+TLS_ALPN_01_PORT = 5001
+CHALLTESTSRV_PORT = 8055
+BOULDER_V1_DIRECTORY_URL = 'http://localhost:4000/directory'
+BOULDER_V2_DIRECTORY_URL = 'http://localhost:4001/directory'
+PEBBLE_DIRECTORY_URL = 'https://localhost:14000/dir'
+PEBBLE_MANAGEMENT_URL = 'https://localhost:15000'
+MOCK_OCSP_SERVER_PORT = 4002

certbot-ci/certbot_integration_tests/utils/misc.py

@@ -3,15 +3,18 @@ Misc module contains stateless functions that could be used during pytest execut
 or outside during setup/teardown of the integration tests environment.
 """
 import contextlib
+import logging
 import errno
 import multiprocessing
 import os
+import re
 import shutil
 import stat
 import subprocess
 import sys
 import tempfile
 import time
+import warnings
 from distutils.version import LooseVersion
 
 import pkg_resources
@@ -22,24 +25,27 @@ from cryptography.hazmat.primitives.asymmetric import ec
 from cryptography.hazmat.primitives.serialization import Encoding, PrivateFormat, NoEncryption
 from six.moves import socketserver, SimpleHTTPServer
 
-from acme import crypto_util
-
-
 RSA_KEY_TYPE = 'rsa'
 ECDSA_KEY_TYPE = 'ecdsa'
 
 
-def check_until_timeout(url):
+def check_until_timeout(url, attempts=30):
     """
     Wait and block until given url responds with status 200, or raise an exception
-    after 150 attempts.
+    after the specified number of attempts.
     :param str url: the URL to test
-    :raise ValueError: exception raised after 150 unsuccessful attempts to reach the URL
+    :param int attempts: the number of times to try to connect to the URL
+    :raise ValueError: exception raised if unable to reach the URL
     """
-    import urllib3
-    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+    try:
+        import urllib3
+        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+    except ImportError:
+        # Handle old versions of request with vendorized urllib3
+        from requests.packages.urllib3.exceptions import InsecureRequestWarning
+        requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 
-    for _ in range(0, 150):
+    for _ in range(attempts):
         time.sleep(1)
         try:
             if requests.get(url, verify=False).status_code == 200:
@@ -47,7 +53,7 @@ def check_until_timeout(url):
         except requests.exceptions.ConnectionError:
             pass
 
-    raise ValueError('Error, url did not respond after 150 attempts: {0}'.format(url))
+    raise ValueError('Error, url did not respond after {0} attempts: {1}'.format(attempts, url))
 
 
 class GracefulTCPServer(socketserver.TCPServer):
@@ -58,6 +64,10 @@ class GracefulTCPServer(socketserver.TCPServer):
     allow_reuse_address = True
 
 
+def _run_server(port):
+    GracefulTCPServer(('', port), SimpleHTTPServer.SimpleHTTPRequestHandler).serve_forever()
+
+
 @contextlib.contextmanager
 def create_http_server(port):
     """
@@ -70,10 +80,7 @@ def create_http_server(port):
     current_cwd = os.getcwd()
     webroot = tempfile.mkdtemp()
 
-    def run():
-        GracefulTCPServer(('', port), SimpleHTTPServer.SimpleHTTPRequestHandler).serve_forever()
-
-    process = multiprocessing.Process(target=run)
+    process = multiprocessing.Process(target=_run_server, args=(port,))
 
     try:
         # SimpleHTTPServer is designed to serve files from the current working directory at the
@@ -115,15 +122,9 @@ def generate_test_file_hooks(config_dir, hook_probe):
     :param str config_dir: current certbot config directory
     :param hook_probe: path to the hook probe to test hook scripts execution
     """
-    if sys.platform == 'win32':
-        extension = 'bat'
-    else:
-        extension = 'sh'
+    hook_path = pkg_resources.resource_filename('certbot_integration_tests', 'assets/hook.py')
 
-    renewal_hooks_dirs = list_renewal_hooks_dirs(config_dir)
-    renewal_deploy_hook_path = os.path.join(renewal_hooks_dirs[1], 'hook.sh')
-
-    for hook_dir in renewal_hooks_dirs:
+    for hook_dir in list_renewal_hooks_dirs(config_dir):
         # We want an equivalent of bash `chmod -p $HOOK_DIR, that does not fail if one folder of
         # the hierarchy already exists. It is not the case of os.makedirs. Python 3 has an
         # optional parameter `exists_ok` to not fail on existing dir, but Python 2.7 does not.
@@ -133,26 +134,25 @@ def generate_test_file_hooks(config_dir, hook_probe):
         except OSError as error:
             if error.errno != errno.EEXIST:
                 raise
-        hook_path = os.path.join(hook_dir, 'hook.{0}'.format(extension))
-        if extension == 'sh':
-            data = '''\
-#!/bin/bash -xe
-if [ "$0" = "{0}" ]; then
-    if [ -z "$RENEWED_DOMAINS" -o -z "$RENEWED_LINEAGE" ]; then
-        echo "Environment variables not properly set!" >&2
-        exit 1
-    fi
-fi
-echo $(basename $(dirname "$0")) >> "{1}"\
-'''.format(renewal_deploy_hook_path, hook_probe)
-        else:
-            # TODO: Write the equivalent bat file for Windows
-            data = '''\
 
-'''
-        with open(hook_path, 'w') as file:
-            file.write(data)
-        os.chmod(hook_path, os.stat(hook_path).st_mode | stat.S_IEXEC)
+        if os.name != 'nt':
+            entrypoint_script_path = os.path.join(hook_dir, 'entrypoint.sh')
+            entrypoint_script = '''\
+#!/usr/bin/env bash
+set -e
+"{0}" "{1}" "{2}" "{3}"
+'''.format(sys.executable, hook_path, entrypoint_script_path, hook_probe)
+        else:
+            entrypoint_script_path = os.path.join(hook_dir, 'entrypoint.bat')
+            entrypoint_script = '''\
+@echo off
+"{0}" "{1}" "{2}" "{3}"
+            '''.format(sys.executable, hook_path, entrypoint_script_path, hook_probe)
+
+        with open(entrypoint_script_path, 'w') as file_h:
+            file_h.write(entrypoint_script)
+
+        os.chmod(entrypoint_script_path, os.stat(entrypoint_script_path).st_mode | stat.S_IEXEC)
 
 
 @contextlib.contextmanager
@@ -189,7 +189,7 @@ for _ in range(0, 10):
     except requests.exceptions.ConnectionError:
         pass
 raise ValueError('Error, url did not respond after 10 attempts: {{0}}'.format(url))
-'''.format(http_server_root, http_port))
+'''.format(http_server_root.replace('\\', '\\\\'), http_port))
         os.chmod(auth_script_path, 0o755)
 
         cleanup_script_path = os.path.join(tempdir, 'cleanup.py')
@@ -200,7 +200,7 @@ import os
 import shutil
 well_known = os.path.join('{0}', '.well-known')
 shutil.rmtree(well_known)
-'''.format(http_server_root))
+'''.format(http_server_root.replace('\\', '\\\\')))
         os.chmod(cleanup_script_path, 0o755)
 
         yield ('{0} {1}'.format(sys.executable, auth_script_path),
@@ -209,18 +209,6 @@ shutil.rmtree(well_known)
         shutil.rmtree(tempdir)
 
 
-def get_certbot_version():
-    """
-    Find the version of the certbot available in PATH.
-    :return str: the certbot version
-    """
-    output = subprocess.check_output(['certbot', '--version'],
-                                     universal_newlines=True, stderr=subprocess.STDOUT)
-    # Typical response is: output = 'certbot 0.31.0.dev0'
-    version_str = output.split(' ')[1].strip()
-    return LooseVersion(version_str)
-
-
 def generate_csr(domains, key_path, csr_path, key_type=RSA_KEY_TYPE):
     """
     Generate a private key, and a CSR for the given domains using this key.
@@ -234,20 +222,30 @@ def generate_csr(domains, key_path, csr_path, key_type=RSA_KEY_TYPE):
         key = crypto.PKey()
         key.generate_key(crypto.TYPE_RSA, 2048)
     elif key_type == ECDSA_KEY_TYPE:
-        key = ec.generate_private_key(ec.SECP384R1(), default_backend())
+        with warnings.catch_warnings():
+            # Ignore a warning on some old versions of cryptography
+            warnings.simplefilter('ignore', category=PendingDeprecationWarning)
+            key = ec.generate_private_key(ec.SECP384R1(), default_backend())
         key = key.private_bytes(encoding=Encoding.PEM, format=PrivateFormat.TraditionalOpenSSL,
                                 encryption_algorithm=NoEncryption())
         key = crypto.load_privatekey(crypto.FILETYPE_PEM, key)
     else:
         raise ValueError('Invalid key type: {0}'.format(key_type))
 
-    key_bytes = crypto.dump_privatekey(crypto.FILETYPE_PEM, key)
-    with open(key_path, 'wb') as file:
-        file.write(key_bytes)
+    with open(key_path, 'wb') as file_h:
+        file_h.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, key))
 
-    csr_bytes = crypto_util.make_csr(key_bytes, domains)
-    with open(csr_path, 'wb') as file:
-        file.write(csr_bytes)
+    req = crypto.X509Req()
+    san = ', '.join(['DNS:{0}'.format(item) for item in domains])
+    san_constraint = crypto.X509Extension(b'subjectAltName', False, san.encode('utf-8'))
+    req.add_extensions([san_constraint])
+
+    req.set_pubkey(key)
+    req.set_version(2)
+    req.sign(key, 'sha256')
+
+    with open(csr_path, 'wb') as file_h:
+        file_h.write(crypto.dump_certificate_request(crypto.FILETYPE_ASN1, req))
 
 
 def read_certificate(cert_path):
@@ -273,4 +271,32 @@ def load_sample_data_path(workspace):
     original = pkg_resources.resource_filename('certbot_integration_tests', 'assets/sample-config')
     copied = os.path.join(workspace, 'sample-config')
     shutil.copytree(original, copied, symlinks=True)
+
+    if os.name == 'nt':
+        # Fix the symlinks on Windows since GIT is not creating them upon checkout
+        for lineage in ['a.encryption-example.com', 'b.encryption-example.com']:
+            current_live = os.path.join(copied, 'live', lineage)
+            for name in os.listdir(current_live):
+                if name != 'README':
+                    current_file = os.path.join(current_live, name)
+                    with open(current_file) as file_h:
+                        src = file_h.read()
+                    os.unlink(current_file)
+                    os.symlink(os.path.join(current_live, src), current_file)
+
     return copied
+
+
+def echo(keyword, path=None):
+    """
+    Generate a platform independent executable command
+    that echoes the given keyword into the given file.
+    :param keyword: the keyword to echo (must be a single keyword)
+    :param path: path to the file were keyword is echoed
+    :return: the executable command
+    """
+    if not re.match(r'^\w+$', keyword):
+        raise ValueError('Error, keyword `{0}` is not a single keyword.'
+                         .format(keyword))
+    return '{0} -c "from __future__ import print_function; print(\'{1}\')"{2}'.format(
+        os.path.basename(sys.executable), keyword, ' >> "{0}"'.format(path) if path else '')

certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py

@@ -0,0 +1,53 @@
+import json
+import os
+import stat
+
+import pkg_resources
+import requests
+
+from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT
+
+PEBBLE_VERSION = 'v2.2.1'
+ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets')
+
+
+def fetch(workspace):
+    suffix = 'linux-amd64' if os.name != 'nt' else 'windows-amd64.exe'
+
+    pebble_path = _fetch_asset('pebble', suffix)
+    challtestsrv_path = _fetch_asset('pebble-challtestsrv', suffix)
+    pebble_config_path = _build_pebble_config(workspace)
+
+    return pebble_path, challtestsrv_path, pebble_config_path
+
+
+def _fetch_asset(asset, suffix):
+    asset_path = os.path.join(ASSETS_PATH, '{0}_{1}_{2}'.format(asset, PEBBLE_VERSION, suffix))
+    if not os.path.exists(asset_path):
+        asset_url = ('https://github.com/letsencrypt/pebble/releases/download/{0}/{1}_{2}'
+                     .format(PEBBLE_VERSION, asset, suffix))
+        response = requests.get(asset_url)
+        response.raise_for_status()
+        with open(asset_path, 'wb') as file_h:
+            file_h.write(response.content)
+    os.chmod(asset_path, os.stat(asset_path).st_mode | stat.S_IEXEC)
+
+    return asset_path
+
+
+def _build_pebble_config(workspace):
+    config_path = os.path.join(workspace, 'pebble-config.json')
+    with open(config_path, 'w') as file_h:
+        file_h.write(json.dumps({
+            'pebble': {
+                'listenAddress': '0.0.0.0:14000',
+                'managementListenAddress': '0.0.0.0:15000',
+                'certificate': os.path.join(ASSETS_PATH, 'cert.pem'),
+                'privateKey': os.path.join(ASSETS_PATH, 'key.pem'),
+                'httpPort': 5002,
+                'tlsPort': 5001,
+                'ocspResponderURL': 'http://127.0.0.1:{0}'.format(MOCK_OCSP_SERVER_PORT),
+            },
+        }))
+
+    return config_path

certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+"""
+This runnable module interfaces itself with the Pebble management interface in order
+to serve a mock OCSP responder during integration tests against Pebble.
+"""
+import datetime
+import re
+
+import requests
+from dateutil import parser
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization, hashes
+from cryptography import x509
+from cryptography.x509 import ocsp
+from six.moves import BaseHTTPServer
+
+from certbot_integration_tests.utils.misc import GracefulTCPServer
+from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT, PEBBLE_MANAGEMENT_URL
+
+
+class _ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+    def do_POST(self):
+        request = requests.get(PEBBLE_MANAGEMENT_URL + '/intermediate-keys/0', verify=False)
+        issuer_key = serialization.load_pem_private_key(request.content, None, default_backend())
+
+        request = requests.get(PEBBLE_MANAGEMENT_URL + '/intermediates/0', verify=False)
+        issuer_cert = x509.load_pem_x509_certificate(request.content, default_backend())
+
+        try:
+            content_len = int(self.headers.getheader('content-length', 0))
+        except AttributeError:
+            content_len = int(self.headers.get('Content-Length'))
+
+        ocsp_request = ocsp.load_der_ocsp_request(self.rfile.read(content_len))
+        response = requests.get('{0}/cert-status-by-serial/{1}'.format(
+            PEBBLE_MANAGEMENT_URL, str(hex(ocsp_request.serial_number)).replace('0x', '')), verify=False)
+
+        if not response.ok:
+            ocsp_response = ocsp.OCSPResponseBuilder.build_unsuccessful(ocsp.OCSPResponseStatus.UNAUTHORIZED)
+        else:
+            data = response.json()
+
+            now = datetime.datetime.utcnow()
+            cert = x509.load_pem_x509_certificate(data['Certificate'].encode(), default_backend())
+            if data['Status'] != 'Revoked':
+                ocsp_status, revocation_time, revocation_reason = ocsp.OCSPCertStatus.GOOD, None, None
+            else:
+                ocsp_status, revocation_reason = ocsp.OCSPCertStatus.REVOKED, x509.ReasonFlags.unspecified
+                revoked_at = re.sub(r'( \+\d{4}).*$', r'\1', data['RevokedAt'])  # "... +0000 UTC" => "+0000"
+                revocation_time = parser.parse(revoked_at)
+
+            ocsp_response = ocsp.OCSPResponseBuilder().add_response(
+                cert=cert, issuer=issuer_cert, algorithm=hashes.SHA1(),
+                cert_status=ocsp_status,
+                this_update=now, next_update=now + datetime.timedelta(hours=1),
+                revocation_time=revocation_time, revocation_reason=revocation_reason
+            ).responder_id(
+                ocsp.OCSPResponderEncoding.NAME, issuer_cert
+            ).sign(issuer_key, hashes.SHA256())
+
+        self.send_response(200)
+        self.end_headers()
+        self.wfile.write(ocsp_response.public_bytes(serialization.Encoding.DER))
+
+
+if __name__ == '__main__':
+    try:
+        GracefulTCPServer(('', MOCK_OCSP_SERVER_PORT), _ProxyHandler).serve_forever()
+    except KeyboardInterrupt:
+        pass

certbot-ci/certbot_integration_tests/utils/proxy.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+import json
+import sys
+import re
+
+import requests
+from six.moves import BaseHTTPServer
+
+from certbot_integration_tests.utils.misc import GracefulTCPServer
+
+
+def _create_proxy(mapping):
+    class ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+        def do_GET(self):
+            headers = {key.lower(): value for key, value in self.headers.items()}
+            backend = [backend for pattern, backend in mapping.items()
+                       if re.match(pattern, headers['host'])][0]
+            response = requests.get(backend + self.path, headers=headers)
+
+            self.send_response(response.status_code)
+            for key, value in response.headers.items():
+                self.send_header(key, value)
+            self.end_headers()
+            self.wfile.write(response.content)
+
+    return ProxyHandler
+
+
+if __name__ == '__main__':
+    http_port = int(sys.argv[1])
+    port_mapping = json.loads(sys.argv[2])
+    httpd = GracefulTCPServer(('', http_port), _create_proxy(port_mapping))
+    try:
+        httpd.serve_forever()
+    except KeyboardInterrupt:
+        pass

certbot-ci/setup.py

@@ -1,24 +1,35 @@
-from setuptools import setup
-from setuptools import find_packages
+import sys
+
+from distutils.version import StrictVersion
+from setuptools import setup, find_packages, __version__ as setuptools_version
 
 
 version = '0.32.0.dev0'
 
 install_requires = [
-    'acme',
     'coverage',
     'cryptography',
     'pyopenssl',
     'pytest',
     'pytest-cov',
-    'pytest-rerunfailures==4.2',
-    'pytest-sugar',
     'pytest-xdist',
+    'python-dateutil',
     'pyyaml',
     'requests',
     'six',
 ]
 
+# Add pywin32 on Windows platforms to handle low-level system calls.
+# This dependency needs to be added using environment markers to avoid its installation on Linux.
+# However environment markers are supported only with setuptools >= 36.2.
+# So this dependency is not added for old Linux distributions with old setuptools,
+# in order to allow these systems to build certbot from sources.
+if StrictVersion(setuptools_version) >= StrictVersion('36.2'):
+    install_requires.append("pywin32>=224 ; sys_platform == 'win32'")
+elif 'bdist_wheel' in sys.argv[1:]:
+    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
+                       'of setuptools. Version 36.2+ of setuptools is required.')
+
 setup(
     name='certbot-ci',
     version=version,
@@ -40,6 +51,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
     ],
@@ -47,4 +59,11 @@ setup(
     packages=find_packages(),
     include_package_data=True,
     install_requires=install_requires,
+
+    entry_points={
+        'console_scripts': [
+            'certbot_test=certbot_integration_tests.utils.certbot_call:main',
+            'run_acme_server=certbot_integration_tests.utils.acme_server:main',
+        ],
+    }
 )

certbot-compatibility-test/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:jessie
+FROM debian:stretch
 MAINTAINER Brad Warren <bmw@eff.org>
 
 # no need to mkdir anything:

certbot-compatibility-test/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.35.0.dev0'
+version = '0.39.0.dev0'
 
 install_requires = [
     'certbot',
@@ -47,6 +47,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
     ],

certbot-dns-cloudflare/Dockerfile

@@ -1,5 +0,0 @@
-FROM certbot/certbot
-
-COPY . src/certbot-dns-cloudflare
-
-RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-cloudflare

certbot-dns-cloudflare/certbot_dns_cloudflare/__init__.py

@@ -22,7 +22,9 @@ Credentials
 
 Use of this plugin requires a configuration file containing Cloudflare API
 credentials, obtained from your Cloudflare
-`account page <https://www.cloudflare.com/a/account/my-account>`_.
+`account page <https://www.cloudflare.com/a/account/my-account>`_. This plugin
+does not currently support Cloudflare's "API Tokens", so please ensure you use
+the "Global API Key" for authentication.
 
 .. code-block:: ini
    :name: credentials.ini

certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py

@@ -10,7 +10,7 @@ from certbot.plugins import dns_common
 
 logger = logging.getLogger(__name__)
 
-ACCOUNT_URL = 'https://www.cloudflare.com/a/account/my-account'
+ACCOUNT_URL = 'https://dash.cloudflare.com/profile/api-tokens'
 
 
 @zope.interface.implementer(interfaces.IAuthenticator)

certbot-dns-cloudflare/local-oldest-requirements.txt

@@ -1,3 +1,3 @@
 # Remember to update setup.py to match the package versions below.
 acme[dev]==0.29.0
-certbot[dev]==0.34.0
+-e .[dev]

certbot-dns-cloudflare/setup.py

@@ -2,13 +2,13 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.35.0.dev0'
+version = '0.39.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
     'acme>=0.29.0',
-    'certbot>=0.34.0',
+    'certbot>=0.39.0.dev0',
     'cloudflare>=1.5.1',
     'mock',
     'setuptools',
@@ -43,6 +43,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-cloudxns/Dockerfile

@@ -1,5 +0,0 @@
-FROM certbot/certbot
-
-COPY . src/certbot-dns-cloudxns
-
-RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-cloudxns

certbot-dns-cloudxns/local-oldest-requirements.txt

@@ -1,3 +1,3 @@
 # Remember to update setup.py to match the package versions below.
 acme[dev]==0.31.0
-certbot[dev]==0.34.0
+-e .[dev]

certbot-dns-cloudxns/setup.py

@@ -2,13 +2,13 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.35.0.dev0'
+version = '0.39.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
     'acme>=0.31.0',
-    'certbot>=0.34.0',
+    'certbot>=0.39.0.dev0',
     'dns-lexicon>=2.2.1',  # Support for >1 TXT record per name
     'mock',
     'setuptools',
@@ -43,6 +43,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-digitalocean/Dockerfile

@@ -1,5 +0,0 @@
-FROM certbot/certbot
-
-COPY . src/certbot-dns-digitalocean
-
-RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-digitalocean

certbot-dns-digitalocean/local-oldest-requirements.txt

@@ -1,3 +1,3 @@
 # Remember to update setup.py to match the package versions below.
 acme[dev]==0.29.0
-certbot[dev]==0.34.0
+-e .[dev]

certbot-dns-digitalocean/setup.py

@@ -2,13 +2,13 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.35.0.dev0'
+version = '0.39.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
     'acme>=0.29.0',
-    'certbot>=0.34.0',
+    'certbot>=0.39.0.dev0',
     'mock',
     'python-digitalocean>=1.11',
     'setuptools',
@@ -44,6 +44,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-dnsimple/Dockerfile

@@ -1,5 +0,0 @@
-FROM certbot/certbot
-
-COPY . src/certbot-dns-dnsimple
-
-RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-dnsimple

certbot-dns-dnsimple/local-oldest-requirements.txt

@@ -1,3 +1,3 @@
 # Remember to update setup.py to match the package versions below.
 acme[dev]==0.31.0
-certbot[dev]==0.34.0
+-e .[dev]

certbot-dns-dnsimple/setup.py

@@ -3,13 +3,13 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.35.0.dev0'
+version = '0.39.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
     'acme>=0.31.0',
-    'certbot>=0.34.0',
+    'certbot>=0.39.0.dev0',
     'mock',
     'setuptools',
     'zope.interface',
@@ -55,6 +55,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-dnsmadeeasy/Dockerfile

@@ -1,5 +0,0 @@
-FROM certbot/certbot
-
-COPY . src/certbot-dns-dnsmadeeasy
-
-RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-dnsmadeeasy

certbot-dns-dnsmadeeasy/local-oldest-requirements.txt

@@ -1,3 +1,3 @@
 # Remember to update setup.py to match the package versions below.
 acme[dev]==0.31.0
-certbot[dev]==0.34.0
+-e .[dev]

certbot-dns-dnsmadeeasy/setup.py

@@ -2,13 +2,13 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.35.0.dev0'
+version = '0.39.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
     'acme>=0.31.0',
-    'certbot>=0.34.0',
+    'certbot>=0.39.0.dev0',
     'dns-lexicon>=2.2.1',  # Support for >1 TXT record per name
     'mock',
     'setuptools',
@@ -43,6 +43,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-gehirn/Dockerfile

@@ -1,5 +0,0 @@
-FROM certbot/certbot
-
-COPY . src/certbot-dns-gehirn
-
-RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-gehirn

certbot-dns-gehirn/local-oldest-requirements.txt

@@ -1,3 +1,3 @@
 # Remember to update setup.py to match the package versions below.
 acme[dev]==0.31.0
-certbot[dev]==0.34.0
+-e .[dev]

certbot-dns-gehirn/setup.py

@@ -2,12 +2,12 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.35.0.dev0'
+version = '0.39.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
     'acme>=0.31.0',
-    'certbot>=0.34.0',
+    'certbot>=0.39.0.dev0',
     'dns-lexicon>=2.1.22',
     'mock',
     'setuptools',
@@ -41,6 +41,8 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-google/Dockerfile

@@ -1,5 +0,0 @@
-FROM certbot/certbot
-
-COPY . src/certbot-dns-google
-
-RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-google

certbot-dns-google/certbot_dns_google/dns_google.py

@@ -274,10 +274,11 @@ class _GoogleClient(object):
                 raise errors.PluginError('Encountered error finding managed zone: {0}'
                                          .format(e))
 
-            if zones:
-                zone_id = zones[0]['id']
-                logger.debug('Found id of %s for %s using name %s', zone_id, domain, zone_name)
-                return zone_id
+            for zone in zones:
+                zone_id = zone['id']
+                if 'privateVisibilityConfig' not in zone:
+                    logger.debug('Found id of %s for %s using name %s', zone_id, domain, zone_name)
+                    return zone_id
 
         raise errors.PluginError('Unable to determine managed zone for {0} using zone names: {1}.'
                                  .format(domain, zone_dns_name_guesses))

certbot-dns-google/local-oldest-requirements.txt

@@ -1,3 +1,3 @@
 # Remember to update setup.py to match the package versions below.
 acme[dev]==0.29.0
-certbot[dev]==0.34.0
+-e .[dev]

certbot-dns-google/setup.py

@@ -2,13 +2,13 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.35.0.dev0'
+version = '0.39.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
     'acme>=0.29.0',
-    'certbot>=0.34.0',
+    'certbot>=0.39.0.dev0',
     # 1.5 is the first version that supports oauth2client>=2.0
     'google-api-python-client>=1.5',
     'mock',
@@ -48,6 +48,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-linode/Dockerfile

@@ -1,5 +0,0 @@
-FROM certbot/certbot
-
-COPY . src/certbot-dns-linode
-
-RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-linode

certbot-dns-linode/local-oldest-requirements.txt

@@ -1,4 +1,4 @@
 # Remember to update setup.py to match the package versions below.
 acme[dev]==0.31.0
-certbot[dev]==0.34.0
+-e .[dev]
 dns-lexicon==2.2.3

certbot-dns-linode/setup.py

@@ -1,12 +1,12 @@
 from setuptools import setup
 from setuptools import find_packages
 
-version = '0.35.0.dev0'
+version = '0.39.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
     'acme>=0.31.0',
-    'certbot>=0.34.0',
+    'certbot>=0.39.0.dev0',
     'dns-lexicon>=2.2.3',
     'mock',
     'setuptools',
@@ -41,6 +41,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-luadns/Dockerfile

@@ -1,5 +0,0 @@
-FROM certbot/certbot
-
-COPY . src/certbot-dns-luadns
-
-RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-luadns

certbot-dns-luadns/local-oldest-requirements.txt

@@ -1,3 +1,3 @@
 # Remember to update setup.py to match the package versions below.
 acme[dev]==0.31.0
-certbot[dev]==0.34.0
+-e .[dev]

certbot-dns-luadns/setup.py

@@ -2,13 +2,13 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.35.0.dev0'
+version = '0.39.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
     'acme>=0.31.0',
-    'certbot>=0.34.0',
+    'certbot>=0.39.0.dev0',
     'dns-lexicon>=2.2.1',  # Support for >1 TXT record per name
     'mock',
     'setuptools',
@@ -43,6 +43,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-nsone/Dockerfile

@@ -1,5 +0,0 @@
-FROM certbot/certbot
-
-COPY . src/certbot-dns-nsone
-
-RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-nsone

certbot-dns-nsone/local-oldest-requirements.txt

@@ -1,3 +1,3 @@
 # Remember to update setup.py to match the package versions below.
 acme[dev]==0.31.0
-certbot[dev]==0.34.0
+-e .[dev]

certbot-dns-nsone/setup.py

@@ -2,13 +2,13 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.35.0.dev0'
+version = '0.39.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
     'acme>=0.31.0',
-    'certbot>=0.34.0',
+    'certbot>=0.39.0.dev0',
     'dns-lexicon>=2.2.1',  # Support for >1 TXT record per name
     'mock',
     'setuptools',
@@ -43,6 +43,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-ovh/Dockerfile

@@ -1,5 +0,0 @@
-FROM certbot/certbot
-
-COPY . src/certbot-dns-ovh
-
-RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-ovh

certbot-dns-ovh/local-oldest-requirements.txt

@@ -1,4 +1,4 @@
 # Remember to update setup.py to match the package versions below.
 acme[dev]==0.31.0
-certbot[dev]==0.34.0
+-e .[dev]
 dns-lexicon==2.7.14

certbot-dns-ovh/setup.py

@@ -2,13 +2,13 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.35.0.dev0'
+version = '0.39.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
     'acme>=0.31.0',
-    'certbot>=0.34.0',
+    'certbot>=0.39.0.dev0',
     'dns-lexicon>=2.7.14',  # Correct proxy use on OVH provider
     'mock',
     'setuptools',
@@ -42,6 +42,8 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',