test snap only

* Add the content interface to Certbot

This commit contains a subset of the changes from 7076a55fd82116d068e2aca7239209b7203917d2.

* Normalise slot parameters

(cherry picked from commit 810941979bcf609c1e0be18e9263abf046b90e82)

.azure-pipelines/main.yml

@@ -1,10 +1,6 @@
 trigger:
-  # apache-parser-v2 is a temporary branch for doing work related to
-  # rewriting the parser in the Apache plugin.
-  - apache-parser-v2
   - master
 pr:
-  - apache-parser-v2
   - master
   - '*.x'
 

.gitignore

@@ -51,3 +51,10 @@ tests/letstest/venv3/
 .certbot_test_workspace
 **/assets/pebble*
 **/assets/challtestsrv*
+
+# snap files
+.snapcraft
+parts
+prime
+stage
+*.snap

.travis.yml

@@ -11,20 +11,23 @@ before_script:
   # Use Travis retry feature for farm tests since they are flaky
   - 'if [[ "$TOXENV" == "travis-test-farm"* ]]; then export TRAVIS_RETRY=travis_retry; fi'
   - export TOX_TESTENV_PASSENV=TRAVIS
+  - 'if [[ "$SNAP" == true ]]; then snap/local/build_and_install.sh; fi'
 
 # Only build pushes to the master branch, PRs, and branches beginning with
-# `test-`, `travis-test-`, or of the form `digit(s).digit(s).x`. This reduces
-# the number of simultaneous Travis runs, which speeds turnaround time on
-# review since there is a cap of on the number of simultaneous runs.
+# `test-`, `travis-test-`, or of the form `digit(s).digit(s).x` or
+# `vdigit(s).digit(s).digit(s)`. As documented at
+# https://docs.travis-ci.com/user/customizing-the-build/#safelisting-or-blocklisting-branches,
+# this includes tags so pushing tags of the form `vdigit(s).digit(s).digit(s)`
+# will also trigger tests. This reduces the number of simultaneous Travis runs,
+# which speeds turnaround time on review since there is a cap of on the number
+# of simultaneous runs.
 branches:
   # When changing these branches, please ensure the documentation under
   # "Running tests in CI" is still correct.
   only:
-    # apache-parser-v2 is a temporary branch for doing work related to
-    # rewriting the parser in the Apache plugin.
-    - apache-parser-v2
     - master
-    - /^\d+\.\d+\.x$/
+    - /^\d+\.\d+\.x$/  # this matches our point release branches
+    - /^v\d+\.\d+\.\d+$/  # this matches our release tags
     - /^(travis-)?test-.*$/
 
 # Jobs for the main test suite are always executed (including on PRs) except for pushes on master.
@@ -32,196 +35,56 @@ not-on-master: &not-on-master
   if: NOT (type = push AND branch = master)
 
 # Jobs for the extended test suite are executed for cron jobs and pushes to
-# non-development branches. See the explanation for apache-parser-v2 above.
+# non-development branches.
 extended-test-suite: &extended-test-suite
-  if: type = cron OR (type = push AND branch NOT IN (apache-parser-v2, master))
+  if: type = cron OR (type = push AND branch != master)
 
 matrix:
   include:
-    # Main test suite
-    - python: "2.7"
-      env: ACME_SERVER=pebble TOXENV=integration
-      <<: *not-on-master
-
-    # This job is always executed, including on master
-    - python: "2.7"
-      env: TOXENV=py27-cover FYI="py27 tests + code coverage"
-
-    - python: "3.7"
-      env: TOXENV=lint
-      <<: *not-on-master
-    - python: "3.5"
-      env: TOXENV=mypy
-      <<: *not-on-master
-    - python: "2.7"
-      # Ubuntu Trusty or older must be used because the oldest version of
-      # cryptography we support cannot be compiled against the version of
-      # OpenSSL in Xenial or newer.
-      dist: trusty
-      env: TOXENV='py27-{acme,apache,apache-v2,certbot,dns,nginx}-oldest'
-      <<: *not-on-master
-    - python: "3.5"
-      env: TOXENV=py35
-      <<: *not-on-master
-    - python: "3.8"
-      env: TOXENV=py38
-      <<: *not-on-master
-    - sudo: required
-      env: TOXENV=apache_compat
-      services: docker
-      before_install:
-      addons:
-      <<: *not-on-master
-    - sudo: required
-      env: TOXENV=le_auto_xenial
-      services: docker
-      <<: *not-on-master
-    - python: "2.7"
-      env: TOXENV=apacheconftest-with-pebble
-      <<: *not-on-master
-    - python: "2.7"
-      env: TOXENV=nginxroundtrip
-      <<: *not-on-master
-
-    # Extended test suite on cron jobs and pushes to tested branches other than master
-    - sudo: required
-      env: TOXENV=nginx_compat
-      services: docker
-      before_install:
-      addons:
-      <<: *extended-test-suite
-    - python: "3.7"
-      env:
-        - TOXENV=travis-test-farm-apache2
-        - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
-      <<: *extended-test-suite
-    - python: "3.7"
-      env:
-        - TOXENV=travis-test-farm-leauto-upgrades
-        - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
-      git:
-        depth: false  # This is needed to have the history to checkout old versions of certbot-auto.
-      <<: *extended-test-suite
-    - python: "3.7"
-      env:
-        - TOXENV=travis-test-farm-certonly-standalone
-        - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
-      <<: *extended-test-suite
-    - python: "3.7"
-      env:
-        - TOXENV=travis-test-farm-sdists
-        - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
-      <<: *extended-test-suite
-    - python: "3.7"
-      env: TOXENV=py37 CERTBOT_NO_PIN=1
-      <<: *extended-test-suite
-    - python: "2.7"
-      env: ACME_SERVER=boulder-v1 TOXENV=integration
+    - stage: "Snap"
       sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "2.7"
-      env: ACME_SERVER=boulder-v2 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "2.7"
-      env: ACME_SERVER=boulder-v1 TOXENV=integration-certbot-oldest
-      # Ubuntu Trusty or older must be used because the oldest version of
-      # cryptography we support cannot be compiled against the version of
-      # OpenSSL in Xenial or newer.
-      dist: trusty
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "2.7"
-      env: ACME_SERVER=boulder-v2 TOXENV=integration-certbot-oldest
-      # Ubuntu Trusty or older must be used because the oldest version of
-      # cryptography we support cannot be compiled against the version of
-      # OpenSSL in Xenial or newer.
-      dist: trusty
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "2.7"
-      env: ACME_SERVER=boulder-v1 TOXENV=integration-nginx-oldest
-      # Ubuntu Trusty or older must be used because the oldest version of
-      # cryptography we support cannot be compiled against the version of
-      # OpenSSL in Xenial or newer.
-      dist: trusty
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "2.7"
-      env: ACME_SERVER=boulder-v2 TOXENV=integration-nginx-oldest
-      # Ubuntu Trusty or older must be used because the oldest version of
-      # cryptography we support cannot be compiled against the version of
-      # OpenSSL in Xenial or newer.
-      dist: trusty
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "3.6"
-      env: TOXENV=py36
-      <<: *extended-test-suite
-    - python: "3.7"
-      env: TOXENV=py37
-      <<: *extended-test-suite
-    - python: "3.5"
-      env: ACME_SERVER=boulder-v1 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "3.5"
-      env: ACME_SERVER=boulder-v2 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "3.6"
-      env: ACME_SERVER=boulder-v1 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "3.6"
-      env: ACME_SERVER=boulder-v2 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "3.7"
-      env: ACME_SERVER=boulder-v1 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "3.7"
-      env: ACME_SERVER=boulder-v2 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "3.8"
-      env: ACME_SERVER=boulder-v1 TOXENV=integration
-      <<: *extended-test-suite
-    - python: "3.8"
-      env: ACME_SERVER=boulder-v2 TOXENV=integration
-      <<: *extended-test-suite
-    - sudo: required
-      env: TOXENV=le_auto_jessie
-      services: docker
-      <<: *extended-test-suite
-    - sudo: required
-      env: TOXENV=le_auto_centos6
-      services: docker
-      <<: *extended-test-suite
-    - sudo: required
-      env: TOXENV=le_auto_oraclelinux6
-      services: docker
-      <<: *extended-test-suite
-    - sudo: required
-      env: TOXENV=docker_dev
-      services: docker
+      env: SNAP=true TOXENV=integration-external,apacheconftest-external-with-pebble
       addons:
         apt:
-          packages:  # don't install nginx and apache
-            - libaugeas0
+          packages:
+          - nginx-light
+        snaps:
+          - name: snapcraft
+            channel: stable
+            confinement: classic
+          - name: lxd
+            channel: stable
+      git:
+        # By default, Travis clones the repo to a depth of 50 commits which can
+        # break the ability to use `git describe` to set the version of the
+        # snap. This setting removes the --depth flag from git commands solving
+        # this problem. See
+        # https://docs.travis-ci.com/user/customizing-the-build#git-clone-depth
+        # for more info.
+        depth: false
+      deploy:
+        # This section relies on credentials stored in a SNAP_TOKEN environment
+        # variable in Travis. See
+        # https://docs.travis-ci.com/user/deployment/snaps/ for more info.
+        # This credential has a maximum lifetime of 1 year and the current
+        # credential will expire on 4/22/2021. The value of SNAP_TOKEN will
+        # need to be updated to use a new credential before then to prevent
+        # automated deploys from breaking. Remembering to do this is also
+        # tracked by https://github.com/certbot/certbot/issues/7931.
+        'on':
+          # Deploy on release tags or nightly runs from any branch. We only try
+          # to deploy from the certbot/certbot repo to prevent errors if forks
+          # of this repo try to run tests.
+          all_branches: true
+          condition: -n $TRAVIS_TAG || $TRAVIS_EVENT_TYPE = cron
+          repo: certbot/certbot
+        provider: snap
+        snap: certbot_*.snap
+        channel: edge
+        # skip_cleanup is needed to prevent Travis from deleting the snaps we
+        # just built and tested. See
+        # https://docs.travis-ci.com/user/deployment#uploading-files-and-skip_cleanup.
+        skip_cleanup: true
       <<: *extended-test-suite
 
 # container-based infrastructure

AUTHORS.md

@@ -237,6 +237,7 @@ Authors
 * [Stefan Weil](https://github.com/stweil)
 * [Steve Desmond](https://github.com/stevedesmond-ca)
 * [sydneyli](https://github.com/sydneyli)
+* [taixx046](https://github.com/taixx046)
 * [Tan Jay Jun](https://github.com/jayjun)
 * [Tapple Gao](https://github.com/tapple)
 * [Telepenin Nikolay](https://github.com/telepenin)

acme/setup.py

@@ -6,7 +6,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-apache/certbot_apache/_internal/configurator.py

@@ -595,6 +595,11 @@ class ApacheConfigurator(common.Installer):
         # cert_key... can all be parsed appropriately
         self.prepare_server_https("443")
 
+        # If we haven't managed to enable mod_ssl by this point, error out
+        if "ssl_module" not in self.parser.modules:
+            raise errors.MisconfigurationError("Could not find ssl_module; "
+                "not installing certificate.")
+
         # Add directives and remove duplicates
         self._add_dummy_ssl_directives(vhost.path)
         self._clean_vhost(vhost)
@@ -609,21 +614,6 @@ class ApacheConfigurator(common.Installer):
             path["chain_path"] = self.parser.find_dir(
                 "SSLCertificateChainFile", None, vhost.path)
 
-        # Handle errors when certificate/key directives cannot be found
-        if not path["cert_path"]:
-            logger.warning(
-                "Cannot find an SSLCertificateFile directive in %s. "
-                "VirtualHost was not modified", vhost.path)
-            raise errors.PluginError(
-                "Unable to find an SSLCertificateFile directive")
-        elif not path["cert_key"]:
-            logger.warning(
-                "Cannot find an SSLCertificateKeyFile directive for "
-                "certificate in %s. VirtualHost was not modified", vhost.path)
-            raise errors.PluginError(
-                "Unable to find an SSLCertificateKeyFile directive for "
-                "certificate")
-
         logger.info("Deploying Certificate to VirtualHost %s", vhost.filep)
 
         if self.version < (2, 4, 8) or (chain_path and not fullchain_path):

certbot-apache/certbot_apache/_internal/parser.py

@@ -741,7 +741,7 @@ class ApacheParser(object):
         """
         if sys.version_info < (3, 6):
             # This strips off final /Z(?ms)
-            return fnmatch.translate(clean_fn_match)[:-7]
+            return fnmatch.translate(clean_fn_match)[:-7]  # pragma: no cover
         # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z
         return fnmatch.translate(clean_fn_match)[4:-3]  # pragma: no cover
 

certbot-apache/setup.py

@@ -6,7 +6,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-apache/tests/configurator_test.py

@@ -455,41 +455,6 @@ class MultipleVhostsTest(util.ApacheTest):
             "SSLCertificateChainFile", "two/cert_chain.pem",
             self.vh_truth[1].path))
 
-    def test_deploy_cert_invalid_vhost(self):
-        """For test cases where the `ApacheConfigurator` class' `_deploy_cert`
-        method is called with an invalid vhost parameter. Currently this tests
-        that a PluginError is appropriately raised when important directives
-        are missing in an SSL module."""
-        self.config.parser.modules["ssl_module"] = None
-        self.config.parser.modules["mod_ssl.c"] = None
-        self.config.parser.modules["socache_shmcb_module"] = None
-
-        def side_effect(*args):
-            """Mocks case where an SSLCertificateFile directive can be found
-            but an SSLCertificateKeyFile directive is missing."""
-            if "SSLCertificateFile" in args:
-                return ["example/cert.pem"]
-            return []
-
-        mock_find_dir = mock.MagicMock(return_value=[])
-        mock_find_dir.side_effect = side_effect
-
-        self.config.parser.find_dir = mock_find_dir
-
-        # Get the default 443 vhost
-        self.config.assoc["random.demo"] = self.vh_truth[1]
-
-        self.assertRaises(
-            errors.PluginError, self.config.deploy_cert, "random.demo",
-            "example/cert.pem", "example/key.pem", "example/cert_chain.pem")
-
-        # Remove side_effect to mock case where both SSLCertificateFile
-        # and SSLCertificateKeyFile directives are missing
-        self.config.parser.find_dir.side_effect = None
-        self.assertRaises(
-            errors.PluginError, self.config.deploy_cert, "random.demo",
-            "example/cert.pem", "example/key.pem", "example/cert_chain.pem")
-
     def test_is_name_vhost(self):
         addr = obj.Addr.fromstring("*:80")
         self.assertTrue(self.config.is_name_vhost(addr))
@@ -1349,6 +1314,16 @@ class MultipleVhostsTest(util.ApacheTest):
                 self.assertTrue(mock_add.called)
         shutil.rmtree(tmp_path)
 
+    def test_deploy_cert_no_mod_ssl(self):
+        # Create
+        ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])
+        self.config.parser.modules["socache_shmcb_module"] = None
+        self.config.prepare_server_https = mock.Mock()
+
+        self.assertRaises(errors.MisconfigurationError, self.config.deploy_cert,
+            "encryption-example.demo", "example/cert.pem", "example/key.pem",
+            "example/cert_chain.pem", "example/fullchain.pem")
+
     @mock.patch("certbot_apache._internal.parser.ApacheParser.parsed_in_original")
     def test_choose_vhost_and_servername_addition_parsed(self, mock_parsed):
         ret_vh = self.vh_truth[8]

certbot-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="1.3.0"
+LE_AUTO_VERSION="1.4.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -910,20 +910,11 @@ elif [ -f /etc/manjaro-release ]; then
   }
   BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION"
 elif [ -f /etc/gentoo-release ]; then
-  Bootstrap() {
-    DeprecationBootstrap "Gentoo" BootstrapGentooCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapGentooCommon $BOOTSTRAP_GENTOO_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif uname | grep -iq FreeBSD ; then
-  Bootstrap() {
-    DeprecationBootstrap "FreeBSD" BootstrapFreeBsd
-  }
-  BOOTSTRAP_VERSION="BootstrapFreeBsd $BOOTSTRAP_FREEBSD_VERSION"
+  DEPRECATED_OS=1
 elif uname | grep -iq Darwin ; then
-  Bootstrap() {
-    DeprecationBootstrap "macOS" BootstrapMac
-  }
-  BOOTSTRAP_VERSION="BootstrapMac $BOOTSTRAP_MAC_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then
   Bootstrap() {
     ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon
@@ -1540,18 +1531,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==1.3.0 \
-    --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \
-    --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8
-acme==1.3.0 \
-    --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \
-    --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d
-certbot-apache==1.3.0 \
-    --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \
-    --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52
-certbot-nginx==1.3.0 \
-    --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \
-    --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b
+certbot==1.4.0 \
+    --hash=sha256:5f8be1e6087d2f1f742caf0048b0f46bac8d3a655d038d5355abd1638523d87e \
+    --hash=sha256:69b5b7925de0d3b693b00a40bf109d85afb24c7199bf616339d74d59a80d8d96
+acme==1.4.0 \
+    --hash=sha256:d2f6799f7fce2414fc1a6753ced91c0ccdf1d0b2cee892c509851db45402fb5b \
+    --hash=sha256:f12cb59762e0b833911b87e95cb16e85a162517ba4aa3440594bdf3b8126fc69
+certbot-apache==1.4.0 \
+    --hash=sha256:1be1a38cb73e950c5cbff941719d326bfd2d1b4fff17b39b7a27377067cd90a6 \
+    --hash=sha256:6067f537deb7f70b979d11ed19846712dbf5c484ca927841805e78d8797b4640
+certbot-nginx==1.4.0 \
+    --hash=sha256:8ee1c7201b40bde7d476894fb06bf8ab0c0cd0ba03c0510bc568e8713e801ccc \
+    --hash=sha256:44a9f74dee7e2f8a32aafaf793280e8fcd4d50a9ffb7c5ed47a0bc591ce6ecca
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------

certbot-ci/certbot_integration_tests/assets/hook.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import print_function
 import os
 import sys
 
@@ -7,5 +8,4 @@ if hook_script_type == 'deploy' and ('RENEWED_DOMAINS' not in os.environ or 'REN
     sys.stderr.write('Environment variables not properly set!\n')
     sys.exit(1)
 
-with open(sys.argv[2], 'a') as file_h:
-    file_h.write(hook_script_type + '\n')
+print(hook_script_type)

certbot-ci/certbot_integration_tests/certbot_tests/assertions.py

@@ -1,4 +1,5 @@
 """This module contains advanced assertions for the certbot integration tests."""
+import io
 import os
 
 try:
@@ -21,7 +22,8 @@ def assert_hook_execution(probe_path, probe_content):
     :param probe_path: path to the file that received the hook output
     :param probe_content: content expected when the hook is executed
     """
-    with open(probe_path, 'r') as file:
+    encoding = 'utf-8' if POSIX_MODE else 'utf-16'
+    with io.open(probe_path, 'rt', encoding=encoding) as file:
         data = file.read()
 
     lines = [line.strip() for line in data.splitlines()]

certbot-ci/certbot_integration_tests/utils/acme_server.py

@@ -86,7 +86,8 @@ class ACMEServer(object):
                                                 'alpine', 'rm', '-rf', '/workspace/boulder'])
                 process.wait()
         finally:
-            shutil.rmtree(self._workspace)
+            if os.path.exists(self._workspace):
+                shutil.rmtree(self._workspace)
         if self._stdout != sys.stdout:
             self._stdout.close()
         print('=> Test infrastructure stopped and cleaned up.')

certbot-ci/certbot_integration_tests/utils/misc.py

@@ -140,13 +140,12 @@ def generate_test_file_hooks(config_dir, hook_probe):
             entrypoint_script = '''\
 #!/usr/bin/env bash
 set -e
-"{0}" "{1}" "{2}" "{3}"
+"{0}" "{1}" "{2}" >> "{3}"
 '''.format(sys.executable, hook_path, entrypoint_script_path, hook_probe)
         else:
-            entrypoint_script_path = os.path.join(hook_dir, 'entrypoint.bat')
+            entrypoint_script_path = os.path.join(hook_dir, 'entrypoint.ps1')
             entrypoint_script = '''\
-@echo off
-"{0}" "{1}" "{2}" "{3}"
+& "{0}" "{1}" "{2}" >> "{3}"
             '''.format(sys.executable, hook_path, entrypoint_script_path, hook_probe)
 
         with open(entrypoint_script_path, 'w') as file_h:

certbot-compatibility-test/setup.py

@@ -5,7 +5,7 @@ from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 install_requires = [
     'certbot',

certbot-dns-cloudflare/setup.py

@@ -6,7 +6,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-cloudxns/setup.py

@@ -6,7 +6,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-digitalocean/setup.py

@@ -6,7 +6,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-dnsimple/setup.py

@@ -7,7 +7,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-dnsmadeeasy/setup.py

@@ -6,7 +6,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-gehirn/setup.py

@@ -6,7 +6,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-google/setup.py

@@ -6,7 +6,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-linode/setup.py

@@ -6,7 +6,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-luadns/setup.py

@@ -6,7 +6,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-nsone/setup.py

@@ -6,7 +6,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-ovh/setup.py

@@ -6,7 +6,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-rfc2136/setup.py

@@ -6,7 +6,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-route53/setup.py

@@ -6,7 +6,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-sakuracloud/setup.py

@@ -6,7 +6,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-nginx/local-oldest-requirements.txt

@@ -1,3 +1,3 @@
 # Remember to update setup.py to match the package versions below.
--e acme[dev]
--e certbot[dev]
+acme[dev]==1.4.0
+certbot[dev]==1.4.0

certbot-nginx/setup.py

@@ -6,13 +6,13 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.4.0.dev0'
+version = '1.5.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    'acme>=1.4.0.dev0',
-    'certbot>=1.4.0.dev0',
+    'acme>=1.4.0',
+    'certbot>=1.4.0',
     'PyOpenSSL',
     'pyparsing>=1.5.5',  # Python3 support
     'setuptools',

certbot/CHANGELOG.md

@@ -2,7 +2,24 @@
 
 Certbot adheres to [Semantic Versioning](https://semver.org/).
 
-## 1.4.0 - master
+## 1.5.0 - master
+
+### Added
+
+*
+
+### Changed
+
+* Improved error message in apache installer when mod_ssl is not available.
+
+### Fixed
+
+* Add support for OCSP responses which use a public key hash ResponderID, fixing
+  interoperability with Sectigo CAs.
+
+More details about these changes can be found on our GitHub repo.
+
+## 1.4.0 - 2020-05-05
 
 ### Added
 
@@ -15,11 +32,16 @@ Certbot adheres to [Semantic Versioning](https://semver.org/).
 * Added TLS-ALPN-01 challenge support in the `acme` library. Support of this
   challenge in the Certbot client is planned to be added in a future release.
 * Added minimal proxy support for OCSP verification.
+* On Windows, hooks are now executed in a Powershell shell instead of a CMD shell,
+  allowing both `*.ps1` and `*.bat` as valid scripts for Certbot.
 
 ### Changed
 
+* Reorganized error message when a user entered an invalid email address.
 * Stop asking interactively if the user would like to add a redirect.
 * `mock` dependency is now conditional on Python 2 in all of our packages.
+* Deprecate certbot-auto on Gentoo, macOS, and FreeBSD.
+* Allow existing but empty archive and live dir to be used when creating new lineage.
 
 ### Fixed
 

certbot/README.rst

@@ -71,16 +71,12 @@ ACME spec: http://ietf-wg-acme.github.io/acme/
 
 ACME working area in github: https://github.com/ietf-wg-acme/acme
 
-|build-status| |container|
+|build-status|
 
 .. |build-status| image:: https://travis-ci.com/certbot/certbot.svg?branch=master
    :target: https://travis-ci.com/certbot/certbot
    :alt: Travis CI status
 
-.. |container| image:: https://quay.io/repository/letsencrypt/letsencrypt/status
-   :target: https://quay.io/repository/letsencrypt/letsencrypt
-   :alt: Docker Repository on Quay.io
-
 .. Do not modify this comment unless you know what you're doing. tag:links-end
 
 System Requirements

certbot/certbot/__init__.py

@@ -1,4 +1,4 @@
 """Certbot client."""
 
 # version number like 1.2.3a0, must have at least 2 parts, like 1.2
-__version__ = '1.4.0.dev0'
+__version__ = '1.5.0.dev0'

certbot/certbot/_internal/cli/paths_parser.py

@@ -38,7 +38,7 @@ def _paths_parser(helpful):
         default_cp = flag_default("auth_chain_path")
     add(["paths", "install"], "--fullchain-path", default=default_cp, type=os.path.abspath,
         help="Accompanying path to a full certificate chain (certificate plus chain).")
-    add("paths", "--chain-path", default=default_cp, type=os.path.abspath,
+    add(["paths", "install"], "--chain-path", default=default_cp, type=os.path.abspath,
         help="Accompanying path to a certificate chain.")
     add("paths", "--config-dir", default=flag_default("config_dir"),
         help=config_help("config_dir"))

certbot/certbot/_internal/hooks.py

@@ -2,14 +2,13 @@
 from __future__ import print_function
 
 import logging
-from subprocess import PIPE
-from subprocess import Popen
 
 from acme.magic_typing import List
 from acme.magic_typing import Set
 from certbot import errors
 from certbot import util
 from certbot.compat import filesystem
+from certbot.compat import misc
 from certbot.compat import os
 from certbot.plugins import util as plug_util
 
@@ -229,36 +228,10 @@ def _run_hook(cmd_name, shell_cmd):
     :type shell_cmd: `list` of `str` or `str`
 
     :returns: stderr if there was any"""
-    err, _ = execute(cmd_name, shell_cmd)
+    err, _ = misc.execute_command(cmd_name, shell_cmd)
     return err
 
 
-def execute(cmd_name, shell_cmd):
-    """Run a command.
-
-    :param str cmd_name: the user facing name of the hook being run
-    :param shell_cmd: shell command to execute
-    :type shell_cmd: `list` of `str` or `str`
-
-    :returns: `tuple` (`str` stderr, `str` stdout)"""
-    logger.info("Running %s command: %s", cmd_name, shell_cmd)
-
-    # universal_newlines causes Popen.communicate()
-    # to return str objects instead of bytes in Python 3
-    cmd = Popen(shell_cmd, shell=True, stdout=PIPE,
-                stderr=PIPE, universal_newlines=True)
-    out, err = cmd.communicate()
-    base_cmd = os.path.basename(shell_cmd.split(None, 1)[0])
-    if out:
-        logger.info('Output from %s command %s:\n%s', cmd_name, base_cmd, out)
-    if cmd.returncode != 0:
-        logger.error('%s command "%s" returned error code %d',
-                     cmd_name, shell_cmd, cmd.returncode)
-    if err:
-        logger.error('Error output from %s command %s:\n%s', cmd_name, base_cmd, err)
-    return err, out
-
-
 def list_hooks(dir_path):
     """List paths to all hooks found in dir_path in sorted order.
 

certbot/certbot/_internal/log.py

@@ -322,15 +322,23 @@ def post_arg_parse_except_hook(exc_type, exc_value, trace, debug, log_path):
         logger.error('Exiting abnormally:', exc_info=exc_info)
     else:
         logger.debug('Exiting abnormally:', exc_info=exc_info)
+        # Use logger to print the error message to take advantage of
+        # our logger printing warnings and errors in red text.
         if issubclass(exc_type, errors.Error):
-            sys.exit(exc_value)
+            logger.error(str(exc_value))
+            sys.exit(1)
         logger.error('An unexpected error occurred:')
         if messages.is_acme_error(exc_value):
             # Remove the ACME error prefix from the exception
             _, _, exc_str = str(exc_value).partition(':: ')
             logger.error(exc_str)
         else:
-            traceback.print_exception(exc_type, exc_value, None)
+            output = traceback.format_exception_only(exc_type, exc_value)
+            # format_exception_only returns a list of strings each
+            # terminated by a newline. We combine them into one string
+            # and remove the final newline before passing it to
+            # logger.error.
+            logger.error(''.join(output).rstrip())
     exit_with_log_path(log_path)
 
 

certbot/certbot/_internal/plugins/manual.py

@@ -9,6 +9,7 @@ from certbot import errors
 from certbot import interfaces
 from certbot import reverter
 from certbot._internal import hooks
+from certbot.compat import misc
 from certbot.compat import os
 from certbot.plugins import common
 
@@ -186,4 +187,4 @@ permitted by DNS standards.)
         self.reverter.recovery_routine()
 
     def _execute_hook(self, hook_name):
-        return hooks.execute(self.option_name(hook_name), self.conf(hook_name))
+        return misc.execute_command(self.option_name(hook_name), self.conf(hook_name))

certbot/certbot/_internal/storage.py

@@ -1007,18 +1007,18 @@ class RenewableCert(interfaces.RenewableCert):
         lineagename = lineagename_for_filename(config_filename)
         archive = full_archive_path(None, cli_config, lineagename)
         live_dir = _full_live_path(cli_config, lineagename)
-        if os.path.exists(archive):
+        if os.path.exists(archive) and (not os.path.isdir(archive) or os.listdir(archive)):
             config_file.close()
             raise errors.CertStorageError(
                 "archive directory exists for " + lineagename)
-        if os.path.exists(live_dir):
+        if os.path.exists(live_dir) and (not os.path.isdir(live_dir) or os.listdir(live_dir)):
             config_file.close()
             raise errors.CertStorageError(
                 "live directory exists for " + lineagename)
-        filesystem.mkdir(archive)
-        filesystem.mkdir(live_dir)
-        logger.debug("Archive directory %s and live "
-                     "directory %s created.", archive, live_dir)
+        for i in (archive, live_dir):
+            if not os.path.exists(i):
+                filesystem.makedirs(i)
+                logger.debug("Creating directory %s.", i)
 
         # Put the data into the appropriate files on disk
         target = {kind: os.path.join(live_dir, kind + ".pem") for kind in ALL_FOUR}

certbot/certbot/compat/filesystem.py

@@ -78,6 +78,35 @@ def copy_ownership_and_apply_mode(src, dst, mode, copy_user, copy_group):
     chmod(dst, mode)
 
 
+# Quite similar to copy_ownership_and_apply_mode, but this time the DACL is copied from
+# the source file on Windows. The DACL stays consistent with the dynamic rights of the
+# equivalent POSIX mode, because ownership and mode are copied altogether on the destination
+# file, so no recomputing of the DACL against the new owner is needed, as it would be
+# for a copy_ownership alone method.
+def copy_ownership_and_mode(src, dst, copy_user=True, copy_group=True):
+    # type: (str, str, bool, bool) -> None
+    """
+    Copy ownership (user and optionally group on Linux) and mode/DACL
+    from the source to the destination.
+    :param str src: Path of the source file
+    :param str dst: Path of the destination file
+    :param bool copy_user: Copy user if `True`
+    :param bool copy_group: Copy group if `True` on Linux (has no effect on Windows)
+    """
+    if POSIX_MODE:
+        # On Linux, we just delegate to chown and chmod.
+        stats = os.stat(src)
+        user_id = stats.st_uid if copy_user else -1
+        group_id = stats.st_gid if copy_group else -1
+        os.chown(dst, user_id, group_id)
+        chmod(dst, stats.st_mode)
+    else:
+        if copy_user:
+            # There is no group handling in Windows
+            _copy_win_ownership(src, dst)
+        _copy_win_mode(src, dst)
+
+
 def check_mode(file_path, mode):
     # type: (str, int) -> bool
     """
@@ -515,6 +544,9 @@ def _analyze_mode(mode):
 
 
 def _copy_win_ownership(src, dst):
+    # Resolve symbolic links
+    src = realpath(src)
+
     security_src = win32security.GetFileSecurity(src, win32security.OWNER_SECURITY_INFORMATION)
     user_src = security_src.GetSecurityDescriptorOwner()
 
@@ -526,6 +558,19 @@ def _copy_win_ownership(src, dst):
     win32security.SetFileSecurity(dst, win32security.OWNER_SECURITY_INFORMATION, security_dst)
 
 
+def _copy_win_mode(src, dst):
+    # Resolve symbolic links
+    src = realpath(src)
+
+    # Copy the DACL from src to dst.
+    security_src = win32security.GetFileSecurity(src, win32security.DACL_SECURITY_INFORMATION)
+    dacl = security_src.GetSecurityDescriptorDacl()
+
+    security_dst = win32security.GetFileSecurity(dst, win32security.DACL_SECURITY_INFORMATION)
+    security_dst.SetSecurityDescriptorDacl(1, dacl, 0)
+    win32security.SetFileSecurity(dst, win32security.DACL_SECURITY_INFORMATION, security_dst)
+
+
 def _generate_windows_flags(rights_desc):
     # Some notes about how each POSIX right is interpreted.
     #

certbot/certbot/compat/misc.py

@@ -4,12 +4,16 @@ particular category.
 """
 from __future__ import absolute_import
 
+import logging
 import select
+import subprocess
 import sys
 
 from certbot import errors
 from certbot.compat import os
 
+from acme.magic_typing import Tuple
+
 try:
     from win32com.shell import shell as shellwin32
     POSIX_MODE = False
@@ -17,6 +21,7 @@ except ImportError:  # pragma: no cover
     POSIX_MODE = True
 
 
+logger = logging.getLogger(__name__)
 
 # For Linux: define OS specific standard binary directories
 STANDARD_BINARY_DIRS = ["/usr/sbin", "/usr/local/bin", "/usr/local/sbin"] if POSIX_MODE else []
@@ -109,3 +114,39 @@ def underscores_for_unsupported_characters_in_path(path):
     # Windows specific
     drive, tail = os.path.splitdrive(path)
     return drive + tail.replace(':', '_')
+
+
+def execute_command(cmd_name, shell_cmd):
+    # type: (str, str) -> Tuple[str, str]
+    """
+    Run a command:
+        - on Linux command will be run by the standard shell selected with Popen(shell=True)
+        - on Windows command will be run in a Powershell shell
+
+    :param str cmd_name: the user facing name of the hook being run
+    :param str shell_cmd: shell command to execute
+
+    :returns: `tuple` (`str` stderr, `str` stdout)
+    """
+    logger.info("Running %s command: %s", cmd_name, shell_cmd)
+
+    if POSIX_MODE:
+        cmd = subprocess.Popen(shell_cmd, shell=True, stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE, universal_newlines=True)
+    else:
+        line = ['powershell.exe', '-Command', shell_cmd]
+        cmd = subprocess.Popen(line, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                               universal_newlines=True)
+
+    # universal_newlines causes Popen.communicate()
+    # to return str objects instead of bytes in Python 3
+    out, err = cmd.communicate()
+    base_cmd = os.path.basename(shell_cmd.split(None, 1)[0])
+    if out:
+        logger.info('Output from %s command %s:\n%s', cmd_name, base_cmd, out)
+    if cmd.returncode != 0:
+        logger.error('%s command "%s" returned error code %d',
+                     cmd_name, shell_cmd, cmd.returncode)
+    if err:
+        logger.error('Error output from %s command %s:\n%s', cmd_name, base_cmd, err)
+    return err, out

certbot/certbot/display/ops.py

@@ -30,7 +30,7 @@ def get_email(invalid=False, optional=True):
 
     """
     invalid_prefix = "There seem to be problems with that address. "
-    msg = "Enter email address (used for urgent renewal and security notices)"
+    msg = "Enter email address (used for urgent renewal and security notices)\n"
     unsafe_suggestion = ("\n\nIf you really want to skip this, you can run "
                          "the client with --register-unsafely-without-email "
                          "but make sure you then backup your account key from "
@@ -64,7 +64,7 @@ def get_email(invalid=False, optional=True):
         if util.safe_email(email):
             return email
         if suggest_unsafe:
-            msg += unsafe_suggestion
+            msg = unsafe_suggestion + msg
             suggest_unsafe = False  # add this message at most once
 
         invalid = bool(email)

certbot/certbot/ocsp.py

@@ -256,7 +256,11 @@ def _check_ocsp_response(response_ocsp, request_ocsp, issuer_cert, cert_path):
 
 def _check_ocsp_response_signature(response_ocsp, issuer_cert, cert_path):
     """Verify an OCSP response signature against certificate issuer or responder"""
-    if response_ocsp.responder_name == issuer_cert.subject:
+    def _key_hash(cert):
+        return x509.SubjectKeyIdentifier.from_public_key(cert.public_key()).digest
+
+    if response_ocsp.responder_name == issuer_cert.subject or \
+       response_ocsp.responder_key_hash == _key_hash(issuer_cert):
         # Case where the OCSP responder is also the certificate issuer
         logger.debug('OCSP response for certificate %s is signed by the certificate\'s issuer.',
                      cert_path)
@@ -267,7 +271,8 @@ def _check_ocsp_response_signature(response_ocsp, issuer_cert, cert_path):
                      cert_path)
 
         responder_certs = [cert for cert in response_ocsp.certificates
-                           if cert.subject == response_ocsp.responder_name]
+                           if response_ocsp.responder_name == cert.subject or \
+                              response_ocsp.responder_key_hash == _key_hash(cert)]
         if not responder_certs:
             raise AssertionError('no matching responder certificate could be found')
 

certbot/docs/cli-help.txt

@@ -113,7 +113,7 @@ optional arguments:
                         case, and to know when to deprecate support for past
                         Python versions and flags. If you wish to hide this
                         information from the Let's Encrypt server, set this to
-                        "". (default: CertbotACMEClient/1.3.0 (certbot(-auto);
+                        "". (default: CertbotACMEClient/1.4.0 (certbot(-auto);
                         OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY
                         (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel).
                         The flags encoded in the user agent are: --duplicate,
@@ -188,10 +188,12 @@ security:
                         supported setups (Apache version >= 2.3.3 ). (default:
                         False)
   --redirect            Automatically redirect all HTTP traffic to HTTPS for
-                        the newly authenticated vhost. (default: Ask)
+                        the newly authenticated vhost. (default: redirect
+                        enabled for install and run, disabled for enhance)
   --no-redirect         Do not automatically redirect all HTTP traffic to
                         HTTPS for the newly authenticated vhost. (default:
-                        Ask)
+                        redirect enabled for install and run, disabled for
+                        enhance)
   --hsts                Add the Strict-Transport-Security header to every HTTP
                         response. Forcing browser to always use SSL for the
                         domain. Defends against SSL Stripping. (default: None)
@@ -213,8 +215,8 @@ testing:
 
   --test-cert, --staging
                         Use the staging server to obtain or revoke test
-                        (invalid) certificates; equivalent to --server https
-                        ://acme-staging-v02.api.letsencrypt.org/directory
+                        (invalid) certificates; equivalent to --server
+                        https://acme-staging-v02.api.letsencrypt.org/directory
                         (default: False)
   --debug               Show tracebacks in case of errors, and allow certbot-
                         auto execution on experimental platforms (default:
@@ -319,8 +321,8 @@ renew:
                         of renewed certificate domains (for example,
                         "example.com www.example.com" (default: None)
   --disable-hook-validation
-                        Ordinarily the commands specified for --pre-hook
-                        /--post-hook/--deploy-hook will be checked for
+                        Ordinarily the commands specified for --pre-
+                        hook/--post-hook/--deploy-hook will be checked for
                         validity, to see if the programs being run are in the
                         $PATH, so that mistakes can be caught early, even when
                         the hooks aren't being run just yet. The validation is
@@ -669,7 +671,11 @@ manual:
   requested when performing an HTTP-01 challenge. An additional cleanup
   script can also be provided and can use the additional variable
   $CERTBOT_AUTH_OUTPUT which contains the stdout output from the auth
-  script.
+  script.For both authenticator and cleanup script, on HTTP-01 and DNS-01
+  challenges,$CERTBOT_REMAINING_CHALLENGES will be equal to the number of
+  challenges that remain after the current one, and $CERTBOT_ALL_DOMAINS
+  contains a comma-separated list of all domains that are challenged for the
+  current certificate.
 
   --manual-auth-hook MANUAL_AUTH_HOOK
                         Path or command to execute for the authentication

certbot/docs/contributing.rst

@@ -117,13 +117,11 @@ either in the same directory as ``foo.py`` or in the ``tests`` subdirectory
 For debugging, we recommend putting
 ``import ipdb; ipdb.set_trace()`` statements inside the source code.
 
-Once you are done with your code changes, and the tests in ``foo_test.py`` pass,
-run all of the unittests for Certbot with ``tox -e py27`` (this uses Python
-2.7).
-
-Once all the unittests pass, check for sufficient test coverage using ``tox -e
-py27-cover``, and then check for code style with ``tox -e lint`` (all files) or
-``pylint --rcfile=.pylintrc path/to/file.py`` (single file at a time).
+Once you are done with your code changes, and the tests in ``foo_test.py``
+pass, run all of the unit tests for Certbot and check for coverage with ``tox
+-e py3-cover``. You should then check for code style with ``tox -e lint`` (all
+files) or ``pylint --rcfile=.pylintrc path/to/file.py`` (single file at a
+time).
 
 Once all of the above is successful, you may run the full test suite using
 ``tox --skip-missing-interpreters``. We recommend running the commands above
@@ -170,7 +168,7 @@ To do so you need:
 - Docker installed, and a user with access to the Docker client,
 - an available `local copy`_ of Certbot.
 
-The virtual environment set up with `python tools/venv.py` contains two commands
+The virtual environment set up with `python tools/venv3.py` contains two commands
 that can be used once the virtual environment is activated:
 
 .. code-block:: shell

certbot/docs/install.rst

@@ -61,6 +61,23 @@ Alternate installation methods
 If you are offline or your operating system doesn't provide a package, you can use
 an alternate method for installing ``certbot``.
 
+.. _snap-install:
+
+Snap
+----
+
+Most modern Linux distributions (basically any that use systemd) can install
+Certbot packaged as a snap. Support for the Certbot snap is currently in its
+beta phase and limited to the x86_64 architecture, but it provides an easy way
+to ensure you have the latest version of Certbot with features like automated
+certificate renewal preconfigured.
+
+You can find instructions for installing the Certbot snap at
+https://certbot.eff.org/instructions by selecting your server software and then
+choosing "snapd" in the "System" dropdown menu. (You should select "snapd"
+regardless of your operating system, as our instructions are the same across
+all systems.)
+
 .. _certbot-auto:
 
 Certbot-Auto

certbot/docs/using.rst

@@ -385,7 +385,7 @@ certificate exists alongside any previously obtained certificates, whether
 or not the previous certificates have expired. The generation of a new
 certificate counts against several rate limits that are intended to prevent
 abuse of the ACME protocol, as described
-`here <https://community.letsencrypt.org/t/rate-limits-for-lets-encrypt/6769>`__.
+`here <https://letsencrypt.org/docs/rate-limits/>`__.
 
 .. _changing:
 
@@ -846,17 +846,15 @@ Example usage for DNS-01 (Cloudflare API v4) (for example purposes only, do not
 Changing the ACME Server
 ========================
 
-By default, Certbot uses Let's Encrypt's initial production server at
-https://acme-v01.api.letsencrypt.org/. You can tell Certbot to use a
+By default, Certbot uses Let's Encrypt's production server at
+https://acme-v02.api.letsencrypt.org/. You can tell Certbot to use a
 different CA by providing ``--server`` on the command line or in a
 :ref:`configuration file <config-file>` with the URL of the server's
 ACME directory. For example, if you would like to use Let's Encrypt's
-new ACMEv2 server, you would add ``--server
-https://acme-v02.api.letsencrypt.org/directory`` to the command line.
-Certbot will automatically select which version of the ACME protocol to
-use based on the contents served at the provided URL.
+staging server, you would add ``--server
+https://acme-staging-v02.api.letsencrypt.org/directory`` to the command line.
 
-If you use ``--server`` to specify an ACME CA that implements a newer
+If you use ``--server`` to specify an ACME CA that implements the standardized
 version of the spec, you may be able to obtain a certificate for a
 wildcard domain. Some CAs (such as Let's Encrypt) require that domain
 validation for wildcard domains must be done through modifications to

certbot/local-oldest-requirements.txt

@@ -1,2 +1,2 @@
 # Remember to update setup.py to match the package versions below.
--e acme[dev]
+acme[dev]==1.4.0

certbot/setup.py

@@ -36,7 +36,7 @@ version = meta['version']
 # specified here to avoid masking the more specific request requirements in
 # acme. See https://github.com/pypa/pip/issues/988 for more info.
 install_requires = [
-    'acme>=1.4.0.dev0',
+    'acme>=1.4.0',
     # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but
     # saying so here causes a runtime error against our temporary fork of 0.9.3
     # in which we added 2.6 support (see #2243), so we relax the requirement.

certbot/tests/compat/filesystem_test.py

@@ -280,14 +280,14 @@ class WindowsMkdirTests(test_util.TempDirTestCase):
         self.assertEqual(original_mkdir, std_os.mkdir)
 
 
-class OwnershipTest(test_util.TempDirTestCase):
-    """Tests about copy_ownership_and_apply_mode and has_same_ownership"""
+class CopyOwnershipAndModeTest(test_util.TempDirTestCase):
+    """Tests about copy_ownership_and_apply_mode, copy_ownership_and_mode and has_same_ownership"""
     def setUp(self):
-        super(OwnershipTest, self).setUp()
+        super(CopyOwnershipAndModeTest, self).setUp()
         self.probe_path = _create_probe(self.tempdir)
 
     @unittest.skipIf(POSIX_MODE, reason='Test specific to Windows security')
-    def test_copy_ownership_windows(self):
+    def test_copy_ownership_and_apply_mode_windows(self):
         system = win32security.ConvertStringSidToSid(SYSTEM_SID)
         security = win32security.SECURITY_ATTRIBUTES().SECURITY_DESCRIPTOR
         security.SetSecurityDescriptorOwner(system, False)
@@ -313,7 +313,7 @@ class OwnershipTest(test_util.TempDirTestCase):
                           if dacl.GetAce(index)[2] == everybody])
 
     @unittest.skipUnless(POSIX_MODE, reason='Test specific to Linux security')
-    def test_copy_ownership_linux(self):
+    def test_copy_ownership_and_apply_mode_linux(self):
         with mock.patch('os.chown') as mock_chown:
             with mock.patch('os.chmod') as mock_chmod:
                 with mock.patch('os.stat') as mock_stat:
@@ -334,6 +334,24 @@ class OwnershipTest(test_util.TempDirTestCase):
 
         self.assertTrue(filesystem.has_same_ownership(path1, path2))
 
+    @unittest.skipIf(POSIX_MODE, reason='Test specific to Windows security')
+    def test_copy_ownership_and_mode_windows(self):
+        src = self.probe_path
+        dst = _create_probe(self.tempdir, name='dst')
+
+        filesystem.chmod(src, 0o700)
+        self.assertTrue(filesystem.check_mode(src, 0o700))
+        self.assertTrue(filesystem.check_mode(dst, 0o744))
+
+        # Checking an actual change of owner is tricky during a unit test, since we do not know
+        # if any user exists beside the current one. So we mock _copy_win_ownership. It's behavior
+        # have been checked theoretically with test_copy_ownership_and_apply_mode_windows.
+        with mock.patch('certbot.compat.filesystem._copy_win_ownership') as mock_copy_owner:
+            filesystem.copy_ownership_and_mode(src, dst)
+
+        mock_copy_owner.assert_called_once_with(src, dst)
+        self.assertTrue(filesystem.check_mode(dst, 0o700))
+
 
 class CheckPermissionsTest(test_util.TempDirTestCase):
     """Tests relative to functions that check modes."""
@@ -537,9 +555,9 @@ def _set_owner(target, security_owner, user):
         target, win32security.OWNER_SECURITY_INFORMATION, security_owner)
 
 
-def _create_probe(tempdir):
+def _create_probe(tempdir, name='probe'):
     filesystem.chmod(tempdir, 0o744)
-    probe_path = os.path.join(tempdir, 'probe')
+    probe_path = os.path.join(tempdir, name)
     util.safe_open(probe_path, 'w', chmod=0o744).close()
     return probe_path
 

certbot/tests/compat/misc_test.py

@@ -0,0 +1,48 @@
+"""Tests for certbot.compat.misc"""
+try:
+    import mock
+except ImportError:  # pragma: no cover
+    from unittest import mock  # type: ignore
+import unittest
+
+from certbot.compat import os
+
+
+class ExecuteTest(unittest.TestCase):
+    """Tests for certbot.compat.misc.execute_command."""
+
+    @classmethod
+    def _call(cls, *args, **kwargs):
+        from certbot.compat.misc import execute_command
+        return execute_command(*args, **kwargs)
+
+    def test_it(self):
+        for returncode in range(0, 2):
+            for stdout in ("", "Hello World!",):
+                for stderr in ("", "Goodbye Cruel World!"):
+                    self._test_common(returncode, stdout, stderr)
+
+    def _test_common(self, returncode, stdout, stderr):
+        given_command = "foo"
+        given_name = "foo-hook"
+        with mock.patch("certbot.compat.misc.subprocess.Popen") as mock_popen:
+            mock_popen.return_value.communicate.return_value = (stdout, stderr)
+            mock_popen.return_value.returncode = returncode
+            with mock.patch("certbot.compat.misc.logger") as mock_logger:
+                self.assertEqual(self._call(given_name, given_command), (stderr, stdout))
+
+        executed_command = mock_popen.call_args[1].get(
+            "args", mock_popen.call_args[0][0])
+        if os.name == 'nt':
+            expected_command = ['powershell.exe', '-Command', given_command]
+        else:
+            expected_command = given_command
+        self.assertEqual(executed_command, expected_command)
+
+        mock_logger.info.assert_any_call("Running %s command: %s",
+                                         given_name, given_command)
+        if stdout:
+            mock_logger.info.assert_any_call(mock.ANY, mock.ANY,
+                                             mock.ANY, stdout)
+        if stderr or returncode:
+            self.assertTrue(mock_logger.error.called)

certbot/tests/hook_test.py

@@ -72,13 +72,13 @@ class HookTest(test_util.ConfigTestCase):
 
     @classmethod
     def _call_with_mock_execute(cls, *args, **kwargs):
-        """Calls self._call after mocking out certbot._internal.hooks.execute.
+        """Calls self._call after mocking out certbot.compat.misc.execute_command.
 
         The mock execute object is returned rather than the return value
         of self._call.
 
         """
-        with mock.patch("certbot._internal.hooks.execute") as mock_execute:
+        with mock.patch("certbot.compat.misc.execute_command") as mock_execute:
             mock_execute.return_value = ("", "")
             cls._call(*args, **kwargs)
         return mock_execute
@@ -292,7 +292,7 @@ class RenewalHookTest(HookTest):
     # pylint: disable=abstract-method
 
     def _call_with_mock_execute(self, *args, **kwargs):
-        """Calls self._call after mocking out certbot._internal.hooks.execute.
+        """Calls self._call after mocking out certbot.compat.misc.execute_command.
 
         The mock execute object is returned rather than the return value
         of self._call. The mock execute object asserts that environment
@@ -313,7 +313,7 @@ class RenewalHookTest(HookTest):
             self.assertEqual(os.environ["RENEWED_LINEAGE"], lineage)
             return ("", "")
 
-        with mock.patch("certbot._internal.hooks.execute") as mock_execute:
+        with mock.patch("certbot.compat.misc.execute_command") as mock_execute:
             mock_execute.side_effect = execute_side_effect
             self._call(*args, **kwargs)
         return mock_execute
@@ -418,42 +418,6 @@ class RenewHookTest(RenewalHookTest):
         mock_execute.assert_called_with("deploy-hook", self.config.renew_hook)
 
 
-class ExecuteTest(unittest.TestCase):
-    """Tests for certbot._internal.hooks.execute."""
-
-    @classmethod
-    def _call(cls, *args, **kwargs):
-        from certbot._internal.hooks import execute
-        return execute(*args, **kwargs)
-
-    def test_it(self):
-        for returncode in range(0, 2):
-            for stdout in ("", "Hello World!",):
-                for stderr in ("", "Goodbye Cruel World!"):
-                    self._test_common(returncode, stdout, stderr)
-
-    def _test_common(self, returncode, stdout, stderr):
-        given_command = "foo"
-        given_name = "foo-hook"
-        with mock.patch("certbot._internal.hooks.Popen") as mock_popen:
-            mock_popen.return_value.communicate.return_value = (stdout, stderr)
-            mock_popen.return_value.returncode = returncode
-            with mock.patch("certbot._internal.hooks.logger") as mock_logger:
-                self.assertEqual(self._call(given_name, given_command), (stderr, stdout))
-
-        executed_command = mock_popen.call_args[1].get(
-            "args", mock_popen.call_args[0][0])
-        self.assertEqual(executed_command, given_command)
-
-        mock_logger.info.assert_any_call("Running %s command: %s",
-                                         given_name, given_command)
-        if stdout:
-            mock_logger.info.assert_any_call(mock.ANY, mock.ANY,
-                                             mock.ANY, stdout)
-        if stderr or returncode:
-            self.assertTrue(mock_logger.error.called)
-
-
 class ListHooksTest(test_util.TempDirTestCase):
     """Tests for certbot._internal.hooks.list_hooks."""
 

certbot/tests/ocsp_test.py

@@ -182,13 +182,23 @@ class OSCPTestCryptography(unittest.TestCase):
 
         with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED,
                         ocsp_lib.OCSPResponseStatus.SUCCESSFUL) as mocks:
+            # OCSP response with ResponseID as Name
             mocks['mock_response'].return_value.responder_name = issuer.subject
+            mocks['mock_response'].return_value.responder_key_hash = None
             self.checker.ocsp_revoked(self.cert_obj)
+            # OCSP response with ResponseID as KeyHash
+            key_hash = x509.SubjectKeyIdentifier.from_public_key(issuer.public_key()).digest
+            mocks['mock_response'].return_value.responder_name = None
+            mocks['mock_response'].return_value.responder_key_hash = key_hash
+            self.checker.ocsp_revoked(self.cert_obj)
+
         # Here responder and issuer are the same. So only the signature of the OCSP
         # response is checked (using the issuer/responder public key).
-        self.assertEqual(mocks['mock_check'].call_count, 1)
-        self.assertEqual(mocks['mock_check'].call_args[0][0].public_numbers(),
-                         issuer.public_key().public_numbers())
+        self.assertEqual(mocks['mock_check'].call_count, 2)
+        self.assertEqual(mocks['mock_check'].call_args_list[0][0][0].public_numbers(),
+            issuer.public_key().public_numbers())
+        self.assertEqual(mocks['mock_check'].call_args_list[1][0][0].public_numbers(),
+            issuer.public_key().public_numbers())
 
     def test_responder_is_authorized_delegate(self):
         issuer = x509.load_pem_x509_certificate(
@@ -198,15 +208,28 @@ class OSCPTestCryptography(unittest.TestCase):
 
         with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED,
                         ocsp_lib.OCSPResponseStatus.SUCCESSFUL) as mocks:
+            # OCSP response with ResponseID as Name
+            mocks['mock_response'].return_value.responder_name = responder.subject
+            mocks['mock_response'].return_value.responder_key_hash = None
             self.checker.ocsp_revoked(self.cert_obj)
+            # OCSP response with ResponseID as KeyHash
+            key_hash = x509.SubjectKeyIdentifier.from_public_key(responder.public_key()).digest
+            mocks['mock_response'].return_value.responder_name = None
+            mocks['mock_response'].return_value.responder_key_hash = key_hash
+            self.checker.ocsp_revoked(self.cert_obj)
+
         # Here responder and issuer are not the same. Two signatures will be checked then,
         # first to verify the responder cert (using the issuer public key), second to
         # to verify the OCSP response itself (using the responder public key).
-        self.assertEqual(mocks['mock_check'].call_count, 2)
+        self.assertEqual(mocks['mock_check'].call_count, 4)
         self.assertEqual(mocks['mock_check'].call_args_list[0][0][0].public_numbers(),
                          issuer.public_key().public_numbers())
         self.assertEqual(mocks['mock_check'].call_args_list[1][0][0].public_numbers(),
                          responder.public_key().public_numbers())
+        self.assertEqual(mocks['mock_check'].call_args_list[2][0][0].public_numbers(),
+                         issuer.public_key().public_numbers())
+        self.assertEqual(mocks['mock_check'].call_args_list[3][0][0].public_numbers(),
+                         responder.public_key().public_numbers())
 
     def test_revoke_resiliency(self):
         # Server return an invalid HTTP response

certbot/tests/storage_test.py

@@ -610,17 +610,25 @@ class RenewableCertTests(BaseRenewableCertTest):
             self.config.renewal_configs_dir, "the-lineage.com-0001.conf")))
         self.assertTrue(os.path.exists(os.path.join(
             self.config.live_dir, "the-lineage.com-0001", "README")))
+        # Allow write to existing but empty dir
+        filesystem.mkdir(os.path.join(self.config.default_archive_dir, "the-lineage.com-0002"))
+        result = storage.RenewableCert.new_lineage(
+            "the-lineage.com", b"cert3", b"privkey3", b"chain3", self.config)
+        self.assertTrue(os.path.exists(os.path.join(
+            self.config.live_dir, "the-lineage.com-0002", "README")))
+        self.assertTrue(filesystem.check_mode(result.key_path, 0o600))
         # Now trigger the detection of already existing files
-        filesystem.mkdir(os.path.join(
-            self.config.live_dir, "the-lineage.com-0002"))
+        shutil.copytree(os.path.join(self.config.live_dir, "the-lineage.com"),
+                        os.path.join(self.config.live_dir, "the-lineage.com-0003"))
         self.assertRaises(errors.CertStorageError,
                           storage.RenewableCert.new_lineage, "the-lineage.com",
-                          b"cert3", b"privkey3", b"chain3", self.config)
-        filesystem.mkdir(os.path.join(self.config.default_archive_dir, "other-example.com"))
+                          b"cert4", b"privkey4", b"chain4", self.config)
+        shutil.copytree(os.path.join(self.config.live_dir, "the-lineage.com"),
+                        os.path.join(self.config.live_dir, "other-example.com"))
         self.assertRaises(errors.CertStorageError,
                           storage.RenewableCert.new_lineage,
-                          "other-example.com", b"cert4",
-                          b"privkey4", b"chain4", self.config)
+                          "other-example.com", b"cert5",
+                          b"privkey5", b"chain5", self.config)
         # Make sure it can accept renewal parameters
         result = storage.RenewableCert.new_lineage(
             "the-lineage.com", b"cert2", b"privkey2", b"chain2", self.config)

letsencrypt-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="1.3.0"
+LE_AUTO_VERSION="1.4.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -910,20 +910,11 @@ elif [ -f /etc/manjaro-release ]; then
   }
   BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION"
 elif [ -f /etc/gentoo-release ]; then
-  Bootstrap() {
-    DeprecationBootstrap "Gentoo" BootstrapGentooCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapGentooCommon $BOOTSTRAP_GENTOO_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif uname | grep -iq FreeBSD ; then
-  Bootstrap() {
-    DeprecationBootstrap "FreeBSD" BootstrapFreeBsd
-  }
-  BOOTSTRAP_VERSION="BootstrapFreeBsd $BOOTSTRAP_FREEBSD_VERSION"
+  DEPRECATED_OS=1
 elif uname | grep -iq Darwin ; then
-  Bootstrap() {
-    DeprecationBootstrap "macOS" BootstrapMac
-  }
-  BOOTSTRAP_VERSION="BootstrapMac $BOOTSTRAP_MAC_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then
   Bootstrap() {
     ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon
@@ -1540,18 +1531,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==1.3.0 \
-    --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \
-    --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8
-acme==1.3.0 \
-    --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \
-    --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d
-certbot-apache==1.3.0 \
-    --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \
-    --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52
-certbot-nginx==1.3.0 \
-    --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \
-    --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b
+certbot==1.4.0 \
+    --hash=sha256:5f8be1e6087d2f1f742caf0048b0f46bac8d3a655d038d5355abd1638523d87e \
+    --hash=sha256:69b5b7925de0d3b693b00a40bf109d85afb24c7199bf616339d74d59a80d8d96
+acme==1.4.0 \
+    --hash=sha256:d2f6799f7fce2414fc1a6753ced91c0ccdf1d0b2cee892c509851db45402fb5b \
+    --hash=sha256:f12cb59762e0b833911b87e95cb16e85a162517ba4aa3440594bdf3b8126fc69
+certbot-apache==1.4.0 \
+    --hash=sha256:1be1a38cb73e950c5cbff941719d326bfd2d1b4fff17b39b7a27377067cd90a6 \
+    --hash=sha256:6067f537deb7f70b979d11ed19846712dbf5c484ca927841805e78d8797b4640
+certbot-nginx==1.4.0 \
+    --hash=sha256:8ee1c7201b40bde7d476894fb06bf8ab0c0cd0ba03c0510bc568e8713e801ccc \
+    --hash=sha256:44a9f74dee7e2f8a32aafaf793280e8fcd4d50a9ffb7c5ed47a0bc591ce6ecca
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------

letsencrypt-auto-source/certbot-auto.asc

@@ -1,11 +1,11 @@
 -----BEGIN PGP SIGNATURE-----
 
-iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl5ewVUACgkQTRfJlc2X
-dfJnZAf+KmxYl1YoP/FlTG5Npb64qaDdxm59SeEVJez6fZh15xq71tRPYR+4xszE
-XTeyGt7uAxjYqeiBJU5xBvGC1Veprhj5AbflVOTP+5yiBr9iNWC35zmgaE63UlZ/
-V94sfL0pkax7wLngil7a0OuzUjikzK3gXOqrY8LoUdr4mAA9AhSjajWHmyY3tpDR
-84GKrVhybIt0sjy/172VuPPbXZKno/clztkKMZHXNrDeL5jgJ15Va4Ts5FK0j9VT
-HQvuazbGkYVCuvlp8Np5ESDje69LCJfPZxl34htoa8WNJoVIOsQWZpoXp5B5huSP
-vGrh4LabZ5UDsl+k11ikHBRUpO7E5w==
-=IgRH
+iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl6x0CQACgkQTRfJlc2X
+dfJMVwf/dQ+Ap/TvIKVkdeIDcAeqycyBpK7CvGGMfkBVD1FPOXvLAaGRu8jpbtKB
+HE2SlCiRW5g+o0iD2zJx+6EMm0hDo64/jK7X7AE04Vz5yolhPujrbxqSMF2CZXZX
+vh9qfzRU+05kjYmOElP/JZxAE3mZyPPK04Ii6gseIjU8NEaGinQQm3oFBDqnaZq6
+DMGqvczaT3kTt8Rr3r2/9XQzr8aF+zpBAteAg7ou31b8nK/hugiX1gfdQL3xF7Gu
+sRPyU14vZeVvoU8n0G0pSWdV//0eV8KmctbQJaU8amrnrFJubM+PKbsRWGSwMtu3
+5PA9aZbXDAB5iXm4huA8sK8IU76FLg==
+=eUK2
 -----END PGP SIGNATURE-----

letsencrypt-auto-source/letsencrypt-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="1.4.0.dev0"
+LE_AUTO_VERSION="1.5.0.dev0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -910,20 +910,11 @@ elif [ -f /etc/manjaro-release ]; then
   }
   BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION"
 elif [ -f /etc/gentoo-release ]; then
-  Bootstrap() {
-    DeprecationBootstrap "Gentoo" BootstrapGentooCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapGentooCommon $BOOTSTRAP_GENTOO_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif uname | grep -iq FreeBSD ; then
-  Bootstrap() {
-    DeprecationBootstrap "FreeBSD" BootstrapFreeBsd
-  }
-  BOOTSTRAP_VERSION="BootstrapFreeBsd $BOOTSTRAP_FREEBSD_VERSION"
+  DEPRECATED_OS=1
 elif uname | grep -iq Darwin ; then
-  Bootstrap() {
-    DeprecationBootstrap "macOS" BootstrapMac
-  }
-  BOOTSTRAP_VERSION="BootstrapMac $BOOTSTRAP_MAC_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then
   Bootstrap() {
     ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon
@@ -1540,18 +1531,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==1.3.0 \
-    --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \
-    --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8
-acme==1.3.0 \
-    --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \
-    --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d
-certbot-apache==1.3.0 \
-    --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \
-    --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52
-certbot-nginx==1.3.0 \
-    --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \
-    --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b
+certbot==1.4.0 \
+    --hash=sha256:5f8be1e6087d2f1f742caf0048b0f46bac8d3a655d038d5355abd1638523d87e \
+    --hash=sha256:69b5b7925de0d3b693b00a40bf109d85afb24c7199bf616339d74d59a80d8d96
+acme==1.4.0 \
+    --hash=sha256:d2f6799f7fce2414fc1a6753ced91c0ccdf1d0b2cee892c509851db45402fb5b \
+    --hash=sha256:f12cb59762e0b833911b87e95cb16e85a162517ba4aa3440594bdf3b8126fc69
+certbot-apache==1.4.0 \
+    --hash=sha256:1be1a38cb73e950c5cbff941719d326bfd2d1b4fff17b39b7a27377067cd90a6 \
+    --hash=sha256:6067f537deb7f70b979d11ed19846712dbf5c484ca927841805e78d8797b4640
+certbot-nginx==1.4.0 \
+    --hash=sha256:8ee1c7201b40bde7d476894fb06bf8ab0c0cd0ba03c0510bc568e8713e801ccc \
+    --hash=sha256:44a9f74dee7e2f8a32aafaf793280e8fcd4d50a9ffb7c5ed47a0bc591ce6ecca
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------

letsencrypt-auto-source/letsencrypt-auto.template

@@ -432,20 +432,11 @@ elif [ -f /etc/manjaro-release ]; then
   }
   BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION"
 elif [ -f /etc/gentoo-release ]; then
-  Bootstrap() {
-    DeprecationBootstrap "Gentoo" BootstrapGentooCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapGentooCommon $BOOTSTRAP_GENTOO_COMMON_VERSION"
+  DEPRECATED_OS=1
 elif uname | grep -iq FreeBSD ; then
-  Bootstrap() {
-    DeprecationBootstrap "FreeBSD" BootstrapFreeBsd
-  }
-  BOOTSTRAP_VERSION="BootstrapFreeBsd $BOOTSTRAP_FREEBSD_VERSION"
+  DEPRECATED_OS=1
 elif uname | grep -iq Darwin ; then
-  Bootstrap() {
-    DeprecationBootstrap "macOS" BootstrapMac
-  }
-  BOOTSTRAP_VERSION="BootstrapMac $BOOTSTRAP_MAC_VERSION"
+  DEPRECATED_OS=1
 elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then
   Bootstrap() {
     ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon

letsencrypt-auto-source/pieces/certbot-requirements.txt

@@ -1,12 +1,12 @@
-certbot==1.3.0 \
-    --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \
-    --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8
-acme==1.3.0 \
-    --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \
-    --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d
-certbot-apache==1.3.0 \
-    --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \
-    --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52
-certbot-nginx==1.3.0 \
-    --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \
-    --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b
+certbot==1.4.0 \
+    --hash=sha256:5f8be1e6087d2f1f742caf0048b0f46bac8d3a655d038d5355abd1638523d87e \
+    --hash=sha256:69b5b7925de0d3b693b00a40bf109d85afb24c7199bf616339d74d59a80d8d96
+acme==1.4.0 \
+    --hash=sha256:d2f6799f7fce2414fc1a6753ced91c0ccdf1d0b2cee892c509851db45402fb5b \
+    --hash=sha256:f12cb59762e0b833911b87e95cb16e85a162517ba4aa3440594bdf3b8126fc69
+certbot-apache==1.4.0 \
+    --hash=sha256:1be1a38cb73e950c5cbff941719d326bfd2d1b4fff17b39b7a27377067cd90a6 \
+    --hash=sha256:6067f537deb7f70b979d11ed19846712dbf5c484ca927841805e78d8797b4640
+certbot-nginx==1.4.0 \
+    --hash=sha256:8ee1c7201b40bde7d476894fb06bf8ab0c0cd0ba03c0510bc568e8713e801ccc \
+    --hash=sha256:44a9f74dee7e2f8a32aafaf793280e8fcd4d50a9ffb7c5ed47a0bc591ce6ecca

pytest.ini

@@ -4,6 +4,13 @@
 [pytest]
 # In general, all warnings are treated as errors. Here are the exceptions:
 #   1- decodestring: https://github.com/rthalley/dnspython/issues/338
+# Warnings being triggered by our plugins using deprecated features in
+# acme/certbot should be fixed by having our plugins no longer using the
+# deprecated code rather than adding them to the list of ignored warnings here.
+# Fixing things in this way prevents us from shipping packages raising our own
+# deprecation warnings and gives time for plugins that don't use the deprecated
+# API to propagate, especially for plugins packaged as an external snap, before
+# we release breaking changes.
 filterwarnings =
     error
     ignore:decodestring:DeprecationWarning

snap/local/build_and_install.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -ex
+
+if [[ -z "$TRAVIS" ]]; then
+    echo "This script makes global changes to the system it is run on so should only be run in CI."
+    exit 1
+fi
+
+sudo /snap/bin/lxd.migrate -yes
+sudo /snap/bin/lxd waitready
+sudo /snap/bin/lxd init --auto
+tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > constraints.txt
+sudo snapcraft --use-lxd
+sudo snap install --dangerous --classic *.snap

snap/snapcraft.yaml

@@ -0,0 +1,93 @@
+name: certbot
+summary: Automatically configure HTTPS using Let's Encrypt
+description: |
+ The objective of Certbot, Let's Encrypt, and the ACME (Automated
+ Certificate Management Environment) protocol is to make it possible
+ to set up an HTTPS server and have it automatically obtain a
+ browser-trusted certificate, without any human intervention. This is
+ accomplished by running a certificate management agent on the web
+ server.
+ 
+ This agent is used to:
+   - Automatically prove to the Let's Encrypt CA that you control the website
+   - Obtain a browser-trusted certificate and set it up on your web server
+   - Keep track of when your certificate is going to expire, and renew it
+   - Help you revoke the certificate if that ever becomes necessary.
+confinement: classic
+grade: devel
+base: core18
+adopt-info: certbot
+
+apps:
+  certbot:
+    command: certbot
+    environment:
+      PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
+      AUGEAS_LENS_LIB: "$SNAP/usr/share/augeas/lenses/dist"
+      LD_LIBRARY_PATH: "$SNAP/usr/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH"
+  renew:
+    command: certbot -q renew
+    daemon: oneshot
+    environment:
+      PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
+      AUGEAS_LENS_LIB: $SNAP/usr/share/augeas/lenses/dist
+      LD_LIBRARY_PATH: "$SNAP/usr/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH"
+    # Run approximately twice a day with randomization
+    timer: 00:00~24:00/2
+
+parts:
+  python-augeas:
+    plugin: python
+    source: git://github.com/basak/python-augeas
+    source-branch: snap
+    python-version: python3
+    build-packages: [libaugeas-dev]
+  acme:
+    plugin: python
+    source: .
+    source-subdir: acme
+    constraints: [$SNAPCRAFT_PART_SRC/constraints.txt]
+    python-version: python3
+  certbot:
+    plugin: python
+    source: .
+    source-subdir: certbot
+    constraints: [$SNAPCRAFT_PART_SRC/constraints.txt]
+    python-version: python3
+    after: [acme]
+    override-pull: |
+        snapcraftctl pull
+        snapcraftctl set-version `cd $SNAPCRAFT_PART_SRC && git describe|sed s/^v//`
+    # Workaround for lack of site-packages leading to empty sitecustomize.py
+    stage:
+      - -usr/lib/python3.6/sitecustomize.py
+  certbot-apache:
+    plugin: python
+    source: .
+    source-subdir: certbot-apache
+    constraints: [$SNAPCRAFT_PART_SRC/constraints.txt]
+    python-version: python3
+    after: [python-augeas, certbot]
+    stage-packages: [libaugeas0]
+    stage:
+      # Prefer cffi
+      - -lib/python3.6/site-packages/augeas.py
+  certbot-nginx:
+    plugin: python
+    source: .
+    source-subdir: certbot-nginx
+    constraints: [$SNAPCRAFT_PART_SRC/constraints.txt]
+    python-version: python3
+    # This is the last step, compile pycache now as there should be no conflicts.
+    override-prime: |
+      snapcraftctl prime
+      ./usr/bin/python3 -m compileall -q .
+    # After certbot-apache to not rebuild duplicates (essentially sharing what was already staged,
+    # like zope)
+    after: [certbot-apache]
+
+plugs:
+  plugin:
+    interface: content
+    content: certbot-1
+    target: $SNAP/certbot-plugin

tests/letstest/README.md

@@ -15,9 +15,10 @@ Simple AWS testfarm scripts for certbot client testing
     are needed, they need to be requested via online webform.
 
 ## Installation and configuration
-These tests require Python 3, awscli, boto3, PyYAML, and fabric 2.0+. If you
-have Python 3 installed, you can use requirements.txt to create a virtual
-environment with a known set of dependencies by running:
+These tests require Python 3, awscli, boto3, PyYAML, and fabric 2.0+. If you're
+on a Debian based system, make sure you also have the python3-venv package
+installed. If you have Python 3 installed, you can use requirements.txt to
+create a virtual environment with a known set of dependencies by running:
 ```
 python3 -m venv venv3
 . ./venv3/bin/activate

tests/letstest/apache2_targets.yaml

@@ -1,6 +1,11 @@
 targets:
   #-----------------------------------------------------------------------------
   #Ubuntu
+  - ami: ami-0545f7036167eb3aa
+    name: ubuntu19.10
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
   - ami: ami-095192256fe1477ad
     name: ubuntu18.04LTS
     type: ubuntu
@@ -36,6 +41,11 @@ targets:
     user: admin
   #-----------------------------------------------------------------------------
   # Fedora
+  - ami: ami-0fcbe88944a53b4c8
+    name: fedora31
+    type: centos
+    virt: hvm
+    user: fedora
   - ami: ami-00bbc6858140f19ed
     name: fedora30
     type: centos

tests/letstest/targets.yaml

@@ -1,6 +1,11 @@
 targets:
   #-----------------------------------------------------------------------------
   #Ubuntu
+  - ami: ami-0545f7036167eb3aa
+    name: ubuntu19.10
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
   - ami: ami-095192256fe1477ad
     name: ubuntu18.04LTS
     type: ubuntu
@@ -50,6 +55,11 @@ targets:
     type: centos
     virt: hvm
     user: ec2-user
+  - ami: ami-0fcbe88944a53b4c8
+    name: fedora31
+    type: centos
+    virt: hvm
+    user: fedora
   - ami: ami-00bbc6858140f19ed
     name: fedora30
     type: centos

tools/_release.sh

@@ -59,7 +59,7 @@ mv "dist.$version" "dist.$version.$(date +%s).bak" || true
 git tag --delete "$tag" || true
 
 tmpvenv=$(mktemp -d)
-VIRTUALENV_NO_DOWNLOAD=1 virtualenv -p python2 $tmpvenv
+python3 -m venv "$tmpvenv"
 . $tmpvenv/bin/activate
 # update setuptools/pip just like in other places in the repo
 pip install -U setuptools
@@ -157,7 +157,7 @@ done
 echo "Testing packages"
 cd "dist.$version"
 # start local PyPI
-python -m SimpleHTTPServer $PORT &
+python -m http.server $PORT &
 # cd .. is NOT done on purpose: we make sure that all subpackages are
 # installed from local PyPI rather than current directory (repo root)
 VIRTUALENV_NO_DOWNLOAD=1 virtualenv ../venv
@@ -202,7 +202,7 @@ done
 # pin pip hashes of the things we just built
 for pkg in $SUBPKGS_IN_AUTO ; do
     echo $pkg==$version \\
-    pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python2 -c 'from sys import stdin; input = stdin.read(); print "   ", input.replace("\n--hash", " \\\n    --hash"),'
+    pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python -c 'from sys import stdin; input = stdin.read(); print("   ", input.replace("\n--hash", " \\\n    --hash"), end="")'
 done > letsencrypt-auto-source/pieces/certbot-requirements.txt
 deactivate
 

tox.cover.py

@@ -14,7 +14,7 @@ DEFAULT_PACKAGES = [
     'certbot_dns_sakuracloud', 'certbot_nginx']
 
 COVER_THRESHOLDS = {
-    'certbot': {'linux': 96, 'windows': 96},
+    'certbot': {'linux': 95, 'windows': 96},
     'acme': {'linux': 100, 'windows': 99},
     'certbot_apache': {'linux': 100, 'windows': 100},
     'certbot_dns_cloudflare': {'linux': 98, 'windows': 98},

tox.ini

@@ -4,7 +4,7 @@
 
 [tox]
 skipsdist = true
-envlist = modification,py3,py27-cover,lint,mypy
+envlist = modification,py3-cover,lint,mypy
 
 [base]
 # pip installs the requested packages in editable mode
@@ -63,8 +63,11 @@ source_paths =
 passenv =
     CERTBOT_NO_PIN
 commands =
-    {[base]install_and_test} {[base]all_packages}
-    python tests/lock_test.py
+    !cover: {[base]install_and_test} {[base]all_packages}
+    !cover: python tests/lock_test.py
+    cover: {[base]install_packages}
+    cover: {[base]pip_install} certbot-apache[dev]
+    cover: python tox.cover.py
 # We always recreate the virtual environment to avoid problems like
 # https://github.com/certbot/certbot/issues/7745.
 recreate = true
@@ -116,20 +119,6 @@ commands =
 setenv =
     {[testenv:py27-oldest]setenv}
 
-[testenv:py27-cover]
-basepython = python2.7
-commands =
-    {[base]install_packages}
-    {[base]pip_install} certbot-apache[dev]
-    python tox.cover.py
-
-[testenv:py37-cover]
-basepython = python3.7
-commands =
-    {[base]install_packages}
-    {[base]pip_install} certbot-apache[dev]
-    python tox.cover.py
-
 [testenv:lint]
 basepython = python3
 # separating into multiple invocations disables cross package
@@ -149,15 +138,22 @@ commands =
 
 [testenv:apacheconftest]
 commands =
-    {[base]pip_install} acme certbot certbot-apache certbot-compatibility-test
+    {[base]pip_install} acme certbot certbot-apache
     {toxinidir}/certbot-apache/tests/apache-conf-files/apache-conf-test --debian-modules
 passenv =
     SERVER
 
+[testenv:apacheconftest-external-with-pebble]
+# Run apacheconftest with pebble and Certbot outside of tox's virtual
+# environment.
+commands =
+    {[base]pip_install} certbot-ci
+    {toxinidir}/certbot-apache/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules
+
 [testenv:apacheconftest-with-pebble]
 commands =
-    {[base]pip_install} acme certbot certbot-apache certbot-ci certbot-compatibility-test
-    {toxinidir}/certbot-apache/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules
+    {[base]pip_install} acme certbot certbot-apache
+    {[testenv:apacheconftest-external-with-pebble]commands}
 
 [testenv:nginxroundtrip]
 commands =
@@ -261,6 +257,14 @@ commands =
         --cov-config=certbot-ci/certbot_integration_tests/.coveragerc
     coverage report --include 'certbot/*' --show-missing --fail-under=62
 
+[testenv:integration-external]
+# Run integration tests with Certbot outside of tox's virtual environment.
+commands =
+    {[base]pip_install} certbot-ci
+    pytest certbot-ci/certbot_integration_tests \
+        --acme-server={env:ACME_SERVER:pebble}
+passenv = DOCKER_*
+
 [testenv:integration-certbot-oldest]
 commands =
     {[base]pip_install} certbot