Release 0.40.0

* Consume constraints file

* Independent pywin32 dependency definition in setup.py and construct.py

.azure-pipelines/INSTALL.md

@@ -112,6 +112,8 @@ steps:
         CODECOV_TOKEN: $(codecov_token)
 ```
 
-- On Azure DevOps, go to you organization, project, pipeline tab
-- Select the pipeline, click "Edit" button, then click "Variables" button
-- Set name (eg `codecov_token`), value, tick "Keep this value secret"
+To set up a variable that is shared between pipelines, follow the instructions
+at
+https://docs.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups.
+When adding variables to a group, don't forget to tick "Keep this value secret"
+if it shouldn't be shared publcily.

.azure-pipelines/advanced.yml

@@ -7,12 +7,15 @@ pr:
   - '*.x'
 # This pipeline is also nightly run on master
 schedules:
-  - cron: "4 0 * * *"
+  - cron: "0 4 * * *"
     displayName: Nightly build
     branches:
       include:
       - master
+    always: true
 
 jobs:
+  # Any addition here should be reflected in the release pipeline.
+  # It is advised to declare all jobs here as templates to improve maintainability.
   - template: templates/tests-suite.yml
-  - template: templates/installer-tests.yml
+  - template: templates/installer-tests.yml

.azure-pipelines/main.yml

@@ -8,4 +8,4 @@ pr:
   - master
 
 jobs:
-  - template: templates/tests-suite.yml
+  - template: templates/tests-suite.yml

.azure-pipelines/release.yml

@@ -0,0 +1,13 @@
+# Release pipeline to build and deploy Certbot for Windows for GitHub release tags
+trigger:
+  tags:
+    include:
+      - v*
+pr: none
+
+jobs:
+  # Any addition here should be reflected in the advanced pipeline.
+  # It is advised to declare all jobs here as templates to improve maintainability.
+  - template: templates/tests-suite.yml
+  - template: templates/installer-tests.yml
+  - template: templates/changelog.yml

.azure-pipelines/templates/changelog.yml

@@ -0,0 +1,14 @@
+jobs:
+  - job: changelog
+    pool:
+      vmImage: vs2017-win2016
+    steps:
+      - bash: |
+          CERTBOT_VERSION="$(python -c "import certbot; print(certbot.__version__)")"
+          "${BUILD_REPOSITORY_LOCALPATH}\tools\extract_changelog.py" "${CERTBOT_VERSION}" >> "${BUILD_ARTIFACTSTAGINGDIRECTORY}/release_notes.md"
+        displayName: Prepare changelog
+      - task: PublishPipelineArtifact@1
+        inputs:
+          path: $(Build.ArtifactStagingDirectory)
+          artifact: changelog
+        displayName: Publish changelog

.azure-pipelines/templates/installer-tests.yml

@@ -18,8 +18,9 @@ jobs:
       - task: PublishPipelineArtifact@1
         inputs:
           path: $(Build.ArtifactStagingDirectory)
-          artifact: WindowsInstaller
-      - script: $(Build.ArtifactStagingDirectory)\certbot-installer-win32.exe /S
+          artifact: windows-installer
+        displayName: Publish Windows installer
+      - script: $(Build.ArtifactStagingDirectory)\certbot-beta-installer-win32.exe /S
         displayName: Install Certbot
       - script: |
           python -m venv venv
@@ -28,4 +29,4 @@ jobs:
       - script: |
           set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
           venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
-        displayName: Run integration tests
+        displayName: Run integration tests

.azure-pipelines/templates/tests-suite.yml

@@ -14,6 +14,8 @@ jobs:
           PYTHON_VERSION: 3.7
           TOXENV: integration-certbot
           PYTEST_ADDOPTS: --numprocesses 4
+    variables:
+    - group: certbot-common
     steps:
     - task: UsePythonVersion@0
       inputs:
@@ -30,7 +32,7 @@ jobs:
         curl -s https://codecov.io/bash -o codecov-bash || echo "Failed to download codecov-bash"
         chmod +x codecov-bash || echo "Failed to apply execute permissions on codecov-bash"
         ./codecov-bash -F windows  || echo "Codecov did not collect coverage reports"
-      condition: eq(variables['TOXENV'], 'py37-cover')
+      condition: in(variables['TOXENV'], 'py37-cover', 'integration-certbot')
       env:
         CODECOV_TOKEN: $(codecov_token)
       displayName: Publish coverage

.codecov.yml

@@ -13,6 +13,6 @@ coverage:
         flags: windows
         # Fixed target instead of auto set by #7173, can
         # be removed when flags in Codecov are added back.
-        target: 97.7
+        target: 97.4
         threshold: 0.1
         base: auto

.travis.yml

@@ -68,7 +68,7 @@ matrix:
     - python: "3.7"
       env: TOXENV=py37
       <<: *not-on-master
-    - python: "3.8-dev"
+    - python: "3.8"
       env: TOXENV=py38
       <<: *not-on-master
     - sudo: required
@@ -234,6 +234,9 @@ matrix:
     - sudo: required
       env: TOXENV=le_auto_centos6
       services: docker
+    - sudo: required
+      env: TOXENV=le_auto_oraclelinux6
+      services: docker
       <<: *extended-test-suite
     - sudo: required
       env: TOXENV=docker_dev

AUTHORS.md

@@ -167,6 +167,7 @@ Authors
 * [Michael Watters](https://github.com/blackknight36)
 * [Michal Moravec](https://github.com/https://github.com/Majkl578)
 * [Michal Papis](https://github.com/mpapis)
+* [Mickaël Schoentgen](https://github.com/BoboTiG)
 * [Minn Soe](https://github.com/MinnSoe)
 * [Min RK](https://github.com/minrk)
 * [Miquel Ruiz](https://github.com/miquelruiz)
@@ -230,6 +231,7 @@ Authors
 * [Stavros Korokithakis](https://github.com/skorokithakis)
 * [Stefan Weil](https://github.com/stweil)
 * [Steve Desmond](https://github.com/stevedesmond-ca)
+* [sydneyli](https://github.com/sydneyli)
 * [Tan Jay Jun](https://github.com/jayjun)
 * [Tapple Gao](https://github.com/tapple)
 * [Telepenin Nikolay](https://github.com/telepenin)

CHANGELOG.md

@@ -2,7 +2,7 @@
 
 Certbot adheres to [Semantic Versioning](https://semver.org/).
 
-## 0.40.0 - master
+## 0.40.0 - 2019-11-05
 
 ### Added
 
@@ -10,7 +10,28 @@ Certbot adheres to [Semantic Versioning](https://semver.org/).
 
 ### Changed
 
-* Removed `--fast` flag from the test farm tests
+* We deprecated support for Python 3.4 in Certbot and its ACME library. Support
+  for Python 3.4 will be removed in the next major release of Certbot.
+  certbot-auto users on RHEL 6 based systems will be asked to enable Software
+  Collections (SCL) repository so Python 3.6 can be installed. certbot-auto can
+  enable the SCL repo for you on CentOS 6 while users on other RHEL 6 based
+  systems will be asked to do this manually.
+* `--server` may now be combined with `--dry-run`. Certbot will, as before, use the
+  staging server instead of the live server when `--dry-run` is used.
+* `--dry-run` now requests fresh authorizations every time, fixing the issue
+  where it was prone to falsely reporting success.
+* Updated certbot-dns-google to depend on newer versions of
+  google-api-python-client and oauth2client.
+* The OS detection logic again uses distro library for Linux OSes
+* certbot.plugins.common.TLSSNI01 has been deprecated and will be removed in a
+  future release.
+* CLI flags --tls-sni-01-port and --tls-sni-01-address have been removed.
+* The values tls-sni and tls-sni-01 for the --preferred-challenges flag are no
+  longer accepted.
+* Removed the flags: `--agree-dev-preview`, `--dialog`, and `--apache-init-script`
+* acme.standalone.BaseRequestHandlerWithLogging and
+  acme.standalone.simple_tls_sni_01_server have been deprecated and will be
+  removed in a future release of the library.
 
 ### Fixed
 

acme/acme/__init__.py

@@ -35,7 +35,7 @@ class _TLSSNI01DeprecationModule(object):
         self.__dict__['_module'] = module
 
     def __getattr__(self, attr):
-        if 'TLSSNI01' in attr:
+        if 'TLSSNI01' in attr or attr == 'BaseRequestHandlerWithLogging':
             warnings.warn('{0} attribute is deprecated, and will be removed soon.'.format(attr),
                           DeprecationWarning, stacklevel=2)
         return getattr(self._module, attr)
@@ -48,3 +48,10 @@ class _TLSSNI01DeprecationModule(object):
 
     def __dir__(self):  # pragma: no cover
         return ['_module'] + dir(self._module)
+
+if sys.version_info[:2] == (3, 4):
+    warnings.warn(
+            "Python 3.4 support will be dropped in the next release of "
+            "acme. Please upgrade your Python version.",
+            PendingDeprecationWarning,
+    ) # pragma: no cover

acme/acme/client.py

@@ -136,7 +136,8 @@ class ClientBase(object):  # pylint: disable=too-many-instance-attributes
         """
         body = messages.UpdateAuthorization(status='deactivated')
         response = self._post(authzr.uri, body)
-        return self._authzr_from_response(response)
+        return self._authzr_from_response(response,
+            authzr.body.identifier, authzr.uri)
 
     def _authzr_from_response(self, response, identifier=None, uri=None):
         authzr = messages.AuthorizationResource(

acme/acme/standalone.py

@@ -7,6 +7,7 @@ import os
 import socket
 import sys
 import threading
+import warnings
 
 from six.moves import BaseHTTPServer  # type: ignore  # pylint: disable=import-error
 from six.moves import http_client  # pylint: disable=import-error
@@ -267,6 +268,9 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 
 def simple_tls_sni_01_server(cli_args, forever=True):
     """Run simple standalone TLSSNI01 server."""
+    warnings.warn(
+        'simple_tls_sni_01_server is deprecated and will be removed soon.',
+        DeprecationWarning, stacklevel=2)
     logging.basicConfig(level=logging.DEBUG)
 
     parser = argparse.ArgumentParser()
@@ -299,7 +303,3 @@ def simple_tls_sni_01_server(cli_args, forever=True):
 
 # Patching ourselves to warn about TLS-SNI challenge deprecation and removal.
 sys.modules[__name__] = _TLSSNI01DeprecationModule(sys.modules[__name__])
-
-
-if __name__ == "__main__":
-    sys.exit(simple_tls_sni_01_server(sys.argv))  # pragma: no cover

acme/acme/standalone_test.py

@@ -6,6 +6,7 @@ import socket
 import threading
 import tempfile
 import unittest
+import warnings
 import time
 from contextlib import closing
 
@@ -67,6 +68,18 @@ class TLSSNI01ServerTest(unittest.TestCase):
                          jose.ComparableX509(self.certs[b'localhost'][1]))
 
 
+class BaseRequestHandlerWithLoggingTest(unittest.TestCase):
+    """Test for acme.standalone.BaseRequestHandlerWithLogging."""
+
+    def test_it(self):
+        with mock.patch('acme.standalone.warnings.warn') as mock_warn:
+            # pylint: disable=unused-variable
+            from acme.standalone import BaseRequestHandlerWithLogging
+        self.assertTrue(mock_warn.called)
+        msg = mock_warn.call_args[0][0]
+        self.assertTrue(msg.startswith('BaseRequestHandlerWithLogging'))
+
+
 class HTTP01ServerTest(unittest.TestCase):
     """Tests for acme.standalone.HTTP01Server."""
 
@@ -266,8 +279,7 @@ class TestSimpleTLSSNI01Server(unittest.TestCase):
             sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
             self.port = sock.getsockname()[1]
 
-        from acme.standalone import simple_tls_sni_01_server
-        self.process = multiprocessing.Process(target=simple_tls_sni_01_server,
+        self.process = multiprocessing.Process(target=_simple_tls_sni_01_server_no_warnings,
                                                args=(['path', '-p', str(self.port)],))
         self.old_cwd = os.getcwd()
         os.chdir(self.test_cwd)
@@ -284,8 +296,8 @@ class TestSimpleTLSSNI01Server(unittest.TestCase):
 
     @mock.patch('acme.standalone.TLSSNI01Server.handle_request')
     def test_mock(self, handle):
-        from acme.standalone import simple_tls_sni_01_server
-        simple_tls_sni_01_server(cli_args=['path', '-p', str(self.port)], forever=False)
+        _simple_tls_sni_01_server_no_warnings(cli_args=['path', '-p', str(self.port)],
+                                              forever=False)
         self.assertEqual(handle.call_count, 1)
 
     def test_live(self):
@@ -302,5 +314,12 @@ class TestSimpleTLSSNI01Server(unittest.TestCase):
                          test_util.load_comparable_cert('rsa2048_cert.pem'))
 
 
+def _simple_tls_sni_01_server_no_warnings(*args, **kwargs):
+    with warnings.catch_warnings():
+        warnings.filterwarnings('ignore', 'simple_tls.*')
+        from acme.standalone import simple_tls_sni_01_server
+        return simple_tls_sni_01_server(*args, **kwargs)
+
+
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

acme/acme/test_util.py

@@ -4,7 +4,6 @@
 
 """
 import os
-import unittest
 import pkg_resources
 
 from cryptography.hazmat.backends import default_backend
@@ -73,23 +72,3 @@ def load_pyopenssl_private_key(*names):
     loader = _guess_loader(
         names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1)
     return crypto.load_privatekey(loader, load_vector(*names))
-
-
-def skip_unless(condition, reason):  # pragma: no cover
-    """Skip tests unless a condition holds.
-
-    This implements the basic functionality of unittest.skipUnless
-    which is only available on Python 2.7+.
-
-    :param bool condition: If ``False``, the test will be skipped
-    :param str reason: the reason for skipping the test
-
-    :rtype: callable
-    :returns: decorator that hides tests unless condition is ``True``
-
-    """
-    if hasattr(unittest, "skipUnless"):
-        return unittest.skipUnless(condition, reason)
-    elif condition:
-        return lambda cls: cls
-    return lambda cls: None

acme/setup.py

@@ -3,7 +3,7 @@ from setuptools import find_packages
 from setuptools.command.test import test as TestCommand
 import sys
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

appveyor.yml

@@ -1,47 +0,0 @@
-image: Visual Studio 2015
-
-environment:
-  matrix:
-    - TOXENV: py35
-    - TOXENV: py37-cover
-    - TOXENV: integration-certbot
-
-branches:
-  only:
-    # apache-parser-v2 is a temporary branch for doing work related to
-    # rewriting the parser in the Apache plugin.
-    - apache-parser-v2
-    - master
-    - /^\d+\.\d+\.x$/ # Version branches like X.X.X
-    - /^test-.*$/
-
-init:
-  # Since master can receive only commits from PR that have already been tested, following
-  # condition avoid to launch all jobs except the coverage one for commits pushed to master.
-  - ps: |
-      if (-Not $Env:APPVEYOR_PULL_REQUEST_NUMBER -And $Env:APPVEYOR_REPO_BRANCH -Eq 'master' `
-          -And -Not ($Env:TOXENV -Like '*-cover'))
-      { $Env:APPVEYOR_SKIP_FINALIZE_ON_EXIT = 'true'; Exit-AppVeyorBuild }
-
-install:
-  # Use Python 3.7 by default
-  - SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%
-  # Using 4 processes is proven to be the most efficient integration tests config for AppVeyor
-  - IF %TOXENV%==integration-certbot SET PYTEST_ADDOPTS=--numprocesses=4
-  # Check env
-  - python --version
-  # Upgrade pip to avoid warnings
-  - python -m pip install --upgrade pip
-  # Ready to install tox and coverage
-  # tools/pip_install.py is used to pin packages to a known working version.
-  - python tools\\pip_install.py tox codecov
-
-build: off
-
-test_script:
-  - set TOX_TESTENV_PASSENV=APPVEYOR
-  # Test env is set by TOXENV env variable
-  - tox
-
-on_success:
-  - if exist .coverage codecov -F windows

certbot-apache/certbot_apache/configurator.py

@@ -174,8 +174,6 @@ class ApacheConfigurator(common.Installer):
                  "(Only Ubuntu/Debian currently)")
         add("ctl", default=DEFAULTS["ctl"],
             help="Full path to Apache control script")
-        util.add_deprecated_argument(
-            add, argument_name="init-script", nargs=1)
 
     def __init__(self, *args, **kwargs):
         """Initialize an Apache Configurator.

certbot-apache/certbot_apache/entrypoint.py

@@ -16,6 +16,7 @@ from certbot_apache import override_suse
 
 OVERRIDE_CLASSES = {
     "arch": override_arch.ArchConfigurator,
+    "cloudlinux": override_centos.CentOSConfigurator,
     "darwin": override_darwin.DarwinConfigurator,
     "debian": override_debian.DebianConfigurator,
     "ubuntu": override_debian.DebianConfigurator,
@@ -23,7 +24,9 @@ OVERRIDE_CLASSES = {
     "centos linux": override_centos.CentOSConfigurator,
     "fedora_old": override_centos.CentOSConfigurator,
     "fedora": override_fedora.FedoraConfigurator,
+    "linuxmint": override_debian.DebianConfigurator,
     "ol": override_centos.CentOSConfigurator,
+    "oracle": override_centos.CentOSConfigurator,
     "redhatenterpriseserver": override_centos.CentOSConfigurator,
     "red hat enterprise linux server": override_centos.CentOSConfigurator,
     "rhel": override_centos.CentOSConfigurator,
@@ -32,6 +35,7 @@ OVERRIDE_CLASSES = {
     "gentoo base system": override_gentoo.GentooConfigurator,
     "opensuse": override_suse.OpenSUSEConfigurator,
     "suse": override_suse.OpenSUSEConfigurator,
+    "sles": override_suse.OpenSUSEConfigurator,
     "scientific": override_centos.CentOSConfigurator,
     "scientific linux": override_centos.CentOSConfigurator,
 }

certbot-apache/certbot_apache/http_01.py

@@ -14,7 +14,7 @@ from certbot_apache.parser import get_aug_path
 logger = logging.getLogger(__name__)
 
 
-class ApacheHttp01(common.TLSSNI01):
+class ApacheHttp01(common.ChallengePerformer):
     """Class that performs HTTP-01 challenges within the Apache configurator."""
 
     CONFIG_TEMPLATE22_PRE = """\

certbot-apache/certbot_apache/tests/configurator_test.py

@@ -108,13 +108,9 @@ class MultipleVhostsTest(util.ApacheTest):
                 exp[k.replace("_", "-")] = ApacheConfigurator.OS_DEFAULTS[k]
         # Special cases
         exp["vhost-root"] = None
-        exp["init-script"] = None
 
         found = set()
         for call in mock_add.call_args_list:
-            # init-script is a special case: deprecated argument
-            if call[0][0] != "init-script":
-                self.assertEqual(exp[call[0][0]], call[1]['default'])
             found.add(call[0][0])
 
         # Make sure that all (and only) the expected values exist
@@ -1296,13 +1292,13 @@ class MultipleVhostsTest(util.ApacheTest):
         account_key = self.rsa512jwk
         achall1 = achallenges.KeyAuthorizationAnnotatedChallenge(
             challb=acme_util.chall_to_challb(
-                challenges.TLSSNI01(
+                challenges.HTTP01(
                     token=b"jIq_Xy1mXGN37tb4L6Xj_es58fW571ZNyXekdZzhh7Q"),
                 "pending"),
             domain="encryption-example.demo", account_key=account_key)
         achall2 = achallenges.KeyAuthorizationAnnotatedChallenge(
             challb=acme_util.chall_to_challb(
-                challenges.TLSSNI01(
+                challenges.HTTP01(
                     token=b"uqnaPzxtrndteOqtrXb0Asl5gOJfWAnnx6QJyvcmlDU"),
                 "pending"),
             domain="certbot.demo", account_key=account_key)

certbot-apache/docs/api/tls_sni_01.rst

@@ -1,5 +0,0 @@
-:mod:`certbot_apache.tls_sni_01`
-------------------------------------
-
-.. automodule:: certbot_apache.tls_sni_01
-   :members:

certbot-apache/setup.py

@@ -4,7 +4,7 @@ from setuptools.command.test import test as TestCommand
 import sys
 
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.39.0"
+LE_AUTO_VERSION="0.40.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -256,20 +256,28 @@ DeprecationBootstrap() {
   fi
 }
 
-MIN_PYTHON_VERSION="2.7"
-MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//')
+MIN_PYTHON_2_VERSION="2.7"
+MIN_PYVER2=$(echo "$MIN_PYTHON_2_VERSION" | sed 's/\.//')
+MIN_PYTHON_3_VERSION="3.5"
+MIN_PYVER3=$(echo "$MIN_PYTHON_3_VERSION" | sed 's/\.//')
 # Sets LE_PYTHON to Python version string and PYVER to the first two
-# digits of the python version
+# digits of the python version.
+# MIN_PYVER and MIN_PYTHON_VERSION are also set by this function, and their
+# values depend on if we try to use Python 3 or Python 2.
 DeterminePythonVersion() {
   # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
   #
   # If no Python is found, PYVER is set to 0.
   if [ "$USE_PYTHON_3" = 1 ]; then
+    MIN_PYVER=$MIN_PYVER3
+    MIN_PYTHON_VERSION=$MIN_PYTHON_3_VERSION
     for LE_PYTHON in "$LE_PYTHON" python3; do
       # Break (while keeping the LE_PYTHON value) if found.
       $EXISTS "$LE_PYTHON" > /dev/null && break
     done
   else
+    MIN_PYVER=$MIN_PYVER2
+    MIN_PYTHON_VERSION=$MIN_PYTHON_2_VERSION
     for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
       # Break (while keeping the LE_PYTHON value) if found.
       $EXISTS "$LE_PYTHON" > /dev/null && break
@@ -285,7 +293,7 @@ DeterminePythonVersion() {
     fi
   fi
 
-  PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
+  PYVER=$("$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//')
   if [ "$PYVER" -lt "$MIN_PYVER" ]; then
     if [ "$1" != "NOCRASH" ]; then
       error "You have an ancient version of Python entombed in your operating system..."
@@ -368,7 +376,9 @@ BootstrapDebCommon() {
 
 # Sets TOOL to the name of the package manager
 # Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG.
-# Enables EPEL if applicable and possible.
+# Note: this function is called both while selecting the bootstrap scripts and
+# during the actual bootstrap. Some things like prompting to user can be done in the latter
+# case, but not in the former one.
 InitializeRPMCommonBase() {
   if type dnf 2>/dev/null
   then
@@ -388,26 +398,6 @@ InitializeRPMCommonBase() {
   if [ "$QUIET" = 1 ]; then
     QUIET_FLAG='--quiet'
   fi
-
-  if ! $TOOL list *virtualenv >/dev/null 2>&1; then
-    echo "To use Certbot, packages from the EPEL repository need to be installed."
-    if ! $TOOL list epel-release >/dev/null 2>&1; then
-      error "Enable the EPEL repository and try running Certbot again."
-      exit 1
-    fi
-    if [ "$ASSUME_YES" = 1 ]; then
-      /bin/echo -n "Enabling the EPEL repository in 3 seconds..."
-      sleep 1s
-      /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
-      sleep 1s
-      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..."
-      sleep 1s
-    fi
-    if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then
-      error "Could not enable EPEL. Aborting bootstrap!"
-      exit 1
-    fi
-  fi
 }
 
 BootstrapRpmCommonBase() {
@@ -488,13 +478,88 @@ BootstrapRpmCommon() {
   BootstrapRpmCommonBase "$python_pkgs"
 }
 
+# If new packages are installed by BootstrapRpmPython3 below, this version
+# number must be increased.
+BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION=1
+
+# Checks if rh-python36 can be installed.
+Python36SclIsAvailable() {
+  InitializeRPMCommonBase >/dev/null 2>&1;
+
+  if "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    return 0
+  fi
+  if "${TOOL}" list centos-release-scl >/dev/null 2>&1; then
+    return 0
+  fi
+  return 1
+}
+
+# Try to enable rh-python36 from SCL if it is necessary and possible.
+EnablePython36SCL() {
+  if "$EXISTS" python3.6 > /dev/null 2> /dev/null; then
+      return 0
+  fi
+  if ! scl --list 2>/dev/null | grep -q rh-python36; then
+      return 0
+  fi
+  set +e
+  . scl_source enable rh-python36
+  set -e
+}
+
+# This bootstrap concerns old RedHat-based distributions that do not ship by default
+# with Python 2.7, but only Python 2.6. We bootstrap them by enabling SCL and installing
+# Python 3.6. Some of these distributions are: CentOS/RHEL/OL/SL 6.
+BootstrapRpmPython3Legacy() {
+  # Tested with:
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  if ! "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    echo "To use Certbot on this operating system, packages from the SCL repository need to be installed."
+    if ! "${TOOL}" list centos-release-scl >/dev/null 2>&1; then
+      error "Enable the SCL repository and try running Certbot again."
+      exit 1
+    fi
+    if [ "${ASSUME_YES}" = 1 ]; then
+      /bin/echo -n "Enabling the SCL repository in 3 seconds... (Press Ctrl-C to cancel)"
+      sleep 1s
+      /bin/echo -ne "\e[0K\rEnabling the SCL repository in 2 seconds... (Press Ctrl-C to cancel)"
+      sleep 1s
+      /bin/echo -e "\e[0K\rEnabling the SCL repository in 1 second... (Press Ctrl-C to cancel)"
+      sleep 1s
+    fi
+    if ! "${TOOL}" install "${YES_FLAG}" "${QUIET_FLAG}" centos-release-scl; then
+      error "Could not enable SCL. Aborting bootstrap!"
+      exit 1
+    fi
+  fi
+
+  # CentOS 6 must use rh-python36 from SCL
+  if "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    python_pkgs="rh-python36-python
+      rh-python36-python-virtualenv
+      rh-python36-python-devel
+    "
+  else
+    error "No supported Python package available to install. Aborting bootstrap!"
+    exit 1
+  fi
+
+  BootstrapRpmCommonBase "${python_pkgs}"
+
+  # Enable SCL rh-python36 after bootstrapping.
+  EnablePython36SCL
+}
+
 # If new packages are installed by BootstrapRpmPython3 below, this version
 # number must be increased.
 BOOTSTRAP_RPM_PYTHON3_VERSION=1
 
 BootstrapRpmPython3() {
   # Tested with:
-  #   - CentOS 6
   #   - Fedora 29
 
   InitializeRPMCommonBase
@@ -505,12 +570,6 @@ BootstrapRpmPython3() {
       python3-virtualenv
       python3-devel
     "
-  # EPEL uses python34
-  elif $TOOL list python34 >/dev/null 2>&1; then
-    python_pkgs="python34
-      python34-devel
-      python34-tools
-    "
   else
     error "No supported Python package available to install. Aborting bootstrap!"
     exit 1
@@ -769,31 +828,50 @@ elif [ -f /etc/redhat-release ]; then
      RPM_DIST_VERSION=0
   fi
 
-  # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then.
-  # RHEL 8 also uses python3 by default.
-  if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then
-    RPM_USE_PYTHON_3=1
-  elif [ "$RPM_DIST_NAME" = "rhel" -a "$RPM_DIST_VERSION" -ge 8 ]; then
-    RPM_USE_PYTHON_3=1
-  elif [ "$RPM_DIST_NAME" = "centos" -a "$RPM_DIST_VERSION" -ge 8 ]; then
-    RPM_USE_PYTHON_3=1
-  else
-    RPM_USE_PYTHON_3=0
-  fi
+  # Handle legacy RPM distributions
+  if [ "$PYVER" -eq 26 ]; then
+    # Check if an automated bootstrap can be achieved on this system.
+    if ! Python36SclIsAvailable; then
+      INTERACTIVE_BOOTSTRAP=1
+    fi
 
-  if [ "$RPM_USE_PYTHON_3" = 1 ]; then
     Bootstrap() {
-      BootstrapMessage "RedHat-based OSes that will use Python3"
-      BootstrapRpmPython3
+      BootstrapMessage "Legacy RedHat-based OSes that will use Python3"
+      BootstrapRpmPython3Legacy
     }
     USE_PYTHON_3=1
-    BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+    BOOTSTRAP_VERSION="BootstrapRpmPython3Legacy $BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION"
+
+    # Try now to enable SCL rh-python36 for systems already bootstrapped
+    # NB: EnablePython36SCL has been defined along with BootstrapRpmPython3Legacy in certbot-auto
+    EnablePython36SCL
   else
-    Bootstrap() {
-      BootstrapMessage "RedHat-based OSes"
-      BootstrapRpmCommon
-    }
-    BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+    # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then.
+    # RHEL 8 also uses python3 by default.
+    if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 ]; then
+      RPM_USE_PYTHON_3=1
+    elif [ "$RPM_DIST_NAME" = "rhel" -a "$RPM_DIST_VERSION" -ge 8 ]; then
+      RPM_USE_PYTHON_3=1
+    elif [ "$RPM_DIST_NAME" = "centos" -a "$RPM_DIST_VERSION" -ge 8 ]; then
+      RPM_USE_PYTHON_3=1
+    else
+      RPM_USE_PYTHON_3=0
+    fi
+
+    if [ "$RPM_USE_PYTHON_3" = 1 ]; then
+      Bootstrap() {
+        BootstrapMessage "RedHat-based OSes that will use Python3"
+        BootstrapRpmPython3
+      }
+      USE_PYTHON_3=1
+      BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+    else
+      Bootstrap() {
+        BootstrapMessage "RedHat-based OSes"
+        BootstrapRpmCommon
+      }
+      BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+    fi
   fi
 
   LE_PYTHON="$prev_le_python"
@@ -1078,8 +1156,15 @@ if [ "$1" = "--le-auto-phase2" ]; then
     # If the selected Bootstrap function isn't a noop and it differs from the
     # previously used version
     if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then
-      # if non-interactive mode or stdin and stdout are connected to a terminal
-      if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
+      # Check if we can rebootstrap without manual user intervention: this requires that
+      # certbot-auto is in non-interactive mode AND selected bootstrap does not claim to
+      # require a manual user intervention.
+      if [ "$NONINTERACTIVE" = 1 -a "$INTERACTIVE_BOOTSTRAP" != 1 ]; then
+        CAN_REBOOTSTRAP=1
+      fi
+      # Check if rebootstrap can be done non-interactively and current shell is non-interactive
+      # (true if stdin and stdout are not attached to a terminal).
+      if [ \( "$CAN_REBOOTSTRAP" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
         if [ -d "$VENV_PATH" ]; then
           rm -rf "$VENV_PATH"
         fi
@@ -1090,12 +1175,21 @@ if [ "$1" = "--le-auto-phase2" ]; then
           ln -s "$VENV_PATH" "$OLD_VENV_PATH"
         fi
         RerunWithArgs "$@"
+      # Otherwise bootstrap needs to be done manually by the user.
       else
-        error "Skipping upgrade because new OS dependencies may need to be installed."
-        error
-        error "To upgrade to a newer version, please run this script again manually so you can"
-        error "approve changes or with --non-interactive on the command line to automatically"
-        error "install any required packages."
+        # If it is because bootstrapping is interactive, --non-interactive will be of no use.
+        if [ "$INTERACTIVE_BOOTSTRAP" = 1 ]; then
+          error "Skipping upgrade because new OS dependencies may need to be installed."
+          error "This requires manual user intervention: please run this script again manually."
+        # If this is because of the environment (eg. non interactive shell without
+        # --non-interactive flag set), help the user in that direction.
+        else
+          error "Skipping upgrade because new OS dependencies may need to be installed."
+          error
+          error "To upgrade to a newer version, please run this script again manually so you can"
+          error "approve changes or with --non-interactive on the command line to automatically"
+          error "install any required packages."
+        fi
         # Set INSTALLED_VERSION to be the same so we don't update the venv
         INSTALLED_VERSION="$LE_AUTO_VERSION"
         # Continue to use OLD_VENV_PATH if the new venv doesn't exist
@@ -1338,18 +1432,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.39.0 \
-    --hash=sha256:f1a70651a6c5137a448f4a8db17b09af619f80a077326caae6b74278bf1db488 \
-    --hash=sha256:885cee1c4d05888af86b626cbbfc29d3c6c842ef4fe8f4a486994cef9daddfe0
-acme==0.39.0 \
-    --hash=sha256:4f8be913df289b981852042719469cc367a7e436256f232c799d0bd1521db710 \
-    --hash=sha256:a2fcb75d16de6804f4b4d773a457ee2f6434ebaf8fd1aa60862a91d4e8f73608
-certbot-apache==0.39.0 \
-    --hash=sha256:c7a8630a85b753a52ca0b8c19e24b8f85ac4ba028292a95745e250c2e72faab9 \
-    --hash=sha256:4651a0212c9ebc3087281dad92ad3cb355bb2730f432d0180a8d23325d11825a
-certbot-nginx==0.39.0 \
-    --hash=sha256:76e5862ad5cc0fbc099df3502987c101c60dee1c188a579eac990edee7a910df \
-    --hash=sha256:ceac88df52d3b27d14c3052b9e90ada327d7e14ecd6e4af7519918182d6138b4
+certbot==0.40.0 \
+    --hash=sha256:b9ff74c4f3d3e06d9c467465f97bcbb07b0f4d778d3c4232ab91583d933dba61 \
+    --hash=sha256:cff166597b3c714c3e7e60b2bcd6089135b375cadca04cf36abd15bfdb22be40
+acme==0.40.0 \
+    --hash=sha256:1b026b07a2099e50dac11cbdb834925f1d9b5691e349b52e9d397a12f3dc4eac \
+    --hash=sha256:f29c1185d1e33919bad6c1f3fece168ee191d96d47f5997117561dc74a454221
+certbot-apache==0.40.0 \
+    --hash=sha256:f1c034a05fbd6cc6fde9494f493a8a6ed0e02e7652e51af16342082bc17387e4 \
+    --hash=sha256:43c3d7628ca6630467c4f57dd30423f031c1c7cbca46f7500293172d0fe3581e
+certbot-nginx==0.40.0 \
+    --hash=sha256:55cd3c90e2851069b536859050374fe2fcfa22c3e862cc0e1811fbce9e52dccc \
+    --hash=sha256:3df8cec22910f2d41ccb4494661ff65f98c52dd441864a53a318b32979256881
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------

certbot-ci/certbot_integration_tests/certbot_tests/test_main.py

@@ -594,3 +594,21 @@ def test_ocsp_status_live(context):
 
     assert output.count('INVALID') == 1, 'Expected {0} to be INVALID'.format(cert)
     assert output.count('REVOKED') == 1, 'Expected {0} to be REVOKED'.format(cert)
+
+
+def test_dry_run_deactivate_authzs(context):
+    """Test that Certbot deactivates authorizations when performing a dry run"""
+
+    name = context.get_domain('dry-run-authz-deactivation')
+    args = ['certonly', '--cert-name', name, '-d', name, '--dry-run']
+    log_line = 'Recreating order after authz deactivation'
+
+    # First order will not need deactivation
+    context.certbot(args)
+    with open(join(context.workspace, 'logs', 'letsencrypt.log'), 'r') as f:
+        assert log_line not in f.read(), 'First order should not have had any authz reuse'
+
+    # Second order will require deactivation
+    context.certbot(args)
+    with open(join(context.workspace, 'logs', 'letsencrypt.log'), 'r') as f:
+        assert log_line in f.read(), 'Second order should have been recreated due to authz reuse'

certbot-ci/certbot_integration_tests/utils/acme_server.py

@@ -125,6 +125,7 @@ class ACMEServer(object):
         environ = os.environ.copy()
         environ['PEBBLE_VA_NOSLEEP'] = '1'
         environ['PEBBLE_WFE_NONCEREJECT'] = '0'
+        environ['PEBBLE_AUTHZREUSE'] = '100'
 
         self._launch_process(
             [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053'],

certbot-compatibility-test/certbot_compatibility_test/testdata/rsa1024_key.pem

@@ -1,15 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIICXAIBAAKBgQCsREbM+UcfsgDy2w56AVGyxsO0HVsbEZHHoEzv7qksIwFgRYMp
-rowwIxD450RQQqjvw9IoXlMVmr1t5szn5KXn9JRO9T5KNCCy3VPx75WBcp6kzd9Q
-2HS1OEOtpilNnDkZ+TJfdgFWPUBYj2o4Md1hPmcvagiIJY5U6speka2bjwIDAQAB
-AoGANCMZ9pF/mDUsmlP4Rq69hkkoFAxKdZ/UqkF256so4mXZ1cRUFTpxzWPfkCWW
-hGAYdzCiG3uo08IYkPmojIqkN1dk5Hcq5eQAmshaPkQHQCHjmPjjcNvgjIXQoGUf
-TpDU2hbY4UAlJlj4ZLh+jGP5Zq8/WrNi8RsI3v9Nagfp/FECQQDgi2q8p1gX0TNh
-d1aEKmSXkR3bxkyFk6oS+pBrAG3+yX27ZayN6Rx6DOs/FcBsOu7fX3PYBziDeEWe
-Lkf1P743AkEAxGYT/LY3puglSz4iJZZzWmRCrVOg41yhfQ+F1BRX43/2vtoU5GyM
-2lUn1vQ2e/rfmnAvfJxc90GeZCIHB1ihaQJBALH8UMLxMtbOMJgVbDKfF9U8ZhqK
-+KT5A1q/2jG2yXmoZU1hroFeQgBMtTvwFfK0VBwjIUQflSBA+Y4EyW0Q9ckCQGvd
-jHitM1+N/H2YwHRYbz5j9mLvnVuCEod3MQ9LpQGj1Eb5y6OxIqL/RgQ+2HW7UXem
-yc3sqvp5pZ5lOesE+JECQETPI64gqxlTIs3nErNMpMynUuTWpaElOcIJTT6icLzB
-Xix67kKXjROO5D58GEYkM0Yi5k7YdUPoQBW7MoIrSIA=
------END RSA PRIVATE KEY-----

certbot-compatibility-test/certbot_compatibility_test/testdata/rsa1024_key2.pem

@@ -1,15 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIICXAIBAAKBgQDCzejLjo8wsz0avrylt7HQyF0+vsKritF70EGmc64cV0XfkCTR
-o+vMXBXMuUY6Kv3hTXV7klgkNYmL7gXAsFGQ4B9qeMnkYn0GcQdI51u076y/26Fu
-37uJg45Q6eApKInJSsyLVMcAT4HUJ6fFnUodFAKR7vTzOQryNW7Et4gA4wIDAQAB
-AoGAKiAU40/krwdTg2ETslJS5W8ums7tkeLnAfs69x+02vQUbA/jpmHoL70KCcdW
-5GU/mWUCrsIqxUm+gL/sBosaV/TF256qUBt2qQCZTN8MbDaNSYiiMnucOfbWdIqx
-Zgls6GUoXQvPic9RUoFSlgfSjo5ezz6el5ihvRMp+wbk24ECQQD3oz4hN029DSZo
-Y3+flmBn77gA0BMUvLa6hmt9b3xT5U/ToCLfbmUvpx7zV1g5era2y9qt/o3UtAbW
-1zCVETgzAkEAyWHv/+RnSXp8/D4YwTVWyeWi862uNBPkuLGP/0zASdwBfBK3uBls
-+VumfSCtp0kt2AXXmScg1fkHdeAVT6AkkQJBAJb2XRnCrRFiwtdAULzo3zx9Vp6o
-OfmaUYrEByMgo5pBYLiSFrA+jFDQgH238YCY3mnxPA517+CLHuA5rtQw+yECQCfm
-gL/pyFE1tLfhsdPuNpDwL9YqLl7hJis1+zrxQRQhRCYKK16NoxrQ/u7B38ZKaIvp
-tGsC5q2elszTJkXNjBECQCVE9QCVx056vHVdPWM8z3GAeV3sJQ01HLLjebTEEz6G
-jH54gk+YYPp4kjCvVUykbnB58BY2n88GQt5Jj5eLuMo=
------END RSA PRIVATE KEY-----

certbot-compatibility-test/certbot_compatibility_test/util.py

@@ -8,13 +8,13 @@ import tarfile
 
 import josepy as jose
 
-from acme import test_util
+from certbot.tests import util as test_util
 from certbot import constants
 
 from certbot_compatibility_test import errors
 
 
-_KEY_BASE = "rsa1024_key.pem"
+_KEY_BASE = "rsa2048_key.pem"
 KEY_PATH = test_util.vector_path(_KEY_BASE)
 KEY = test_util.load_pyopenssl_private_key(_KEY_BASE)
 JWK = jose.JWKRSA(key=test_util.load_rsa_private_key(_KEY_BASE))

certbot-compatibility-test/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 install_requires = [
     'certbot',

certbot-dns-cloudflare/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-cloudxns/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-digitalocean/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-dnsimple/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-dnsmadeeasy/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-gehirn/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-google/setup.py

@@ -2,18 +2,16 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
     'acme>=0.29.0',
     'certbot>=0.39.0',
-    # 1.5 is the first version that supports oauth2client>=2.0
-    'google-api-python-client>=1.5',
+    'google-api-python-client>=1.5.5',
     'mock',
-    # for oauth2client.service_account.ServiceAccountCredentials
-    'oauth2client>=2.0',
+    'oauth2client>=4.0',
     'setuptools',
     'zope.interface',
     # already a dependency of google-api-python-client, but added for consistency

certbot-dns-linode/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-luadns/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-nsone/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-ovh/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-rfc2136/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-route53/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-sakuracloud/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-nginx/certbot_nginx/configurator.py

@@ -1107,7 +1107,7 @@ class NginxConfigurator(common.Installer):
     ###########################################################################
     def get_chall_pref(self, unused_domain):  # pylint: disable=no-self-use
         """Return list of challenge preferences."""
-        return [challenges.HTTP01, challenges.TLSSNI01]
+        return [challenges.HTTP01]
 
     # Entry point in main.py for performing challenges
     def perform(self, achalls):

certbot-nginx/certbot_nginx/http_01.py

@@ -29,10 +29,10 @@ class NginxHttp01(common.ChallengePerformer):
     :param list indices: Meant to hold indices of challenges in a
         larger array. NginxHttp01 is capable of solving many challenges
         at once which causes an indexing issue within NginxConfigurator
-        who must return all responses in order.  Imagine NginxConfigurator
-        maintaining state about where all of the http-01 Challenges,
-        TLS-SNI-01 Challenges belong in the response array.  This is an
-        optional utility.
+        who must return all responses in order. Imagine
+        NginxConfigurator maintaining state about where all of the
+        challenges, possibly of different types, belong in the response
+        array. This is an optional utility.
 
     """
 

certbot-nginx/certbot_nginx/tests/configurator_test.py

@@ -97,7 +97,7 @@ class NginxConfiguratorTest(util.NginxTest):
             errors.PluginError, self.config.enhance, 'myhost', 'unknown_enhancement')
 
     def test_get_chall_pref(self):
-        self.assertEqual([challenges.HTTP01, challenges.TLSSNI01],
+        self.assertEqual([challenges.HTTP01],
                          self.config.get_chall_pref('myhost'))
 
     def test_save(self):

certbot-nginx/certbot_nginx/tests/http_01_test.py

@@ -73,11 +73,11 @@ class HttpPerformTest(util.NginxTest):
             self.http01.add_chall(achall)
             acme_responses.append(achall.response(self.account_key))
 
-        sni_responses = self.http01.perform()
+        http_responses = self.http01.perform()
 
-        self.assertEqual(len(sni_responses), 4)
+        self.assertEqual(len(http_responses), 4)
         for i in six.moves.range(4):
-            self.assertEqual(sni_responses[i], acme_responses[i])
+            self.assertEqual(http_responses[i], acme_responses[i])
 
     def test_mod_config(self):
         self.http01.add_chall(self.achalls[0])

certbot-nginx/docs/api/tls_sni_01.rst

@@ -1,5 +0,0 @@
-:mod:`certbot_nginx.tls_sni_01`
------------------------------------
-
-.. automodule:: certbot_nginx.tls_sni_01
-   :members:

certbot-nginx/setup.py

@@ -4,7 +4,7 @@ from setuptools.command.test import test as TestCommand
 import sys
 
 
-version = '0.40.0.dev0'
+version = '0.40.0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot/__init__.py

@@ -1,4 +1,4 @@
 """Certbot client."""
 
 # version number like 1.2.3a0, must have at least 2 parts, like 1.2
-__version__ = '0.40.0.dev0'
+__version__ = '0.40.0'

certbot/auth_handler.py

@@ -7,8 +7,9 @@ import zope.component
 
 from acme import challenges
 from acme import messages
+from acme import errors as acme_errors
 # pylint: disable=unused-import, no-name-in-module
-from acme.magic_typing import Dict, List
+from acme.magic_typing import Dict, List, Tuple
 # pylint: enable=unused-import, no-name-in-module
 from certbot import achallenges
 from certbot import errors
@@ -97,6 +98,31 @@ class AuthHandler(object):
 
             return authzrs_validated
 
+    def deactivate_valid_authorizations(self, orderr):
+        # type: (messages.OrderResource) -> Tuple[List, List]
+        """
+        Deactivate all `valid` authorizations in the order, so that they cannot be re-used
+        in subsequent orders.
+        :param messages.OrderResource orderr: must have authorizations filled in
+        :returns: tuple of list of successfully deactivated authorizations, and
+                  list of unsuccessfully deactivated authorizations.
+        :rtype: tuple
+        """
+        to_deactivate = [authzr for authzr in orderr.authorizations
+                         if authzr.body.status == messages.STATUS_VALID]
+        deactivated = []
+        failed = []
+
+        for authzr in to_deactivate:
+            try:
+                authzr = self.acme.deactivate_authorization(authzr)
+                deactivated.append(authzr)
+            except acme_errors.Error as e:
+                failed.append(authzr)
+                logger.debug('Failed to deactivate authorization %s: %s', authzr.uri, e)
+
+        return (deactivated, failed)
+
     def _poll_authorizations(self, authzrs, max_retries, best_effort):
         """
         Poll the ACME CA server, to wait for confirmation that authorizations have their challenges
@@ -182,9 +208,6 @@ class AuthHandler(object):
 
             achalls.extend(self._challenge_factory(authzr, path))
 
-        if any(isinstance(achall.chall, challenges.TLSSNI01) for achall in achalls):
-            logger.warning("TLS-SNI-01 is deprecated, and will stop working soon.")
-
         return achalls
 
     def _get_chall_pref(self, domain):

certbot/cli.py

@@ -163,24 +163,6 @@ def report_config_interaction(modified, modifiers):
         VAR_MODIFIERS.setdefault(var, set()).update(modifiers)
 
 
-def possible_deprecation_warning(config):
-    "A deprecation warning for users with the old, not-self-upgrading letsencrypt-auto."
-    if cli_command != LEAUTO:
-        return
-    if config.no_self_upgrade:
-        # users setting --no-self-upgrade might be hanging on a client version like 0.3.0
-        # or 0.5.0 which is the new script, but doesn't set CERTBOT_AUTO; they don't
-        # need warnings
-        return
-    if "CERTBOT_AUTO" not in os.environ:
-        logger.warning("You are running with an old copy of letsencrypt-auto"
-            " that does not receive updates, and is less reliable than more"
-            " recent versions. The letsencrypt client has also been renamed"
-            " to Certbot. We recommend upgrading to the latest certbot-auto"
-            " script, or using native OS packages.")
-        logger.debug("Deprecation warning circumstances: %s / %s", sys.argv[0], os.environ)
-
-
 class _Default(object):
     """A class to use as a default to detect if a value is set by a user"""
 
@@ -642,20 +624,25 @@ class HelpfulArgumentParser(object):
             raise errors.Error(
                 "Parameters --hsts and --auto-hsts cannot be used simultaneously.")
 
-        possible_deprecation_warning(parsed_args)
-
         return parsed_args
 
     def set_test_server(self, parsed_args):
         """We have --staging/--dry-run; perform sanity check and set config.server"""
 
-        if parsed_args.server not in (flag_default("server"), constants.STAGING_URI):
-            conflicts = ["--staging"] if parsed_args.staging else []
-            conflicts += ["--dry-run"] if parsed_args.dry_run else []
-            raise errors.Error("--server value conflicts with {0}".format(
-                " and ".join(conflicts)))
+        # Flag combinations should produce these results:
+        #                             | --staging      | --dry-run   |
+        # ------------------------------------------------------------
+        # | --server acme-v02         | Use staging    | Use staging |
+        # | --server acme-staging-v02 | Use staging    | Use staging |
+        # | --server <other>          | Conflict error | Use <other> |
 
-        parsed_args.server = constants.STAGING_URI
+        default_servers = (flag_default("server"), constants.STAGING_URI)
+
+        if parsed_args.staging and parsed_args.server not in default_servers:
+            raise errors.Error("--server value conflicts with --staging")
+
+        if parsed_args.server in default_servers:
+            parsed_args.server = constants.STAGING_URI
 
         if parsed_args.dry_run:
             if self.verb not in ["certonly", "renew"]:
@@ -1259,20 +1246,6 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False):  # pylint: dis
         default=flag_default("autorenew"), dest="autorenew",
         help="Disable auto renewal of certificates.")
 
-    helpful.add_deprecated_argument("--agree-dev-preview", 0)
-    helpful.add_deprecated_argument("--dialog", 0)
-
-    # Deprecation of tls-sni-01 related cli flags
-    # TODO: remove theses flags completely in few releases
-    class _DeprecatedTLSSNIAction(util._ShowWarning):  # pylint: disable=protected-access
-        def __call__(self, parser, namespace, values, option_string=None):
-            super(_DeprecatedTLSSNIAction, self).__call__(parser, namespace, values, option_string)
-            namespace.https_port = values
-    helpful.add(
-        ["testing", "standalone", "apache", "nginx"], "--tls-sni-01-port",
-        type=int, action=_DeprecatedTLSSNIAction, help=argparse.SUPPRESS)
-    helpful.add_deprecated_argument("--tls-sni-01-address", 1)
-
     # Populate the command line parameters for new style enhancements
     enhancements.populate_cli(helpful.add)
 
@@ -1559,18 +1532,10 @@ def parse_preferred_challenges(pref_challs):
     :raises errors.Error: if pref_challs is invalid
 
     """
-    aliases = {"dns": "dns-01", "http": "http-01", "tls-sni": "tls-sni-01"}
+    aliases = {"dns": "dns-01", "http": "http-01"}
     challs = [c.strip() for c in pref_challs]
     challs = [aliases.get(c, c) for c in challs]
 
-    # Ignore tls-sni-01 from the list, and generates a deprecation warning
-    # TODO: remove this option completely in few releases
-    if "tls-sni-01" in challs:
-        logger.warning('TLS-SNI-01 support is deprecated. This value is being dropped from the '
-                       'setting of --preferred-challenges and future versions of Certbot will '
-                       'error if it is included.')
-        challs = [chall for chall in challs if chall != "tls-sni-01"]
-
     unrecognized = ", ".join(name for name in challs
                              if name not in challenges.Challenge.TYPES)
     if unrecognized:

certbot/client.py

@@ -15,7 +15,7 @@ from acme import client as acme_client
 from acme import crypto_util as acme_crypto_util
 from acme import errors as acme_errors
 from acme import messages
-from acme.magic_typing import Optional  # pylint: disable=unused-import,no-name-in-module
+from acme.magic_typing import Optional, List  # pylint: disable=unused-import,no-name-in-module
 
 import certbot
 from certbot import account
@@ -366,6 +366,7 @@ class Client(object):
             return cert, chain, key, csr
 
     def _get_order_and_authorizations(self, csr_pem, best_effort):
+        # type: (str, bool) -> List[messages.OrderResource]
         """Request a new order and complete its authorizations.
 
         :param str csr_pem: A CSR in PEM format.
@@ -381,6 +382,17 @@ class Client(object):
         except acme_errors.WildcardUnsupportedError:
             raise errors.Error("The currently selected ACME CA endpoint does"
                                " not support issuing wildcard certificates.")
+
+        # For a dry run, ensure we have an order with fresh authorizations
+        if orderr and self.config.dry_run:
+            deactivated, failed = self.auth_handler.deactivate_valid_authorizations(orderr)
+            if deactivated:
+                logger.debug("Recreating order after authz deactivations")
+                orderr = self.acme.new_order(csr_pem)
+            if failed:
+                logger.warning("Certbot was unable to obtain fresh authorizations for every domain"
+                               ". The dry run will continue, but results may not be accurate.")
+
         authzr = self.auth_handler.handle_authorizations(orderr, best_effort)
         return orderr.update(authorizations=authzr)
 

certbot/compat/filesystem.py

@@ -546,13 +546,7 @@ def _generate_windows_flags(rights_desc):
     if rights_desc['write']:
         flag = flag | (ntsecuritycon.FILE_ALL_ACCESS
                        ^ ntsecuritycon.FILE_GENERIC_READ
-                       ^ ntsecuritycon.FILE_GENERIC_EXECUTE
-                       # Despite bit `512` being present in ntsecuritycon.FILE_ALL_ACCESS, it is
-                       # not effectively applied to the file or the directory.
-                       # As _generate_windows_flags is also used to compare two dacls, we remove
-                       # it right now to have flags that contain only the bits effectively applied
-                       # by Windows.
-                       ^ 512)
+                       ^ ntsecuritycon.FILE_GENERIC_EXECUTE)
     if rights_desc['execute']:
         flag = flag | ntsecuritycon.FILE_GENERIC_EXECUTE
 

certbot/main.py

@@ -1368,6 +1368,10 @@ def main(cli_args=None):
         if config.func != plugins_cmd:
             raise
 
+    if sys.version_info[:2] == (3, 4):
+        logger.warning("Python 3.4 support will be dropped in the next release "
+                    "of Certbot - please upgrade your Python version.")
+
     set_displayer(config)
 
     # Reporter

certbot/plugins/common.py

@@ -2,6 +2,7 @@
 import logging
 import re
 import shutil
+import sys
 import tempfile
 import warnings
 
@@ -503,3 +504,34 @@ def dir_setup(test_dir, pkg):  # pragma: no cover
         test_configs, os.path.join(temp_dir, test_dir), symlinks=True)
 
     return temp_dir, config_dir, work_dir
+
+
+# This class takes a similar approach to the cryptography project to deprecate attributes
+# in public modules. See the _ModuleWithDeprecation class here:
+# https://github.com/pyca/cryptography/blob/91105952739442a74582d3e62b3d2111365b0dc7/src/cryptography/utils.py#L129
+class _TLSSNI01DeprecationModule(object):
+    """
+    Internal class delegating to a module, and displaying warnings when
+    attributes related to TLS-SNI-01 are accessed.
+    """
+    def __init__(self, module):
+        self.__dict__['_module'] = module
+
+    def __getattr__(self, attr):
+        if attr == 'TLSSNI01':
+            warnings.warn('TLSSNI01 is deprecated and will be removed soon.',
+                          DeprecationWarning, stacklevel=2)
+        return getattr(self._module, attr)
+
+    def __setattr__(self, attr, value):  # pragma: no cover
+        setattr(self._module, attr, value)
+
+    def __delattr__(self, attr):  # pragma: no cover
+        delattr(self._module, attr)
+
+    def __dir__(self):  # pragma: no cover
+        return ['_module'] + dir(self._module)
+
+
+# Patching ourselves to warn about TLS-SNI challenge deprecation and removal.
+sys.modules[__name__] = _TLSSNI01DeprecationModule(sys.modules[__name__])

certbot/plugins/common_test.py

@@ -352,6 +352,11 @@ class TLSSNI01Test(unittest.TestCase):
         self.assertEqual(self.sni.get_z_domain(achall),
             achall.response(achall.account_key).z_domain.decode("utf-8"))
 
+    def test_warning(self):
+        with mock.patch('certbot.plugins.common.warnings.warn') as mock_warn:
+            from certbot.plugins.common import TLSSNI01  # pylint: disable=unused-variable
+        self.assertTrue(mock_warn.call_args[0][0].startswith('TLSSNI01'))
+
 
 class InstallVersionControlledFileTest(test_util.TempDirTestCase):
     """Tests for certbot.plugins.common.install_version_controlled_file."""

certbot/renewal.py

@@ -434,7 +434,7 @@ def handle_renewal_request(config):  # pylint: disable=too-many-locals,too-many-
                 if should_renew(lineage_config, renewal_candidate):
                     # Apply random sleep upon first renewal if needed
                     if apply_random_sleep:
-                        sleep_time = random.randint(1, 60 * 8)
+                        sleep_time = random.uniform(1, 60 * 8)
                         logger.info("Non-interactive renewal: random delay of %s seconds",
                                     sleep_time)
                         time.sleep(sleep_time)

certbot/tests/acme_util.py

@@ -18,12 +18,10 @@ KEY = util.load_rsa_private_key('rsa512_key.pem')
 # Challenges
 HTTP01 = challenges.HTTP01(
     token=b"evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ+PCt92wr+oA")
-TLSSNI01 = challenges.TLSSNI01(
-    token=jose.b64decode(b"evaGxfADs6pSRb2LAv9IZf17Dt3juxGJyPCt92wrDoA"))
 DNS01 = challenges.DNS01(token=b"17817c66b60ce2e4012dfad92657527a")
 DNS01_2 = challenges.DNS01(token=b"cafecafecafecafecafecafe0feedbac")
 
-CHALLENGES = [HTTP01, TLSSNI01, DNS01]
+CHALLENGES = [HTTP01, DNS01]
 
 
 def gen_combos(challbs):
@@ -47,21 +45,19 @@ def chall_to_challb(chall, status):  # pylint: disable=redefined-outer-name
 
 
 # Pending ChallengeBody objects
-TLSSNI01_P = chall_to_challb(TLSSNI01, messages.STATUS_PENDING)
 HTTP01_P = chall_to_challb(HTTP01, messages.STATUS_PENDING)
 DNS01_P = chall_to_challb(DNS01, messages.STATUS_PENDING)
 DNS01_P_2 = chall_to_challb(DNS01_2, messages.STATUS_PENDING)
 
-CHALLENGES_P = [HTTP01_P, TLSSNI01_P, DNS01_P]
+CHALLENGES_P = [HTTP01_P, DNS01_P]
 
 
 # AnnotatedChallenge objects
 HTTP01_A = auth_handler.challb_to_achall(HTTP01_P, JWK, "example.com")
-TLSSNI01_A = auth_handler.challb_to_achall(TLSSNI01_P, JWK, "example.net")
 DNS01_A = auth_handler.challb_to_achall(DNS01_P, JWK, "example.org")
 DNS01_A_2 = auth_handler.challb_to_achall(DNS01_P_2, JWK, "esimerkki.example.org")
 
-ACHALLENGES = [HTTP01_A, TLSSNI01_A, DNS01_A]
+ACHALLENGES = [HTTP01_A, DNS01_A]
 
 
 def gen_authzr(authz_status, domain, challs, statuses, combos=True):

certbot/tests/auth_handler_test.py

@@ -9,6 +9,7 @@ import zope.component
 from acme import challenges
 from acme import client as acme_client
 from acme import messages
+from acme import errors as acme_errors
 
 from certbot import achallenges
 from certbot import errors
@@ -39,11 +40,11 @@ class ChallengeFactoryTest(unittest.TestCase):
         self.assertEqual(
             [achall.chall for achall in achalls], acme_util.CHALLENGES)
 
-    def test_one_tls_sni(self):
-        achalls = self.handler._challenge_factory(self.authzr, [1])
+    def test_one_http(self):
+        achalls = self.handler._challenge_factory(self.authzr, [0])
 
         self.assertEqual(
-            [achall.chall for achall in achalls], [acme_util.TLSSNI01])
+            [achall.chall for achall in achalls], [acme_util.HTTP01])
 
     def test_unrecognized(self):
         authzr = acme_util.gen_authzr(
@@ -73,7 +74,7 @@ class HandleAuthorizationsTest(unittest.TestCase):  # pylint: disable=too-many-p
 
         self.mock_auth = mock.MagicMock(name="ApacheConfigurator")
 
-        self.mock_auth.get_chall_pref.return_value = [challenges.TLSSNI01]
+        self.mock_auth.get_chall_pref.return_value = [challenges.HTTP01]
 
         self.mock_auth.perform.side_effect = gen_auth_resp
 
@@ -90,7 +91,7 @@ class HandleAuthorizationsTest(unittest.TestCase):  # pylint: disable=too-many-p
     def tearDown(self):
         logging.disable(logging.NOTSET)
 
-    def _test_name1_tls_sni_01_1_common(self, combos):
+    def _test_name1_http_01_1_common(self, combos):
         authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos)
         mock_order = mock.MagicMock(authorizations=[authzr])
 
@@ -109,44 +110,42 @@ class HandleAuthorizationsTest(unittest.TestCase):  # pylint: disable=too-many-p
             self.assertTrue(mock_time.sleep.call_args_list[1][0][0] > 3)
 
             self.assertEqual(self.mock_auth.cleanup.call_count, 1)
-            # Test if list first element is TLSSNI01, use typ because it is an achall
+            # Test if list first element is http-01, use typ because it is an achall
             self.assertEqual(
-                self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01")
+                self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01")
 
             self.assertEqual(len(authzr), 1)
 
-    def test_name1_tls_sni_01_1_acme_1(self):
-        self._test_name1_tls_sni_01_1_common(combos=True)
+    def test_name1_http_01_1_acme_1(self):
+        self._test_name1_http_01_1_common(combos=True)
 
-    def test_name1_tls_sni_01_1_acme_2(self):
+    def test_name1_http_01_1_acme_2(self):
         self.mock_net.acme_version = 2
-        self._test_name1_tls_sni_01_1_common(combos=False)
+        self._test_name1_http_01_1_common(combos=False)
 
-    def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_1(self):
+    def test_name1_http_01_1_dns_1_acme_1(self):
         self.mock_net.poll.side_effect = _gen_mock_on_poll()
-        self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01)
         self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01)
 
         authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=False)
         mock_order = mock.MagicMock(authorizations=[authzr])
         authzr = self.handler.handle_authorizations(mock_order)
 
-        self.assertEqual(self.mock_net.answer_challenge.call_count, 3)
+        self.assertEqual(self.mock_net.answer_challenge.call_count, 2)
 
         self.assertEqual(self.mock_net.poll.call_count, 1)
 
         self.assertEqual(self.mock_auth.cleanup.call_count, 1)
-        # Test if list first element is TLSSNI01, use typ because it is an achall
+        # Test if list first element is http-01, use typ because it is an achall
         for achall in self.mock_auth.cleanup.call_args[0][0]:
-            self.assertTrue(achall.typ in ["tls-sni-01", "http-01", "dns-01"])
+            self.assertTrue(achall.typ in ["http-01", "dns-01"])
 
         # Length of authorizations list
         self.assertEqual(len(authzr), 1)
 
-    def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_2(self):
+    def test_name1_http_01_1_dns_1_acme_2(self):
         self.mock_net.acme_version = 2
         self.mock_net.poll.side_effect = _gen_mock_on_poll()
-        self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01)
         self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01)
 
         authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=False)
@@ -160,12 +159,12 @@ class HandleAuthorizationsTest(unittest.TestCase):  # pylint: disable=too-many-p
         self.assertEqual(self.mock_auth.cleanup.call_count, 1)
         cleaned_up_achalls = self.mock_auth.cleanup.call_args[0][0]
         self.assertEqual(len(cleaned_up_achalls), 1)
-        self.assertEqual(cleaned_up_achalls[0].typ, "tls-sni-01")
+        self.assertEqual(cleaned_up_achalls[0].typ, "http-01")
 
         # Length of authorizations list
         self.assertEqual(len(authzr), 1)
 
-    def _test_name3_tls_sni_01_3_common(self, combos):
+    def _test_name3_http_01_3_common(self, combos):
         self.mock_net.request_domain_challenges.side_effect = functools.partial(
             gen_dom_authzr, challs=acme_util.CHALLENGES, combos=combos)
 
@@ -186,12 +185,12 @@ class HandleAuthorizationsTest(unittest.TestCase):  # pylint: disable=too-many-p
 
         self.assertEqual(len(authzr), 3)
 
-    def test_name3_tls_sni_01_3_common_acme_1(self):
-        self._test_name3_tls_sni_01_3_common(combos=True)
+    def test_name3_http_01_3_common_acme_1(self):
+        self._test_name3_http_01_3_common(combos=True)
 
-    def test_name3_tls_sni_01_3_common_acme_2(self):
+    def test_name3_http_01_3_common_acme_2(self):
         self.mock_net.acme_version = 2
-        self._test_name3_tls_sni_01_3_common(combos=False)
+        self._test_name3_http_01_3_common(combos=False)
 
     def test_debug_challenges(self):
         zope.component.provideUtility(
@@ -257,7 +256,7 @@ class HandleAuthorizationsTest(unittest.TestCase):  # pylint: disable=too-many-p
     def _test_preferred_challenges_not_supported_common(self, combos):
         authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos)]
         mock_order = mock.MagicMock(authorizations=authzrs)
-        self.handler.pref_challs.append(challenges.HTTP01.typ)
+        self.handler.pref_challs.append(challenges.DNS01.typ)
         self.assertRaises(
             errors.AuthorizationError, self.handler.handle_authorizations, mock_order)
 
@@ -283,7 +282,7 @@ class HandleAuthorizationsTest(unittest.TestCase):  # pylint: disable=too-many-p
 
         self.assertEqual(self.mock_auth.cleanup.call_count, 1)
         self.assertEqual(
-            self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01")
+            self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01")
 
     def test_answer_error(self):
         self.mock_net.answer_challenge.side_effect = errors.AuthorizationError
@@ -295,7 +294,7 @@ class HandleAuthorizationsTest(unittest.TestCase):  # pylint: disable=too-many-p
             errors.AuthorizationError, self.handler.handle_authorizations, mock_order)
         self.assertEqual(self.mock_auth.cleanup.call_count, 1)
         self.assertEqual(
-            self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01")
+            self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01")
 
     def test_incomplete_authzr_error(self):
         authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)]
@@ -308,7 +307,7 @@ class HandleAuthorizationsTest(unittest.TestCase):  # pylint: disable=too-many-p
         self.assertTrue('Some challenges have failed.' in str(error.exception))
         self.assertEqual(self.mock_auth.cleanup.call_count, 1)
         self.assertEqual(
-            self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01")
+            self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01")
 
     def test_best_effort(self):
         def _conditional_mock_on_poll(authzr):
@@ -344,27 +343,58 @@ class HandleAuthorizationsTest(unittest.TestCase):  # pylint: disable=too-many-p
         self.assertTrue('All challenges have failed.' in str(error.exception))
 
     def test_validated_challenge_not_rerun(self):
-        # With pending challenge, we expect the challenge to be tried, and fail.
+        # With a pending challenge that is not supported by the plugin, we
+        # expect an exception to be raised.
         authzr = acme_util.gen_authzr(
                 messages.STATUS_PENDING, "0",
-                [acme_util.HTTP01],
+                [acme_util.DNS01],
                 [messages.STATUS_PENDING], False)
         mock_order = mock.MagicMock(authorizations=[authzr])
         self.assertRaises(
             errors.AuthorizationError, self.handler.handle_authorizations, mock_order)
 
-        # With validated challenge; we expect the challenge not be tried again, and succeed.
+        # With a validated challenge that is not supported by the plugin, we
+        # expect the challenge to not be solved again and
+        # handle_authorizations() to succeed.
         authzr = acme_util.gen_authzr(
                 messages.STATUS_VALID, "0",
-                [acme_util.HTTP01],
+                [acme_util.DNS01],
                 [messages.STATUS_VALID], False)
         mock_order = mock.MagicMock(authorizations=[authzr])
         self.handler.handle_authorizations(mock_order)
 
-    @mock.patch("certbot.auth_handler.logger")
-    def test_tls_sni_logs(self, logger):
-        self._test_name1_tls_sni_01_1_common(combos=True)
-        self.assertTrue("deprecated" in logger.warning.call_args[0][0])
+    def test_valid_authzrs_deactivated(self):
+        """When we deactivate valid authzrs in an orderr, we expect them to become deactivated
+        and to receive a list of deactivated authzrs in return."""
+        def _mock_deactivate(authzr):
+            if authzr.body.status == messages.STATUS_VALID:
+                if authzr.body.identifier.value == "is_valid_but_will_fail":
+                    raise acme_errors.Error("Mock deactivation ACME error")
+                authzb = authzr.body.update(status=messages.STATUS_DEACTIVATED)
+                authzr = messages.AuthorizationResource(body=authzb)
+            else: # pragma: no cover
+                raise errors.Error("Can't deactivate non-valid authz")
+            return authzr
+
+        to_deactivate = [("is_valid", messages.STATUS_VALID),
+                         ("is_pending", messages.STATUS_PENDING),
+                         ("is_valid_but_will_fail", messages.STATUS_VALID)]
+
+        to_deactivate = [acme_util.gen_authzr(a[1], a[0], [acme_util.HTTP01],
+                         [a[1], False]) for a in to_deactivate]
+        orderr = mock.MagicMock(authorizations=to_deactivate)
+
+        self.mock_net.deactivate_authorization.side_effect = _mock_deactivate
+
+        authzrs, failed = self.handler.deactivate_valid_authorizations(orderr)
+
+        self.assertEqual(self.mock_net.deactivate_authorization.call_count, 2)
+        self.assertEqual(len(authzrs), 1)
+        self.assertEqual(len(failed), 1)
+        self.assertEqual(authzrs[0].body.identifier.value, "is_valid")
+        self.assertEqual(authzrs[0].body.status, messages.STATUS_DEACTIVATED)
+        self.assertEqual(failed[0].body.identifier.value, "is_valid_but_will_fail")
+        self.assertEqual(failed[0].body.status, messages.STATUS_VALID)
 
 
 def _gen_mock_on_poll(status=messages.STATUS_VALID, retry=0, wait_value=1):
@@ -417,9 +447,9 @@ class GenChallengePathTest(unittest.TestCase):
         return gen_challenge_path(challbs, preferences, combinations)
 
     def test_common_case(self):
-        """Given TLSSNI01 and HTTP01 with appropriate combos."""
-        challbs = (acme_util.TLSSNI01_P, acme_util.HTTP01_P)
-        prefs = [challenges.TLSSNI01, challenges.HTTP01]
+        """Given DNS01 and HTTP01 with appropriate combos."""
+        challbs = (acme_util.DNS01_P, acme_util.HTTP01_P)
+        prefs = [challenges.DNS01, challenges.HTTP01]
         combos = ((0,), (1,))
 
         # Smart then trivial dumb path test
@@ -430,8 +460,8 @@ class GenChallengePathTest(unittest.TestCase):
         self.assertTrue(self._call(challbs[::-1], prefs, None))
 
     def test_not_supported(self):
-        challbs = (acme_util.DNS01_P, acme_util.TLSSNI01_P)
-        prefs = [challenges.TLSSNI01]
+        challbs = (acme_util.DNS01_P, acme_util.HTTP01_P)
+        prefs = [challenges.HTTP01]
         combos = ((0, 1),)
 
         # smart path fails because no challs in perfs satisfies combos
@@ -459,19 +489,19 @@ class ReportFailedAuthzrsTest(unittest.TestCase):
 
         http_01 = messages.ChallengeBody(**kwargs)
 
-        kwargs["chall"] = acme_util.TLSSNI01
-        tls_sni_01 = messages.ChallengeBody(**kwargs)
+        kwargs["chall"] = acme_util.HTTP01
+        http_01 = messages.ChallengeBody(**kwargs)
 
         self.authzr1 = mock.MagicMock()
         self.authzr1.body.identifier.value = 'example.com'
-        self.authzr1.body.challenges = [http_01, tls_sni_01]
+        self.authzr1.body.challenges = [http_01, http_01]
 
         kwargs["error"] = messages.Error(typ="dnssec", detail="detail")
-        tls_sni_01_diff = messages.ChallengeBody(**kwargs)
+        http_01_diff = messages.ChallengeBody(**kwargs)
 
         self.authzr2 = mock.MagicMock()
         self.authzr2.body.identifier.value = 'foo.bar'
-        self.authzr2.body.challenges = [tls_sni_01_diff]
+        self.authzr2.body.challenges = [http_01_diff]
 
     @test_util.patch_get_utility()
     def test_same_error_and_domain(self, mock_zope):

certbot/tests/cli_test.py

@@ -249,13 +249,6 @@ class ParseTest(unittest.TestCase):  # pylint: disable=too-many-public-methods
         expected = [challenges.HTTP01.typ, challenges.DNS01.typ]
         self.assertEqual(namespace.pref_challs, expected)
 
-        # TODO: to be removed once tls-sni deprecation logic is removed
-        with mock.patch('certbot.cli.logger.warning') as mock_warn:
-            self.assertEqual(self.parse(['--preferred-challenges', 'http, tls-sni']).pref_challs,
-                             [challenges.HTTP01.typ])
-        self.assertEqual(mock_warn.call_count, 1)
-        self.assertTrue('deprecated' in mock_warn.call_args[0][0])
-
         short_args = ['--preferred-challenges', 'jumping-over-the-moon']
         # argparse.ArgumentError makes argparse print more information
         # to stderr and call sys.exit()
@@ -272,16 +265,6 @@ class ParseTest(unittest.TestCase):  # pylint: disable=too-many-public-methods
         self.assertTrue(namespace.must_staple)
         self.assertTrue(namespace.staple)
 
-    def test_no_gui(self):
-        args = ['renew', '--dialog']
-        with mock.patch("certbot.util.logger.warning") as mock_warn:
-            namespace = self.parse(args)
-
-        self.assertTrue(namespace.noninteractive_mode)
-        self.assertEqual(mock_warn.call_count, 1)
-        self.assertTrue("is deprecated" in mock_warn.call_args[0][0])
-        self.assertEqual("--dialog", mock_warn.call_args[0][1])
-
     def _check_server_conflict_message(self, parser_args, conflicting_args):
         try:
             self.parse(parser_args)
@@ -333,16 +316,26 @@ class ParseTest(unittest.TestCase):  # pylint: disable=too-many-public-methods
 
         self._assert_dry_run_flag_worked(self.parse(short_args + ['auth']), True)
         self._assert_dry_run_flag_worked(self.parse(short_args + ['renew']), True)
+        self._assert_dry_run_flag_worked(self.parse(short_args + ['certonly']), True)
+
         short_args += ['certonly']
-        self._assert_dry_run_flag_worked(self.parse(short_args), True)
 
-        short_args += '--server example.com'.split()
-        conflicts = ['--dry-run']
-        self._check_server_conflict_message(short_args, '--dry-run')
+        # `--dry-run --server example.com` should emit example.com
+        self.assertEqual(self.parse(short_args + ['--server', 'example.com']).server,
+                         'example.com')
 
-        short_args += ['--staging']
-        conflicts += ['--staging']
-        self._check_server_conflict_message(short_args, conflicts)
+        # `--dry-run --server STAGING_URI` should emit STAGING_URI
+        self.assertEqual(self.parse(short_args + ['--server', constants.STAGING_URI]).server,
+                         constants.STAGING_URI)
+
+        # `--dry-run --server LIVE` should emit STAGING_URI
+        self.assertEqual(self.parse(short_args + ['--server', cli.flag_default("server")]).server,
+                         constants.STAGING_URI)
+
+        # `--dry-run --server example.com --staging` should emit an error
+        conflicts = ['--staging']
+        self._check_server_conflict_message(short_args + ['--server', 'example.com', '--staging'],
+                                            conflicts)
 
     def test_option_was_set(self):
         key_size_option = 'rsa_key_size'

certbot/tests/client_test.py

@@ -256,6 +256,7 @@ class ClientTest(ClientTestCommon):
     def _mock_obtain_certificate(self):
         self.client.auth_handler = mock.MagicMock()
         self.client.auth_handler.handle_authorizations.return_value = [None]
+        self.client.auth_handler.deactivate_valid_authorizations.return_value = ([], [])
         self.acme.finalize_order.return_value = self.eg_order
         self.acme.new_order.return_value = self.eg_order
         self.eg_order.update.return_value = self.eg_order
@@ -360,6 +361,47 @@ class ClientTest(ClientTestCommon):
         mock_crypto.init_save_csr.assert_not_called()
         self.assertEqual(mock_crypto.cert_and_chain_from_fullchain.call_count, 1)
 
+    @mock.patch("certbot.client.logger")
+    @mock.patch("certbot.client.crypto_util")
+    @mock.patch("certbot.client.acme_crypto_util")
+    def test_obtain_certificate_dry_run_authz_deactivations_failed(self, mock_acme_crypto,
+                                                                   mock_crypto, mock_log):
+        from acme import messages
+        csr = util.CSR(form="pem", file=None, data=CSR_SAN)
+        mock_acme_crypto.make_csr.return_value = CSR_SAN
+        mock_crypto.make_key.return_value = mock.sentinel.key_pem
+        key = util.Key(file=None, pem=mock.sentinel.key_pem)
+        self._set_mock_from_fullchain(mock_crypto.cert_and_chain_from_fullchain)
+
+        self._mock_obtain_certificate()
+        self.client.config.dry_run = True
+
+        # Two authzs that are already valid and should get deactivated (dry run)
+        authzrs = self._authzr_from_domains(["example.com", "www.example.com"])
+        for authzr in authzrs:
+            authzr.body.status = messages.STATUS_VALID
+
+        # One deactivation succeeds, one fails
+        auth_handler = self.client.auth_handler
+        auth_handler.deactivate_valid_authorizations.return_value = ([authzrs[0]], [authzrs[1]])
+
+        # Certificate should get issued despite one failed deactivation
+        self.eg_order.authorizations = authzrs
+        self.client.auth_handler.handle_authorizations.return_value = authzrs
+        with test_util.patch_get_utility():
+            result = self.client.obtain_certificate(self.eg_domains)
+        self.assertEqual(result, (mock.sentinel.cert, mock.sentinel.chain, key, csr))
+        self._check_obtain_certificate(1)
+
+        # Deactivation success/failure should have been handled properly
+        self.assertEqual(auth_handler.deactivate_valid_authorizations.call_count, 1,
+                        "Deactivate authorizations should be called")
+        self.assertEqual(self.acme.new_order.call_count, 2,
+                        "Order should be recreated due to successfully deactivated authorizations")
+        mock_log.warning.assert_called_with("Certbot was unable to obtain fresh authorizations for"
+                                            " every domain. The dry run will continue, but results"
+                                            " may not be accurate.")
+
     def _set_mock_from_fullchain(self, mock_from_fullchain):
         mock_cert = mock.Mock()
         mock_cert.encode.return_value = mock.sentinel.cert

certbot/tests/compat/filesystem_test.py

@@ -89,8 +89,8 @@ class WindowsChmodTests(TempDirTestCase):
         self.assertEqual(len(system_aces), 1)
         self.assertEqual(len(admin_aces), 1)
 
-        self.assertEqual(system_aces[0][1], ntsecuritycon.FILE_ALL_ACCESS ^ 512)
-        self.assertEqual(admin_aces[0][1], ntsecuritycon.FILE_ALL_ACCESS ^ 512)
+        self.assertEqual(system_aces[0][1], ntsecuritycon.FILE_ALL_ACCESS)
+        self.assertEqual(admin_aces[0][1], ntsecuritycon.FILE_ALL_ACCESS)
 
     def test_read_flag(self):
         self._test_flag(4, ntsecuritycon.FILE_GENERIC_READ)
@@ -101,12 +101,10 @@ class WindowsChmodTests(TempDirTestCase):
     def test_write_flag(self):
         self._test_flag(2, (ntsecuritycon.FILE_ALL_ACCESS
                             ^ ntsecuritycon.FILE_GENERIC_READ
-                            ^ ntsecuritycon.FILE_GENERIC_EXECUTE
-                            ^ 512))
+                            ^ ntsecuritycon.FILE_GENERIC_EXECUTE))
 
     def test_full_flag(self):
-        self._test_flag(7, (ntsecuritycon.FILE_ALL_ACCESS
-                            ^ 512))
+        self._test_flag(7, ntsecuritycon.FILE_ALL_ACCESS)
 
     def _test_flag(self, everyone_mode, windows_flag):
         # Note that flag is tested against `everyone`, not `user`, because practically these unit

certbot/tests/main_test.py

@@ -1381,11 +1381,6 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
                     jose.ComparableX509(cert),
                     mock.ANY)
 
-    def test_agree_dev_preview_config(self):
-        with mock.patch('certbot.main.run') as mocked_run:
-            self._call(['-c', test_util.vector_path('cli.ini')])
-        self.assertTrue(mocked_run.called)
-
     @mock.patch('certbot.log.post_arg_parse_setup')
     def test_register(self, _):
         with mock.patch('certbot.main.client') as mocked_client:

certbot/tests/renewal_test.py

@@ -75,15 +75,10 @@ class RestoreRequiredConfigElementsTest(test_util.ConfigTestCase):
     @mock.patch('certbot.renewal.cli.set_by_cli')
     def test_pref_challs_list(self, mock_set_by_cli):
         mock_set_by_cli.return_value = False
-        # TODO: remove tls-sni and related assertions to logger.warning call once
-        #  the deprecation logic has been removed
-        renewalparams = {'pref_challs': 'tls-sni, http-01, dns'.split(',')}
-        with mock.patch('certbot.renewal.cli.logger.warning') as mock_warn:
-            self._call(self.config, renewalparams)
+        renewalparams = {'pref_challs': 'http-01, dns'.split(',')}
+        self._call(self.config, renewalparams)
         expected = [challenges.HTTP01.typ, challenges.DNS01.typ]
         self.assertEqual(self.config.pref_challs, expected)
-        self.assertEqual(mock_warn.call_count, 1)
-        self.assertTrue('deprecated' in mock_warn.call_args[0][0])
 
     @mock.patch('certbot.renewal.cli.set_by_cli')
     def test_pref_challs_str(self, mock_set_by_cli):

certbot/tests/util.py

@@ -94,26 +94,6 @@ def load_pyopenssl_private_key(*names):
     return OpenSSL.crypto.load_privatekey(loader, load_vector(*names))
 
 
-def skip_unless(condition, reason):  # pragma: no cover
-    """Skip tests unless a condition holds.
-
-    This implements the basic functionality of unittest.skipUnless
-    which is only available on Python 2.7+.
-
-    :param bool condition: If ``False``, the test will be skipped
-    :param str reason: the reason for skipping the test
-
-    :rtype: callable
-    :returns: decorator that hides tests unless condition is ``True``
-
-    """
-    if hasattr(unittest, "skipUnless"):
-        return unittest.skipUnless(condition, reason)
-    elif condition:
-        return lambda cls: cls
-    return lambda cls: None
-
-
 def make_lineage(config_dir, testfile):
     """Creates a lineage defined by testfile.
 

certbot/tests/util_test.py

@@ -1,6 +1,7 @@
 """Tests for certbot.util."""
 import argparse
 import errno
+import sys
 import unittest
 
 import mock
@@ -473,74 +474,92 @@ class IsWildcardDomainTest(unittest.TestCase):
 class OsInfoTest(unittest.TestCase):
     """Test OS / distribution detection"""
 
-    def test_systemd_os_release(self):
-        from certbot.util import (get_os_info, get_systemd_os_info,
-                                  get_os_info_ua)
+    @mock.patch("certbot.util.distro")
+    @unittest.skipUnless(sys.platform.startswith("linux"), "requires Linux")
+    def test_systemd_os_release_like(self, m_distro):
+        import certbot.util as cbutil
+        m_distro.like.return_value = "first debian third"
+        id_likes = cbutil.get_systemd_os_like()
+        self.assertEqual(len(id_likes), 3)
+        self.assertTrue("debian" in id_likes)
 
-        with mock.patch('certbot.compat.os.path.isfile', return_value=True):
-            self.assertEqual(get_os_info(
-                test_util.vector_path("os-release"))[0], 'systemdos')
-            self.assertEqual(get_os_info(
-                test_util.vector_path("os-release"))[1], '42')
-            self.assertEqual(get_systemd_os_info(os.devnull), ("", ""))
-            self.assertEqual(get_os_info_ua(
-                test_util.vector_path("os-release")), "SystemdOS")
-        with mock.patch('certbot.compat.os.path.isfile', return_value=False):
-            self.assertEqual(get_systemd_os_info(), ("", ""))
+    @mock.patch("certbot.util.distro")
+    @unittest.skipUnless(sys.platform.startswith("linux"), "requires Linux")
+    def test_get_os_info_ua(self, m_distro):
+        import certbot.util as cbutil
+        with mock.patch('platform.system_alias',
+                        return_value=('linux', '42', '42')):
+            m_distro.name.return_value = ""
+            m_distro.linux_distribution.return_value = ("something", "1.0", "codename")
+            cbutil.get_python_os_info(pretty=True)
+            self.assertEqual(cbutil.get_os_info_ua(),
+                            " ".join(cbutil.get_python_os_info(pretty=True)))
 
-    def test_systemd_os_release_like(self):
-        from certbot.util import get_systemd_os_like
+        m_distro.name.return_value = "whatever"
+        self.assertEqual(cbutil.get_os_info_ua(), "whatever")
 
-        with mock.patch('certbot.compat.os.path.isfile', return_value=True):
-            id_likes = get_systemd_os_like(test_util.vector_path(
-                "os-release"))
-            self.assertEqual(len(id_likes), 3)
-            self.assertTrue("debian" in id_likes)
+    @mock.patch("certbot.util.distro")
+    @unittest.skipUnless(sys.platform.startswith("linux"), "requires Linux")
+    def test_get_os_info(self, m_distro):
+        import certbot.util as cbutil
+        with mock.patch("platform.system") as mock_platform:
+            m_distro.linux_distribution.return_value = ("name", "version", 'x')
+            mock_platform.return_value = "linux"
+            self.assertEqual(cbutil.get_os_info(), ("name", "version"))
+
+            m_distro.linux_distribution.return_value = ("something", "else")
+            self.assertEqual(cbutil.get_os_info(), ("something", "else"))
+
+    @mock.patch("warnings.warn")
+    @mock.patch("certbot.util.distro")
+    @unittest.skipUnless(sys.platform.startswith("linux"), "requires Linux")
+    def test_get_systemd_os_info_deprecation(self, _, mock_warn):
+        import certbot.util as cbutil
+        cbutil.get_systemd_os_info()
+        self.assertTrue(mock_warn.called)
 
     @mock.patch("certbot.util.subprocess.Popen")
     def test_non_systemd_os_info(self, popen_mock):
-        from certbot.util import (get_os_info, get_python_os_info,
-                                     get_os_info_ua)
-        with mock.patch('certbot.compat.os.path.isfile', return_value=False):
+        import certbot.util as cbutil
+        with mock.patch('certbot.util._USE_DISTRO', False):
             with mock.patch('platform.system_alias',
                             return_value=('NonSystemD', '42', '42')):
-                self.assertEqual(get_os_info()[0], 'nonsystemd')
-                self.assertEqual(get_os_info_ua(),
-                                 " ".join(get_python_os_info()))
+                self.assertEqual(cbutil.get_python_os_info()[0], 'nonsystemd')
 
             with mock.patch('platform.system_alias',
                             return_value=('darwin', '', '')):
                 comm_mock = mock.Mock()
                 comm_attrs = {'communicate.return_value':
-                              ('42.42.42', 'error')}
+                            ('42.42.42', 'error')}
                 comm_mock.configure_mock(**comm_attrs)
                 popen_mock.return_value = comm_mock
-                self.assertEqual(get_os_info()[0], 'darwin')
-                self.assertEqual(get_os_info()[1], '42.42.42')
-
-            with mock.patch('platform.system_alias',
-                            return_value=('linux', '', '')):
-                with mock.patch('platform.linux_distribution',
-                                side_effect=AttributeError,
-                                create=True):
-                    with mock.patch('distro.linux_distribution',
-                                    return_value=('', '', '')):
-                        self.assertEqual(get_python_os_info(), ("linux", ""))
-
-                    with mock.patch('distro.linux_distribution',
-                                    return_value=('testdist', '42', '')):
-                        self.assertEqual(get_python_os_info(), ("testdist", "42"))
+                self.assertEqual(cbutil.get_python_os_info()[0], 'darwin')
+                self.assertEqual(cbutil.get_python_os_info()[1], '42.42.42')
 
             with mock.patch('platform.system_alias',
                             return_value=('freebsd', '9.3-RC3-p1', '')):
-                self.assertEqual(get_python_os_info(), ("freebsd", "9"))
+                self.assertEqual(cbutil.get_python_os_info(), ("freebsd", "9"))
 
             with mock.patch('platform.system_alias',
                             return_value=('windows', '', '')):
                 with mock.patch('platform.win32_ver',
                                 return_value=('4242', '95', '2', '')):
-                    self.assertEqual(get_python_os_info(),
-                                     ("windows", "95"))
+                    self.assertEqual(cbutil.get_python_os_info(),
+                                    ("windows", "95"))
+
+    @mock.patch("certbot.util.distro")
+    @unittest.skipUnless(sys.platform.startswith("linux"), "requires Linux")
+    def test_python_os_info_notfound(self, m_distro):
+        import certbot.util as cbutil
+        m_distro.linux_distribution.return_value = ('', '', '')
+        self.assertEqual(cbutil.get_python_os_info()[0], "linux")
+
+    @mock.patch("certbot.util.distro")
+    @unittest.skipUnless(sys.platform.startswith("linux"), "requires Linux")
+    def test_python_os_info_custom(self, m_distro):
+        import certbot.util as cbutil
+        m_distro.linux_distribution.return_value = ('testdist', '42', '')
+        self.assertEqual(cbutil.get_python_os_info(), ("testdist", "42"))
 
 
 class AtexitRegisterTest(unittest.TestCase):

certbot/util.py

@@ -12,9 +12,10 @@ import platform
 import re
 import socket
 import subprocess
+import sys
+import warnings
 
 import configargparse
-import distro
 import six
 
 from acme.magic_typing import Tuple, Union  # pylint: disable=unused-import, no-name-in-module
@@ -25,6 +26,12 @@ from certbot import lock
 from certbot.compat import os
 from certbot.compat import filesystem
 
+if sys.platform.startswith('linux'):
+    import distro
+    _USE_DISTRO = True
+else:
+    _USE_DISTRO = False
+
 logger = logging.getLogger(__name__)
 
 
@@ -277,77 +284,59 @@ def get_filtered_names(all_names):
             logger.debug('Not suggesting name "%s"', name, exc_info=True)
     return filtered_names
 
-
-def get_os_info(filepath="/etc/os-release"):
+def get_os_info():
     """
     Get OS name and version
 
-    :param str filepath: File path of os-release file
     :returns: (os_name, os_version)
     :rtype: `tuple` of `str`
     """
 
-    if os.path.isfile(filepath):
-        # Systemd os-release parsing might be viable
-        os_name, os_version = get_systemd_os_info(filepath=filepath)
-        if os_name:
-            return os_name, os_version
+    return get_python_os_info(pretty=False)
 
-    # Fallback to platform module
-    return get_python_os_info()
-
-
-def get_os_info_ua(filepath="/etc/os-release"):
+def get_os_info_ua():
     """
     Get OS name and version string for User Agent
 
-    :param str filepath: File path of os-release file
     :returns: os_ua
     :rtype: `str`
     """
+    if _USE_DISTRO:
+        os_info = distro.name(pretty=True)
 
-    if os.path.isfile(filepath):
-        os_ua = get_var_from_file("PRETTY_NAME", filepath=filepath)
-        if not os_ua:
-            os_ua = get_var_from_file("NAME", filepath=filepath)
-        if os_ua:
-            return os_ua
+    if not _USE_DISTRO or not os_info:
+        return " ".join(get_python_os_info(pretty=True))
+    return os_info
 
-    # Fallback
-    return " ".join(get_python_os_info())
-
-
-def get_systemd_os_info(filepath="/etc/os-release"):
+def get_systemd_os_info():
     """
     Parse systemd /etc/os-release for distribution information
 
-    :param str filepath: File path of os-release file
     :returns: (os_name, os_version)
     :rtype: `tuple` of `str`
     """
 
-    os_name = get_var_from_file("ID", filepath=filepath)
-    os_version = get_var_from_file("VERSION_ID", filepath=filepath)
+    warnings.warn(
+        "The get_sytemd_os_like() function is deprecated and will be removed in "
+        "a future release.", DeprecationWarning, stacklevel=2)
+    return get_os_info()[:2]
 
-    return (os_name, os_version)
-
-
-def get_systemd_os_like(filepath="/etc/os-release"):
+def get_systemd_os_like():
     """
     Get a list of strings that indicate the distribution likeness to
     other distributions.
 
-    :param str filepath: File path of os-release file
     :returns: List of distribution acronyms
     :rtype: `list` of `str`
     """
 
-    return get_var_from_file("ID_LIKE", filepath).split(" ")
-
+    if _USE_DISTRO:
+        return distro.like().split(" ")
+    return []
 
 def get_var_from_file(varname, filepath="/etc/os-release"):
     """
-    Get single value from systemd /etc/os-release
+    Get single value from a file formatted like systemd /etc/os-release
 
     :param str varname: Name of variable to fetch
     :param str filepath: File path of os-release file
@@ -367,7 +356,6 @@ def get_var_from_file(varname, filepath="/etc/os-release"):
             return _normalize_string(line.strip()[len(var_string):])
     return ""
 
-
 def _normalize_string(orig):
     """
     Helper function for get_var_from_file() to remove quotes
@@ -375,12 +363,13 @@ def _normalize_string(orig):
     """
     return orig.replace('"', '').replace("'", "").strip()
 
-
-def get_python_os_info():
+def get_python_os_info(pretty=False):
     """
     Get Operating System type/distribution and major version
     using python platform module
 
+    :param bool pretty: If the returned OS name should be in longer (pretty) form
+
     :returns: (os_name, os_version)
     :rtype: `tuple` of `str`
     """
@@ -391,8 +380,8 @@ def get_python_os_info():
     )
     os_type, os_ver, _ = info
     os_type = os_type.lower()
-    if os_type.startswith('linux'):
-        info = _get_linux_distribution()
+    if os_type.startswith('linux') and _USE_DISTRO:
+        info = distro.linux_distribution(pretty)
         # On arch, distro.linux_distribution() is reportedly ('','',''),
         # so handle it defensively
         if info[0]:
@@ -424,14 +413,6 @@ def get_python_os_info():
         os_ver = ''
     return os_type, os_ver
 
-def _get_linux_distribution():
-    """Gets the linux distribution name from the underlying OS"""
-
-    try:
-        return platform.linux_distribution()
-    except AttributeError:
-        return distro.linux_distribution()
-
 # Just make sure we don't get pwned... Make sure that it also doesn't
 # start with a period or have two consecutive periods <- this needs to
 # be done in addition to the regex

docs/cli-help.txt

@@ -113,7 +113,7 @@ optional arguments:
                         case, and to know when to deprecate support for past
                         Python versions and flags. If you wish to hide this
                         information from the Let's Encrypt server, set this to
-                        "". (default: CertbotACMEClient/0.39.0
+                        "". (default: CertbotACMEClient/0.40.0
                         (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX
                         Installer/YYY (SUBCOMMAND; flags: FLAGS)
                         Py/major.minor.patchlevel). The flags encoded in the

docs/contributing.rst

@@ -197,15 +197,20 @@ using an HTTP-01 challenge on a machine with Python 3:
 Code components and layout
 ==========================
 
+The following components of the Certbot repository are distributed to users:
+
 acme
   contains all protocol specific code
 certbot
   main client code
 certbot-apache and certbot-nginx
   client code to configure specific web servers
-certbot.egg-info
-  configuration for packaging Certbot
-
+certbot-dns-*
+  client code to configure DNS providers
+certbot-auto and letsencrypt-auto
+  shell scripts to install Certbot and its dependencies on UNIX systems
+windows installer
+  Installs Certbot on Windows and is built using the files in windows-installer/
 
 Plugin-architecture
 -------------------
@@ -234,7 +239,7 @@ Authenticators
 
 Authenticators are plugins that prove control of a domain name by solving a
 challenge provided by the ACME server. ACME currently defines several types of
-challenges: HTTP, TLS-SNI (deprecated), TLS-ALPR, and DNS, represented by classes in `acme.challenges`.
+challenges: HTTP, TLS-ALPN, and DNS, represented by classes in `acme.challenges`.
 An authenticator plugin should implement support for at least one challenge type.
 
 An Authenticator indicates which challenges it supports by implementing

docs/using.rst

@@ -917,8 +917,9 @@ Certbot accepts a global configuration file that applies its options to all invo
 of Certbot. Certificate specific configuration choices should be set in the ``.conf``
 files that can be found in ``/etc/letsencrypt/renewal``.
 
-By default no cli.ini file is created, after creating one
-it is possible to specify the location of this configuration file with
+By default no cli.ini file is created (though it may exist already if you installed Certbot
+via a package manager, for instance).
+After creating one it is possible to specify the location of this configuration file with
 ``certbot --config cli.ini`` (or shorter ``-c cli.ini``). An
 example configuration file is shown below:
 

letsencrypt-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.39.0"
+LE_AUTO_VERSION="0.40.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -256,20 +256,28 @@ DeprecationBootstrap() {
   fi
 }
 
-MIN_PYTHON_VERSION="2.7"
-MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//')
+MIN_PYTHON_2_VERSION="2.7"
+MIN_PYVER2=$(echo "$MIN_PYTHON_2_VERSION" | sed 's/\.//')
+MIN_PYTHON_3_VERSION="3.5"
+MIN_PYVER3=$(echo "$MIN_PYTHON_3_VERSION" | sed 's/\.//')
 # Sets LE_PYTHON to Python version string and PYVER to the first two
-# digits of the python version
+# digits of the python version.
+# MIN_PYVER and MIN_PYTHON_VERSION are also set by this function, and their
+# values depend on if we try to use Python 3 or Python 2.
 DeterminePythonVersion() {
   # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
   #
   # If no Python is found, PYVER is set to 0.
   if [ "$USE_PYTHON_3" = 1 ]; then
+    MIN_PYVER=$MIN_PYVER3
+    MIN_PYTHON_VERSION=$MIN_PYTHON_3_VERSION
     for LE_PYTHON in "$LE_PYTHON" python3; do
       # Break (while keeping the LE_PYTHON value) if found.
       $EXISTS "$LE_PYTHON" > /dev/null && break
     done
   else
+    MIN_PYVER=$MIN_PYVER2
+    MIN_PYTHON_VERSION=$MIN_PYTHON_2_VERSION
     for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
       # Break (while keeping the LE_PYTHON value) if found.
       $EXISTS "$LE_PYTHON" > /dev/null && break
@@ -285,7 +293,7 @@ DeterminePythonVersion() {
     fi
   fi
 
-  PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
+  PYVER=$("$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//')
   if [ "$PYVER" -lt "$MIN_PYVER" ]; then
     if [ "$1" != "NOCRASH" ]; then
       error "You have an ancient version of Python entombed in your operating system..."
@@ -368,7 +376,9 @@ BootstrapDebCommon() {
 
 # Sets TOOL to the name of the package manager
 # Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG.
-# Enables EPEL if applicable and possible.
+# Note: this function is called both while selecting the bootstrap scripts and
+# during the actual bootstrap. Some things like prompting to user can be done in the latter
+# case, but not in the former one.
 InitializeRPMCommonBase() {
   if type dnf 2>/dev/null
   then
@@ -388,26 +398,6 @@ InitializeRPMCommonBase() {
   if [ "$QUIET" = 1 ]; then
     QUIET_FLAG='--quiet'
   fi
-
-  if ! $TOOL list *virtualenv >/dev/null 2>&1; then
-    echo "To use Certbot, packages from the EPEL repository need to be installed."
-    if ! $TOOL list epel-release >/dev/null 2>&1; then
-      error "Enable the EPEL repository and try running Certbot again."
-      exit 1
-    fi
-    if [ "$ASSUME_YES" = 1 ]; then
-      /bin/echo -n "Enabling the EPEL repository in 3 seconds..."
-      sleep 1s
-      /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
-      sleep 1s
-      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..."
-      sleep 1s
-    fi
-    if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then
-      error "Could not enable EPEL. Aborting bootstrap!"
-      exit 1
-    fi
-  fi
 }
 
 BootstrapRpmCommonBase() {
@@ -488,13 +478,88 @@ BootstrapRpmCommon() {
   BootstrapRpmCommonBase "$python_pkgs"
 }
 
+# If new packages are installed by BootstrapRpmPython3 below, this version
+# number must be increased.
+BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION=1
+
+# Checks if rh-python36 can be installed.
+Python36SclIsAvailable() {
+  InitializeRPMCommonBase >/dev/null 2>&1;
+
+  if "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    return 0
+  fi
+  if "${TOOL}" list centos-release-scl >/dev/null 2>&1; then
+    return 0
+  fi
+  return 1
+}
+
+# Try to enable rh-python36 from SCL if it is necessary and possible.
+EnablePython36SCL() {
+  if "$EXISTS" python3.6 > /dev/null 2> /dev/null; then
+      return 0
+  fi
+  if ! scl --list 2>/dev/null | grep -q rh-python36; then
+      return 0
+  fi
+  set +e
+  . scl_source enable rh-python36
+  set -e
+}
+
+# This bootstrap concerns old RedHat-based distributions that do not ship by default
+# with Python 2.7, but only Python 2.6. We bootstrap them by enabling SCL and installing
+# Python 3.6. Some of these distributions are: CentOS/RHEL/OL/SL 6.
+BootstrapRpmPython3Legacy() {
+  # Tested with:
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  if ! "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    echo "To use Certbot on this operating system, packages from the SCL repository need to be installed."
+    if ! "${TOOL}" list centos-release-scl >/dev/null 2>&1; then
+      error "Enable the SCL repository and try running Certbot again."
+      exit 1
+    fi
+    if [ "${ASSUME_YES}" = 1 ]; then
+      /bin/echo -n "Enabling the SCL repository in 3 seconds... (Press Ctrl-C to cancel)"
+      sleep 1s
+      /bin/echo -ne "\e[0K\rEnabling the SCL repository in 2 seconds... (Press Ctrl-C to cancel)"
+      sleep 1s
+      /bin/echo -e "\e[0K\rEnabling the SCL repository in 1 second... (Press Ctrl-C to cancel)"
+      sleep 1s
+    fi
+    if ! "${TOOL}" install "${YES_FLAG}" "${QUIET_FLAG}" centos-release-scl; then
+      error "Could not enable SCL. Aborting bootstrap!"
+      exit 1
+    fi
+  fi
+
+  # CentOS 6 must use rh-python36 from SCL
+  if "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    python_pkgs="rh-python36-python
+      rh-python36-python-virtualenv
+      rh-python36-python-devel
+    "
+  else
+    error "No supported Python package available to install. Aborting bootstrap!"
+    exit 1
+  fi
+
+  BootstrapRpmCommonBase "${python_pkgs}"
+
+  # Enable SCL rh-python36 after bootstrapping.
+  EnablePython36SCL
+}
+
 # If new packages are installed by BootstrapRpmPython3 below, this version
 # number must be increased.
 BOOTSTRAP_RPM_PYTHON3_VERSION=1
 
 BootstrapRpmPython3() {
   # Tested with:
-  #   - CentOS 6
   #   - Fedora 29
 
   InitializeRPMCommonBase
@@ -505,12 +570,6 @@ BootstrapRpmPython3() {
       python3-virtualenv
       python3-devel
     "
-  # EPEL uses python34
-  elif $TOOL list python34 >/dev/null 2>&1; then
-    python_pkgs="python34
-      python34-devel
-      python34-tools
-    "
   else
     error "No supported Python package available to install. Aborting bootstrap!"
     exit 1
@@ -769,31 +828,50 @@ elif [ -f /etc/redhat-release ]; then
      RPM_DIST_VERSION=0
   fi
 
-  # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then.
-  # RHEL 8 also uses python3 by default.
-  if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then
-    RPM_USE_PYTHON_3=1
-  elif [ "$RPM_DIST_NAME" = "rhel" -a "$RPM_DIST_VERSION" -ge 8 ]; then
-    RPM_USE_PYTHON_3=1
-  elif [ "$RPM_DIST_NAME" = "centos" -a "$RPM_DIST_VERSION" -ge 8 ]; then
-    RPM_USE_PYTHON_3=1
-  else
-    RPM_USE_PYTHON_3=0
-  fi
+  # Handle legacy RPM distributions
+  if [ "$PYVER" -eq 26 ]; then
+    # Check if an automated bootstrap can be achieved on this system.
+    if ! Python36SclIsAvailable; then
+      INTERACTIVE_BOOTSTRAP=1
+    fi
 
-  if [ "$RPM_USE_PYTHON_3" = 1 ]; then
     Bootstrap() {
-      BootstrapMessage "RedHat-based OSes that will use Python3"
-      BootstrapRpmPython3
+      BootstrapMessage "Legacy RedHat-based OSes that will use Python3"
+      BootstrapRpmPython3Legacy
     }
     USE_PYTHON_3=1
-    BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+    BOOTSTRAP_VERSION="BootstrapRpmPython3Legacy $BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION"
+
+    # Try now to enable SCL rh-python36 for systems already bootstrapped
+    # NB: EnablePython36SCL has been defined along with BootstrapRpmPython3Legacy in certbot-auto
+    EnablePython36SCL
   else
-    Bootstrap() {
-      BootstrapMessage "RedHat-based OSes"
-      BootstrapRpmCommon
-    }
-    BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+    # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then.
+    # RHEL 8 also uses python3 by default.
+    if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 ]; then
+      RPM_USE_PYTHON_3=1
+    elif [ "$RPM_DIST_NAME" = "rhel" -a "$RPM_DIST_VERSION" -ge 8 ]; then
+      RPM_USE_PYTHON_3=1
+    elif [ "$RPM_DIST_NAME" = "centos" -a "$RPM_DIST_VERSION" -ge 8 ]; then
+      RPM_USE_PYTHON_3=1
+    else
+      RPM_USE_PYTHON_3=0
+    fi
+
+    if [ "$RPM_USE_PYTHON_3" = 1 ]; then
+      Bootstrap() {
+        BootstrapMessage "RedHat-based OSes that will use Python3"
+        BootstrapRpmPython3
+      }
+      USE_PYTHON_3=1
+      BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+    else
+      Bootstrap() {
+        BootstrapMessage "RedHat-based OSes"
+        BootstrapRpmCommon
+      }
+      BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+    fi
   fi
 
   LE_PYTHON="$prev_le_python"
@@ -1078,8 +1156,15 @@ if [ "$1" = "--le-auto-phase2" ]; then
     # If the selected Bootstrap function isn't a noop and it differs from the
     # previously used version
     if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then
-      # if non-interactive mode or stdin and stdout are connected to a terminal
-      if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
+      # Check if we can rebootstrap without manual user intervention: this requires that
+      # certbot-auto is in non-interactive mode AND selected bootstrap does not claim to
+      # require a manual user intervention.
+      if [ "$NONINTERACTIVE" = 1 -a "$INTERACTIVE_BOOTSTRAP" != 1 ]; then
+        CAN_REBOOTSTRAP=1
+      fi
+      # Check if rebootstrap can be done non-interactively and current shell is non-interactive
+      # (true if stdin and stdout are not attached to a terminal).
+      if [ \( "$CAN_REBOOTSTRAP" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
         if [ -d "$VENV_PATH" ]; then
           rm -rf "$VENV_PATH"
         fi
@@ -1090,12 +1175,21 @@ if [ "$1" = "--le-auto-phase2" ]; then
           ln -s "$VENV_PATH" "$OLD_VENV_PATH"
         fi
         RerunWithArgs "$@"
+      # Otherwise bootstrap needs to be done manually by the user.
       else
-        error "Skipping upgrade because new OS dependencies may need to be installed."
-        error
-        error "To upgrade to a newer version, please run this script again manually so you can"
-        error "approve changes or with --non-interactive on the command line to automatically"
-        error "install any required packages."
+        # If it is because bootstrapping is interactive, --non-interactive will be of no use.
+        if [ "$INTERACTIVE_BOOTSTRAP" = 1 ]; then
+          error "Skipping upgrade because new OS dependencies may need to be installed."
+          error "This requires manual user intervention: please run this script again manually."
+        # If this is because of the environment (eg. non interactive shell without
+        # --non-interactive flag set), help the user in that direction.
+        else
+          error "Skipping upgrade because new OS dependencies may need to be installed."
+          error
+          error "To upgrade to a newer version, please run this script again manually so you can"
+          error "approve changes or with --non-interactive on the command line to automatically"
+          error "install any required packages."
+        fi
         # Set INSTALLED_VERSION to be the same so we don't update the venv
         INSTALLED_VERSION="$LE_AUTO_VERSION"
         # Continue to use OLD_VENV_PATH if the new venv doesn't exist
@@ -1338,18 +1432,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.39.0 \
-    --hash=sha256:f1a70651a6c5137a448f4a8db17b09af619f80a077326caae6b74278bf1db488 \
-    --hash=sha256:885cee1c4d05888af86b626cbbfc29d3c6c842ef4fe8f4a486994cef9daddfe0
-acme==0.39.0 \
-    --hash=sha256:4f8be913df289b981852042719469cc367a7e436256f232c799d0bd1521db710 \
-    --hash=sha256:a2fcb75d16de6804f4b4d773a457ee2f6434ebaf8fd1aa60862a91d4e8f73608
-certbot-apache==0.39.0 \
-    --hash=sha256:c7a8630a85b753a52ca0b8c19e24b8f85ac4ba028292a95745e250c2e72faab9 \
-    --hash=sha256:4651a0212c9ebc3087281dad92ad3cb355bb2730f432d0180a8d23325d11825a
-certbot-nginx==0.39.0 \
-    --hash=sha256:76e5862ad5cc0fbc099df3502987c101c60dee1c188a579eac990edee7a910df \
-    --hash=sha256:ceac88df52d3b27d14c3052b9e90ada327d7e14ecd6e4af7519918182d6138b4
+certbot==0.40.0 \
+    --hash=sha256:b9ff74c4f3d3e06d9c467465f97bcbb07b0f4d778d3c4232ab91583d933dba61 \
+    --hash=sha256:cff166597b3c714c3e7e60b2bcd6089135b375cadca04cf36abd15bfdb22be40
+acme==0.40.0 \
+    --hash=sha256:1b026b07a2099e50dac11cbdb834925f1d9b5691e349b52e9d397a12f3dc4eac \
+    --hash=sha256:f29c1185d1e33919bad6c1f3fece168ee191d96d47f5997117561dc74a454221
+certbot-apache==0.40.0 \
+    --hash=sha256:f1c034a05fbd6cc6fde9494f493a8a6ed0e02e7652e51af16342082bc17387e4 \
+    --hash=sha256:43c3d7628ca6630467c4f57dd30423f031c1c7cbca46f7500293172d0fe3581e
+certbot-nginx==0.40.0 \
+    --hash=sha256:55cd3c90e2851069b536859050374fe2fcfa22c3e862cc0e1811fbce9e52dccc \
+    --hash=sha256:3df8cec22910f2d41ccb4494661ff65f98c52dd441864a53a318b32979256881
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------

letsencrypt-auto-source/Dockerfile.redhat6

@@ -1,9 +1,13 @@
 # For running tests, build a docker image with a passwordless sudo and a trust
 # store we can manipulate.
 
-FROM centos:6
+ARG REDHAT_DIST_FLAVOR
+FROM ${REDHAT_DIST_FLAVOR}:6
 
-RUN yum install -y epel-release
+ARG REDHAT_DIST_FLAVOR
+
+RUN curl -O https://dl.fedoraproject.org/pub/epel/epel-release-latest-6.noarch.rpm \
+ && rpm -ivh epel-release-latest-6.noarch.rpm
 
 # Install pip and sudo:
 RUN yum install -y python-pip sudo
@@ -27,11 +31,18 @@ RUN mkdir -p /home/lea/certbot
 COPY ./tests/certs/ca/my-root-ca.crt.pem /usr/local/share/ca-certificates/
 RUN update-ca-trust
 
-# Copy code:
+# Copy current letsencrypt-auto:
 COPY . /home/lea/certbot/letsencrypt-auto-source
 
+# Fetch previous letsencrypt-auto that was installing python 3.4
+RUN curl https://raw.githubusercontent.com/certbot/certbot/v0.38.0/letsencrypt-auto-source/letsencrypt-auto \
+    -o /home/lea/certbot/letsencrypt-auto-source/letsencrypt-auto_py_34 \
+ && chmod +x /home/lea/certbot/letsencrypt-auto-source/letsencrypt-auto_py_34
+
+RUN cp /home/lea/certbot/letsencrypt-auto-source/tests/${REDHAT_DIST_FLAVOR}6_tests.sh /home/lea/certbot/letsencrypt-auto-source/tests/redhat6_tests.sh \
+ && chmod +x /home/lea/certbot/letsencrypt-auto-source/tests/redhat6_tests.sh
+
 USER lea
 WORKDIR /home/lea
 
-RUN sudo chmod +x certbot/letsencrypt-auto-source/tests/centos6_tests.sh
-CMD sudo certbot/letsencrypt-auto-source/tests/centos6_tests.sh
+CMD ["sudo", "certbot/letsencrypt-auto-source/tests/redhat6_tests.sh"]

letsencrypt-auto-source/certbot-auto.asc

@@ -1,11 +1,11 @@
 -----BEGIN PGP SIGNATURE-----
 
-iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl2TsPMACgkQTRfJlc2X
-dfJHUAf+NcnvHzowhLr1rkR11CSKMCMgwUee7Nm0QHnVPf09+Dd9mvuaRptuua1D
-Qvtcb3F4OQ6/3khy3fzGXIcEe9kuI2+boe+ZA0dfmmzo4ELzpWUadXkuonYybZFE
-JAaICgLLHOkiRL8J8ZTmXZI4tbFSsxTLMNOwoMZ6oGgp2plj2rm85L4Z+vUlfaTf
-wcs/glbBtbYfW3WWapMsMWwgrE62Q/OOhBjbkPCywFRQDwwaXz6QPrvi+k6gLCqs
-Okvg5bY2hP70tU1i9wxp2DAfF/P/5i2hVSWktRdMolUTTTeczLW81allmmDRJcAi
-4xrj6wYhN7olMZrTpakXb7zRR9/MGQ==
-=Ag2y
+iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl3B4KcACgkQTRfJlc2X
+dfJKHAf+PUViUdwbaXUMNfDRo7g6v44RA0RIj+SG3cjLsX2E/A2G70KndfUC/9KS
+cgYpFZ3h/2y3fLLsYgDIOPRhAKLgrk+LFKrtDsUbOLF7K3eS70KQmDxYFXNzw0jc
+34zhc9BKsKrqX6a80LprkVtbEuRlE58JaXyqjMW8NvGvLXNV8qCZK8xG8SrCkVnU
+KFlXgHAl3UFibm3yJOlIjHikuOaU0jlDbO/S2WfkkgV3BWQkngUKu+9gr+ItV3We
+GMidJljIoho8CqYQnLWtsjhOmjLQogsUKZJSg/riAxrDW3cCEmF4EaV/S8lNnSiL
+f49WauHsGdfIaFabl8HVG7h+R3Uh8Q==
+=JXq9
 -----END PGP SIGNATURE-----

letsencrypt-auto-source/letsencrypt-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.40.0.dev0"
+LE_AUTO_VERSION="0.40.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -256,20 +256,28 @@ DeprecationBootstrap() {
   fi
 }
 
-MIN_PYTHON_VERSION="2.7"
-MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//')
+MIN_PYTHON_2_VERSION="2.7"
+MIN_PYVER2=$(echo "$MIN_PYTHON_2_VERSION" | sed 's/\.//')
+MIN_PYTHON_3_VERSION="3.5"
+MIN_PYVER3=$(echo "$MIN_PYTHON_3_VERSION" | sed 's/\.//')
 # Sets LE_PYTHON to Python version string and PYVER to the first two
-# digits of the python version
+# digits of the python version.
+# MIN_PYVER and MIN_PYTHON_VERSION are also set by this function, and their
+# values depend on if we try to use Python 3 or Python 2.
 DeterminePythonVersion() {
   # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
   #
   # If no Python is found, PYVER is set to 0.
   if [ "$USE_PYTHON_3" = 1 ]; then
+    MIN_PYVER=$MIN_PYVER3
+    MIN_PYTHON_VERSION=$MIN_PYTHON_3_VERSION
     for LE_PYTHON in "$LE_PYTHON" python3; do
       # Break (while keeping the LE_PYTHON value) if found.
       $EXISTS "$LE_PYTHON" > /dev/null && break
     done
   else
+    MIN_PYVER=$MIN_PYVER2
+    MIN_PYTHON_VERSION=$MIN_PYTHON_2_VERSION
     for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
       # Break (while keeping the LE_PYTHON value) if found.
       $EXISTS "$LE_PYTHON" > /dev/null && break
@@ -285,7 +293,7 @@ DeterminePythonVersion() {
     fi
   fi
 
-  PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
+  PYVER=$("$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//')
   if [ "$PYVER" -lt "$MIN_PYVER" ]; then
     if [ "$1" != "NOCRASH" ]; then
       error "You have an ancient version of Python entombed in your operating system..."
@@ -368,7 +376,9 @@ BootstrapDebCommon() {
 
 # Sets TOOL to the name of the package manager
 # Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG.
-# Enables EPEL if applicable and possible.
+# Note: this function is called both while selecting the bootstrap scripts and
+# during the actual bootstrap. Some things like prompting to user can be done in the latter
+# case, but not in the former one.
 InitializeRPMCommonBase() {
   if type dnf 2>/dev/null
   then
@@ -388,26 +398,6 @@ InitializeRPMCommonBase() {
   if [ "$QUIET" = 1 ]; then
     QUIET_FLAG='--quiet'
   fi
-
-  if ! $TOOL list *virtualenv >/dev/null 2>&1; then
-    echo "To use Certbot, packages from the EPEL repository need to be installed."
-    if ! $TOOL list epel-release >/dev/null 2>&1; then
-      error "Enable the EPEL repository and try running Certbot again."
-      exit 1
-    fi
-    if [ "$ASSUME_YES" = 1 ]; then
-      /bin/echo -n "Enabling the EPEL repository in 3 seconds..."
-      sleep 1s
-      /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
-      sleep 1s
-      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..."
-      sleep 1s
-    fi
-    if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then
-      error "Could not enable EPEL. Aborting bootstrap!"
-      exit 1
-    fi
-  fi
 }
 
 BootstrapRpmCommonBase() {
@@ -488,13 +478,88 @@ BootstrapRpmCommon() {
   BootstrapRpmCommonBase "$python_pkgs"
 }
 
+# If new packages are installed by BootstrapRpmPython3 below, this version
+# number must be increased.
+BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION=1
+
+# Checks if rh-python36 can be installed.
+Python36SclIsAvailable() {
+  InitializeRPMCommonBase >/dev/null 2>&1;
+
+  if "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    return 0
+  fi
+  if "${TOOL}" list centos-release-scl >/dev/null 2>&1; then
+    return 0
+  fi
+  return 1
+}
+
+# Try to enable rh-python36 from SCL if it is necessary and possible.
+EnablePython36SCL() {
+  if "$EXISTS" python3.6 > /dev/null 2> /dev/null; then
+      return 0
+  fi
+  if ! scl --list 2>/dev/null | grep -q rh-python36; then
+      return 0
+  fi
+  set +e
+  . scl_source enable rh-python36
+  set -e
+}
+
+# This bootstrap concerns old RedHat-based distributions that do not ship by default
+# with Python 2.7, but only Python 2.6. We bootstrap them by enabling SCL and installing
+# Python 3.6. Some of these distributions are: CentOS/RHEL/OL/SL 6.
+BootstrapRpmPython3Legacy() {
+  # Tested with:
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  if ! "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    echo "To use Certbot on this operating system, packages from the SCL repository need to be installed."
+    if ! "${TOOL}" list centos-release-scl >/dev/null 2>&1; then
+      error "Enable the SCL repository and try running Certbot again."
+      exit 1
+    fi
+    if [ "${ASSUME_YES}" = 1 ]; then
+      /bin/echo -n "Enabling the SCL repository in 3 seconds... (Press Ctrl-C to cancel)"
+      sleep 1s
+      /bin/echo -ne "\e[0K\rEnabling the SCL repository in 2 seconds... (Press Ctrl-C to cancel)"
+      sleep 1s
+      /bin/echo -e "\e[0K\rEnabling the SCL repository in 1 second... (Press Ctrl-C to cancel)"
+      sleep 1s
+    fi
+    if ! "${TOOL}" install "${YES_FLAG}" "${QUIET_FLAG}" centos-release-scl; then
+      error "Could not enable SCL. Aborting bootstrap!"
+      exit 1
+    fi
+  fi
+
+  # CentOS 6 must use rh-python36 from SCL
+  if "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    python_pkgs="rh-python36-python
+      rh-python36-python-virtualenv
+      rh-python36-python-devel
+    "
+  else
+    error "No supported Python package available to install. Aborting bootstrap!"
+    exit 1
+  fi
+
+  BootstrapRpmCommonBase "${python_pkgs}"
+
+  # Enable SCL rh-python36 after bootstrapping.
+  EnablePython36SCL
+}
+
 # If new packages are installed by BootstrapRpmPython3 below, this version
 # number must be increased.
 BOOTSTRAP_RPM_PYTHON3_VERSION=1
 
 BootstrapRpmPython3() {
   # Tested with:
-  #   - CentOS 6
   #   - Fedora 29
 
   InitializeRPMCommonBase
@@ -505,12 +570,6 @@ BootstrapRpmPython3() {
       python3-virtualenv
       python3-devel
     "
-  # EPEL uses python34
-  elif $TOOL list python34 >/dev/null 2>&1; then
-    python_pkgs="python34
-      python34-devel
-      python34-tools
-    "
   else
     error "No supported Python package available to install. Aborting bootstrap!"
     exit 1
@@ -769,31 +828,50 @@ elif [ -f /etc/redhat-release ]; then
      RPM_DIST_VERSION=0
   fi
 
-  # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then.
-  # RHEL 8 also uses python3 by default.
-  if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then
-    RPM_USE_PYTHON_3=1
-  elif [ "$RPM_DIST_NAME" = "rhel" -a "$RPM_DIST_VERSION" -ge 8 ]; then
-    RPM_USE_PYTHON_3=1
-  elif [ "$RPM_DIST_NAME" = "centos" -a "$RPM_DIST_VERSION" -ge 8 ]; then
-    RPM_USE_PYTHON_3=1
-  else
-    RPM_USE_PYTHON_3=0
-  fi
+  # Handle legacy RPM distributions
+  if [ "$PYVER" -eq 26 ]; then
+    # Check if an automated bootstrap can be achieved on this system.
+    if ! Python36SclIsAvailable; then
+      INTERACTIVE_BOOTSTRAP=1
+    fi
 
-  if [ "$RPM_USE_PYTHON_3" = 1 ]; then
     Bootstrap() {
-      BootstrapMessage "RedHat-based OSes that will use Python3"
-      BootstrapRpmPython3
+      BootstrapMessage "Legacy RedHat-based OSes that will use Python3"
+      BootstrapRpmPython3Legacy
     }
     USE_PYTHON_3=1
-    BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+    BOOTSTRAP_VERSION="BootstrapRpmPython3Legacy $BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION"
+
+    # Try now to enable SCL rh-python36 for systems already bootstrapped
+    # NB: EnablePython36SCL has been defined along with BootstrapRpmPython3Legacy in certbot-auto
+    EnablePython36SCL
   else
-    Bootstrap() {
-      BootstrapMessage "RedHat-based OSes"
-      BootstrapRpmCommon
-    }
-    BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+    # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then.
+    # RHEL 8 also uses python3 by default.
+    if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 ]; then
+      RPM_USE_PYTHON_3=1
+    elif [ "$RPM_DIST_NAME" = "rhel" -a "$RPM_DIST_VERSION" -ge 8 ]; then
+      RPM_USE_PYTHON_3=1
+    elif [ "$RPM_DIST_NAME" = "centos" -a "$RPM_DIST_VERSION" -ge 8 ]; then
+      RPM_USE_PYTHON_3=1
+    else
+      RPM_USE_PYTHON_3=0
+    fi
+
+    if [ "$RPM_USE_PYTHON_3" = 1 ]; then
+      Bootstrap() {
+        BootstrapMessage "RedHat-based OSes that will use Python3"
+        BootstrapRpmPython3
+      }
+      USE_PYTHON_3=1
+      BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+    else
+      Bootstrap() {
+        BootstrapMessage "RedHat-based OSes"
+        BootstrapRpmCommon
+      }
+      BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+    fi
   fi
 
   LE_PYTHON="$prev_le_python"
@@ -1078,8 +1156,15 @@ if [ "$1" = "--le-auto-phase2" ]; then
     # If the selected Bootstrap function isn't a noop and it differs from the
     # previously used version
     if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then
-      # if non-interactive mode or stdin and stdout are connected to a terminal
-      if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
+      # Check if we can rebootstrap without manual user intervention: this requires that
+      # certbot-auto is in non-interactive mode AND selected bootstrap does not claim to
+      # require a manual user intervention.
+      if [ "$NONINTERACTIVE" = 1 -a "$INTERACTIVE_BOOTSTRAP" != 1 ]; then
+        CAN_REBOOTSTRAP=1
+      fi
+      # Check if rebootstrap can be done non-interactively and current shell is non-interactive
+      # (true if stdin and stdout are not attached to a terminal).
+      if [ \( "$CAN_REBOOTSTRAP" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
         if [ -d "$VENV_PATH" ]; then
           rm -rf "$VENV_PATH"
         fi
@@ -1090,12 +1175,21 @@ if [ "$1" = "--le-auto-phase2" ]; then
           ln -s "$VENV_PATH" "$OLD_VENV_PATH"
         fi
         RerunWithArgs "$@"
+      # Otherwise bootstrap needs to be done manually by the user.
       else
-        error "Skipping upgrade because new OS dependencies may need to be installed."
-        error
-        error "To upgrade to a newer version, please run this script again manually so you can"
-        error "approve changes or with --non-interactive on the command line to automatically"
-        error "install any required packages."
+        # If it is because bootstrapping is interactive, --non-interactive will be of no use.
+        if [ "$INTERACTIVE_BOOTSTRAP" = 1 ]; then
+          error "Skipping upgrade because new OS dependencies may need to be installed."
+          error "This requires manual user intervention: please run this script again manually."
+        # If this is because of the environment (eg. non interactive shell without
+        # --non-interactive flag set), help the user in that direction.
+        else
+          error "Skipping upgrade because new OS dependencies may need to be installed."
+          error
+          error "To upgrade to a newer version, please run this script again manually so you can"
+          error "approve changes or with --non-interactive on the command line to automatically"
+          error "install any required packages."
+        fi
         # Set INSTALLED_VERSION to be the same so we don't update the venv
         INSTALLED_VERSION="$LE_AUTO_VERSION"
         # Continue to use OLD_VENV_PATH if the new venv doesn't exist
@@ -1338,18 +1432,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.39.0 \
-    --hash=sha256:f1a70651a6c5137a448f4a8db17b09af619f80a077326caae6b74278bf1db488 \
-    --hash=sha256:885cee1c4d05888af86b626cbbfc29d3c6c842ef4fe8f4a486994cef9daddfe0
-acme==0.39.0 \
-    --hash=sha256:4f8be913df289b981852042719469cc367a7e436256f232c799d0bd1521db710 \
-    --hash=sha256:a2fcb75d16de6804f4b4d773a457ee2f6434ebaf8fd1aa60862a91d4e8f73608
-certbot-apache==0.39.0 \
-    --hash=sha256:c7a8630a85b753a52ca0b8c19e24b8f85ac4ba028292a95745e250c2e72faab9 \
-    --hash=sha256:4651a0212c9ebc3087281dad92ad3cb355bb2730f432d0180a8d23325d11825a
-certbot-nginx==0.39.0 \
-    --hash=sha256:76e5862ad5cc0fbc099df3502987c101c60dee1c188a579eac990edee7a910df \
-    --hash=sha256:ceac88df52d3b27d14c3052b9e90ada327d7e14ecd6e4af7519918182d6138b4
+certbot==0.40.0 \
+    --hash=sha256:b9ff74c4f3d3e06d9c467465f97bcbb07b0f4d778d3c4232ab91583d933dba61 \
+    --hash=sha256:cff166597b3c714c3e7e60b2bcd6089135b375cadca04cf36abd15bfdb22be40
+acme==0.40.0 \
+    --hash=sha256:1b026b07a2099e50dac11cbdb834925f1d9b5691e349b52e9d397a12f3dc4eac \
+    --hash=sha256:f29c1185d1e33919bad6c1f3fece168ee191d96d47f5997117561dc74a454221
+certbot-apache==0.40.0 \
+    --hash=sha256:f1c034a05fbd6cc6fde9494f493a8a6ed0e02e7652e51af16342082bc17387e4 \
+    --hash=sha256:43c3d7628ca6630467c4f57dd30423f031c1c7cbca46f7500293172d0fe3581e
+certbot-nginx==0.40.0 \
+    --hash=sha256:55cd3c90e2851069b536859050374fe2fcfa22c3e862cc0e1811fbce9e52dccc \
+    --hash=sha256:3df8cec22910f2d41ccb4494661ff65f98c52dd441864a53a318b32979256881
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------

letsencrypt-auto-source/letsencrypt-auto.template

@@ -256,20 +256,28 @@ DeprecationBootstrap() {
   fi
 }
 
-MIN_PYTHON_VERSION="2.7"
-MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//')
+MIN_PYTHON_2_VERSION="2.7"
+MIN_PYVER2=$(echo "$MIN_PYTHON_2_VERSION" | sed 's/\.//')
+MIN_PYTHON_3_VERSION="3.5"
+MIN_PYVER3=$(echo "$MIN_PYTHON_3_VERSION" | sed 's/\.//')
 # Sets LE_PYTHON to Python version string and PYVER to the first two
-# digits of the python version
+# digits of the python version.
+# MIN_PYVER and MIN_PYTHON_VERSION are also set by this function, and their
+# values depend on if we try to use Python 3 or Python 2.
 DeterminePythonVersion() {
   # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
   #
   # If no Python is found, PYVER is set to 0.
   if [ "$USE_PYTHON_3" = 1 ]; then
+    MIN_PYVER=$MIN_PYVER3
+    MIN_PYTHON_VERSION=$MIN_PYTHON_3_VERSION
     for LE_PYTHON in "$LE_PYTHON" python3; do
       # Break (while keeping the LE_PYTHON value) if found.
       $EXISTS "$LE_PYTHON" > /dev/null && break
     done
   else
+    MIN_PYVER=$MIN_PYVER2
+    MIN_PYTHON_VERSION=$MIN_PYTHON_2_VERSION
     for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
       # Break (while keeping the LE_PYTHON value) if found.
       $EXISTS "$LE_PYTHON" > /dev/null && break
@@ -285,7 +293,7 @@ DeterminePythonVersion() {
     fi
   fi
 
-  PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
+  PYVER=$("$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//')
   if [ "$PYVER" -lt "$MIN_PYVER" ]; then
     if [ "$1" != "NOCRASH" ]; then
       error "You have an ancient version of Python entombed in your operating system..."
@@ -298,6 +306,7 @@ DeterminePythonVersion() {
 {{ bootstrappers/deb_common.sh }}
 {{ bootstrappers/rpm_common_base.sh }}
 {{ bootstrappers/rpm_common.sh }}
+{{ bootstrappers/rpm_python3_legacy.sh }}
 {{ bootstrappers/rpm_python3.sh }}
 {{ bootstrappers/suse_common.sh }}
 {{ bootstrappers/arch_common.sh }}
@@ -344,31 +353,50 @@ elif [ -f /etc/redhat-release ]; then
      RPM_DIST_VERSION=0
   fi
 
-  # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then.
-  # RHEL 8 also uses python3 by default.
-  if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then
-    RPM_USE_PYTHON_3=1
-  elif [ "$RPM_DIST_NAME" = "rhel" -a "$RPM_DIST_VERSION" -ge 8 ]; then
-    RPM_USE_PYTHON_3=1
-  elif [ "$RPM_DIST_NAME" = "centos" -a "$RPM_DIST_VERSION" -ge 8 ]; then
-    RPM_USE_PYTHON_3=1
-  else
-    RPM_USE_PYTHON_3=0
-  fi
+  # Handle legacy RPM distributions
+  if [ "$PYVER" -eq 26 ]; then
+    # Check if an automated bootstrap can be achieved on this system.
+    if ! Python36SclIsAvailable; then
+      INTERACTIVE_BOOTSTRAP=1
+    fi
 
-  if [ "$RPM_USE_PYTHON_3" = 1 ]; then
     Bootstrap() {
-      BootstrapMessage "RedHat-based OSes that will use Python3"
-      BootstrapRpmPython3
+      BootstrapMessage "Legacy RedHat-based OSes that will use Python3"
+      BootstrapRpmPython3Legacy
     }
     USE_PYTHON_3=1
-    BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+    BOOTSTRAP_VERSION="BootstrapRpmPython3Legacy $BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION"
+
+    # Try now to enable SCL rh-python36 for systems already bootstrapped
+    # NB: EnablePython36SCL has been defined along with BootstrapRpmPython3Legacy in certbot-auto
+    EnablePython36SCL
   else
-    Bootstrap() {
-      BootstrapMessage "RedHat-based OSes"
-      BootstrapRpmCommon
-    }
-    BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+    # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then.
+    # RHEL 8 also uses python3 by default.
+    if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 ]; then
+      RPM_USE_PYTHON_3=1
+    elif [ "$RPM_DIST_NAME" = "rhel" -a "$RPM_DIST_VERSION" -ge 8 ]; then
+      RPM_USE_PYTHON_3=1
+    elif [ "$RPM_DIST_NAME" = "centos" -a "$RPM_DIST_VERSION" -ge 8 ]; then
+      RPM_USE_PYTHON_3=1
+    else
+      RPM_USE_PYTHON_3=0
+    fi
+
+    if [ "$RPM_USE_PYTHON_3" = 1 ]; then
+      Bootstrap() {
+        BootstrapMessage "RedHat-based OSes that will use Python3"
+        BootstrapRpmPython3
+      }
+      USE_PYTHON_3=1
+      BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+    else
+      Bootstrap() {
+        BootstrapMessage "RedHat-based OSes"
+        BootstrapRpmCommon
+      }
+      BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+    fi
   fi
 
   LE_PYTHON="$prev_le_python"
@@ -545,8 +573,15 @@ if [ "$1" = "--le-auto-phase2" ]; then
     # If the selected Bootstrap function isn't a noop and it differs from the
     # previously used version
     if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then
-      # if non-interactive mode or stdin and stdout are connected to a terminal
-      if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
+      # Check if we can rebootstrap without manual user intervention: this requires that
+      # certbot-auto is in non-interactive mode AND selected bootstrap does not claim to
+      # require a manual user intervention.
+      if [ "$NONINTERACTIVE" = 1 -a "$INTERACTIVE_BOOTSTRAP" != 1 ]; then
+        CAN_REBOOTSTRAP=1
+      fi
+      # Check if rebootstrap can be done non-interactively and current shell is non-interactive
+      # (true if stdin and stdout are not attached to a terminal).
+      if [ \( "$CAN_REBOOTSTRAP" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
         if [ -d "$VENV_PATH" ]; then
           rm -rf "$VENV_PATH"
         fi
@@ -557,12 +592,21 @@ if [ "$1" = "--le-auto-phase2" ]; then
           ln -s "$VENV_PATH" "$OLD_VENV_PATH"
         fi
         RerunWithArgs "$@"
+      # Otherwise bootstrap needs to be done manually by the user.
       else
-        error "Skipping upgrade because new OS dependencies may need to be installed."
-        error
-        error "To upgrade to a newer version, please run this script again manually so you can"
-        error "approve changes or with --non-interactive on the command line to automatically"
-        error "install any required packages."
+        # If it is because bootstrapping is interactive, --non-interactive will be of no use.
+        if [ "$INTERACTIVE_BOOTSTRAP" = 1 ]; then
+          error "Skipping upgrade because new OS dependencies may need to be installed."
+          error "This requires manual user intervention: please run this script again manually."
+        # If this is because of the environment (eg. non interactive shell without
+        # --non-interactive flag set), help the user in that direction.
+        else
+          error "Skipping upgrade because new OS dependencies may need to be installed."
+          error
+          error "To upgrade to a newer version, please run this script again manually so you can"
+          error "approve changes or with --non-interactive on the command line to automatically"
+          error "install any required packages."
+        fi
         # Set INSTALLED_VERSION to be the same so we don't update the venv
         INSTALLED_VERSION="$LE_AUTO_VERSION"
         # Continue to use OLD_VENV_PATH if the new venv doesn't exist

letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh

@@ -3,7 +3,9 @@
 
 # Sets TOOL to the name of the package manager
 # Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG.
-# Enables EPEL if applicable and possible.
+# Note: this function is called both while selecting the bootstrap scripts and
+# during the actual bootstrap. Some things like prompting to user can be done in the latter
+# case, but not in the former one.
 InitializeRPMCommonBase() {
   if type dnf 2>/dev/null
   then
@@ -23,26 +25,6 @@ InitializeRPMCommonBase() {
   if [ "$QUIET" = 1 ]; then
     QUIET_FLAG='--quiet'
   fi
-
-  if ! $TOOL list *virtualenv >/dev/null 2>&1; then
-    echo "To use Certbot, packages from the EPEL repository need to be installed."
-    if ! $TOOL list epel-release >/dev/null 2>&1; then
-      error "Enable the EPEL repository and try running Certbot again."
-      exit 1
-    fi
-    if [ "$ASSUME_YES" = 1 ]; then
-      /bin/echo -n "Enabling the EPEL repository in 3 seconds..."
-      sleep 1s
-      /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
-      sleep 1s
-      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..."
-      sleep 1s
-    fi
-    if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then
-      error "Could not enable EPEL. Aborting bootstrap!"
-      exit 1
-    fi
-  fi
 }
 
 BootstrapRpmCommonBase() {

letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh

@@ -4,7 +4,6 @@ BOOTSTRAP_RPM_PYTHON3_VERSION=1
 
 BootstrapRpmPython3() {
   # Tested with:
-  #   - CentOS 6
   #   - Fedora 29
 
   InitializeRPMCommonBase
@@ -15,12 +14,6 @@ BootstrapRpmPython3() {
       python3-virtualenv
       python3-devel
     "
-  # EPEL uses python34
-  elif $TOOL list python34 >/dev/null 2>&1; then
-    python_pkgs="python34
-      python34-devel
-      python34-tools
-    "
   else
     error "No supported Python package available to install. Aborting bootstrap!"
     exit 1

letsencrypt-auto-source/pieces/bootstrappers/rpm_python3_legacy.sh

@@ -0,0 +1,75 @@
+# If new packages are installed by BootstrapRpmPython3 below, this version
+# number must be increased.
+BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION=1
+
+# Checks if rh-python36 can be installed.
+Python36SclIsAvailable() {
+  InitializeRPMCommonBase >/dev/null 2>&1;
+
+  if "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    return 0
+  fi
+  if "${TOOL}" list centos-release-scl >/dev/null 2>&1; then
+    return 0
+  fi
+  return 1
+}
+
+# Try to enable rh-python36 from SCL if it is necessary and possible.
+EnablePython36SCL() {
+  if "$EXISTS" python3.6 > /dev/null 2> /dev/null; then
+      return 0
+  fi
+  if ! scl --list 2>/dev/null | grep -q rh-python36; then
+      return 0
+  fi
+  set +e
+  . scl_source enable rh-python36
+  set -e
+}
+
+# This bootstrap concerns old RedHat-based distributions that do not ship by default
+# with Python 2.7, but only Python 2.6. We bootstrap them by enabling SCL and installing
+# Python 3.6. Some of these distributions are: CentOS/RHEL/OL/SL 6.
+BootstrapRpmPython3Legacy() {
+  # Tested with:
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  if ! "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    echo "To use Certbot on this operating system, packages from the SCL repository need to be installed."
+    if ! "${TOOL}" list centos-release-scl >/dev/null 2>&1; then
+      error "Enable the SCL repository and try running Certbot again."
+      exit 1
+    fi
+    if [ "${ASSUME_YES}" = 1 ]; then
+      /bin/echo -n "Enabling the SCL repository in 3 seconds... (Press Ctrl-C to cancel)"
+      sleep 1s
+      /bin/echo -ne "\e[0K\rEnabling the SCL repository in 2 seconds... (Press Ctrl-C to cancel)"
+      sleep 1s
+      /bin/echo -e "\e[0K\rEnabling the SCL repository in 1 second... (Press Ctrl-C to cancel)"
+      sleep 1s
+    fi
+    if ! "${TOOL}" install "${YES_FLAG}" "${QUIET_FLAG}" centos-release-scl; then
+      error "Could not enable SCL. Aborting bootstrap!"
+      exit 1
+    fi
+  fi
+
+  # CentOS 6 must use rh-python36 from SCL
+  if "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    python_pkgs="rh-python36-python
+      rh-python36-python-virtualenv
+      rh-python36-python-devel
+    "
+  else
+    error "No supported Python package available to install. Aborting bootstrap!"
+    exit 1
+  fi
+
+  BootstrapRpmCommonBase "${python_pkgs}"
+
+  # Enable SCL rh-python36 after bootstrapping.
+  EnablePython36SCL
+}

letsencrypt-auto-source/pieces/certbot-requirements.txt

@@ -1,12 +1,12 @@
-certbot==0.39.0 \
-    --hash=sha256:f1a70651a6c5137a448f4a8db17b09af619f80a077326caae6b74278bf1db488 \
-    --hash=sha256:885cee1c4d05888af86b626cbbfc29d3c6c842ef4fe8f4a486994cef9daddfe0
-acme==0.39.0 \
-    --hash=sha256:4f8be913df289b981852042719469cc367a7e436256f232c799d0bd1521db710 \
-    --hash=sha256:a2fcb75d16de6804f4b4d773a457ee2f6434ebaf8fd1aa60862a91d4e8f73608
-certbot-apache==0.39.0 \
-    --hash=sha256:c7a8630a85b753a52ca0b8c19e24b8f85ac4ba028292a95745e250c2e72faab9 \
-    --hash=sha256:4651a0212c9ebc3087281dad92ad3cb355bb2730f432d0180a8d23325d11825a
-certbot-nginx==0.39.0 \
-    --hash=sha256:76e5862ad5cc0fbc099df3502987c101c60dee1c188a579eac990edee7a910df \
-    --hash=sha256:ceac88df52d3b27d14c3052b9e90ada327d7e14ecd6e4af7519918182d6138b4
+certbot==0.40.0 \
+    --hash=sha256:b9ff74c4f3d3e06d9c467465f97bcbb07b0f4d778d3c4232ab91583d933dba61 \
+    --hash=sha256:cff166597b3c714c3e7e60b2bcd6089135b375cadca04cf36abd15bfdb22be40
+acme==0.40.0 \
+    --hash=sha256:1b026b07a2099e50dac11cbdb834925f1d9b5691e349b52e9d397a12f3dc4eac \
+    --hash=sha256:f29c1185d1e33919bad6c1f3fece168ee191d96d47f5997117561dc74a454221
+certbot-apache==0.40.0 \
+    --hash=sha256:f1c034a05fbd6cc6fde9494f493a8a6ed0e02e7652e51af16342082bc17387e4 \
+    --hash=sha256:43c3d7628ca6630467c4f57dd30423f031c1c7cbca46f7500293172d0fe3581e
+certbot-nginx==0.40.0 \
+    --hash=sha256:55cd3c90e2851069b536859050374fe2fcfa22c3e862cc0e1811fbce9e52dccc \
+    --hash=sha256:3df8cec22910f2d41ccb4494661ff65f98c52dd441864a53a318b32979256881

letsencrypt-auto-source/rebuild_dependencies.py

@@ -62,7 +62,7 @@ CERTBOT_REPO_PATH = dirname(dirname(abspath(__file__)))
 #     without pinned dependencies, and respecting input authoritative requirements
 #   - `certbot plugins` is called to check we have an healthy environment
 #   - finally current set of dependencies is extracted out of the docker using pip freeze
-SCRIPT = """\
+SCRIPT = r"""\
 #!/bin/sh
 set -e
 

letsencrypt-auto-source/tests/centos6_tests.sh

@@ -1,81 +1,140 @@
 #!/bin/bash
 # Start by making sure your system is up-to-date:
-yum update -y > /dev/null
-yum install -y centos-release-scl > /dev/null
-yum install -y python27 > /dev/null 2> /dev/null
+yum update -y >/dev/null
+yum install -y centos-release-scl >/dev/null
+yum install -y python27 >/dev/null 2>/dev/null
 
+LE_AUTO_PY_34="certbot/letsencrypt-auto-source/letsencrypt-auto_py_34"
 LE_AUTO="certbot/letsencrypt-auto-source/letsencrypt-auto"
 
+# Last version of certbot-auto that was bootstraping Python 3.4 for CentOS 6 users
+INITIAL_CERTBOT_VERSION_PY34="certbot 0.38.0"
+
 # we're going to modify env variables, so do this in a subshell
 (
-source /opt/rh/python27/enable
+  . scl_source enable python27
 
-# ensure python 3 isn't installed
-python3 --version 2> /dev/null
-RESULT=$?
-if [ $RESULT -eq 0 ]; then
-  error "Python3 is already installed."
-  exit 1
-fi
+  # ensure python 3 isn't installed
+  python3 --version >/dev/null 2>/dev/null
+  RESULT=$?
+  if [ $RESULT -eq 0 ]; then
+    echo "ERROR: Python3 is already installed."
+    exit 1
+  fi
 
-# ensure python2.7 is available
-python2.7 --version 2> /dev/null
-RESULT=$?
-if [ $RESULT -ne 0 ]; then
-  error "Python3 is not available."
-  exit 1
-fi
+  # ensure python2.7 is available
+  python2.7 --version >/dev/null 2>/dev/null
+  RESULT=$?
+  if [ $RESULT -ne 0 ]; then
+    echo "ERROR: Python2.7 is not available."
+    exit 1
+  fi
 
-# bootstrap, but don't install python 3.
-"$LE_AUTO" --no-self-upgrade -n > /dev/null 2> /dev/null
+  # bootstrap, but don't install python 3.
+  "$LE_AUTO" --no-self-upgrade -n >/dev/null 2>/dev/null
 
-# ensure python 3 isn't installed
-python3 --version 2> /dev/null
-RESULT=$?
-if [ $RESULT -eq 0 ]; then
-  error "letsencrypt-auto installed Python3 even though Python2.7 is present."
-  exit 1
-fi
+  # ensure python 3 isn't installed
+  python3 --version >/dev/null 2>/dev/null
+  RESULT=$?
+  if [ $RESULT -eq 0 ]; then
+    echo "ERROR: letsencrypt-auto installed Python3 even though Python2.7 is present."
+    exit 1
+  fi
 
-echo ""
-echo "PASSED: Did not upgrade to Python3 when Python2.7 is present."
+  echo "PASSED: Did not upgrade to Python3 when Python2.7 is present."
 )
 
 # ensure python2.7 isn't available
-python2.7 --version 2> /dev/null
+python2.7 --version >/dev/null 2>/dev/null
 RESULT=$?
 if [ $RESULT -eq 0 ]; then
-  error "Python2.7 is still available."
+  error "ERROR: Python2.7 is still available."
   exit 1
 fi
 
 # Skip self upgrade due to Python 3 not being available.
 if ! "$LE_AUTO" 2>&1 | grep -q "WARNING: couldn't find Python"; then
-  echo "Python upgrade failure warning not printed!"
+  echo "ERROR: Python upgrade failure warning not printed!"
   exit 1
 fi
 
-# bootstrap, this time installing python3
-"$LE_AUTO" --no-self-upgrade -n > /dev/null 2> /dev/null
+# bootstrap from the old letsencrypt-auto, this time installing python3.4
+"$LE_AUTO_PY_34" --no-self-upgrade -n >/dev/null 2>/dev/null
 
-# ensure python 3 is installed
-python3 --version > /dev/null
+# ensure python 3.4 is installed
+python3.4 --version >/dev/null 2>/dev/null
 RESULT=$?
 if [ $RESULT -ne 0 ]; then
-  error "letsencrypt-auto failed to install Python3 when only Python2.6 is present."
+  echo "ERROR: letsencrypt-auto failed to install Python3.4 using letsencrypt-auto < 0.37.0 when only Python2.6 is present."
   exit 1
 fi
 
-echo "PASSED: Successfully upgraded to Python3 when only Python2.6 is present."
-echo ""
+echo "PASSED: Successfully upgraded to Python3.4 using letsencrypt-auto < 0.37.0 when only Python2.6 is present."
 
-export VENV_PATH=$(mktemp -d)
-"$LE_AUTO" -n --no-bootstrap --no-self-upgrade --version >/dev/null 2>&1
-if [ "$($VENV_PATH/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1)" != 3 ]; then
-  echo "Python 3 wasn't used with --no-bootstrap!"
+# As "certbot-auto" (so without implicit --non-interactive flag set), check that the script
+# refuses to install SCL Python 3.6 when run in a non interactive shell (simulated here
+# using | tee /dev/null) if --non-interactive flag is not provided.
+cp "$LE_AUTO" /tmp/certbot-auto
+# NB: Readline has an issue on all Python versions for CentOS 6, making `certbot --version`
+# output an unprintable ASCII character on a new line at the end.
+# So we take the second last line of the output.
+version=$(/tmp/certbot-auto --version 2>/dev/null | tee /dev/null | tail -2 | head -1)
+
+if [ "$version" != "$INITIAL_CERTBOT_VERSION_PY34" ]; then
+  echo "ERROR: certbot-auto upgraded certbot in a non-interactive shell with --non-interactive flag not set."
   exit 1
 fi
-unset VENV_PATH
+
+echo "PASSED: certbot-auto did not upgrade certbot in a non-interactive shell with --non-interactive flag not set."
+
+if [ -f /opt/rh/rh-python36/enable ]; then
+  echo "ERROR: certbot-auto installed Python3.6 in a non-interactive shell with --non-interactive flag not set."
+  exit 1
+fi
+
+echo "PASSED: certbot-auto did not install Python3.6 in a non-interactive shell with --non-interactive flag not set."
+
+# now bootstrap from current letsencrypt-auto, that will install python3.6 from SCL
+"$LE_AUTO" --no-self-upgrade -n >/dev/null 2>/dev/null
+
+# Following test is exectued in a subshell, to not leak any environment variable
+(
+  # enable SCL rh-python36
+  . scl_source enable rh-python36
+
+  # ensure python 3.6 is installed
+  python3.6 --version >/dev/null 2>/dev/null
+  RESULT=$?
+  if [ $RESULT -ne 0 ]; then
+    echo "ERROR: letsencrypt-auto failed to install Python3.6 using current letsencrypt-auto when only Python2.6/Python3.4 are present."
+    exit 1
+  fi
+
+  echo "PASSED: Successfully upgraded to Python3.6 using current letsencrypt-auto when only Python2.6/Python3.4 are present."
+)
+
+# Following test is exectued in a subshell, to not leak any environment variable
+(
+  export VENV_PATH=$(mktemp -d)
+  "$LE_AUTO" -n --no-bootstrap --no-self-upgrade --version >/dev/null 2>&1
+  if [ "$($VENV_PATH/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1-2)" != "3.6" ]; then
+    echo "ERROR: Python 3.6 wasn't used with --no-bootstrap!"
+    exit 1
+  fi
+)
+
+# Following test is exectued in a subshell, to not leak any environment variable
+(
+  # enable SCL rh-python36
+  . scl_source enable rh-python36
+
+  # ensure everything works fine with certbot-auto bootstrap when python 3.6 is already enabled
+  export VENV_PATH=$(mktemp -d)
+  if ! "$LE_AUTO" --no-self-upgrade -n --version >/dev/null 2>/dev/null; then
+    echo "ERROR: Certbot-auto broke when Python 3.6 SCL is already enabled."
+    exit 1
+  fi
+)
 
 # test using python3
 pytest -v -s certbot/letsencrypt-auto-source/tests

letsencrypt-auto-source/tests/oraclelinux6_tests.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+set -eo pipefail
+# Start by making sure your system is up-to-date:
+yum update -y >/dev/null
+
+LE_AUTO_PY_34="certbot/letsencrypt-auto-source/letsencrypt-auto_py_34"
+LE_AUTO="certbot/letsencrypt-auto-source/letsencrypt-auto"
+
+# Apply installation instructions from official documentation:
+# https://certbot.eff.org/lets-encrypt/centosrhel6-other
+cp "$LE_AUTO" /usr/local/bin/certbot-auto
+chown root /usr/local/bin/certbot-auto
+chmod 0755 /usr/local/bin/certbot-auto
+LE_AUTO=/usr/local/bin/certbot-auto
+
+# Last version of certbot-auto that was bootstraping Python 3.4 for CentOS 6 users
+INITIAL_CERTBOT_VERSION_PY34="certbot 0.38.0"
+
+# Check bootstrap from current certbot-auto will fail, because SCL is not enabled.
+set +o pipefail
+if ! "$LE_AUTO" -n 2>&1 | grep -q "Enable the SCL repository and try running Certbot again."; then
+  echo "ERROR: Bootstrap was not aborted although SCL was not installed!"
+  exit 1
+fi
+set -o pipefail
+
+echo "PASSED: Bootstrap was aborted since SCL was not installed."
+
+# Bootstrap from the old letsencrypt-auto, Python 3.4 will be installed from EPEL.
+"$LE_AUTO_PY_34" --no-self-upgrade -n --install-only >/dev/null 2>/dev/null
+
+# Ensure Python 3.4 is installed
+if ! command -v python3.4 &>/dev/null; then
+  echo "ERROR: old letsencrypt-auto failed to install Python3.4 using letsencrypt-auto < 0.37.0 when only Python2.6 is present."
+  exit 1
+fi
+
+echo "PASSED: Bootstrap from old letsencrypt-auto succeeded and installed Python 3.4"
+
+# Expect certbot-auto to skip rebootstrapping with a warning since SCL is not installed.
+if ! "$LE_AUTO" --non-interactive --version 2>&1 | grep -q "This requires manual user intervention"; then
+  echo "FAILED: Script certbot-auto did not print a warning about needing manual intervention!"
+  exit 1
+fi
+
+echo "PASSED: Script certbot-auto did not rebootstrap."
+
+# NB: Readline has an issue on all Python versions for OL 6, making `certbot --version`
+# output an unprintable ASCII character on a new line at the end.
+# So we take the second last line of the output.
+version=$($LE_AUTO --version 2>/dev/null | tail -2 | head -1)
+
+if [ "$version" != "$INITIAL_CERTBOT_VERSION_PY34" ]; then
+  echo "ERROR: Script certbot-auto upgraded certbot in a non-interactive shell while SCL was not enabled."
+  exit 1
+fi
+
+echo "PASSED: Script certbot-auto did not upgrade certbot but started it successfully while SCL was not enabled."
+
+# Enable SCL
+yum install -y oracle-softwarecollection-release-el6 >/dev/null
+
+# Expect certbot-auto to bootstrap successfully since SCL is available.
+"$LE_AUTO" -n --version &>/dev/null
+
+if [ "$(/opt/eff.org/certbot/venv/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1-2)" != "3.6" ]; then
+  echo "ERROR: Script certbot-auto failed to bootstrap and install Python 3.6 while SCL is available."
+  exit 1
+fi
+
+if ! /opt/eff.org/certbot/venv/bin/certbot --version > /dev/null 2> /dev/null; then
+  echo "ERROR: Script certbot-auto did not install certbot correctly while SCL is enabled."
+  exit 1
+fi
+
+echo "PASSED: Script certbot-auto correctly bootstraped Certbot using rh-python36 when SCL is available."
+
+# Expect certbot-auto will be totally silent now that everything has been correctly boostraped.
+OUTPUT_LEN=$("$LE_AUTO" --install-only --no-self-upgrade --quiet 2>&1 | wc -c)
+if [ "$OUTPUT_LEN" != 0 ]; then
+    echo certbot-auto produced unexpected output!
+    exit 1
+fi
+
+echo "PASSED: Script certbot-auto did not print anything in quiet mode."

local-oldest-requirements.txt

@@ -1,2 +1,2 @@
 # Remember to update setup.py to match the package versions below.
-acme[dev]==0.29.0
+acme[dev]==0.40.0

pull_request_template.md

@@ -1,7 +1,9 @@
 ## Pull Request Checklist
 
-- [ ] Edit the `master` section of `CHANGELOG.md` to include a description of
-  the change being made.
+- [ ] If the change being made is to a [distributed
+  component](https://certbot.eff.org/docs/contributing.html#code-components-and-layout),
+  edit the `master` section of `CHANGELOG.md` to include a description of the
+  change being made.
 - [ ] Add [mypy type
   annotations](https://certbot.eff.org/docs/contributing.html#mypy-type-annotations)
   for any functions that were added or modified.

pytest.ini

@@ -18,3 +18,4 @@ filterwarnings =
     ignore:.*collections\.abc:DeprecationWarning
     ignore:The `color_scheme` argument is deprecated:DeprecationWarning:IPython.*
     ignore:.*view_config_changes:DeprecationWarning
+    ignore:.*get_systemd_os_info:DeprecationWarning

setup.py

@@ -34,7 +34,7 @@ version = meta['version']
 # specified here to avoid masking the more specific request requirements in
 # acme. See https://github.com/pypa/pip/issues/988 for more info.
 install_requires = [
-    'acme>=0.29.0',
+    'acme>=0.40.0',
     # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but
     # saying so here causes a runtime error against our temporary fork of 0.9.3
     # in which we added 2.6 support (see #2243), so we relax the requirement.
@@ -59,7 +59,7 @@ install_requires = [
 # However environment markers are supported only with setuptools >= 36.2.
 # So this dependency is not added for old Linux distributions with old setuptools,
 # in order to allow these systems to build certbot from sources.
-pywin32_req = 'pywin32>=224'
+pywin32_req = 'pywin32>=225'  # do not forget to edit pywin32 dependency accordingly in windows-installer/construct.py
 if StrictVersion(setuptools_version) >= StrictVersion('36.2'):
     install_requires.append(pywin32_req + " ; sys_platform == 'win32'")
 elif 'bdist_wheel' in sys.argv[1:]:

tests/letstest/scripts/test_apache2.sh

@@ -51,7 +51,7 @@ fi
 . tests/letstest/scripts/set_python_envvars.sh
 
 "$VENV_SCRIPT" -e acme[dev] -e .[dev,docs] -e certbot-apache
-sudo "$VENV_PATH/bin/certbot" -v --debug --text --agree-dev-preview --agree-tos \
+sudo "$VENV_PATH/bin/certbot" -v --debug --text --agree-tos \
                    --renew-by-default --redirect --register-unsafely-without-email \
                    --domain $PUBLIC_HOSTNAME --server $BOULDER_URL
 if [ $? -ne 0 ] ; then

tests/letstest/scripts/test_leauto_upgrades.sh

@@ -117,6 +117,8 @@ if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then
 fi
 
 if [ "$RUN_RHEL6_TESTS" = 1 ]; then
+    # Add the SCL python release to PATH in order to resolve python3 command
+    PATH="/opt/rh/rh-python36/root/usr/bin:$PATH"
     if ! command -v python3; then
         echo "Python3 wasn't properly installed"
         exit 1

tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh

@@ -32,7 +32,7 @@ mkdir -p "$OLD_VENV_BIN"
 touch "$OLD_VENV_BIN/letsencrypt"
 
 letsencrypt-auto certonly --no-self-upgrade -v --standalone --debug \
-                   --text --agree-dev-preview --agree-tos \
+                   --text --agree-tos \
                    --renew-by-default --redirect \
                    --register-unsafely-without-email \
                    --domain $PUBLIC_HOSTNAME --server $BOULDER_URL

tests/letstest/scripts/test_sdists.sh

@@ -1,8 +1,21 @@
 #!/bin/sh -xe
 
 cd letsencrypt
+
+# If we're on a RHEL 6 based system, we can be confident Python is already
+# installed because the package manager is written in Python.
+if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then
+    # RHEL/CentOS 6 will need a special treatment, so we need to detect that environment
+    RUN_RHEL6_TESTS=1
+fi
+
 letsencrypt-auto-source/letsencrypt-auto --install-only -n --debug
 
+if [ "$RUN_RHEL6_TESTS" = 1 ]; then
+  # Enable the SCL Python 3.6 installed by letsencrypt-auto bootstrap
+  PATH="/opt/rh/rh-python36/root/usr/bin:$PATH"
+fi
+
 PLUGINS="certbot-apache certbot-nginx"
 PYTHON_MAJOR_VERSION=$(/opt/eff.org/certbot/venv/bin/python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1)
 TEMP_DIR=$(mktemp -d)

tools/dev_constraints.txt

@@ -4,6 +4,7 @@
 # files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt).
 alabaster==0.7.10
 apipkg==1.4
+appnope==0.1.0
 asn1crypto==0.22.0
 astroid==1.6.5
 attrs==17.3.0
@@ -15,6 +16,7 @@ botocore==1.12.36
 cloudflare==1.5.1
 codecov==2.0.15
 configparser==3.7.4
+contextlib2==0.6.0.post1
 coverage==4.5.4
 decorator==4.1.2
 dns-lexicon==3.2.1
@@ -23,9 +25,11 @@ docutils==0.12
 execnet==1.5.0
 future==0.16.0
 futures==3.1.1
-google-api-python-client==1.5
+filelock==3.0.12
+google-api-python-client==1.5.5
 httplib2==0.10.3
 imagesize==0.7.1
+importlib-metadata==0.23
 ipdb==0.10.2
 ipython==5.5.0
 ipython-genutils==0.2.0
@@ -38,14 +42,16 @@ logger==1.4
 logilab-common==1.4.1
 MarkupSafe==1.0
 mccabe==0.6.1
+more-itertools==5.0.0
 mypy==0.600
 ndg-httpsclient==0.3.2
-oauth2client==2.0.0
+oauth2client==4.0.0
+packaging==19.2
 pathlib2==2.3.0
 pexpect==4.7.0
 pickleshare==0.7.4
 pkginfo==1.4.2
-pluggy==0.5.2
+pluggy==0.13.0
 prompt-toolkit==1.0.15
 ptyprocess==0.6.0
 py==1.8.0
@@ -64,7 +70,7 @@ pytest-sugar==0.9.2
 pytest-rerunfailures==4.2
 python-dateutil==2.6.1
 python-digitalocean==1.11
-pywin32==224
+pywin32==225
 PyYAML==3.13
 repoze.sphinx.autointerface==0.8
 requests-file==1.4.2
@@ -79,13 +85,15 @@ Sphinx==1.7.5
 sphinx-rtd-theme==0.2.4
 sphinxcontrib-websupport==1.0.1
 tldextract==2.2.0
+toml==0.10.0
 tox==3.14.0
 tqdm==4.19.4
 traitlets==4.3.2
 twine==1.11.0
 typed-ast==1.1.0
 typing==3.6.4
-uritemplate==0.6
+uritemplate==3.0.0
 virtualenv==16.6.2
 wcwidth==0.1.7
 wrapt==1.11.1
+zipp==0.6.0

tools/extract_changelog.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+from __future__ import print_function
+import sys
+import os
+import re
+
+
+CERTBOT_ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+NEW_SECTION_PATTERN = re.compile(r'^##\s*[\d.]+\s*-\s*[\d-]+$')
+
+
+def main():
+    version = sys.argv[1]
+
+    section_pattern = re.compile(r'^##\s*{0}\s*-\s*[\d-]+$'
+                                 .format(version.replace('.', '\\.')))
+
+    with open(os.path.join(CERTBOT_ROOT, 'CHANGELOG.md')) as file_h:
+        lines = file_h.read().splitlines()
+
+    changelog = []
+
+    i = 0
+    while i < len(lines):
+        if section_pattern.match(lines[i]):
+            i = i + 1
+            while i < len(lines):
+                if NEW_SECTION_PATTERN.match(lines[i]):
+                    break
+                changelog.append(lines[i])
+                i = i + 1
+        i = i + 1
+
+    changelog = [entry for entry in changelog if entry]
+
+    print('\n'.join(changelog))
+
+
+if __name__ == '__main__':
+    main()

tools/oldest_constraints.txt

@@ -16,6 +16,7 @@ pyOpenSSL==0.13.1
 pyparsing==1.5.6
 pyRFC3339==1.0
 python-augeas==0.5.0
+oauth2client==4.0.0
 six==1.9.0
 # setuptools 0.9.8 is the actual version packaged, but some other dependencies
 # in this file require setuptools>=1.0 and there are no relevant changes for us
@@ -35,11 +36,12 @@ idna==2.0
 pbr==1.8.0
 pytz==2012rc0
 
+# Debian Buster constraints
+google-api-python-client==1.5.5
+
 # Our setup.py constraints
 cloudflare==1.5.1
 cryptography==1.2.3
-google-api-python-client==1.5
-oauth2client==2.0
 parsedatetime==1.3
 pyparsing==1.5.5
 python-digitalocean==1.11

tox.cover.py

@@ -56,9 +56,9 @@ def cover(package):
 
 def main():
     description = """
-This script is used by tox.ini (and thus by Travis CI and AppVeyor) in order
-to generate separate stats for each package. It should be removed once those
-packages are moved to a separate repo.
+This script is used by tox.ini (and thus by Travis CI and Azure Pipelines) in
+order to generate separate stats for each package. It should be removed once
+those packages are moved to a separate repo.
 
 Option -e makes sure we fail fast and don't submit to codecov."""
     parser = argparse.ArgumentParser(description=description)

tox.ini

@@ -206,7 +206,17 @@ passenv = DOCKER_*
 # At the moment, this tests under Python 2.6 only, as only that version is
 # readily available on the CentOS 6 Docker image.
 commands =
-    docker build -f letsencrypt-auto-source/Dockerfile.centos6 -t lea letsencrypt-auto-source
+    docker build -f letsencrypt-auto-source/Dockerfile.redhat6 --build-arg REDHAT_DIST_FLAVOR=centos -t lea letsencrypt-auto-source
+    docker run --rm -t -i lea
+whitelist_externals =
+    docker
+passenv = DOCKER_*
+
+[testenv:le_auto_oraclelinux6]
+# At the moment, this tests under Python 2.6 only, as only that version is
+# readily available on the Oracle Linux 6 Docker image.
+commands =
+    docker build -f letsencrypt-auto-source/Dockerfile.redhat6 --build-arg REDHAT_DIST_FLAVOR=oraclelinux -t lea letsencrypt-auto-source
     docker run --rm -t -i lea
 whitelist_externals =
     docker

windows-installer/construct.py

@@ -1,15 +1,17 @@
 #!/usr/bin/env python3
+import contextlib
 import ctypes
 import struct
 import subprocess
 import os
 import sys
 import shutil
+import tempfile
 import time
 
-
 PYTHON_VERSION = (3, 7, 4)
 PYTHON_BITNESS = 32
+PYWIN32_VERSION = 225  # do not forget to edit pywin32 dependency accordingly in setup.py
 
 
 def main():
@@ -42,9 +44,10 @@ def _compile_wheels(repo_path, build_path, venv_python):
     # certbot_packages.extend([name for name in os.listdir(repo_path) if name.startswith('certbot-dns-')])
     wheels_project = [os.path.join(repo_path, package) for package in certbot_packages]
 
-    command = [venv_python, '-m', 'pip', 'wheel', '-w', wheels_path]
-    command.extend(wheels_project)
-    subprocess.check_call(command)
+    with _prepare_constraints(repo_path) as constraints_file_path:
+        command = [venv_python, '-m', 'pip', 'wheel', '-w', wheels_path, '--constraint', constraints_file_path]
+        command.extend(wheels_project)
+        subprocess.check_call(command)
 
 
 def _prepare_build_tools(venv_path, venv_python, repo_path):
@@ -55,6 +58,23 @@ def _prepare_build_tools(venv_path, venv_python, repo_path):
     subprocess.check_call([venv_python, os.path.join(repo_path, 'tools', 'pip_install.py'), 'wheel', 'pynsist'])
 
 
+@contextlib.contextmanager
+def _prepare_constraints(repo_path):
+    requirements = os.path.join(repo_path, 'letsencrypt-auto-source', 'pieces', 'dependency-requirements.txt')
+    constraints = subprocess.check_output(
+        [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), requirements],
+        universal_newlines=True)
+    workdir = tempfile.mkdtemp()
+    try:
+        constraints_file_path = os.path.join(workdir, 'constraints.txt')
+        with open(constraints_file_path, 'a') as file_h:
+            file_h.write(constraints)
+            file_h.write('pywin32=={0}'.format(PYWIN32_VERSION))
+        yield constraints_file_path
+    finally:
+        shutil.rmtree(workdir)
+
+
 def _copy_assets(build_path, repo_path):
     print('Copy assets')
     if os.path.exists(build_path):
@@ -113,7 +133,7 @@ target=$INSTDIR\\run.bat
 [Build]
 directory=nsis
 nsi_template=template.nsi
-installer_name=certbot-installer-{installer_suffix}.exe
+installer_name=certbot-beta-installer-{installer_suffix}.exe
 
 [Python]
 version={python_version}