Re-enable all tests

# Conflicts:
#	CHANGELOG.md

.azure-pipelines/INSTALL.md

@@ -112,6 +112,8 @@ steps:
         CODECOV_TOKEN: $(codecov_token)
 ```
 
-- On Azure DevOps, go to you organization, project, pipeline tab
-- Select the pipeline, click "Edit" button, then click "Variables" button
-- Set name (eg `codecov_token`), value, tick "Keep this value secret"
+To set up a variable that is shared between pipelines, follow the instructions
+at
+https://docs.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups.
+When adding variables to a group, don't forget to tick "Keep this value secret"
+if it shouldn't be shared publcily.

.azure-pipelines/advanced.yml

@@ -7,12 +7,13 @@ pr:
   - '*.x'
 # This pipeline is also nightly run on master
 schedules:
-  - cron: "4 0 * * *"
+  - cron: "0 4 * * *"
     displayName: Nightly build
     branches:
       include:
       - master
+    always: true
 
 jobs:
   - template: templates/tests-suite.yml
-  - template: templates/installer-tests.yml
+  - template: templates/installer-tests.yml

.azure-pipelines/main.yml

@@ -8,4 +8,4 @@ pr:
   - master
 
 jobs:
-  - template: templates/tests-suite.yml
+  - template: templates/tests-suite.yml

.azure-pipelines/templates/installer-tests.yml

@@ -28,4 +28,4 @@ jobs:
       - script: |
           set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
           venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
-        displayName: Run integration tests
+        displayName: Run integration tests

.azure-pipelines/templates/tests-suite.yml

@@ -14,6 +14,8 @@ jobs:
           PYTHON_VERSION: 3.7
           TOXENV: integration-certbot
           PYTEST_ADDOPTS: --numprocesses 4
+    variables:
+    - group: certbot-common
     steps:
     - task: UsePythonVersion@0
       inputs:

.travis.yml

@@ -234,6 +234,9 @@ matrix:
     - sudo: required
       env: TOXENV=le_auto_centos6
       services: docker
+    - sudo: required
+      env: TOXENV=le_auto_oraclelinux6
+      services: docker
       <<: *extended-test-suite
     - sudo: required
       env: TOXENV=docker_dev
@@ -298,15 +301,15 @@ script: '$TRAVIS_RETRY tox'
 
 after_success: '[ "$TOXENV" == "py27-cover" ] && codecov -F linux'
 
-notifications:
-  email: false
-  irc:
-    channels:
-      # This is set to a secure variable to prevent forks from sending
-      # notifications. This value was created by installing
-      # https://github.com/travis-ci/travis.rb and running
-      # `travis encrypt "chat.freenode.net#certbot-devel"`.
-      - secure: "EWW66E2+KVPZyIPR8ViENZwfcup4Gx3/dlimmAZE0WuLwxDCshBBOd3O8Rf6pBokEoZlXM5eDT6XdyJj8n0DLslgjO62pExdunXpbcMwdY7l1ELxX2/UbnDTE6UnPYa09qVBHNG7156Z6yE0x2lH4M9Ykvp0G0cubjPQHylAwo0="
-    on_cancel: never
-    on_success: never
-    on_failure: always
+#notifications:
+#  email: false
+#  irc:
+#    channels:
+#      # This is set to a secure variable to prevent forks from sending
+#      # notifications. This value was created by installing
+#      # https://github.com/travis-ci/travis.rb and running
+#      # `travis encrypt "chat.freenode.net#certbot-devel"`.
+#      - secure: "EWW66E2+KVPZyIPR8ViENZwfcup4Gx3/dlimmAZE0WuLwxDCshBBOd3O8Rf6pBokEoZlXM5eDT6XdyJj8n0DLslgjO62pExdunXpbcMwdY7l1ELxX2/UbnDTE6UnPYa09qVBHNG7156Z6yE0x2lH4M9Ykvp0G0cubjPQHylAwo0="
+#    on_cancel: never
+#    on_success: never
+#    on_failure: always

CHANGELOG.md

@@ -11,6 +11,11 @@ Certbot adheres to [Semantic Versioning](https://semver.org/).
 ### Changed
 
 * Removed `--fast` flag from the test farm tests
+* `--server` may now be combined with `--dry-run`. Certbot will, as before, use the
+  staging server instead of the live server when `--dry-run` is used.
+* Updated certbot-dns-google to depend on newer versions of
+  google-api-python-client and oauth2client.
+* Migrated CentOS 6 certbot-auto users from Python 3.4 to Python 3.6.
 
 ### Fixed
 

appveyor.yml

@@ -1,47 +0,0 @@
-image: Visual Studio 2015
-
-environment:
-  matrix:
-    - TOXENV: py35
-    - TOXENV: py37-cover
-    - TOXENV: integration-certbot
-
-branches:
-  only:
-    # apache-parser-v2 is a temporary branch for doing work related to
-    # rewriting the parser in the Apache plugin.
-    - apache-parser-v2
-    - master
-    - /^\d+\.\d+\.x$/ # Version branches like X.X.X
-    - /^test-.*$/
-
-init:
-  # Since master can receive only commits from PR that have already been tested, following
-  # condition avoid to launch all jobs except the coverage one for commits pushed to master.
-  - ps: |
-      if (-Not $Env:APPVEYOR_PULL_REQUEST_NUMBER -And $Env:APPVEYOR_REPO_BRANCH -Eq 'master' `
-          -And -Not ($Env:TOXENV -Like '*-cover'))
-      { $Env:APPVEYOR_SKIP_FINALIZE_ON_EXIT = 'true'; Exit-AppVeyorBuild }
-
-install:
-  # Use Python 3.7 by default
-  - SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%
-  # Using 4 processes is proven to be the most efficient integration tests config for AppVeyor
-  - IF %TOXENV%==integration-certbot SET PYTEST_ADDOPTS=--numprocesses=4
-  # Check env
-  - python --version
-  # Upgrade pip to avoid warnings
-  - python -m pip install --upgrade pip
-  # Ready to install tox and coverage
-  # tools/pip_install.py is used to pin packages to a known working version.
-  - python tools\\pip_install.py tox codecov
-
-build: off
-
-test_script:
-  - set TOX_TESTENV_PASSENV=APPVEYOR
-  # Test env is set by TOXENV env variable
-  - tox
-
-on_success:
-  - if exist .coverage codecov -F windows

certbot-dns-google/setup.py

@@ -9,11 +9,9 @@ version = '0.40.0.dev0'
 install_requires = [
     'acme>=0.29.0',
     'certbot>=0.39.0',
-    # 1.5 is the first version that supports oauth2client>=2.0
-    'google-api-python-client>=1.5',
+    'google-api-python-client>=1.5.5',
     'mock',
-    # for oauth2client.service_account.ServiceAccountCredentials
-    'oauth2client>=2.0',
+    'oauth2client>=4.0',
     'setuptools',
     'zope.interface',
     # already a dependency of google-api-python-client, but added for consistency

certbot/cli.py

@@ -163,24 +163,6 @@ def report_config_interaction(modified, modifiers):
         VAR_MODIFIERS.setdefault(var, set()).update(modifiers)
 
 
-def possible_deprecation_warning(config):
-    "A deprecation warning for users with the old, not-self-upgrading letsencrypt-auto."
-    if cli_command != LEAUTO:
-        return
-    if config.no_self_upgrade:
-        # users setting --no-self-upgrade might be hanging on a client version like 0.3.0
-        # or 0.5.0 which is the new script, but doesn't set CERTBOT_AUTO; they don't
-        # need warnings
-        return
-    if "CERTBOT_AUTO" not in os.environ:
-        logger.warning("You are running with an old copy of letsencrypt-auto"
-            " that does not receive updates, and is less reliable than more"
-            " recent versions. The letsencrypt client has also been renamed"
-            " to Certbot. We recommend upgrading to the latest certbot-auto"
-            " script, or using native OS packages.")
-        logger.debug("Deprecation warning circumstances: %s / %s", sys.argv[0], os.environ)
-
-
 class _Default(object):
     """A class to use as a default to detect if a value is set by a user"""
 
@@ -642,20 +624,25 @@ class HelpfulArgumentParser(object):
             raise errors.Error(
                 "Parameters --hsts and --auto-hsts cannot be used simultaneously.")
 
-        possible_deprecation_warning(parsed_args)
-
         return parsed_args
 
     def set_test_server(self, parsed_args):
         """We have --staging/--dry-run; perform sanity check and set config.server"""
 
-        if parsed_args.server not in (flag_default("server"), constants.STAGING_URI):
-            conflicts = ["--staging"] if parsed_args.staging else []
-            conflicts += ["--dry-run"] if parsed_args.dry_run else []
-            raise errors.Error("--server value conflicts with {0}".format(
-                " and ".join(conflicts)))
+        # Flag combinations should produce these results:
+        #                             | --staging      | --dry-run   |
+        # ------------------------------------------------------------
+        # | --server acme-v02         | Use staging    | Use staging |
+        # | --server acme-staging-v02 | Use staging    | Use staging |
+        # | --server <other>          | Conflict error | Use <other> |
 
-        parsed_args.server = constants.STAGING_URI
+        default_servers = (flag_default("server"), constants.STAGING_URI)
+
+        if parsed_args.staging and parsed_args.server not in default_servers:
+            raise errors.Error("--server value conflicts with --staging")
+
+        if parsed_args.server in default_servers:
+            parsed_args.server = constants.STAGING_URI
 
         if parsed_args.dry_run:
             if self.verb not in ["certonly", "renew"]:

certbot/compat/filesystem.py

@@ -546,13 +546,7 @@ def _generate_windows_flags(rights_desc):
     if rights_desc['write']:
         flag = flag | (ntsecuritycon.FILE_ALL_ACCESS
                        ^ ntsecuritycon.FILE_GENERIC_READ
-                       ^ ntsecuritycon.FILE_GENERIC_EXECUTE
-                       # Despite bit `512` being present in ntsecuritycon.FILE_ALL_ACCESS, it is
-                       # not effectively applied to the file or the directory.
-                       # As _generate_windows_flags is also used to compare two dacls, we remove
-                       # it right now to have flags that contain only the bits effectively applied
-                       # by Windows.
-                       ^ 512)
+                       ^ ntsecuritycon.FILE_GENERIC_EXECUTE)
     if rights_desc['execute']:
         flag = flag | ntsecuritycon.FILE_GENERIC_EXECUTE
 

certbot/tests/cli_test.py

@@ -333,16 +333,26 @@ class ParseTest(unittest.TestCase):  # pylint: disable=too-many-public-methods
 
         self._assert_dry_run_flag_worked(self.parse(short_args + ['auth']), True)
         self._assert_dry_run_flag_worked(self.parse(short_args + ['renew']), True)
+        self._assert_dry_run_flag_worked(self.parse(short_args + ['certonly']), True)
+
         short_args += ['certonly']
-        self._assert_dry_run_flag_worked(self.parse(short_args), True)
 
-        short_args += '--server example.com'.split()
-        conflicts = ['--dry-run']
-        self._check_server_conflict_message(short_args, '--dry-run')
+        # `--dry-run --server example.com` should emit example.com
+        self.assertEqual(self.parse(short_args + ['--server', 'example.com']).server,
+                         'example.com')
 
-        short_args += ['--staging']
-        conflicts += ['--staging']
-        self._check_server_conflict_message(short_args, conflicts)
+        # `--dry-run --server STAGING_URI` should emit STAGING_URI
+        self.assertEqual(self.parse(short_args + ['--server', constants.STAGING_URI]).server,
+                         constants.STAGING_URI)
+
+        # `--dry-run --server LIVE` should emit STAGING_URI
+        self.assertEqual(self.parse(short_args + ['--server', cli.flag_default("server")]).server,
+                         constants.STAGING_URI)
+
+        # `--dry-run --server example.com --staging` should emit an error
+        conflicts = ['--staging']
+        self._check_server_conflict_message(short_args + ['--server', 'example.com', '--staging'],
+                                            conflicts)
 
     def test_option_was_set(self):
         key_size_option = 'rsa_key_size'

certbot/tests/compat/filesystem_test.py

@@ -89,8 +89,8 @@ class WindowsChmodTests(TempDirTestCase):
         self.assertEqual(len(system_aces), 1)
         self.assertEqual(len(admin_aces), 1)
 
-        self.assertEqual(system_aces[0][1], ntsecuritycon.FILE_ALL_ACCESS ^ 512)
-        self.assertEqual(admin_aces[0][1], ntsecuritycon.FILE_ALL_ACCESS ^ 512)
+        self.assertEqual(system_aces[0][1], ntsecuritycon.FILE_ALL_ACCESS)
+        self.assertEqual(admin_aces[0][1], ntsecuritycon.FILE_ALL_ACCESS)
 
     def test_read_flag(self):
         self._test_flag(4, ntsecuritycon.FILE_GENERIC_READ)
@@ -101,12 +101,10 @@ class WindowsChmodTests(TempDirTestCase):
     def test_write_flag(self):
         self._test_flag(2, (ntsecuritycon.FILE_ALL_ACCESS
                             ^ ntsecuritycon.FILE_GENERIC_READ
-                            ^ ntsecuritycon.FILE_GENERIC_EXECUTE
-                            ^ 512))
+                            ^ ntsecuritycon.FILE_GENERIC_EXECUTE))
 
     def test_full_flag(self):
-        self._test_flag(7, (ntsecuritycon.FILE_ALL_ACCESS
-                            ^ 512))
+        self._test_flag(7, ntsecuritycon.FILE_ALL_ACCESS)
 
     def _test_flag(self, everyone_mode, windows_flag):
         # Note that flag is tested against `everyone`, not `user`, because practically these unit

docs/using.rst

@@ -917,8 +917,9 @@ Certbot accepts a global configuration file that applies its options to all invo
 of Certbot. Certificate specific configuration choices should be set in the ``.conf``
 files that can be found in ``/etc/letsencrypt/renewal``.
 
-By default no cli.ini file is created, after creating one
-it is possible to specify the location of this configuration file with
+By default no cli.ini file is created (though it may exist already if you installed Certbot
+via a package manager, for instance).
+After creating one it is possible to specify the location of this configuration file with
 ``certbot --config cli.ini`` (or shorter ``-c cli.ini``). An
 example configuration file is shown below:
 

letsencrypt-auto-source/Dockerfile.redhat6

@@ -1,9 +1,13 @@
 # For running tests, build a docker image with a passwordless sudo and a trust
 # store we can manipulate.
 
-FROM centos:6
+ARG REDHAT_DIST_FLAVOR
+FROM ${REDHAT_DIST_FLAVOR}:6
 
-RUN yum install -y epel-release
+ARG REDHAT_DIST_FLAVOR
+
+RUN curl -O https://dl.fedoraproject.org/pub/epel/epel-release-latest-6.noarch.rpm \
+ && rpm -ivh epel-release-latest-6.noarch.rpm
 
 # Install pip and sudo:
 RUN yum install -y python-pip sudo
@@ -27,11 +31,18 @@ RUN mkdir -p /home/lea/certbot
 COPY ./tests/certs/ca/my-root-ca.crt.pem /usr/local/share/ca-certificates/
 RUN update-ca-trust
 
-# Copy code:
+# Copy current letsencrypt-auto:
 COPY . /home/lea/certbot/letsencrypt-auto-source
 
+# Fetch previous letsencrypt-auto that was installing python 3.4
+RUN curl https://raw.githubusercontent.com/certbot/certbot/v0.38.0/letsencrypt-auto-source/letsencrypt-auto \
+    -o /home/lea/certbot/letsencrypt-auto-source/letsencrypt-auto_py_34 \
+ && chmod +x /home/lea/certbot/letsencrypt-auto-source/letsencrypt-auto_py_34
+
+RUN cp /home/lea/certbot/letsencrypt-auto-source/tests/${REDHAT_DIST_FLAVOR}6_tests.sh /home/lea/certbot/letsencrypt-auto-source/tests/redhat6_tests.sh \
+ && chmod +x /home/lea/certbot/letsencrypt-auto-source/tests/redhat6_tests.sh
+
 USER lea
 WORKDIR /home/lea
 
-RUN sudo chmod +x certbot/letsencrypt-auto-source/tests/centos6_tests.sh
-CMD sudo certbot/letsencrypt-auto-source/tests/centos6_tests.sh
+CMD ["sudo", "certbot/letsencrypt-auto-source/tests/redhat6_tests.sh"]

letsencrypt-auto-source/letsencrypt-auto

@@ -256,20 +256,28 @@ DeprecationBootstrap() {
   fi
 }
 
-MIN_PYTHON_VERSION="2.7"
-MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//')
+MIN_PYTHON_2_VERSION="2.7"
+MIN_PYVER2=$(echo "$MIN_PYTHON_2_VERSION" | sed 's/\.//')
+MIN_PYTHON_3_VERSION="3.5"
+MIN_PYVER3=$(echo "$MIN_PYTHON_3_VERSION" | sed 's/\.//')
 # Sets LE_PYTHON to Python version string and PYVER to the first two
-# digits of the python version
+# digits of the python version.
+# MIN_PYVER and MIN_PYTHON_VERSION are also set by this function, and their
+# values depend on if we try to use Python 3 or Python 2.
 DeterminePythonVersion() {
   # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
   #
   # If no Python is found, PYVER is set to 0.
   if [ "$USE_PYTHON_3" = 1 ]; then
+    MIN_PYVER=$MIN_PYVER3
+    MIN_PYTHON_VERSION=$MIN_PYTHON_3_VERSION
     for LE_PYTHON in "$LE_PYTHON" python3; do
       # Break (while keeping the LE_PYTHON value) if found.
       $EXISTS "$LE_PYTHON" > /dev/null && break
     done
   else
+    MIN_PYVER=$MIN_PYVER2
+    MIN_PYTHON_VERSION=$MIN_PYTHON_2_VERSION
     for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
       # Break (while keeping the LE_PYTHON value) if found.
       $EXISTS "$LE_PYTHON" > /dev/null && break
@@ -285,7 +293,7 @@ DeterminePythonVersion() {
     fi
   fi
 
-  PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
+  PYVER=$("$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//')
   if [ "$PYVER" -lt "$MIN_PYVER" ]; then
     if [ "$1" != "NOCRASH" ]; then
       error "You have an ancient version of Python entombed in your operating system..."
@@ -368,7 +376,9 @@ BootstrapDebCommon() {
 
 # Sets TOOL to the name of the package manager
 # Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG.
-# Enables EPEL if applicable and possible.
+# Note: this function is called both while selecting the bootstrap scripts and
+# during the actual bootstrap. Some things like prompting to user can be done in the latter
+# case, but not in the former one.
 InitializeRPMCommonBase() {
   if type dnf 2>/dev/null
   then
@@ -388,26 +398,6 @@ InitializeRPMCommonBase() {
   if [ "$QUIET" = 1 ]; then
     QUIET_FLAG='--quiet'
   fi
-
-  if ! $TOOL list *virtualenv >/dev/null 2>&1; then
-    echo "To use Certbot, packages from the EPEL repository need to be installed."
-    if ! $TOOL list epel-release >/dev/null 2>&1; then
-      error "Enable the EPEL repository and try running Certbot again."
-      exit 1
-    fi
-    if [ "$ASSUME_YES" = 1 ]; then
-      /bin/echo -n "Enabling the EPEL repository in 3 seconds..."
-      sleep 1s
-      /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
-      sleep 1s
-      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..."
-      sleep 1s
-    fi
-    if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then
-      error "Could not enable EPEL. Aborting bootstrap!"
-      exit 1
-    fi
-  fi
 }
 
 BootstrapRpmCommonBase() {
@@ -488,13 +478,88 @@ BootstrapRpmCommon() {
   BootstrapRpmCommonBase "$python_pkgs"
 }
 
+# If new packages are installed by BootstrapRpmPython3 below, this version
+# number must be increased.
+BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION=1
+
+# Checks if rh-python36 can be installed.
+Python36SclIsAvailable() {
+  InitializeRPMCommonBase >/dev/null 2>&1;
+
+  if "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    return 0
+  fi
+  if "${TOOL}" list centos-release-scl >/dev/null 2>&1; then
+    return 0
+  fi
+  return 1
+}
+
+# Try to enable rh-python36 from SCL if it is necessary and possible.
+EnablePython36SCL() {
+  if "$EXISTS" python3.6 > /dev/null 2> /dev/null; then
+      return 0
+  fi
+  if ! scl --list 2>/dev/null | grep -q rh-python36; then
+      return 0
+  fi
+  set +e
+  . scl_source enable rh-python36
+  set -e
+}
+
+# This bootstrap concerns old RedHat-based distributions that do not ship by default
+# with Python 2.7, but only Python 2.6. We bootstrap them by enabling SCL and installing
+# Python 3.6. Some of these distributions are: CentOS/RHEL/OL/SL 6.
+BootstrapRpmPython3Legacy() {
+  # Tested with:
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  if ! "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    echo "To use Certbot on this operating system, packages from the SCL repository need to be installed."
+    if ! "${TOOL}" list centos-release-scl >/dev/null 2>&1; then
+      error "Enable the SCL repository and try running Certbot again."
+      exit 1
+    fi
+    if [ "${ASSUME_YES}" = 1 ]; then
+      /bin/echo -n "Enabling the SCL repository in 3 seconds... (Press Ctrl-C to cancel)"
+      sleep 1s
+      /bin/echo -ne "\e[0K\rEnabling the SCL repository in 2 seconds... (Press Ctrl-C to cancel)"
+      sleep 1s
+      /bin/echo -e "\e[0K\rEnabling the SCL repository in 1 second... (Press Ctrl-C to cancel)"
+      sleep 1s
+    fi
+    if ! "${TOOL}" install "${YES_FLAG}" "${QUIET_FLAG}" centos-release-scl; then
+      error "Could not enable SCL. Aborting bootstrap!"
+      exit 1
+    fi
+  fi
+
+  # CentOS 6 must use rh-python36 from SCL
+  if "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    python_pkgs="rh-python36-python
+      rh-python36-python-virtualenv
+      rh-python36-python-devel
+    "
+  else
+    error "No supported Python package available to install. Aborting bootstrap!"
+    exit 1
+  fi
+
+  BootstrapRpmCommonBase "${python_pkgs}"
+
+  # Enable SCL rh-python36 after bootstrapping.
+  EnablePython36SCL
+}
+
 # If new packages are installed by BootstrapRpmPython3 below, this version
 # number must be increased.
 BOOTSTRAP_RPM_PYTHON3_VERSION=1
 
 BootstrapRpmPython3() {
   # Tested with:
-  #   - CentOS 6
   #   - Fedora 29
 
   InitializeRPMCommonBase
@@ -505,12 +570,6 @@ BootstrapRpmPython3() {
       python3-virtualenv
       python3-devel
     "
-  # EPEL uses python34
-  elif $TOOL list python34 >/dev/null 2>&1; then
-    python_pkgs="python34
-      python34-devel
-      python34-tools
-    "
   else
     error "No supported Python package available to install. Aborting bootstrap!"
     exit 1
@@ -769,31 +828,50 @@ elif [ -f /etc/redhat-release ]; then
      RPM_DIST_VERSION=0
   fi
 
-  # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then.
-  # RHEL 8 also uses python3 by default.
-  if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then
-    RPM_USE_PYTHON_3=1
-  elif [ "$RPM_DIST_NAME" = "rhel" -a "$RPM_DIST_VERSION" -ge 8 ]; then
-    RPM_USE_PYTHON_3=1
-  elif [ "$RPM_DIST_NAME" = "centos" -a "$RPM_DIST_VERSION" -ge 8 ]; then
-    RPM_USE_PYTHON_3=1
-  else
-    RPM_USE_PYTHON_3=0
-  fi
+  # Handle legacy RPM distributions
+  if [ "$PYVER" -eq 26 ]; then
+    # Check if an automated bootstrap can be achieved on this system.
+    if ! Python36SclIsAvailable; then
+      INTERACTIVE_BOOTSTRAP=1
+    fi
 
-  if [ "$RPM_USE_PYTHON_3" = 1 ]; then
     Bootstrap() {
-      BootstrapMessage "RedHat-based OSes that will use Python3"
-      BootstrapRpmPython3
+      BootstrapMessage "Legacy RedHat-based OSes that will use Python3"
+      BootstrapRpmPython3Legacy
     }
     USE_PYTHON_3=1
-    BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+    BOOTSTRAP_VERSION="BootstrapRpmPython3Legacy $BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION"
+
+    # Try now to enable SCL rh-python36 for systems already bootstrapped
+    # NB: EnablePython36SCL has been defined along with BootstrapRpmPython3Legacy in certbot-auto
+    EnablePython36SCL
   else
-    Bootstrap() {
-      BootstrapMessage "RedHat-based OSes"
-      BootstrapRpmCommon
-    }
-    BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+    # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then.
+    # RHEL 8 also uses python3 by default.
+    if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 ]; then
+      RPM_USE_PYTHON_3=1
+    elif [ "$RPM_DIST_NAME" = "rhel" -a "$RPM_DIST_VERSION" -ge 8 ]; then
+      RPM_USE_PYTHON_3=1
+    elif [ "$RPM_DIST_NAME" = "centos" -a "$RPM_DIST_VERSION" -ge 8 ]; then
+      RPM_USE_PYTHON_3=1
+    else
+      RPM_USE_PYTHON_3=0
+    fi
+
+    if [ "$RPM_USE_PYTHON_3" = 1 ]; then
+      Bootstrap() {
+        BootstrapMessage "RedHat-based OSes that will use Python3"
+        BootstrapRpmPython3
+      }
+      USE_PYTHON_3=1
+      BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+    else
+      Bootstrap() {
+        BootstrapMessage "RedHat-based OSes"
+        BootstrapRpmCommon
+      }
+      BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+    fi
   fi
 
   LE_PYTHON="$prev_le_python"
@@ -1078,8 +1156,15 @@ if [ "$1" = "--le-auto-phase2" ]; then
     # If the selected Bootstrap function isn't a noop and it differs from the
     # previously used version
     if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then
-      # if non-interactive mode or stdin and stdout are connected to a terminal
-      if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
+      # Check if we can rebootstrap without manual user intervention: this requires that
+      # certbot-auto is in non-interactive mode AND selected bootstrap does not claim to
+      # require a manual user intervention.
+      if [ "$NONINTERACTIVE" = 1 -a "$INTERACTIVE_BOOTSTRAP" != 1 ]; then
+        CAN_REBOOTSTRAP=1
+      fi
+      # Check if rebootstrap can be done non-interactively and current shell is non-interactive
+      # (true if stdin and stdout are not attached to a terminal).
+      if [ \( "$CAN_REBOOTSTRAP" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
         if [ -d "$VENV_PATH" ]; then
           rm -rf "$VENV_PATH"
         fi
@@ -1090,12 +1175,21 @@ if [ "$1" = "--le-auto-phase2" ]; then
           ln -s "$VENV_PATH" "$OLD_VENV_PATH"
         fi
         RerunWithArgs "$@"
+      # Otherwise bootstrap needs to be done manually by the user.
       else
-        error "Skipping upgrade because new OS dependencies may need to be installed."
-        error
-        error "To upgrade to a newer version, please run this script again manually so you can"
-        error "approve changes or with --non-interactive on the command line to automatically"
-        error "install any required packages."
+        # If it is because bootstrapping is interactive, --non-interactive will be of no use.
+        if [ "$INTERACTIVE_BOOTSTRAP" = 1 ]; then
+          error "Skipping upgrade because new OS dependencies may need to be installed."
+          error "This requires manual user intervention: please run this script again manually."
+        # If this is because of the environment (eg. non interactive shell without
+        # --non-interactive flag set), help the user in that direction.
+        else
+          error "Skipping upgrade because new OS dependencies may need to be installed."
+          error
+          error "To upgrade to a newer version, please run this script again manually so you can"
+          error "approve changes or with --non-interactive on the command line to automatically"
+          error "install any required packages."
+        fi
         # Set INSTALLED_VERSION to be the same so we don't update the venv
         INSTALLED_VERSION="$LE_AUTO_VERSION"
         # Continue to use OLD_VENV_PATH if the new venv doesn't exist

letsencrypt-auto-source/letsencrypt-auto.template

@@ -256,20 +256,28 @@ DeprecationBootstrap() {
   fi
 }
 
-MIN_PYTHON_VERSION="2.7"
-MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//')
+MIN_PYTHON_2_VERSION="2.7"
+MIN_PYVER2=$(echo "$MIN_PYTHON_2_VERSION" | sed 's/\.//')
+MIN_PYTHON_3_VERSION="3.5"
+MIN_PYVER3=$(echo "$MIN_PYTHON_3_VERSION" | sed 's/\.//')
 # Sets LE_PYTHON to Python version string and PYVER to the first two
-# digits of the python version
+# digits of the python version.
+# MIN_PYVER and MIN_PYTHON_VERSION are also set by this function, and their
+# values depend on if we try to use Python 3 or Python 2.
 DeterminePythonVersion() {
   # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
   #
   # If no Python is found, PYVER is set to 0.
   if [ "$USE_PYTHON_3" = 1 ]; then
+    MIN_PYVER=$MIN_PYVER3
+    MIN_PYTHON_VERSION=$MIN_PYTHON_3_VERSION
     for LE_PYTHON in "$LE_PYTHON" python3; do
       # Break (while keeping the LE_PYTHON value) if found.
       $EXISTS "$LE_PYTHON" > /dev/null && break
     done
   else
+    MIN_PYVER=$MIN_PYVER2
+    MIN_PYTHON_VERSION=$MIN_PYTHON_2_VERSION
     for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
       # Break (while keeping the LE_PYTHON value) if found.
       $EXISTS "$LE_PYTHON" > /dev/null && break
@@ -285,7 +293,7 @@ DeterminePythonVersion() {
     fi
   fi
 
-  PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
+  PYVER=$("$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//')
   if [ "$PYVER" -lt "$MIN_PYVER" ]; then
     if [ "$1" != "NOCRASH" ]; then
       error "You have an ancient version of Python entombed in your operating system..."
@@ -298,6 +306,7 @@ DeterminePythonVersion() {
 {{ bootstrappers/deb_common.sh }}
 {{ bootstrappers/rpm_common_base.sh }}
 {{ bootstrappers/rpm_common.sh }}
+{{ bootstrappers/rpm_python3_legacy.sh }}
 {{ bootstrappers/rpm_python3.sh }}
 {{ bootstrappers/suse_common.sh }}
 {{ bootstrappers/arch_common.sh }}
@@ -344,31 +353,50 @@ elif [ -f /etc/redhat-release ]; then
      RPM_DIST_VERSION=0
   fi
 
-  # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then.
-  # RHEL 8 also uses python3 by default.
-  if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then
-    RPM_USE_PYTHON_3=1
-  elif [ "$RPM_DIST_NAME" = "rhel" -a "$RPM_DIST_VERSION" -ge 8 ]; then
-    RPM_USE_PYTHON_3=1
-  elif [ "$RPM_DIST_NAME" = "centos" -a "$RPM_DIST_VERSION" -ge 8 ]; then
-    RPM_USE_PYTHON_3=1
-  else
-    RPM_USE_PYTHON_3=0
-  fi
+  # Handle legacy RPM distributions
+  if [ "$PYVER" -eq 26 ]; then
+    # Check if an automated bootstrap can be achieved on this system.
+    if ! Python36SclIsAvailable; then
+      INTERACTIVE_BOOTSTRAP=1
+    fi
 
-  if [ "$RPM_USE_PYTHON_3" = 1 ]; then
     Bootstrap() {
-      BootstrapMessage "RedHat-based OSes that will use Python3"
-      BootstrapRpmPython3
+      BootstrapMessage "Legacy RedHat-based OSes that will use Python3"
+      BootstrapRpmPython3Legacy
     }
     USE_PYTHON_3=1
-    BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+    BOOTSTRAP_VERSION="BootstrapRpmPython3Legacy $BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION"
+
+    # Try now to enable SCL rh-python36 for systems already bootstrapped
+    # NB: EnablePython36SCL has been defined along with BootstrapRpmPython3Legacy in certbot-auto
+    EnablePython36SCL
   else
-    Bootstrap() {
-      BootstrapMessage "RedHat-based OSes"
-      BootstrapRpmCommon
-    }
-    BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+    # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then.
+    # RHEL 8 also uses python3 by default.
+    if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 ]; then
+      RPM_USE_PYTHON_3=1
+    elif [ "$RPM_DIST_NAME" = "rhel" -a "$RPM_DIST_VERSION" -ge 8 ]; then
+      RPM_USE_PYTHON_3=1
+    elif [ "$RPM_DIST_NAME" = "centos" -a "$RPM_DIST_VERSION" -ge 8 ]; then
+      RPM_USE_PYTHON_3=1
+    else
+      RPM_USE_PYTHON_3=0
+    fi
+
+    if [ "$RPM_USE_PYTHON_3" = 1 ]; then
+      Bootstrap() {
+        BootstrapMessage "RedHat-based OSes that will use Python3"
+        BootstrapRpmPython3
+      }
+      USE_PYTHON_3=1
+      BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+    else
+      Bootstrap() {
+        BootstrapMessage "RedHat-based OSes"
+        BootstrapRpmCommon
+      }
+      BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+    fi
   fi
 
   LE_PYTHON="$prev_le_python"
@@ -545,8 +573,15 @@ if [ "$1" = "--le-auto-phase2" ]; then
     # If the selected Bootstrap function isn't a noop and it differs from the
     # previously used version
     if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then
-      # if non-interactive mode or stdin and stdout are connected to a terminal
-      if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
+      # Check if we can rebootstrap without manual user intervention: this requires that
+      # certbot-auto is in non-interactive mode AND selected bootstrap does not claim to
+      # require a manual user intervention.
+      if [ "$NONINTERACTIVE" = 1 -a "$INTERACTIVE_BOOTSTRAP" != 1 ]; then
+        CAN_REBOOTSTRAP=1
+      fi
+      # Check if rebootstrap can be done non-interactively and current shell is non-interactive
+      # (true if stdin and stdout are not attached to a terminal).
+      if [ \( "$CAN_REBOOTSTRAP" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
         if [ -d "$VENV_PATH" ]; then
           rm -rf "$VENV_PATH"
         fi
@@ -557,12 +592,21 @@ if [ "$1" = "--le-auto-phase2" ]; then
           ln -s "$VENV_PATH" "$OLD_VENV_PATH"
         fi
         RerunWithArgs "$@"
+      # Otherwise bootstrap needs to be done manually by the user.
       else
-        error "Skipping upgrade because new OS dependencies may need to be installed."
-        error
-        error "To upgrade to a newer version, please run this script again manually so you can"
-        error "approve changes or with --non-interactive on the command line to automatically"
-        error "install any required packages."
+        # If it is because bootstrapping is interactive, --non-interactive will be of no use.
+        if [ "$INTERACTIVE_BOOTSTRAP" = 1 ]; then
+          error "Skipping upgrade because new OS dependencies may need to be installed."
+          error "This requires manual user intervention: please run this script again manually."
+        # If this is because of the environment (eg. non interactive shell without
+        # --non-interactive flag set), help the user in that direction.
+        else
+          error "Skipping upgrade because new OS dependencies may need to be installed."
+          error
+          error "To upgrade to a newer version, please run this script again manually so you can"
+          error "approve changes or with --non-interactive on the command line to automatically"
+          error "install any required packages."
+        fi
         # Set INSTALLED_VERSION to be the same so we don't update the venv
         INSTALLED_VERSION="$LE_AUTO_VERSION"
         # Continue to use OLD_VENV_PATH if the new venv doesn't exist

letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh

@@ -3,7 +3,9 @@
 
 # Sets TOOL to the name of the package manager
 # Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG.
-# Enables EPEL if applicable and possible.
+# Note: this function is called both while selecting the bootstrap scripts and
+# during the actual bootstrap. Some things like prompting to user can be done in the latter
+# case, but not in the former one.
 InitializeRPMCommonBase() {
   if type dnf 2>/dev/null
   then
@@ -23,26 +25,6 @@ InitializeRPMCommonBase() {
   if [ "$QUIET" = 1 ]; then
     QUIET_FLAG='--quiet'
   fi
-
-  if ! $TOOL list *virtualenv >/dev/null 2>&1; then
-    echo "To use Certbot, packages from the EPEL repository need to be installed."
-    if ! $TOOL list epel-release >/dev/null 2>&1; then
-      error "Enable the EPEL repository and try running Certbot again."
-      exit 1
-    fi
-    if [ "$ASSUME_YES" = 1 ]; then
-      /bin/echo -n "Enabling the EPEL repository in 3 seconds..."
-      sleep 1s
-      /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
-      sleep 1s
-      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..."
-      sleep 1s
-    fi
-    if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then
-      error "Could not enable EPEL. Aborting bootstrap!"
-      exit 1
-    fi
-  fi
 }
 
 BootstrapRpmCommonBase() {

letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh

@@ -4,7 +4,6 @@ BOOTSTRAP_RPM_PYTHON3_VERSION=1
 
 BootstrapRpmPython3() {
   # Tested with:
-  #   - CentOS 6
   #   - Fedora 29
 
   InitializeRPMCommonBase
@@ -15,12 +14,6 @@ BootstrapRpmPython3() {
       python3-virtualenv
       python3-devel
     "
-  # EPEL uses python34
-  elif $TOOL list python34 >/dev/null 2>&1; then
-    python_pkgs="python34
-      python34-devel
-      python34-tools
-    "
   else
     error "No supported Python package available to install. Aborting bootstrap!"
     exit 1

letsencrypt-auto-source/pieces/bootstrappers/rpm_python3_legacy.sh

@@ -0,0 +1,75 @@
+# If new packages are installed by BootstrapRpmPython3 below, this version
+# number must be increased.
+BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION=1
+
+# Checks if rh-python36 can be installed.
+Python36SclIsAvailable() {
+  InitializeRPMCommonBase >/dev/null 2>&1;
+
+  if "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    return 0
+  fi
+  if "${TOOL}" list centos-release-scl >/dev/null 2>&1; then
+    return 0
+  fi
+  return 1
+}
+
+# Try to enable rh-python36 from SCL if it is necessary and possible.
+EnablePython36SCL() {
+  if "$EXISTS" python3.6 > /dev/null 2> /dev/null; then
+      return 0
+  fi
+  if ! scl --list 2>/dev/null | grep -q rh-python36; then
+      return 0
+  fi
+  set +e
+  . scl_source enable rh-python36
+  set -e
+}
+
+# This bootstrap concerns old RedHat-based distributions that do not ship by default
+# with Python 2.7, but only Python 2.6. We bootstrap them by enabling SCL and installing
+# Python 3.6. Some of these distributions are: CentOS/RHEL/OL/SL 6.
+BootstrapRpmPython3Legacy() {
+  # Tested with:
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  if ! "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    echo "To use Certbot on this operating system, packages from the SCL repository need to be installed."
+    if ! "${TOOL}" list centos-release-scl >/dev/null 2>&1; then
+      error "Enable the SCL repository and try running Certbot again."
+      exit 1
+    fi
+    if [ "${ASSUME_YES}" = 1 ]; then
+      /bin/echo -n "Enabling the SCL repository in 3 seconds... (Press Ctrl-C to cancel)"
+      sleep 1s
+      /bin/echo -ne "\e[0K\rEnabling the SCL repository in 2 seconds... (Press Ctrl-C to cancel)"
+      sleep 1s
+      /bin/echo -e "\e[0K\rEnabling the SCL repository in 1 second... (Press Ctrl-C to cancel)"
+      sleep 1s
+    fi
+    if ! "${TOOL}" install "${YES_FLAG}" "${QUIET_FLAG}" centos-release-scl; then
+      error "Could not enable SCL. Aborting bootstrap!"
+      exit 1
+    fi
+  fi
+
+  # CentOS 6 must use rh-python36 from SCL
+  if "${TOOL}" list rh-python36 >/dev/null 2>&1; then
+    python_pkgs="rh-python36-python
+      rh-python36-python-virtualenv
+      rh-python36-python-devel
+    "
+  else
+    error "No supported Python package available to install. Aborting bootstrap!"
+    exit 1
+  fi
+
+  BootstrapRpmCommonBase "${python_pkgs}"
+
+  # Enable SCL rh-python36 after bootstrapping.
+  EnablePython36SCL
+}

letsencrypt-auto-source/tests/centos6_tests.sh

@@ -1,81 +1,140 @@
 #!/bin/bash
 # Start by making sure your system is up-to-date:
-yum update -y > /dev/null
-yum install -y centos-release-scl > /dev/null
-yum install -y python27 > /dev/null 2> /dev/null
+yum update -y >/dev/null
+yum install -y centos-release-scl >/dev/null
+yum install -y python27 >/dev/null 2>/dev/null
 
+LE_AUTO_PY_34="certbot/letsencrypt-auto-source/letsencrypt-auto_py_34"
 LE_AUTO="certbot/letsencrypt-auto-source/letsencrypt-auto"
 
+# Last version of certbot-auto that was bootstraping Python 3.4 for CentOS 6 users
+INITIAL_CERTBOT_VERSION_PY34="certbot 0.38.0"
+
 # we're going to modify env variables, so do this in a subshell
 (
-source /opt/rh/python27/enable
+  . scl_source enable python27
 
-# ensure python 3 isn't installed
-python3 --version 2> /dev/null
-RESULT=$?
-if [ $RESULT -eq 0 ]; then
-  error "Python3 is already installed."
-  exit 1
-fi
+  # ensure python 3 isn't installed
+  python3 --version >/dev/null 2>/dev/null
+  RESULT=$?
+  if [ $RESULT -eq 0 ]; then
+    echo "ERROR: Python3 is already installed."
+    exit 1
+  fi
 
-# ensure python2.7 is available
-python2.7 --version 2> /dev/null
-RESULT=$?
-if [ $RESULT -ne 0 ]; then
-  error "Python3 is not available."
-  exit 1
-fi
+  # ensure python2.7 is available
+  python2.7 --version >/dev/null 2>/dev/null
+  RESULT=$?
+  if [ $RESULT -ne 0 ]; then
+    echo "ERROR: Python2.7 is not available."
+    exit 1
+  fi
 
-# bootstrap, but don't install python 3.
-"$LE_AUTO" --no-self-upgrade -n > /dev/null 2> /dev/null
+  # bootstrap, but don't install python 3.
+  "$LE_AUTO" --no-self-upgrade -n >/dev/null 2>/dev/null
 
-# ensure python 3 isn't installed
-python3 --version 2> /dev/null
-RESULT=$?
-if [ $RESULT -eq 0 ]; then
-  error "letsencrypt-auto installed Python3 even though Python2.7 is present."
-  exit 1
-fi
+  # ensure python 3 isn't installed
+  python3 --version >/dev/null 2>/dev/null
+  RESULT=$?
+  if [ $RESULT -eq 0 ]; then
+    echo "ERROR: letsencrypt-auto installed Python3 even though Python2.7 is present."
+    exit 1
+  fi
 
-echo ""
-echo "PASSED: Did not upgrade to Python3 when Python2.7 is present."
+  echo "PASSED: Did not upgrade to Python3 when Python2.7 is present."
 )
 
 # ensure python2.7 isn't available
-python2.7 --version 2> /dev/null
+python2.7 --version >/dev/null 2>/dev/null
 RESULT=$?
 if [ $RESULT -eq 0 ]; then
-  error "Python2.7 is still available."
+  error "ERROR: Python2.7 is still available."
   exit 1
 fi
 
 # Skip self upgrade due to Python 3 not being available.
 if ! "$LE_AUTO" 2>&1 | grep -q "WARNING: couldn't find Python"; then
-  echo "Python upgrade failure warning not printed!"
+  echo "ERROR: Python upgrade failure warning not printed!"
   exit 1
 fi
 
-# bootstrap, this time installing python3
-"$LE_AUTO" --no-self-upgrade -n > /dev/null 2> /dev/null
+# bootstrap from the old letsencrypt-auto, this time installing python3.4
+"$LE_AUTO_PY_34" --no-self-upgrade -n >/dev/null 2>/dev/null
 
-# ensure python 3 is installed
-python3 --version > /dev/null
+# ensure python 3.4 is installed
+python3.4 --version >/dev/null 2>/dev/null
 RESULT=$?
 if [ $RESULT -ne 0 ]; then
-  error "letsencrypt-auto failed to install Python3 when only Python2.6 is present."
+  echo "ERROR: letsencrypt-auto failed to install Python3.4 using letsencrypt-auto < 0.37.0 when only Python2.6 is present."
   exit 1
 fi
 
-echo "PASSED: Successfully upgraded to Python3 when only Python2.6 is present."
-echo ""
+echo "PASSED: Successfully upgraded to Python3.4 using letsencrypt-auto < 0.37.0 when only Python2.6 is present."
 
-export VENV_PATH=$(mktemp -d)
-"$LE_AUTO" -n --no-bootstrap --no-self-upgrade --version >/dev/null 2>&1
-if [ "$($VENV_PATH/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1)" != 3 ]; then
-  echo "Python 3 wasn't used with --no-bootstrap!"
+# As "certbot-auto" (so without implicit --non-interactive flag set), check that the script
+# refuses to install SCL Python 3.6 when run in a non interactive shell (simulated here
+# using | tee /dev/null) if --non-interactive flag is not provided.
+cp "$LE_AUTO" /tmp/certbot-auto
+# NB: Readline has an issue on all Python versions for CentOS 6, making `certbot --version`
+# output an unprintable ASCII character on a new line at the end.
+# So we take the second last line of the output.
+version=$(/tmp/certbot-auto --version 2>/dev/null | tee /dev/null | tail -2 | head -1)
+
+if [ "$version" != "$INITIAL_CERTBOT_VERSION_PY34" ]; then
+  echo "ERROR: certbot-auto upgraded certbot in a non-interactive shell with --non-interactive flag not set."
   exit 1
 fi
-unset VENV_PATH
+
+echo "PASSED: certbot-auto did not upgrade certbot in a non-interactive shell with --non-interactive flag not set."
+
+if [ -f /opt/rh/rh-python36/enable ]; then
+  echo "ERROR: certbot-auto installed Python3.6 in a non-interactive shell with --non-interactive flag not set."
+  exit 1
+fi
+
+echo "PASSED: certbot-auto did not install Python3.6 in a non-interactive shell with --non-interactive flag not set."
+
+# now bootstrap from current letsencrypt-auto, that will install python3.6 from SCL
+"$LE_AUTO" --no-self-upgrade -n >/dev/null 2>/dev/null
+
+# Following test is exectued in a subshell, to not leak any environment variable
+(
+  # enable SCL rh-python36
+  . scl_source enable rh-python36
+
+  # ensure python 3.6 is installed
+  python3.6 --version >/dev/null 2>/dev/null
+  RESULT=$?
+  if [ $RESULT -ne 0 ]; then
+    echo "ERROR: letsencrypt-auto failed to install Python3.6 using current letsencrypt-auto when only Python2.6/Python3.4 are present."
+    exit 1
+  fi
+
+  echo "PASSED: Successfully upgraded to Python3.6 using current letsencrypt-auto when only Python2.6/Python3.4 are present."
+)
+
+# Following test is exectued in a subshell, to not leak any environment variable
+(
+  export VENV_PATH=$(mktemp -d)
+  "$LE_AUTO" -n --no-bootstrap --no-self-upgrade --version >/dev/null 2>&1
+  if [ "$($VENV_PATH/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1-2)" != "3.6" ]; then
+    echo "ERROR: Python 3.6 wasn't used with --no-bootstrap!"
+    exit 1
+  fi
+)
+
+# Following test is exectued in a subshell, to not leak any environment variable
+(
+  # enable SCL rh-python36
+  . scl_source enable rh-python36
+
+  # ensure everything works fine with certbot-auto bootstrap when python 3.6 is already enabled
+  export VENV_PATH=$(mktemp -d)
+  if ! "$LE_AUTO" --no-self-upgrade -n --version >/dev/null 2>/dev/null; then
+    echo "ERROR: Certbot-auto broke when Python 3.6 SCL is already enabled."
+    exit 1
+  fi
+)
 
 # test using python3
 pytest -v -s certbot/letsencrypt-auto-source/tests

letsencrypt-auto-source/tests/oraclelinux6_tests.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+set -eo pipefail
+# Start by making sure your system is up-to-date:
+yum update -y >/dev/null
+
+LE_AUTO_PY_34="certbot/letsencrypt-auto-source/letsencrypt-auto_py_34"
+LE_AUTO="certbot/letsencrypt-auto-source/letsencrypt-auto"
+
+# Apply installation instructions from official documentation:
+# https://certbot.eff.org/lets-encrypt/centosrhel6-other
+cp "$LE_AUTO" /usr/local/bin/certbot-auto
+chown root /usr/local/bin/certbot-auto
+chmod 0755 /usr/local/bin/certbot-auto
+LE_AUTO=/usr/local/bin/certbot-auto
+
+# Last version of certbot-auto that was bootstraping Python 3.4 for CentOS 6 users
+INITIAL_CERTBOT_VERSION_PY34="certbot 0.38.0"
+
+# Check bootstrap from current certbot-auto will fail, because SCL is not enabled.
+set +o pipefail
+if ! "$LE_AUTO" -n 2>&1 | grep -q "Enable the SCL repository and try running Certbot again."; then
+  echo "ERROR: Bootstrap was not aborted although SCL was not installed!"
+  exit 1
+fi
+set -o pipefail
+
+echo "PASSED: Bootstrap was aborted since SCL was not installed."
+
+# Bootstrap from the old letsencrypt-auto, Python 3.4 will be installed from EPEL.
+"$LE_AUTO_PY_34" --no-self-upgrade -n --install-only >/dev/null 2>/dev/null
+
+# Ensure Python 3.4 is installed
+if ! command -v python3.4 &>/dev/null; then
+  echo "ERROR: old letsencrypt-auto failed to install Python3.4 using letsencrypt-auto < 0.37.0 when only Python2.6 is present."
+  exit 1
+fi
+
+echo "PASSED: Bootstrap from old letsencrypt-auto succeeded and installed Python 3.4"
+
+# Expect certbot-auto to skip rebootstrapping with a warning since SCL is not installed.
+if ! "$LE_AUTO" --non-interactive --version 2>&1 | grep -q "This requires manual user intervention"; then
+  echo "FAILED: Script certbot-auto managed to start Certbot in interactive shell while SCL is not enabled!"
+  exit 1
+fi
+
+echo "PASSED: Script certbot-auto did not rebootstrap."
+
+# NB: Readline has an issue on all Python versions for OL 6, making `certbot --version`
+# output an unprintable ASCII character on a new line at the end.
+# So we take the second last line of the output.
+version=$($LE_AUTO --version 2>/dev/null | tail -2 | head -1)
+
+if [ "$version" != "$INITIAL_CERTBOT_VERSION_PY34" ]; then
+  echo "ERROR: Script certbot-auto upgraded certbot in a non-interactive shell while SCL was not enabled."
+  exit 1
+fi
+
+echo "PASSED: Script certbot-auto did not upgrade certbot but started it successfully while SCL was not enabled."
+
+# Enable SCL
+yum install -y oracle-softwarecollection-release-el6 >/dev/null
+
+# Expect certbot-auto to bootstrap successfully since SCL is available.
+"$LE_AUTO" -n --version &>/dev/null
+
+if [ "$(/opt/eff.org/certbot/venv/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1-2)" != "3.6" ]; then
+  echo "ERROR: Script certbot-auto failed to bootstrap and install Python 3.6 while SCL is available."
+  exit 1
+fi
+
+if ! /opt/eff.org/certbot/venv/bin/certbot --version > /dev/null 2> /dev/null; then
+  echo "ERROR: Script certbot-auto did not install certbot correctly while SCL is enabled."
+  exit 1
+fi
+
+echo "PASSED: Script certbot-auto correctly bootstraped Certbot using rh-python36 when SCL is available."
+
+# Expect certbot-auto will be totally silent now that everything has been correctly boostraped.
+OUTPUT_LEN=$("$LE_AUTO" --install-only --no-self-upgrade --quiet 2>&1 | wc -c)
+if [ "$OUTPUT_LEN" != 0 ]; then
+    echo certbot-auto produced unexpected output!
+    exit 1
+fi
+
+echo "PASSED: Script certbot-auto did not print anything in quiet mode."

setup.py

@@ -59,7 +59,7 @@ install_requires = [
 # However environment markers are supported only with setuptools >= 36.2.
 # So this dependency is not added for old Linux distributions with old setuptools,
 # in order to allow these systems to build certbot from sources.
-pywin32_req = 'pywin32>=224'
+pywin32_req = 'pywin32>=225'
 if StrictVersion(setuptools_version) >= StrictVersion('36.2'):
     install_requires.append(pywin32_req + " ; sys_platform == 'win32'")
 elif 'bdist_wheel' in sys.argv[1:]:

tests/letstest/scripts/test_leauto_upgrades.sh

@@ -117,6 +117,8 @@ if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then
 fi
 
 if [ "$RUN_RHEL6_TESTS" = 1 ]; then
+    # Add the SCL python release to PATH in order to resolve python3 command
+    PATH="/opt/rh/rh-python36/root/usr/bin:$PATH"
     if ! command -v python3; then
         echo "Python3 wasn't properly installed"
         exit 1

tests/letstest/scripts/test_sdists.sh

@@ -1,8 +1,21 @@
 #!/bin/sh -xe
 
 cd letsencrypt
+
+# If we're on a RHEL 6 based system, we can be confident Python is already
+# installed because the package manager is written in Python.
+if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then
+    # RHEL/CentOS 6 will need a special treatment, so we need to detect that environment
+    RUN_RHEL6_TESTS=1
+fi
+
 letsencrypt-auto-source/letsencrypt-auto --install-only -n --debug
 
+if [ "$RUN_RHEL6_TESTS" = 1 ]; then
+  # Enable the SCL Python 3.6 installed by letsencrypt-auto bootstrap
+  PATH="/opt/rh/rh-python36/root/usr/bin:$PATH"
+fi
+
 PLUGINS="certbot-apache certbot-nginx"
 PYTHON_MAJOR_VERSION=$(/opt/eff.org/certbot/venv/bin/python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1)
 TEMP_DIR=$(mktemp -d)

tools/dev_constraints.txt

@@ -4,6 +4,7 @@
 # files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt).
 alabaster==0.7.10
 apipkg==1.4
+appnope==0.1.0
 asn1crypto==0.22.0
 astroid==1.6.5
 attrs==17.3.0
@@ -15,6 +16,7 @@ botocore==1.12.36
 cloudflare==1.5.1
 codecov==2.0.15
 configparser==3.7.4
+contextlib2==0.6.0.post1
 coverage==4.5.4
 decorator==4.1.2
 dns-lexicon==3.2.1
@@ -23,9 +25,11 @@ docutils==0.12
 execnet==1.5.0
 future==0.16.0
 futures==3.1.1
-google-api-python-client==1.5
+filelock==3.0.12
+google-api-python-client==1.5.5
 httplib2==0.10.3
 imagesize==0.7.1
+importlib-metadata==0.23
 ipdb==0.10.2
 ipython==5.5.0
 ipython-genutils==0.2.0
@@ -38,14 +42,16 @@ logger==1.4
 logilab-common==1.4.1
 MarkupSafe==1.0
 mccabe==0.6.1
+more-itertools==5.0.0
 mypy==0.600
 ndg-httpsclient==0.3.2
-oauth2client==2.0.0
+oauth2client==4.0.0
+packaging==19.2
 pathlib2==2.3.0
 pexpect==4.7.0
 pickleshare==0.7.4
 pkginfo==1.4.2
-pluggy==0.5.2
+pluggy==0.13.0
 prompt-toolkit==1.0.15
 ptyprocess==0.6.0
 py==1.8.0
@@ -64,7 +70,7 @@ pytest-sugar==0.9.2
 pytest-rerunfailures==4.2
 python-dateutil==2.6.1
 python-digitalocean==1.11
-pywin32==224
+pywin32==225
 PyYAML==3.13
 repoze.sphinx.autointerface==0.8
 requests-file==1.4.2
@@ -79,13 +85,15 @@ Sphinx==1.7.5
 sphinx-rtd-theme==0.2.4
 sphinxcontrib-websupport==1.0.1
 tldextract==2.2.0
+toml==0.10.0
 tox==3.14.0
 tqdm==4.19.4
 traitlets==4.3.2
 twine==1.11.0
 typed-ast==1.1.0
 typing==3.6.4
-uritemplate==0.6
+uritemplate==3.0.0
 virtualenv==16.6.2
 wcwidth==0.1.7
 wrapt==1.11.1
+zipp==0.6.0

tools/oldest_constraints.txt

@@ -16,6 +16,7 @@ pyOpenSSL==0.13.1
 pyparsing==1.5.6
 pyRFC3339==1.0
 python-augeas==0.5.0
+oauth2client==4.0.0
 six==1.9.0
 # setuptools 0.9.8 is the actual version packaged, but some other dependencies
 # in this file require setuptools>=1.0 and there are no relevant changes for us
@@ -35,11 +36,12 @@ idna==2.0
 pbr==1.8.0
 pytz==2012rc0
 
+# Debian Buster constraints
+google-api-python-client==1.5.5
+
 # Our setup.py constraints
 cloudflare==1.5.1
 cryptography==1.2.3
-google-api-python-client==1.5
-oauth2client==2.0
 parsedatetime==1.3
 pyparsing==1.5.5
 python-digitalocean==1.11

tox.cover.py

@@ -56,9 +56,9 @@ def cover(package):
 
 def main():
     description = """
-This script is used by tox.ini (and thus by Travis CI and AppVeyor) in order
-to generate separate stats for each package. It should be removed once those
-packages are moved to a separate repo.
+This script is used by tox.ini (and thus by Travis CI and Azure Pipelines) in
+order to generate separate stats for each package. It should be removed once
+those packages are moved to a separate repo.
 
 Option -e makes sure we fail fast and don't submit to codecov."""
     parser = argparse.ArgumentParser(description=description)

tox.ini

@@ -206,7 +206,17 @@ passenv = DOCKER_*
 # At the moment, this tests under Python 2.6 only, as only that version is
 # readily available on the CentOS 6 Docker image.
 commands =
-    docker build -f letsencrypt-auto-source/Dockerfile.centos6 -t lea letsencrypt-auto-source
+    docker build -f letsencrypt-auto-source/Dockerfile.redhat6 --build-arg REDHAT_DIST_FLAVOR=centos -t lea letsencrypt-auto-source
+    docker run --rm -t -i lea
+whitelist_externals =
+    docker
+passenv = DOCKER_*
+
+[testenv:le_auto_oraclelinux6]
+# At the moment, this tests under Python 2.6 only, as only that version is
+# readily available on the Oracle Linux 6 Docker image.
+commands =
+    docker build -f letsencrypt-auto-source/Dockerfile.redhat6 --build-arg REDHAT_DIST_FLAVOR=oraclelinux -t lea letsencrypt-auto-source
     docker run --rm -t -i lea
 whitelist_externals =
     docker