Add tests to prevent future regression

Add 0.21.0 changelog

CHANGELOG.md

@@ -2,6 +2,51 @@
 
 Certbot adheres to [Semantic Versioning](http://semver.org/).
 
+## 0.21.0 - 2018-01-17
+
+### Added
+
+* Support for the HTTP-01 challenge type was added to our Apache and Nginx
+  plugins. For those not aware, Let's Encrypt disabled the TLS-SNI-01 challenge
+  type which was what was previously being used by our Apache and Nginx plugins
+  last week due to a security issue. For more information about Let's Encrypt's
+  change, click
+  [here](https://community.letsencrypt.org/t/2018-01-11-update-regarding-acme-tls-sni-and-shared-hosting-infrastructure/50188).
+  Our Apache and Nginx plugins will automatically switch to use HTTP-01 so no
+  changes need to be made to your Certbot configuration, however, you should
+  make sure your server is accessible on port 80 and isn't behind an external
+  proxy doing things like redirecting all traffic from HTTP to HTTPS. HTTP to
+  HTTPS redirects inside Apache and Nginx are fine.
+* IPv6 support was added to the Nginx plugin.
+* Support for automatically creating server blocks based on the default server
+  block was added to the Nginx plugin.
+* The flags --delete-after-revoke and --no-delete-after-revoke were added
+  allowing users to control whether the revoke subcommand also deletes the
+  certificates it is revoking.
+
+### Changed
+
+* We deprecated support for Python 2.6 and Python 3.3 in Certbot and its ACME
+  library. Support for these versions of Python will be removed in the next
+  major release of Certbot. If you are using certbot-auto on a RHEL 6 based
+  system, it will guide you through the process of installing Python 3.
+* We split our implementation of JOSE (Javascript Object Signing and
+  Encryption) out of our ACME library and into a separate package named josepy.
+  This package is available on [PyPI](https://pypi.python.org/pypi/josepy) and
+  on [GitHub](https://github.com/certbot/josepy).
+* We updated the ciphersuites used in Apache to the new [values recommended by
+  Mozilla](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28default.29).
+  The major change here is adding ChaCha20 to the list of supported
+  ciphersuites.
+
+### Fixed
+
+* An issue with our Apache plugin on Gentoo due to differences in their
+  apache2ctl command have been resolved.
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/47?closed=1
+
 ## 0.20.0 - 2017-12-06
 
 ### Added

acme/acme/__init__.py

@@ -13,9 +13,10 @@ supported version: `draft-ietf-acme-01`_.
 import sys
 import warnings
 
-if sys.version_info[:2] == (3, 3):
-    warnings.warn(
-            "Python 3.3 support will be dropped in the next release of "
-            "acme. Please upgrade your Python version.",
-            PendingDeprecationWarning,
-    ) #pragma: no cover
+for (major, minor) in [(2, 6), (3, 3)]:
+    if sys.version_info[:2] == (major, minor):
+        warnings.warn(
+                "Python {0}.{1} support will be dropped in the next release of "
+                "acme. Please upgrade your Python version.".format(major, minor),
+                DeprecationWarning,
+        ) #pragma: no cover

acme/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-apache/certbot_apache/configurator.py

@@ -24,9 +24,10 @@ from certbot_apache import apache_util
 from certbot_apache import augeas_configurator
 from certbot_apache import constants
 from certbot_apache import display_ops
-from certbot_apache import tls_sni_01
+from certbot_apache import http_01
 from certbot_apache import obj
 from certbot_apache import parser
+from certbot_apache import tls_sni_01
 
 from collections import defaultdict
 
@@ -435,12 +436,35 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 return True
         return False
 
-    def _find_best_vhost(self, target_name):
+    def find_best_http_vhost(self, target, filter_defaults, port="80"):
+        """Returns non-HTTPS vhost objects found from the Apache config
+
+        :param str target: Domain name of the desired VirtualHost
+        :param bool filter_defaults: whether _default_ vhosts should be
+            included if it is the best match
+        :param str port: port number the vhost should be listening on
+
+        :returns: VirtualHost object that's the best match for target name
+        :rtype: `obj.VirtualHost` or None
+        """
+        filtered_vhosts = []
+        for vhost in self.vhosts:
+            if any(a.is_wildcard() or a.get_port() == port for a in vhost.addrs) and not vhost.ssl:
+                filtered_vhosts.append(vhost)
+        return self._find_best_vhost(target, filtered_vhosts, filter_defaults)
+
+    def _find_best_vhost(self, target_name, vhosts=None, filter_defaults=True):
         """Finds the best vhost for a target_name.
 
         This does not upgrade a vhost to HTTPS... it only finds the most
         appropriate vhost for the given target_name.
 
+        :param str target_name: domain handled by the desired vhost
+        :param vhosts: vhosts to consider
+        :type vhosts: `collections.Iterable` of :class:`~certbot_apache.obj.VirtualHost`
+        :param bool filter_defaults: whether a vhost with a _default_
+            addr is acceptable
+
         :returns: VHost or None
 
         """
@@ -452,7 +476,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         # Points 1 - Address name with no SSL
         best_candidate = None
         best_points = 0
-        for vhost in self.vhosts:
+
+        if vhosts is None:
+            vhosts = self.vhosts
+
+        for vhost in vhosts:
             if vhost.modmacro is True:
                 continue
             names = vhost.get_names()
@@ -476,8 +504,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         # No winners here... is there only one reasonable vhost?
         if best_candidate is None:
-            # reasonable == Not all _default_ addrs
-            vhosts = self._non_default_vhosts()
+            if filter_defaults:
+                vhosts = self._non_default_vhosts(vhosts)
             # remove mod_macro hosts from reasonable vhosts
             reasonable_vhosts = [vh for vh
                                  in vhosts if vh.modmacro is False]
@@ -486,9 +514,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         return best_candidate
 
-    def _non_default_vhosts(self):
+    def _non_default_vhosts(self, vhosts):
         """Return all non _default_ only vhosts."""
-        return [vh for vh in self.vhosts if not all(
+        return [vh for vh in vhosts if not all(
             addr.get_addr() == "_default_" for addr in vh.addrs
         )]
 
@@ -736,31 +764,43 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         """
 
-        # If nonstandard port, add service definition for matching
-        if port != "443":
+        self.prepare_https_modules(temp)
+        self.ensure_listen(port, https=True)
+
+    def ensure_listen(self, port, https=False):
+        """Make sure that Apache is listening on the port. Checks if the
+        Listen statement for the port already exists, and adds it to the
+        configuration if necessary.
+
+        :param str port: Port number to check and add Listen for if not in
+            place already
+        :param bool https: If the port will be used for HTTPS
+
+        """
+
+        # If HTTPS requested for nonstandard port, add service definition
+        if https and port != "443":
             port_service = "%s %s" % (port, "https")
         else:
             port_service = port
 
-        self.prepare_https_modules(temp)
         # Check for Listen <port>
         # Note: This could be made to also look for ip:443 combo
         listens = [self.parser.get_arg(x).split()[0] for
                    x in self.parser.find_dir("Listen")]
 
-        # In case no Listens are set (which really is a broken apache config)
-        if not listens:
-            listens = ["80"]
-
         # Listen already in place
         if self._has_port_already(listens, port):
             return
 
         listen_dirs = set(listens)
 
+        if not listens:
+            listen_dirs.add(port_service)
+
         for listen in listens:
             # For any listen statement, check if the machine also listens on
-            # Port 443. If not, add such a listen statement.
+            # the given port. If not, add such a listen statement.
             if len(listen.split(":")) == 1:
                 # Its listening to all interfaces
                 if port not in listen_dirs and port_service not in listen_dirs:
@@ -772,11 +812,39 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 if "%s:%s" % (ip, port_service) not in listen_dirs and (
                    "%s:%s" % (ip, port_service) not in listen_dirs):
                     listen_dirs.add("%s:%s" % (ip, port_service))
-        self._add_listens(listen_dirs, listens, port)
+        if https:
+            self._add_listens_https(listen_dirs, listens, port)
+        else:
+            self._add_listens_http(listen_dirs, listens, port)
 
-    def _add_listens(self, listens, listens_orig, port):
-        """Helper method for prepare_server_https to figure out which new
-        listen statements need adding
+    def _add_listens_http(self, listens, listens_orig, port):
+        """Helper method for ensure_listen to figure out which new
+        listen statements need adding for listening HTTP on port
+
+        :param set listens: Set of all needed Listen statements
+        :param list listens_orig: List of existing listen statements
+        :param string port: Port number we're adding
+        """
+
+        new_listens = listens.difference(listens_orig)
+
+        if port in new_listens:
+            # We have wildcard, skip the rest
+            self.parser.add_dir(parser.get_aug_path(self.parser.loc["listen"]),
+                                "Listen", port)
+            self.save_notes += "Added Listen %s directive to %s\n" % (
+                port, self.parser.loc["listen"])
+        else:
+            for listen in new_listens:
+                self.parser.add_dir(parser.get_aug_path(
+                    self.parser.loc["listen"]), "Listen", listen.split(" "))
+                self.save_notes += ("Added Listen %s directive to "
+                                    "%s\n") % (listen,
+                                               self.parser.loc["listen"])
+
+    def _add_listens_https(self, listens, listens_orig, port):
+        """Helper method for ensure_listen to figure out which new
+        listen statements need adding for listening HTTPS on port
 
         :param set listens: Set of all needed Listen statements
         :param list listens_orig: List of existing listen statements
@@ -1201,7 +1269,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                             "insert_cert_file_path")
         self.parser.add_dir(vh_path, "SSLCertificateKeyFile",
                             "insert_key_file_path")
-        self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf)
+        # Only include the TLS configuration if not already included
+        existing_inc = self.parser.find_dir("Include", self.mod_ssl_conf, vh_path)
+        if not existing_inc:
+            self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf)
 
     def _add_servername_alias(self, target_name, vhost):
         vh_path = vhost.path
@@ -1855,7 +1926,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
     ###########################################################################
     def get_chall_pref(self, unused_domain):  # pylint: disable=no-self-use
         """Return list of challenge preferences."""
-        return [challenges.TLSSNI01]
+        return [challenges.TLSSNI01, challenges.HTTP01]
 
     def perform(self, achalls):
         """Perform the configuration related challenge.
@@ -1867,16 +1938,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         """
         self._chall_out.update(achalls)
         responses = [None] * len(achalls)
-        chall_doer = tls_sni_01.ApacheTlsSni01(self)
+        http_doer = http_01.ApacheHttp01(self)
+        sni_doer = tls_sni_01.ApacheTlsSni01(self)
 
         for i, achall in enumerate(achalls):
             # Currently also have chall_doer hold associated index of the
             # challenge. This helps to put all of the responses back together
             # when they are all complete.
-            chall_doer.add_chall(achall, i)
+            if isinstance(achall.chall, challenges.HTTP01):
+                http_doer.add_chall(achall, i)
+            else:  # tls-sni-01
+                sni_doer.add_chall(achall, i)
 
-        sni_response = chall_doer.perform()
-        if sni_response:
+        http_response = http_doer.perform()
+        sni_response = sni_doer.perform()
+        if http_response or sni_response:
             # Must reload in order to activate the challenges.
             # Handled here because we may be able to load up other challenge
             # types
@@ -1886,14 +1962,18 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             # of identifying when the new configuration is being used.
             time.sleep(3)
 
-            # Go through all of the challenges and assign them to the proper
-            # place in the responses return value. All responses must be in the
-            # same order as the original challenges.
-            for i, resp in enumerate(sni_response):
-                responses[chall_doer.indices[i]] = resp
+            self._update_responses(responses, http_response, http_doer)
+            self._update_responses(responses, sni_response, sni_doer)
 
         return responses
 
+    def _update_responses(self, responses, chall_response, chall_doer):
+        # Go through all of the challenges and assign them to the proper
+        # place in the responses return value. All responses must be in the
+        # same order as the original challenges.
+        for i, resp in enumerate(chall_response):
+            responses[chall_doer.indices[i]] = resp
+
     def cleanup(self, achalls):
         """Revert all challenges."""
         self._chall_out.difference_update(achalls)

certbot-apache/certbot_apache/http_01.py

@@ -0,0 +1,150 @@
+"""A class that performs HTTP-01 challenges for Apache"""
+import logging
+import os
+
+from certbot import errors
+
+from certbot.plugins import common
+
+logger = logging.getLogger(__name__)
+
+class ApacheHttp01(common.TLSSNI01):
+    """Class that performs HTTP-01 challenges within the Apache configurator."""
+
+    CONFIG_TEMPLATE22 = """\
+        RewriteEngine on
+        RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [L]
+
+        <Directory {0}>
+            Order Allow,Deny
+            Allow from all
+        </Directory>
+    """
+
+    CONFIG_TEMPLATE24 = """\
+        RewriteEngine on
+        RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [END]
+
+        <Directory {0}>
+            Require all granted
+        </Directory>
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(ApacheHttp01, self).__init__(*args, **kwargs)
+        self.challenge_conf = os.path.join(
+            self.configurator.conf("challenge-location"),
+            "le_http_01_challenge.conf")
+        self.challenge_dir = os.path.join(
+            self.configurator.config.work_dir,
+            "http_challenges")
+        self.moded_vhosts = set()
+
+    def perform(self):
+        """Perform all HTTP-01 challenges."""
+        if not self.achalls:
+            return []
+        # Save any changes to the configuration as a precaution
+        # About to make temporary changes to the config
+        self.configurator.save("Changes before challenge setup", True)
+
+        self.configurator.ensure_listen(str(
+            self.configurator.config.http01_port))
+        self.prepare_http01_modules()
+
+        responses = self._set_up_challenges()
+
+        self._mod_config()
+        # Save reversible changes
+        self.configurator.save("HTTP Challenge", True)
+
+        return responses
+
+    def prepare_http01_modules(self):
+        """Make sure that we have the needed modules available for http01"""
+
+        if self.configurator.conf("handle-modules"):
+            needed_modules = ["rewrite"]
+            if self.configurator.version < (2, 4):
+                needed_modules.append("authz_host")
+            else:
+                needed_modules.append("authz_core")
+            for mod in needed_modules:
+                if mod + "_module" not in self.configurator.parser.modules:
+                    self.configurator.enable_mod(mod, temp=True)
+
+    def _mod_config(self):
+        for chall in self.achalls:
+            vh = self.configurator.find_best_http_vhost(
+                chall.domain, filter_defaults=False,
+                port=str(self.configurator.config.http01_port))
+            if vh:
+                self._set_up_include_directive(vh)
+            else:
+                for vh in self._relevant_vhosts():
+                    self._set_up_include_directive(vh)
+
+        self.configurator.reverter.register_file_creation(
+            True, self.challenge_conf)
+
+        if self.configurator.version < (2, 4):
+            config_template = self.CONFIG_TEMPLATE22
+        else:
+            config_template = self.CONFIG_TEMPLATE24
+
+        config_text = config_template.format(self.challenge_dir)
+
+        logger.debug("writing a config file with text:\n %s", config_text)
+        with open(self.challenge_conf, "w") as new_conf:
+            new_conf.write(config_text)
+
+    def _relevant_vhosts(self):
+        http01_port = str(self.configurator.config.http01_port)
+        relevant_vhosts = []
+        for vhost in self.configurator.vhosts:
+            if any(a.is_wildcard() or a.get_port() == http01_port for a in vhost.addrs):
+                if not vhost.ssl:
+                    relevant_vhosts.append(vhost)
+        if not relevant_vhosts:
+            raise errors.PluginError(
+                "Unable to find a virtual host listening on port {0} which is"
+                " currently needed for Certbot to prove to the CA that you"
+                " control your domain. Please add a virtual host for port"
+                " {0}.".format(http01_port))
+
+        return relevant_vhosts
+
+    def _set_up_challenges(self):
+        if not os.path.isdir(self.challenge_dir):
+            os.makedirs(self.challenge_dir)
+            os.chmod(self.challenge_dir, 0o755)
+
+        responses = []
+        for achall in self.achalls:
+            responses.append(self._set_up_challenge(achall))
+
+        return responses
+
+    def _set_up_challenge(self, achall):
+        response, validation = achall.response_and_validation()
+
+        name = os.path.join(self.challenge_dir, achall.chall.encode("token"))
+
+        self.configurator.reverter.register_file_creation(True, name)
+        with open(name, 'wb') as f:
+            f.write(validation.encode())
+        os.chmod(name, 0o644)
+
+        return response
+
+    def _set_up_include_directive(self, vhost):
+        """Includes override configuration to the beginning of VirtualHost.
+        Note that this include isn't added to Augeas search tree"""
+
+        if vhost not in self.moded_vhosts:
+            logger.debug(
+                "Adding a temporary challenge validation Include for name: %s " +
+                "in: %s", vhost.name, vhost.filep)
+            self.configurator.parser.add_dir_beginning(
+                vhost.path, "Include", self.challenge_conf)
+            self.moded_vhosts.add(vhost)

certbot-apache/certbot_apache/override_debian.py

@@ -140,5 +140,5 @@ class DebianConfigurator(configurator.ApacheConfigurator):
                 "a2dismod are configured correctly for certbot.")
 
         self.reverter.register_undo_command(
-            temp, [self.conf("dismod"), mod_name])
+            temp, [self.conf("dismod"), "-f", mod_name])
         util.run_script([self.conf("enmod"), mod_name])

certbot-apache/certbot_apache/parser.py

@@ -332,6 +332,23 @@ class ApacheParser(object):
         else:
             self.aug.set(aug_conf_path + "/directive[last()]/arg", args)
 
+    def add_dir_beginning(self, aug_conf_path, dirname, args):
+        """Adds the directive to the beginning of defined aug_conf_path.
+
+        :param str aug_conf_path: Augeas configuration path to add directive
+        :param str dirname: Directive to add
+        :param args: Value of the directive. ie. Listen 443, 443 is arg
+        :type args: list or str
+        """
+        first_dir = aug_conf_path + "/directive[1]"
+        self.aug.insert(first_dir, "directive", True)
+        self.aug.set(first_dir, dirname)
+        if isinstance(args, list):
+            for i, value in enumerate(args, 1):
+                self.aug.set(first_dir + "/arg[%d]" % (i), value)
+        else:
+            self.aug.set(first_dir + "/arg", args)
+
     def find_dir(self, directive, arg=None, start=None, exclude=True):
         """Finds directive in the configuration.
 

certbot-apache/certbot_apache/tests/configurator_test.py

@@ -126,7 +126,7 @@ class MultipleVhostsTest(util.ApacheTest):
         names = self.config.get_all_names()
         self.assertEqual(names, set(
             ["certbot.demo", "ocspvhost.com", "encryption-example.demo",
-             "nonsym.link", "vhost.in.rootconf"]
+             "nonsym.link", "vhost.in.rootconf", "www.certbot.demo"]
         ))
 
     @certbot_util.patch_get_utility()
@@ -146,7 +146,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
         names = self.config.get_all_names()
         # Names get filtered, only 5 are returned
-        self.assertEqual(len(names), 7)
+        self.assertEqual(len(names), 8)
         self.assertTrue("zombo.com" in names)
         self.assertTrue("google.com" in names)
         self.assertTrue("certbot.demo" in names)
@@ -260,6 +260,20 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertRaises(
             errors.PluginError, self.config.choose_vhost, "none.com")
 
+    def test_find_best_http_vhost_default(self):
+        vh = obj.VirtualHost(
+            "fp", "ap", set([obj.Addr.fromstring("_default_:80")]), False, True)
+        self.config.vhosts = [vh]
+        self.assertEqual(self.config.find_best_http_vhost("foo.bar", False), vh)
+
+    def test_find_best_http_vhost_port(self):
+        port = "8080"
+        vh = obj.VirtualHost(
+            "fp", "ap", set([obj.Addr.fromstring("*:" + port)]),
+            False, True, "encryption-example.demo")
+        self.config.vhosts.append(vh)
+        self.assertEqual(self.config.find_best_http_vhost("foo.bar", False, port), vh)
+
     def test_findbest_continues_on_short_domain(self):
         # pylint: disable=protected-access
         chosen_vhost = self.config._find_best_vhost("purple.com")
@@ -305,7 +319,8 @@ class MultipleVhostsTest(util.ApacheTest):
 
     def test_non_default_vhosts(self):
         # pylint: disable=protected-access
-        self.assertEqual(len(self.config._non_default_vhosts()), 8)
+        vhosts = self.config._non_default_vhosts(self.config.vhosts)
+        self.assertEqual(len(vhosts), 8)
 
     def test_deploy_cert_enable_new_vhost(self):
         # Create
@@ -320,6 +335,33 @@ class MultipleVhostsTest(util.ApacheTest):
             "example/cert_chain.pem", "example/fullchain.pem")
         self.assertTrue(ssl_vhost.enabled)
 
+    def test_no_duplicate_include(self):
+        def mock_find_dir(directive, argument, _):
+            """Mock method for parser.find_dir"""
+            if directive == "Include" and argument.endswith("options-ssl-apache.conf"):
+                return ["/path/to/whatever"]
+
+        mock_add = mock.MagicMock()
+        self.config.parser.add_dir = mock_add
+        self.config._add_dummy_ssl_directives(self.vh_truth[0])  # pylint: disable=protected-access
+        tried_to_add = False
+        for a in mock_add.call_args_list:
+            if a[0][1] == "Include" and a[0][2] == self.config.mod_ssl_conf:
+                tried_to_add = True
+        # Include should be added, find_dir is not patched, and returns falsy
+        self.assertTrue(tried_to_add)
+
+        self.config.parser.find_dir = mock_find_dir
+        mock_add.reset_mock()
+
+        self.config._add_dummy_ssl_directives(self.vh_truth[0])  # pylint: disable=protected-access
+        tried_to_add = []
+        for a in mock_add.call_args_list:
+            tried_to_add.append(a[0][1] == "Include" and
+                                a[0][2] == self.config.mod_ssl_conf)
+        # Include shouldn't be added, as patched find_dir "finds" existing one
+        self.assertFalse(any(tried_to_add))
+
     def test_deploy_cert(self):
         self.config.parser.modules.add("ssl_module")
         self.config.parser.modules.add("mod_ssl.c")
@@ -424,6 +466,43 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertTrue(self.config.parser.find_dir(
             "NameVirtualHost", "*:80"))
 
+    def test_add_listen_80(self):
+        mock_find = mock.Mock()
+        mock_add_dir = mock.Mock()
+        mock_find.return_value = []
+        self.config.parser.find_dir = mock_find
+        self.config.parser.add_dir = mock_add_dir
+        self.config.ensure_listen("80")
+        self.assertTrue(mock_add_dir.called)
+        self.assertTrue(mock_find.called)
+        self.assertEqual(mock_add_dir.call_args[0][1], "Listen")
+        self.assertEqual(mock_add_dir.call_args[0][2], "80")
+
+    def test_add_listen_80_named(self):
+        mock_find = mock.Mock()
+        mock_find.return_value = ["test1", "test2", "test3"]
+        mock_get = mock.Mock()
+        mock_get.side_effect = ["1.2.3.4:80", "[::1]:80", "1.1.1.1:443"]
+        mock_add_dir = mock.Mock()
+
+        self.config.parser.find_dir = mock_find
+        self.config.parser.get_arg = mock_get
+        self.config.parser.add_dir = mock_add_dir
+
+        self.config.ensure_listen("80")
+        self.assertEqual(mock_add_dir.call_count, 0)
+
+        # Reset return lists and inputs
+        mock_add_dir.reset_mock()
+        mock_get.side_effect = ["1.2.3.4:80", "[::1]:80", "1.1.1.1:443"]
+
+        # Test
+        self.config.ensure_listen("8080")
+        self.assertEqual(mock_add_dir.call_count, 3)
+        self.assertTrue(mock_add_dir.called)
+        self.assertEqual(mock_add_dir.call_args[0][1], "Listen")
+        self.assertEqual(mock_add_dir.call_args[0][2], ['1.2.3.4:8080'])
+
     def test_prepare_server_https(self):
         mock_enable = mock.Mock()
         self.config.enable_mod = mock_enable
@@ -435,7 +514,6 @@ class MultipleVhostsTest(util.ApacheTest):
         # This will test the Add listen
         self.config.parser.find_dir = mock_find
         self.config.parser.add_dir_to_ifmodssl = mock_add_dir
-
         self.config.prepare_server_https("443")
         # Changing the order these modules are enabled breaks the reverter
         self.assertEqual(mock_enable.call_args_list[0][0][0], "socache_shmcb")
@@ -676,23 +754,33 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config._add_name_vhost_if_necessary(self.vh_truth[0])
         self.assertEqual(self.config.add_name_vhost.call_count, 2)
 
+    @mock.patch("certbot_apache.configurator.http_01.ApacheHttp01.perform")
     @mock.patch("certbot_apache.configurator.tls_sni_01.ApacheTlsSni01.perform")
     @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
-    def test_perform(self, mock_restart, mock_perform):
+    def test_perform(self, mock_restart, mock_tls_perform, mock_http_perform):
         # Only tests functionality specific to configurator.perform
         # Note: As more challenges are offered this will have to be expanded
-        account_key, achall1, achall2 = self.get_achalls()
+        account_key, achalls = self.get_key_and_achalls()
 
-        expected = [
-            achall1.response(account_key),
-            achall2.response(account_key),
-        ]
+        all_expected = []
+        http_expected = []
+        tls_expected = []
+        for achall in achalls:
+            response = achall.response(account_key)
+            if isinstance(achall.chall, challenges.HTTP01):
+                http_expected.append(response)
+            else:
+                tls_expected.append(response)
+            all_expected.append(response)
 
-        mock_perform.return_value = expected
-        responses = self.config.perform([achall1, achall2])
+        mock_http_perform.return_value = http_expected
+        mock_tls_perform.return_value = tls_expected
 
-        self.assertEqual(mock_perform.call_count, 1)
-        self.assertEqual(responses, expected)
+        responses = self.config.perform(achalls)
+
+        self.assertEqual(mock_http_perform.call_count, 1)
+        self.assertEqual(mock_tls_perform.call_count, 1)
+        self.assertEqual(responses, all_expected)
 
         self.assertEqual(mock_restart.call_count, 1)
 
@@ -700,29 +788,32 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg")
     def test_cleanup(self, mock_cfg, mock_restart):
         mock_cfg.return_value = ""
-        _, achall1, achall2 = self.get_achalls()
+        _, achalls = self.get_key_and_achalls()
 
-        self.config._chall_out.add(achall1)  # pylint: disable=protected-access
-        self.config._chall_out.add(achall2)  # pylint: disable=protected-access
+        for achall in achalls:
+            self.config._chall_out.add(achall)  # pylint: disable=protected-access
 
-        self.config.cleanup([achall1])
-        self.assertFalse(mock_restart.called)
-
-        self.config.cleanup([achall2])
-        self.assertTrue(mock_restart.called)
+        for i, achall in enumerate(achalls):
+            self.config.cleanup([achall])
+            if i == len(achalls) - 1:
+                self.assertTrue(mock_restart.called)
+            else:
+                self.assertFalse(mock_restart.called)
 
     @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
     @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg")
     def test_cleanup_no_errors(self, mock_cfg, mock_restart):
         mock_cfg.return_value = ""
-        _, achall1, achall2 = self.get_achalls()
+        _, achalls = self.get_key_and_achalls()
+        self.config.http_doer = mock.MagicMock()
 
-        self.config._chall_out.add(achall1)  # pylint: disable=protected-access
+        for achall in achalls:
+            self.config._chall_out.add(achall)  # pylint: disable=protected-access
 
-        self.config.cleanup([achall2])
+        self.config.cleanup([achalls[-1]])
         self.assertFalse(mock_restart.called)
 
-        self.config.cleanup([achall1, achall2])
+        self.config.cleanup(achalls)
         self.assertTrue(mock_restart.called)
 
     @mock.patch("certbot.util.run_script")
@@ -1151,7 +1242,7 @@ class MultipleVhostsTest(util.ApacheTest):
         not_rewriterule = "NotRewriteRule ^ ..."
         self.assertFalse(self.config._sift_rewrite_rule(not_rewriterule))
 
-    def get_achalls(self):
+    def get_key_and_achalls(self):
         """Return testing achallenges."""
         account_key = self.rsa512jwk
         achall1 = achallenges.KeyAuthorizationAnnotatedChallenge(
@@ -1166,8 +1257,12 @@ class MultipleVhostsTest(util.ApacheTest):
                     token=b"uqnaPzxtrndteOqtrXb0Asl5gOJfWAnnx6QJyvcmlDU"),
                 "pending"),
             domain="certbot.demo", account_key=account_key)
+        achall3 = achallenges.KeyAuthorizationAnnotatedChallenge(
+            challb=acme_util.chall_to_challb(
+                challenges.HTTP01(token=(b'x' * 16)), "pending"),
+            domain="example.org", account_key=account_key)
 
-        return account_key, achall1, achall2
+        return account_key, (achall1, achall2, achall3)
 
     def test_make_addrs_sni_ready(self):
         self.config.version = (2, 2)

certbot-apache/certbot_apache/tests/http_01_test.py

@@ -0,0 +1,194 @@
+"""Test for certbot_apache.http_01."""
+import mock
+import os
+import unittest
+
+from acme import challenges
+
+from certbot import achallenges
+from certbot import errors
+
+from certbot.tests import acme_util
+
+from certbot_apache.tests import util
+
+
+NUM_ACHALLS = 3
+
+
+class ApacheHttp01TestMeta(type):
+    """Generates parmeterized tests for testing perform."""
+    def __new__(mcs, name, bases, class_dict):
+
+        def _gen_test(num_achalls, minor_version):
+            def _test(self):
+                achalls = self.achalls[:num_achalls]
+                vhosts = self.vhosts[:num_achalls]
+                self.config.version = (2, minor_version)
+                self.common_perform_test(achalls, vhosts)
+            return _test
+
+        for i in range(1, NUM_ACHALLS + 1):
+            for j in (2, 4):
+                test_name = "test_perform_{0}_{1}".format(i, j)
+                class_dict[test_name] = _gen_test(i, j)
+        return type.__new__(mcs, name, bases, class_dict)
+
+
+class ApacheHttp01Test(util.ApacheTest):
+    """Test for certbot_apache.http_01.ApacheHttp01."""
+
+    __metaclass__ = ApacheHttp01TestMeta
+
+    def setUp(self, *args, **kwargs):
+        super(ApacheHttp01Test, self).setUp(*args, **kwargs)
+
+        self.account_key = self.rsa512jwk
+        self.achalls = []
+        vh_truth = util.get_vh_truth(
+            self.temp_dir, "debian_apache_2_4/multiple_vhosts")
+        # Takes the vhosts for encryption-example.demo, certbot.demo, and
+        # vhost.in.rootconf
+        self.vhosts = [vh_truth[0], vh_truth[3], vh_truth[10]]
+
+        for i in range(NUM_ACHALLS):
+            self.achalls.append(
+                achallenges.KeyAuthorizationAnnotatedChallenge(
+                    challb=acme_util.chall_to_challb(
+                        challenges.HTTP01(token=((chr(ord('a') + i).encode() * 16))),
+                        "pending"),
+                    domain=self.vhosts[i].name, account_key=self.account_key))
+
+        modules = ["rewrite", "authz_core", "authz_host"]
+        for mod in modules:
+            self.config.parser.modules.add("mod_{0}.c".format(mod))
+            self.config.parser.modules.add(mod + "_module")
+
+        from certbot_apache.http_01 import ApacheHttp01
+        self.http = ApacheHttp01(self.config)
+
+    def test_empty_perform(self):
+        self.assertFalse(self.http.perform())
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod")
+    def test_enable_modules_22(self, mock_enmod):
+        self.config.version = (2, 2)
+        self.config.parser.modules.remove("authz_host_module")
+        self.config.parser.modules.remove("mod_authz_host.c")
+
+        enmod_calls = self.common_enable_modules_test(mock_enmod)
+        self.assertEqual(enmod_calls[0][0][0], "authz_host")
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod")
+    def test_enable_modules_24(self, mock_enmod):
+        self.config.parser.modules.remove("authz_core_module")
+        self.config.parser.modules.remove("mod_authz_core.c")
+
+        enmod_calls = self.common_enable_modules_test(mock_enmod)
+        self.assertEqual(enmod_calls[0][0][0], "authz_core")
+
+    def common_enable_modules_test(self, mock_enmod):
+        """Tests enabling mod_rewrite and other modules."""
+        self.config.parser.modules.remove("rewrite_module")
+        self.config.parser.modules.remove("mod_rewrite.c")
+
+        self.http.prepare_http01_modules()
+
+        self.assertTrue(mock_enmod.called)
+        calls = mock_enmod.call_args_list
+        other_calls = []
+        for call in calls:
+            if "rewrite" != call[0][0]:
+                other_calls.append(call)
+
+        # If these lists are equal, we never enabled mod_rewrite
+        self.assertNotEqual(calls, other_calls)
+        return other_calls
+
+    def test_same_vhost(self):
+        vhost = next(v for v in self.config.vhosts if v.name == "certbot.demo")
+        achalls = [
+            achallenges.KeyAuthorizationAnnotatedChallenge(
+                challb=acme_util.chall_to_challb(
+                    challenges.HTTP01(token=((b'a' * 16))),
+                    "pending"),
+                domain=vhost.name, account_key=self.account_key),
+            achallenges.KeyAuthorizationAnnotatedChallenge(
+                challb=acme_util.chall_to_challb(
+                    challenges.HTTP01(token=((b'b' * 16))),
+                    "pending"),
+                domain=next(iter(vhost.aliases)), account_key=self.account_key)
+        ]
+        self.common_perform_test(achalls, [vhost])
+
+    def test_anonymous_vhost(self):
+        vhosts = [v for v in self.config.vhosts if not v.ssl]
+        achalls = [
+            achallenges.KeyAuthorizationAnnotatedChallenge(
+                challb=acme_util.chall_to_challb(
+                    challenges.HTTP01(token=((b'a' * 16))),
+                    "pending"),
+                domain="something.nonexistent", account_key=self.account_key)]
+        self.common_perform_test(achalls, vhosts)
+
+    def test_no_vhost(self):
+        for achall in self.achalls:
+            self.http.add_chall(achall)
+        self.config.config.http01_port = 12345
+        self.assertRaises(errors.PluginError, self.http.perform)
+
+    def common_perform_test(self, achalls, vhosts):
+        """Tests perform with the given achalls."""
+        challenge_dir = self.http.challenge_dir
+        self.assertFalse(os.path.exists(challenge_dir))
+        for achall in achalls:
+            self.http.add_chall(achall)
+
+        expected_response = [
+            achall.response(self.account_key) for achall in achalls]
+        self.assertEqual(self.http.perform(), expected_response)
+
+        self.assertTrue(os.path.isdir(self.http.challenge_dir))
+        self._has_min_permissions(self.http.challenge_dir, 0o755)
+        self._test_challenge_conf()
+
+        for achall in achalls:
+            self._test_challenge_file(achall)
+
+        for vhost in vhosts:
+            if not vhost.ssl:
+                matches = self.config.parser.find_dir("Include",
+                                                      self.http.challenge_conf,
+                                                      vhost.path)
+                self.assertEqual(len(matches), 1)
+
+        self.assertTrue(os.path.exists(challenge_dir))
+
+    def _test_challenge_conf(self):
+        with open(self.http.challenge_conf) as f:
+            conf_contents = f.read()
+
+        self.assertTrue("RewriteEngine on" in conf_contents)
+        self.assertTrue("RewriteRule" in conf_contents)
+        self.assertTrue(self.http.challenge_dir in conf_contents)
+        if self.config.version < (2, 4):
+            self.assertTrue("Allow from all" in conf_contents)
+        else:
+            self.assertTrue("Require all granted" in conf_contents)
+
+    def _test_challenge_file(self, achall):
+        name = os.path.join(self.http.challenge_dir, achall.chall.encode("token"))
+        validation = achall.validation(self.account_key)
+
+        self._has_min_permissions(name, 0o644)
+        with open(name, 'rb') as f:
+            self.assertEqual(f.read(), validation.encode())
+
+    def _has_min_permissions(self, path, min_mode):
+        """Tests the given file has at least the permissions in mode."""
+        st_mode = os.stat(path).st_mode
+        self.assertEqual(st_mode, st_mode | min_mode)
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/parser_test.py

@@ -66,6 +66,23 @@ class BasicParserTest(util.ParserTest):
         for i, match in enumerate(matches):
             self.assertEqual(self.parser.aug.get(match), str(i + 1))
 
+    def test_add_dir_beginning(self):
+        aug_default = "/files" + self.parser.loc["default"]
+        self.parser.add_dir_beginning(aug_default,
+                                      "AddDirectiveBeginning",
+                                      "testBegin")
+
+        self.assertTrue(
+            self.parser.find_dir("AddDirectiveBeginning", "testBegin", aug_default))
+
+        self.assertEqual(
+            self.parser.aug.get(aug_default+"/directive[1]"),
+                                "AddDirectiveBeginning")
+        self.parser.add_dir_beginning(aug_default, "AddList", ["1", "2", "3", "4"])
+        matches = self.parser.find_dir("AddList", None, aug_default)
+        for i, match in enumerate(matches):
+            self.assertEqual(self.parser.aug.get(match), str(i + 1))
+
     def test_empty_arg(self):
         self.assertEquals(None,
                           self.parser.get_arg("/files/whatever/nonexistent"))

certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf

@@ -1,5 +1,6 @@
 <VirtualHost *:80>
 ServerName certbot.demo
+ServerAlias www.certbot.demo
 ServerAdmin webmaster@localhost
 
 DocumentRoot /var/www-certbot-reworld/static/

certbot-apache/certbot_apache/tests/tls_sni_01_test.py

@@ -1,6 +1,6 @@
 """Test for certbot_apache.tls_sni_01."""
-import unittest
 import shutil
+import unittest
 
 import mock
 
@@ -16,8 +16,8 @@ from six.moves import xrange  # pylint: disable=redefined-builtin, import-error
 class TlsSniPerformTest(util.ApacheTest):
     """Test the ApacheTlsSni01 challenge."""
 
-    auth_key = common_test.TLSSNI01Test.auth_key
-    achalls = common_test.TLSSNI01Test.achalls
+    auth_key = common_test.AUTH_KEY
+    achalls = common_test.ACHALLS
 
     def setUp(self):  # pylint: disable=arguments-differ
         super(TlsSniPerformTest, self).setUp()

certbot-apache/certbot_apache/tests/util.py

@@ -103,6 +103,7 @@ def get_apache_configurator(  # pylint: disable=too-many-arguments, too-many-loc
         apache_challenge_location=config_path,
         backup_dir=backups,
         config_dir=config_dir,
+        http01_port=80,
         temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"),
         in_progress_dir=os.path.join(backups, "IN_PROGRESS"),
         work_dir=work_dir)
@@ -169,7 +170,7 @@ def get_vh_truth(temp_dir, config_name):
                 os.path.join(prefix, "certbot.conf"),
                 os.path.join(aug_pre, "certbot.conf/VirtualHost"),
                 set([obj.Addr.fromstring("*:80")]), False, True,
-                "certbot.demo"),
+                "certbot.demo", aliases=["www.certbot.demo"]),
             obj.VirtualHost(
                 os.path.join(prefix, "mod_macro-example.conf"),
                 os.path.join(aug_pre,

certbot-apache/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.20.0"
+LE_AUTO_VERSION="0.21.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -68,10 +68,12 @@ for arg in "$@" ; do
       NO_BOOTSTRAP=1;;
     --help)
       HELP=1;;
-    --noninteractive|--non-interactive|renew)
-      ASSUME_YES=1;;
+    --noninteractive|--non-interactive)
+      NONINTERACTIVE=1;;
     --quiet)
       QUIET=1;;
+    renew)
+      ASSUME_YES=1;;
     --verbose)
       VERBOSE=1;;
     -[!-]*)
@@ -93,7 +95,7 @@ done
 
 if [ $BASENAME = "letsencrypt-auto" ]; then
   # letsencrypt-auto does not respect --help or --yes for backwards compatibility
-  ASSUME_YES=1
+  NONINTERACTIVE=1
   HELP=0
 fi
 
@@ -244,23 +246,42 @@ DeprecationBootstrap() {
   fi
 }
 
-
+MIN_PYTHON_VERSION="2.6"
+MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//')
+# Sets LE_PYTHON to Python version string and PYVER to the first two
+# digits of the python version
 DeterminePythonVersion() {
-  for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
-    # Break (while keeping the LE_PYTHON value) if found.
-    $EXISTS "$LE_PYTHON" > /dev/null && break
-  done
-  if [ "$?" != "0" ]; then
-    error "Cannot find any Pythons; please install one!"
-    exit 1
+  # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
+  #
+  # If no Python is found, PYVER is set to 0.
+  if [ "$USE_PYTHON_3" = 1 ]; then
+    for LE_PYTHON in "$LE_PYTHON" python3; do
+      # Break (while keeping the LE_PYTHON value) if found.
+      $EXISTS "$LE_PYTHON" > /dev/null && break
+    done
+  else
+    for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
+      # Break (while keeping the LE_PYTHON value) if found.
+      $EXISTS "$LE_PYTHON" > /dev/null && break
+    done
+  fi
+  if [ "$?" != "0" ]; then
+    if [ "$1" != "NOCRASH" ]; then
+      error "Cannot find any Pythons; please install one!"
+      exit 1
+    else
+      PYVER=0
+      return 0
+    fi
   fi
-  export LE_PYTHON
 
   PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
-  if [ "$PYVER" -lt 26 ]; then
-    error "You have an ancient version of Python entombed in your operating system..."
-    error "This isn't going to work; you'll need at least version 2.6."
-    exit 1
+  if [ "$PYVER" -lt "$MIN_PYVER" ]; then
+    if [ "$1" != "NOCRASH" ]; then
+      error "You have an ancient version of Python entombed in your operating system..."
+      error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION."
+      exit 1
+    fi
   fi
 }
 
@@ -384,23 +405,19 @@ BootstrapDebCommon() {
   fi
 }
 
-# If new packages are installed by BootstrapRpmCommon below, this version
-# number must be increased.
-BOOTSTRAP_RPM_COMMON_VERSION=1
-
-BootstrapRpmCommon() {
-  # Tested with:
-  #   - Fedora 20, 21, 22, 23 (x64)
-  #   - Centos 7 (x64: on DigitalOcean droplet)
-  #   - CentOS 7 Minimal install in a Hyper-V VM
-  #   - CentOS 6 (EPEL must be installed manually)
+# If new packages are installed by BootstrapRpmCommonBase below, version
+# numbers in rpm_common.sh and rpm_python3.sh must be increased.
 
+# Sets TOOL to the name of the package manager
+# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG.
+# Enables EPEL if applicable and possible.
+InitializeRPMCommonBase() {
   if type dnf 2>/dev/null
   then
-    tool=dnf
+    TOOL=dnf
   elif type yum 2>/dev/null
   then
-    tool=yum
+    TOOL=yum
 
   else
     error "Neither yum nor dnf found. Aborting bootstrap!"
@@ -408,15 +425,15 @@ BootstrapRpmCommon() {
   fi
 
   if [ "$ASSUME_YES" = 1 ]; then
-    yes_flag="-y"
+    YES_FLAG="-y"
   fi
   if [ "$QUIET" = 1 ]; then
     QUIET_FLAG='--quiet'
   fi
 
-  if ! $tool list *virtualenv >/dev/null 2>&1; then
+  if ! $TOOL list *virtualenv >/dev/null 2>&1; then
     echo "To use Certbot, packages from the EPEL repository need to be installed."
-    if ! $tool list epel-release >/dev/null 2>&1; then
+    if ! $TOOL list epel-release >/dev/null 2>&1; then
       error "Enable the EPEL repository and try running Certbot again."
       exit 1
     fi
@@ -425,14 +442,20 @@ BootstrapRpmCommon() {
       sleep 1s
       /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
       sleep 1s
-      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..."
+      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..."
       sleep 1s
     fi
-    if ! $tool install $yes_flag $QUIET_FLAG epel-release; then
+    if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then
       error "Could not enable EPEL. Aborting bootstrap!"
       exit 1
     fi
   fi
+}
+
+BootstrapRpmCommonBase() {
+  # Arguments: whitespace-delimited python packages to install
+
+  InitializeRPMCommonBase # This call is superfluous in practice
 
   pkgs="
     gcc
@@ -444,10 +467,39 @@ BootstrapRpmCommon() {
     ca-certificates
   "
 
-  # Most RPM distros use the "python" or "python-" naming convention.  Let's try that first.
-  if $tool list python >/dev/null 2>&1; then
+  # Add the python packages
+  pkgs="$pkgs
+    $1
+  "
+
+  if $TOOL list installed "httpd" >/dev/null 2>&1; then
     pkgs="$pkgs
-      python
+      mod_ssl
+    "
+  fi
+
+  if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then
+    error "Could not install OS dependencies. Aborting bootstrap!"
+    exit 1
+  fi
+}
+
+# If new packages are installed by BootstrapRpmCommon below, this version
+# number must be increased.
+BOOTSTRAP_RPM_COMMON_VERSION=1
+
+BootstrapRpmCommon() {
+  # Tested with:
+  #   - Fedora 20, 21, 22, 23 (x64)
+  #   - Centos 7 (x64: on DigitalOcean droplet)
+  #   - CentOS 7 Minimal install in a Hyper-V VM
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  # Most RPM distros use the "python" or "python-" naming convention.  Let's try that first.
+  if $TOOL list python >/dev/null 2>&1; then
+    python_pkgs="$python
       python-devel
       python-virtualenv
       python-tools
@@ -455,9 +507,8 @@ BootstrapRpmCommon() {
     "
   # Fedora 26 starts to use the prefix python2 for python2 based packages.
   # this elseif is theoretically for any Fedora over version 26:
-  elif $tool list python2 >/dev/null 2>&1; then
-    pkgs="$pkgs
-      python2
+  elif $TOOL list python2 >/dev/null 2>&1; then
+    python_pkgs="$python2
       python2-libs
       python2-setuptools
       python2-devel
@@ -468,8 +519,7 @@ BootstrapRpmCommon() {
   # Some distros and older versions of current distros use a "python27"
   # instead of the "python" or "python-" naming convention.
   else
-    pkgs="$pkgs
-      python27
+    python_pkgs="$python27
       python27-devel
       python27-virtualenv
       python27-tools
@@ -477,16 +527,31 @@ BootstrapRpmCommon() {
     "
   fi
 
-  if $tool list installed "httpd" >/dev/null 2>&1; then
-    pkgs="$pkgs
-      mod_ssl
-    "
-  fi
+  BootstrapRpmCommonBase "$python_pkgs"
+}
 
-  if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then
-    error "Could not install OS dependencies. Aborting bootstrap!"
+# If new packages are installed by BootstrapRpmPython3 below, this version
+# number must be increased.
+BOOTSTRAP_RPM_PYTHON3_VERSION=1
+
+BootstrapRpmPython3() {
+  # Tested with:
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  # EPEL uses python34
+  if $TOOL list python34 >/dev/null 2>&1; then
+    python_pkgs="python34
+      python34-devel
+      python34-tools
+    "
+  else
+    error "No supported Python package available to install. Aborting bootstrap!"
     exit 1
   fi
+
+  BootstrapRpmCommonBase "$python_pkgs"
 }
 
 # If new packages are installed by BootstrapSuseCommon below, this version
@@ -715,11 +780,27 @@ elif [ -f /etc/mageia-release ]; then
   }
   BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION"
 elif [ -f /etc/redhat-release ]; then
-  Bootstrap() {
-    BootstrapMessage "RedHat-based OSes"
-    BootstrapRpmCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+  # Run DeterminePythonVersion to decide on the basis of available Python versions
+  # whether to use 2.x or 3.x on RedHat-like systems.
+  # Then, revert LE_PYTHON to its previous state.
+  prev_le_python="$LE_PYTHON"
+  unset LE_PYTHON
+  DeterminePythonVersion "NOCRASH"
+  if [ "$PYVER" -eq 26 ]; then
+    Bootstrap() {
+      BootstrapMessage "RedHat-based OSes that will use Python3"
+      BootstrapRpmPython3
+    }
+    USE_PYTHON_3=1
+    BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+  else
+    Bootstrap() {
+      BootstrapMessage "RedHat-based OSes"
+      BootstrapRpmCommon
+    }
+    BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+  fi
+  LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
   Bootstrap() {
     BootstrapMessage "openSUSE-based OSes"
@@ -816,7 +897,11 @@ TempDir() {
   mktemp -d 2>/dev/null || mktemp -d -t 'le'  # Linux || macOS
 }
 
-
+# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise,
+# returns a non-zero number.
+OldVenvExists() {
+    [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ]
+}
 
 if [ "$1" = "--le-auto-phase2" ]; then
   # Phase 2: Create venv, install LE, and run.
@@ -824,14 +909,26 @@ if [ "$1" = "--le-auto-phase2" ]; then
   shift 1  # the --le-auto-phase2 arg
   SetPrevBootstrapVersion
 
+  if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then
+    unset LE_PYTHON
+  fi
+
   INSTALLED_VERSION="none"
-  if [ -d "$VENV_PATH" ]; then
+  if [ -d "$VENV_PATH" ] || OldVenvExists; then
     # If the selected Bootstrap function isn't a noop and it differs from the
     # previously used version
     if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then
       # if non-interactive mode or stdin and stdout are connected to a terminal
       if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
-        rm -rf "$VENV_PATH"
+        if [ -d "$VENV_PATH" ]; then
+          rm -rf "$VENV_PATH"
+        fi
+        # In the case the old venv was just a symlink to the new one,
+        # OldVenvExists is now false because we deleted the venv at VENV_PATH.
+        if OldVenvExists; then
+          rm -rf "$OLD_VENV_PATH"
+          ln -s "$VENV_PATH" "$OLD_VENV_PATH"
+        fi
         RerunWithArgs "$@"
       else
         error "Skipping upgrade because new OS dependencies may need to be installed."
@@ -841,6 +938,10 @@ if [ "$1" = "--le-auto-phase2" ]; then
         error "install any required packages."
         # Set INSTALLED_VERSION to be the same so we don't update the venv
         INSTALLED_VERSION="$LE_AUTO_VERSION"
+        # Continue to use OLD_VENV_PATH if the new venv doesn't exist
+        if [ ! -d "$VENV_PATH" ]; then
+          VENV_BIN="$OLD_VENV_PATH/bin"
+        fi
       fi
     elif [ -f "$VENV_BIN/letsencrypt" ]; then
       # --version output ran through grep due to python-cryptography DeprecationWarnings
@@ -858,10 +959,18 @@ if [ "$1" = "--le-auto-phase2" ]; then
     say "Creating virtual environment..."
     DeterminePythonVersion
     rm -rf "$VENV_PATH"
-    if [ "$VERBOSE" = 1 ]; then
-      virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
+    if [ "$PYVER" -le 27 ]; then
+      if [ "$VERBOSE" = 1 ]; then
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
+      else
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
+      fi
     else
-      virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
+      if [ "$VERBOSE" = 1 ]; then
+        "$LE_PYTHON" -m venv "$VENV_PATH"
+      else
+        "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null
+      fi
     fi
 
     if [ -n "$BOOTSTRAP_VERSION" ]; then
@@ -983,9 +1092,16 @@ idna==2.5 \
 ipaddress==1.0.16 \
     --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \
     --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0
+josepy==1.0.1 \
+    --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \
+    --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc
 linecache2==1.0.0 \
     --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \
     --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c
+# Using an older version of mock here prevents regressions of #5276.
+mock==1.3.0 \
+    --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \
+    --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6
 ordereddict==1.1 \
     --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f
 packaging==16.8 \
@@ -1062,10 +1178,6 @@ zope.interface==4.1.3 \
     --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \
     --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \
     --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392
-# Using an older version of mock here prevents regressions of #5276.
-mock==1.3.0 \
-    --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \
-    --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6
 
 # Contains the requirements for the letsencrypt package.
 #
@@ -1078,18 +1190,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.20.0 \
-    --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \
-    --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88
-acme==0.20.0 \
-    --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \
-    --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5
-certbot-apache==0.20.0 \
-    --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \
-    --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f
-certbot-nginx==0.20.0 \
-    --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \
-    --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae
+certbot==0.21.0 \
+    --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \
+    --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857
+acme==0.21.0 \
+    --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \
+    --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d
+certbot-apache==0.21.0 \
+    --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \
+    --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b
+certbot-nginx==0.21.0 \
+    --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \
+    --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
@@ -1319,9 +1431,10 @@ else
   # upgrading. Phase 1 checks the version of the latest release of
   # certbot-auto (which is always the same as that of the certbot
   # package). Phase 2 checks the version of the locally installed certbot.
+  export PHASE_1_VERSION="$LE_AUTO_VERSION"
 
   if [ ! -f "$VENV_BIN/letsencrypt" ]; then
-    if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then
+    if ! OldVenvExists; then
       if [ "$HELP" = 1 ]; then
         echo "$USAGE"
         exit 0
@@ -1353,17 +1466,22 @@ On failure, return non-zero.
 
 """
 
-from __future__ import print_function
+from __future__ import print_function, unicode_literals
 
 from distutils.version import LooseVersion
 from json import loads
 from os import devnull, environ
 from os.path import dirname, join
 import re
+import ssl
 from subprocess import check_call, CalledProcessError
 from sys import argv, exit
-from urllib2 import build_opener, HTTPHandler, HTTPSHandler
-from urllib2 import HTTPError, URLError
+try:
+    from urllib2 import build_opener, HTTPHandler, HTTPSHandler
+    from urllib2 import HTTPError, URLError
+except ImportError:
+    from urllib.request import build_opener, HTTPHandler, HTTPSHandler
+    from urllib.error import HTTPError, URLError
 
 PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY-----
 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq
@@ -1385,8 +1503,11 @@ class HttpsGetter(object):
     def __init__(self):
         """Build an HTTPS opener."""
         # Based on pip 1.4.1's URLOpener
-        # This verifies certs on only Python >=2.7.9.
-        self._opener = build_opener(HTTPSHandler())
+        # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set.
+        if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'):
+            self._opener = build_opener(HTTPSHandler(context=cert_none_context()))
+        else:
+            self._opener = build_opener(HTTPSHandler())
         # Strip out HTTPHandler to prevent MITM spoof:
         for handler in self._opener.handlers:
             if isinstance(handler, HTTPHandler):
@@ -1408,7 +1529,7 @@ class HttpsGetter(object):
 
 def write(contents, dir, filename):
     """Write something to a file in a certain directory."""
-    with open(join(dir, filename), 'w') as file:
+    with open(join(dir, filename), 'wb') as file:
         file.write(contents)
 
 
@@ -1416,13 +1537,13 @@ def latest_stable_version(get):
     """Return the latest stable release of letsencrypt."""
     metadata = loads(get(
         environ.get('LE_AUTO_JSON_URL',
-                    'https://pypi.python.org/pypi/certbot/json')))
+                    'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8'))
     # metadata['info']['version'] actually returns the latest of any kind of
     # release release, contrary to https://wiki.python.org/moin/PyPIJSON.
     # The regex is a sufficient regex for picking out prereleases for most
     # packages, LE included.
     return str(max(LooseVersion(r) for r
-                   in metadata['releases'].iterkeys()
+                   in metadata['releases'].keys()
                    if re.match('^[0-9.]+$', r)))
 
 
@@ -1439,7 +1560,7 @@ def verified_new_le_auto(get, tag, temp_dir):
         'letsencrypt-auto-source/') % tag
     write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto')
     write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig')
-    write(PUBLIC_KEY, temp_dir, 'public_key.pem')
+    write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem')
     try:
         with open(devnull, 'w') as dev_null:
             check_call(['openssl', 'dgst', '-sha256', '-verify',
@@ -1454,6 +1575,14 @@ def verified_new_le_auto(get, tag, temp_dir):
                             "certbot-auto.", exc)
 
 
+def cert_none_context():
+    """Create a SSLContext object to not check hostname."""
+    # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this.
+    context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+    context.verify_mode = ssl.CERT_NONE
+    return context
+
+
 def main():
     get = HttpsGetter().get
     flag = argv[1]
@@ -1475,8 +1604,10 @@ if __name__ == '__main__':
 
 UNLIKELY_EOF
     # ---------------------------------------------------------------------------
-    DeterminePythonVersion
-    if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then
+    DeterminePythonVersion "NOCRASH"
+    if [ "$PYVER" -lt "$MIN_PYVER" ]; then
+      error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates."
+    elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then
       error "WARNING: unable to check for updates."
     elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then
       say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..."

certbot-compatibility-test/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.22.0.dev0'
 
 install_requires = [
     'certbot',

certbot-dns-cloudflare/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-cloudxns/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-digitalocean/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-dnsimple/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-dnsmadeeasy/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-google/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-luadns/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-nsone/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-rfc2136/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-route53/setup.py

@@ -3,7 +3,7 @@ import sys
 from distutils.core import setup
 from setuptools import find_packages
 
-version = '0.21.0.dev0'
+version = '0.22.0.dev0'
 
 install_requires = [
     'acme=={0}'.format(version),

certbot-nginx/certbot_nginx/configurator.py

@@ -26,6 +26,7 @@ from certbot_nginx import constants
 from certbot_nginx import nginxparser
 from certbot_nginx import parser
 from certbot_nginx import tls_sni_01
+from certbot_nginx import http_01
 
 
 logger = logging.getLogger(__name__)
@@ -208,7 +209,8 @@ class NginxConfigurator(common.Installer):
 
         :param str target_name: domain name
         :param bool create_if_no_match: If we should create a new vhost from default
-            when there is no match found
+            when there is no match found. If we can't choose a default, raise a
+            MisconfigurationError.
 
         :returns: ssl vhost associated with name
         :rtype: :class:`~certbot_nginx.obj.VirtualHost`
@@ -259,9 +261,9 @@ class NginxConfigurator(common.Installer):
                     ipv6only_present = True
         return (ipv6_active, ipv6only_present)
 
-    def _vhost_from_duplicated_default(self, domain):
+    def _vhost_from_duplicated_default(self, domain, port=None):
         if self.new_vhost is None:
-            default_vhost = self._get_default_vhost()
+            default_vhost = self._get_default_vhost(port)
             self.new_vhost = self.parser.duplicate_vhost(default_vhost, delete_default=True)
             self.new_vhost.names = set()
 
@@ -276,15 +278,16 @@ class NginxConfigurator(common.Installer):
             name_block[0].append(name)
         self.parser.add_server_directives(vhost, name_block, replace=True)
 
-    def _get_default_vhost(self):
+    def _get_default_vhost(self, port):
         vhost_list = self.parser.get_vhosts()
         # if one has default_server set, return that one
         default_vhosts = []
         for vhost in vhost_list:
             for addr in vhost.addrs:
                 if addr.default:
-                    default_vhosts.append(vhost)
-                    break
+                    if port is None or self._port_matches(port, addr.get_port()):
+                        default_vhosts.append(vhost)
+                        break
 
         if len(default_vhosts) == 1:
             return default_vhosts[0]
@@ -366,7 +369,7 @@ class NginxConfigurator(common.Installer):
         return sorted(matches, key=lambda x: x['rank'])
 
 
-    def choose_redirect_vhost(self, target_name, port):
+    def choose_redirect_vhost(self, target_name, port, create_if_no_match=False):
         """Chooses a single virtual host for redirect enhancement.
 
         Chooses the vhost most closely matching target_name that is
@@ -380,12 +383,27 @@ class NginxConfigurator(common.Installer):
 
         :param str target_name: domain name
         :param str port: port number
+        :param bool create_if_no_match: If we should create a new vhost from default
+            when there is no match found. If we can't choose a default, raise a
+            MisconfigurationError.
+
         :returns: vhost associated with name
         :rtype: :class:`~certbot_nginx.obj.VirtualHost`
 
         """
         matches = self._get_redirect_ranked_matches(target_name, port)
-        return self._select_best_name_match(matches)
+        vhost = self._select_best_name_match(matches)
+        if not vhost and create_if_no_match:
+            vhost = self._vhost_from_duplicated_default(target_name, port=port)
+        return vhost
+
+    def _port_matches(self, test_port, matching_port):
+        # test_port is a number, matching is a number or "" or None
+        if matching_port == "" or matching_port is None:
+            # if no port is specified, Nginx defaults to listening on port 80.
+            return test_port == self.DEFAULT_LISTEN_PORT
+        else:
+            return test_port == matching_port
 
     def _get_redirect_ranked_matches(self, target_name, port):
         """Gets a ranked list of plaintextish port-listening vhosts matching target_name
@@ -394,20 +412,13 @@ class NginxConfigurator(common.Installer):
         Rank by how well these match target_name.
 
         :param str target_name: The name to match
-        :param str port: port number
+        :param str port: port number as a string
         :returns: list of dicts containing the vhost, the matching name, and
             the numerical rank
         :rtype: list
 
         """
         all_vhosts = self.parser.get_vhosts()
-        def _port_matches(test_port, matching_port):
-            # test_port is a number, matching is a number or "" or None
-            if matching_port == "" or matching_port is None:
-                # if no port is specified, Nginx defaults to listening on port 80.
-                return test_port == self.DEFAULT_LISTEN_PORT
-            else:
-                return test_port == matching_port
 
         def _vhost_matches(vhost, port):
             found_matching_port = False
@@ -417,7 +428,7 @@ class NginxConfigurator(common.Installer):
                 found_matching_port = (port == self.DEFAULT_LISTEN_PORT)
             else:
                 for addr in vhost.addrs:
-                    if _port_matches(port, addr.get_port()) and addr.ssl == False:
+                    if self._port_matches(port, addr.get_port()) and addr.ssl == False:
                         found_matching_port = True
 
             if found_matching_port:
@@ -840,7 +851,7 @@ class NginxConfigurator(common.Installer):
     ###########################################################################
     def get_chall_pref(self, unused_domain):  # pylint: disable=no-self-use
         """Return list of challenge preferences."""
-        return [challenges.TLSSNI01]
+        return [challenges.TLSSNI01, challenges.HTTP01]
 
     # Entry point in main.py for performing challenges
     def perform(self, achalls):
@@ -853,15 +864,20 @@ class NginxConfigurator(common.Installer):
         """
         self._chall_out += len(achalls)
         responses = [None] * len(achalls)
-        chall_doer = tls_sni_01.NginxTlsSni01(self)
+        sni_doer = tls_sni_01.NginxTlsSni01(self)
+        http_doer = http_01.NginxHttp01(self)
 
         for i, achall in enumerate(achalls):
             # Currently also have chall_doer hold associated index of the
             # challenge. This helps to put all of the responses back together
             # when they are all complete.
-            chall_doer.add_chall(achall, i)
+            if isinstance(achall.chall, challenges.HTTP01):
+                http_doer.add_chall(achall, i)
+            else:  # tls-sni-01
+                sni_doer.add_chall(achall, i)
 
-        sni_response = chall_doer.perform()
+        sni_response = sni_doer.perform()
+        http_response = http_doer.perform()
         # Must restart in order to activate the challenges.
         # Handled here because we may be able to load up other challenge types
         self.restart()
@@ -869,8 +885,9 @@ class NginxConfigurator(common.Installer):
         # Go through all of the challenges and assign them to the proper place
         # in the responses return value. All responses must be in the same order
         # as the original challenges.
-        for i, resp in enumerate(sni_response):
-            responses[chall_doer.indices[i]] = resp
+        for chall_response, chall_doer in ((sni_response, sni_doer), (http_response, http_doer)):
+            for i, resp in enumerate(chall_response):
+                responses[chall_doer.indices[i]] = resp
 
         return responses
 

certbot-nginx/certbot_nginx/http_01.py

@@ -0,0 +1,203 @@
+"""A class that performs HTTP-01 challenges for Nginx"""
+
+import logging
+import os
+
+from acme import challenges
+
+from certbot import errors
+from certbot.plugins import common
+
+from certbot_nginx import obj
+from certbot_nginx import nginxparser
+
+
+logger = logging.getLogger(__name__)
+
+
+class NginxHttp01(common.ChallengePerformer):
+    """HTTP-01 authenticator for Nginx
+
+    :ivar configurator: NginxConfigurator object
+    :type configurator: :class:`~nginx.configurator.NginxConfigurator`
+
+    :ivar list achalls: Annotated
+        class:`~certbot.achallenges.KeyAuthorizationAnnotatedChallenge`
+        challenges
+
+    :param list indices: Meant to hold indices of challenges in a
+        larger array. NginxHttp01 is capable of solving many challenges
+        at once which causes an indexing issue within NginxConfigurator
+        who must return all responses in order.  Imagine NginxConfigurator
+        maintaining state about where all of the http-01 Challenges,
+        TLS-SNI-01 Challenges belong in the response array.  This is an
+        optional utility.
+
+    """
+
+    def __init__(self, configurator):
+        super(NginxHttp01, self).__init__(configurator)
+        self.challenge_conf = os.path.join(
+            configurator.config.config_dir, "le_http_01_cert_challenge.conf")
+        self._ipv6 = None
+        self._ipv6only = None
+
+    def perform(self):
+        """Perform a challenge on Nginx.
+
+        :returns: list of :class:`certbot.acme.challenges.HTTP01Response`
+        :rtype: list
+
+        """
+        if not self.achalls:
+            return []
+
+        responses = [x.response(x.account_key) for x in self.achalls]
+
+        # Set up the configuration
+        self._mod_config()
+
+        # Save reversible changes
+        self.configurator.save("HTTP Challenge", True)
+
+        return responses
+
+    def _mod_config(self):
+        """Modifies Nginx config to include server_names_hash_bucket_size directive
+           and server challenge blocks.
+
+        :raises .MisconfigurationError:
+            Unable to find a suitable HTTP block in which to include
+            authenticator hosts.
+        """
+        included = False
+        include_directive = ['\n', 'include', ' ', self.challenge_conf]
+        root = self.configurator.parser.config_root
+
+        bucket_directive = ['\n', 'server_names_hash_bucket_size', ' ', '128']
+
+        main = self.configurator.parser.parsed[root]
+        for line in main:
+            if line[0] == ['http']:
+                body = line[1]
+                found_bucket = False
+                posn = 0
+                for inner_line in body:
+                    if inner_line[0] == bucket_directive[1]:
+                        if int(inner_line[1]) < int(bucket_directive[3]):
+                            body[posn] = bucket_directive
+                        found_bucket = True
+                    posn += 1
+                if not found_bucket:
+                    body.insert(0, bucket_directive)
+                if include_directive not in body:
+                    body.insert(0, include_directive)
+                included = True
+                break
+        if not included:
+            raise errors.MisconfigurationError(
+                'Certbot could not find a block to include '
+                'challenges in %s.' % root)
+        config = [self._make_or_mod_server_block(achall) for achall in self.achalls]
+        config = [x for x in config if x is not None]
+        config = nginxparser.UnspacedList(config)
+
+        self.configurator.reverter.register_file_creation(
+            True, self.challenge_conf)
+
+        with open(self.challenge_conf, "w") as new_conf:
+            nginxparser.dump(config, new_conf)
+
+    def _default_listen_addresses(self):
+        """Finds addresses for a challenge block to listen on.
+        :returns: list of :class:`certbot_nginx.obj.Addr` to apply
+        :rtype: list
+        """
+        addresses = []
+        default_addr = "%s" % self.configurator.config.http01_port
+        ipv6_addr = "[::]:{0}".format(
+            self.configurator.config.http01_port)
+        port = self.configurator.config.http01_port
+
+        if self._ipv6 is None or self._ipv6only is None:
+            self._ipv6, self._ipv6only = self.configurator.ipv6_info(port)
+        ipv6, ipv6only = self._ipv6, self._ipv6only
+
+        if ipv6:
+            # If IPv6 is active in Nginx configuration
+            if not ipv6only:
+                # If ipv6only=on is not already present in the config
+                ipv6_addr = ipv6_addr + " ipv6only=on"
+            addresses = [obj.Addr.fromstring(default_addr),
+                         obj.Addr.fromstring(ipv6_addr)]
+            logger.info(("Using default addresses %s and %s for authentication."),
+                        default_addr,
+                        ipv6_addr)
+        else:
+            addresses = [obj.Addr.fromstring(default_addr)]
+            logger.info("Using default address %s for authentication.",
+                        default_addr)
+        return addresses
+
+    def _get_validation_path(self, achall):
+        return os.sep + os.path.join(challenges.HTTP01.URI_ROOT_PATH, achall.chall.encode("token"))
+
+    def _make_server_block(self, achall):
+        """Creates a server block for a challenge.
+        :param achall: Annotated HTTP-01 challenge
+        :type achall:
+            :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge`
+        :param list addrs: addresses of challenged domain
+            :class:`list` of type :class:`~nginx.obj.Addr`
+        :returns: server block for the challenge host
+        :rtype: list
+        """
+        addrs = self._default_listen_addresses()
+        block = [['listen', ' ', addr.to_string(include_default=False)] for addr in addrs]
+
+        # Ensure we 404 on any other request by setting a root
+        document_root = os.path.join(
+            self.configurator.config.work_dir, "http_01_nonexistent")
+
+        validation = achall.validation(achall.account_key)
+        validation_path = self._get_validation_path(achall)
+
+        block.extend([['server_name', ' ', achall.domain],
+                      ['root', ' ', document_root],
+                      [['location', ' ', '=', ' ', validation_path],
+                        [['default_type', ' ', 'text/plain'],
+                         ['return', ' ', '200', ' ', validation]]]])
+        # TODO: do we want to return something else if they otherwise access this block?
+        return [['server'], block]
+
+    def _make_or_mod_server_block(self, achall):
+        """Modifies a server block to respond to a challenge.
+
+        :param achall: Annotated HTTP-01 challenge
+        :type achall:
+            :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge`
+
+        """
+        try:
+            vhost = self.configurator.choose_redirect_vhost(achall.domain,
+                '%i' % self.configurator.config.http01_port, create_if_no_match=True)
+        except errors.MisconfigurationError:
+            # Couldn't find either a matching name+port server block
+            # or a port+default_server block, so create a dummy block
+            return self._make_server_block(achall)
+
+        # Modify existing server block
+        validation = achall.validation(achall.account_key)
+        validation_path = self._get_validation_path(achall)
+
+        location_directive = [[['location', ' ', '=', ' ', validation_path],
+                               [['default_type', ' ', 'text/plain'],
+                                ['return', ' ', '200', ' ', validation]]]]
+
+        self.configurator.parser.add_server_directives(vhost,
+            location_directive, replace=False)
+
+        rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)',
+                                ' ', '$1', ' ', 'break']]
+        self.configurator.parser.add_server_directives(vhost,
+            rewrite_directive, replace=False, insert_at_top=True)

certbot-nginx/certbot_nginx/parser.py

@@ -276,7 +276,7 @@ class NginxParser(object):
 
         return False
 
-    def add_server_directives(self, vhost, directives, replace):
+    def add_server_directives(self, vhost, directives, replace, insert_at_top=False):
         """Add or replace directives in the server block identified by vhost.
 
         This method modifies vhost to be fully consistent with the new directives.
@@ -293,10 +293,12 @@ class NginxParser(object):
             whose information we use to match on
         :param list directives: The directives to add
         :param bool replace: Whether to only replace existing directives
+        :param bool insert_at_top: True if the directives need to be inserted at the top
+            of the server block instead of the bottom
 
         """
         self._modify_server_directives(vhost,
-            functools.partial(_add_directives, directives, replace))
+            functools.partial(_add_directives, directives, replace, insert_at_top))
 
     def remove_server_directives(self, vhost, directive_name, match_func=None):
         """Remove all directives of type directive_name.
@@ -521,10 +523,10 @@ def _is_ssl_on_directive(entry):
             len(entry) == 2 and entry[0] == 'ssl' and
             entry[1] == 'on')
 
-def _add_directives(directives, replace, block):
+def _add_directives(directives, replace, insert_at_top, block):
     """Adds or replaces directives in a config block.
 
-    When replace=False, it's an error to try and add a directive that already
+    When replace=False, it's an error to try and add a nonrepeatable directive that already
     exists in the config block with a conflicting value.
 
     When replace=True and a directive with the same name already exists in the
@@ -535,17 +537,18 @@ def _add_directives(directives, replace, block):
 
     :param list directives: The new directives.
     :param bool replace: Described above.
+    :param bool insert_at_top: Described above.
     :param list block: The block to replace in
 
     """
     for directive in directives:
-        _add_directive(block, directive, replace)
+        _add_directive(block, directive, replace, insert_at_top)
     if block and '\n' not in block[-1]:  # could be "   \n  " or ["\n"] !
         block.append(nginxparser.UnspacedList('\n'))
 
 
 INCLUDE = 'include'
-REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE])
+REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'location', 'rewrite'])
 COMMENT = ' managed by Certbot'
 COMMENT_BLOCK = [' ', '#', COMMENT]
 
@@ -597,7 +600,7 @@ def _find_location(block, directive_name, match_func=None):
     return next((index for index, line in enumerate(block) \
         if line and line[0] == directive_name and (match_func is None or match_func(line))), None)
 
-def _add_directive(block, directive, replace):
+def _add_directive(block, directive, replace, insert_at_top):
     """Adds or replaces a single directive in a config block.
 
     See _add_directives for more documentation.
@@ -619,7 +622,7 @@ def _add_directive(block, directive, replace):
             block[location] = directive
             comment_directive(block, location)
             return
-    # Append directive. Fail if the name is not a repeatable directive name,
+    # Append or prepend directive. Fail if the name is not a repeatable directive name,
     # and there is already a copy of that directive with a different value
     # in the config file.
 
@@ -652,8 +655,15 @@ def _add_directive(block, directive, replace):
                     _comment_out_directive(block, included_dir_loc, directive[1])
 
     if can_append(location, directive_name):
-        block.append(directive)
-        comment_directive(block, len(block) - 1)
+        if insert_at_top:
+            # Add a newline so the comment doesn't comment
+            # out existing directives
+            block.insert(0, nginxparser.UnspacedList('\n'))
+            block.insert(0, directive)
+            comment_directive(block, 0)
+        else:
+            block.append(directive)
+            comment_directive(block, len(block) - 1)
     elif block[location] != directive:
         raise errors.MisconfigurationError(err_fmt.format(directive, block[location]))
 

certbot-nginx/certbot_nginx/tests/configurator_test.py

@@ -100,7 +100,7 @@ class NginxConfiguratorTest(util.NginxTest):
             errors.PluginError, self.config.enhance, 'myhost', 'unknown_enhancement')
 
     def test_get_chall_pref(self):
-        self.assertEqual([challenges.TLSSNI01],
+        self.assertEqual([challenges.TLSSNI01, challenges.HTTP01],
                          self.config.get_chall_pref('myhost'))
 
     def test_save(self):
@@ -291,9 +291,11 @@ class NginxConfiguratorTest(util.NginxTest):
                          parsed_migration_conf[0])
 
     @mock.patch("certbot_nginx.configurator.tls_sni_01.NginxTlsSni01.perform")
+    @mock.patch("certbot_nginx.configurator.http_01.NginxHttp01.perform")
     @mock.patch("certbot_nginx.configurator.NginxConfigurator.restart")
     @mock.patch("certbot_nginx.configurator.NginxConfigurator.revert_challenge_config")
-    def test_perform_and_cleanup(self, mock_revert, mock_restart, mock_perform):
+    def test_perform_and_cleanup(self, mock_revert, mock_restart, mock_http_perform,
+        mock_tls_perform):
         # Only tests functionality specific to configurator.perform
         # Note: As more challenges are offered this will have to be expanded
         achall1 = achallenges.KeyAuthorizationAnnotatedChallenge(
@@ -304,7 +306,7 @@ class NginxConfiguratorTest(util.NginxTest):
             ), domain="localhost", account_key=self.rsa512jwk)
         achall2 = achallenges.KeyAuthorizationAnnotatedChallenge(
             challb=messages.ChallengeBody(
-                chall=challenges.TLSSNI01(token=b"m8TdO1qik4JVFtgPPurJmg"),
+                chall=challenges.HTTP01(token=b"m8TdO1qik4JVFtgPPurJmg"),
                 uri="https://ca.org/chall1_uri",
                 status=messages.Status("pending"),
             ), domain="example.com", account_key=self.rsa512jwk)
@@ -314,10 +316,12 @@ class NginxConfiguratorTest(util.NginxTest):
             achall2.response(self.rsa512jwk),
         ]
 
-        mock_perform.return_value = expected
+        mock_tls_perform.return_value = expected[:1]
+        mock_http_perform.return_value = expected[1:]
         responses = self.config.perform([achall1, achall2])
 
-        self.assertEqual(mock_perform.call_count, 1)
+        self.assertEqual(mock_tls_perform.call_count, 1)
+        self.assertEqual(mock_http_perform.call_count, 1)
         self.assertEqual(responses, expected)
 
         self.config.cleanup([achall1, achall2])

certbot-nginx/certbot_nginx/tests/http_01_test.py

@@ -0,0 +1,113 @@
+"""Tests for certbot_nginx.http_01"""
+import unittest
+import shutil
+
+import mock
+import six
+
+from acme import challenges
+
+from certbot import achallenges
+
+from certbot.plugins import common_test
+from certbot.tests import acme_util
+
+from certbot_nginx.tests import util
+
+
+class HttpPerformTest(util.NginxTest):
+    """Test the NginxHttp01 challenge."""
+
+    account_key = common_test.AUTH_KEY
+    achalls = [
+        achallenges.KeyAuthorizationAnnotatedChallenge(
+            challb=acme_util.chall_to_challb(
+                challenges.HTTP01(token=b"kNdwjwOeX0I_A8DXt9Msmg"), "pending"),
+            domain="www.example.com", account_key=account_key),
+        achallenges.KeyAuthorizationAnnotatedChallenge(
+            challb=acme_util.chall_to_challb(
+                challenges.HTTP01(
+                    token=b"\xba\xa9\xda?<m\xaewmx\xea\xad\xadv\xf4\x02\xc9y"
+                          b"\x80\xe2_X\t\xe7\xc7\xa4\t\xca\xf7&\x945"
+                ), "pending"),
+            domain="ipv6.com", account_key=account_key),
+        achallenges.KeyAuthorizationAnnotatedChallenge(
+            challb=acme_util.chall_to_challb(
+                challenges.HTTP01(
+                    token=b"\x8c\x8a\xbf_-f\\cw\xee\xd6\xf8/\xa5\xe3\xfd"
+                          b"\xeb9\xf1\xf5\xb9\xefVM\xc9w\xa4u\x9c\xe1\x87\xb4"
+                ), "pending"),
+            domain="www.example.org", account_key=account_key),
+        achallenges.KeyAuthorizationAnnotatedChallenge(
+            challb=acme_util.chall_to_challb(
+                challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"),
+            domain="migration.com", account_key=account_key),
+    ]
+
+    def setUp(self):
+        super(HttpPerformTest, self).setUp()
+
+        config = util.get_nginx_configurator(
+            self.config_path, self.config_dir, self.work_dir, self.logs_dir)
+
+        from certbot_nginx import http_01
+        self.http01 = http_01.NginxHttp01(config)
+
+    def tearDown(self):
+        shutil.rmtree(self.temp_dir)
+        shutil.rmtree(self.config_dir)
+        shutil.rmtree(self.work_dir)
+
+    def test_perform0(self):
+        responses = self.http01.perform()
+        self.assertEqual([], responses)
+
+    @mock.patch("certbot_nginx.configurator.NginxConfigurator.save")
+    def test_perform1(self, mock_save):
+        self.http01.add_chall(self.achalls[0])
+        response = self.achalls[0].response(self.account_key)
+
+        responses = self.http01.perform()
+
+        self.assertEqual([response], responses)
+        self.assertEqual(mock_save.call_count, 1)
+
+    def test_perform2(self):
+        acme_responses = []
+        for achall in self.achalls:
+            self.http01.add_chall(achall)
+            acme_responses.append(achall.response(self.account_key))
+
+        sni_responses = self.http01.perform()
+
+        self.assertEqual(len(sni_responses), 4)
+        for i in six.moves.range(4):
+            self.assertEqual(sni_responses[i], acme_responses[i])
+
+    def test_mod_config(self):
+        self.http01.add_chall(self.achalls[0])
+        self.http01.add_chall(self.achalls[2])
+
+        self.http01._mod_config()  # pylint: disable=protected-access
+
+        self.http01.configurator.save()
+
+        self.http01.configurator.parser.load()
+
+        # vhosts = self.http01.configurator.parser.get_vhosts()
+
+        # for vhost in vhosts:
+        #     pass
+            # if the name matches
+            # check that the location block is in there and is correct
+
+            # if vhost.addrs == set(v_addr1):
+            #     response = self.achalls[0].response(self.account_key)
+            # else:
+            #     response = self.achalls[2].response(self.account_key)
+            #     self.assertEqual(vhost.addrs, set(v_addr2_print))
+            # self.assertEqual(vhost.names, set([response.z_domain.decode('ascii')]))
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py

@@ -20,7 +20,7 @@ from certbot_nginx.tests import util
 class TlsSniPerformTest(util.NginxTest):
     """Test the NginxTlsSni01 challenge."""
 
-    account_key = common_test.TLSSNI01Test.auth_key
+    account_key = common_test.AUTH_KEY
     achalls = [
         achallenges.KeyAuthorizationAnnotatedChallenge(
             challb=acme_util.chall_to_challb(

certbot-nginx/certbot_nginx/tests/util.py

@@ -64,6 +64,7 @@ def get_nginx_configurator(
                     in_progress_dir=os.path.join(backups, "IN_PROGRESS"),
                     server="https://acme-server.org:443/new",
                     tls_sni_01_port=5001,
+                    http01_port=80
                 ),
                 name="nginx",
                 version=version)

certbot-nginx/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-nginx/tests/boulder-integration.conf.sh

@@ -49,10 +49,10 @@ http {
 
   server {
     # IPv4.
-    listen 8081;
+    listen 5002;
     # IPv6.
-    listen [::]:8081 default ipv6only=on;
-    server_name nginx.wtf;
+    listen [::]:5002 default ipv6only=on;
+    server_name nginx.wtf nginx2.wtf;
 
     root $root/webroot;
 

certbot-nginx/tests/boulder-integration.sh

@@ -22,13 +22,20 @@ certbot_test_nginx () {
         "$@"
 }
 
-certbot_test_nginx --domains nginx.wtf run
-echo | openssl s_client -connect localhost:5001 \
-    | openssl x509 -out $root/nginx.pem
-diff -q $root/nginx.pem $root/conf/live/nginx.wtf/cert.pem
+test_deployment_and_rollback() {
+    # Arguments: certname
+    echo | openssl s_client -connect localhost:5001 \
+        | openssl x509 -out $root/nginx.pem
+    diff -q $root/nginx.pem "$root/conf/live/$1/cert.pem"
 
-certbot_test_nginx rollback --checkpoints 9001
-diff -q <(echo "$original") $nginx_conf
+    certbot_test_nginx rollback --checkpoints 9001
+    diff -q <(echo "$original") $nginx_conf
+}
+
+certbot_test_nginx --domains nginx.wtf run
+test_deployment_and_rollback nginx.wtf
+certbot_test_nginx --domains nginx2.wtf --preferred-challenges http
+test_deployment_and_rollback nginx2.wtf
 
 # note: not reached if anything above fails, hence "killall" at the
 # top

certbot/__init__.py

@@ -1,4 +1,4 @@
 """Certbot client."""
 
 # version number like 1.2.3a0, must have at least 2 parts, like 1.2
-__version__ = '0.21.0.dev0'
+__version__ = '0.22.0.dev0'

certbot/main.py

@@ -4,6 +4,7 @@ import functools
 import logging.handlers
 import os
 import sys
+import warnings
 
 import configobj
 import josepy as jose
@@ -1217,9 +1218,17 @@ def main(cli_args=sys.argv[1:]):
         # Let plugins_cmd be run as un-privileged user.
         if config.func != plugins_cmd:
             raise
-    if sys.version_info[:2] == (3, 3):
-        logger.warning("Python 3.3 support will be dropped in the next release "
-                    "of Certbot - please upgrade your Python version.")
+    deprecation_fmt = (
+        "Python %s.%s support will be dropped in the next "
+        "release of Certbot - please upgrade your Python version.")
+    # We use the warnings system for Python 2.6 and logging for Python 3
+    # because DeprecationWarnings are only reported by default in Python <= 2.6
+    # and warnings can be disabled by the user.
+    if sys.version_info[:2] == (2, 6):
+        warning = deprecation_fmt % sys.version_info[:2]
+        warnings.warn(warning, DeprecationWarning)
+    elif sys.version_info[:2] == (3, 3):
+        logger.warning(deprecation_fmt, *sys.version_info[:2])
 
     set_displayer(config)
 

certbot/plugins/common.py

@@ -315,23 +315,28 @@ class Addr(object):
         return result
 
 
-class TLSSNI01(object):
-    """Abstract base for TLS-SNI-01 challenge performers"""
+class ChallengePerformer(object):
+    """Abstract base for challenge performers.
+
+    :ivar configurator: Authenticator and installer plugin
+    :ivar achalls: Annotated challenges
+    :vartype achalls: `list` of `.KeyAuthorizationAnnotatedChallenge`
+    :ivar indices: Holds the indices of challenges from a larger array
+        so the user of the class doesn't have to.
+    :vartype indices: `list` of `int`
+
+    """
 
     def __init__(self, configurator):
         self.configurator = configurator
         self.achalls = []
         self.indices = []
-        self.challenge_conf = os.path.join(
-            configurator.config.config_dir, "le_tls_sni_01_cert_challenge.conf")
-        # self.completed = 0
 
     def add_chall(self, achall, idx=None):
-        """Add challenge to TLSSNI01 object to perform at once.
+        """Store challenge to be performed when perform() is called.
 
         :param .KeyAuthorizationAnnotatedChallenge achall: Annotated
-            TLSSNI01 challenge.
-
+            challenge.
         :param int idx: index to challenge in a larger array
 
         """
@@ -339,6 +344,27 @@ class TLSSNI01(object):
         if idx is not None:
             self.indices.append(idx)
 
+    def perform(self):
+        """Perform all added challenges.
+
+        :returns: challenge respones
+        :rtype: `list` of `acme.challenges.KeyAuthorizationChallengeResponse`
+
+
+        """
+        raise NotImplementedError()
+
+
+class TLSSNI01(ChallengePerformer):
+    # pylint: disable=abstract-method
+    """Abstract base for TLS-SNI-01 challenge performers"""
+
+    def __init__(self, configurator):
+        super(TLSSNI01, self).__init__(configurator)
+        self.challenge_conf = os.path.join(
+            configurator.config.config_dir, "le_tls_sni_01_cert_challenge.conf")
+        # self.completed = 0
+
     def get_cert_path(self, achall):
         """Returns standardized name for challenge certificate.
 

certbot/plugins/common_test.py

@@ -18,6 +18,17 @@ from certbot import errors
 from certbot.tests import acme_util
 from certbot.tests import util as test_util
 
+AUTH_KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem"))
+ACHALLS = [
+    achallenges.KeyAuthorizationAnnotatedChallenge(
+        challb=acme_util.chall_to_challb(
+            challenges.TLSSNI01(token=b'token1'), "pending"),
+        domain="encryption-example.demo", account_key=AUTH_KEY),
+    achallenges.KeyAuthorizationAnnotatedChallenge(
+        challb=acme_util.chall_to_challb(
+            challenges.TLSSNI01(token=b'token2'), "pending"),
+        domain="certbot.demo", account_key=AUTH_KEY),
+]
 
 class NamespaceFunctionsTest(unittest.TestCase):
     """Tests for certbot.plugins.common.*_namespace functions."""
@@ -261,21 +272,27 @@ class AddrTest(unittest.TestCase):
         self.assertEqual(set_c, set_d)
 
 
+class ChallengePerformerTest(unittest.TestCase):
+    """Tests for certbot.plugins.common.ChallengePerformer."""
+
+    def setUp(self):
+        configurator = mock.MagicMock()
+
+        from certbot.plugins.common import ChallengePerformer
+        self.performer = ChallengePerformer(configurator)
+
+    def test_add_chall(self):
+        self.performer.add_chall(ACHALLS[0], 0)
+        self.assertEqual(1, len(self.performer.achalls))
+        self.assertEqual([0], self.performer.indices)
+
+    def test_perform(self):
+        self.assertRaises(NotImplementedError, self.performer.perform)
+
+
 class TLSSNI01Test(unittest.TestCase):
     """Tests for certbot.plugins.common.TLSSNI01."""
 
-    auth_key = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem"))
-    achalls = [
-        achallenges.KeyAuthorizationAnnotatedChallenge(
-            challb=acme_util.chall_to_challb(
-                challenges.TLSSNI01(token=b'token1'), "pending"),
-            domain="encryption-example.demo", account_key=auth_key),
-        achallenges.KeyAuthorizationAnnotatedChallenge(
-            challb=acme_util.chall_to_challb(
-                challenges.TLSSNI01(token=b'token2'), "pending"),
-            domain="certbot.demo", account_key=auth_key),
-    ]
-
     def setUp(self):
         self.tempdir = tempfile.mkdtemp()
         configurator = mock.MagicMock()
@@ -288,11 +305,6 @@ class TLSSNI01Test(unittest.TestCase):
     def tearDown(self):
         shutil.rmtree(self.tempdir)
 
-    def test_add_chall(self):
-        self.sni.add_chall(self.achalls[0], 0)
-        self.assertEqual(1, len(self.sni.achalls))
-        self.assertEqual([0], self.sni.indices)
-
     def test_setup_challenge_cert(self):
         # This is a helper function that can be used for handling
         # open context managers more elegantly. It avoids dealing with
@@ -325,7 +337,7 @@ class TLSSNI01Test(unittest.TestCase):
             OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key))
 
     def test_get_z_domain(self):
-        achall = self.achalls[0]
+        achall = ACHALLS[0]
         self.assertEqual(self.sni.get_z_domain(achall),
             achall.response(achall.account_key).z_domain.decode("utf-8"))
 

docs/cli-help.txt

@@ -107,7 +107,7 @@ optional arguments:
                         case, and to know when to deprecate support for past
                         Python versions and flags. If you wish to hide this
                         information from the Let's Encrypt server, set this to
-                        "". (default: CertbotACMEClient/0.20.0 (certbot;
+                        "". (default: CertbotACMEClient/0.21.0 (certbot;
                         Ubuntu 16.04.3 LTS) Authenticator/XXX Installer/YYY
                         (SUBCOMMAND; flags: FLAGS) Py/2.7.12). The flags
                         encoded in the user agent are: --duplicate, --force-
@@ -331,6 +331,14 @@ revoke:
   --reason {unspecified,keycompromise,affiliationchanged,superseded,cessationofoperation}
                         Specify reason for revoking certificate. (default:
                         unspecified)
+  --delete-after-revoke
+                        Delete certificates after revoking them. (default:
+                        None)
+  --no-delete-after-revoke
+                        Do not delete certificates after revoking them. This
+                        option should be used with caution because the 'renew'
+                        subcommand will attempt to renew undeleted revoked
+                        certificates. (default: None)
 
 register:
   Options for account registration & modification

letsencrypt-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.20.0"
+LE_AUTO_VERSION="0.21.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -68,10 +68,12 @@ for arg in "$@" ; do
       NO_BOOTSTRAP=1;;
     --help)
       HELP=1;;
-    --noninteractive|--non-interactive|renew)
-      ASSUME_YES=1;;
+    --noninteractive|--non-interactive)
+      NONINTERACTIVE=1;;
     --quiet)
       QUIET=1;;
+    renew)
+      ASSUME_YES=1;;
     --verbose)
       VERBOSE=1;;
     -[!-]*)
@@ -93,7 +95,7 @@ done
 
 if [ $BASENAME = "letsencrypt-auto" ]; then
   # letsencrypt-auto does not respect --help or --yes for backwards compatibility
-  ASSUME_YES=1
+  NONINTERACTIVE=1
   HELP=0
 fi
 
@@ -244,23 +246,42 @@ DeprecationBootstrap() {
   fi
 }
 
-
+MIN_PYTHON_VERSION="2.6"
+MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//')
+# Sets LE_PYTHON to Python version string and PYVER to the first two
+# digits of the python version
 DeterminePythonVersion() {
-  for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
-    # Break (while keeping the LE_PYTHON value) if found.
-    $EXISTS "$LE_PYTHON" > /dev/null && break
-  done
-  if [ "$?" != "0" ]; then
-    error "Cannot find any Pythons; please install one!"
-    exit 1
+  # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
+  #
+  # If no Python is found, PYVER is set to 0.
+  if [ "$USE_PYTHON_3" = 1 ]; then
+    for LE_PYTHON in "$LE_PYTHON" python3; do
+      # Break (while keeping the LE_PYTHON value) if found.
+      $EXISTS "$LE_PYTHON" > /dev/null && break
+    done
+  else
+    for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
+      # Break (while keeping the LE_PYTHON value) if found.
+      $EXISTS "$LE_PYTHON" > /dev/null && break
+    done
+  fi
+  if [ "$?" != "0" ]; then
+    if [ "$1" != "NOCRASH" ]; then
+      error "Cannot find any Pythons; please install one!"
+      exit 1
+    else
+      PYVER=0
+      return 0
+    fi
   fi
-  export LE_PYTHON
 
   PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
-  if [ "$PYVER" -lt 26 ]; then
-    error "You have an ancient version of Python entombed in your operating system..."
-    error "This isn't going to work; you'll need at least version 2.6."
-    exit 1
+  if [ "$PYVER" -lt "$MIN_PYVER" ]; then
+    if [ "$1" != "NOCRASH" ]; then
+      error "You have an ancient version of Python entombed in your operating system..."
+      error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION."
+      exit 1
+    fi
   fi
 }
 
@@ -384,23 +405,19 @@ BootstrapDebCommon() {
   fi
 }
 
-# If new packages are installed by BootstrapRpmCommon below, this version
-# number must be increased.
-BOOTSTRAP_RPM_COMMON_VERSION=1
-
-BootstrapRpmCommon() {
-  # Tested with:
-  #   - Fedora 20, 21, 22, 23 (x64)
-  #   - Centos 7 (x64: on DigitalOcean droplet)
-  #   - CentOS 7 Minimal install in a Hyper-V VM
-  #   - CentOS 6 (EPEL must be installed manually)
+# If new packages are installed by BootstrapRpmCommonBase below, version
+# numbers in rpm_common.sh and rpm_python3.sh must be increased.
 
+# Sets TOOL to the name of the package manager
+# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG.
+# Enables EPEL if applicable and possible.
+InitializeRPMCommonBase() {
   if type dnf 2>/dev/null
   then
-    tool=dnf
+    TOOL=dnf
   elif type yum 2>/dev/null
   then
-    tool=yum
+    TOOL=yum
 
   else
     error "Neither yum nor dnf found. Aborting bootstrap!"
@@ -408,15 +425,15 @@ BootstrapRpmCommon() {
   fi
 
   if [ "$ASSUME_YES" = 1 ]; then
-    yes_flag="-y"
+    YES_FLAG="-y"
   fi
   if [ "$QUIET" = 1 ]; then
     QUIET_FLAG='--quiet'
   fi
 
-  if ! $tool list *virtualenv >/dev/null 2>&1; then
+  if ! $TOOL list *virtualenv >/dev/null 2>&1; then
     echo "To use Certbot, packages from the EPEL repository need to be installed."
-    if ! $tool list epel-release >/dev/null 2>&1; then
+    if ! $TOOL list epel-release >/dev/null 2>&1; then
       error "Enable the EPEL repository and try running Certbot again."
       exit 1
     fi
@@ -425,14 +442,20 @@ BootstrapRpmCommon() {
       sleep 1s
       /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
       sleep 1s
-      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..."
+      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..."
       sleep 1s
     fi
-    if ! $tool install $yes_flag $QUIET_FLAG epel-release; then
+    if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then
       error "Could not enable EPEL. Aborting bootstrap!"
       exit 1
     fi
   fi
+}
+
+BootstrapRpmCommonBase() {
+  # Arguments: whitespace-delimited python packages to install
+
+  InitializeRPMCommonBase # This call is superfluous in practice
 
   pkgs="
     gcc
@@ -444,10 +467,39 @@ BootstrapRpmCommon() {
     ca-certificates
   "
 
-  # Most RPM distros use the "python" or "python-" naming convention.  Let's try that first.
-  if $tool list python >/dev/null 2>&1; then
+  # Add the python packages
+  pkgs="$pkgs
+    $1
+  "
+
+  if $TOOL list installed "httpd" >/dev/null 2>&1; then
     pkgs="$pkgs
-      python
+      mod_ssl
+    "
+  fi
+
+  if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then
+    error "Could not install OS dependencies. Aborting bootstrap!"
+    exit 1
+  fi
+}
+
+# If new packages are installed by BootstrapRpmCommon below, this version
+# number must be increased.
+BOOTSTRAP_RPM_COMMON_VERSION=1
+
+BootstrapRpmCommon() {
+  # Tested with:
+  #   - Fedora 20, 21, 22, 23 (x64)
+  #   - Centos 7 (x64: on DigitalOcean droplet)
+  #   - CentOS 7 Minimal install in a Hyper-V VM
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  # Most RPM distros use the "python" or "python-" naming convention.  Let's try that first.
+  if $TOOL list python >/dev/null 2>&1; then
+    python_pkgs="$python
       python-devel
       python-virtualenv
       python-tools
@@ -455,9 +507,8 @@ BootstrapRpmCommon() {
     "
   # Fedora 26 starts to use the prefix python2 for python2 based packages.
   # this elseif is theoretically for any Fedora over version 26:
-  elif $tool list python2 >/dev/null 2>&1; then
-    pkgs="$pkgs
-      python2
+  elif $TOOL list python2 >/dev/null 2>&1; then
+    python_pkgs="$python2
       python2-libs
       python2-setuptools
       python2-devel
@@ -468,8 +519,7 @@ BootstrapRpmCommon() {
   # Some distros and older versions of current distros use a "python27"
   # instead of the "python" or "python-" naming convention.
   else
-    pkgs="$pkgs
-      python27
+    python_pkgs="$python27
       python27-devel
       python27-virtualenv
       python27-tools
@@ -477,16 +527,31 @@ BootstrapRpmCommon() {
     "
   fi
 
-  if $tool list installed "httpd" >/dev/null 2>&1; then
-    pkgs="$pkgs
-      mod_ssl
-    "
-  fi
+  BootstrapRpmCommonBase "$python_pkgs"
+}
 
-  if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then
-    error "Could not install OS dependencies. Aborting bootstrap!"
+# If new packages are installed by BootstrapRpmPython3 below, this version
+# number must be increased.
+BOOTSTRAP_RPM_PYTHON3_VERSION=1
+
+BootstrapRpmPython3() {
+  # Tested with:
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  # EPEL uses python34
+  if $TOOL list python34 >/dev/null 2>&1; then
+    python_pkgs="python34
+      python34-devel
+      python34-tools
+    "
+  else
+    error "No supported Python package available to install. Aborting bootstrap!"
     exit 1
   fi
+
+  BootstrapRpmCommonBase "$python_pkgs"
 }
 
 # If new packages are installed by BootstrapSuseCommon below, this version
@@ -715,11 +780,27 @@ elif [ -f /etc/mageia-release ]; then
   }
   BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION"
 elif [ -f /etc/redhat-release ]; then
-  Bootstrap() {
-    BootstrapMessage "RedHat-based OSes"
-    BootstrapRpmCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+  # Run DeterminePythonVersion to decide on the basis of available Python versions
+  # whether to use 2.x or 3.x on RedHat-like systems.
+  # Then, revert LE_PYTHON to its previous state.
+  prev_le_python="$LE_PYTHON"
+  unset LE_PYTHON
+  DeterminePythonVersion "NOCRASH"
+  if [ "$PYVER" -eq 26 ]; then
+    Bootstrap() {
+      BootstrapMessage "RedHat-based OSes that will use Python3"
+      BootstrapRpmPython3
+    }
+    USE_PYTHON_3=1
+    BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+  else
+    Bootstrap() {
+      BootstrapMessage "RedHat-based OSes"
+      BootstrapRpmCommon
+    }
+    BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+  fi
+  LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
   Bootstrap() {
     BootstrapMessage "openSUSE-based OSes"
@@ -816,7 +897,11 @@ TempDir() {
   mktemp -d 2>/dev/null || mktemp -d -t 'le'  # Linux || macOS
 }
 
-
+# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise,
+# returns a non-zero number.
+OldVenvExists() {
+    [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ]
+}
 
 if [ "$1" = "--le-auto-phase2" ]; then
   # Phase 2: Create venv, install LE, and run.
@@ -824,14 +909,26 @@ if [ "$1" = "--le-auto-phase2" ]; then
   shift 1  # the --le-auto-phase2 arg
   SetPrevBootstrapVersion
 
+  if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then
+    unset LE_PYTHON
+  fi
+
   INSTALLED_VERSION="none"
-  if [ -d "$VENV_PATH" ]; then
+  if [ -d "$VENV_PATH" ] || OldVenvExists; then
     # If the selected Bootstrap function isn't a noop and it differs from the
     # previously used version
     if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then
       # if non-interactive mode or stdin and stdout are connected to a terminal
       if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
-        rm -rf "$VENV_PATH"
+        if [ -d "$VENV_PATH" ]; then
+          rm -rf "$VENV_PATH"
+        fi
+        # In the case the old venv was just a symlink to the new one,
+        # OldVenvExists is now false because we deleted the venv at VENV_PATH.
+        if OldVenvExists; then
+          rm -rf "$OLD_VENV_PATH"
+          ln -s "$VENV_PATH" "$OLD_VENV_PATH"
+        fi
         RerunWithArgs "$@"
       else
         error "Skipping upgrade because new OS dependencies may need to be installed."
@@ -841,6 +938,10 @@ if [ "$1" = "--le-auto-phase2" ]; then
         error "install any required packages."
         # Set INSTALLED_VERSION to be the same so we don't update the venv
         INSTALLED_VERSION="$LE_AUTO_VERSION"
+        # Continue to use OLD_VENV_PATH if the new venv doesn't exist
+        if [ ! -d "$VENV_PATH" ]; then
+          VENV_BIN="$OLD_VENV_PATH/bin"
+        fi
       fi
     elif [ -f "$VENV_BIN/letsencrypt" ]; then
       # --version output ran through grep due to python-cryptography DeprecationWarnings
@@ -858,10 +959,18 @@ if [ "$1" = "--le-auto-phase2" ]; then
     say "Creating virtual environment..."
     DeterminePythonVersion
     rm -rf "$VENV_PATH"
-    if [ "$VERBOSE" = 1 ]; then
-      virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
+    if [ "$PYVER" -le 27 ]; then
+      if [ "$VERBOSE" = 1 ]; then
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
+      else
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
+      fi
     else
-      virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
+      if [ "$VERBOSE" = 1 ]; then
+        "$LE_PYTHON" -m venv "$VENV_PATH"
+      else
+        "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null
+      fi
     fi
 
     if [ -n "$BOOTSTRAP_VERSION" ]; then
@@ -983,9 +1092,16 @@ idna==2.5 \
 ipaddress==1.0.16 \
     --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \
     --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0
+josepy==1.0.1 \
+    --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \
+    --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc
 linecache2==1.0.0 \
     --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \
     --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c
+# Using an older version of mock here prevents regressions of #5276.
+mock==1.3.0 \
+    --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \
+    --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6
 ordereddict==1.1 \
     --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f
 packaging==16.8 \
@@ -1062,10 +1178,6 @@ zope.interface==4.1.3 \
     --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \
     --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \
     --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392
-# Using an older version of mock here prevents regressions of #5276.
-mock==1.3.0 \
-    --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \
-    --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6
 
 # Contains the requirements for the letsencrypt package.
 #
@@ -1078,18 +1190,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.20.0 \
-    --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \
-    --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88
-acme==0.20.0 \
-    --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \
-    --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5
-certbot-apache==0.20.0 \
-    --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \
-    --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f
-certbot-nginx==0.20.0 \
-    --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \
-    --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae
+certbot==0.21.0 \
+    --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \
+    --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857
+acme==0.21.0 \
+    --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \
+    --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d
+certbot-apache==0.21.0 \
+    --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \
+    --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b
+certbot-nginx==0.21.0 \
+    --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \
+    --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
@@ -1319,9 +1431,10 @@ else
   # upgrading. Phase 1 checks the version of the latest release of
   # certbot-auto (which is always the same as that of the certbot
   # package). Phase 2 checks the version of the locally installed certbot.
+  export PHASE_1_VERSION="$LE_AUTO_VERSION"
 
   if [ ! -f "$VENV_BIN/letsencrypt" ]; then
-    if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then
+    if ! OldVenvExists; then
       if [ "$HELP" = 1 ]; then
         echo "$USAGE"
         exit 0
@@ -1353,17 +1466,22 @@ On failure, return non-zero.
 
 """
 
-from __future__ import print_function
+from __future__ import print_function, unicode_literals
 
 from distutils.version import LooseVersion
 from json import loads
 from os import devnull, environ
 from os.path import dirname, join
 import re
+import ssl
 from subprocess import check_call, CalledProcessError
 from sys import argv, exit
-from urllib2 import build_opener, HTTPHandler, HTTPSHandler
-from urllib2 import HTTPError, URLError
+try:
+    from urllib2 import build_opener, HTTPHandler, HTTPSHandler
+    from urllib2 import HTTPError, URLError
+except ImportError:
+    from urllib.request import build_opener, HTTPHandler, HTTPSHandler
+    from urllib.error import HTTPError, URLError
 
 PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY-----
 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq
@@ -1385,8 +1503,11 @@ class HttpsGetter(object):
     def __init__(self):
         """Build an HTTPS opener."""
         # Based on pip 1.4.1's URLOpener
-        # This verifies certs on only Python >=2.7.9.
-        self._opener = build_opener(HTTPSHandler())
+        # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set.
+        if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'):
+            self._opener = build_opener(HTTPSHandler(context=cert_none_context()))
+        else:
+            self._opener = build_opener(HTTPSHandler())
         # Strip out HTTPHandler to prevent MITM spoof:
         for handler in self._opener.handlers:
             if isinstance(handler, HTTPHandler):
@@ -1408,7 +1529,7 @@ class HttpsGetter(object):
 
 def write(contents, dir, filename):
     """Write something to a file in a certain directory."""
-    with open(join(dir, filename), 'w') as file:
+    with open(join(dir, filename), 'wb') as file:
         file.write(contents)
 
 
@@ -1416,13 +1537,13 @@ def latest_stable_version(get):
     """Return the latest stable release of letsencrypt."""
     metadata = loads(get(
         environ.get('LE_AUTO_JSON_URL',
-                    'https://pypi.python.org/pypi/certbot/json')))
+                    'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8'))
     # metadata['info']['version'] actually returns the latest of any kind of
     # release release, contrary to https://wiki.python.org/moin/PyPIJSON.
     # The regex is a sufficient regex for picking out prereleases for most
     # packages, LE included.
     return str(max(LooseVersion(r) for r
-                   in metadata['releases'].iterkeys()
+                   in metadata['releases'].keys()
                    if re.match('^[0-9.]+$', r)))
 
 
@@ -1439,7 +1560,7 @@ def verified_new_le_auto(get, tag, temp_dir):
         'letsencrypt-auto-source/') % tag
     write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto')
     write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig')
-    write(PUBLIC_KEY, temp_dir, 'public_key.pem')
+    write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem')
     try:
         with open(devnull, 'w') as dev_null:
             check_call(['openssl', 'dgst', '-sha256', '-verify',
@@ -1454,6 +1575,14 @@ def verified_new_le_auto(get, tag, temp_dir):
                             "certbot-auto.", exc)
 
 
+def cert_none_context():
+    """Create a SSLContext object to not check hostname."""
+    # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this.
+    context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+    context.verify_mode = ssl.CERT_NONE
+    return context
+
+
 def main():
     get = HttpsGetter().get
     flag = argv[1]
@@ -1475,8 +1604,10 @@ if __name__ == '__main__':
 
 UNLIKELY_EOF
     # ---------------------------------------------------------------------------
-    DeterminePythonVersion
-    if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then
+    DeterminePythonVersion "NOCRASH"
+    if [ "$PYVER" -lt "$MIN_PYVER" ]; then
+      error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates."
+    elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then
       error "WARNING: unable to check for updates."
     elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then
       say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..."

letsencrypt-auto-source/certbot-auto.asc

@@ -1,11 +1,11 @@
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v2
 
-iQEcBAABCAAGBQJaKHMlAAoJEE0XyZXNl3Xy6OEH/iPg6D6+zco4NHMwxYIcTWVt
-XE4u3CjuLcEVsvEnJYNSA48NHyi9rIqMHd+IneLU+lCG2D7eBsisNNyVPIgHktTf
-p9i0WoZB+axe1glv9FJSZvjvr2d/ic4/wYHBF1c+szb9p8Z7o5Lhqa9/gtLJ/SZX
-OGU0wok4hPIB6emq5zvmi/+r1AiOECXE26lZ0STp6wDkvz+ahTJSk6UaPCDY+Az4
-X2VmnRSks/gk7Q8cloFnyiPXyFMQHdGIBRrIXsSix90QqmNUF7iYb8sbHksU23EI
-/LmIwSJlDm6KNOO2nllBB/uIg2ki7g0z7R4uf7XF4im+P95PAL/tQQ45lVj8DXE=
-=Is56
+iQEcBAABCAAGBQJaX+JUAAoJEE0XyZXNl3XyUCkH/jowI7yayXREoBUWpLuByd/n
+e1wGLQjnZYkxv/AJGJ63G3QvwpzmIqo3r/6K4ARlUcdOnepZRDpF6jC4F5q9vBwW
+AvUVU2B7e6mC6l/jXNepS8xowEwkQptQBDfnqh8TTeTb3rQTFod8X41skZ2633HL
+RX4ditKaGMbcswMn6+5/juz0YK5ujVdVTcMeMcZKP2tvPJ9Y08YdpY6IdrM0Mfhn
+IqssjM06CzsiYHeNOXfRY4vAPw4Oq/md3bf6ZpPCee1HPiDm0NvHtTemWBkPIehf
+yy0U8JIDIZha4WKo3yifbZFL5Zf5czVkrtqQ3DBRcLrCFtBh2aTVsIMJkpW/wFo=
+=d/hS
 -----END PGP SIGNATURE-----

letsencrypt-auto-source/letsencrypt-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.21.0.dev0"
+LE_AUTO_VERSION="0.22.0.dev0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -254,7 +254,7 @@ DeterminePythonVersion() {
   # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
   #
   # If no Python is found, PYVER is set to 0.
-  if [ -n "$USE_PYTHON_3" ]; then
+  if [ "$USE_PYTHON_3" = 1 ]; then
     for LE_PYTHON in "$LE_PYTHON" python3; do
       # Break (while keeping the LE_PYTHON value) if found.
       $EXISTS "$LE_PYTHON" > /dev/null && break
@@ -274,7 +274,6 @@ DeterminePythonVersion() {
       return 0
     fi
   fi
-  export LE_PYTHON
 
   PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
   if [ "$PYVER" -lt "$MIN_PYVER" ]; then
@@ -443,7 +442,7 @@ InitializeRPMCommonBase() {
       sleep 1s
       /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
       sleep 1s
-      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..."
+      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..."
       sleep 1s
     fi
     if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then
@@ -781,6 +780,9 @@ elif [ -f /etc/mageia-release ]; then
   }
   BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION"
 elif [ -f /etc/redhat-release ]; then
+  # Run DeterminePythonVersion to decide on the basis of available Python versions
+  # whether to use 2.x or 3.x on RedHat-like systems.
+  # Then, revert LE_PYTHON to its previous state.
   prev_le_python="$LE_PYTHON"
   unset LE_PYTHON
   DeterminePythonVersion "NOCRASH"
@@ -798,7 +800,7 @@ elif [ -f /etc/redhat-release ]; then
     }
     BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
   fi
-  export LE_PYTHON="$prev_le_python"
+  LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
   Bootstrap() {
     BootstrapMessage "openSUSE-based OSes"
@@ -895,7 +897,11 @@ TempDir() {
   mktemp -d 2>/dev/null || mktemp -d -t 'le'  # Linux || macOS
 }
 
-
+# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise,
+# returns a non-zero number.
+OldVenvExists() {
+    [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ]
+}
 
 if [ "$1" = "--le-auto-phase2" ]; then
   # Phase 2: Create venv, install LE, and run.
@@ -903,14 +909,26 @@ if [ "$1" = "--le-auto-phase2" ]; then
   shift 1  # the --le-auto-phase2 arg
   SetPrevBootstrapVersion
 
+  if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then
+    unset LE_PYTHON
+  fi
+
   INSTALLED_VERSION="none"
-  if [ -d "$VENV_PATH" ]; then
+  if [ -d "$VENV_PATH" ] || OldVenvExists; then
     # If the selected Bootstrap function isn't a noop and it differs from the
     # previously used version
     if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then
       # if non-interactive mode or stdin and stdout are connected to a terminal
       if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
-        rm -rf "$VENV_PATH"
+        if [ -d "$VENV_PATH" ]; then
+          rm -rf "$VENV_PATH"
+        fi
+        # In the case the old venv was just a symlink to the new one,
+        # OldVenvExists is now false because we deleted the venv at VENV_PATH.
+        if OldVenvExists; then
+          rm -rf "$OLD_VENV_PATH"
+          ln -s "$VENV_PATH" "$OLD_VENV_PATH"
+        fi
         RerunWithArgs "$@"
       else
         error "Skipping upgrade because new OS dependencies may need to be installed."
@@ -920,6 +938,10 @@ if [ "$1" = "--le-auto-phase2" ]; then
         error "install any required packages."
         # Set INSTALLED_VERSION to be the same so we don't update the venv
         INSTALLED_VERSION="$LE_AUTO_VERSION"
+        # Continue to use OLD_VENV_PATH if the new venv doesn't exist
+        if [ ! -d "$VENV_PATH" ]; then
+          VENV_BIN="$OLD_VENV_PATH/bin"
+        fi
       fi
     elif [ -f "$VENV_BIN/letsencrypt" ]; then
       # --version output ran through grep due to python-cryptography DeprecationWarnings
@@ -1168,18 +1190,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.20.0 \
-    --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \
-    --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88
-acme==0.20.0 \
-    --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \
-    --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5
-certbot-apache==0.20.0 \
-    --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \
-    --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f
-certbot-nginx==0.20.0 \
-    --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \
-    --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae
+certbot==0.21.0 \
+    --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \
+    --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857
+acme==0.21.0 \
+    --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \
+    --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d
+certbot-apache==0.21.0 \
+    --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \
+    --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b
+certbot-nginx==0.21.0 \
+    --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \
+    --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
@@ -1409,9 +1431,10 @@ else
   # upgrading. Phase 1 checks the version of the latest release of
   # certbot-auto (which is always the same as that of the certbot
   # package). Phase 2 checks the version of the locally installed certbot.
+  export PHASE_1_VERSION="$LE_AUTO_VERSION"
 
   if [ ! -f "$VENV_BIN/letsencrypt" ]; then
-    if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then
+    if ! OldVenvExists; then
       if [ "$HELP" = 1 ]; then
         echo "$USAGE"
         exit 0
@@ -1482,7 +1505,7 @@ class HttpsGetter(object):
         # Based on pip 1.4.1's URLOpener
         # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set.
         if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'):
-            self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context()))
+            self._opener = build_opener(HTTPSHandler(context=cert_none_context()))
         else:
             self._opener = build_opener(HTTPSHandler())
         # Strip out HTTPHandler to prevent MITM spoof:
@@ -1520,7 +1543,7 @@ def latest_stable_version(get):
     # The regex is a sufficient regex for picking out prereleases for most
     # packages, LE included.
     return str(max(LooseVersion(r) for r
-                   in iter(metadata['releases'].keys())
+                   in metadata['releases'].keys()
                    if re.match('^[0-9.]+$', r)))
 
 
@@ -1552,7 +1575,7 @@ def verified_new_le_auto(get, tag, temp_dir):
                             "certbot-auto.", exc)
 
 
-def create_CERT_NONE_context():
+def cert_none_context():
     """Create a SSLContext object to not check hostname."""
     # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this.
     context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)

letsencrypt-auto-source/letsencrypt-auto.template

@@ -254,7 +254,7 @@ DeterminePythonVersion() {
   # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
   #
   # If no Python is found, PYVER is set to 0.
-  if [ -n "$USE_PYTHON_3" ]; then
+  if [ "$USE_PYTHON_3" = 1 ]; then
     for LE_PYTHON in "$LE_PYTHON" python3; do
       # Break (while keeping the LE_PYTHON value) if found.
       $EXISTS "$LE_PYTHON" > /dev/null && break
@@ -274,7 +274,6 @@ DeterminePythonVersion() {
       return 0
     fi
   fi
-  export LE_PYTHON
 
   PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
   if [ "$PYVER" -lt "$MIN_PYVER" ]; then
@@ -320,6 +319,9 @@ elif [ -f /etc/mageia-release ]; then
   }
   BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION"
 elif [ -f /etc/redhat-release ]; then
+  # Run DeterminePythonVersion to decide on the basis of available Python versions
+  # whether to use 2.x or 3.x on RedHat-like systems.
+  # Then, revert LE_PYTHON to its previous state.
   prev_le_python="$LE_PYTHON"
   unset LE_PYTHON
   DeterminePythonVersion "NOCRASH"
@@ -337,7 +339,7 @@ elif [ -f /etc/redhat-release ]; then
     }
     BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
   fi
-  export LE_PYTHON="$prev_le_python"
+  LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
   Bootstrap() {
     BootstrapMessage "openSUSE-based OSes"
@@ -434,7 +436,11 @@ TempDir() {
   mktemp -d 2>/dev/null || mktemp -d -t 'le'  # Linux || macOS
 }
 
-
+# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise,
+# returns a non-zero number.
+OldVenvExists() {
+    [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ]
+}
 
 if [ "$1" = "--le-auto-phase2" ]; then
   # Phase 2: Create venv, install LE, and run.
@@ -442,14 +448,26 @@ if [ "$1" = "--le-auto-phase2" ]; then
   shift 1  # the --le-auto-phase2 arg
   SetPrevBootstrapVersion
 
+  if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then
+    unset LE_PYTHON
+  fi
+
   INSTALLED_VERSION="none"
-  if [ -d "$VENV_PATH" ]; then
+  if [ -d "$VENV_PATH" ] || OldVenvExists; then
     # If the selected Bootstrap function isn't a noop and it differs from the
     # previously used version
     if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then
       # if non-interactive mode or stdin and stdout are connected to a terminal
       if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
-        rm -rf "$VENV_PATH"
+        if [ -d "$VENV_PATH" ]; then
+          rm -rf "$VENV_PATH"
+        fi
+        # In the case the old venv was just a symlink to the new one,
+        # OldVenvExists is now false because we deleted the venv at VENV_PATH.
+        if OldVenvExists; then
+          rm -rf "$OLD_VENV_PATH"
+          ln -s "$VENV_PATH" "$OLD_VENV_PATH"
+        fi
         RerunWithArgs "$@"
       else
         error "Skipping upgrade because new OS dependencies may need to be installed."
@@ -459,6 +477,10 @@ if [ "$1" = "--le-auto-phase2" ]; then
         error "install any required packages."
         # Set INSTALLED_VERSION to be the same so we don't update the venv
         INSTALLED_VERSION="$LE_AUTO_VERSION"
+        # Continue to use OLD_VENV_PATH if the new venv doesn't exist
+        if [ ! -d "$VENV_PATH" ]; then
+          VENV_BIN="$OLD_VENV_PATH/bin"
+        fi
       fi
     elif [ -f "$VENV_BIN/letsencrypt" ]; then
       # --version output ran through grep due to python-cryptography DeprecationWarnings
@@ -568,9 +590,10 @@ else
   # upgrading. Phase 1 checks the version of the latest release of
   # certbot-auto (which is always the same as that of the certbot
   # package). Phase 2 checks the version of the locally installed certbot.
+  export PHASE_1_VERSION="$LE_AUTO_VERSION"
 
   if [ ! -f "$VENV_BIN/letsencrypt" ]; then
-    if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then
+    if ! OldVenvExists; then
       if [ "$HELP" = 1 ]; then
         echo "$USAGE"
         exit 0

letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh

@@ -35,7 +35,7 @@ InitializeRPMCommonBase() {
       sleep 1s
       /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
       sleep 1s
-      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..."
+      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..."
       sleep 1s
     fi
     if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then

letsencrypt-auto-source/pieces/certbot-requirements.txt

@@ -1,12 +1,12 @@
-certbot==0.20.0 \
-    --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \
-    --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88
-acme==0.20.0 \
-    --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \
-    --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5
-certbot-apache==0.20.0 \
-    --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \
-    --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f
-certbot-nginx==0.20.0 \
-    --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \
-    --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae
+certbot==0.21.0 \
+    --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \
+    --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857
+acme==0.21.0 \
+    --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \
+    --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d
+certbot-apache==0.21.0 \
+    --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \
+    --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b
+certbot-nginx==0.21.0 \
+    --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \
+    --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642

letsencrypt-auto-source/pieces/fetch.py

@@ -50,7 +50,7 @@ class HttpsGetter(object):
         # Based on pip 1.4.1's URLOpener
         # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set.
         if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'):
-            self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context()))
+            self._opener = build_opener(HTTPSHandler(context=cert_none_context()))
         else:
             self._opener = build_opener(HTTPSHandler())
         # Strip out HTTPHandler to prevent MITM spoof:
@@ -88,7 +88,7 @@ def latest_stable_version(get):
     # The regex is a sufficient regex for picking out prereleases for most
     # packages, LE included.
     return str(max(LooseVersion(r) for r
-                   in iter(metadata['releases'].keys())
+                   in metadata['releases'].keys()
                    if re.match('^[0-9.]+$', r)))
 
 
@@ -120,7 +120,7 @@ def verified_new_le_auto(get, tag, temp_dir):
                             "certbot-auto.", exc)
 
 
-def create_CERT_NONE_context():
+def cert_none_context():
     """Create a SSLContext object to not check hostname."""
     # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this.
     context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)

tests/letstest/scripts/test_leauto_upgrades.sh

@@ -64,10 +64,45 @@ iQIDAQAB
 -----END PUBLIC KEY-----
 "
 
-if ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then
+if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then
+    if command -v python3; then
+        echo "Didn't expect Python 3 to be installed!"
+        exit 1
+    fi
+    cp letsencrypt-auto cb-auto
+    if ! ./cb-auto -v --debug --version 2>&1 | grep 0.5.0 ; then
+        echo "Certbot shouldn't have updated to a new version!"
+        exit 1
+    fi
+    if [ -d "/opt/eff.org" ]; then
+        echo "New directory shouldn't have been created!"
+        exit 1
+    fi
+    # Create a 2nd venv at the new path to ensure we properly handle this case
+    export VENV_PATH="/opt/eff.org/certbot/venv"
+    if ! sudo -E ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.5.0 ; then
+        echo second installation appeared to fail
+        exit 1
+    fi
+    unset VENV_PATH
+    EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2)
+    if ! ./cb-auto -v --debug --version -n 2>&1 | grep "$EXPECTED_VERSION" ; then
+        echo "Certbot didn't upgrade as expected!"
+        exit 1
+    fi
+    if ! command -v python3; then
+        echo "Python3 wasn't properly installed"
+        exit 1
+    fi
+    if [ "$(/opt/eff.org/certbot/venv/bin/python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1)" != 3 ]; then
+        echo "Python3 wasn't used in venv!"
+        exit 1
+    fi
+elif ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then
     echo upgrade appeared to fail
     exit 1
 fi
+
 echo upgrade appeared to be successful
 
 if [ "$(tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt)" != "/opt/eff.org/certbot/venv" ]; then

tools/dev_constraints.txt

@@ -21,7 +21,7 @@ futures==3.1.1
 google-api-python-client==1.5
 httplib2==0.10.3
 imagesize==0.7.1
-ipdb==0.10.3
+ipdb==0.10.2
 ipython==5.5.0
 ipython-genutils==0.2.0
 Jinja2==2.9.6