run apache test farm tests

I have

   Define: goto=http://datasets.datalad.org/?dir=/shub

which seems to be perfectly fine by apache.  But the = in URL throws
certbot off causing an error, forbidding certificates updates for any
of the sites etc.

It seems only logical to simply split once by a single (if present) =.

I have not looked/analyzed into the other spot of use of
.partition("=") in parse_define_file -- may be similar approach needs
to be implemented there as well.

.azure-pipelines/templates/jobs/extended-tests-jobs.yml

@@ -8,86 +8,12 @@ jobs:
       - group: certbot-common
     strategy:
       matrix:
-        linux-py36:
-          PYTHON_VERSION: 3.6
-          TOXENV: py36
-        linux-py37:
-          PYTHON_VERSION: 3.7
-          TOXENV: py37
-        linux-py38:
-          PYTHON_VERSION: 3.8
-          TOXENV: py38
-        linux-py37-nopin:
-          PYTHON_VERSION: 3.7
-          TOXENV: py37
-          CERTBOT_NO_PIN: 1
-        linux-external-mock:
-          TOXENV: external-mock
-        linux-boulder-v1-integration-certbot-oldest:
-          PYTHON_VERSION: 3.6
-          TOXENV: integration-certbot-oldest
-          ACME_SERVER: boulder-v1
-        linux-boulder-v2-integration-certbot-oldest:
-          PYTHON_VERSION: 3.6
-          TOXENV: integration-certbot-oldest
-          ACME_SERVER: boulder-v2
-        linux-boulder-v1-integration-nginx-oldest:
-          PYTHON_VERSION: 3.6
-          TOXENV: integration-nginx-oldest
-          ACME_SERVER: boulder-v1
-        linux-boulder-v2-integration-nginx-oldest:
-          PYTHON_VERSION: 3.6
-          TOXENV: integration-nginx-oldest
-          ACME_SERVER: boulder-v2
-        linux-boulder-v1-py36-integration:
-          PYTHON_VERSION: 3.6
-          TOXENV: integration
-          ACME_SERVER: boulder-v1
-        linux-boulder-v2-py36-integration:
-          PYTHON_VERSION: 3.6
-          TOXENV: integration
-          ACME_SERVER: boulder-v2
-        linux-boulder-v1-py37-integration:
-          PYTHON_VERSION: 3.7
-          TOXENV: integration
-          ACME_SERVER: boulder-v1
-        linux-boulder-v2-py37-integration:
-          PYTHON_VERSION: 3.7
-          TOXENV: integration
-          ACME_SERVER: boulder-v2
-        linux-boulder-v1-py38-integration:
-          PYTHON_VERSION: 3.8
-          TOXENV: integration
-          ACME_SERVER: boulder-v1
-        linux-boulder-v2-py38-integration:
-          PYTHON_VERSION: 3.8
-          TOXENV: integration
-          ACME_SERVER: boulder-v2
-        linux-boulder-v1-py39-integration:
-          PYTHON_VERSION: 3.9
-          TOXENV: integration
-          ACME_SERVER: boulder-v1
-        linux-boulder-v2-py39-integration:
-          PYTHON_VERSION: 3.9
-          TOXENV: integration
-          ACME_SERVER: boulder-v2
-        nginx-compat:
-          TOXENV: nginx_compat
-        linux-integration-rfc2136:
-          IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.8
-          TOXENV: integration-dns-rfc2136
-        docker-dev:
-          TOXENV: docker_dev
         macos-farmtest-apache2:
           # We run one of these test farm tests on macOS to help ensure the
           # tests continue to work on the platform.
           IMAGE_NAME: macOS-10.15
           PYTHON_VERSION: 3.8
           TOXENV: test-farm-apache2
-        farmtest-sdists:
-          PYTHON_VERSION: 3.7
-          TOXENV: test-farm-sdists
     pool:
       vmImage: $(IMAGE_NAME)
     steps:

.azure-pipelines/templates/stages/test-and-package-stage.yml

@@ -1,6 +1,4 @@
 stages:
   - stage: TestAndPackage
     jobs:
-      - template: ../jobs/standard-tests-jobs.yml
       - template: ../jobs/extended-tests-jobs.yml
-      - template: ../jobs/packaging-jobs.yml

acme/acme/client.py

@@ -658,7 +658,10 @@ class ClientV2(ClientBase):
         response = self._post(self.directory['newOrder'], order)
         body = messages.Order.from_json(response.json())
         authorizations = []
-        for url in body.authorizations:
+        # pylint has trouble understanding our josepy based objects which use
+        # things like custom metaclass logic. body.authorizations should be a
+        # list of strings containing URLs so let's disable this check here.
+        for url in body.authorizations:  # pylint: disable=not-an-iterable
             authorizations.append(self._authzr_from_response(self._post_as_get(url), uri=url))
         return messages.OrderResource(
             body=body,

acme/acme/standalone.py

@@ -8,6 +8,7 @@ import socket
 import socketserver
 import threading
 from typing import List
+from typing import Optional
 
 from acme import challenges
 from acme import crypto_util
@@ -66,6 +67,9 @@ class BaseDualNetworkedServers:
         self.threads: List[threading.Thread] = []
         self.servers: List[socketserver.BaseServer] = []
 
+        # Preserve socket error for re-raising, if no servers can be started
+        last_socket_err: Optional[socket.error] = None
+
         # Must try True first.
         # Ubuntu, for example, will fail to bind to IPv4 if we've already bound
         # to IPv6. But that's ok, since it will accept IPv4 connections on the IPv6
@@ -82,7 +86,8 @@ class BaseDualNetworkedServers:
                 logger.debug(
                     "Successfully bound to %s:%s using %s", new_address[0],
                     new_address[1], "IPv6" if ip_version else "IPv4")
-            except socket.error:
+            except socket.error as e:
+                last_socket_err = e
                 if self.servers:
                     # Already bound using IPv6.
                     logger.debug(
@@ -101,7 +106,10 @@ class BaseDualNetworkedServers:
                 # bind to the same port for both servers.
                 port = server.socket.getsockname()[1]
         if not self.servers:
-            raise socket.error("Could not bind to IPv4 or IPv6.")
+            if last_socket_err:
+                raise last_socket_err
+            else: # pragma: no cover
+                raise socket.error("Could not bind to IPv4 or IPv6.")
 
     def serve_forever(self):
         """Wraps socketserver.TCPServer.serve_forever"""

acme/setup.py

@@ -19,17 +19,16 @@ install_requires = [
     'setuptools>=39.0.1',
 ]
 
-dev_extras = [
-    'pytest',
-    'pytest-xdist',
-    'tox',
-]
-
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',
 ]
 
+test_extras = [
+    'pytest',
+    'pytest-xdist',
+]
+
 setup(
     name='acme',
     version=version,
@@ -57,7 +56,7 @@ setup(
     include_package_data=True,
     install_requires=install_requires,
     extras_require={
-        'dev': dev_extras,
         'docs': docs_extras,
+        'test': test_extras,
     },
 )

acme/tests/standalone_test.py

@@ -190,12 +190,18 @@ class BaseDualNetworkedServersTest(unittest.TestCase):
 
     @mock.patch("socket.socket.bind")
     def test_fail_to_bind(self, mock_bind):
-        mock_bind.side_effect = socket.error
+        from errno import EADDRINUSE
         from acme.standalone import BaseDualNetworkedServers
-        self.assertRaises(socket.error, BaseDualNetworkedServers,
-                          BaseDualNetworkedServersTest.SingleProtocolServer,
-                          ('', 0),
-                          socketserver.BaseRequestHandler)
+
+        mock_bind.side_effect = socket.error(EADDRINUSE, "Fake addr in use error")
+
+        with self.assertRaises(socket.error) as em:
+            BaseDualNetworkedServers(
+                BaseDualNetworkedServersTest.SingleProtocolServer,
+                ('', 0), socketserver.BaseRequestHandler)
+
+        self.assertEqual(em.exception.errno, EADDRINUSE)
+
 
     def test_ports_equal(self):
         from acme.standalone import BaseDualNetworkedServers

certbot-apache/certbot_apache/_internal/apache_util.py

@@ -153,13 +153,10 @@ def parse_defines(apachectl):
         return {}
 
     for match in matches:
-        if match.count("=") > 1:
-            logger.error("Unexpected number of equal signs in "
-                         "runtime config dump.")
-            raise errors.PluginError(
-                "Error parsing Apache runtime variables")
-        parts = match.partition("=")
-        variables[parts[0]] = parts[2]
+        # Value could also contain = so split only once
+        parts = match.split('=', 1)
+        value = parts[1] if len(parts) == 2 else ''
+        variables[parts[0]] = value
 
     return variables
 

certbot-apache/certbot_apache/_internal/entrypoint.py

@@ -10,6 +10,7 @@ from certbot_apache._internal import override_debian
 from certbot_apache._internal import override_fedora
 from certbot_apache._internal import override_gentoo
 from certbot_apache._internal import override_suse
+from certbot_apache._internal import override_void
 
 OVERRIDE_CLASSES = {
     "arch": override_arch.ArchConfigurator,
@@ -35,6 +36,7 @@ OVERRIDE_CLASSES = {
     "sles": override_suse.OpenSUSEConfigurator,
     "scientific": override_centos.CentOSConfigurator,
     "scientific linux": override_centos.CentOSConfigurator,
+    "void": override_void.VoidConfigurator,
 }
 
 

certbot-apache/certbot_apache/_internal/http_01.py

@@ -95,10 +95,10 @@ class ApacheHttp01(common.ChallengePerformer):
     def _mod_config(self):
         selected_vhosts: List[VirtualHost] = []
         http_port = str(self.configurator.config.http01_port)
+
+        # Search for VirtualHosts matching by name
         for chall in self.achalls:
-            # Search for matching VirtualHosts
-            for vh in self._matching_vhosts(chall.domain):
-                selected_vhosts.append(vh)
+            selected_vhosts += self._matching_vhosts(chall.domain)
 
         # Ensure that we have one or more VirtualHosts that we can continue
         # with. (one that listens to port configured with --http-01-port)
@@ -107,9 +107,13 @@ class ApacheHttp01(common.ChallengePerformer):
             if any(a.is_wildcard() or a.get_port() == http_port for a in vhost.addrs):
                 found = True
 
-        if not found:
-            for vh in self._relevant_vhosts():
-                selected_vhosts.append(vh)
+        # If there's at least one elgible VirtualHost, also add all unnamed VirtualHosts
+        # because they might match at runtime (#8890)
+        if found:
+            selected_vhosts += self._unnamed_vhosts()
+        # Otherwise, add every Virtualhost which listens on the right port
+        else:
+            selected_vhosts += self._relevant_vhosts()
 
         # Add the challenge configuration
         for vh in selected_vhosts:
@@ -167,6 +171,10 @@ class ApacheHttp01(common.ChallengePerformer):
 
         return relevant_vhosts
 
+    def _unnamed_vhosts(self) -> List[VirtualHost]:
+        """Return all VirtualHost objects with no ServerName"""
+        return [vh for vh in self.configurator.vhosts if vh.name is None]
+
     def _set_up_challenges(self):
         if not os.path.isdir(self.challenge_dir):
             old_umask = filesystem.umask(0o022)

certbot-apache/certbot_apache/_internal/override_void.py

@@ -0,0 +1,23 @@
+""" Distribution specific override class for Void Linux """
+import zope.interface
+
+from certbot import interfaces
+from certbot_apache._internal import configurator
+from certbot_apache._internal.configurator import OsOptions
+
+
+@zope.interface.provider(interfaces.IPluginFactory)
+class VoidConfigurator(configurator.ApacheConfigurator):
+    """Void Linux specific ApacheConfigurator override class"""
+
+    OS_DEFAULTS = OsOptions(
+        server_root="/etc/apache",
+        vhost_root="/etc/apache/extra",
+        vhost_files="*.conf",
+        logs_root="/var/log/httpd",
+        ctl="apachectl",
+        version_cmd=['apachectl', '-v'],
+        restart_cmd=['apachectl', 'graceful'],
+        conftest_cmd=['apachectl', 'configtest'],
+        challenge_location="/etc/apache/extra",
+    )

certbot-apache/tests/http_01_test.py

@@ -125,6 +125,18 @@ class ApacheHttp01Test(util.ApacheTest):
                 domain="duplicate.example.com", account_key=self.account_key)]
         self.common_perform_test(achalls, vhosts)
 
+    def test_configure_name_and_blank(self):
+        domain = "certbot.demo"
+        vhosts = [v for v in self.config.vhosts if v.name == domain or v.name is None]
+        achalls = [
+            achallenges.KeyAuthorizationAnnotatedChallenge(
+                challb=acme_util.chall_to_challb(
+                    challenges.HTTP01(token=((b'a' * 16))),
+                    "pending"),
+                domain=domain, account_key=self.account_key),
+        ]
+        self.common_perform_test(achalls, vhosts)
+
     def test_no_vhost(self):
         for achall in self.achalls:
             self.http.add_chall(achall)

certbot-nginx/certbot_nginx/_internal/configurator.py

@@ -769,9 +769,6 @@ class NginxConfigurator(common.Installer):
         except (KeyError, ValueError):
             raise errors.PluginError(
                 "Unsupported enhancement: {0}".format(enhancement))
-        except errors.PluginError:
-            logger.error("Failed %s for %s", enhancement, domain)
-            raise
 
     def _has_certbot_redirect(self, vhost, domain):
         test_redirect_block = _test_block_from_block(_redirect_block_for_domain(domain))
@@ -791,6 +788,10 @@ class NginxConfigurator(common.Installer):
         :raises .errors.PluginError: If no viable HTTPS host can be created or
             set with header header_substring.
         """
+        if not header_substring in constants.HEADER_ARGS:
+            raise errors.NotSupportedError(
+                f"{header_substring} is not supported by the nginx plugin.")
+
         vhosts = self.choose_vhosts(domain)
         if not vhosts:
             raise errors.PluginError(

certbot-nginx/certbot_nginx/_internal/nginxparser.py

@@ -9,7 +9,6 @@ from pyparsing import Combine
 from pyparsing import Forward
 from pyparsing import Group
 from pyparsing import Literal
-from pyparsing import OneOrMore
 from pyparsing import Optional
 from pyparsing import QuotedString
 from pyparsing import Regex
@@ -57,7 +56,7 @@ class RawNginxParser:
     block_innards = Group(ZeroOrMore(contents) + space).leaveWhitespace()
     block << block_begin + left_bracket + block_innards + right_bracket
 
-    script = OneOrMore(contents) + space + stringEnd
+    script = ZeroOrMore(contents) + space + stringEnd
     script.parseWithTabs().leaveWhitespace()
 
     def __init__(self, source):

certbot-nginx/tests/configurator_test.py

@@ -43,7 +43,7 @@ class NginxConfiguratorTest(util.NginxTest):
 
     def test_prepare(self):
         self.assertEqual((1, 6, 2), self.config.version)
-        self.assertEqual(13, len(self.config.parser.parsed))
+        self.assertEqual(14, len(self.config.parser.parsed))
 
     @mock.patch("certbot_nginx._internal.configurator.util.exe_exists")
     @mock.patch("certbot_nginx._internal.configurator.subprocess.run")

certbot-nginx/tests/nginxparser_test.py

@@ -350,6 +350,10 @@ class TestRawNginxParser(unittest.TestCase):
         self.assertEqual(loads("blag${dfgdfg};"), [['blag${dfgdfg}']])
         self.assertRaises(ParseException, loads, "blag${dfgdf{g};")
 
+        # empty file
+        parsed = loads("")
+        self.assertEqual(parsed, [])
+
 
 class TestUnspacedList(unittest.TestCase):
     """Test the UnspacedList data structure"""

certbot-nginx/tests/parser_test.py

@@ -50,7 +50,7 @@ class NginxParserTest(util.NginxTest):
         nparser = parser.NginxParser(self.config_path)
         nparser.load()
         self.assertEqual({nparser.abs_path(x) for x in
-                          ['foo.conf', 'nginx.conf', 'server.conf',
+                          ['foo.conf', 'nginx.conf', 'server.conf', 'mime.types',
                            'sites-enabled/default',
                            'sites-enabled/both.com',
                            'sites-enabled/example.com',
@@ -89,7 +89,7 @@ class NginxParserTest(util.NginxTest):
         # pylint: disable=protected-access
         parsed = nparser._parse_files(nparser.abs_path(
             'sites-enabled/example.com.test'))
-        self.assertEqual(3, len(glob.glob(nparser.abs_path('*.test'))))
+        self.assertEqual(4, len(glob.glob(nparser.abs_path('*.test'))))
         self.assertEqual(10, len(
             glob.glob(nparser.abs_path('sites-enabled/*.test'))))
         self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'],

certbot/CHANGELOG.md

@@ -6,7 +6,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/).
 
 ### Added
 
-*
+* Add Void Linux overrides for certbot-apache.
 
 ### Changed
 
@@ -16,10 +16,17 @@ Certbot adheres to [Semantic Versioning](https://semver.org/).
   of the Certbot package will now always require acme>=X and version Y of a
   plugin package will always require acme>=Y and certbot=>Y. Specifying
   dependencies in this way simplifies testing and development.
+* The Apache authenticator now always configures virtual hosts which do not have
+  an explicit `ServerName`. This should make it work more reliably with the
+  default Apache configuration in Debian-based environments.
 
 ### Fixed
 
-*
+* When we increased the logging level on our nginx "Could not parse file" message,
+  it caused a previously-existing inability to parse empty files to become more
+  visible. We have now added the ability to correctly parse empty files, so that
+  message should only show for more significant errors.
+* Fixed parsing of `Define`d values the apache component to allow for `=` in the value.
 
 More details about these changes can be found on our GitHub repo.
 

certbot/certbot/_internal/cli/__init__.py

@@ -71,6 +71,11 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False):
         default=flag_default("verbose_count"), help="This flag can be used "
         "multiple times to incrementally increase the verbosity of output, "
         "e.g. -vvv.")
+    # This is for developers to set the level in the cli.ini, and overrides
+    # the --verbose flag
+    helpful.add(
+        None, "--verbose-level", dest="verbose_level",
+        default=flag_default("verbose_level"), help=argparse.SUPPRESS)
     helpful.add(
         None, "-t", "--text", dest="text_mode", action="store_true",
         default=flag_default("text_mode"), help=argparse.SUPPRESS)
@@ -449,6 +454,7 @@ def set_by_cli(var):
         plugins = plugins_disco.PluginsRegistry.find_all()
         # reconstructed_args == sys.argv[1:], or whatever was passed to main()
         reconstructed_args = helpful_parser.args + [helpful_parser.verb]
+
         detector = set_by_cli.detector = prepare_and_parse_args(  # type: ignore
             plugins, reconstructed_args, detect_defaults=True)
         # propagate plugin requests: eg --standalone modifies config.authenticator

certbot/certbot/_internal/client.py

@@ -2,7 +2,7 @@
 import datetime
 import logging
 import platform
-from typing import Optional
+from typing import List, Optional, Union
 
 from cryptography.hazmat.backends import default_backend
 # See https://github.com/pyca/cryptography/issues/4275
@@ -598,7 +598,8 @@ class Client:
             with error_handler.ErrorHandler(self._rollback_and_restart, msg):
                 self.installer.restart()
 
-    def apply_enhancement(self, domains, enhancement, options=None):
+    def apply_enhancement(self, domains: List[str], enhancement: str,
+                          options: Optional[Union[List[str], str]] = None) -> None:
         """Applies an enhancement on all domains.
 
         :param list domains: list of ssl_vhosts (as strings)
@@ -612,33 +613,28 @@ class Client:
 
 
         """
-        msg = f"Could not set up {enhancement} enhancement"
-        with error_handler.ErrorHandler(self._recovery_routine_with_msg, msg):
+        enh_label = options if enhancement == "ensure-http-header" else enhancement
+        with error_handler.ErrorHandler(self._recovery_routine_with_msg, None):
             for dom in domains:
                 try:
                     self.installer.enhance(dom, enhancement, options)
                 except errors.PluginEnhancementAlreadyPresent:
-                    if enhancement == "ensure-http-header":
-                        logger.info("Enhancement %s was already set.",
-                                options)
-                    else:
-                        logger.info("Enhancement %s was already set.",
-                                enhancement)
+                    logger.info("Enhancement %s was already set.", enh_label)
                 except errors.PluginError:
-                    logger.error("Unable to set enhancement %s for %s",
-                            enhancement, dom)
+                    logger.error("Unable to set the %s enhancement for %s.", enh_label, dom)
                     raise
 
-            self.installer.save("Add enhancement %s" % (enhancement))
+            self.installer.save(f"Add enhancement {enh_label}")
 
-    def _recovery_routine_with_msg(self, success_msg):
+    def _recovery_routine_with_msg(self, success_msg: Optional[str]) -> None:
         """Calls the installer's recovery routine and prints success_msg
 
         :param str success_msg: message to show on successful recovery
 
         """
         self.installer.recovery_routine()
-        display_util.notify(success_msg)
+        if success_msg:
+            display_util.notify(success_msg)
 
     def _rollback_and_restart(self, success_msg):
         """Rollback the most recent checkpoint and restart the webserver

certbot/certbot/_internal/constants.py

@@ -22,7 +22,8 @@ CLI_DEFAULTS = dict(
     ],
 
     # Main parser
-    verbose_count=-int(logging.WARNING / 10),
+    verbose_count=0,
+    verbose_level=None,
     text_mode=False,
     max_log_backups=1000,
     preconfigured_renewal=False,
@@ -142,6 +143,9 @@ REVOCATION_REASONS = {
 QUIET_LOGGING_LEVEL = logging.ERROR
 """Logging level to use in quiet mode."""
 
+DEFAULT_LOGGING_LEVEL = logging.WARNING
+"""Default logging level to use when not in quiet mode."""
+
 RENEWER_DEFAULTS = dict(
     renewer_enabled="yes",
     renew_before_expiry="30 days",

certbot/certbot/_internal/log.py

@@ -120,8 +120,11 @@ def post_arg_parse_setup(config):
 
     if config.quiet:
         level = constants.QUIET_LOGGING_LEVEL
+    elif config.verbose_level is not None:
+        level = constants.DEFAULT_LOGGING_LEVEL - int(config.verbose_level) * 10
     else:
-        level = -config.verbose_count * 10
+        level = constants.DEFAULT_LOGGING_LEVEL - config.verbose_count * 10
+
     stderr_handler.setLevel(level)
     logger.debug('Root logging level set at %d', level)
 

certbot/certbot/_internal/main.py

@@ -507,6 +507,13 @@ def _report_next_steps(config: interfaces.IConfig, installer_err: Optional[error
                 "Certificates created using --csr will not be renewed automatically by Certbot. "
                 "You will need to renew the certificate before it expires, by running the same "
                 "Certbot command again.")
+        elif _is_interactive_only_auth(config):
+            steps.append(
+                "This certificate will not be renewed automatically. Autorenewal of "
+                "--manual certificates requires the use of an authentication hook script "
+                "(--manual-auth-hook) but one was not provided. To renew this certificate, repeat "
+                f"this same {cli.cli_command} command before the certificate's expiry date."
+            )
         elif not config.preconfigured_renewal:
             steps.append(
                 "The certificate will need to be renewed before it expires. Certbot can "
@@ -556,6 +563,11 @@ def _report_new_cert(config, cert_path, fullchain_path, key_path=None):
 
     assert cert_path and fullchain_path, "No certificates saved to report."
 
+    renewal_msg = ""
+    if config.preconfigured_renewal and not _is_interactive_only_auth(config):
+        renewal_msg = ("\nCertbot has set up a scheduled task to automatically renew this "
+                       "certificate in the background.")
+
     display_util.notify(
         ("\nSuccessfully received certificate.\n"
         "Certificate is saved at: {cert_path}\n{key_msg}"
@@ -564,13 +576,22 @@ def _report_new_cert(config, cert_path, fullchain_path, key_path=None):
             cert_path=fullchain_path,
             expiry=crypto_util.notAfter(cert_path).date(),
             key_msg="Key is saved at:         {}\n".format(key_path) if key_path else "",
-            renewal_msg="\nCertbot has set up a scheduled task to automatically renew this "
-                        "certificate in the background." if config.preconfigured_renewal else "",
+            renewal_msg=renewal_msg,
             nl="\n" if config.verb == "run" else "" # Normalize spacing across verbs
         )
     )
 
 
+def _is_interactive_only_auth(config: interfaces.IConfig) -> bool:
+    """ Whether the current authenticator params only support interactive renewal.
+    """
+    # --manual without --manual-auth-hook can never autorenew
+    if config.authenticator == "manual" and config.manual_auth_hook is None:
+        return True
+
+    return False
+
+
 def _csr_report_new_cert(config: interfaces.IConfig, cert_path: Optional[str],
                          chain_path: Optional[str], fullchain_path: Optional[str]):
     """ --csr variant of _report_new_cert.
@@ -1398,7 +1419,7 @@ def certonly(config, plugins):
     if config.csr:
         cert_path, chain_path, fullchain_path = _csr_get_and_save_cert(config, le_client)
         _csr_report_new_cert(config, cert_path, chain_path, fullchain_path)
-        _report_next_steps(config, None, None)
+        _report_next_steps(config, None, None, new_or_renewed_cert=not config.dry_run)
         _suggest_donation_if_appropriate(config)
         eff.handle_subscription(config, le_client.account)
         return
@@ -1417,7 +1438,8 @@ def certonly(config, plugins):
     fullchain_path = lineage.fullchain_path if lineage else None
     key_path = lineage.key_path if lineage else None
     _report_new_cert(config, cert_path, fullchain_path, key_path)
-    _report_next_steps(config, None, lineage, new_or_renewed_cert=should_get_cert)
+    _report_next_steps(config, None, lineage,
+                       new_or_renewed_cert=should_get_cert and not config.dry_run)
     _suggest_donation_if_appropriate(config)
     eff.handle_subscription(config, le_client.account)
 

certbot/certbot/_internal/plugins/standalone.py

@@ -5,6 +5,7 @@ import logging
 import socket
 from typing import DefaultDict
 from typing import Dict
+from typing import List
 from typing import Set
 from typing import Tuple
 from typing import TYPE_CHECKING
@@ -184,6 +185,14 @@ class Authenticator(common.Plugin):
             if not self.served[servers]:
                 self.servers.stop(port)
 
+    def auth_hint(self, failed_achalls: List[achallenges.AnnotatedChallenge]) -> str:
+        port, addr = self.config.http01_port, self.config.http01_address
+        neat_addr = f"{addr}:{port}" if addr else f"port {port}"
+        return ("The Certificate Authority failed to download the challenge files from "
+                f"the temporary standalone webserver started by Certbot on {neat_addr}. "
+                "Ensure that the listed domains point to this machine and that it can "
+                "accept inbound connections from the internet.")
+
 
 def _handle_perform_error(error):
     if error.socket_error.errno == errno.EACCES:

certbot/docs/using.rst

@@ -57,10 +57,11 @@ standalone_ Y    N    | Uses a "standalone" webserver to obtain a certificate.
                       | domain. Doing domain validation in this way is
                       | the only way to obtain wildcard certificates from Let's
                       | Encrypt.
-manual_     Y    N    | Helps you obtain a certificate by giving you instructions to  http-01_ (80) or
-                      | perform domain validation yourself. Additionally allows you   dns-01_ (53)
-                      | to specify scripts to automate the validation task in a
-                      | customized way.
+manual_     Y    N    | Obtain a certificate by manually following instructions to    http-01_ (80) or
+                      | perform domain validation yourself. Certificates created this dns-01_ (53)
+                      | way do not support autorenewal.
+                      | Autorenewal may be enabled by providing an authentication
+                      | hook script to automate the domain validation steps.
 =========== ==== ==== =============================================================== =============================
 
 .. |dns_plugs| replace:: :ref:`DNS plugins <dns_plugins>`
@@ -229,11 +230,21 @@ For example, for the domain ``example.com``, a zone file entry would look like:
 
         _acme-challenge.example.com. 300 IN TXT "gfj9Xq...Rg85nM"
 
+.. _manual-renewal:
 
-Additionally you can specify scripts to prepare for validation and
-perform the authentication procedure and/or clean up after it by using
-the ``--manual-auth-hook`` and ``--manual-cleanup-hook`` flags. This is
-described in more depth in the hooks_ section.
+**Renewal with the manual plugin**
+
+Certificates created using ``--manual`` **do not** support automatic renewal unless
+combined with an `authentication hook script <#hooks>`_  via ``--manual-auth-hook``
+to automatically set up the required HTTP and/or TXT challenges.
+
+If you can use one of the other plugins_ which support autorenewal to create
+your certificate, doing so is highly recommended.
+
+To manually renew a certificate using ``--manual`` without hooks, repeat the same
+``certbot --manual`` command you used to create the certificate originally. As this
+will require you to copy and paste new HTTP files or DNS TXT records, the command
+cannot be automated with a cron job.
 
 .. _combination:
 
@@ -286,6 +297,10 @@ dns-lightsail_     Y    N    DNS Authentication using Amazon Lightsail DNS API
 dns-inwx_          Y    Y    DNS Authentication for INWX through the XML API
 dns-azure_         Y    N    DNS Authentication using Azure DNS
 dns-godaddy_       Y    N    DNS Authentication using Godaddy DNS
+njalla_            Y    N    DNS Authentication for njalla
+DuckDNS_           Y    N    DNS Authentication for DuckDNS
+Porkbun_           Y    N    DNS Authentication for Porkbun
+Infomaniak_        Y    N    DNS Authentication using Infomaniak Domains API
 ================== ==== ==== ===============================================================
 
 .. _haproxy: https://github.com/greenhost/certbot-haproxy
@@ -302,6 +317,10 @@ dns-godaddy_       Y    N    DNS Authentication using Godaddy DNS
 .. _dns-inwx: https://github.com/oGGy990/certbot-dns-inwx/
 .. _dns-azure: https://github.com/binkhq/certbot-dns-azure
 .. _dns-godaddy: https://github.com/miigotu/certbot-dns-godaddy
+.. _njalla: https://github.com/chaptergy/certbot-dns-njalla
+.. _DuckDNS: https://github.com/infinityofspace/certbot_dns_duckdns
+.. _Porkbun: https://github.com/infinityofspace/certbot_dns_porkbun
+.. _Infomaniak: https://github.com/Infomaniak/certbot-dns-infomaniak
 
 If you're interested, you can also :ref:`write your own plugin <dev-plugin>`.
 
@@ -522,6 +541,10 @@ Renewing certificates
 .. seealso:: Most Certbot installations come with automatic
    renewal out of the box. See `Automated Renewals`_ for more details.
 
+.. seealso:: Users of the `Manual`_ plugin should note that ``--manual`` certificates
+   will not renew automatically, unless combined with authentication hook scripts.
+   See `Renewal with the manual plugin <#manual-renewal>`_.
+
 As of version 0.10.0, Certbot supports a ``renew`` action to check
 all installed certificates for impending expiry and attempt to renew
 them. The simplest form is simply
@@ -710,7 +733,7 @@ Setting up automated renewal
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If you think you may need to set up automated renewal, follow these instructions to set up a
-scheduled task to automatically renew your certificates in the background. If you are unsure 
+scheduled task to automatically renew your certificates in the background. If you are unsure
 whether your system has a pre-installed scheduled task for Certbot, it is safe to follow these
 instructions to create one.
 

certbot/examples/dev-cli.ini

@@ -13,8 +13,6 @@ domains = example.com
 text = True
 agree-tos = True
 debug = True
-# Unfortunately, it's not possible to specify "verbose" multiple times
-# (correspondingly to -vvvvvv)
-verbose = True
+verbose-level = 2 # -vv (debug)
 
 authenticator = standalone

certbot/setup.py

@@ -67,22 +67,13 @@ install_requires = [
 ]
 
 dev_extras = [
-    'astroid',
     'azure-devops',
-    'coverage',
     'ipdb',
-    'mypy',
     'PyGithub',
-    # 1.1.0+ is required for poetry to use the poetry-core library for the
-    # build system declared in tools/pinning/pyproject.toml.
-    'poetry>=1.1.0',
-    'pylint',
-    'pytest',
-    'pytest-cov',
-    'pytest-xdist',
-    # typing-extensions is required to import typing.Protocol and make the mypy checks
-    # pass (along with pylint about non-existent objects) on Python 3.6 & 3.7
-    'typing-extensions',
+    'pip',
+    # poetry 1.2.0+ is required for it to pin pip, setuptools, and wheel. See
+    # https://github.com/python-poetry/poetry/issues/1584.
+    'poetry>=1.2.0a1',
     'tox',
     'twine',
     'wheel',
@@ -96,6 +87,21 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+test_extras = [
+    'coverage',
+    'mypy',
+    'pylint',
+    'pytest',
+    'pytest-cov',
+    'pytest-xdist',
+    # typing-extensions is required to import typing.Protocol and make the mypy checks
+    # pass (along with pylint about non-existent objects) on Python 3.6 & 3.7
+    'typing-extensions',
+]
+
+
+all_extras = dev_extras + docs_extras + test_extras
+
 setup(
     name='certbot',
     version=version,
@@ -132,8 +138,10 @@ setup(
 
     install_requires=install_requires,
     extras_require={
+        'all': all_extras,
         'dev': dev_extras,
         'docs': docs_extras,
+        'test': test_extras,
     },
 
     entry_points={

certbot/tests/client_test.py

@@ -712,7 +712,7 @@ class EnhanceConfigTest(ClientTestCommon):
 
         if enhance_error:
             self.assertEqual(mock_logger.error.call_count, 1)
-            self.assertIn('Unable to set enhancement', mock_logger.error.call_args_list[0][0][0])
+            self.assertEqual('Unable to set the %s enhancement for %s.', mock_logger.error.call_args_list[0][0][0])
         if restart_error:
             mock_logger.critical.assert_called_with(
                 'Rolling back to previous server configuration...')

certbot/tests/log_test.py

@@ -122,7 +122,7 @@ class PostArgParseSetupTest(test_util.ConfigTestCase):
         if self.config.quiet:
             self.assertEqual(level, constants.QUIET_LOGGING_LEVEL)
         else:
-            self.assertEqual(level, -self.config.verbose_count * 10)
+            self.assertEqual(level, constants.DEFAULT_LOGGING_LEVEL)
 
     def test_debug(self):
         self.config.debug = True

certbot/tests/main_test.py

@@ -271,6 +271,21 @@ class CertonlyTest(unittest.TestCase):
         self._call(('certonly --webroot --cert-name example.com').split())
         self.assertIs(mock_choose_names.called, True)
 
+    @mock.patch('certbot._internal.main._report_next_steps')
+    @mock.patch('certbot._internal.main._get_and_save_cert')
+    @mock.patch('certbot._internal.main._csr_get_and_save_cert')
+    @mock.patch('certbot._internal.cert_manager.lineage_for_certname')
+    def test_dryrun_next_steps_no_cert_saved(self, mock_lineage, mock_csr_get_cert,
+                                             unused_mock_get_cert, mock_report_next_steps):
+        """certonly --dry-run shouldn't report creation of a certificate in NEXT STEPS."""
+        mock_lineage.return_value = None
+        mock_csr_get_cert.return_value = ("/cert", "/chain", "/fullchain")
+        for flag in (f"--csr {CSR}", "-d example.com"):
+            self._call(f"certonly {flag} --webroot --cert-name example.com --dry-run".split())
+            mock_report_next_steps.assert_called_once_with(
+                mock.ANY, mock.ANY, mock.ANY, new_or_renewed_cert=False)
+            mock_report_next_steps.reset_mock()
+
 
 class FindDomainsOrCertnameTest(unittest.TestCase):
     """Tests for certbot._internal.main._find_domains_or_certname."""
@@ -1886,6 +1901,71 @@ class ReportNewCertTest(unittest.TestCase):
             'This certificate expires on 1970-01-01.'
         )
 
+    def test_manual_no_hooks_report(self):
+        """Shouldn't get a message about autorenewal if no --manual-auth-hook"""
+        self._call(mock.Mock(dry_run=False, authenticator='manual', manual_auth_hook=None),
+                  '/path/to/cert.pem', '/path/to/fullchain.pem',
+                  '/path/to/privkey.pem')
+
+        self.mock_notify.assert_called_with(
+            '\nSuccessfully received certificate.\n'
+            'Certificate is saved at: /path/to/fullchain.pem\n'
+            'Key is saved at:         /path/to/privkey.pem\n'
+            'This certificate expires on 1970-01-01.\n'
+            'These files will be updated when the certificate renews.'
+        )
+
+
+class ReportNextStepsTest(unittest.TestCase):
+    """Tests for certbot._internal.main._report_next_steps"""
+
+    def setUp(self):
+        self.config = mock.MagicMock(
+            cert_name="example.com", preconfigured_renewal=True,
+            csr=None, authenticator="nginx", manual_auth_hook=None)
+        notify_patch = mock.patch('certbot._internal.main.display_util.notify')
+        self.mock_notify = notify_patch.start()
+        self.addCleanup(notify_patch.stop)
+        self.old_stdout = sys.stdout
+        sys.stdout = io.StringIO()
+
+    def tearDown(self):
+        sys.stdout = self.old_stdout
+
+    @classmethod
+    def _call(cls, *args, **kwargs):
+        from certbot._internal.main import _report_next_steps
+        _report_next_steps(*args, **kwargs)
+
+    def _output(self) -> str:
+        self.mock_notify.assert_called_once()
+        return self.mock_notify.call_args_list[0][0][0]
+
+    def test_report(self):
+        """No steps for a normal renewal"""
+        self.config.authenticator = "manual"
+        self.config.manual_auth_hook = "/bin/true"
+        self._call(self.config, None, None)
+        self.mock_notify.assert_not_called()
+
+    def test_csr_report(self):
+        """--csr requires manual renewal"""
+        self.config.csr = "foo.csr"
+        self._call(self.config, None, None)
+        self.assertIn("--csr will not be renewed", self._output())
+
+    def test_manual_no_hook_renewal(self):
+        """--manual without a hook requires manual renewal"""
+        self.config.authenticator = "manual"
+        self._call(self.config, None, None)
+        self.assertIn("--manual certificates requires", self._output())
+
+    def test_no_preconfigured_renewal(self):
+        """No --preconfigured-renewal needs manual cron setup"""
+        self.config.preconfigured_renewal = False
+        self._call(self.config, None, None)
+        self.assertIn("https://certbot.org/renewal-setup", self._output())
+
 
 class UpdateAccountTest(test_util.ConfigTestCase):
     """Tests for certbot._internal.main.update_account"""

certbot/tests/plugins/standalone_test.py

@@ -177,6 +177,13 @@ class AuthenticatorTest(unittest.TestCase):
             "server1": set(), "server2": set()})
         self.auth.servers.stop.assert_called_with(2)
 
+    def test_auth_hint(self):
+        self.config.http01_port = "80"
+        self.config.http01_address = None
+        self.assertIn("on port 80", self.auth.auth_hint([]))
+        self.config.http01_address = "127.0.0.1"
+        self.assertIn("on 127.0.0.1:80", self.auth.auth_hint([]))
+
 
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

letstest/scripts/test_apache2.sh

@@ -45,7 +45,7 @@ if [ $? -ne 0 ] ; then
     exit 1
 fi
 
-tools/venv.py -e acme[dev] -e certbot[dev,docs] -e certbot-apache -e certbot-ci
+tools/venv.py -e acme -e certbot -e certbot-apache -e certbot-ci tox
 PEBBLE_LOGS="acme_server.log"
 PEBBLE_URL="https://localhost:14000/dir"
 # We configure Pebble to use port 80 for http-01 validation rather than an

tools/dev_constraints.txt

@@ -71,6 +71,7 @@ parso==0.7.0
 pathlib2==2.3.5
 pexpect==4.7.0
 pickleshare==0.7.5
+pip==20.2.4
 pkginfo==1.4.2
 pluggy==0.13.0
 ply==3.4
@@ -125,5 +126,6 @@ uritemplate==3.0.0
 virtualenv==16.6.2
 wcwidth==0.1.8
 websocket-client==0.56.0
+wheel==0.35.1
 wrapt==1.11.2
 zipp==0.6.0

tools/pinning/pin.sh

@@ -6,13 +6,11 @@ set -euo pipefail
 
 WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 REPO_ROOT="$(dirname "$(dirname "${WORK_DIR}")")"
-PIPSTRAP_CONSTRAINTS="${REPO_ROOT}/tools/pipstrap_constraints.txt"
 RELATIVE_SCRIPT_PATH="$(realpath --relative-to "$REPO_ROOT" "$WORK_DIR")/$(basename "${BASH_SOURCE[0]}")"
 REQUIREMENTS_FILE="$REPO_ROOT/tools/requirements.txt"
-STRIP_HASHES="${REPO_ROOT}/tools/strip_hashes.py"
 
-if ! command -v poetry >/dev/null; then
-    echo "Please install poetry."
+if ! command -v poetry >/dev/null || [ $(poetry --version | grep -oE '[0-9]+\.[0-9]+' | sed 's/\.//') -lt 12 ]; then
+    echo "Please install poetry 1.2+."
     echo "You may need to recreate Certbot's virtual environment and activate it."
     exit 1
 fi
@@ -37,13 +35,6 @@ trap 'rm poetry.lock; rm $TEMP_REQUIREMENTS' EXIT
 poetry export -o "${TEMP_REQUIREMENTS}" --without-hashes
 # We need to remove local packages from the requirements file.
 sed -i '/^acme @/d; /certbot/d;' "${TEMP_REQUIREMENTS}"
-# Poetry currently will not include pip, setuptools, or wheel in lockfiles or
-# requirements files. This was resolved by
-# https://github.com/python-poetry/poetry/pull/2826, but as of writing this it
-# hasn't been included in a release yet. For now, we continue to keep
-# pipstrap's pinning separate which has the added benefit of having it continue
-# to check hashes when pipstrap is run directly.
-"${STRIP_HASHES}" "${PIPSTRAP_CONSTRAINTS}" >>  "${TEMP_REQUIREMENTS}"
 
 cat << EOF > "$REQUIREMENTS_FILE"
 # This file was generated by $RELATIVE_SCRIPT_PATH and can be updated using

tools/pinning/pyproject.toml

@@ -12,8 +12,8 @@ python = "^3.6"
 # Any local packages that have dependencies on other local packages must be
 # listed below before the package it depends on. For instance, certbot depends
 # on acme so certbot must be listed before acme.
-certbot-ci = {path = "../../certbot-ci", extras = ["docs"]}
-certbot-compatibility-test = {path = "../../certbot-compatibility-test", extras = ["docs"]}
+certbot-ci = {path = "../../certbot-ci"}
+certbot-compatibility-test = {path = "../../certbot-compatibility-test"}
 certbot-dns-cloudflare = {path = "../../certbot-dns-cloudflare", extras = ["docs"]}
 certbot-dns-cloudxns = {path = "../../certbot-dns-cloudxns", extras = ["docs"]}
 certbot-dns-digitalocean = {path = "../../certbot-dns-digitalocean", extras = ["docs"]}
@@ -28,10 +28,10 @@ certbot-dns-ovh = {path = "../../certbot-dns-ovh", extras = ["docs"]}
 certbot-dns-rfc2136 = {path = "../../certbot-dns-rfc2136", extras = ["docs"]}
 certbot-dns-route53 = {path = "../../certbot-dns-route53", extras = ["docs"]}
 certbot-dns-sakuracloud = {path = "../../certbot-dns-sakuracloud", extras = ["docs"]}
-certbot-nginx = {path = "../../certbot-nginx", extras = ["docs"]}
+certbot-nginx = {path = "../../certbot-nginx"}
 certbot-apache = {path = "../../certbot-apache", extras = ["dev"]}
-certbot = {path = "../../certbot", extras = ["dev", "docs"]}
-acme = {path = "../../acme", extras = ["dev", "docs"]}
+certbot = {path = "../../certbot", extras = ["all"]}
+acme = {path = "../../acme", extras = ["docs", "test"]}
 letstest = {path = "../../letstest"}
 windows-installer = {path = "../../windows-installer"}
 
@@ -51,6 +51,12 @@ awscli = ">=1.19.62"
 # as a dependency here to ensure a version of cython is pinned for extra
 # stability.
 cython = "*"
+# mypy 0.900 stopped including stubs containing type information for 3rd party
+# libraries by default. This breaks our tests so let's continue to pin it back
+# for now. See
+# https://mypy-lang.blogspot.com/2021/05/the-upcoming-switch-to-modular-typeshed.html
+# for more info.
+mypy = "<0.900"
 # We install mock in our "external-mock" tox environment to test that we didn't
 # break Certbot's test API which used to always use mock objects from the 3rd
 # party mock library. We list the mock dependency here so that is pinned, but
@@ -58,6 +64,15 @@ cython = "*"
 # needed. This dependency can be removed here once Certbot's support for the
 # 3rd party mock library has been dropped.
 mock = "*"
+# pip's new dependency resolver fails on local packages that depend on each
+# other when those packages are requested with extras such as 'certbot[dev]' so
+# let's pin it back for now. See https://github.com/pypa/pip/issues/9204.
+pip = "20.2.4"
+# poetry 1.2.0+ is required for it to pin pip, setuptools, and wheel. See
+# https://github.com/python-poetry/poetry/issues/1584. This version is required
+# here in addition to certbot/setup.py because otherwise the pre-release
+# version of poetry will not be installed.
+poetry = ">=1.2.0a1"
 # We were originally pinning back python-augeas for certbot-auto because we
 # found the way older versions of the library linked to Augeas were more
 # reliable. That's no longer a concern, however, we continue to pin back the

tools/pip_install.py

@@ -16,7 +16,6 @@ import tempfile
 
 import merge_requirements as merge_module
 import readlink
-import strip_hashes
 
 
 # Once this code doesn't need to support Python 2, we can simply use

tools/pipstrap.py

@@ -1,15 +1,10 @@
 #!/usr/bin/env python
 """Uses pip to upgrade Python packaging tools to pinned versions."""
-import os
-
 import pip_install
 
-_REQUIREMENTS_PATH = os.path.join(os.path.dirname(__file__), "pipstrap_constraints.txt")
-
 
 def main():
-    pip_install_args = '--requirement "{0}"'.format(_REQUIREMENTS_PATH)
-    pip_install.pip_install_with_print(pip_install_args)
+    pip_install.main('pip setuptools wheel'.split())
 
 
 if __name__ == '__main__':

tools/pipstrap_constraints.txt

@@ -1,18 +0,0 @@
-# Constraints for pipstrap.py
-#
-# We include the hashes of the packages here for extra verification of
-# the packages downloaded from PyPI. This is especially valuable in our
-# builds of Certbot that we ship to our users such as our Docker images.
-#
-# An older version of setuptools is currently used here in order to keep
-# compatibility with Python 2 since newer versions of setuptools have dropped
-# support for it.
-pip==20.2.4 \
-    --hash=sha256:51f1c7514530bd5c145d8f13ed936ad6b8bfcb8cf74e10403d0890bc986f0033 \
-    --hash=sha256:85c99a857ea0fb0aedf23833d9be5c40cf253fe24443f0829c7b472e23c364a1
-setuptools==54.1.2 \
-    --hash=sha256:dd20743f36b93cbb8724f4d2ccd970dce8b6e6e823a13aa7e5751bb4e674c20b \
-    --hash=sha256:ebd0148faf627b569c8d2a1b20f5d3b09c873f12739d71c7ee88f037d5be82ff
-wheel==0.35.1 \
-    --hash=sha256:497add53525d16c173c2c1c733b8f655510e909ea78cc0e29d374243544b77a2 \
-    --hash=sha256:99a22d87add3f634ff917310a3d87e499f19e663413a52eb9232c447aa646c9f

tools/requirements.txt

@@ -8,148 +8,150 @@
 alabaster==0.7.12; python_version >= "3.6"
 apacheconfig==0.3.2; python_version >= "3.6"
 apipkg==1.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-appdirs==1.4.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-appnope==0.1.2
+appdirs==1.4.4; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0"
+appnope==0.1.2; python_version == "3.6" and sys_platform == "darwin" or python_version >= "3.7" and sys_platform == "darwin"
 astroid==2.5.6; python_version >= "3.6" and python_version < "4.0"
 atomicwrites==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.4.0"
 attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-awscli==1.19.83; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
+awscli==1.19.91; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
 azure-devops==6.0.0b4; python_version >= "3.6"
 babel==2.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
-backcall==0.2.0
+backcall==0.2.0; python_version == "3.6" or python_version >= "3.7"
 bcrypt==3.2.0; python_version >= "3.6"
-beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0"
+beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0" or python_version >= "3.6"
 bleach==3.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-boto3==1.17.83; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-botocore==1.20.83; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
+boto3==1.17.91; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+botocore==1.20.91; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0"
 cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 cachetools==4.2.2; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
-cachy==0.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-certifi==2020.12.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
+cachy==0.3.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0"
+certifi==2021.5.30; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6"
+cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" or python_version >= "3.6"
 chardet==4.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-cleo==0.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-clikit==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
+cleo==1.0.0a3; python_version >= "3.6" and python_version < "4.0"
 cloudflare==2.8.15; python_version >= "3.6"
-colorama==0.4.3; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
+colorama==0.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" and sys_platform == "win32" or python_version >= "3.6" and python_version < "4.0" and sys_platform == "win32" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" and platform_system == "Windows" or python_version >= "3.6" and python_full_version >= "3.5.0" and platform_system == "Windows" or python_version >= "3.6" and python_full_version >= "3.5.0" or python_version == "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or python_version == "3.6" and sys_platform == "win32" and python_full_version >= "3.5.0" or python_version >= "3.7" and python_full_version < "3.0.0" and sys_platform == "win32" or python_version >= "3.7" and sys_platform == "win32" and python_full_version >= "3.5.0"
 configargparse==1.4.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 configobj==5.0.6; python_version >= "3.6"
 coverage==5.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6"
-crashtest==0.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0")
-cryptography==3.4.7; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux"
+crashtest==0.3.1; python_version >= "3.6" and python_version < "4.0"
+cryptography==3.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6" or python_version >= "3.6" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and sys_platform == "linux"
 cython==0.29.23; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
-decorator==5.0.9
+dataclasses==0.8; python_version >= "3.6" and python_version < "3.7"
+decorator==5.0.9; python_version == "3.6" or python_version > "3.6" or python_version >= "3.5" or python_version >= "3.7"
 deprecated==1.2.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
-distlib==0.3.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-distro==1.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+distlib==0.3.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" or python_version >= "3.6"
+distro==1.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_version >= "3.6"
 dns-lexicon==3.6.0; python_version >= "3.6" and python_version < "4.0"
 dnspython==2.1.0; python_version >= "3.6"
 docker-compose==1.26.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 docker==4.2.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 dockerpty==0.4.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 docopt==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-docutils==0.15.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
+docutils==0.15.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.3.0"
+entrypoints==0.3; python_version >= "3.6" and python_version < "4.0"
 execnet==1.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 fabric==2.6.0; python_version >= "3.6"
-filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "4.0"
-google-api-core==1.28.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-google-api-python-client==2.6.0; python_version >= "3.6"
+filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" or python_version >= "3.6" and python_version < "4.0"
+google-api-core==1.30.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+google-api-python-client==2.8.0; python_version >= "3.6"
 google-auth-httplib2==0.1.0; python_version >= "3.6"
-google-auth==1.30.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+google-auth==1.31.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-html5lib==1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
+html5lib==1.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0"
 httplib2==0.19.1; python_version >= "3.6"
-idna==2.10; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4.0"
+idna==2.10; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0"
 imagesize==1.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
-importlib-metadata==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") or python_version < "3.8" and python_version >= "3.6" and python_full_version >= "3.5.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0")
-importlib-resources==5.1.4; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.7"
+importlib-metadata==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_version >= "3.6" and python_version < "3.8" and python_full_version >= "3.5.0"
+importlib-resources==5.1.4; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_version < "3.7" and python_full_version >= "3.4.0"
 iniconfig==1.1.1; python_version >= "3.6"
 invoke==1.5.0; python_version >= "3.6"
-ipdb==0.13.8; python_version >= "3.6"
-ipython-genutils==0.2.0; python_version == "3.6"
+ipdb==0.13.9; python_version >= "3.6"
+ipython-genutils==0.2.0
 ipython==7.16.1; python_version == "3.6"
-ipython==7.24.0; python_version >= "3.7"
+ipython==7.24.1; python_version >= "3.7"
 isodate==0.6.0; python_version >= "3.6"
 isort==5.8.0; python_version >= "3.6" and python_version < "4.0"
-jedi==0.18.0
-jeepney==0.6.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux"
-jinja2==3.0.1; python_version >= "3.6"
+jedi==0.18.0; python_version == "3.6" or python_version >= "3.7"
+jeepney==0.6.0; python_version >= "3.6" and python_version < "4.0" and sys_platform == "linux"
+jinja2==3.0.1; python_version >= "3.6" or python_version >= "3.6"
 jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 josepy==1.8.0; python_version >= "3.6"
 jsonlines==2.0.0; python_version >= "3.6"
 jsonpickle==2.0.0; python_version >= "3.6"
 jsonschema==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-keyring==21.8.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0")
+keyring==22.3.0; python_version >= "3.6" and python_version < "4.0" or python_version >= "3.6"
 lazy-object-proxy==1.6.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0"
 lockfile==0.12.2
 markupsafe==2.0.1; python_version >= "3.6"
 matplotlib-inline==0.1.2; python_version >= "3.7"
 mccabe==0.6.1; python_version >= "3.6" and python_version < "4.0"
 mock==4.0.3; python_version >= "3.6"
-msgpack==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
+msgpack==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0"
 msrest==0.6.21; python_version >= "3.6"
 mypy-extensions==0.4.3; python_version >= "3.6"
-mypy==0.812; python_version >= "3.6"
+mypy==0.812; python_version >= "3.5"
 oauth2client==4.1.3; python_version >= "3.6"
-oauthlib==3.1.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
-packaging==20.9; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0"
-paramiko==2.7.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+oauthlib==3.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
+packaging==20.9; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" or python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" or python_version >= "3.6" and python_full_version >= "3.5.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+paramiko==2.7.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6"
 parsedatetime==2.6; python_version >= "3.6"
 parso==0.8.2; python_version == "3.6"
-pastel==0.2.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 pathlib2==2.3.5; python_version >= "3.6"
-pexpect==4.8.0
-pickleshare==0.7.5
-pkginfo==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-pluggy==0.13.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+pexpect==4.8.0; python_version >= "3.6" and python_version < "4.0" or python_version == "3.6" and sys_platform != "win32" or python_version >= "3.7" and sys_platform != "win32"
+pickleshare==0.7.5; python_version == "3.6" or python_version >= "3.7"
+pip==20.2.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
+pkginfo==1.7.0; python_version >= "3.6" and python_version < "4.0" or python_version >= "3.6"
+pluggy==0.13.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.5.0"
 ply==3.11; python_version >= "3.6"
-poetry-core==1.0.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-poetry==1.1.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-prompt-toolkit==3.0.3
-protobuf==3.17.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-ptyprocess==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-pyasn1==0.4.8; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
+poetry-core==1.1.0a5; python_version >= "3.6" and python_version < "4.0"
+poetry==1.2.0a1; python_version >= "3.6" and python_version < "4.0"
+prompt-toolkit==3.0.3; python_version == "3.6" or python_version >= "3.7"
+protobuf==3.17.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+ptyprocess==0.7.0; python_version >= "3.6" and python_version < "4.0"
+py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.5.0"
+pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version >= "3.6"
+pyasn1==0.4.8; python_version >= "3.5" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") or python_version >= "3.6"
 pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 pygithub==1.55; python_version >= "3.6"
-pygments==2.9.0
+pygments==2.9.0; python_version >= "3.6" or python_version == "3.6" or python_version >= "3.7"
 pyjwt==2.1.0; python_version >= "3.6"
-pylev==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-pylint==2.8.2; python_version >= "3.6" and python_version < "4.0"
+pylev==1.4.0; python_version >= "3.6" and python_version < "4.0"
+pylint==2.8.3; python_version >= "3.6" and python_version < "4.0"
 pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
 pynsist==2.7; python_version >= "3.6"
 pyopenssl==20.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-pyparsing==2.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+pyparsing==2.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0"
 pypiwin32==223; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6")
 pyrfc3339==1.1; python_version >= "3.6"
 pyrsistent==0.17.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-pytest-cov==2.12.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+pytest-cov==2.12.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 pytest-forked==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-pytest-xdist==2.2.1; python_version >= "3.6"
-pytest==6.2.4; python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6")
+pytest-xdist==2.2.1; python_version >= "3.6" or python_version >= "3.6"
+pytest==6.2.4; python_version >= "3.6" or python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 python-augeas==0.5.0
-python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" or python_full_version >= "3.6.0" and python_version >= "3.6"
 python-digitalocean==1.16.0; python_version >= "3.6"
 python-dotenv==0.17.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0"
-pywin32-ctypes==0.2.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "win32"
-pywin32==300; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6")
-pyyaml==5.4.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.6.0" and python_version >= "3.6" and python_version < "4.0"
+pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" or python_version >= "3.6" or python_full_version >= "3.6.0" and python_version >= "3.6"
+pywin32-ctypes==0.2.0; python_version >= "3.6" and python_version < "4.0" and sys_platform == "win32"
+pywin32==301; sys_platform == "win32" and python_version >= "3.6" or sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6")
+pyyaml==5.4.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0"
 readme-renderer==29.0; python_version >= "3.6"
 repoze.sphinx.autointerface==0.8; python_version >= "3.6"
 requests-download==0.1.2; python_version >= "3.6"
 requests-file==1.5.1; python_version >= "3.6" and python_version < "4.0"
 requests-oauthlib==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
-requests-toolbelt==0.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-requests==2.25.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.6.0" and python_version < "4.0"
+requests-toolbelt==0.9.1; python_version >= "3.6" and python_version < "4.0" or python_version >= "3.6" or python_version >= "3.6"
+requests==2.25.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 rfc3986==1.5.0; python_version >= "3.6"
-rsa==4.7.2; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
+rsa==4.7.2; python_version >= "3.5" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") or python_version >= "3.6" and python_version < "4"
 s3transfer==0.4.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-secretstorage==3.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux"
-shellingham==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-six==1.16.0; python_version == "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version == "3.6"
+secretstorage==3.3.1; python_version >= "3.6" and python_version < "4.0" and sys_platform == "linux"
+setuptools==57.0.0; python_version >= "3.6" or python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version == "3.6" or python_version >= "3.7"
+shellingham==1.4.0; python_version >= "3.6" and python_version < "4.0"
+six==1.16.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_full_version >= "3.3.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0" or python_version == "3.6" and python_full_version < "3.0.0" or python_version == "3.6" and python_full_version >= "3.3.0"
 snowballstemmer==2.1.0; python_version >= "3.6"
 soupsieve==2.2.1; python_version >= "3.6"
 sphinx-rtd-theme==0.5.2; python_version >= "3.6"
@@ -162,27 +164,25 @@ sphinxcontrib-qthelp==1.0.3; python_version >= "3.6"
 sphinxcontrib-serializinghtml==1.1.5; python_version >= "3.6"
 texttable==1.6.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 tldextract==3.1.0; python_version >= "3.6" and python_version < "4.0"
-toml==0.10.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0"
-tomlkit==0.7.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
+toml==0.10.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version == "3.6" and python_full_version < "3.0.0" or python_version > "3.6" and python_full_version < "3.0.0" or python_version == "3.6" and python_full_version >= "3.3.0" or python_version > "3.6" and python_full_version >= "3.3.0" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0"
+tomlkit==0.7.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0"
 tox==3.23.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 tqdm==4.61.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
 traitlets==4.3.3
 twine==3.3.0; python_version >= "3.6"
-typed-ast==1.4.3; implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6"
+typed-ast==1.4.3; python_version >= "3.6" or implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6"
 typing-extensions==3.10.0.0; python_version >= "3.6"
 uritemplate==3.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-urllib3==1.26.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6"
-virtualenv==20.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
+urllib3==1.26.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6"
+virtualenv==20.4.4; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" or python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 wcwidth==0.2.5; python_version == "3.6"
-webencodings==0.5.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-websocket-client==0.59.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-wrapt==1.12.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0")
+webencodings==0.5.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
+websocket-client==0.59.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_full_version >= "3.5.0" and python_version >= "3.6"
+wheel==0.36.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
+wrapt==1.12.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" or python_version >= "3.6" and python_version < "4.0"
 yarg==0.1.9; python_version >= "3.6"
-zipp==3.4.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version < "3.7" and python_version >= "3.6" and python_full_version >= "3.5.0"
+zipp==3.4.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_version >= "3.6" and python_version < "3.8" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_version < "3.7" and python_full_version >= "3.4.0"
 zope.component==5.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 zope.event==4.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 zope.hookable==5.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 zope.interface==5.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-pip==20.2.4
-setuptools==54.1.2
-wheel==0.35.1

tools/strip_hashes.py

@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-"""Removes hash information from requirement files passed to it as file path
-arguments or simply piped to stdin."""
-
-import re
-import sys
-
-
-def process_entries(entries):
-    """Strips off hash strings from dependencies.
-
-    :param list entries: List of entries
-
-    :returns: list of dependencies without hashes
-    :rtype: list
-    """
-    out_lines = []
-    for e in entries:
-        e = e.strip()
-        search = re.search(r'^(\S*==\S*).*$', e)
-        if search:
-            out_lines.append(search.group(1))
-    return out_lines
-
-def main(*paths):
-    """
-    Reads dependency definitions from a (list of) file(s) provided on the
-    command line. If no command line arguments are present, data is read from
-    stdin instead.
-
-    Hashes are removed from returned entries.
-    """
-
-    deps = []
-    if paths:
-        for path in paths:
-            with open(path) as file_h:
-                deps += process_entries(file_h.readlines())
-    else:
-        # Need to check if interactive to avoid blocking if nothing is piped
-        if not sys.stdin.isatty():
-            stdin_data = []
-            for line in sys.stdin:
-                stdin_data.append(line)
-            deps += process_entries(stdin_data)
-
-    return "\n".join(deps)
-
-if __name__ == '__main__':
-    print(main(*sys.argv[1:]))  # pylint: disable=star-args

tools/venv.py

@@ -24,8 +24,8 @@ import sys
 import time
 
 REQUIREMENTS = [
-    '-e acme[dev]',
-    '-e certbot[dev,docs]',
+    '-e acme[test]',
+    '-e certbot[all]',
     '-e certbot-apache',
     '-e certbot-dns-cloudflare',
     '-e certbot-dns-cloudxns',

tox.ini

@@ -19,7 +19,7 @@ install_packages = python {toxinidir}/tools/pip_install_editable.py {[base]all_p
 # behavior with substitutions that contain line continuations, see
 # https://github.com/tox-dev/tox/issues/2069 for more info.
 dns_packages = certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud
-all_packages = acme[dev] certbot[dev] certbot-apache {[base]dns_packages} certbot-nginx
+all_packages = acme[test] certbot[test] certbot-apache {[base]dns_packages} certbot-nginx
 source_paths = acme/acme certbot/certbot certbot-ci/certbot_integration_tests certbot-apache/certbot_apache certbot-compatibility-test/certbot_compatibility_test certbot-dns-cloudflare/certbot_dns_cloudflare certbot-dns-cloudxns/certbot_dns_cloudxns certbot-dns-digitalocean/certbot_dns_digitalocean certbot-dns-dnsimple/certbot_dns_dnsimple certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy certbot-dns-gehirn/certbot_dns_gehirn certbot-dns-google/certbot_dns_google certbot-dns-linode/certbot_dns_linode certbot-dns-luadns/certbot_dns_luadns certbot-dns-nsone/certbot_dns_nsone certbot-dns-ovh/certbot_dns_ovh certbot-dns-rfc2136/certbot_dns_rfc2136 certbot-dns-route53/certbot_dns_route53 certbot-dns-sakuracloud/certbot_dns_sakuracloud certbot-nginx/certbot_nginx tests/lock_test.py
 
 [testenv]
@@ -54,7 +54,7 @@ setenv =
 basepython =
     {[testenv:oldest]basepython}
 commands =
-    {[base]install_and_test} acme[dev]
+    {[base]install_and_test} acme[test]
 setenv =
     {[testenv:oldest]setenv}
 
@@ -62,7 +62,7 @@ setenv =
 basepython =
     {[testenv:oldest]basepython}
 commands =
-    {[base]pip_install} acme[dev] certbot[dev] certbot-apache
+    {[base]pip_install} acme[test] certbot[test] certbot-apache
     pytest certbot-apache
 setenv =
     {[testenv:oldest]setenv}
@@ -71,7 +71,7 @@ setenv =
 basepython =
     {[testenv:oldest]basepython}
 commands =
-    {[base]pip_install} acme[dev] certbot[dev] certbot-apache[dev]
+    {[base]pip_install} acme[test] certbot[test] certbot-apache[dev]
     pytest certbot-apache
 setenv =
     {[testenv:oldest]setenv}
@@ -80,7 +80,7 @@ setenv =
 basepython =
     {[testenv:oldest]basepython}
 commands =
-    {[base]pip_install} acme[dev] certbot[dev]
+    {[base]pip_install} acme[test] certbot[test]
     pytest certbot
 setenv =
     {[testenv:oldest]setenv}
@@ -89,7 +89,7 @@ setenv =
 basepython =
     {[testenv:oldest]basepython}
 commands =
-    {[base]pip_install} acme[dev] certbot[dev] {[base]dns_packages}
+    {[base]pip_install} acme[test] certbot[test] {[base]dns_packages}
     pytest {[base]dns_packages}
 setenv =
     {[testenv:oldest]setenv}
@@ -98,7 +98,7 @@ setenv =
 basepython =
     {[testenv:oldest]basepython}
 commands =
-    {[base]pip_install} acme[dev] certbot[dev] certbot-nginx
+    {[base]pip_install} acme[test] certbot[test] certbot-nginx
     pytest certbot-nginx
     python tests/lock_test.py
 setenv =