rewrite coverage tests

This reverts commit 561037bfad.

.azure-pipelines/templates/jobs/extended-tests-jobs.yml

@@ -17,6 +17,8 @@ jobs:
         linux-py310:
           PYTHON_VERSION: 3.10
           TOXENV: py310
+        linux-isolated:
+          TOXENV: 'isolated-{acme,certbot,apache,cloudflare,digitalocean,dnsimple,dnsmadeeasy,gehirn,google,linode,luadns,nsone,ovh,rfc2136,route53,sakuracloud,nginx}'
         linux-boulder-v2-integration-certbot-oldest:
           PYTHON_VERSION: 3.7
           TOXENV: integration-certbot-oldest

.azure-pipelines/templates/jobs/standard-tests-jobs.yml

@@ -14,7 +14,7 @@ jobs:
         windows-py37:
           IMAGE_NAME: windows-2019
           PYTHON_VERSION: 3.7
-          TOXENV: py37-win
+          TOXENV: py-win
         windows-py39-cover:
           IMAGE_NAME: windows-2019
           PYTHON_VERSION: 3.9
@@ -23,14 +23,10 @@ jobs:
           IMAGE_NAME: windows-2019
           PYTHON_VERSION: 3.9
           TOXENV: integration-certbot
-        linux-oldest-tests-1:
+        linux-oldest:
           IMAGE_NAME: ubuntu-22.04
           PYTHON_VERSION: 3.7
-          TOXENV: '{acme,apache,apache-v2,certbot}-oldest'
-        linux-oldest-tests-2:
-          IMAGE_NAME: ubuntu-22.04
-          PYTHON_VERSION: 3.7
-          TOXENV: '{dns,nginx}-oldest'
+          TOXENV: oldest
         linux-py37:
           IMAGE_NAME: ubuntu-22.04
           PYTHON_VERSION: 3.7
@@ -43,7 +39,7 @@ jobs:
           TOXENV: lint-posix
         linux-mypy:
           IMAGE_NAME: ubuntu-22.04
-          TOXENV: mypy-posix
+          TOXENV: mypy
         linux-integration:
           IMAGE_NAME: ubuntu-22.04
           PYTHON_VERSION: 3.8

.coveragerc

@@ -1,5 +1,24 @@
 [run]
 omit = */setup.py
+source =
+    acme
+    certbot
+    certbot-apache
+    certbot-dns-cloudflare
+    certbot-dns-digitalocean
+    certbot-dns-dnsimple
+    certbot-dns-dnsmadeeasy
+    certbot-dns-gehirn
+    certbot-dns-google
+    certbot-dns-linode
+    certbot-dns-luadns
+    certbot-dns-nsone
+    certbot-dns-ovh
+    certbot-dns-rfc2136
+    certbot-dns-route53
+    certbot-dns-sakuracloud
+    certbot-nginx
 
 [report]
 omit = */setup.py
+show_missing = True

acme/acme/_internal/tests/crypto_util_test.py

@@ -29,11 +29,9 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
         from acme.crypto_util import SSLSocket
 
         class _TestServer(socketserver.TCPServer):
-
-            def server_bind(self):  # pylint: disable=missing-docstring
-                self.socket = SSLSocket(socket.socket(),
-                        certs)
-                socketserver.TCPServer.server_bind(self)
+            def __init__(self, *args, **kwargs):
+                super().__init__(*args, **kwargs)
+                self.socket = SSLSocket(self.socket, certs)
 
         self.server = _TestServer(('', 0), socketserver.BaseRequestHandler)
         self.port = self.server.socket.getsockname()[1]
@@ -44,6 +42,7 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
         if self.server_thread.is_alive():
             # The thread may have already terminated.
             self.server_thread.join()  # pragma: no cover
+        self.server.server_close()
 
     def _probe(self, name):
         from acme.crypto_util import probe_sni

acme/acme/_internal/tests/standalone_test.py

@@ -55,6 +55,7 @@ class HTTP01ServerTest(unittest.TestCase):
     def tearDown(self):
         self.server.shutdown()
         self.thread.join()
+        self.server.server_close()
 
     def test_index(self):
         response = requests.get(
@@ -88,25 +89,25 @@ class HTTP01ServerTest(unittest.TestCase):
 
     def test_timely_shutdown(self):
         from acme.standalone import HTTP01Server
-        server = HTTP01Server(('', 0), resources=set(), timeout=0.05)
-        server_thread = threading.Thread(target=server.serve_forever)
-        server_thread.start()
+        with HTTP01Server(('', 0), resources=set(), timeout=0.05) as server:
+            server_thread = threading.Thread(target=server.serve_forever)
+            server_thread.start()
 
-        client = socket.socket()
-        client.connect(('localhost', server.socket.getsockname()[1]))
+            with socket.socket() as client:
+                client.connect(('localhost', server.socket.getsockname()[1]))
 
-        stop_thread = threading.Thread(target=server.shutdown)
-        stop_thread.start()
-        server_thread.join(5.)
+                stop_thread = threading.Thread(target=server.shutdown)
+                stop_thread.start()
+                server_thread.join(5.)
 
-        is_hung = server_thread.is_alive()
-        try:
-            client.shutdown(socket.SHUT_RDWR)
-        except: # pragma: no cover, pylint: disable=bare-except
-            # may raise error because socket could already be closed
-            pass
+                is_hung = server_thread.is_alive()
+                try:
+                    client.shutdown(socket.SHUT_RDWR)
+                except: # pragma: no cover, pylint: disable=bare-except
+                    # may raise error because socket could already be closed
+                    pass
 
-        assert not is_hung, 'Server shutdown should not be hung'
+                assert not is_hung, 'Server shutdown should not be hung'
 
 
 @unittest.skipIf(not challenges.TLSALPN01.is_supported(), "pyOpenSSL too old")
@@ -133,6 +134,7 @@ class TLSALPN01ServerTest(unittest.TestCase):
     def tearDown(self):
         self.server.shutdown()  # pylint: disable=no-member
         self.thread.join()
+        self.server.server_close()
 
     # TODO: This is not implemented yet, see comments in standalone.py
     # def test_certs(self):
@@ -214,6 +216,8 @@ class BaseDualNetworkedServersTest(unittest.TestCase):
             if prev_port:
                 assert prev_port == port
             prev_port = port
+        for server in servers.servers:
+            server.server_close()
 
 
 class HTTP01DualNetworkedServersTest(unittest.TestCase):

acme/acme/crypto_util.py

@@ -136,27 +136,33 @@ class SSLSocket:  # pylint: disable=too-few-public-methods
     def accept(self) -> Tuple[FakeConnection, Any]:  # pylint: disable=missing-function-docstring
         sock, addr = self.sock.accept()
 
-        context = SSL.Context(self.method)
-        context.set_options(SSL.OP_NO_SSLv2)
-        context.set_options(SSL.OP_NO_SSLv3)
-        context.set_tlsext_servername_callback(self._pick_certificate_cb)
-        if self.alpn_selection is not None:
-            context.set_alpn_select_callback(self.alpn_selection)
-
-        ssl_sock = self.FakeConnection(SSL.Connection(context, sock))
-        ssl_sock.set_accept_state()
-
-        # This log line is especially desirable because without it requests to
-        # our standalone TLSALPN server would not be logged.
-        logger.debug("Performing handshake with %s", addr)
         try:
-            ssl_sock.do_handshake()
-        except SSL.Error as error:
-            # _pick_certificate_cb might have returned without
-            # creating SSL context (wrong server name)
-            raise socket.error(error)
+            context = SSL.Context(self.method)
+            context.set_options(SSL.OP_NO_SSLv2)
+            context.set_options(SSL.OP_NO_SSLv3)
+            context.set_tlsext_servername_callback(self._pick_certificate_cb)
+            if self.alpn_selection is not None:
+                context.set_alpn_select_callback(self.alpn_selection)
 
-        return ssl_sock, addr
+            ssl_sock = self.FakeConnection(SSL.Connection(context, sock))
+            ssl_sock.set_accept_state()
+
+            # This log line is especially desirable because without it requests to
+            # our standalone TLSALPN server would not be logged.
+            logger.debug("Performing handshake with %s", addr)
+            try:
+                ssl_sock.do_handshake()
+            except SSL.Error as error:
+                # _pick_certificate_cb might have returned without
+                # creating SSL context (wrong server name)
+                raise socket.error(error)
+
+            return ssl_sock, addr
+        except:
+            # If we encounter any error, close the new socket before reraising
+            # the exception.
+            sock.close()
+            raise
 
 
 def probe_sni(name: bytes, host: bytes, port: int = 443, timeout: int = 300,  # pylint: disable=too-many-arguments

certbot-apache/certbot_apache/_internal/tests/autohsts_test.py

@@ -128,11 +128,11 @@ class AutoHSTSTest(util.ApacheTest):
                           max_val
 
     def test_autohsts_update_noop(self):
-        with mock.patch("time.time") as mock_time:
+        with mock.patch("certbot_apache._internal.configurator.time") as mock_time_module:
             # Time mock is used to make sure that the execution does not
             # continue when no autohsts entries exist in pluginstorage
             self.config.update_autohsts(mock.MagicMock())
-            assert mock_time.called is False
+            assert not mock_time_module.time.called
 
     def test_autohsts_make_permanent_noop(self):
         self.config.storage.put = mock.MagicMock()

tools/install_and_test.py

@@ -1,32 +0,0 @@
-#!/usr/bin/env python
-# pip installs the requested packages in editable mode and runs unit tests on
-# them. Each package is installed and tested in the order they are provided
-# before the script moves on to the next package. Packages are installed using
-# pinned versions of all of our dependencies. See pip_install.py for more
-# information on the versions pinned to.
-import os
-import re
-import subprocess
-import sys
-
-
-def call_with_print(command):
-    print(command)
-    subprocess.check_call(command, shell=True)
-
-
-def main(args):
-    script_dir = os.path.dirname(os.path.abspath(__file__))
-    command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')]
-
-    for requirement in args:
-        current_command = command[:]
-        current_command.append(requirement)
-        call_with_print(' '.join(current_command))
-        pkg = re.sub(r'\[\w+\]', '', requirement)
-
-        call_with_print(' '.join([
-            sys.executable, '-m', 'pytest', pkg]))
-
-if __name__ == '__main__':
-    main(sys.argv[1:])

tools/oldest_constraints.txt

@@ -2,69 +2,69 @@
 # that script.
 apacheconfig==0.3.2 ; python_full_version < "3.8.0" and python_version >= "3.7"
 asn1crypto==0.24.0 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0"
-astroid==2.12.13 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
-attrs==22.1.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
+astroid==2.15.2 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
+attrs==22.2.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 boto3==1.15.15 ; python_full_version < "3.8.0" and python_version >= "3.7"
 botocore==1.18.15 ; python_full_version < "3.8.0" and python_version >= "3.7"
 certifi==2022.12.7 ; python_full_version < "3.8.0" and python_version >= "3.7"
-cffi==1.9.1 ; python_full_version < "3.8.0" and python_version >= "3.7"
+cffi==1.11.5 ; python_full_version < "3.8.0" and python_version >= "3.7"
 chardet==3.0.4 ; python_full_version < "3.8.0" and python_version >= "3.7"
 cloudflare==1.5.1 ; python_full_version < "3.8.0" and python_version >= "3.7"
 colorama==0.4.6 ; python_full_version < "3.8.0" and sys_platform == "win32" and python_version >= "3.7"
 configargparse==0.10.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 configobj==5.0.6 ; python_full_version < "3.8.0" and python_version >= "3.7"
-coverage==6.5.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
+coverage==7.2.3 ; python_version >= "3.7" and python_full_version < "3.8.0"
 cryptography==3.2.1 ; python_full_version < "3.8.0" and python_version >= "3.7"
-cython==0.29.32 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0"
+cython==0.29.34 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0"
 dill==0.3.6 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
 distlib==0.3.6 ; python_version >= "3.7" and python_full_version < "3.8.0"
 distro==1.0.1 ; python_full_version < "3.8.0" and python_version >= "3.7"
 dns-lexicon==3.2.1 ; python_full_version < "3.8.0" and python_version >= "3.7"
 dnspython==1.15.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
-exceptiongroup==1.0.4 ; python_version >= "3.7" and python_full_version < "3.8.0"
+exceptiongroup==1.1.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
 execnet==1.9.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
-filelock==3.8.2 ; python_version >= "3.7" and python_full_version < "3.8.0"
+filelock==3.11.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 funcsigs==0.4 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0"
-future==0.18.2 ; python_full_version < "3.8.0" and python_version >= "3.7"
+future==0.18.3 ; python_full_version < "3.8.0" and python_version >= "3.7"
 google-api-python-client==1.5.5 ; python_full_version < "3.8.0" and python_version >= "3.7"
 httplib2==0.9.2 ; python_full_version < "3.8.0" and python_version >= "3.7"
 idna==2.6 ; python_full_version < "3.8.0" and python_version >= "3.7"
-importlib-metadata==5.1.0 ; python_version >= "3.7" and python_version < "3.8"
-iniconfig==1.1.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
+importlib-metadata==6.1.0 ; python_version >= "3.7" and python_version < "3.8"
+iniconfig==2.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 ipaddress==1.0.16 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0"
-isort==5.10.1 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
+isort==5.11.5 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
 jmespath==0.10.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 josepy==1.13.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
-lazy-object-proxy==1.8.0 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
+lazy-object-proxy==1.9.0 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
 logger==1.4 ; python_full_version < "3.8.0" and python_version >= "3.7"
 mccabe==0.7.0 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
-mypy-extensions==0.4.3 ; python_version >= "3.7" and python_full_version < "3.8.0"
-mypy==0.991 ; python_version >= "3.7" and python_full_version < "3.8.0"
+mypy-extensions==1.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
+mypy==1.2.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 ndg-httpsclient==0.3.2 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0"
 oauth2client==4.0.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
-packaging==22.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
+packaging==23.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 parsedatetime==2.4 ; python_full_version < "3.8.0" and python_version >= "3.7"
 pbr==1.8.0 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0"
-pip==22.3.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
-platformdirs==2.6.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
+pip==23.0.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
+platformdirs==3.2.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 pluggy==1.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 ply==3.4 ; python_full_version < "3.8.0" and python_version >= "3.7"
 py==1.11.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 pyasn1-modules==0.0.10 ; python_full_version < "3.8.0" and python_version >= "3.7"
 pyasn1==0.1.9 ; python_full_version < "3.8.0" and python_version >= "3.7"
 pycparser==2.14 ; python_full_version < "3.8.0" and python_version >= "3.7"
-pylint==2.15.8 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
+pylint==2.17.2 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
 pyopenssl==17.5.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 pyparsing==2.2.1 ; python_full_version < "3.8.0" and python_version >= "3.7"
 pyrfc3339==1.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 pytest-cov==4.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
-pytest-xdist==3.1.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
-pytest==7.2.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
+pytest-xdist==3.2.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
+pytest==7.2.2 ; python_version >= "3.7" and python_full_version < "3.8.0"
 python-augeas==0.5.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 python-dateutil==2.8.2 ; python_full_version < "3.8.0" and python_version >= "3.7"
 python-digitalocean==1.11 ; python_full_version < "3.8.0" and python_version >= "3.7"
 pytz==2019.3 ; python_full_version < "3.8.0" and python_version >= "3.7"
-pywin32==305 ; python_version >= "3.7" and python_full_version < "3.8.0" and sys_platform == "win32"
+pywin32==306 ; python_version >= "3.7" and python_full_version < "3.8.0" and sys_platform == "win32"
 pyyaml==6.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 requests-file==1.5.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
 requests==2.20.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
@@ -74,22 +74,22 @@ setuptools==41.6.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 six==1.11.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 tldextract==3.4.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 tomli==2.0.1 ; python_full_version < "3.8.0" and python_version >= "3.7"
-tomlkit==0.11.6 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
+tomlkit==0.11.7 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
 tox==1.9.2 ; python_version >= "3.7" and python_full_version < "3.8.0"
 typed-ast==1.5.4 ; python_version < "3.8" and python_version >= "3.7"
 types-cryptography==3.3.23.2 ; python_version >= "3.7" and python_full_version < "3.8.0"
-types-pyopenssl==22.1.0.2 ; python_version >= "3.7" and python_full_version < "3.8.0"
-types-pyrfc3339==1.1.1.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
-types-python-dateutil==2.8.19.4 ; python_version >= "3.7" and python_full_version < "3.8.0"
-types-pytz==2022.6.0.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
-types-requests==2.28.11.5 ; python_version >= "3.7" and python_full_version < "3.8.0"
-types-setuptools==65.6.0.2 ; python_version >= "3.7" and python_full_version < "3.8.0"
-types-six==1.16.21.4 ; python_version >= "3.7" and python_full_version < "3.8.0"
-types-urllib3==1.26.25.4 ; python_version >= "3.7" and python_full_version < "3.8.0"
-typing-extensions==4.4.0 ; python_version >= "3.7" and python_version < "3.8"
+types-pyopenssl==23.0.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
+types-pyrfc3339==1.1.1.4 ; python_version >= "3.7" and python_full_version < "3.8.0"
+types-python-dateutil==2.8.19.12 ; python_version >= "3.7" and python_full_version < "3.8.0"
+types-pytz==2023.3.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
+types-requests==2.28.11.17 ; python_version >= "3.7" and python_full_version < "3.8.0"
+types-setuptools==67.6.0.7 ; python_version >= "3.7" and python_full_version < "3.8.0"
+types-six==1.16.21.8 ; python_version >= "3.7" and python_full_version < "3.8.0"
+types-urllib3==1.26.25.10 ; python_version >= "3.7" and python_full_version < "3.8.0"
+typing-extensions==4.5.0 ; python_version < "3.8" and python_version >= "3.7"
 uritemplate==3.0.1 ; python_full_version < "3.8.0" and python_version >= "3.7"
 urllib3==1.24.2 ; python_full_version < "3.8.0" and python_version >= "3.7"
-virtualenv==20.17.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
+virtualenv==20.21.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 wheel==0.33.6 ; python_full_version < "3.8.0" and python_version >= "3.7"
-wrapt==1.14.1 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
-zipp==3.11.0 ; python_version >= "3.7" and python_version < "3.8"
+wrapt==1.15.0 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
+zipp==3.15.0 ; python_version >= "3.7" and python_version < "3.8"

tools/pinning/oldest/pyproject.toml

@@ -48,7 +48,7 @@ apacheconfig = "0.3.2"
 asn1crypto = "0.24.0"
 boto3 = "1.15.15"
 botocore = "1.18.15"
-cffi = "1.9.1"
+cffi = "1.11.5"
 chardet = "3.0.4"
 cloudflare = "1.5.1"
 configobj = "5.0.6"

tools/pip_install_editable.py

@@ -1,17 +0,0 @@
-#!/usr/bin/env python
-# pip installs packages in editable mode using pip_install.py
-import sys
-
-import pip_install
-
-
-def main(args):
-    new_args = []
-    for arg in args:
-        new_args.append('-e')
-        new_args.append(arg)
-
-    pip_install.main(new_args)
-
-if __name__ == '__main__':
-    main(sys.argv[1:])

tox.cover.py

@@ -1,87 +0,0 @@
-#!/usr/bin/env python
-from __future__ import print_function
-
-import argparse
-import os
-import subprocess
-import sys
-
-DEFAULT_PACKAGES = [
-    'certbot', 'acme', 'certbot_apache', 'certbot_dns_cloudflare',
-    'certbot_dns_digitalocean', 'certbot_dns_dnsimple', 'certbot_dns_dnsmadeeasy',
-    'certbot_dns_gehirn', 'certbot_dns_google', 'certbot_dns_linode', 'certbot_dns_luadns',
-    'certbot_dns_nsone', 'certbot_dns_ovh', 'certbot_dns_rfc2136', 'certbot_dns_route53',
-    'certbot_dns_sakuracloud', 'certbot_nginx']
-
-COVER_THRESHOLDS = {
-    'certbot': {'linux': 94, 'windows': 96},
-    'acme': {'linux': 100, 'windows': 99},
-    'certbot_apache': {'linux': 100, 'windows': 100},
-    'certbot_dns_cloudflare': {'linux': 98, 'windows': 98},
-    'certbot_dns_digitalocean': {'linux': 98, 'windows': 98},
-    'certbot_dns_dnsimple': {'linux': 98, 'windows': 98},
-    'certbot_dns_dnsmadeeasy': {'linux': 99, 'windows': 99},
-    'certbot_dns_gehirn': {'linux': 97, 'windows': 97},
-    'certbot_dns_google': {'linux': 99, 'windows': 99},
-    'certbot_dns_linode': {'linux': 98, 'windows': 98},
-    'certbot_dns_luadns': {'linux': 98, 'windows': 98},
-    'certbot_dns_nsone': {'linux': 99, 'windows': 99},
-    'certbot_dns_ovh': {'linux': 97, 'windows': 97},
-    'certbot_dns_rfc2136': {'linux': 99, 'windows': 99},
-    'certbot_dns_route53': {'linux': 92, 'windows': 92},
-    'certbot_dns_sakuracloud': {'linux': 97, 'windows': 97},
-    'certbot_nginx': {'linux': 97, 'windows': 97},
-}
-
-SKIP_PROJECTS_ON_WINDOWS = ['certbot-apache']
-
-
-def cover(package):
-    threshold = COVER_THRESHOLDS.get(package)['windows' if os.name == 'nt' else 'linux']
-
-    pkg_dir = package.replace('_', '-')
-
-    if os.name == 'nt' and pkg_dir in SKIP_PROJECTS_ON_WINDOWS:
-        print((
-            'Info: currently {0} is not supported on Windows and will not be tested/covered.'
-            .format(pkg_dir)))
-        return
-
-    subprocess.check_call([sys.executable, '-m', 'pytest',
-                           '--cov', pkg_dir, '--cov-append', '--cov-report=', pkg_dir])
-    try:
-        subprocess.check_call([
-            sys.executable, '-m', 'coverage', 'report', '--fail-under',
-            str(threshold), '--include', '{0}/*'.format(pkg_dir),
-            '--show-missing'])
-    except subprocess.CalledProcessError as err:
-        print(err)
-        print('Test coverage on', pkg_dir,
-              'did not meet threshold of {0}%.'.format(threshold))
-        sys.exit(1)
-
-
-def main():
-    description = """
-This script is used by tox.ini (and thus by Travis CI and Azure Pipelines) in
-order to generate separate stats for each package. It should be removed once
-those packages are moved to a separate repo."""
-    parser = argparse.ArgumentParser(description=description)
-    parser.add_argument('--packages', nargs='+')
-
-    args = parser.parse_args()
-
-    packages = args.packages or DEFAULT_PACKAGES
-
-    # --cov-append is on, make sure stats are correct
-    try:
-        os.remove('.coverage')
-    except OSError:
-        pass
-
-    for package in packages:
-        cover(package)
-
-
-if __name__ == '__main__':
-    main()

tox.ini

@@ -1,42 +1,66 @@
-# Tox (https://tox.readthedocs.io/) is a tool for running tests in
-# multiple virtualenvs. To use it, "pip install tox" and then run
-# "tox" from this directory.
-
 [tox]
-skipsdist = true
 # mypy doesn't current pass for us on Windows. Fixing that is being tracked by
 # https://github.com/certbot/certbot/issues/7803.
-envlist = {cover,lint}-{win,posix},mypy-posix
+envlist = {cover,lint}-{win,posix},mypy
+skipsdist = true
 
 [base]
-# pip installs the requested packages in editable mode
-pip_install = python {toxinidir}/tools/pip_install_editable.py
-# pip installs the requested packages in editable mode and runs unit tests on
-# them. Each package is installed and tested in the order they are provided
-# before the script moves on to the next package. All dependencies are pinned
-# to a specific version for increased stability for developers.
-install_and_test = python {toxinidir}/tools/install_and_test.py
-# Packages are listed on one line because tox seems to have inconsistent
+pytest = python -m pytest {posargs}
+# Paths are listed on one line because tox seems to have inconsistent
 # behavior with substitutions that contain line continuations, see
 # https://github.com/tox-dev/tox/issues/2069 for more info.
-dns_packages = certbot-dns-cloudflare certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud
-win_all_packages = acme[test] certbot[test] {[base]dns_packages} certbot-nginx
-all_packages = {[base]win_all_packages} certbot-apache
 source_paths = acme/acme certbot/certbot certbot-apache/certbot_apache certbot-ci/certbot_integration_tests certbot-ci/snap_integration_tests certbot-ci/windows_installer_integration_tests certbot-compatibility-test/certbot_compatibility_test certbot-dns-cloudflare/certbot_dns_cloudflare certbot-dns-digitalocean/certbot_dns_digitalocean certbot-dns-dnsimple/certbot_dns_dnsimple certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy certbot-dns-gehirn/certbot_dns_gehirn certbot-dns-google/certbot_dns_google certbot-dns-linode/certbot_dns_linode certbot-dns-luadns/certbot_dns_luadns certbot-dns-nsone/certbot_dns_nsone certbot-dns-ovh/certbot_dns_ovh certbot-dns-rfc2136/certbot_dns_rfc2136 certbot-dns-route53/certbot_dns_route53 certbot-dns-sakuracloud/certbot_dns_sakuracloud certbot-nginx/certbot_nginx
 
 [testenv]
 platform =
     win: win32
     posix: ^(?!.*win32).*$
-commands =
-    win: {[base]install_and_test} {[base]win_all_packages}
-    !win: {[base]install_and_test} {[base]all_packages}
-# We always recreate the virtual environment to avoid problems like
-# https://github.com/certbot/certbot/issues/7745.
-recreate = true
 setenv =
     PYTEST_ADDOPTS = {env:PYTEST_ADDOPTS:--numprocesses auto}
     PYTHONHASHSEED = 0
+# The default install command is python -I -m pip install {opts} {packages}
+install_command = python -I {toxinidir}/tools/pip_install.py {opts} {packages}
+deps =
+    -e acme[test]
+    -e certbot[test]
+    !win: -e certbot-apache[dev]
+    -e certbot-dns-cloudflare
+    -e certbot-dns-digitalocean
+    -e certbot-dns-dnsimple
+    -e certbot-dns-dnsmadeeasy
+    -e certbot-dns-gehirn
+    -e certbot-dns-google
+    -e certbot-dns-linode
+    -e certbot-dns-luadns
+    -e certbot-dns-nsone
+    -e certbot-dns-ovh
+    -e certbot-dns-rfc2136
+    -e certbot-dns-route53
+    -e certbot-dns-sakuracloud
+    -e certbot-nginx
+whitelist_externals =
+    echo
+    false
+# This and the next few testenvs are a workaround for
+# https://github.com/tox-dev/tox/issues/2858.
+commands =
+    echo "Unrecognized environment name {envname}"
+    false
+
+[testenv:py-win]
+commands =
+    {[base]pytest} acme certbot certbot-dns-cloudflare certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud certbot-nginx
+
+[testenv:py{,-posix}]
+# We want to test everything we do on Windows plus the Apache plugin.
+commands =
+    {[testenv:py-win]commands} certbot-apache
+
+[testenv:py3{,7,8,9,10,11}]
+commands = {[testenv:py]commands}
+
+[testenv:py3.{7,8,9,10,11}]
+commands = {[testenv:py]commands}
 
 [testenv:oldest]
 # Setting basepython allows the tests to fail fast if that version of Python
@@ -46,117 +70,137 @@ setenv =
 # This version should be kept in sync with the one declared in
 # tools/pinning/oldest/pyproject.toml.
 basepython = python3.7
-commands =
-    {[testenv]commands}
-setenv =
-    {[testenv]setenv}
-    CERTBOT_OLDEST=1
+setenv = CERTBOT_OLDEST=1
+commands = {[testenv:py]commands}
 
-[testenv:acme-oldest]
-basepython =
-    {[testenv:oldest]basepython}
+[testenv:cover{,-posix}]
+coverage_report = python -m coverage report
+# These coverage report commands are used on both posix and windows
+common_coverage_report_commands =
+    {[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-cloudflare/*
+    {[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-digitalocean/*
+    {[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-dnsimple/*
+    {[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-dnsmadeeasy/*
+    {[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-gehirn/*
+    {[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-google/*
+    {[testenv:cover]coverage_report} --fail-under 100 --include certbot-dns-linode/*
+    {[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-luadns/*
+    {[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-nsone/*
+    {[testenv:cover]coverage_report} --fail-under 98 --include certbot-dns-ovh/*
+    {[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-rfc2136/*
+    {[testenv:cover]coverage_report} --fail-under 94 --include certbot-dns-route53/*
+    {[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-sakuracloud/*
+    {[testenv:cover]coverage_report} --fail-under 98 --include certbot-nginx/*
 commands =
-    {[base]install_and_test} acme[test]
-setenv =
-    {[testenv:oldest]setenv}
+    {[testenv:py]commands} --cov --cov-report=
+    {[testenv:cover]coverage_report} --fail-under 100 --include acme/*
+    {[testenv:cover]coverage_report} --fail-under 95 --include certbot/*
+    {[testenv:cover]coverage_report} --fail-under 100 --include certbot-apache/*
+    {[testenv:cover]common_coverage_report_commands}
 
-[testenv:apache-oldest]
-basepython =
-    {[testenv:oldest]basepython}
+[testenv:cover-win]
 commands =
-    {[base]pip_install} acme[test] certbot[test] certbot-apache
-    pytest certbot-apache
-setenv =
-    {[testenv:oldest]setenv}
-
-[testenv:apache-v2-oldest]
-basepython =
-    {[testenv:oldest]basepython}
-commands =
-    {[base]pip_install} acme[test] certbot[test] certbot-apache[dev]
-    pytest certbot-apache
-setenv =
-    {[testenv:oldest]setenv}
-
-[testenv:certbot-oldest]
-basepython =
-    {[testenv:oldest]basepython}
-commands =
-    {[base]pip_install} acme[test] certbot[test]
-    pytest certbot
-setenv =
-    {[testenv:oldest]setenv}
-
-[testenv:dns-oldest]
-basepython =
-    {[testenv:oldest]basepython}
-commands =
-    {[base]pip_install} acme[test] certbot[test] {[base]dns_packages}
-    pytest {[base]dns_packages}
-setenv =
-    {[testenv:oldest]setenv}
-
-[testenv:nginx-oldest]
-basepython =
-    {[testenv:oldest]basepython}
-commands =
-    {[base]pip_install} acme[test] certbot[test] certbot-nginx
-    pytest certbot-nginx
-setenv =
-    {[testenv:oldest]setenv}
-
-[testenv:cover{,-win,-posix}]
-commands =
-    win: {[base]pip_install} {[base]win_all_packages}
-    !win: {[base]pip_install} {[base]all_packages} certbot-apache[dev]
-    python tox.cover.py
+    {[testenv:py-win]commands} --cov --cov-report=
+    {[testenv:cover]coverage_report} --fail-under 99 --include acme/*
+    {[testenv:cover]coverage_report} --fail-under 96 --include certbot/*
+    {[testenv:cover]common_coverage_report_commands}
 
 [testenv:lint{,-win,-posix}]
-basepython = python3
-# separating into multiple invocations disables cross package
-# duplicate code checking; if one of the commands fails, others will
-# continue, but tox return code will reflect previous error
-commands =
-    win: {[base]pip_install} {[base]win_all_packages}
-    !win: {[base]pip_install} {[base]all_packages}
-    python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths}
+commands = python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths}
 
-[testenv:mypy{,-win,-posix}]
-basepython = python3
+[testenv:mypy]
+deps =
+    {[testenv]deps}
+    -e certbot-ci
+commands = mypy {[base]source_paths}
+
+[testenv:isolated-acme]
+description = Tests acme without any Certbot components installed
+deps = -e acme[test]
+commands = {[base]pytest} acme
+
+[testenv:isolated-certbot]
+description = Tests Certbot without any additional plugins installed
+deps =
+    {[testenv:isolated-acme]deps}
+    -e certbot[test]
+commands = {[base]pytest} certbot
+
+[testenv:isolated-{apache,cloudflare,digitalocean,dnsimple,dnsmadeeasy,gehirn,google,linode,luadns,nsone,ovh,rfc2136,route53,sakuracloud,nginx}]
+description = Tests the plugin without installing any other plugins
+deps =
+    {[testenv:isolated-certbot]deps}
+    apache: -e certbot-apache[dev]
+    cloudflare: -e certbot-dns-cloudflare
+    digitalocean: -e certbot-dns-digitalocean
+    dnsimple: -e certbot-dns-dnsimple
+    dnsmadeeasy: -e certbot-dns-dnsmadeeasy
+    gehirn: -e certbot-dns-gehirn
+    google: -e certbot-dns-google
+    linode: -e certbot-dns-linode
+    luadns: -e certbot-dns-luadns
+    nsone: -e certbot-dns-nsone
+    ovh: -e certbot-dns-ovh
+    rfc2136: -e certbot-dns-rfc2136
+    route53: -e certbot-dns-route53
+    sakuracloud: -e certbot-dns-sakuracloud
+    nginx: -e certbot-nginx
 commands =
-    win: {[base]pip_install} {[base]win_all_packages}
-    !win: {[base]pip_install} {[base]all_packages} certbot-ci
-    mypy {[base]source_paths}
+    apache: {[base]pytest} certbot-apache
+    cloudflare: {[base]pytest} certbot-dns-cloudflare
+    digitalocean: {[base]pytest} certbot-dns-digitalocean
+    dnsimple: {[base]pytest} certbot-dns-dnsimple
+    dnsmadeeasy: {[base]pytest} certbot-dns-dnsmadeeasy
+    gehirn: {[base]pytest} certbot-dns-gehirn
+    google: {[base]pytest} certbot-dns-google
+    linode: {[base]pytest} certbot-dns-linode
+    luadns: {[base]pytest} certbot-dns-luadns
+    nsone: {[base]pytest} certbot-dns-nsone
+    ovh: {[base]pytest} certbot-dns-ovh
+    rfc2136: {[base]pytest} certbot-dns-rfc2136
+    route53: {[base]pytest} certbot-dns-route53
+    sakuracloud: {[base]pytest} certbot-dns-sakuracloud
+    nginx: {[base]pytest} certbot-nginx
 
 [testenv:apacheconftest]
+deps =
+    -e acme
+    -e certbot
+    -e certbot-apache
 commands =
-    {[base]pip_install} acme certbot certbot-apache
     {toxinidir}/certbot-apache/certbot_apache/_internal/tests/apache-conf-files/apache-conf-test --debian-modules
 passenv =
     SERVER
 
 [testenv:apacheconftest-external-with-pebble]
-# Run apacheconftest with pebble and Certbot outside of tox's virtual
-# environment.
+description = Run apacheconftest with pebble and Certbot outside of tox's virtual environment.
+deps =
+    -e certbot-ci
 commands =
-    {[base]pip_install} certbot-ci
     {toxinidir}/certbot-apache/certbot_apache/_internal/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules
 
 [testenv:apacheconftest-with-pebble]
-commands =
-    {[base]pip_install} acme certbot certbot-apache
-    {[testenv:apacheconftest-external-with-pebble]commands}
+deps =
+    {[testenv:apacheconftest]deps}
+    {[testenv:apacheconftest-external-with-pebble]deps}
+commands = {[testenv:apacheconftest-external-with-pebble]commands}
 
 [testenv:nginxroundtrip]
+deps =
+    -e acme
+    -e certbot
+    -e certbot-apache
+    -e certbot-nginx
 commands =
-    {[base]pip_install} acme certbot certbot-apache certbot-nginx
     python certbot-compatibility-test/nginx/roundtrip.py certbot-compatibility-test/nginx/nginx-roundtrip-testdata
 
 [testenv:modification]
+deps =
 commands =
     python {toxinidir}/tests/modification-check.py
 
 [testenv:apache_compat]
+deps =
 commands =
     docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile .
     docker build -t apache-compat -f certbot-compatibility-test/Dockerfile-apache .
@@ -167,6 +211,7 @@ passenv =
     DOCKER_*
 
 [testenv:nginx_compat]
+deps =
 commands =
     docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile .
     docker build -t nginx-compat -f certbot-compatibility-test/Dockerfile-nginx .
@@ -177,8 +222,8 @@ passenv =
     DOCKER_*
 
 [testenv:docker_dev]
-# tests the Dockerfile-dev file to ensure development with it works
-# as expected
+description = Tests the Dockerfile-dev file to ensure development with it works as expected
+deps =
 commands =
     docker-compose run --rm --service-ports development bash -c 'tox -e lint'
 whitelist_externals =
@@ -186,9 +231,13 @@ whitelist_externals =
 passenv = DOCKER_*
 
 [testenv:integration]
+deps =
+    -e acme
+    -e certbot
+    -e certbot-nginx
+    -e certbot-ci
 commands =
-    {[base]pip_install} acme certbot certbot-nginx certbot-ci
-    pytest certbot-ci/certbot_integration_tests \
+    {[base]pytest} certbot-ci/certbot_integration_tests \
         --acme-server={env:ACME_SERVER:pebble} \
         --cov=acme --cov=certbot --cov=certbot_nginx --cov-report= \
         --cov-config=certbot-ci/certbot_integration_tests/.coveragerc
@@ -197,18 +246,25 @@ commands =
 passenv = DOCKER_*
 
 [testenv:integration-certbot]
+deps =
+    -e acme
+    -e certbot
+    -e certbot-ci
 commands =
-    {[base]pip_install} acme certbot certbot-ci
-    pytest certbot-ci/certbot_integration_tests/certbot_tests \
+    {[base]pytest} certbot-ci/certbot_integration_tests/certbot_tests \
         --acme-server={env:ACME_SERVER:pebble} \
         --cov=acme --cov=certbot --cov-report= \
         --cov-config=certbot-ci/certbot_integration_tests/.coveragerc
     coverage report --include 'certbot/*' --show-missing --fail-under=62
 
 [testenv:integration-dns-rfc2136]
+deps =
+    -e acme
+    -e certbot
+    -e certbot-dns-rfc2136
+    -e certbot-ci
 commands =
-    {[base]pip_install} acme certbot certbot-dns-rfc2136 certbot-ci
-    pytest certbot-ci/certbot_integration_tests/rfc2136_tests \
+    {[base]pytest} certbot-ci/certbot_integration_tests/rfc2136_tests \
         --acme-server=pebble --dns-server=bind \
         --numprocesses=1 \
         --cov=acme --cov=certbot --cov=certbot_dns_rfc2136 --cov-report= \
@@ -217,46 +273,45 @@ commands =
     coverage report --include 'certbot-dns-rfc2136/*' --show-missing --fail-under=86
 
 [testenv:integration-external]
-# Run integration tests with Certbot outside of tox's virtual environment.
+description = Run integration tests with Certbot outside of tox's virtual environment.
+deps =
+    -e certbot-ci
 commands =
-    {[base]pip_install} certbot-ci
-    pytest certbot-ci/certbot_integration_tests \
+    {[base]pytest} certbot-ci/certbot_integration_tests \
         --acme-server={env:ACME_SERVER:pebble}
 passenv = DOCKER_*
 
 [testenv:integration-certbot-oldest]
+deps =
+    -e acme
+    -e certbot
+    -e certbot-ci
 basepython =
     {[testenv:oldest]basepython}
 commands =
-    {[base]pip_install} acme certbot certbot-ci
-    pytest certbot-ci/certbot_integration_tests/certbot_tests \
+    {[base]pytest} certbot-ci/certbot_integration_tests/certbot_tests \
         --acme-server={env:ACME_SERVER:pebble}
 passenv = DOCKER_*
 setenv = {[testenv:oldest]setenv}
 
 [testenv:integration-nginx-oldest]
+deps =
+    -e acme
+    -e certbot
+    -e certbot-nginx
+    -e certbot-ci
 basepython =
     {[testenv:oldest]basepython}
 commands =
-    {[base]pip_install} acme certbot certbot-nginx certbot-ci
-    pytest certbot-ci/certbot_integration_tests/nginx_tests \
+    {[base]pytest} certbot-ci/certbot_integration_tests/nginx_tests \
         --acme-server={env:ACME_SERVER:pebble}
 passenv = DOCKER_*
 setenv = {[testenv:oldest]setenv}
 
-[testenv:test-farm-tests-base]
-changedir = letstest
-# The package to install is in the current working directory because of the
-# value of changedir.
-commands = {[base]pip_install} .
+[testenv:test-farm-apache2]
 passenv =
     AWS_*
 setenv = AWS_DEFAULT_REGION=us-east-1
-
-[testenv:test-farm-apache2]
-changedir = {[testenv:test-farm-tests-base]changedir}
-commands =
-    {[testenv:test-farm-tests-base]commands}
-    {toxinidir}/tools/retry.sh letstest targets/targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir}
-passenv = {[testenv:test-farm-tests-base]passenv}
-setenv = {[testenv:test-farm-tests-base]setenv}
+changedir = letstest
+deps = -e {toxinidir}/letstest
+commands = {toxinidir}/tools/retry.sh letstest targets/targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir}