fix tests with updated coverage

Fixes #8425

This PR upgrades mypy to the latest version available, 0.812.

Given the advanced type inference capabilities provided by this newer version, this PRs also fixes various type inconsistencies that are now detected. Here are the non obvious changes done to fix types:
* typing in mixins has been solved using `Protocol` classes, as recommended by mypy (https://mypy.readthedocs.io/en/latest/more_types.html#mixin-classes, https://mypy.readthedocs.io/en/stable/protocols.html)
* `cast` when we are playing with `Union` types

This PR also disables the strict optional checks that have been enable by default in recent versions of mypy. Once this PR is merged, I will create an issue to study how these checks can be enabled.

`typing.Protocol` is available only since Python 3.8. To keep compatibility with Python 3.6, I try to import the class `Protocol` from `typing`, and fallback to assign `object` to `Protocol` if that fails. This way the code is working with all versions of Python, but the mypy check can be run only with Python 3.8+ because it needs the protocol feature. As a consequence, tox runs mypy under Python 3.8.

Alternatives are:
* importing `typing_extensions`, that proposes backport of newest typing features to Python 3.6, but this implies to add a dependency to Certbot just to run mypy
* redesign the concerned classes to not use mixins, or use them differently, but this implies to modify the code itself even if there is nothing wrong with it and it is just a matter of instructing mypy to understand in which context the mixins can be used
* ignoring type for these classes with `# type: ignore` but we loose the benefit of mypy for them

* Upgrade mypy

* First step for acme

* Cast for the rescue

* Fixing types for certbot

* Fix typing for certbot-nginx

* Finalize type fixes, configure no optional strict check for mypy in tox

* Align requirements

* Isort

* Pylint

* Protocol for python 3.6

* Use Python 3.9 for mypy, make code compatible with Python 3.8<

* Pylint and mypy

* Pragma no cover

* Pythonic NotImplemented constant

* More type definitions

* Add comments

* Simplify typing logic

* Use vararg tuple

* Relax constraints on mypy

* Add more type

* Do not silence error if target is not defined

* Conditionally import Protocol for type checking only

* Clean up imports

* Add comments

* Align python version linting with mypy and coverage

* Just ignore types in an unused module

* Add comments

* Fix lint

.azure-pipelines/templates/jobs/extended-tests-jobs.yml

@@ -21,6 +21,8 @@ jobs:
           PYTHON_VERSION: 3.7
           TOXENV: py37
           CERTBOT_NO_PIN: 1
+        linux-external-mock:
+          TOXENV: external-mock
         linux-boulder-v1-integration-certbot-oldest:
           PYTHON_VERSION: 3.6
           TOXENV: integration-certbot-oldest

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -12,6 +12,9 @@ jobs:
             DOCKER_ARCH: arm32v6
           arm64v8:
             DOCKER_ARCH: arm64v8
+    # The default timeout of 60 minutes is a little low for compiling
+    # cryptography on ARM architectures.
+    timeoutInMinutes: 180
     steps:
       - bash: set -e && tools/docker/build.sh $(dockerTag) $DOCKER_ARCH
         displayName: Build the Docker images
@@ -59,7 +62,13 @@ jobs:
           versionSpec: 3.8
           architecture: x86
           addToPath: true
-      - script: python windows-installer/construct.py
+      - script: |
+          python -m venv venv
+          venv\Scripts\python tools\pipstrap.py
+          venv\Scripts\python tools\pip_install.py -e windows-installer
+        displayName: Prepare Windows installer build environment
+      - script: |
+          venv\Scripts\construct-windows-installer
         displayName: Build Certbot installer
       - task: CopyFiles@2
         inputs:
@@ -116,13 +125,17 @@ jobs:
   - job: snaps_build
     pool:
       vmImage: ubuntu-18.04
+    strategy:
+      matrix:
+        amd64:
+          SNAP_ARCH: amd64
+        # Do not run the heavy non-amd64 builds for test branches
+        ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
+          armhf:
+            SNAP_ARCH: armhf
+          arm64:
+            SNAP_ARCH: arm64
     timeoutInMinutes: 0
-    variables:
-      # Do not run the heavy non-amd64 builds for test branches
-      ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
-        ARCHS: amd64 arm64 armhf
-      ${{ if startsWith(variables['Build.SourceBranchName'], 'test-') }}:
-        ARCHS: amd64
     steps:
       - script: |
           set -e
@@ -144,7 +157,7 @@ jobs:
           git config --global user.name "$(Build.RequestedFor)"
           mkdir -p ~/.local/share/snapcraft/provider/launchpad
           cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials
-          python3 tools/snap/build_remote.py ALL --archs ${ARCHS} --timeout 19800
+          python3 tools/snap/build_remote.py ALL --archs ${SNAP_ARCH} --timeout 19800
         displayName: Build snaps
       - script: |
           set -e
@@ -154,7 +167,7 @@ jobs:
       - task: PublishPipelineArtifact@1
         inputs:
           path: $(Build.ArtifactStagingDirectory)
-          artifact: snaps
+          artifact: snaps_$(SNAP_ARCH)
         displayName: Store snaps artifacts
   - job: snap_run
     dependsOn: snaps_build
@@ -175,12 +188,12 @@ jobs:
         displayName: Install dependencies
       - task: DownloadPipelineArtifact@2
         inputs:
-          artifact: snaps
+          artifact: snaps_amd64
           path: $(Build.SourcesDirectory)/snap
         displayName: Retrieve Certbot snaps
       - script: |
           set -e
-          sudo snap install --dangerous --classic snap/certbot_*_amd64.snap
+          sudo snap install --dangerous --classic snap/certbot_*.snap
         displayName: Install Certbot snap
       - script: |
           set -e
@@ -202,7 +215,7 @@ jobs:
           addToPath: true
       - task: DownloadPipelineArtifact@2
         inputs:
-          artifact: snaps
+          artifact: snaps_amd64
           path: $(Build.SourcesDirectory)/snap
         displayName: Retrieve Certbot snaps
       - script: |

.azure-pipelines/templates/jobs/standard-tests-jobs.yml

@@ -40,13 +40,13 @@ jobs:
           IMAGE_NAME: ubuntu-18.04
           PYTHON_VERSION: 3.9
           TOXENV: py39-cover
-        linux-py37-lint:
+        linux-py39-lint:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.7
+          PYTHON_VERSION: 3.9
           TOXENV: lint
-        linux-py36-mypy:
+        linux-py39-mypy:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.6
+          PYTHON_VERSION: 3.9
           TOXENV: mypy
         linux-integration:
           IMAGE_NAME: ubuntu-18.04
@@ -73,6 +73,6 @@ jobs:
       - template: ../steps/tox-steps.yml
   - job: test_sphinx_builds
     pool:
-      vmImage: ubuntu-latest
+      vmImage: ubuntu-20.04
     steps:
       - template: ../steps/sphinx-steps.yml

.azure-pipelines/templates/stages/deploy-stage.yml

@@ -37,6 +37,14 @@ stages:
           vmImage: ubuntu-18.04
         variables:
           - group: certbot-common
+        strategy:
+          matrix:
+            amd64:
+              SNAP_ARCH: amd64
+            arm32v6:
+              SNAP_ARCH: armhf
+            arm64v8:
+              SNAP_ARCH: arm64
         steps:
           - bash: |
               set -e
@@ -46,7 +54,7 @@ stages:
             displayName: Install dependencies
           - task: DownloadPipelineArtifact@2
             inputs:
-              artifact: snaps
+              artifact: snaps_$(SNAP_ARCH)
               path: $(Build.SourcesDirectory)/snap
             displayName: Retrieve Certbot snaps
           - task: DownloadSecureFile@1
@@ -55,8 +63,7 @@ stages:
               secureFile: snapcraft.cfg
           - bash: |
               set -e
-              mkdir -p .snapcraft
-              ln -s $(snapcraftCfg.secureFilePath) .snapcraft/snapcraft.cfg
+              snapcraft login --with $(snapcraftCfg.secureFilePath)
               for SNAP_FILE in snap/*.snap; do
                 tools/retry.sh eval snapcraft upload --release=${{ parameters.snapReleaseChannel }} "${SNAP_FILE}"
               done

.azure-pipelines/templates/stages/notify-failure-stage.yml

@@ -5,7 +5,7 @@ stages:
         variables:
           - group: certbot-common
         pool:
-          vmImage: ubuntu-latest
+          vmImage: ubuntu-20.04
         steps:
           - bash: |
               set -e

.dockerignore

@@ -8,5 +8,4 @@
 .git
 .tox
 venv
-venv3
 docs

.envrc

@@ -3,7 +3,7 @@
 # activated and then deactivated when you cd elsewhere. Developers have to have
 # direnv set up and run `direnv allow` to allow this file to execute on their
 # system. You can find more information at https://direnv.net/.
-. venv3/bin/activate
+. venv/bin/activate
 # direnv doesn't support modifying PS1 so we unset it to squelch the error
 # it'll otherwise print about this being done in the activate script. See
 # https://github.com/direnv/direnv/wiki/PS1. If you would like your shell

.gitignore

@@ -11,9 +11,11 @@ dist*/
 letsencrypt.log
 certbot.log
 letsencrypt-auto-source/letsencrypt-auto.sig.lzma.base64
+poetry.lock
 
 # coverage
 .coverage
+.coverage.*
 /htmlcov/
 
 /.vagrant

.pylintrc

@@ -8,7 +8,10 @@ jobs=0
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-#init-hook=
+# CERTBOT COMMENT
+# This is needed for pylint to import linter_plugin.py since
+# https://github.com/PyCQA/pylint/pull/3396.
+init-hook="import pylint.config, os, sys; sys.path.append(os.path.dirname(pylint.config.PYLINTRC))"
 
 # Profiled execution.
 profile=no
@@ -254,7 +257,7 @@ ignore-mixin-members=yes
 # List of module names for which member attributes should not be checked
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis
-ignored-modules=pkg_resources,confargparse,argparse,six.moves,six.moves.urllib
+ignored-modules=pkg_resources,confargparse,argparse
 # import errors ignored only in 1.4.4
 # https://bitbucket.org/logilab/pylint/commits/cd000904c9e2
 

Dockerfile-dev

@@ -15,6 +15,6 @@ RUN apt-get update && \
            /tmp/* \
            /var/tmp/*
 
-RUN VENV_NAME="../venv3" python3 tools/venv3.py
+RUN VENV_NAME="../venv" python3 tools/venv.py
 
-ENV PATH /opt/certbot/venv3/bin:$PATH
+ENV PATH /opt/certbot/venv/bin:$PATH

ISSUE_TEMPLATE.md

@@ -7,7 +7,7 @@ questions.
 ## My operating system is (include version):
 
 
-## I installed Certbot with (certbot-auto, OS package manager, pip, etc):
+## I installed Certbot with (snap, OS package manager, pip, certbot-auto, etc):
 
 
 ## I ran this command and it produced this output:

acme/acme/challenges.py

@@ -5,18 +5,19 @@ import functools
 import hashlib
 import logging
 import socket
+from typing import Type
 
 from cryptography.hazmat.primitives import hashes  # type: ignore
 import josepy as jose
-import requests
-import six
-from OpenSSL import SSL  # type: ignore # https://github.com/python/typeshed/issues/2052
 from OpenSSL import crypto
+from OpenSSL import SSL  # type: ignore # https://github.com/python/typeshed/issues/2052
+import requests
 
 from acme import crypto_util
 from acme import errors
 from acme import fields
-from acme.mixins import ResourceMixin, TypeMixin
+from acme.mixins import ResourceMixin
+from acme.mixins import TypeMixin
 
 logger = logging.getLogger(__name__)
 
@@ -24,7 +25,7 @@ logger = logging.getLogger(__name__)
 class Challenge(jose.TypedJSONObjectWithFields):
     # _fields_to_partial_json
     """ACME challenge."""
-    TYPES = {}  # type: dict
+    TYPES: dict = {}
 
     @classmethod
     def from_json(cls, jobj):
@@ -38,7 +39,7 @@ class Challenge(jose.TypedJSONObjectWithFields):
 class ChallengeResponse(ResourceMixin, TypeMixin, jose.TypedJSONObjectWithFields):
     # _fields_to_partial_json
     """ACME challenge response."""
-    TYPES = {}  # type: dict
+    TYPES: dict = {}
     resource_type = 'challenge'
     resource = fields.Resource(resource_type)
 
@@ -145,16 +146,15 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse):
         return jobj
 
 
-@six.add_metaclass(abc.ABCMeta)
-class KeyAuthorizationChallenge(_TokenChallenge):
+class KeyAuthorizationChallenge(_TokenChallenge, metaclass=abc.ABCMeta):
     """Challenge based on Key Authorization.
 
     :param response_cls: Subclass of `KeyAuthorizationChallengeResponse`
         that will be used to generate ``response``.
     :param str typ: type of the challenge
     """
-    typ = NotImplemented
-    response_cls = NotImplemented
+    typ: str = NotImplemented
+    response_cls: Type[KeyAuthorizationChallengeResponse] = NotImplemented
     thumbprint_hash_function = (
         KeyAuthorizationChallengeResponse.thumbprint_hash_function)
 

acme/acme/client.py

@@ -4,10 +4,16 @@ import collections
 import datetime
 from email.utils import parsedate_tz
 import heapq
+import http.client as http_client
 import logging
 import re
-import sys
 import time
+from typing import cast
+from typing import Dict
+from typing import List
+from typing import Set
+from typing import Text
+from typing import Union
 
 import josepy as jose
 import OpenSSL
@@ -15,38 +21,21 @@ import requests
 from requests.adapters import HTTPAdapter
 from requests.utils import parse_header_links
 from requests_toolbelt.adapters.source import SourceAddressAdapter
-import six
-from six.moves import http_client
 
 from acme import crypto_util
 from acme import errors
 from acme import jws
 from acme import messages
-from acme.magic_typing import Dict
-from acme.magic_typing import List
-from acme.magic_typing import Set
-from acme.magic_typing import Text
 from acme.mixins import VersionedLEACMEMixin
 
 logger = logging.getLogger(__name__)
 
-# Prior to Python 2.7.9 the stdlib SSL module did not allow a user to configure
-# many important security related options. On these platforms we use PyOpenSSL
-# for SSL, which does allow these options to be configured.
-# https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning
-if sys.version_info < (2, 7, 9):  # pragma: no cover
-    try:
-        requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3()  # type: ignore
-    except AttributeError:
-        import urllib3.contrib.pyopenssl
-        urllib3.contrib.pyopenssl.inject_into_urllib3()
-
 DEFAULT_NETWORK_TIMEOUT = 45
 
 DER_CONTENT_TYPE = 'application/pkix-cert'
 
 
-class ClientBase(object):
+class ClientBase:
     """ACME client base object.
 
     :ivar messages.Directory directory:
@@ -125,8 +114,9 @@ class ClientBase(object):
         """
         return self.update_registration(regr, update={'status': 'deactivated'})
 
-    def deactivate_authorization(self, authzr):
-        # type: (messages.AuthorizationResource) -> messages.AuthorizationResource
+    def deactivate_authorization(self,
+                                 authzr: messages.AuthorizationResource
+                                 ) -> messages.AuthorizationResource:
         """Deactivate authorization.
 
         :param messages.AuthorizationResource authzr: The Authorization resource
@@ -260,7 +250,7 @@ class Client(ClientBase):
         if net is None:
             net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl)
 
-        if isinstance(directory, six.string_types):
+        if isinstance(directory, str):
             directory = messages.Directory.from_json(
                 net.get(directory).json())
         super(Client, self).__init__(directory=directory,
@@ -436,7 +426,7 @@ class Client(ClientBase):
 
         """
         assert max_attempts > 0
-        attempts = collections.defaultdict(int) # type: Dict[messages.AuthorizationResource, int]
+        attempts: Dict[messages.AuthorizationResource, int] = collections.defaultdict(int)
         exhausted = set()
 
         # priority queue with datetime.datetime (based on Retry-After) as key,
@@ -475,7 +465,7 @@ class Client(ClientBase):
                     exhausted.add(authzr)
 
         if exhausted or any(authzr.body.status == messages.STATUS_INVALID
-                            for authzr in six.itervalues(updated)):
+                            for authzr in updated.values()):
             raise errors.PollError(exhausted, updated)
 
         updated_authzrs = tuple(updated[authzr] for authzr in authzrs)
@@ -549,7 +539,7 @@ class Client(ClientBase):
         :rtype: `list` of `OpenSSL.crypto.X509` wrapped in `.ComparableX509`
 
         """
-        chain = [] # type: List[jose.ComparableX509]
+        chain: List[jose.ComparableX509] = []
         uri = certr.cert_chain_uri
         while uri is not None and len(chain) < max_length:
             response, cert = self._get_cert(uri)
@@ -808,7 +798,7 @@ class ClientV2(ClientBase):
                 if 'rel' in l and 'url' in l and l['rel'] == relation_type]
 
 
-class BackwardsCompatibleClientV2(object):
+class BackwardsCompatibleClientV2:
     """ACME client wrapper that tends towards V2-style calls, but
     supports V1 servers.
 
@@ -830,6 +820,7 @@ class BackwardsCompatibleClientV2(object):
     def __init__(self, net, key, server):
         directory = messages.Directory.from_json(net.get(server).json())
         self.acme_version = self._acme_version_from_directory(directory)
+        self.client: Union[Client, ClientV2]
         if self.acme_version == 1:
             self.client = Client(directory, key=key, net=net)
         else:
@@ -849,16 +840,18 @@ class BackwardsCompatibleClientV2(object):
             if check_tos_cb is not None:
                 check_tos_cb(tos)
         if self.acme_version == 1:
-            regr = self.client.register(regr)
+            client_v1 = cast(Client, self.client)
+            regr = client_v1.register(regr)
             if regr.terms_of_service is not None:
                 _assess_tos(regr.terms_of_service)
-                return self.client.agree_to_tos(regr)
+                return client_v1.agree_to_tos(regr)
             return regr
         else:
-            if "terms_of_service" in self.client.directory.meta:
-                _assess_tos(self.client.directory.meta.terms_of_service)
+            client_v2 = cast(ClientV2, self.client)
+            if "terms_of_service" in client_v2.directory.meta:
+                _assess_tos(client_v2.directory.meta.terms_of_service)
                 regr = regr.update(terms_of_service_agreed=True)
-            return self.client.new_account(regr)
+            return client_v2.new_account(regr)
 
     def new_order(self, csr_pem):
         """Request a new Order object from the server.
@@ -876,14 +869,15 @@ class BackwardsCompatibleClientV2(object):
 
         """
         if self.acme_version == 1:
+            client_v1 = cast(Client, self.client)
             csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)
             # pylint: disable=protected-access
             dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr)
             authorizations = []
             for domain in dnsNames:
-                authorizations.append(self.client.request_domain_challenges(domain))
+                authorizations.append(client_v1.request_domain_challenges(domain))
             return messages.OrderResource(authorizations=authorizations, csr_pem=csr_pem)
-        return self.client.new_order(csr_pem)
+        return cast(ClientV2, self.client).new_order(csr_pem)
 
     def finalize_order(self, orderr, deadline, fetch_alternative_chains=False):
         """Finalize an order and obtain a certificate.
@@ -898,8 +892,9 @@ class BackwardsCompatibleClientV2(object):
 
         """
         if self.acme_version == 1:
+            client_v1 = cast(Client, self.client)
             csr_pem = orderr.csr_pem
-            certr = self.client.request_issuance(
+            certr = client_v1.request_issuance(
                 jose.ComparableX509(
                     OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)),
                     orderr.authorizations)
@@ -907,7 +902,7 @@ class BackwardsCompatibleClientV2(object):
             chain = None
             while datetime.datetime.now() < deadline:
                 try:
-                    chain = self.client.fetch_chain(certr)
+                    chain = client_v1.fetch_chain(certr)
                     break
                 except errors.Error:
                     time.sleep(1)
@@ -922,7 +917,8 @@ class BackwardsCompatibleClientV2(object):
             chain = crypto_util.dump_pyopenssl_chain(chain).decode()
 
             return orderr.update(fullchain_pem=(cert + chain))
-        return self.client.finalize_order(orderr, deadline, fetch_alternative_chains)
+        return cast(ClientV2, self.client).finalize_order(
+            orderr, deadline, fetch_alternative_chains)
 
     def revoke(self, cert, rsn):
         """Revoke certificate.
@@ -948,10 +944,10 @@ class BackwardsCompatibleClientV2(object):
         Always return False for ACMEv1 servers, as it doesn't use External Account Binding."""
         if self.acme_version == 1:
             return False
-        return self.client.external_account_required()
+        return cast(ClientV2, self.client).external_account_required()
 
 
-class ClientNetwork(object):
+class ClientNetwork:
     """Wrapper around requests that signs POSTs for authentication.
 
     Also adds user agent, and handles Content-Type.
@@ -981,7 +977,7 @@ class ClientNetwork(object):
         self.account = account
         self.alg = alg
         self.verify_ssl = verify_ssl
-        self._nonces = set() # type: Set[Text]
+        self._nonces: Set[Text] = set()
         self.user_agent = user_agent
         self.session = requests.Session()
         self._default_timeout = timeout
@@ -1141,6 +1137,7 @@ class ClientNetwork(object):
 
         # If content is DER, log the base64 of it instead of raw bytes, to keep
         # binary data out of the logs.
+        debug_content: Union[bytes, str]
         if response.headers.get("Content-Type") == DER_CONTENT_TYPE:
             debug_content = base64.b64encode(response.content)
         else:

acme/acme/crypto_util.py

@@ -5,15 +5,15 @@ import logging
 import os
 import re
 import socket
+from typing import Callable
+from typing import Tuple
+from typing import Union
 
 import josepy as jose
 from OpenSSL import crypto
 from OpenSSL import SSL  # type: ignore # https://github.com/python/typeshed/issues/2052
 
 from acme import errors
-from acme.magic_typing import Callable
-from acme.magic_typing import Tuple
-from acme.magic_typing import Union
 
 logger = logging.getLogger(__name__)
 
@@ -27,7 +27,7 @@ logger = logging.getLogger(__name__)
 _DEFAULT_SSL_METHOD = SSL.SSLv23_METHOD  # type: ignore
 
 
-class _DefaultCertSelection(object):
+class _DefaultCertSelection:
     def __init__(self, certs):
         self.certs = certs
 
@@ -36,7 +36,7 @@ class _DefaultCertSelection(object):
         return self.certs.get(server_name, None)
 
 
-class SSLSocket(object):  # pylint: disable=too-few-public-methods
+class SSLSocket:  # pylint: disable=too-few-public-methods
     """SSL wrapper for sockets.
 
     :ivar socket sock: Original wrapped socket.
@@ -93,7 +93,7 @@ class SSLSocket(object):  # pylint: disable=too-few-public-methods
             new_context.set_alpn_select_callback(self.alpn_selection)
         connection.set_context(new_context)
 
-    class FakeConnection(object):
+    class FakeConnection:
         """Fake OpenSSL.SSL.Connection."""
 
         # pylint: disable=missing-function-docstring
@@ -168,7 +168,7 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu
                 source_address[1]
             ) if any(source_address) else ""
         )
-        socket_tuple = (host, port)  # type: Tuple[str, int]
+        socket_tuple: Tuple[str, int] = (host, port)
         sock = socket.create_connection(socket_tuple, **socket_kwargs)  # type: ignore
     except socket.error as error:
         raise errors.Error(error)
@@ -256,7 +256,7 @@ def _pyopenssl_cert_or_req_san(cert_or_req):
 
     if isinstance(cert_or_req, crypto.X509):
         # pylint: disable=line-too-long
-        func = crypto.dump_certificate # type: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]]
+        func: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]] = crypto.dump_certificate
     else:
         func = crypto.dump_certificate_request
     text = func(crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8")

acme/acme/jws.py

@@ -14,7 +14,9 @@ class Header(jose.Header):
     kid = jose.Field('kid', omitempty=True)
     url = jose.Field('url', omitempty=True)
 
-    @nonce.decoder
+    # Mypy does not understand the josepy magic happening here, and falsely claims
+    # that nonce is redefined. Let's ignore the type check here.
+    @nonce.decoder  # type: ignore
     def nonce(value):  # pylint: disable=no-self-argument,missing-function-docstring
         try:
             return jose.decode_b64jose(value)

acme/acme/magic_typing.py

@@ -1,16 +1,17 @@
-"""Shim class to not have to depend on typing module in prod."""
-import sys
+"""Simple shim around the typing module.
 
+This was useful when this code supported Python 2 and typing wasn't always
+available. This code is being kept for now for backwards compatibility.
 
-class TypingClass(object):
+"""
+import warnings
+from typing import *  # pylint: disable=wildcard-import, unused-wildcard-import
+from typing import Collection, IO  # type: ignore
+
+warnings.warn("acme.magic_typing is deprecated and will be removed in a future release.",
+              DeprecationWarning)
+
+class TypingClass:
     """Ignore import errors by getting anything"""
     def __getattr__(self, name):
-        return None
-
-try:
-    # mypy doesn't respect modifying sys.modules
-    from typing import *  # pylint: disable=wildcard-import, unused-wildcard-import
-    from typing import Collection, IO  # type: ignore
-except ImportError:
-    # mypy complains because TypingClass is not a module
-    sys.modules[__name__] = TypingClass()  # type: ignore
+        return None  # pragma: no cover

acme/acme/messages.py

@@ -1,8 +1,11 @@
 """ACME protocol messages."""
+from collections.abc import Hashable
 import json
+from typing import Any
+from typing import Dict
+from typing import Type
 
 import josepy as jose
-import six
 
 from acme import challenges
 from acme import errors
@@ -11,13 +14,6 @@ from acme import jws
 from acme import util
 from acme.mixins import ResourceMixin
 
-try:
-    from collections.abc import Hashable
-except ImportError:  # pragma: no cover
-    from collections import Hashable
-
-
-
 OLD_ERROR_PREFIX = "urn:acme:error:"
 ERROR_PREFIX = "urn:ietf:params:acme:error:"
 
@@ -68,7 +64,6 @@ def is_acme_error(err):
     return False
 
 
-@six.python_2_unicode_compatible
 class Error(jose.JSONObjectWithFields, errors.Error):
     """ACME error.
 
@@ -95,7 +90,9 @@ class Error(jose.JSONObjectWithFields, errors.Error):
             raise ValueError("The supplied code: %s is not a known ACME error"
                              " code" % code)
         typ = ERROR_PREFIX + code
-        return cls(typ=typ, **kwargs)
+        # Mypy will not understand that the Error constructor accepts a named argument
+        # "typ" because of josepy magic. Let's ignore the type check here.
+        return cls(typ=typ, **kwargs)  # type: ignore
 
     @property
     def description(self):
@@ -132,7 +129,7 @@ class Error(jose.JSONObjectWithFields, errors.Error):
 class _Constant(jose.JSONDeSerializable, Hashable):  # type: ignore
     """ACME constant."""
     __slots__ = ('name',)
-    POSSIBLE_NAMES = NotImplemented
+    POSSIBLE_NAMES: Dict[str, '_Constant'] = NotImplemented
 
     def __init__(self, name):
         super(_Constant, self).__init__()
@@ -158,13 +155,10 @@ class _Constant(jose.JSONDeSerializable, Hashable):  # type: ignore
     def __hash__(self):
         return hash((self.__class__, self.name))
 
-    def __ne__(self, other):
-        return not self == other
-
 
 class Status(_Constant):
     """ACME "status" field."""
-    POSSIBLE_NAMES = {}  # type: dict
+    POSSIBLE_NAMES: dict = {}
 STATUS_UNKNOWN = Status('unknown')
 STATUS_PENDING = Status('pending')
 STATUS_PROCESSING = Status('processing')
@@ -177,7 +171,7 @@ STATUS_DEACTIVATED = Status('deactivated')
 
 class IdentifierType(_Constant):
     """ACME identifier type."""
-    POSSIBLE_NAMES = {}  # type: dict
+    POSSIBLE_NAMES: Dict[str, 'IdentifierType'] = {}
 IDENTIFIER_FQDN = IdentifierType('dns')  # IdentifierDNS in Boulder
 
 
@@ -195,7 +189,7 @@ class Identifier(jose.JSONObjectWithFields):
 class Directory(jose.JSONDeSerializable):
     """Directory."""
 
-    _REGISTERED_TYPES = {}  # type: dict
+    _REGISTERED_TYPES: Dict[str, Type[Any]] = {}
 
     class Meta(jose.JSONObjectWithFields):
         """Directory Meta."""
@@ -229,7 +223,7 @@ class Directory(jose.JSONDeSerializable):
         return getattr(key, 'resource_type', key)
 
     @classmethod
-    def register(cls, resource_body_cls):
+    def register(cls, resource_body_cls: Type[Any]) -> Type[Any]:
         """Register resource."""
         resource_type = resource_body_cls.resource_type
         assert resource_type not in cls._REGISTERED_TYPES
@@ -285,7 +279,7 @@ class ResourceBody(jose.JSONObjectWithFields):
     """ACME Resource Body."""
 
 
-class ExternalAccountBinding(object):
+class ExternalAccountBinding:
     """ACME External Account Binding"""
 
     @classmethod
@@ -539,7 +533,9 @@ class Authorization(ResourceBody):
     expires = fields.RFC3339Field('expires', omitempty=True)
     wildcard = jose.Field('wildcard', omitempty=True)
 
-    @challenges.decoder
+    # Mypy does not understand the josepy magic happening here, and falsely claims
+    # that challenge is redefined. Let's ignore the type check here.
+    @challenges.decoder  # type: ignore
     def challenges(value):  # pylint: disable=no-self-argument,missing-function-docstring
         return tuple(ChallengeBody.from_json(chall) for chall in value)
 
@@ -638,7 +634,9 @@ class Order(ResourceBody):
     expires = fields.RFC3339Field('expires', omitempty=True)
     error = jose.Field('error', omitempty=True, decoder=Error.from_json)
 
-    @identifiers.decoder
+    # Mypy does not understand the josepy magic happening here, and falsely claims
+    # that identifiers is redefined. Let's ignore the type check here.
+    @identifiers.decoder  # type: ignore
     def identifiers(value):  # pylint: disable=no-self-argument,missing-function-docstring
         return tuple(Identifier.from_json(identifier) for identifier in value)
 

acme/acme/mixins.py

@@ -1,7 +1,7 @@
 """Useful mixins for Challenge and Resource objects"""
 
 
-class VersionedLEACMEMixin(object):
+class VersionedLEACMEMixin:
     """This mixin stores the version of Let's Encrypt's endpoint being used."""
     @property
     def le_acme_version(self):

acme/acme/standalone.py

@@ -1,17 +1,16 @@
 """Support for standalone client challenge solvers. """
 import collections
 import functools
+import http.client as http_client
+import http.server as BaseHTTPServer
 import logging
 import socket
+import socketserver
 import threading
-
-from six.moves import BaseHTTPServer  # type: ignore
-from six.moves import http_client
-from six.moves import socketserver  # type: ignore
+from typing import List
 
 from acme import challenges
 from acme import crypto_util
-from acme.magic_typing import List
 
 logger = logging.getLogger(__name__)
 
@@ -54,7 +53,7 @@ class ACMEServerMixin:
     allow_reuse_address = True
 
 
-class BaseDualNetworkedServers(object):
+class BaseDualNetworkedServers:
     """Base class for a pair of IPv6 and IPv4 servers that tries to do everything
        it's asked for both servers, but where failures in one server don't
        affect the other.
@@ -64,8 +63,8 @@ class BaseDualNetworkedServers(object):
 
     def __init__(self, ServerClass, server_address, *remaining_args, **kwargs):
         port = server_address[1]
-        self.threads = [] # type: List[threading.Thread]
-        self.servers = [] # type: List[ACMEServerMixin]
+        self.threads: List[threading.Thread] = []
+        self.servers: List[socketserver.BaseServer] = []
 
         # Must try True first.
         # Ubuntu, for example, will fail to bind to IPv4 if we've already bound
@@ -204,8 +203,24 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 
     def __init__(self, *args, **kwargs):
         self.simple_http_resources = kwargs.pop("simple_http_resources", set())
-        self.timeout = kwargs.pop('timeout', 30)
+        self._timeout = kwargs.pop('timeout', 30)
         BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
+        self.server: HTTP01Server
+
+    # In parent class BaseHTTPRequestHandler, 'timeout' is a class-level property but we
+    # need to define its value during the initialization phase in HTTP01RequestHandler.
+    # However MyPy does not appreciate that we dynamically shadow a class-level property
+    # with an instance-level property (eg. self.timeout = ... in __init__()). So to make
+    # everyone happy, we statically redefine 'timeout' as a method property, and set the
+    # timeout value in a new internal instance-level property _timeout.
+    @property
+    def timeout(self):
+        """
+        The default timeout this server should apply to requests.
+        :return: timeout to apply
+        :rtype: int
+        """
+        return self._timeout
 
     def log_message(self, format, *args):  # pylint: disable=redefined-builtin
         """Log arbitrary message."""

acme/acme/util.py

@@ -1,7 +1,6 @@
 """ACME utilities."""
-import six
 
 
 def map_keys(dikt, func):
     """Map dictionary keys."""
-    return {func(key): value for key, value in six.iteritems(dikt)}
+    return {func(key): value for key, value in dikt.items()}

acme/docs/conf.py

@@ -87,7 +87,6 @@ language = 'en'
 # directories to ignore when looking for source files.
 exclude_patterns = [
     '_build',
-    'man/*'
 ]
 
 # The reST default role (used for this markup: `text`) to use for all

acme/docs/man/jws.rst

@@ -1 +1,3 @@
+:orphan:
+
 .. literalinclude:: ../jws-help.txt

acme/setup.py

@@ -1,11 +1,9 @@
-from distutils.version import LooseVersion
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
@@ -17,21 +15,11 @@ install_requires = [
     'PyOpenSSL>=17.3.0',
     'pyrfc3339',
     'pytz',
-    'requests[security]>=2.6.0',  # security extras added in 2.4.1
+    'requests>=2.6.0',
     'requests-toolbelt>=0.3.0',
     'setuptools>=39.0.1',
-    'six>=1.11.0',
 ]
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 dev_extras = [
     'pytest',
     'pytest-xdist',

acme/tests/challenges_test.py

@@ -1,14 +1,11 @@
 """Tests for acme.challenges."""
+import urllib.parse as urllib_parse
 import unittest
+from unittest import mock
 
 import josepy as jose
 import OpenSSL
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
 import requests
-from six.moves.urllib import parse as urllib_parse
 
 from acme import errors
 

acme/tests/client_test.py

@@ -2,17 +2,15 @@
 # pylint: disable=too-many-lines
 import copy
 import datetime
+import http.client as http_client
 import json
 import unittest
+from typing import Dict
+from unittest import mock
 
 import josepy as jose
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
 import OpenSSL
 import requests
-from six.moves import http_client  # pylint: disable=import-error
 
 from acme import challenges
 from acme import errors
@@ -64,7 +62,7 @@ class ClientTestBase(unittest.TestCase):
         self.contact = ('mailto:cert-admin@example.com', 'tel:+12025551212')
         reg = messages.Registration(
             contact=self.contact, key=KEY.public_key())
-        the_arg = dict(reg)  # type: Dict
+        the_arg: Dict = dict(reg)
         self.new_reg = messages.NewRegistration(**the_arg)
         self.regr = messages.RegistrationResource(
             body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1')

acme/tests/crypto_util_test.py

@@ -1,14 +1,14 @@
 """Tests for acme.crypto_util."""
 import itertools
 import socket
+import socketserver
 import threading
 import time
 import unittest
+from typing import List
 
 import josepy as jose
 import OpenSSL
-import six
-from six.moves import socketserver  # type: ignore  # pylint: disable=import-error
 
 from acme import errors
 import test_util
@@ -27,8 +27,6 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
 
         class _TestServer(socketserver.TCPServer):
 
-            # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init
-
             def server_bind(self):  # pylint: disable=missing-docstring
                 self.socket = SSLSocket(socket.socket(),
                         certs)
@@ -62,7 +60,6 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
         self.assertRaises(errors.Error, self._probe, b'bar')
 
     def test_probe_connection_error(self):
-        # pylint has a hard time with six
         self.server.server_close()
         original_timeout = socket.getdefaulttimeout()
         try:
@@ -121,9 +118,9 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase):
     @classmethod
     def _get_idn_names(cls):
         """Returns expected names from '{cert,csr}-idnsans.pem'."""
-        chars = [six.unichr(i) for i in itertools.chain(range(0x3c3, 0x400),
-                                                        range(0x641, 0x6fc),
-                                                        range(0x1820, 0x1877))]
+        chars = [chr(i) for i in itertools.chain(range(0x3c3, 0x400),
+                                                 range(0x641, 0x6fc),
+                                                 range(0x1820, 0x1877))]
         return [''.join(chars[i: i + 45]) + '.invalid'
                 for i in range(0, len(chars), 45)]
 
@@ -184,7 +181,7 @@ class RandomSnTest(unittest.TestCase):
 
     def setUp(self):
         self.cert_count = 5
-        self.serial_num = [] # type: List[int]
+        self.serial_num: List[int] = []
         self.key = OpenSSL.crypto.PKey()
         self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
 

acme/tests/errors_test.py

@@ -1,10 +1,6 @@
 """Tests for acme.errors."""
 import unittest
-
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+from unittest import mock
 
 
 class BadNonceTest(unittest.TestCase):

acme/tests/magic_typing_test.py

@@ -1,11 +1,8 @@
 """Tests for acme.magic_typing."""
 import sys
 import unittest
-
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+import warnings
+from unittest import mock
 
 
 class MagicTypingTest(unittest.TestCase):
@@ -13,32 +10,21 @@ class MagicTypingTest(unittest.TestCase):
     def test_import_success(self):
         try:
             import typing as temp_typing
-        except ImportError: # pragma: no cover
-            temp_typing = None # pragma: no cover
+        except ImportError:  # pragma: no cover
+            temp_typing = None  # pragma: no cover
         typing_class_mock = mock.MagicMock()
         text_mock = mock.MagicMock()
         typing_class_mock.Text = text_mock
         sys.modules['typing'] = typing_class_mock
         if 'acme.magic_typing' in sys.modules:
-            del sys.modules['acme.magic_typing'] # pragma: no cover
-        from acme.magic_typing import Text
+            del sys.modules['acme.magic_typing']  # pragma: no cover
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", category=DeprecationWarning)
+            from acme.magic_typing import Text
         self.assertEqual(Text, text_mock)
         del sys.modules['acme.magic_typing']
         sys.modules['typing'] = temp_typing
 
-    def test_import_failure(self):
-        try:
-            import typing as temp_typing
-        except ImportError: # pragma: no cover
-            temp_typing = None # pragma: no cover
-        sys.modules['typing'] = None
-        if 'acme.magic_typing' in sys.modules:
-            del sys.modules['acme.magic_typing'] # pragma: no cover
-        from acme.magic_typing import Text
-        self.assertTrue(Text is None)
-        del sys.modules['acme.magic_typing']
-        sys.modules['typing'] = temp_typing
-
 
 if __name__ == '__main__':
     unittest.main()  # pragma: no cover

acme/tests/messages_test.py

@@ -1,11 +1,9 @@
 """Tests for acme.messages."""
+from typing import Dict
 import unittest
+from unittest import mock
 
 import josepy as jose
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
 
 from acme import challenges
 import test_util
@@ -84,7 +82,7 @@ class ConstantTest(unittest.TestCase):
         from acme.messages import _Constant
 
         class MockConstant(_Constant):  # pylint: disable=missing-docstring
-            POSSIBLE_NAMES = {} # type: Dict
+            POSSIBLE_NAMES: Dict = {}
 
         self.MockConstant = MockConstant  # pylint: disable=invalid-name
         self.const_a = MockConstant('a')

acme/tests/standalone_test.py

@@ -1,16 +1,14 @@
 """Tests for acme.standalone."""
+import http.client as http_client
 import socket
+import socketserver
 import threading
 import unittest
+from typing import Set
+from unittest import mock
 
 import josepy as jose
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
 import requests
-from six.moves import http_client  # pylint: disable=import-error
-from six.moves import socketserver  # type: ignore  # pylint: disable=import-error
 
 from acme import challenges
 from acme import crypto_util
@@ -44,7 +42,7 @@ class HTTP01ServerTest(unittest.TestCase):
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))
-        self.resources = set() # type: Set
+        self.resources: Set = set()
 
         from acme.standalone import HTTP01Server
         self.server = HTTP01Server(('', 0), resources=self.resources)
@@ -221,7 +219,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase):
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))
-        self.resources = set() # type: Set
+        self.resources: Set = set()
 
         from acme.standalone import HTTP01DualNetworkedServers
         self.servers = HTTP01DualNetworkedServers(('', 0), resources=self.resources)

certbot-apache/certbot_apache/_internal/apache_util.py

@@ -9,7 +9,6 @@ import pkg_resources
 
 from certbot import errors
 from certbot import util
-
 from certbot.compat import os
 
 logger = logging.getLogger(__name__)

certbot-apache/certbot_apache/_internal/apacheparser.py

@@ -1,4 +1,5 @@
 """ apacheconfig implementation of the ParserNode interfaces """
+from typing import Tuple
 
 from certbot_apache._internal import assertions
 from certbot_apache._internal import interfaces
@@ -21,7 +22,7 @@ class ApacheParserNode(interfaces.ParserNode):
         self.metadata = metadata
         self._raw = self.metadata["ac_ast"]
 
-    def save(self, msg): # pragma: no cover
+    def save(self, msg):  # pragma: no cover
         pass
 
     def find_ancestors(self, name):  # pylint: disable=unused-variable
@@ -83,7 +84,7 @@ class ApacheBlockNode(ApacheDirectiveNode):
 
     def __init__(self, **kwargs):
         super(ApacheBlockNode, self).__init__(**kwargs)
-        self.children = ()
+        self.children: Tuple[ApacheParserNode, ...] = ()
 
     def __eq__(self, other):  # pragma: no cover
         if isinstance(other, self.__class__):

certbot-apache/certbot_apache/_internal/assertions.py

@@ -3,7 +3,6 @@ import fnmatch
 
 from certbot_apache._internal import interfaces
 
-
 PASS = "CERTBOT_PASS_ASSERT"
 
 

certbot-apache/certbot_apache/_internal/augeasparser.py

@@ -64,10 +64,10 @@ Translates over to:
     "/files/etc/apache2/apache2.conf/bLoCk[1]",
 ]
 """
-from acme.magic_typing import Set
+from typing import Set
+
 from certbot import errors
 from certbot.compat import os
-
 from certbot_apache._internal import apache_util
 from certbot_apache._internal import assertions
 from certbot_apache._internal import interfaces
@@ -355,7 +355,7 @@ class AugeasBlockNode(AugeasDirectiveNode):
         ownpath = self.metadata.get("augeaspath")
 
         directives = self.parser.find_dir(name, start=ownpath, exclude=exclude)
-        already_parsed = set()  # type: Set[str]
+        already_parsed: Set[str] = set()
         for directive in directives:
             # Remove the /arg part from the Augeas path
             directive = directive.partition("/arg")[0]

certbot-apache/certbot_apache/_internal/configurator.py

@@ -1,28 +1,24 @@
 """Apache Configurator."""
 # pylint: disable=too-many-lines
 from collections import defaultdict
-from distutils.version import LooseVersion
 import copy
+from distutils.version import LooseVersion
 import fnmatch
 import logging
 import re
 import socket
 import time
+from typing import cast
+from typing import DefaultDict
+from typing import Dict
+from typing import List
+from typing import Set
+from typing import Union
 
 import zope.component
 import zope.interface
-try:
-    import apacheconfig
-    HAS_APACHECONFIG = True
-except ImportError:  # pragma: no cover
-    HAS_APACHECONFIG = False
 
 from acme import challenges
-from acme.magic_typing import DefaultDict
-from acme.magic_typing import Dict
-from acme.magic_typing import List
-from acme.magic_typing import Set
-from acme.magic_typing import Union
 from certbot import errors
 from certbot import interfaces
 from certbot import util
@@ -41,6 +37,13 @@ from certbot_apache._internal import http_01
 from certbot_apache._internal import obj
 from certbot_apache._internal import parser
 
+try:
+    import apacheconfig
+    HAS_APACHECONFIG = True
+except ImportError:  # pragma: no cover
+    HAS_APACHECONFIG = False
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -154,9 +157,9 @@ class ApacheConfigurator(common.Installer):
                 self.options[o] = self.OS_DEFAULTS[o]
 
         # Special cases
-        self.options["version_cmd"][0] = self.option("ctl")
-        self.options["restart_cmd"][0] = self.option("ctl")
-        self.options["conftest_cmd"][0] = self.option("ctl")
+        cast(List[str], self.options["version_cmd"])[0] = self.option("ctl")
+        cast(List[str], self.options["restart_cmd"])[0] = self.option("ctl")
+        cast(List[str], self.options["conftest_cmd"])[0] = self.option("ctl")
 
     @classmethod
     def add_parser_arguments(cls, add):
@@ -210,23 +213,23 @@ class ApacheConfigurator(common.Installer):
         super(ApacheConfigurator, self).__init__(*args, **kwargs)
 
         # Add name_server association dict
-        self.assoc = {}  # type: Dict[str, obj.VirtualHost]
+        self.assoc: Dict[str, obj.VirtualHost] = {}
         # Outstanding challenges
-        self._chall_out = set()  # type: Set[KeyAuthorizationAnnotatedChallenge]
+        self._chall_out: Set[KeyAuthorizationAnnotatedChallenge] = set()
         # List of vhosts configured per wildcard domain on this run.
         # used by deploy_cert() and enhance()
-        self._wildcard_vhosts = {}  # type: Dict[str, List[obj.VirtualHost]]
+        self._wildcard_vhosts: Dict[str, List[obj.VirtualHost]] = {}
         # Maps enhancements to vhosts we've enabled the enhancement for
-        self._enhanced_vhosts = defaultdict(set)  # type: DefaultDict[str, Set[obj.VirtualHost]]
+        self._enhanced_vhosts: DefaultDict[str, Set[obj.VirtualHost]] = defaultdict(set)
         # Temporary state for AutoHSTS enhancement
-        self._autohsts = {}  # type: Dict[str, Dict[str, Union[int, float]]]
+        self._autohsts: Dict[str, Dict[str, Union[int, float]]] = {}
         # Reverter save notes
         self.save_notes = ""
         # Should we use ParserNode implementation instead of the old behavior
         self.USE_PARSERNODE = use_parsernode
         # Saves the list of file paths that were parsed initially, and
         # not added to parser tree by self.conf("vhost-root") for example.
-        self.parsed_paths = []  # type: List[str]
+        self.parsed_paths: List[str] = []
         # These will be set in the prepare function
         self._prepared = False
         self.parser = None
@@ -832,7 +835,7 @@ class ApacheConfigurator(common.Installer):
         :rtype: set
 
         """
-        all_names = set()  # type: Set[str]
+        all_names: Set[str] = set()
 
         vhost_macro = []
 
@@ -996,8 +999,8 @@ class ApacheConfigurator(common.Installer):
 
         """
         # Search base config, and all included paths for VirtualHosts
-        file_paths = {}  # type: Dict[str, str]
-        internal_paths = defaultdict(set)  # type: DefaultDict[str, Set[str]]
+        file_paths: Dict[str, str] = {}
+        internal_paths: DefaultDict[str, Set[str]] = defaultdict(set)
         vhs = []
         # Make a list of parser paths because the parser_paths
         # dictionary may be modified during the loop.
@@ -2156,7 +2159,7 @@ class ApacheConfigurator(common.Installer):
         # There can be other RewriteRule directive lines in vhost config.
         # rewrite_args_dict keys are directive ids and the corresponding value
         # for each is a list of arguments to that directive.
-        rewrite_args_dict = defaultdict(list)  # type: DefaultDict[str, List[str]]
+        rewrite_args_dict: DefaultDict[str, List[str]] = defaultdict(list)
         pat = r'(.*directive\[\d+\]).*'
         for match in rewrite_path:
             m = re.match(pat, match)
@@ -2250,7 +2253,7 @@ class ApacheConfigurator(common.Installer):
         if ssl_vhost.aliases:
             serveralias = "ServerAlias " + " ".join(ssl_vhost.aliases)
 
-        rewrite_rule_args = []  # type: List[str]
+        rewrite_rule_args: List[str] = []
         if self.get_version() >= (2, 3, 9):
             rewrite_rule_args = constants.REWRITE_HTTPS_ARGS_WITH_END
         else:

certbot-apache/certbot_apache/_internal/dualparser.py

@@ -1,10 +1,10 @@
 """ Dual ParserNode implementation """
+from certbot_apache._internal import apacheparser
 from certbot_apache._internal import assertions
 from certbot_apache._internal import augeasparser
-from certbot_apache._internal import apacheparser
 
 
-class DualNodeBase(object):
+class DualNodeBase:
     """ Dual parser interface for in development testing. This is used as the
     base class for dual parser interface classes. This class handles runtime
     attribute value assertions."""

certbot-apache/certbot_apache/_internal/http_01.py

@@ -1,9 +1,9 @@
 """A class that performs HTTP-01 challenges for Apache"""
-import logging
 import errno
+import logging
+from typing import List
+from typing import Set
 
-from acme.magic_typing import List
-from acme.magic_typing import Set
 from certbot import errors
 from certbot.compat import filesystem
 from certbot.compat import os
@@ -57,7 +57,7 @@ class ApacheHttp01(common.ChallengePerformer):
         self.challenge_dir = os.path.join(
             self.configurator.config.work_dir,
             "http_challenges")
-        self.moded_vhosts = set()  # type: Set[VirtualHost]
+        self.moded_vhosts: Set[VirtualHost] = set()
 
     def perform(self):
         """Perform all HTTP-01 challenges."""
@@ -93,7 +93,7 @@ class ApacheHttp01(common.ChallengePerformer):
                     self.configurator.enable_mod(mod, temp=True)
 
     def _mod_config(self):
-        selected_vhosts = []  # type: List[VirtualHost]
+        selected_vhosts: List[VirtualHost] = []
         http_port = str(self.configurator.config.http01_port)
         for chall in self.achalls:
             # Search for matching VirtualHosts

certbot-apache/certbot_apache/_internal/interfaces.py

@@ -100,12 +100,9 @@ For this reason the internal representation of data should not ignore the case.
 """
 
 import abc
-import six
 
 
-
-@six.add_metaclass(abc.ABCMeta)
-class ParserNode(object):
+class ParserNode(object, metaclass=abc.ABCMeta):
     """
     ParserNode is the basic building block of the tree of such nodes,
     representing the structure of the configuration. It is largely meant to keep
@@ -204,9 +201,7 @@ class ParserNode(object):
         """
 
 
-# Linter rule exclusion done because of https://github.com/PyCQA/pylint/issues/179
-@six.add_metaclass(abc.ABCMeta)  # pylint: disable=abstract-method
-class CommentNode(ParserNode):
+class CommentNode(ParserNode, metaclass=abc.ABCMeta):
     """
     CommentNode class is used for representation of comments within the parsed
     configuration structure. Because of the nature of comments, it is not able
@@ -249,8 +244,7 @@ class CommentNode(ParserNode):
                                           metadata=kwargs.get('metadata', {}))  # pragma: no cover
 
 
-@six.add_metaclass(abc.ABCMeta)
-class DirectiveNode(ParserNode):
+class DirectiveNode(ParserNode, metaclass=abc.ABCMeta):
     """
     DirectiveNode class represents a configuration directive within the configuration.
     It can have zero or more parameters attached to it. Because of the nature of
@@ -325,8 +319,7 @@ class DirectiveNode(ParserNode):
         """
 
 
-@six.add_metaclass(abc.ABCMeta)
-class BlockNode(DirectiveNode):
+class BlockNode(DirectiveNode, metaclass=abc.ABCMeta):
     """
     BlockNode class represents a block of nested configuration directives, comments
     and other blocks as its children. A BlockNode can have zero or more parameters

certbot-apache/certbot_apache/_internal/obj.py

@@ -1,7 +1,7 @@
 """Module contains classes used by the Apache Configurator."""
 import re
+from typing import Set
 
-from acme.magic_typing import Set
 from certbot.plugins import common
 
 
@@ -20,9 +20,6 @@ class Addr(common.Addr):
                      self.is_wildcard() and other.is_wildcard()))
         return False
 
-    def __ne__(self, other):
-        return not self.__eq__(other)
-
     def __repr__(self):
         return "certbot_apache._internal.obj.Addr(" + repr(self.tup) + ")"
 
@@ -98,7 +95,7 @@ class Addr(common.Addr):
         return self.get_addr_obj(port)
 
 
-class VirtualHost(object):
+class VirtualHost:
     """Represents an Apache Virtualhost.
 
     :ivar str filep: file path of VH
@@ -140,7 +137,7 @@ class VirtualHost(object):
 
     def get_names(self):
         """Return a set of all names."""
-        all_names = set()  # type: Set[str]
+        all_names: Set[str] = set()
         all_names.update(self.aliases)
         # Strip out any scheme:// and <port> field from servername
         if self.name is not None:
@@ -191,9 +188,6 @@ class VirtualHost(object):
 
         return False
 
-    def __ne__(self, other):
-        return not self.__eq__(other)
-
     def __hash__(self):
         return hash((self.filep, self.path,
                      tuple(self.addrs), tuple(self.get_names()),
@@ -251,7 +245,7 @@ class VirtualHost(object):
 
         # already_found acts to keep everything very conservative.
         # Don't allow multiple ip:ports in same set.
-        already_found = set()  # type: Set[str]
+        already_found: Set[str] = set()
 
         for addr in vhost.addrs:
             for local_addr in self.addrs:

certbot-apache/certbot_apache/_internal/override_centos.py

@@ -1,9 +1,10 @@
 """ Distribution specific override class for CentOS family (RHEL, Fedora) """
 import logging
+from typing import cast
+from typing import List
 
 import zope.interface
 
-from acme.magic_typing import List
 from certbot import errors
 from certbot import interfaces
 from certbot import util
@@ -76,7 +77,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
         alternative restart cmd used in CentOS.
         """
         super(CentOSConfigurator, self)._prepare_options()
-        self.options["restart_cmd_alt"][0] = self.option("ctl")
+        cast(List[str], self.options["restart_cmd_alt"])[0] = self.option("ctl")
 
     def get_parser(self):
         """Initializes the ApacheParser"""
@@ -102,9 +103,9 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
 
         loadmods = self.parser.find_dir("LoadModule", "ssl_module", exclude=False)
 
-        correct_ifmods = []  # type: List[str]
-        loadmod_args = []  # type: List[str]
-        loadmod_paths = []  # type: List[str]
+        correct_ifmods: List[str] = []
+        loadmod_args: List[str] = []
+        loadmod_paths: List[str] = []
         for m in loadmods:
             noarg_path = m.rpartition("/")[0]
             path_args = self.parser.get_all_args(noarg_path)

certbot-apache/certbot_apache/_internal/override_fedora.py

@@ -1,4 +1,7 @@
 """ Distribution specific override class for Fedora 29+ """
+from typing import cast
+from typing import List
+
 import zope.interface
 
 from certbot import errors
@@ -69,9 +72,9 @@ class FedoraConfigurator(configurator.ApacheConfigurator):
         of Fedora to restart httpd.
         """
         super(FedoraConfigurator, self)._prepare_options()
-        self.options["restart_cmd"][0] = 'apachectl'
-        self.options["restart_cmd_alt"][0] = 'apachectl'
-        self.options["conftest_cmd"][0] = 'apachectl'
+        cast(List[str], self.options["restart_cmd"])[0] = 'apachectl'
+        cast(List[str], self.options["restart_cmd_alt"])[0] = 'apachectl'
+        cast(List[str], self.options["conftest_cmd"])[0] = 'apachectl'
 
 
 class FedoraParser(parser.ApacheParser):

certbot-apache/certbot_apache/_internal/override_gentoo.py

@@ -1,4 +1,7 @@
 """ Distribution specific override class for Gentoo Linux """
+from typing import cast
+from typing import List
+
 import zope.interface
 
 from certbot import interfaces
@@ -36,7 +39,7 @@ class GentooConfigurator(configurator.ApacheConfigurator):
         alternative restart cmd used in Gentoo.
         """
         super(GentooConfigurator, self)._prepare_options()
-        self.options["restart_cmd_alt"][0] = self.option("ctl")
+        cast(List[str], self.options["restart_cmd_alt"])[0] = self.option("ctl")
 
     def get_parser(self):
         """Initializes the ApacheParser"""

certbot-apache/certbot_apache/_internal/parser.py

@@ -3,12 +3,9 @@ import copy
 import fnmatch
 import logging
 import re
-import sys
+from typing import Dict
+from typing import List
 
-import six
-
-from acme.magic_typing import Dict
-from acme.magic_typing import List
 from certbot import errors
 from certbot.compat import os
 from certbot_apache._internal import apache_util
@@ -17,7 +14,7 @@ from certbot_apache._internal import constants
 logger = logging.getLogger(__name__)
 
 
-class ApacheParser(object):
+class ApacheParser:
     """Class handles the fine details of parsing the Apache Configuration.
 
     .. todo:: Make parsing general... remove sites-available etc...
@@ -51,9 +48,9 @@ class ApacheParser(object):
                 "version 1.2.0 or higher, please make sure you have you have "
                 "those installed.")
 
-        self.modules = {}  # type: Dict[str, str]
-        self.parser_paths = {}  # type: Dict[str, List[str]]
-        self.variables = {}  # type: Dict[str, str]
+        self.modules: Dict[str, str] = {}
+        self.parser_paths: Dict[str, List[str]] = {}
+        self.variables: Dict[str, str] = {}
 
         # Find configuration root and make sure augeas can parse it.
         self.root = os.path.abspath(root)
@@ -266,7 +263,7 @@ class ApacheParser(object):
             the iteration issue.  Else... parse and enable mods at same time.
 
         """
-        mods = {}  # type: Dict[str, str]
+        mods: Dict[str, str] = {}
         matches = self.find_dir("LoadModule")
         iterator = iter(matches)
         # Make sure prev_size != cur_size for do: while: iteration
@@ -275,7 +272,7 @@ class ApacheParser(object):
         while len(mods) != prev_size:
             prev_size = len(mods)
 
-            for match_name, match_filename in six.moves.zip(
+            for match_name, match_filename in zip(
                     iterator, iterator):
                 mod_name = self.get_arg(match_name)
                 mod_filename = self.get_arg(match_filename)
@@ -553,7 +550,7 @@ class ApacheParser(object):
         else:
             arg_suffix = "/*[self::arg=~regexp('%s')]" % case_i(arg)
 
-        ordered_matches = []  # type: List[str]
+        ordered_matches: List[str] = []
 
         # TODO: Wildcards should be included in alphabetical order
         # https://httpd.apache.org/docs/2.4/mod/core.html#include
@@ -738,9 +735,6 @@ class ApacheParser(object):
         :rtype: str
 
         """
-        if sys.version_info < (3, 6):
-            # This strips off final /Z(?ms)
-            return fnmatch.translate(clean_fn_match)[:-7]  # pragma: no cover
         # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z
         return fnmatch.translate(clean_fn_match)[4:-3]  # pragma: no cover
 

certbot-apache/setup.py

@@ -1,11 +1,7 @@
-from distutils.version import LooseVersion
-import sys
-
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -18,15 +14,6 @@ install_requires = [
     'zope.interface',
 ]
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 dev_extras = [
     'apacheconfig>=0.3.2',
 ]

certbot-apache/tests/augeasnode_test.py

@@ -1,4 +1,6 @@
 """Tests for AugeasParserNode classes"""
+from typing import List
+
 try:
     import mock
 except ImportError: # pragma: no cover
@@ -107,7 +109,7 @@ class AugeasParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-
 
     def test_set_parameters(self):
         servernames = self.config.parser_root.find_directives("servername")
-        names = []  # type: List[str]
+        names: List[str] = []
         for servername in servernames:
             names += servername.parameters
         self.assertFalse("going_to_set_this" in names)

certbot-apache/tests/autohsts_test.py

@@ -7,7 +7,6 @@ try:
     import mock
 except ImportError: # pragma: no cover
     from unittest import mock # type: ignore
-import six  # pylint: disable=unused-import  # six is used in mock.patch()
 
 from certbot import errors
 from certbot_apache._internal import constants

certbot-apache/tests/configurator_test.py

@@ -10,7 +10,6 @@ try:
     import mock
 except ImportError: # pragma: no cover
     from unittest import mock # type: ignore
-import six  # pylint: disable=unused-import  # six is used in mock.patch()
 
 from acme import challenges
 from certbot import achallenges
@@ -726,7 +725,7 @@ class MultipleVhostsTest(util.ApacheTest):
         # This calls open
         self.config.reverter.register_file_creation = mock.Mock()
         mock_open.side_effect = IOError
-        with mock.patch("six.moves.builtins.open", mock_open):
+        with mock.patch("builtins.open", mock_open):
             self.assertRaises(
                 errors.PluginError,
                 self.config.make_vhost_ssl, self.vh_truth[0])
@@ -1834,7 +1833,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
 
     def test_open_module_file(self):
         mock_open = mock.mock_open(read_data="testing 12 3")
-        with mock.patch("six.moves.builtins.open", mock_open):
+        with mock.patch("builtins.open", mock_open):
             self.assertEqual(self.config._open_module_file("/nonsense/"), "testing 12 3")
 
 if __name__ == "__main__":

certbot-apache/tests/http_01_test.py

@@ -1,6 +1,7 @@
 """Test for certbot_apache._internal.http_01."""
 import unittest
 import errno
+from typing import List
 
 try:
     import mock
@@ -26,7 +27,7 @@ class ApacheHttp01Test(util.ApacheTest):
         super(ApacheHttp01Test, self).setUp(*args, **kwargs)
 
         self.account_key = self.rsa512jwk
-        self.achalls = []  # type: List[achallenges.KeyAuthorizationAnnotatedChallenge]
+        self.achalls: List[achallenges.KeyAuthorizationAnnotatedChallenge] = []
         vh_truth = util.get_vh_truth(
             self.temp_dir, "debian_apache_2_4/multiple_vhosts")
         # Takes the vhosts for encryption-example.demo, certbot.demo

certbot-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="1.11.0"
+LE_AUTO_VERSION="1.13.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -803,8 +803,10 @@ if [ -f /etc/debian_version ]; then
 elif [ -f /etc/mageia-release ]; then
   # Mageia has both /etc/mageia-release and /etc/redhat-release
   DEPRECATED_OS=1
+  NO_SELF_UPGRADE=1
 elif [ -f /etc/redhat-release ]; then
   DEPRECATED_OS=1
+  NO_SELF_UPGRADE=1
   # Run DeterminePythonVersion to decide on the basis of available Python versions
   # whether to use 2.x or 3.x on RedHat-like systems.
   # Then, revert LE_PYTHON to its previous state.
@@ -863,22 +865,31 @@ elif [ -f /etc/redhat-release ]; then
   LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
   DEPRECATED_OS=1
+  NO_SELF_UPGRADE=1
 elif [ -f /etc/arch-release ]; then
   DEPRECATED_OS=1
+  NO_SELF_UPGRADE=1
 elif [ -f /etc/manjaro-release ]; then
   DEPRECATED_OS=1
+  NO_SELF_UPGRADE=1
 elif [ -f /etc/gentoo-release ]; then
   DEPRECATED_OS=1
+  NO_SELF_UPGRADE=1
 elif uname | grep -iq FreeBSD ; then
   DEPRECATED_OS=1
+  NO_SELF_UPGRADE=1
 elif uname | grep -iq Darwin ; then
   DEPRECATED_OS=1
+  NO_SELF_UPGRADE=1
 elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then
   DEPRECATED_OS=1
+  NO_SELF_UPGRADE=1
 elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then
   DEPRECATED_OS=1
+  NO_SELF_UPGRADE=1
 else
   DEPRECATED_OS=1
+  NO_SELF_UPGRADE=1
 fi
 
 # We handle this case after determining the normal bootstrap version to allow
@@ -1107,7 +1118,9 @@ if [ "$1" = "--le-auto-phase2" ]; then
     fi
 
     if [ -f "$VENV_BIN/letsencrypt" -a "$INSTALL_ONLY" != 1 ]; then
-      error "Certbot will no longer receive updates."
+      error "certbot-auto and its Certbot installation will no longer receive updates."
+      error "You will not receive any bug fixes including those fixing server compatibility"
+      error "or security problems."
       error "Please visit https://certbot.eff.org/ to check for other alternatives."
       "$VENV_BIN/letsencrypt" "$@"
       exit 0
@@ -1475,18 +1488,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==1.11.0 \
-    --hash=sha256:b7faa66c40a1ce5a31bfc8668d8feb5d2db6f7af9e791079a6d95c77b6593bf4 \
-    --hash=sha256:6b0ce04e55379aff0a47f873fa05c084538ad0f4a9b79f33108dbb0a7a668b43
-acme==1.11.0 \
-    --hash=sha256:77d6ce61b155315d7d7031489bbd245c0ea42c0453a04d4304393414e741a56d \
-    --hash=sha256:092eb09a074a935da4c10f66cb8634ffb2cc2d2cc1035d2998d608996efab924
-certbot-apache==1.11.0 \
-    --hash=sha256:ea7ac88733aad91a89c700289effda2a0c0658778da1ae2c54a0aefaee351285 \
-    --hash=sha256:3ed001427ec0b49324f2b9af7170fa6e6e88948fa51c3678b07bf17f8138863d
-certbot-nginx==1.11.0 \
-    --hash=sha256:79de69782a1199e577787ff9790dee02a44aac17dbecd6a7287593030842a306 \
-    --hash=sha256:9afe611f99a78b8898941b8ad7bdcf7f3c2b6e0fce27125268f7c713e64b34ee
+certbot==1.13.0 \
+    --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \
+    --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458
+acme==1.13.0 \
+    --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \
+    --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088
+certbot-apache==1.13.0 \
+    --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \
+    --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1
+certbot-nginx==1.13.0 \
+    --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \
+    --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------

certbot-ci/certbot_integration_tests/assets/hook.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import print_function
 import os
 import sys
 

certbot-ci/certbot_integration_tests/certbot_tests/context.py

@@ -7,14 +7,14 @@ import tempfile
 from certbot_integration_tests.utils import certbot_call
 
 
-class IntegrationTestsContext(object):
+class IntegrationTestsContext:
     """General fixture describing a certbot integration tests context"""
     def __init__(self, request):
         self.request = request
 
-        if hasattr(request.config, 'slaveinput'):  # Worker node
-            self.worker_id = request.config.slaveinput['slaveid']
-            acme_xdist = request.config.slaveinput['acme_xdist']
+        if hasattr(request.config, 'workerinput'):  # Worker node
+            self.worker_id = request.config.workerinput['workerid']
+            acme_xdist = request.config.workerinput['acme_xdist']
         else:  # Primary node
             self.worker_id = 'primary'
             acme_xdist = request.config.acme_xdist

certbot-ci/certbot_integration_tests/certbot_tests/test_main.py

@@ -1,5 +1,4 @@
 """Module executing integration tests against certbot core."""
-from __future__ import print_function
 
 import os
 from os.path import exists
@@ -9,19 +8,20 @@ import shutil
 import subprocess
 import time
 
-from cryptography.hazmat.primitives.asymmetric.ec import SECP256R1, SECP384R1, SECP521R1
+from cryptography.hazmat.primitives.asymmetric.ec import SECP256R1
+from cryptography.hazmat.primitives.asymmetric.ec import SECP384R1
+from cryptography.hazmat.primitives.asymmetric.ec import SECP521R1
 from cryptography.x509 import NameOID
-
 import pytest
 
 from certbot_integration_tests.certbot_tests import context as certbot_context
 from certbot_integration_tests.certbot_tests.assertions import assert_cert_count_for_lineage
 from certbot_integration_tests.certbot_tests.assertions import assert_elliptic_key
-from certbot_integration_tests.certbot_tests.assertions import assert_rsa_key
 from certbot_integration_tests.certbot_tests.assertions import assert_equals_group_owner
 from certbot_integration_tests.certbot_tests.assertions import assert_equals_group_permissions
 from certbot_integration_tests.certbot_tests.assertions import assert_equals_world_read_permissions
 from certbot_integration_tests.certbot_tests.assertions import assert_hook_execution
+from certbot_integration_tests.certbot_tests.assertions import assert_rsa_key
 from certbot_integration_tests.certbot_tests.assertions import assert_saved_renew_hook
 from certbot_integration_tests.certbot_tests.assertions import assert_world_no_permissions
 from certbot_integration_tests.certbot_tests.assertions import assert_world_read_permissions

certbot-ci/certbot_integration_tests/conftest.py

@@ -6,7 +6,6 @@ for a directory a specific configuration using built-in pytest hooks.
 
 See https://docs.pytest.org/en/latest/reference.html#hook-reference
 """
-from __future__ import print_function
 import contextlib
 import subprocess
 import sys
@@ -35,7 +34,7 @@ def pytest_configure(config):
     Standard pytest hook used to add a configuration logic for each node of a pytest run.
     :param config: the current pytest configuration
     """
-    if not hasattr(config, 'slaveinput'):  # If true, this is the primary node
+    if not hasattr(config, 'workerinput'):  # If true, this is the primary node
         with _print_on_err():
             _setup_primary_node(config)
 
@@ -45,8 +44,8 @@ def pytest_configure_node(node):
     Standard pytest-xdist hook used to configure a worker node.
     :param node: current worker node
     """
-    node.slaveinput['acme_xdist'] = node.config.acme_xdist
-    node.slaveinput['dns_xdist'] = node.config.dns_xdist
+    node.workerinput['acme_xdist'] = node.config.acme_xdist
+    node.workerinput['dns_xdist'] = node.config.dns_xdist
 
 
 @contextlib.contextmanager

certbot-ci/certbot_integration_tests/nginx_tests/test_main.py

@@ -1,8 +1,8 @@
 """Module executing integration tests against certbot with nginx plugin."""
 import os
 import ssl
-
 from typing import List
+
 import pytest
 
 from certbot_integration_tests.nginx_tests import context as nginx_context
@@ -32,8 +32,8 @@ def test_context(request):
         '--preferred-challenges', 'http'
     ], {'default_server': False}),
 ], indirect=['context'])
-def test_certificate_deployment(certname_pattern, params, context):
-    # type: (str, List[str], nginx_context.IntegrationTestsContext) -> None
+def test_certificate_deployment(certname_pattern: str, params: List[str],
+                                context: nginx_context.IntegrationTestsContext) -> None:
     """
     Test various scenarios to deploy a certificate to nginx using certbot.
     """

certbot-ci/certbot_integration_tests/rfc2136_tests/context.py

@@ -1,7 +1,7 @@
 """Module to handle the context of RFC2136 integration tests."""
 
-import tempfile
 from contextlib import contextmanager
+import tempfile
 
 from pkg_resources import resource_filename
 from pytest import skip
@@ -18,8 +18,8 @@ class IntegrationTestsContext(certbot_context.IntegrationTestsContext):
         self.request = request
 
         self._dns_xdist = None
-        if hasattr(request.config, 'slaveinput'):  # Worker node
-            self._dns_xdist = request.config.slaveinput['dns_xdist']
+        if hasattr(request.config, 'workerinput'):  # Worker node
+            self._dns_xdist = request.config.workerinput['dns_xdist']
         else:  # Primary node
             self._dns_xdist = request.config.dns_xdist
 

certbot-ci/certbot_integration_tests/utils/acme_server.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 """Module to setup an ACME CA server environment able to run multiple tests in parallel"""
-from __future__ import print_function
 
 import argparse
 import errno
@@ -12,18 +11,18 @@ import subprocess
 import sys
 import tempfile
 import time
-
 from typing import List
+
 import requests
 
+# pylint: disable=wildcard-import,unused-wildcard-import
 from certbot_integration_tests.utils import misc
 from certbot_integration_tests.utils import pebble_artifacts
 from certbot_integration_tests.utils import proxy
-# pylint: disable=wildcard-import,unused-wildcard-import
 from certbot_integration_tests.utils.constants import *
 
 
-class ACMEServer(object):
+class ACMEServer:
     """
     ACMEServer configures and handles the lifecycle of an ACME CA server and an HTTP reverse proxy
     instance, to allow parallel execution of integration tests against the unique http-01 port
@@ -52,7 +51,7 @@ class ACMEServer(object):
         self._acme_type = 'pebble' if acme_server == 'pebble' else 'boulder'
         self._proxy = http_proxy
         self._workspace = tempfile.mkdtemp()
-        self._processes = []  # type: List[subprocess.Popen]
+        self._processes: List[subprocess.Popen] = []
         self._stdout = sys.stdout if stdout else open(os.devnull, 'w')
         self._dns_server = dns_server
         self._http_01_port = http_01_port

certbot-ci/certbot_integration_tests/utils/certbot_call.py

@@ -1,11 +1,10 @@
 #!/usr/bin/env python
 """Module to call certbot in test mode"""
-from __future__ import absolute_import
 
+from distutils.version import LooseVersion
 import os
 import subprocess
 import sys
-from distutils.version import LooseVersion
 
 import certbot_integration_tests
 # pylint: disable=wildcard-import,unused-wildcard-import

certbot-ci/certbot_integration_tests/utils/dns_server.py

@@ -1,7 +1,5 @@
 #!/usr/bin/env python
 """Module to setup an RFC2136-capable DNS server"""
-from __future__ import print_function
-
 import os
 import os.path
 import shutil
@@ -21,7 +19,7 @@ BIND_BIND_ADDRESS = ("127.0.0.1", 45953)
 BIND_TEST_QUERY = bytearray.fromhex("0011cb37000000010000000000000000010003")
 
 
-class DNSServer(object):
+class DNSServer:
     """
     DNSServer configures and handles the lifetime of an RFC2136-capable server.
     DNServer provides access to the dns_xdist parameter, listing the address and port
@@ -40,7 +38,7 @@ class DNSServer(object):
 
         self.bind_root = tempfile.mkdtemp()
 
-        self.process = None  # type: subprocess.Popen
+        self.process: subprocess.Popen = None
 
         self.dns_xdist = {"address": BIND_BIND_ADDRESS[0], "port": BIND_BIND_ADDRESS[1]}
 
@@ -113,8 +111,7 @@ class DNSServer(object):
             self.stop()
             raise
 
-    def _wait_until_ready(self, attempts=30):
-        # type: (int) -> None
+    def _wait_until_ready(self, attempts: int = 30) -> None:
         """
         Polls the DNS server over TCP until it gets a response, or until
         it runs out of attempts and raises a ValueError.

certbot-ci/certbot_integration_tests/utils/misc.py

@@ -4,10 +4,12 @@ or outside during setup/teardown of the integration tests environment.
 """
 import contextlib
 import errno
+import http.server as SimpleHTTPServer
 import multiprocessing
 import os
 import re
 import shutil
+import socketserver
 import stat
 import sys
 import tempfile
@@ -23,11 +25,9 @@ from cryptography.x509 import load_pem_x509_certificate
 from OpenSSL import crypto
 import pkg_resources
 import requests
-from six.moves import SimpleHTTPServer
-from six.moves import socketserver
 
-from certbot_integration_tests.utils.constants import \
-     PEBBLE_ALTERNATE_ROOTS, PEBBLE_MANAGEMENT_URL
+from certbot_integration_tests.utils.constants import PEBBLE_ALTERNATE_ROOTS
+from certbot_integration_tests.utils.constants import PEBBLE_MANAGEMENT_URL
 
 RSA_KEY_TYPE = 'rsa'
 ECDSA_KEY_TYPE = 'ecdsa'
@@ -311,7 +311,7 @@ def echo(keyword, path=None):
     if not re.match(r'^\w+$', keyword):
         raise ValueError('Error, keyword `{0}` is not a single keyword.'
                          .format(keyword))
-    return '{0} -c "from __future__ import print_function; print(\'{1}\')"{2}'.format(
+    return '{0} -c "print(\'{1}\')"{2}'.format(
         os.path.basename(sys.executable), keyword, ' >> "{0}"'.format(path) if path else '')
 
 

certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py

@@ -7,7 +7,8 @@ import stat
 import pkg_resources
 import requests
 
-from certbot_integration_tests.utils.constants import DEFAULT_HTTP_01_PORT, MOCK_OCSP_SERVER_PORT
+from certbot_integration_tests.utils.constants import DEFAULT_HTTP_01_PORT
+from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT
 
 PEBBLE_VERSION = 'v2.3.0'
 ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets')

certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py

@@ -4,6 +4,7 @@ This runnable module interfaces itself with the Pebble management interface in o
 to serve a mock OCSP responder during integration tests against Pebble.
 """
 import datetime
+import http.server as BaseHTTPServer
 import re
 
 from cryptography import x509
@@ -13,7 +14,6 @@ from cryptography.hazmat.primitives import serialization
 from cryptography.x509 import ocsp
 from dateutil import parser
 import requests
-from six.moves import BaseHTTPServer
 
 from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT
 from certbot_integration_tests.utils.constants import PEBBLE_MANAGEMENT_URL
@@ -29,10 +29,7 @@ class _ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
         request = requests.get(PEBBLE_MANAGEMENT_URL + '/intermediates/0', verify=False)
         issuer_cert = x509.load_pem_x509_certificate(request.content, default_backend())
 
-        try:
-            content_len = int(self.headers.getheader('content-length', 0))
-        except AttributeError:
-            content_len = int(self.headers.get('Content-Length'))
+        content_len = int(self.headers.get('Content-Length'))
 
         ocsp_request = ocsp.load_der_ocsp_request(self.rfile.read(content_len))
         response = requests.get('{0}/cert-status-by-serial/{1}'.format(

certbot-ci/certbot_integration_tests/utils/proxy.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python
 # pylint: disable=missing-module-docstring
 
+import http.server as BaseHTTPServer
 import json
 import re
 import sys
 
 import requests
-from six.moves import BaseHTTPServer
 
 from certbot_integration_tests.utils.misc import GracefulTCPServer
 

certbot-ci/setup.py

@@ -7,6 +7,13 @@ from setuptools import setup
 
 version = '0.32.0.dev0'
 
+# setuptools 36.2+ is needed for support for environment markers
+min_setuptools_version='36.2'
+# This conditional isn't necessary, but it provides better error messages to
+# people who try to install this package with older versions of setuptools.
+if LooseVersion(setuptools_version) < LooseVersion(min_setuptools_version):
+    raise RuntimeError(f'setuptools {min_setuptools_version}+ is required')
+
 install_requires = [
     'coverage',
     'cryptography',
@@ -14,24 +21,17 @@ install_requires = [
     'pyopenssl',
     'pytest',
     'pytest-cov',
-    'pytest-xdist',
+    # This version is needed for "worker" attributes we currently use like
+    # "workerinput".  See https://github.com/pytest-dev/pytest-xdist/pull/268.
+    'pytest-xdist>=1.22.1',
     'python-dateutil',
+    # This dependency needs to be added using environment markers to avoid its
+    # installation on Linux.
+    'pywin32>=300 ; sys_platform == "win32"',
     'pyyaml',
     'requests',
-    'six'
 ]
 
-# Add pywin32 on Windows platforms to handle low-level system calls.
-# This dependency needs to be added using environment markers to avoid its installation on Linux.
-# However environment markers are supported only with setuptools >= 36.2.
-# So this dependency is not added for old Linux distributions with old setuptools,
-# in order to allow these systems to build certbot from sources.
-if LooseVersion(setuptools_version) >= LooseVersion('36.2'):
-    install_requires.append("pywin32>=224 ; sys_platform == 'win32'")
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-
 setup(
     name='certbot-ci',
     version=version,

certbot-ci/snap_integration_tests/dns_tests/test_main.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
-import pytest
-import subprocess
 import glob
 import os
 import re
+import subprocess
+
+import pytest
 
 
 @pytest.fixture(autouse=True, scope="module")

certbot-ci/windows_installer_integration_tests/conftest.py

@@ -6,7 +6,7 @@ for a directory a specific configuration using built-in pytest hooks.
 
 See https://docs.pytest.org/en/latest/reference.html#hook-reference
 """
-from __future__ import print_function
+
 import os
 
 ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))

certbot-ci/windows_installer_integration_tests/test_main.py

@@ -1,8 +1,8 @@
 import os
+import re
+import subprocess
 import time
 import unittest
-import subprocess
-import re
 
 
 @unittest.skipIf(os.name != 'nt', reason='Windows installer tests must be run on Windows.')

certbot-compatibility-test/Dockerfile

@@ -8,11 +8,11 @@ RUN apt-get update && \
 WORKDIR /opt/certbot/src
 
 # We copy all contents of the build directory to allow us to easily use
-# things like tools/venv3.py which expects all of our packages to be available.
+# things like tools/venv.py which expects all of our packages to be available.
 COPY . .
 
-RUN tools/venv3.py
-ENV PATH /opt/certbot/src/venv3/bin:$PATH
+RUN tools/venv.py
+ENV PATH /opt/certbot/src/venv/bin:$PATH
 
 # install in editable mode (-e) to save space: it's not possible to
 # "rm -rf /opt/certbot/src" (it's stays in the underlaying image);

certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py

@@ -2,11 +2,8 @@
 import os
 import shutil
 import subprocess
+from unittest import mock
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
 import zope.interface
 
 from certbot import errors as le_errors

certbot-compatibility-test/certbot_compatibility_test/configurators/common.py

@@ -11,7 +11,7 @@ from certbot_compatibility_test import util
 logger = logging.getLogger(__name__)
 
 
-class Proxy(object):
+class Proxy:
     """A common base for compatibility test configurators"""
 
     @classmethod

certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py

@@ -2,10 +2,10 @@
 import os
 import shutil
 import subprocess
+from typing import Set
 
 import zope.interface
 
-from acme.magic_typing import Set
 from certbot._internal import configuration
 from certbot_compatibility_test import errors
 from certbot_compatibility_test import interfaces
@@ -68,7 +68,7 @@ def _get_server_root(config):
 
 def _get_names(config):
     """Returns all and testable domain names in config"""
-    all_names = set()  # type: Set[str]
+    all_names: Set[str] = set()
     for root, _dirs, files in os.walk(config):
         for this_file in files:
             update_names = _get_server_names(root, this_file)

certbot-compatibility-test/certbot_compatibility_test/test_driver.py

@@ -8,6 +8,8 @@ import shutil
 import sys
 import tempfile
 import time
+from typing import List
+from typing import Tuple
 
 import OpenSSL
 from urllib3.util import connection
@@ -15,8 +17,6 @@ from urllib3.util import connection
 from acme import challenges
 from acme import crypto_util
 from acme import messages
-from acme.magic_typing import List
-from acme.magic_typing import Tuple
 from certbot import achallenges
 from certbot import errors as le_errors
 from certbot.tests import acme_util
@@ -178,7 +178,7 @@ def test_enhancements(plugin, domains):
                      "enhancements")
         return False
 
-    domains_and_info = [(domain, []) for domain in domains]  # type: List[Tuple[str, List[bool]]]
+    domains_and_info: List[Tuple[str, List[bool]]] = [(domain, []) for domain in domains]
 
     for domain, info in domains_and_info:
         try:

certbot-compatibility-test/certbot_compatibility_test/validator.py

@@ -3,8 +3,6 @@ import logging
 import socket
 
 import requests
-import six
-from six.moves import xrange
 
 from acme import crypto_util
 from acme import errors as acme_errors
@@ -12,18 +10,18 @@ from acme import errors as acme_errors
 logger = logging.getLogger(__name__)
 
 
-class Validator(object):
+class Validator:
     """Collection of functions to test a live webserver's configuration"""
 
     def certificate(self, cert, name, alt_host=None, port=443):
         """Verifies the certificate presented at name is cert"""
         if alt_host is None:
             host = socket.gethostbyname(name).encode()
-        elif isinstance(alt_host, six.binary_type):
+        elif isinstance(alt_host, bytes):
             host = alt_host
         else:
             host = alt_host.encode()
-        name = name if isinstance(name, six.binary_type) else name.encode()
+        name = name if isinstance(name, bytes) else name.encode()
 
         try:
             presented_cert = crypto_util.probe_sni(name, host, port)
@@ -62,7 +60,7 @@ class Validator(object):
         else:
             response = requests.get(url, allow_redirects=False)
 
-        return response.status_code in xrange(300, 309)
+        return response.status_code in range(300, 309)
 
     def hsts(self, name):
         """Test for HTTP Strict Transport Security header"""

certbot-compatibility-test/certbot_compatibility_test/validator_test.py

@@ -1,10 +1,7 @@
 """Tests for certbot_compatibility_test.validator."""
 import unittest
+from unittest import mock
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
 import OpenSSL
 import requests
 

certbot-compatibility-test/setup.py

@@ -1,29 +1,17 @@
-from distutils.version import LooseVersion
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 install_requires = [
     'certbot',
     'certbot-apache',
-    'six',
     'requests',
     'zope.interface',
 ]
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 if sys.version_info < (2, 7, 9):
     # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning)
     install_requires.append('ndg-httpsclient')

certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py

@@ -1,13 +1,12 @@
 """DNS Authenticator for Cloudflare."""
 import logging
+from typing import Any
+from typing import Dict
+from typing import List
 
 import CloudFlare
 import zope.interface
 
-from acme.magic_typing import Any
-from acme.magic_typing import Dict
-from acme.magic_typing import List
-
 from certbot import errors
 from certbot import interfaces
 from certbot.plugins import dns_common
@@ -85,7 +84,7 @@ class Authenticator(dns_common.DNSAuthenticator):
         return _CloudflareClient(self.credentials.conf('email'), self.credentials.conf('api-key'))
 
 
-class _CloudflareClient(object):
+class _CloudflareClient:
     """
     Encapsulates all communication with the Cloudflare API.
     """
@@ -173,7 +172,7 @@ class _CloudflareClient(object):
         """
 
         zone_name_guesses = dns_common.base_domain_name_guesses(domain)
-        zones = []  # type: List[Dict[str, Any]]
+        zones: List[Dict[str, Any]] = []
         code = msg = None
 
         for zone_name in zone_name_guesses:

certbot-dns-cloudflare/setup.py

@@ -1,12 +1,10 @@
-from distutils.version import LooseVersion
 import os
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -27,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]:
 if os.environ.get('SNAP_BUILD'):
     install_requires.append('packaging')
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',

certbot-dns-cloudxns/setup.py

@@ -1,12 +1,10 @@
-from distutils.version import LooseVersion
 import os
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -27,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]:
 if os.environ.get('SNAP_BUILD'):
     install_requires.append('packaging')
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',

certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py

@@ -21,6 +21,7 @@ class Authenticator(dns_common.DNSAuthenticator):
 
     description = 'Obtain certificates using a DNS TXT record (if you are ' + \
                   'using DigitalOcean for DNS).'
+    ttl = 30
 
     def __init__(self, *args, **kwargs):
         super(Authenticator, self).__init__(*args, **kwargs)
@@ -45,7 +46,8 @@ class Authenticator(dns_common.DNSAuthenticator):
         )
 
     def _perform(self, domain, validation_name, validation):
-        self._get_digitalocean_client().add_txt_record(domain, validation_name, validation)
+        self._get_digitalocean_client().add_txt_record(domain, validation_name, validation,
+                                                       self.ttl)
 
     def _cleanup(self, domain, validation_name, validation):
         self._get_digitalocean_client().del_txt_record(domain, validation_name, validation)
@@ -54,7 +56,7 @@ class Authenticator(dns_common.DNSAuthenticator):
         return _DigitalOceanClient(self.credentials.conf('token'))
 
 
-class _DigitalOceanClient(object):
+class _DigitalOceanClient:
     """
     Encapsulates all communication with the DigitalOcean API.
     """
@@ -62,13 +64,15 @@ class _DigitalOceanClient(object):
     def __init__(self, token):
         self.manager = digitalocean.Manager(token=token)
 
-    def add_txt_record(self, domain_name, record_name, record_content):
+    def add_txt_record(self, domain_name: str, record_name: str, record_content: str,
+                       record_ttl: int):
         """
         Add a TXT record using the supplied information.
 
         :param str domain_name: The domain to use to associate the record with.
         :param str record_name: The record name (typically beginning with '_acme-challenge.').
         :param str record_content: The record content (typically the challenge validation).
+        :param int record_ttl: The record TTL.
         :raises certbot.errors.PluginError: if an error occurs communicating with the DigitalOcean
                                             API
         """
@@ -89,7 +93,8 @@ class _DigitalOceanClient(object):
             result = domain.create_new_domain_record(
                 type='TXT',
                 name=self._compute_record_name(domain, record_name),
-                data=record_content)
+                data=record_content,
+                ttl=record_ttl) # ttl kwarg is only effective starting python-digitalocean 1.15.0
 
             record_id = result['domain_record']['id']
 
@@ -99,7 +104,7 @@ class _DigitalOceanClient(object):
             raise errors.PluginError('Error adding TXT record using the DigitalOcean API: {0}'
                                      .format(e))
 
-    def del_txt_record(self, domain_name, record_name, record_content):
+    def del_txt_record(self, domain_name: str, record_name: str, record_content: str):
         """
         Delete a TXT record using the supplied information.
 

certbot-dns-digitalocean/setup.py

@@ -1,19 +1,16 @@
-from distutils.version import LooseVersion
 import os
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    'python-digitalocean>=1.11',
+    'python-digitalocean>=1.11', # 1.15.0 or newer is recommended for TTL support
     'setuptools>=39.0.1',
-    'six>=1.11.0',
     'zope.interface',
 ]
 
@@ -28,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]:
 if os.environ.get('SNAP_BUILD'):
     install_requires.append('packaging')
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',

certbot-dns-digitalocean/tests/dns_digitalocean_test.py

@@ -40,7 +40,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic
     def test_perform(self):
         self.auth.perform([self.achall])
 
-        expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY)]
+        expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY, 30)]
         self.assertEqual(expected, self.mock_client.mock_calls)
 
     def test_cleanup(self):
@@ -58,6 +58,7 @@ class DigitalOceanClientTest(unittest.TestCase):
     record_prefix = "_acme-challenge"
     record_name = record_prefix + "." + DOMAIN
     record_content = "bar"
+    record_ttl = 60
 
     def setUp(self):
         from certbot_dns_digitalocean._internal.dns_digitalocean import _DigitalOceanClient
@@ -78,25 +79,27 @@ class DigitalOceanClientTest(unittest.TestCase):
 
         self.manager.get_all_domains.return_value = [wrong_domain_mock, domain_mock]
 
-        self.digitalocean_client.add_txt_record(DOMAIN, self.record_name, self.record_content)
+        self.digitalocean_client.add_txt_record(DOMAIN, self.record_name, self.record_content,
+                                                self.record_ttl)
 
         domain_mock.create_new_domain_record.assert_called_with(type='TXT',
                                                                 name=self.record_prefix,
-                                                                data=self.record_content)
+                                                                data=self.record_content,
+                                                                ttl=self.record_ttl)
 
     def test_add_txt_record_fail_to_find_domain(self):
         self.manager.get_all_domains.return_value = []
 
         self.assertRaises(errors.PluginError,
                           self.digitalocean_client.add_txt_record,
-                          DOMAIN, self.record_name, self.record_content)
+                          DOMAIN, self.record_name, self.record_content, self.record_ttl)
 
     def test_add_txt_record_error_finding_domain(self):
         self.manager.get_all_domains.side_effect = API_ERROR
 
         self.assertRaises(errors.PluginError,
                           self.digitalocean_client.add_txt_record,
-                          DOMAIN, self.record_name, self.record_content)
+                          DOMAIN, self.record_name, self.record_content, self.record_ttl)
 
     def test_add_txt_record_error_creating_record(self):
         domain_mock = mock.MagicMock()
@@ -107,7 +110,7 @@ class DigitalOceanClientTest(unittest.TestCase):
 
         self.assertRaises(errors.PluginError,
                           self.digitalocean_client.add_txt_record,
-                          DOMAIN, self.record_name, self.record_content)
+                          DOMAIN, self.record_name, self.record_content, self.record_ttl)
 
     def test_del_txt_record(self):
         first_record_mock = mock.MagicMock()

certbot-dns-dnsimple/setup.py

@@ -1,12 +1,10 @@
-from distutils.version import LooseVersion
 import os
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -26,15 +24,6 @@ elif 'bdist_wheel' in sys.argv[1:]:
 if os.environ.get('SNAP_BUILD'):
     install_requires.append('packaging')
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 # This package normally depends on dns-lexicon>=3.2.1 to address the
 # problem described in https://github.com/AnalogJ/lexicon/issues/387,
 # however, the fix there has been backported to older versions of

certbot-dns-dnsmadeeasy/setup.py

@@ -1,12 +1,10 @@
-from distutils.version import LooseVersion
 import os
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -27,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]:
 if os.environ.get('SNAP_BUILD'):
     install_requires.append('packaging')
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',

certbot-dns-gehirn/setup.py

@@ -1,12 +1,10 @@
-from distutils.version import LooseVersion
 import os
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
@@ -26,15 +24,6 @@ elif 'bdist_wheel' in sys.argv[1:]:
 if os.environ.get('SNAP_BUILD'):
     install_requires.append('packaging')
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',

certbot-dns-google/certbot_dns_google/_internal/dns_google.py

@@ -76,7 +76,7 @@ class Authenticator(dns_common.DNSAuthenticator):
         return _GoogleClient(self.conf('credentials'))
 
 
-class _GoogleClient(object):
+class _GoogleClient:
     """
     Encapsulates all communication with the Google Cloud DNS API.
     """

certbot-dns-google/setup.py

@@ -1,12 +1,10 @@
-from distutils.version import LooseVersion
 import os
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -30,15 +28,6 @@ elif 'bdist_wheel' in sys.argv[1:]:
 if os.environ.get('SNAP_BUILD'):
     install_requires.append('packaging')
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',

certbot-dns-google/tests/dns_google_test.py

@@ -401,7 +401,7 @@ class GoogleClientTest(unittest.TestCase):
             self.assertRaises(ServerNotFoundError, _GoogleClient.get_project_id)
 
 
-class DummyResponse(object):
+class DummyResponse:
     """
     Dummy object to create a fake HTTPResponse (the actual one requires a socket and we only
      need the status attribute)

certbot-dns-linode/setup.py

@@ -1,12 +1,10 @@
-from distutils.version import LooseVersion
 import os
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
@@ -26,15 +24,6 @@ elif 'bdist_wheel' in sys.argv[1:]:
 if os.environ.get('SNAP_BUILD'):
     install_requires.append('packaging')
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',

certbot-dns-luadns/setup.py

@@ -1,12 +1,10 @@
-from distutils.version import LooseVersion
 import os
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -27,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]:
 if os.environ.get('SNAP_BUILD'):
     install_requires.append('packaging')
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',

certbot-dns-nsone/setup.py

@@ -1,12 +1,10 @@
-from distutils.version import LooseVersion
 import os
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -27,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]:
 if os.environ.get('SNAP_BUILD'):
     install_requires.append('packaging')
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',

certbot-dns-ovh/setup.py

@@ -1,12 +1,10 @@
-from distutils.version import LooseVersion
 import os
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -27,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]:
 if os.environ.get('SNAP_BUILD'):
     install_requires.append('packaging')
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',

certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py

@@ -88,7 +88,7 @@ class Authenticator(dns_common.DNSAuthenticator):
                                                   dns.tsig.HMAC_MD5))
 
 
-class _RFC2136Client(object):
+class _RFC2136Client:
     """
     Encapsulates all communication with the target DNS server.
     """

certbot-dns-rfc2136/setup.py

@@ -1,12 +1,10 @@
-from distutils.version import LooseVersion
 import os
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -27,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]:
 if os.environ.get('SNAP_BUILD'):
     install_requires.append('packaging')
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',

certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py

@@ -2,15 +2,15 @@
 import collections
 import logging
 import time
+from typing import DefaultDict
+from typing import Dict
+from typing import List
 
 import boto3
 from botocore.exceptions import ClientError
 from botocore.exceptions import NoCredentialsError
 import zope.interface
 
-from acme.magic_typing import DefaultDict
-from acme.magic_typing import Dict
-from acme.magic_typing import List
 from certbot import errors
 from certbot import interfaces
 from certbot.plugins import dns_common
@@ -39,7 +39,7 @@ class Authenticator(dns_common.DNSAuthenticator):
     def __init__(self, *args, **kwargs):
         super(Authenticator, self).__init__(*args, **kwargs)
         self.r53 = boto3.client("route53")
-        self._resource_records = collections.defaultdict(list) # type: DefaultDict[str, List[Dict[str, str]]]
+        self._resource_records: DefaultDict[str, List[Dict[str, str]]] = collections.defaultdict(list)
 
     def more_info(self):  # pylint: disable=missing-function-docstring
         return "Solve a DNS01 challenge using AWS Route53"

certbot-dns-route53/setup.py

@@ -1,12 +1,10 @@
-from distutils.version import LooseVersion
 import os
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -27,15 +25,6 @@ elif 'bdist_wheel' in sys.argv[1:]:
 if os.environ.get('SNAP_BUILD'):
     install_requires.append('packaging')
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',

certbot-dns-sakuracloud/setup.py

@@ -1,12 +1,10 @@
-from distutils.version import LooseVersion
 import os
 import sys
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.12.0.dev0'
+version = '1.14.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
@@ -26,15 +24,6 @@ elif 'bdist_wheel' in sys.argv[1:]:
 if os.environ.get('SNAP_BUILD'):
     install_requires.append('packaging')
 
-setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))
-if setuptools_known_environment_markers:
-    install_requires.append('mock ; python_version < "3.3"')
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-elif sys.version_info < (3,3):
-    install_requires.append('mock')
-
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',

certbot-nginx/certbot_nginx/_internal/configurator.py

@@ -1,3 +1,4 @@
+# pylint: disable=too-many-lines
 """Nginx Configuration"""
 from distutils.version import LooseVersion
 import logging
@@ -6,6 +7,12 @@ import socket
 import subprocess
 import tempfile
 import time
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Set
+from typing import Text
+from typing import Tuple
 
 import OpenSSL
 import pkg_resources
@@ -13,10 +20,6 @@ import zope.interface
 
 from acme import challenges
 from acme import crypto_util as acme_crypto_util
-from acme.magic_typing import Dict
-from acme.magic_typing import List
-from acme.magic_typing import Set
-from acme.magic_typing import Text
 from certbot import crypto_util
 from certbot import errors
 from certbot import interfaces
@@ -105,18 +108,18 @@ class NginxConfigurator(common.Installer):
         self.save_notes = ""
 
         # For creating new vhosts if no names match
-        self.new_vhost = None
+        self.new_vhost: Optional[obj.VirtualHost] = None
 
         # List of vhosts configured per wildcard domain on this run.
         # used by deploy_cert() and enhance()
-        self._wildcard_vhosts = {} # type: Dict[str, List[obj.VirtualHost]]
-        self._wildcard_redirect_vhosts = {} # type: Dict[str, List[obj.VirtualHost]]
+        self._wildcard_vhosts: Dict[str, List[obj.VirtualHost]] = {}
+        self._wildcard_redirect_vhosts: Dict[str, List[obj.VirtualHost]] = {}
 
         # Add number of outstanding challenges
         self._chall_out = 0
 
         # These will be set in the prepare function
-        self.parser = None
+        self.parser: Optional[parser.NginxParser] = None
         self.version = version
         self.openssl_version = openssl_version
         self._enhance_func = {"redirect": self._enable_redirect,
@@ -377,10 +380,13 @@ class NginxConfigurator(common.Installer):
                     ipv6only_present = True
         return (ipv6_active, ipv6only_present)
 
-    def _vhost_from_duplicated_default(self, domain, allow_port_mismatch, port):
+    def _vhost_from_duplicated_default(self, domain: str, allow_port_mismatch: bool, port: str
+                                       ) -> obj.VirtualHost:
         """if allow_port_mismatch is False, only server blocks with matching ports will be
            used as a default server block template.
         """
+        assert self.parser is not None # prepare should already have been called here
+
         if self.new_vhost is None:
             default_vhost = self._get_default_vhost(domain, allow_port_mismatch, port)
             self.new_vhost = self.parser.duplicate_vhost(default_vhost,
@@ -509,7 +515,7 @@ class NginxConfigurator(common.Installer):
                 match['rank'] += NO_SSL_MODIFIER
         return sorted(matches, key=lambda x: x['rank'])
 
-    def choose_redirect_vhosts(self, target_name, port, create_if_no_match=False):
+    def choose_redirect_vhosts(self, target_name: str, port: str) -> List[obj.VirtualHost]:
         """Chooses a single virtual host for redirect enhancement.
 
         Chooses the vhost most closely matching target_name that is
@@ -523,9 +529,6 @@ class NginxConfigurator(common.Installer):
 
         :param str target_name: domain name
         :param str port: port number
-        :param bool create_if_no_match: If we should create a new vhost from default
-            when there is no match found. If we can't choose a default, raise a
-            MisconfigurationError.
 
         :returns: vhosts associated with name
         :rtype: list of :class:`~certbot_nginx._internal.obj.VirtualHost`
@@ -538,32 +541,75 @@ class NginxConfigurator(common.Installer):
         else:
             matches = self._get_redirect_ranked_matches(target_name, port)
             vhosts = [x for x in [self._select_best_name_match(matches)]if x is not None]
-        if not vhosts and create_if_no_match:
-            vhosts = [self._vhost_from_duplicated_default(target_name, False, port)]
         return vhosts
 
-    def _port_matches(self, test_port, matching_port):
+    def choose_auth_vhosts(self, target_name: str) -> Tuple[List[obj.VirtualHost],
+                                                            List[obj.VirtualHost]]:
+        """Returns a list of HTTP and HTTPS vhosts with a server_name matching target_name.
+
+        If no HTTP vhost exists, one will be cloned from the default vhost. If that fails, no HTTP
+        vhost will be returned.
+
+        :param str target_name: non-wildcard domain name
+
+        :returns: tuple of HTTP and HTTPS virtualhosts
+        :rtype: tuple of :class:`~certbot_nginx._internal.obj.VirtualHost`
+
+        """
+        vhosts = [m['vhost'] for m in self._get_ranked_matches(target_name) if m and 'vhost' in m]
+        http_vhosts = [vh for vh in vhosts if
+                       self._vhost_listening(vh, str(self.config.http01_port), False)]
+        https_vhosts = [vh for vh in vhosts if
+                        self._vhost_listening(vh, str(self.config.https_port), True)]
+
+        # If no HTTP vhost matches, try create one from the default_server on http01_port.
+        if not http_vhosts:
+            try:
+                http_vhosts = [self._vhost_from_duplicated_default(target_name, False,
+                                                                   str(self.config.http01_port))]
+            except errors.MisconfigurationError:
+                http_vhosts = []
+
+        return http_vhosts, https_vhosts
+
+    def _port_matches(self, test_port: str, matching_port: str) -> bool:
         # test_port is a number, matching is a number or "" or None
         if matching_port == "" or matching_port is None:
             # if no port is specified, Nginx defaults to listening on port 80.
             return test_port == self.DEFAULT_LISTEN_PORT
         return test_port == matching_port
 
-    def _vhost_listening_on_port_no_ssl(self, vhost, port):
-        found_matching_port = False
-        if not vhost.addrs:
-            # if there are no listen directives at all, Nginx defaults to
-            # listening on port 80.
-            found_matching_port = (port == self.DEFAULT_LISTEN_PORT)
-        else:
-            for addr in vhost.addrs:
-                if self._port_matches(port, addr.get_port()) and not addr.ssl:
-                    found_matching_port = True
+    def _vhost_listening(self, vhost: obj.VirtualHost, port: str, ssl: bool) -> bool:
+        """Tests whether a vhost has an address listening on a port with SSL enabled or disabled.
 
-        if found_matching_port:
-            # make sure we don't have an 'ssl on' directive
-            return not self.parser.has_ssl_on_directive(vhost)
-        return False
+        :param `obj.VirtualHost` vhost: The vhost whose addresses will be tested
+        :param port str: The port number as a string that the address should be bound to
+        :param bool ssl: Whether SSL should be enabled or disabled on the address
+
+        :returns: Whether the vhost has an address listening on the port and protocol.
+        :rtype: bool
+
+        """
+        assert self.parser is not None # prepare should already have been called here
+
+        # if the 'ssl on' directive is present on the vhost, all its addresses have SSL enabled
+        all_addrs_are_ssl = self.parser.has_ssl_on_directive(vhost)
+
+        # if we want ssl vhosts: either 'ssl on' or 'addr.ssl' should be enabled
+        # if we want plaintext vhosts: neither 'ssl on' nor 'addr.ssl' should be enabled
+        _ssl_matches = lambda addr: addr.ssl or all_addrs_are_ssl if ssl else \
+                                    not addr.ssl and not all_addrs_are_ssl
+
+        # if there are no listen directives at all, Nginx defaults to
+        # listening on port 80.
+        if not vhost.addrs:
+            return port == self.DEFAULT_LISTEN_PORT and ssl == all_addrs_are_ssl
+
+        return any(self._port_matches(port, addr.get_port()) and _ssl_matches(addr)
+                   for addr in vhost.addrs)
+
+    def _vhost_listening_on_port_no_ssl(self, vhost: obj.VirtualHost, port: str) -> bool:
+        return self._vhost_listening(vhost, port, False)
 
     def _get_redirect_ranked_matches(self, target_name, port):
         """Gets a ranked list of plaintextish port-listening vhosts matching target_name
@@ -595,7 +641,7 @@ class NginxConfigurator(common.Installer):
         :rtype: set
 
         """
-        all_names = set()  # type: Set[str]
+        all_names: Set[str] = set()
 
         for vhost in self.parser.get_vhosts():
             try:
@@ -1176,7 +1222,7 @@ def nginx_restart(nginx_ctl, nginx_conf, sleep_duration):
 
     """
     try:
-        reload_output = u"" # type: Text
+        reload_output: Text = u""
         with tempfile.TemporaryFile() as out:
             proc = subprocess.Popen([nginx_ctl, "-c", nginx_conf, "-s", "reload"],
                                     env=util.env_no_snap_for_external_calls(),

certbot-nginx/certbot_nginx/_internal/constants.py

@@ -1,8 +1,7 @@
 """nginx plugin constants."""
 import platform
-
-from acme.magic_typing import Any
-from acme.magic_typing import Dict
+from typing import Any
+from typing import Dict
 
 FREEBSD_DARWIN_SERVER_ROOT = "/usr/local/etc/nginx"
 LINUX_SERVER_ROOT = "/etc/nginx"
@@ -15,11 +14,11 @@ elif platform.system() in ('NetBSD',):
 else:
     server_root_tmp = LINUX_SERVER_ROOT
 
-CLI_DEFAULTS = dict(
+CLI_DEFAULTS: Dict[str, Any] = dict(
     server_root=server_root_tmp,
     ctl="nginx",
     sleep_seconds=1
-) # type: Dict[str, Any]
+)
 """CLI defaults."""
 
 

certbot-nginx/certbot_nginx/_internal/http_01.py

@@ -2,9 +2,11 @@
 
 import io
 import logging
+from typing import List
+from typing import Optional
 
 from acme import challenges
-from acme.magic_typing import List
+from certbot import achallenges
 from certbot import errors
 from certbot.compat import os
 from certbot.plugins import common
@@ -111,7 +113,7 @@ class NginxHttp01(common.ChallengePerformer):
         :returns: list of :class:`certbot_nginx._internal.obj.Addr` to apply
         :rtype: list
         """
-        addresses = [] # type: List[obj.Addr]
+        addresses: List[obj.Addr] = []
         default_addr = "%s" % self.configurator.config.http01_port
         ipv6_addr = "[::]:{0}".format(
             self.configurator.config.http01_port)
@@ -138,13 +140,12 @@ class NginxHttp01(common.ChallengePerformer):
     def _get_validation_path(self, achall):
         return os.sep + os.path.join(challenges.HTTP01.URI_ROOT_PATH, achall.chall.encode("token"))
 
-    def _make_server_block(self, achall):
+    def _make_server_block(self, achall: achallenges.KeyAuthorizationAnnotatedChallenge) -> List:
         """Creates a server block for a challenge.
+
         :param achall: Annotated HTTP-01 challenge
-        :type achall:
-            :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge`
-        :param list addrs: addresses of challenged domain
-            :class:`list` of type :class:`~nginx.obj.Addr`
+        :type achall: :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge`
+
         :returns: server block for the challenge host
         :rtype: list
         """
@@ -172,34 +173,35 @@ class NginxHttp01(common.ChallengePerformer):
         return location_directive
 
 
-    def _make_or_mod_server_block(self, achall):
-        """Modifies a server block to respond to a challenge.
+    def _make_or_mod_server_block(self, achall: achallenges.KeyAuthorizationAnnotatedChallenge
+                                  ) -> Optional[List]:
+        """Modifies server blocks to respond to a challenge. Returns a new HTTP server block
+           to add to the configuration if an existing one can't be found.
 
         :param achall: Annotated HTTP-01 challenge
-        :type achall:
-            :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge`
+        :type achall: :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge`
+
+        :returns: new server block to be added, if any
+        :rtype: list
 
         """
-        try:
-            vhosts = self.configurator.choose_redirect_vhosts(achall.domain,
-                '%i' % self.configurator.config.http01_port, create_if_no_match=True)
-        except errors.MisconfigurationError:
+        http_vhosts, https_vhosts = self.configurator.choose_auth_vhosts(achall.domain)
+
+        new_vhost: Optional[list] = None
+        if not http_vhosts:
             # Couldn't find either a matching name+port server block
             # or a port+default_server block, so create a dummy block
-            return self._make_server_block(achall)
+            new_vhost = self._make_server_block(achall)
 
-        # len is max 1 because Nginx doesn't authenticate wildcards
-        # if len were or vhosts None, we would have errored
-        vhost = vhosts[0]
+        # Modify any existing server blocks
+        for vhost in set(http_vhosts + https_vhosts):
+            location_directive = [self._location_directive_for_achall(achall)]
 
-        # Modify existing server block
-        location_directive = [self._location_directive_for_achall(achall)]
+            self.configurator.parser.add_server_directives(vhost, location_directive)
 
-        self.configurator.parser.add_server_directives(vhost,
-            location_directive)
+            rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)',
+                                    ' ', '$1', ' ', 'break']]
+            self.configurator.parser.add_server_directives(vhost,
+                rewrite_directive, insert_at_top=True)
 
-        rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)',
-                                ' ', '$1', ' ', 'break']]
-        self.configurator.parser.add_server_directives(vhost,
-            rewrite_directive, insert_at_top=True)
-        return None
+        return new_vhost

certbot-nginx/certbot_nginx/_internal/nginxparser.py

@@ -2,6 +2,8 @@
 # Forked from https://github.com/fatiherikli/nginxparser (MIT Licensed)
 import copy
 import logging
+from typing import Any
+from typing import IO
 
 from pyparsing import Combine
 from pyparsing import Forward
@@ -15,12 +17,11 @@ from pyparsing import restOfLine
 from pyparsing import stringEnd
 from pyparsing import White
 from pyparsing import ZeroOrMore
-import six
-from acme.magic_typing import IO, Any # pylint: disable=unused-import
 
 logger = logging.getLogger(__name__)
 
-class RawNginxParser(object):
+
+class RawNginxParser:
     # pylint: disable=pointless-statement
     """A class that parses nginx configuration with pyparsing."""
 
@@ -70,7 +71,7 @@ class RawNginxParser(object):
         """Returns the parsed tree as a list."""
         return self.parse().asList()
 
-class RawNginxDumper(object):
+class RawNginxDumper:
     """A class that dumps nginx configuration from the provided tree."""
     def __init__(self, blocks):
         self.blocks = blocks
@@ -79,7 +80,7 @@ class RawNginxDumper(object):
         """Iterates the dumped nginx content."""
         blocks = blocks or self.blocks
         for b0 in blocks:
-            if isinstance(b0, six.string_types):
+            if isinstance(b0, str):
                 yield b0
                 continue
             item = copy.deepcopy(b0)
@@ -96,7 +97,7 @@ class RawNginxDumper(object):
                 yield '}'
             else: # not a block - list of strings
                 semicolon = ";"
-                if isinstance(item[0], six.string_types) and item[0].strip() == '#': # comment
+                if isinstance(item[0], str) and item[0].strip() == '#': # comment
                     semicolon = ""
                 yield "".join(item) + semicolon
 
@@ -105,56 +106,8 @@ class RawNginxDumper(object):
         return ''.join(self)
 
 
-# Shortcut functions to respect Python's serialization interface
-# (like pyyaml, picker or json)
+spacey = lambda x: (isinstance(x, str) and x.isspace()) or x == ''
 
-def loads(source):
-    """Parses from a string.
-
-    :param str source: The string to parse
-    :returns: The parsed tree
-    :rtype: list
-
-    """
-    return UnspacedList(RawNginxParser(source).as_list())
-
-
-def load(_file):
-    """Parses from a file.
-
-    :param file _file: The file to parse
-    :returns: The parsed tree
-    :rtype: list
-
-    """
-    return loads(_file.read())
-
-
-def dumps(blocks):
-    # type: (UnspacedList) -> six.text_type
-    """Dump to a Unicode string.
-
-    :param UnspacedList block: The parsed tree
-    :rtype: six.text_type
-
-    """
-    return six.text_type(RawNginxDumper(blocks.spaced))
-
-
-def dump(blocks, _file):
-    # type: (UnspacedList, IO[Any]) -> None
-    """Dump to a file.
-
-    :param UnspacedList block: The parsed tree
-    :param IO[Any] _file: The file stream to dump to. It must be opened with
-                          Unicode encoding.
-    :rtype: None
-
-    """
-    _file.write(dumps(blocks))
-
-
-spacey = lambda x: (isinstance(x, six.string_types) and x.isspace()) or x == ''
 
 class UnspacedList(list):
     """Wrap a list [of lists], making any whitespace entries magically invisible"""
@@ -274,3 +227,50 @@ class UnspacedList(list):
                 idx -= 1
             pos += 1
         return idx0 + spaces
+
+
+# Shortcut functions to respect Python's serialization interface
+# (like pyyaml, picker or json)
+
+def loads(source):
+    """Parses from a string.
+
+    :param str source: The string to parse
+    :returns: The parsed tree
+    :rtype: list
+
+    """
+    return UnspacedList(RawNginxParser(source).as_list())
+
+
+def load(_file):
+    """Parses from a file.
+
+    :param file _file: The file to parse
+    :returns: The parsed tree
+    :rtype: list
+
+    """
+    return loads(_file.read())
+
+
+def dumps(blocks: UnspacedList) -> str:
+    """Dump to a Unicode string.
+
+    :param UnspacedList block: The parsed tree
+    :rtype: six.text_type
+
+    """
+    return str(RawNginxDumper(blocks.spaced))
+
+
+def dump(blocks: UnspacedList, _file: IO[Any]) -> None:
+    """Dump to a file.
+
+    :param UnspacedList block: The parsed tree
+    :param IO[Any] _file: The file stream to dump to. It must be opened with
+                          Unicode encoding.
+    :rtype: None
+
+    """
+    _file.write(dumps(blocks))

certbot-nginx/certbot_nginx/_internal/obj.py

@@ -1,8 +1,6 @@
 """Module contains classes used by the Nginx Configurator."""
 import re
 
-import six
-
 from certbot.plugins import common
 
 ADD_HEADER_DIRECTIVE = 'add_header'
@@ -144,7 +142,7 @@ class Addr(common.Addr):
         return False
 
 
-class VirtualHost(object):
+class VirtualHost:
     """Represents an Nginx Virtualhost.
 
     :ivar str filep: file path of VH
@@ -211,7 +209,7 @@ class VirtualHost(object):
     def contains_list(self, test):
         """Determine if raw server block contains test list at top level
         """
-        for i in six.moves.range(0, len(self.raw) - len(test) + 1):
+        for i in range(0, len(self.raw) - len(test) + 1):
             if self.raw[i:i + len(test)] == test:
                 return True
         return False
@@ -250,7 +248,7 @@ def _find_directive(directives, directive_name, match_content=None):
     """Find a directive of type directive_name in directives. If match_content is given,
        Searches for `match_content` in the directive arguments.
     """
-    if not directives or isinstance(directives, six.string_types):
+    if not directives or isinstance(directives, str):
         return None
 
     # If match_content is None, just match on directive type. Otherwise, match on