test windows installer only

We were originally using `socket.errno` with a `type: ignore` and a comment suggesting that this attribute needs to be included in the typeshed. This is incorrect.

While it's true that [socket imports errno](43682f1e39/Lib/socket.py (L58)), it's not intended to be part of its API. https://docs.python.org/3/library/socket.html has no mention of it.

Instead, we should be using the standard `errno` module and remove this `type: ignore`.

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -1,55 +1,4 @@
 jobs:
-  - job: docker_build
-    pool:
-      vmImage: ubuntu-18.04
-    strategy:
-      matrix:
-        amd64:
-          DOCKER_ARCH: amd64
-        # Do not run the heavy non-amd64 builds for test branches
-        ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
-          arm32v6:
-            DOCKER_ARCH: arm32v6
-          arm64v8:
-            DOCKER_ARCH: arm64v8
-    steps:
-      - bash: set -e && tools/docker/build.sh $(dockerTag) $DOCKER_ARCH
-        displayName: Build the Docker images
-      # We don't filter for the Docker Hub organization to continue to allow
-      # easy testing of these scripts on forks.
-      - bash: |
-          set -e
-          DOCKER_IMAGES=$(docker images --filter reference='*/certbot' --filter reference='*/dns-*' --format '{{.Repository}}')
-          docker save --output images.tar $DOCKER_IMAGES
-        displayName: Save the Docker images
-      # If the name of the tar file or artifact changes, the deploy stage will
-      # also need to be updated.
-      - bash: set -e && mv images.tar $(Build.ArtifactStagingDirectory)
-        displayName: Prepare Docker artifact
-      - task: PublishPipelineArtifact@1
-        inputs:
-          path: $(Build.ArtifactStagingDirectory)
-          artifact: docker_$(DOCKER_ARCH)
-        displayName: Store Docker artifact
-  - job: docker_run
-    dependsOn: docker_build
-    pool:
-      vmImage: ubuntu-18.04
-    steps:
-      - task: DownloadPipelineArtifact@2
-        inputs:
-          artifact: docker_amd64
-          path: $(Build.SourcesDirectory)
-        displayName: Retrieve Docker images
-      - bash: set -e && docker load --input $(Build.SourcesDirectory)/images.tar
-        displayName: Load Docker images
-      - bash: |
-          set -ex
-          DOCKER_IMAGES=$(docker images --filter reference='*/certbot' --filter reference='*/dns-*' --format '{{.Repository}}:{{.Tag}}')
-          for DOCKER_IMAGE in ${DOCKER_IMAGES}
-            do docker run --rm "${DOCKER_IMAGE}" plugins --prepare
-          done
-        displayName: Run integration tests for Docker images
   - job: installer_build
     pool:
       vmImage: vs2017-win2016
@@ -59,7 +8,13 @@ jobs:
           versionSpec: 3.8
           architecture: x86
           addToPath: true
-      - script: python windows-installer/construct.py
+      - script: |
+          python -m venv venv
+          venv\Scripts\python tools\pipstrap.py
+          venv\Scripts\python tools\pip_install.py -e windows-installer
+        displayName: Prepare Windows installer build environment
+      - script: |
+          venv\Scripts\construct-windows-installer
         displayName: Build Certbot installer
       - task: CopyFiles@2
         inputs:
@@ -113,105 +68,3 @@ jobs:
           set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
           venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
         displayName: Run certbot integration tests
-  - job: snaps_build
-    pool:
-      vmImage: ubuntu-18.04
-    timeoutInMinutes: 0
-    variables:
-      # Do not run the heavy non-amd64 builds for test branches
-      ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
-        ARCHS: amd64 arm64 armhf
-      ${{ if startsWith(variables['Build.SourceBranchName'], 'test-') }}:
-        ARCHS: amd64
-    steps:
-      - script: |
-          set -e
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends snapd
-          sudo snap install --classic snapcraft
-        displayName: Install dependencies
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: 3.8
-          addToPath: true
-      - task: DownloadSecureFile@1
-        name: credentials
-        inputs:
-          secureFile: launchpad-credentials
-      - script: |
-          set -e
-          git config --global user.email "$(Build.RequestedForEmail)"
-          git config --global user.name "$(Build.RequestedFor)"
-          mkdir -p ~/.local/share/snapcraft/provider/launchpad
-          cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials
-          python3 tools/snap/build_remote.py ALL --archs ${ARCHS} --timeout 19800
-        displayName: Build snaps
-      - script: |
-          set -e
-          mv *.snap $(Build.ArtifactStagingDirectory)
-          mv certbot-dns-*/*.snap $(Build.ArtifactStagingDirectory)
-        displayName: Prepare artifacts
-      - task: PublishPipelineArtifact@1
-        inputs:
-          path: $(Build.ArtifactStagingDirectory)
-          artifact: snaps
-        displayName: Store snaps artifacts
-  - job: snap_run
-    dependsOn: snaps_build
-    pool:
-      vmImage: ubuntu-18.04
-    steps:
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: 3.8
-          addToPath: true
-      - script: |
-          set -e
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends nginx-light snapd
-          python3 -m venv venv
-          venv/bin/python tools/pipstrap.py
-          venv/bin/python tools/pip_install.py -U tox
-        displayName: Install dependencies
-      - task: DownloadPipelineArtifact@2
-        inputs:
-          artifact: snaps
-          path: $(Build.SourcesDirectory)/snap
-        displayName: Retrieve Certbot snaps
-      - script: |
-          set -e
-          sudo snap install --dangerous --classic snap/certbot_*_amd64.snap
-        displayName: Install Certbot snap
-      - script: |
-          set -e
-          venv/bin/python -m tox -e integration-external,apacheconftest-external-with-pebble
-        displayName: Run tox
-  - job: snap_dns_run
-    dependsOn: snaps_build
-    pool:
-      vmImage: ubuntu-18.04
-    steps:
-      - script: |
-          set -e
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends snapd
-        displayName: Install dependencies
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: 3.8
-          addToPath: true
-      - task: DownloadPipelineArtifact@2
-        inputs:
-          artifact: snaps
-          path: $(Build.SourcesDirectory)/snap
-        displayName: Retrieve Certbot snaps
-      - script: |
-          set -e
-          python3 -m venv venv
-          venv/bin/python tools/pipstrap.py
-          venv/bin/python tools/pip_install.py -e certbot-ci
-        displayName: Prepare Certbot-CI
-      - script: |
-          set -e
-          sudo -E venv/bin/pytest certbot-ci/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap --snap-arch amd64
-        displayName: Test DNS plugins snaps

.azure-pipelines/templates/jobs/standard-tests-jobs.yml

@@ -40,13 +40,13 @@ jobs:
           IMAGE_NAME: ubuntu-18.04
           PYTHON_VERSION: 3.9
           TOXENV: py39-cover
-        linux-py37-lint:
+        linux-py39-lint:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.7
+          PYTHON_VERSION: 3.9
           TOXENV: lint
-        linux-py36-mypy:
+        linux-py39-mypy:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.6
+          PYTHON_VERSION: 3.9
           TOXENV: mypy
         linux-integration:
           IMAGE_NAME: ubuntu-18.04

.azure-pipelines/templates/stages/deploy-stage.yml

@@ -37,6 +37,14 @@ stages:
           vmImage: ubuntu-18.04
         variables:
           - group: certbot-common
+        strategy:
+          matrix:
+            amd64:
+              SNAP_ARCH: amd64
+            arm32v6:
+              SNAP_ARCH: armhf
+            arm64v8:
+              SNAP_ARCH: arm64
         steps:
           - bash: |
               set -e
@@ -46,7 +54,7 @@ stages:
             displayName: Install dependencies
           - task: DownloadPipelineArtifact@2
             inputs:
-              artifact: snaps
+              artifact: snaps_$(SNAP_ARCH)
               path: $(Build.SourcesDirectory)/snap
             displayName: Retrieve Certbot snaps
           - task: DownloadSecureFile@1
@@ -55,8 +63,7 @@ stages:
               secureFile: snapcraft.cfg
           - bash: |
               set -e
-              mkdir -p .snapcraft
-              ln -s $(snapcraftCfg.secureFilePath) .snapcraft/snapcraft.cfg
+              snapcraft login --with $(snapcraftCfg.secureFilePath)
               for SNAP_FILE in snap/*.snap; do
                 tools/retry.sh eval snapcraft upload --release=${{ parameters.snapReleaseChannel }} "${SNAP_FILE}"
               done

.azure-pipelines/templates/stages/test-and-package-stage.yml

@@ -1,6 +1,4 @@
 stages:
   - stage: TestAndPackage
     jobs:
-      - template: ../jobs/standard-tests-jobs.yml
-      - template: ../jobs/extended-tests-jobs.yml
       - template: ../jobs/packaging-jobs.yml

.gitignore

@@ -11,6 +11,7 @@ dist*/
 letsencrypt.log
 certbot.log
 letsencrypt-auto-source/letsencrypt-auto.sig.lzma.base64
+poetry.lock
 
 # coverage
 .coverage

.pylintrc

@@ -8,7 +8,10 @@ jobs=0
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-#init-hook=
+# CERTBOT COMMENT
+# This is needed for pylint to import linter_plugin.py since
+# https://github.com/PyCQA/pylint/pull/3396.
+init-hook="import pylint.config, os, sys; sys.path.append(os.path.dirname(pylint.config.PYLINTRC))"
 
 # Profiled execution.
 profile=no

ISSUE_TEMPLATE.md

@@ -7,7 +7,7 @@ questions.
 ## My operating system is (include version):
 
 
-## I installed Certbot with (certbot-auto, OS package manager, pip, etc):
+## I installed Certbot with (snap, OS package manager, pip, certbot-auto, etc):
 
 
 ## I ran this command and it produced this output:

acme/acme/challenges.py

@@ -5,17 +5,19 @@ import functools
 import hashlib
 import logging
 import socket
+from typing import Type
 
 from cryptography.hazmat.primitives import hashes  # type: ignore
 import josepy as jose
-import requests
-from OpenSSL import SSL  # type: ignore # https://github.com/python/typeshed/issues/2052
 from OpenSSL import crypto
+from OpenSSL import SSL  # type: ignore # https://github.com/python/typeshed/issues/2052
+import requests
 
 from acme import crypto_util
 from acme import errors
 from acme import fields
-from acme.mixins import ResourceMixin, TypeMixin
+from acme.mixins import ResourceMixin
+from acme.mixins import TypeMixin
 
 logger = logging.getLogger(__name__)
 
@@ -23,7 +25,7 @@ logger = logging.getLogger(__name__)
 class Challenge(jose.TypedJSONObjectWithFields):
     # _fields_to_partial_json
     """ACME challenge."""
-    TYPES = {}  # type: dict
+    TYPES: dict = {}
 
     @classmethod
     def from_json(cls, jobj):
@@ -37,7 +39,7 @@ class Challenge(jose.TypedJSONObjectWithFields):
 class ChallengeResponse(ResourceMixin, TypeMixin, jose.TypedJSONObjectWithFields):
     # _fields_to_partial_json
     """ACME challenge response."""
-    TYPES = {}  # type: dict
+    TYPES: dict = {}
     resource_type = 'challenge'
     resource = fields.Resource(resource_type)
 
@@ -151,8 +153,8 @@ class KeyAuthorizationChallenge(_TokenChallenge, metaclass=abc.ABCMeta):
         that will be used to generate ``response``.
     :param str typ: type of the challenge
     """
-    typ = NotImplemented
-    response_cls = NotImplemented
+    typ: str = NotImplemented
+    response_cls: Type[KeyAuthorizationChallengeResponse] = NotImplemented
     thumbprint_hash_function = (
         KeyAuthorizationChallengeResponse.thumbprint_hash_function)
 

acme/acme/client.py

@@ -8,6 +8,12 @@ import http.client as http_client
 import logging
 import re
 import time
+from typing import cast
+from typing import Dict
+from typing import List
+from typing import Set
+from typing import Text
+from typing import Union
 
 import josepy as jose
 import OpenSSL
@@ -20,10 +26,6 @@ from acme import crypto_util
 from acme import errors
 from acme import jws
 from acme import messages
-from acme.magic_typing import Dict
-from acme.magic_typing import List
-from acme.magic_typing import Set
-from acme.magic_typing import Text
 from acme.mixins import VersionedLEACMEMixin
 
 logger = logging.getLogger(__name__)
@@ -112,8 +114,9 @@ class ClientBase:
         """
         return self.update_registration(regr, update={'status': 'deactivated'})
 
-    def deactivate_authorization(self, authzr):
-        # type: (messages.AuthorizationResource) -> messages.AuthorizationResource
+    def deactivate_authorization(self,
+                                 authzr: messages.AuthorizationResource
+                                 ) -> messages.AuthorizationResource:
         """Deactivate authorization.
 
         :param messages.AuthorizationResource authzr: The Authorization resource
@@ -423,7 +426,7 @@ class Client(ClientBase):
 
         """
         assert max_attempts > 0
-        attempts = collections.defaultdict(int) # type: Dict[messages.AuthorizationResource, int]
+        attempts: Dict[messages.AuthorizationResource, int] = collections.defaultdict(int)
         exhausted = set()
 
         # priority queue with datetime.datetime (based on Retry-After) as key,
@@ -536,7 +539,7 @@ class Client(ClientBase):
         :rtype: `list` of `OpenSSL.crypto.X509` wrapped in `.ComparableX509`
 
         """
-        chain = [] # type: List[jose.ComparableX509]
+        chain: List[jose.ComparableX509] = []
         uri = certr.cert_chain_uri
         while uri is not None and len(chain) < max_length:
             response, cert = self._get_cert(uri)
@@ -817,6 +820,7 @@ class BackwardsCompatibleClientV2:
     def __init__(self, net, key, server):
         directory = messages.Directory.from_json(net.get(server).json())
         self.acme_version = self._acme_version_from_directory(directory)
+        self.client: Union[Client, ClientV2]
         if self.acme_version == 1:
             self.client = Client(directory, key=key, net=net)
         else:
@@ -836,16 +840,18 @@ class BackwardsCompatibleClientV2:
             if check_tos_cb is not None:
                 check_tos_cb(tos)
         if self.acme_version == 1:
-            regr = self.client.register(regr)
+            client_v1 = cast(Client, self.client)
+            regr = client_v1.register(regr)
             if regr.terms_of_service is not None:
                 _assess_tos(regr.terms_of_service)
-                return self.client.agree_to_tos(regr)
+                return client_v1.agree_to_tos(regr)
             return regr
         else:
-            if "terms_of_service" in self.client.directory.meta:
-                _assess_tos(self.client.directory.meta.terms_of_service)
+            client_v2 = cast(ClientV2, self.client)
+            if "terms_of_service" in client_v2.directory.meta:
+                _assess_tos(client_v2.directory.meta.terms_of_service)
                 regr = regr.update(terms_of_service_agreed=True)
-            return self.client.new_account(regr)
+            return client_v2.new_account(regr)
 
     def new_order(self, csr_pem):
         """Request a new Order object from the server.
@@ -863,14 +869,15 @@ class BackwardsCompatibleClientV2:
 
         """
         if self.acme_version == 1:
+            client_v1 = cast(Client, self.client)
             csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)
             # pylint: disable=protected-access
             dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr)
             authorizations = []
             for domain in dnsNames:
-                authorizations.append(self.client.request_domain_challenges(domain))
+                authorizations.append(client_v1.request_domain_challenges(domain))
             return messages.OrderResource(authorizations=authorizations, csr_pem=csr_pem)
-        return self.client.new_order(csr_pem)
+        return cast(ClientV2, self.client).new_order(csr_pem)
 
     def finalize_order(self, orderr, deadline, fetch_alternative_chains=False):
         """Finalize an order and obtain a certificate.
@@ -885,8 +892,9 @@ class BackwardsCompatibleClientV2:
 
         """
         if self.acme_version == 1:
+            client_v1 = cast(Client, self.client)
             csr_pem = orderr.csr_pem
-            certr = self.client.request_issuance(
+            certr = client_v1.request_issuance(
                 jose.ComparableX509(
                     OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)),
                     orderr.authorizations)
@@ -894,7 +902,7 @@ class BackwardsCompatibleClientV2:
             chain = None
             while datetime.datetime.now() < deadline:
                 try:
-                    chain = self.client.fetch_chain(certr)
+                    chain = client_v1.fetch_chain(certr)
                     break
                 except errors.Error:
                     time.sleep(1)
@@ -909,7 +917,8 @@ class BackwardsCompatibleClientV2:
             chain = crypto_util.dump_pyopenssl_chain(chain).decode()
 
             return orderr.update(fullchain_pem=(cert + chain))
-        return self.client.finalize_order(orderr, deadline, fetch_alternative_chains)
+        return cast(ClientV2, self.client).finalize_order(
+            orderr, deadline, fetch_alternative_chains)
 
     def revoke(self, cert, rsn):
         """Revoke certificate.
@@ -935,7 +944,7 @@ class BackwardsCompatibleClientV2:
         Always return False for ACMEv1 servers, as it doesn't use External Account Binding."""
         if self.acme_version == 1:
             return False
-        return self.client.external_account_required()
+        return cast(ClientV2, self.client).external_account_required()
 
 
 class ClientNetwork:
@@ -968,7 +977,7 @@ class ClientNetwork:
         self.account = account
         self.alg = alg
         self.verify_ssl = verify_ssl
-        self._nonces = set() # type: Set[Text]
+        self._nonces: Set[Text] = set()
         self.user_agent = user_agent
         self.session = requests.Session()
         self._default_timeout = timeout
@@ -1128,6 +1137,7 @@ class ClientNetwork:
 
         # If content is DER, log the base64 of it instead of raw bytes, to keep
         # binary data out of the logs.
+        debug_content: Union[bytes, str]
         if response.headers.get("Content-Type") == DER_CONTENT_TYPE:
             debug_content = base64.b64encode(response.content)
         else:

acme/acme/crypto_util.py

@@ -5,15 +5,15 @@ import logging
 import os
 import re
 import socket
+from typing import Callable
+from typing import Tuple
+from typing import Union
 
 import josepy as jose
 from OpenSSL import crypto
 from OpenSSL import SSL  # type: ignore # https://github.com/python/typeshed/issues/2052
 
 from acme import errors
-from acme.magic_typing import Callable
-from acme.magic_typing import Tuple
-from acme.magic_typing import Union
 
 logger = logging.getLogger(__name__)
 
@@ -168,7 +168,7 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu
                 source_address[1]
             ) if any(source_address) else ""
         )
-        socket_tuple = (host, port)  # type: Tuple[str, int]
+        socket_tuple: Tuple[str, int] = (host, port)
         sock = socket.create_connection(socket_tuple, **socket_kwargs)  # type: ignore
     except socket.error as error:
         raise errors.Error(error)
@@ -256,7 +256,7 @@ def _pyopenssl_cert_or_req_san(cert_or_req):
 
     if isinstance(cert_or_req, crypto.X509):
         # pylint: disable=line-too-long
-        func = crypto.dump_certificate # type: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]]
+        func: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]] = crypto.dump_certificate
     else:
         func = crypto.dump_certificate_request
     text = func(crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8")

acme/acme/errors.py

@@ -28,13 +28,8 @@ class NonceError(ClientError):
 
 class BadNonce(NonceError):
     """Bad nonce error."""
-    def __init__(self, nonce, error, *args, **kwargs):
-        # MyPy complains here that there is too many arguments for BaseException constructor.
-        # This is an error fixed in typeshed, see https://github.com/python/mypy/issues/4183
-        # The fix is included in MyPy>=0.740, but upgrading it would bring dozen of errors due to
-        #   new types definitions. So we ignore the error until the code base is fixed to match
-        #   with MyPy>=0.740 referential.
-        super(BadNonce, self).__init__(*args, **kwargs)  # type: ignore
+    def __init__(self, nonce, error, *args):
+        super(BadNonce, self).__init__(*args)
         self.nonce = nonce
         self.error = error
 
@@ -52,9 +47,8 @@ class MissingNonce(NonceError):
     :ivar requests.Response ~.response: HTTP Response
 
     """
-    def __init__(self, response, *args, **kwargs):
-        # See comment in BadNonce constructor above for an explanation of type: ignore here.
-        super(MissingNonce, self).__init__(*args, **kwargs)  # type: ignore
+    def __init__(self, response, *args):
+        super(MissingNonce, self).__init__(*args)
         self.response = response
 
     def __str__(self):

acme/acme/jws.py

@@ -14,7 +14,9 @@ class Header(jose.Header):
     kid = jose.Field('kid', omitempty=True)
     url = jose.Field('url', omitempty=True)
 
-    @nonce.decoder
+    # Mypy does not understand the josepy magic happening here, and falsely claims
+    # that nonce is redefined. Let's ignore the type check here.
+    @nonce.decoder  # type: ignore
     def nonce(value):  # pylint: disable=no-self-argument,missing-function-docstring
         try:
             return jose.decode_b64jose(value)

acme/acme/magic_typing.py

@@ -4,9 +4,12 @@ This was useful when this code supported Python 2 and typing wasn't always
 available. This code is being kept for now for backwards compatibility.
 
 """
+import warnings
 from typing import *  # pylint: disable=wildcard-import, unused-wildcard-import
 from typing import Collection, IO  # type: ignore
 
+warnings.warn("acme.magic_typing is deprecated and will be removed in a future release.",
+              DeprecationWarning)
 
 class TypingClass:
     """Ignore import errors by getting anything"""

acme/acme/messages.py

@@ -1,6 +1,9 @@
 """ACME protocol messages."""
-import json
 from collections.abc import Hashable
+import json
+from typing import Any
+from typing import Dict
+from typing import Type
 
 import josepy as jose
 
@@ -87,7 +90,9 @@ class Error(jose.JSONObjectWithFields, errors.Error):
             raise ValueError("The supplied code: %s is not a known ACME error"
                              " code" % code)
         typ = ERROR_PREFIX + code
-        return cls(typ=typ, **kwargs)
+        # Mypy will not understand that the Error constructor accepts a named argument
+        # "typ" because of josepy magic. Let's ignore the type check here.
+        return cls(typ=typ, **kwargs)  # type: ignore
 
     @property
     def description(self):
@@ -124,7 +129,7 @@ class Error(jose.JSONObjectWithFields, errors.Error):
 class _Constant(jose.JSONDeSerializable, Hashable):  # type: ignore
     """ACME constant."""
     __slots__ = ('name',)
-    POSSIBLE_NAMES = NotImplemented
+    POSSIBLE_NAMES: Dict[str, '_Constant'] = NotImplemented
 
     def __init__(self, name):
         super(_Constant, self).__init__()
@@ -153,7 +158,7 @@ class _Constant(jose.JSONDeSerializable, Hashable):  # type: ignore
 
 class Status(_Constant):
     """ACME "status" field."""
-    POSSIBLE_NAMES = {}  # type: dict
+    POSSIBLE_NAMES: dict = {}
 STATUS_UNKNOWN = Status('unknown')
 STATUS_PENDING = Status('pending')
 STATUS_PROCESSING = Status('processing')
@@ -166,7 +171,7 @@ STATUS_DEACTIVATED = Status('deactivated')
 
 class IdentifierType(_Constant):
     """ACME identifier type."""
-    POSSIBLE_NAMES = {}  # type: dict
+    POSSIBLE_NAMES: Dict[str, 'IdentifierType'] = {}
 IDENTIFIER_FQDN = IdentifierType('dns')  # IdentifierDNS in Boulder
 
 
@@ -184,7 +189,7 @@ class Identifier(jose.JSONObjectWithFields):
 class Directory(jose.JSONDeSerializable):
     """Directory."""
 
-    _REGISTERED_TYPES = {}  # type: dict
+    _REGISTERED_TYPES: Dict[str, Type[Any]] = {}
 
     class Meta(jose.JSONObjectWithFields):
         """Directory Meta."""
@@ -218,7 +223,7 @@ class Directory(jose.JSONDeSerializable):
         return getattr(key, 'resource_type', key)
 
     @classmethod
-    def register(cls, resource_body_cls):
+    def register(cls, resource_body_cls: Type[Any]) -> Type[Any]:
         """Register resource."""
         resource_type = resource_body_cls.resource_type
         assert resource_type not in cls._REGISTERED_TYPES
@@ -528,7 +533,9 @@ class Authorization(ResourceBody):
     expires = fields.RFC3339Field('expires', omitempty=True)
     wildcard = jose.Field('wildcard', omitempty=True)
 
-    @challenges.decoder
+    # Mypy does not understand the josepy magic happening here, and falsely claims
+    # that challenge is redefined. Let's ignore the type check here.
+    @challenges.decoder  # type: ignore
     def challenges(value):  # pylint: disable=no-self-argument,missing-function-docstring
         return tuple(ChallengeBody.from_json(chall) for chall in value)
 
@@ -627,7 +634,9 @@ class Order(ResourceBody):
     expires = fields.RFC3339Field('expires', omitempty=True)
     error = jose.Field('error', omitempty=True, decoder=Error.from_json)
 
-    @identifiers.decoder
+    # Mypy does not understand the josepy magic happening here, and falsely claims
+    # that identifiers is redefined. Let's ignore the type check here.
+    @identifiers.decoder  # type: ignore
     def identifiers(value):  # pylint: disable=no-self-argument,missing-function-docstring
         return tuple(Identifier.from_json(identifier) for identifier in value)
 

acme/acme/standalone.py

@@ -7,10 +7,10 @@ import logging
 import socket
 import socketserver
 import threading
+from typing import List
 
 from acme import challenges
 from acme import crypto_util
-from acme.magic_typing import List
 
 logger = logging.getLogger(__name__)
 
@@ -63,8 +63,8 @@ class BaseDualNetworkedServers:
 
     def __init__(self, ServerClass, server_address, *remaining_args, **kwargs):
         port = server_address[1]
-        self.threads = [] # type: List[threading.Thread]
-        self.servers = [] # type: List[ACMEServerMixin]
+        self.threads: List[threading.Thread] = []
+        self.servers: List[socketserver.BaseServer] = []
 
         # Must try True first.
         # Ubuntu, for example, will fail to bind to IPv4 if we've already bound
@@ -203,8 +203,24 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 
     def __init__(self, *args, **kwargs):
         self.simple_http_resources = kwargs.pop("simple_http_resources", set())
-        self.timeout = kwargs.pop('timeout', 30)
+        self._timeout = kwargs.pop('timeout', 30)
         BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
+        self.server: HTTP01Server
+
+    # In parent class BaseHTTPRequestHandler, 'timeout' is a class-level property but we
+    # need to define its value during the initialization phase in HTTP01RequestHandler.
+    # However MyPy does not appreciate that we dynamically shadow a class-level property
+    # with an instance-level property (eg. self.timeout = ... in __init__()). So to make
+    # everyone happy, we statically redefine 'timeout' as a method property, and set the
+    # timeout value in a new internal instance-level property _timeout.
+    @property
+    def timeout(self):
+        """
+        The default timeout this server should apply to requests.
+        :return: timeout to apply
+        :rtype: int
+        """
+        return self._timeout
 
     def log_message(self, format, *args):  # pylint: disable=redefined-builtin
         """Log arbitrary message."""

acme/setup.py

@@ -3,7 +3,7 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

acme/tests/client_test.py

@@ -5,6 +5,7 @@ import datetime
 import http.client as http_client
 import json
 import unittest
+from typing import Dict
 from unittest import mock
 
 import josepy as jose
@@ -61,7 +62,7 @@ class ClientTestBase(unittest.TestCase):
         self.contact = ('mailto:cert-admin@example.com', 'tel:+12025551212')
         reg = messages.Registration(
             contact=self.contact, key=KEY.public_key())
-        the_arg = dict(reg)  # type: Dict
+        the_arg: Dict = dict(reg)
         self.new_reg = messages.NewRegistration(**the_arg)
         self.regr = messages.RegistrationResource(
             body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1')

acme/tests/crypto_util_test.py

@@ -5,6 +5,7 @@ import socketserver
 import threading
 import time
 import unittest
+from typing import List
 
 import josepy as jose
 import OpenSSL
@@ -180,7 +181,7 @@ class RandomSnTest(unittest.TestCase):
 
     def setUp(self):
         self.cert_count = 5
-        self.serial_num = [] # type: List[int]
+        self.serial_num: List[int] = []
         self.key = OpenSSL.crypto.PKey()
         self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
 

acme/tests/magic_typing_test.py

@@ -1,6 +1,7 @@
 """Tests for acme.magic_typing."""
 import sys
 import unittest
+import warnings
 from unittest import mock
 
 
@@ -9,15 +10,17 @@ class MagicTypingTest(unittest.TestCase):
     def test_import_success(self):
         try:
             import typing as temp_typing
-        except ImportError: # pragma: no cover
-            temp_typing = None # pragma: no cover
+        except ImportError:  # pragma: no cover
+            temp_typing = None  # pragma: no cover
         typing_class_mock = mock.MagicMock()
         text_mock = mock.MagicMock()
         typing_class_mock.Text = text_mock
         sys.modules['typing'] = typing_class_mock
         if 'acme.magic_typing' in sys.modules:
-            del sys.modules['acme.magic_typing'] # pragma: no cover
-        from acme.magic_typing import Text
+            del sys.modules['acme.magic_typing']  # pragma: no cover
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", category=DeprecationWarning)
+            from acme.magic_typing import Text
         self.assertEqual(Text, text_mock)
         del sys.modules['acme.magic_typing']
         sys.modules['typing'] = temp_typing

acme/tests/messages_test.py

@@ -1,4 +1,5 @@
 """Tests for acme.messages."""
+from typing import Dict
 import unittest
 from unittest import mock
 
@@ -81,7 +82,7 @@ class ConstantTest(unittest.TestCase):
         from acme.messages import _Constant
 
         class MockConstant(_Constant):  # pylint: disable=missing-docstring
-            POSSIBLE_NAMES = {} # type: Dict
+            POSSIBLE_NAMES: Dict = {}
 
         self.MockConstant = MockConstant  # pylint: disable=invalid-name
         self.const_a = MockConstant('a')

acme/tests/standalone_test.py

@@ -4,6 +4,7 @@ import socket
 import socketserver
 import threading
 import unittest
+from typing import Set
 from unittest import mock
 
 import josepy as jose
@@ -41,7 +42,7 @@ class HTTP01ServerTest(unittest.TestCase):
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))
-        self.resources = set() # type: Set
+        self.resources: Set = set()
 
         from acme.standalone import HTTP01Server
         self.server = HTTP01Server(('', 0), resources=self.resources)
@@ -218,7 +219,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase):
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))
-        self.resources = set() # type: Set
+        self.resources: Set = set()
 
         from acme.standalone import HTTP01DualNetworkedServers
         self.servers = HTTP01DualNetworkedServers(('', 0), resources=self.resources)

certbot-apache/certbot_apache/_internal/apache_util.py

@@ -9,7 +9,6 @@ import pkg_resources
 
 from certbot import errors
 from certbot import util
-
 from certbot.compat import os
 
 logger = logging.getLogger(__name__)

certbot-apache/certbot_apache/_internal/apacheparser.py

@@ -1,4 +1,5 @@
 """ apacheconfig implementation of the ParserNode interfaces """
+from typing import Tuple
 
 from certbot_apache._internal import assertions
 from certbot_apache._internal import interfaces
@@ -21,7 +22,7 @@ class ApacheParserNode(interfaces.ParserNode):
         self.metadata = metadata
         self._raw = self.metadata["ac_ast"]
 
-    def save(self, msg): # pragma: no cover
+    def save(self, msg):  # pragma: no cover
         pass
 
     def find_ancestors(self, name):  # pylint: disable=unused-variable
@@ -83,7 +84,7 @@ class ApacheBlockNode(ApacheDirectiveNode):
 
     def __init__(self, **kwargs):
         super(ApacheBlockNode, self).__init__(**kwargs)
-        self.children = ()
+        self.children: Tuple[ApacheParserNode, ...] = ()
 
     def __eq__(self, other):  # pragma: no cover
         if isinstance(other, self.__class__):

certbot-apache/certbot_apache/_internal/assertions.py

@@ -3,7 +3,6 @@ import fnmatch
 
 from certbot_apache._internal import interfaces
 
-
 PASS = "CERTBOT_PASS_ASSERT"
 
 

certbot-apache/certbot_apache/_internal/augeasparser.py

@@ -64,10 +64,10 @@ Translates over to:
     "/files/etc/apache2/apache2.conf/bLoCk[1]",
 ]
 """
-from acme.magic_typing import Set
+from typing import Set
+
 from certbot import errors
 from certbot.compat import os
-
 from certbot_apache._internal import apache_util
 from certbot_apache._internal import assertions
 from certbot_apache._internal import interfaces
@@ -355,7 +355,7 @@ class AugeasBlockNode(AugeasDirectiveNode):
         ownpath = self.metadata.get("augeaspath")
 
         directives = self.parser.find_dir(name, start=ownpath, exclude=exclude)
-        already_parsed = set()  # type: Set[str]
+        already_parsed: Set[str] = set()
         for directive in directives:
             # Remove the /arg part from the Augeas path
             directive = directive.partition("/arg")[0]

certbot-apache/certbot_apache/_internal/configurator.py

@@ -1,28 +1,24 @@
 """Apache Configurator."""
 # pylint: disable=too-many-lines
 from collections import defaultdict
-from distutils.version import LooseVersion
 import copy
+from distutils.version import LooseVersion
 import fnmatch
 import logging
 import re
 import socket
 import time
+from typing import cast
+from typing import DefaultDict
+from typing import Dict
+from typing import List
+from typing import Set
+from typing import Union
 
 import zope.component
 import zope.interface
-try:
-    import apacheconfig
-    HAS_APACHECONFIG = True
-except ImportError:  # pragma: no cover
-    HAS_APACHECONFIG = False
 
 from acme import challenges
-from acme.magic_typing import DefaultDict
-from acme.magic_typing import Dict
-from acme.magic_typing import List
-from acme.magic_typing import Set
-from acme.magic_typing import Union
 from certbot import errors
 from certbot import interfaces
 from certbot import util
@@ -41,6 +37,13 @@ from certbot_apache._internal import http_01
 from certbot_apache._internal import obj
 from certbot_apache._internal import parser
 
+try:
+    import apacheconfig
+    HAS_APACHECONFIG = True
+except ImportError:  # pragma: no cover
+    HAS_APACHECONFIG = False
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -154,9 +157,9 @@ class ApacheConfigurator(common.Installer):
                 self.options[o] = self.OS_DEFAULTS[o]
 
         # Special cases
-        self.options["version_cmd"][0] = self.option("ctl")
-        self.options["restart_cmd"][0] = self.option("ctl")
-        self.options["conftest_cmd"][0] = self.option("ctl")
+        cast(List[str], self.options["version_cmd"])[0] = self.option("ctl")
+        cast(List[str], self.options["restart_cmd"])[0] = self.option("ctl")
+        cast(List[str], self.options["conftest_cmd"])[0] = self.option("ctl")
 
     @classmethod
     def add_parser_arguments(cls, add):
@@ -210,23 +213,23 @@ class ApacheConfigurator(common.Installer):
         super(ApacheConfigurator, self).__init__(*args, **kwargs)
 
         # Add name_server association dict
-        self.assoc = {}  # type: Dict[str, obj.VirtualHost]
+        self.assoc: Dict[str, obj.VirtualHost] = {}
         # Outstanding challenges
-        self._chall_out = set()  # type: Set[KeyAuthorizationAnnotatedChallenge]
+        self._chall_out: Set[KeyAuthorizationAnnotatedChallenge] = set()
         # List of vhosts configured per wildcard domain on this run.
         # used by deploy_cert() and enhance()
-        self._wildcard_vhosts = {}  # type: Dict[str, List[obj.VirtualHost]]
+        self._wildcard_vhosts: Dict[str, List[obj.VirtualHost]] = {}
         # Maps enhancements to vhosts we've enabled the enhancement for
-        self._enhanced_vhosts = defaultdict(set)  # type: DefaultDict[str, Set[obj.VirtualHost]]
+        self._enhanced_vhosts: DefaultDict[str, Set[obj.VirtualHost]] = defaultdict(set)
         # Temporary state for AutoHSTS enhancement
-        self._autohsts = {}  # type: Dict[str, Dict[str, Union[int, float]]]
+        self._autohsts: Dict[str, Dict[str, Union[int, float]]] = {}
         # Reverter save notes
         self.save_notes = ""
         # Should we use ParserNode implementation instead of the old behavior
         self.USE_PARSERNODE = use_parsernode
         # Saves the list of file paths that were parsed initially, and
         # not added to parser tree by self.conf("vhost-root") for example.
-        self.parsed_paths = []  # type: List[str]
+        self.parsed_paths: List[str] = []
         # These will be set in the prepare function
         self._prepared = False
         self.parser = None
@@ -832,7 +835,7 @@ class ApacheConfigurator(common.Installer):
         :rtype: set
 
         """
-        all_names = set()  # type: Set[str]
+        all_names: Set[str] = set()
 
         vhost_macro = []
 
@@ -996,8 +999,8 @@ class ApacheConfigurator(common.Installer):
 
         """
         # Search base config, and all included paths for VirtualHosts
-        file_paths = {}  # type: Dict[str, str]
-        internal_paths = defaultdict(set)  # type: DefaultDict[str, Set[str]]
+        file_paths: Dict[str, str] = {}
+        internal_paths: DefaultDict[str, Set[str]] = defaultdict(set)
         vhs = []
         # Make a list of parser paths because the parser_paths
         # dictionary may be modified during the loop.
@@ -2156,7 +2159,7 @@ class ApacheConfigurator(common.Installer):
         # There can be other RewriteRule directive lines in vhost config.
         # rewrite_args_dict keys are directive ids and the corresponding value
         # for each is a list of arguments to that directive.
-        rewrite_args_dict = defaultdict(list)  # type: DefaultDict[str, List[str]]
+        rewrite_args_dict: DefaultDict[str, List[str]] = defaultdict(list)
         pat = r'(.*directive\[\d+\]).*'
         for match in rewrite_path:
             m = re.match(pat, match)
@@ -2250,7 +2253,7 @@ class ApacheConfigurator(common.Installer):
         if ssl_vhost.aliases:
             serveralias = "ServerAlias " + " ".join(ssl_vhost.aliases)
 
-        rewrite_rule_args = []  # type: List[str]
+        rewrite_rule_args: List[str] = []
         if self.get_version() >= (2, 3, 9):
             rewrite_rule_args = constants.REWRITE_HTTPS_ARGS_WITH_END
         else:

certbot-apache/certbot_apache/_internal/dualparser.py

@@ -1,7 +1,7 @@
 """ Dual ParserNode implementation """
+from certbot_apache._internal import apacheparser
 from certbot_apache._internal import assertions
 from certbot_apache._internal import augeasparser
-from certbot_apache._internal import apacheparser
 
 
 class DualNodeBase:

certbot-apache/certbot_apache/_internal/http_01.py

@@ -1,9 +1,9 @@
 """A class that performs HTTP-01 challenges for Apache"""
-import logging
 import errno
+import logging
+from typing import List
+from typing import Set
 
-from acme.magic_typing import List
-from acme.magic_typing import Set
 from certbot import errors
 from certbot.compat import filesystem
 from certbot.compat import os
@@ -57,7 +57,7 @@ class ApacheHttp01(common.ChallengePerformer):
         self.challenge_dir = os.path.join(
             self.configurator.config.work_dir,
             "http_challenges")
-        self.moded_vhosts = set()  # type: Set[VirtualHost]
+        self.moded_vhosts: Set[VirtualHost] = set()
 
     def perform(self):
         """Perform all HTTP-01 challenges."""
@@ -93,7 +93,7 @@ class ApacheHttp01(common.ChallengePerformer):
                     self.configurator.enable_mod(mod, temp=True)
 
     def _mod_config(self):
-        selected_vhosts = []  # type: List[VirtualHost]
+        selected_vhosts: List[VirtualHost] = []
         http_port = str(self.configurator.config.http01_port)
         for chall in self.achalls:
             # Search for matching VirtualHosts

certbot-apache/certbot_apache/_internal/interfaces.py

@@ -102,7 +102,6 @@ For this reason the internal representation of data should not ignore the case.
 import abc
 
 
-
 class ParserNode(object, metaclass=abc.ABCMeta):
     """
     ParserNode is the basic building block of the tree of such nodes,

certbot-apache/certbot_apache/_internal/obj.py

@@ -1,7 +1,7 @@
 """Module contains classes used by the Apache Configurator."""
 import re
+from typing import Set
 
-from acme.magic_typing import Set
 from certbot.plugins import common
 
 
@@ -137,7 +137,7 @@ class VirtualHost:
 
     def get_names(self):
         """Return a set of all names."""
-        all_names = set()  # type: Set[str]
+        all_names: Set[str] = set()
         all_names.update(self.aliases)
         # Strip out any scheme:// and <port> field from servername
         if self.name is not None:
@@ -245,7 +245,7 @@ class VirtualHost:
 
         # already_found acts to keep everything very conservative.
         # Don't allow multiple ip:ports in same set.
-        already_found = set()  # type: Set[str]
+        already_found: Set[str] = set()
 
         for addr in vhost.addrs:
             for local_addr in self.addrs:

certbot-apache/certbot_apache/_internal/override_centos.py

@@ -1,9 +1,10 @@
 """ Distribution specific override class for CentOS family (RHEL, Fedora) """
 import logging
+from typing import cast
+from typing import List
 
 import zope.interface
 
-from acme.magic_typing import List
 from certbot import errors
 from certbot import interfaces
 from certbot import util
@@ -76,7 +77,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
         alternative restart cmd used in CentOS.
         """
         super(CentOSConfigurator, self)._prepare_options()
-        self.options["restart_cmd_alt"][0] = self.option("ctl")
+        cast(List[str], self.options["restart_cmd_alt"])[0] = self.option("ctl")
 
     def get_parser(self):
         """Initializes the ApacheParser"""
@@ -102,9 +103,9 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
 
         loadmods = self.parser.find_dir("LoadModule", "ssl_module", exclude=False)
 
-        correct_ifmods = []  # type: List[str]
-        loadmod_args = []  # type: List[str]
-        loadmod_paths = []  # type: List[str]
+        correct_ifmods: List[str] = []
+        loadmod_args: List[str] = []
+        loadmod_paths: List[str] = []
         for m in loadmods:
             noarg_path = m.rpartition("/")[0]
             path_args = self.parser.get_all_args(noarg_path)

certbot-apache/certbot_apache/_internal/override_fedora.py

@@ -1,4 +1,7 @@
 """ Distribution specific override class for Fedora 29+ """
+from typing import cast
+from typing import List
+
 import zope.interface
 
 from certbot import errors
@@ -69,9 +72,9 @@ class FedoraConfigurator(configurator.ApacheConfigurator):
         of Fedora to restart httpd.
         """
         super(FedoraConfigurator, self)._prepare_options()
-        self.options["restart_cmd"][0] = 'apachectl'
-        self.options["restart_cmd_alt"][0] = 'apachectl'
-        self.options["conftest_cmd"][0] = 'apachectl'
+        cast(List[str], self.options["restart_cmd"])[0] = 'apachectl'
+        cast(List[str], self.options["restart_cmd_alt"])[0] = 'apachectl'
+        cast(List[str], self.options["conftest_cmd"])[0] = 'apachectl'
 
 
 class FedoraParser(parser.ApacheParser):

certbot-apache/certbot_apache/_internal/override_gentoo.py

@@ -1,4 +1,7 @@
 """ Distribution specific override class for Gentoo Linux """
+from typing import cast
+from typing import List
+
 import zope.interface
 
 from certbot import interfaces
@@ -36,7 +39,7 @@ class GentooConfigurator(configurator.ApacheConfigurator):
         alternative restart cmd used in Gentoo.
         """
         super(GentooConfigurator, self)._prepare_options()
-        self.options["restart_cmd_alt"][0] = self.option("ctl")
+        cast(List[str], self.options["restart_cmd_alt"])[0] = self.option("ctl")
 
     def get_parser(self):
         """Initializes the ApacheParser"""

certbot-apache/certbot_apache/_internal/parser.py

@@ -3,11 +3,9 @@ import copy
 import fnmatch
 import logging
 import re
-import sys
+from typing import Dict
+from typing import List
 
-
-from acme.magic_typing import Dict
-from acme.magic_typing import List
 from certbot import errors
 from certbot.compat import os
 from certbot_apache._internal import apache_util
@@ -50,9 +48,9 @@ class ApacheParser:
                 "version 1.2.0 or higher, please make sure you have you have "
                 "those installed.")
 
-        self.modules = {}  # type: Dict[str, str]
-        self.parser_paths = {}  # type: Dict[str, List[str]]
-        self.variables = {}  # type: Dict[str, str]
+        self.modules: Dict[str, str] = {}
+        self.parser_paths: Dict[str, List[str]] = {}
+        self.variables: Dict[str, str] = {}
 
         # Find configuration root and make sure augeas can parse it.
         self.root = os.path.abspath(root)
@@ -265,7 +263,7 @@ class ApacheParser:
             the iteration issue.  Else... parse and enable mods at same time.
 
         """
-        mods = {}  # type: Dict[str, str]
+        mods: Dict[str, str] = {}
         matches = self.find_dir("LoadModule")
         iterator = iter(matches)
         # Make sure prev_size != cur_size for do: while: iteration
@@ -552,7 +550,7 @@ class ApacheParser:
         else:
             arg_suffix = "/*[self::arg=~regexp('%s')]" % case_i(arg)
 
-        ordered_matches = []  # type: List[str]
+        ordered_matches: List[str] = []
 
         # TODO: Wildcards should be included in alphabetical order
         # https://httpd.apache.org/docs/2.4/mod/core.html#include
@@ -737,9 +735,6 @@ class ApacheParser:
         :rtype: str
 
         """
-        if sys.version_info < (3, 6):
-            # This strips off final /Z(?ms)
-            return fnmatch.translate(clean_fn_match)[:-7]  # pragma: no cover
         # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z
         return fnmatch.translate(clean_fn_match)[4:-3]  # pragma: no cover
 

certbot-apache/setup.py

@@ -1,7 +1,7 @@
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-apache/tests/augeasnode_test.py

@@ -1,4 +1,6 @@
 """Tests for AugeasParserNode classes"""
+from typing import List
+
 try:
     import mock
 except ImportError: # pragma: no cover
@@ -107,7 +109,7 @@ class AugeasParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-
 
     def test_set_parameters(self):
         servernames = self.config.parser_root.find_directives("servername")
-        names = []  # type: List[str]
+        names: List[str] = []
         for servername in servernames:
             names += servername.parameters
         self.assertFalse("going_to_set_this" in names)

certbot-apache/tests/http_01_test.py

@@ -1,6 +1,7 @@
 """Test for certbot_apache._internal.http_01."""
 import unittest
 import errno
+from typing import List
 
 try:
     import mock
@@ -26,7 +27,7 @@ class ApacheHttp01Test(util.ApacheTest):
         super(ApacheHttp01Test, self).setUp(*args, **kwargs)
 
         self.account_key = self.rsa512jwk
-        self.achalls = []  # type: List[achallenges.KeyAuthorizationAnnotatedChallenge]
+        self.achalls: List[achallenges.KeyAuthorizationAnnotatedChallenge] = []
         vh_truth = util.get_vh_truth(
             self.temp_dir, "debian_apache_2_4/multiple_vhosts")
         # Takes the vhosts for encryption-example.demo, certbot.demo

certbot-ci/certbot_integration_tests/assets/hook.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import print_function
 import os
 import sys
 

certbot-ci/certbot_integration_tests/certbot_tests/context.py

@@ -12,9 +12,9 @@ class IntegrationTestsContext:
     def __init__(self, request):
         self.request = request
 
-        if hasattr(request.config, 'slaveinput'):  # Worker node
-            self.worker_id = request.config.slaveinput['slaveid']
-            acme_xdist = request.config.slaveinput['acme_xdist']
+        if hasattr(request.config, 'workerinput'):  # Worker node
+            self.worker_id = request.config.workerinput['workerid']
+            acme_xdist = request.config.workerinput['acme_xdist']
         else:  # Primary node
             self.worker_id = 'primary'
             acme_xdist = request.config.acme_xdist

certbot-ci/certbot_integration_tests/certbot_tests/test_main.py

@@ -1,5 +1,4 @@
 """Module executing integration tests against certbot core."""
-from __future__ import print_function
 
 import os
 from os.path import exists
@@ -9,19 +8,20 @@ import shutil
 import subprocess
 import time
 
-from cryptography.hazmat.primitives.asymmetric.ec import SECP256R1, SECP384R1, SECP521R1
+from cryptography.hazmat.primitives.asymmetric.ec import SECP256R1
+from cryptography.hazmat.primitives.asymmetric.ec import SECP384R1
+from cryptography.hazmat.primitives.asymmetric.ec import SECP521R1
 from cryptography.x509 import NameOID
-
 import pytest
 
 from certbot_integration_tests.certbot_tests import context as certbot_context
 from certbot_integration_tests.certbot_tests.assertions import assert_cert_count_for_lineage
 from certbot_integration_tests.certbot_tests.assertions import assert_elliptic_key
-from certbot_integration_tests.certbot_tests.assertions import assert_rsa_key
 from certbot_integration_tests.certbot_tests.assertions import assert_equals_group_owner
 from certbot_integration_tests.certbot_tests.assertions import assert_equals_group_permissions
 from certbot_integration_tests.certbot_tests.assertions import assert_equals_world_read_permissions
 from certbot_integration_tests.certbot_tests.assertions import assert_hook_execution
+from certbot_integration_tests.certbot_tests.assertions import assert_rsa_key
 from certbot_integration_tests.certbot_tests.assertions import assert_saved_renew_hook
 from certbot_integration_tests.certbot_tests.assertions import assert_world_no_permissions
 from certbot_integration_tests.certbot_tests.assertions import assert_world_read_permissions

certbot-ci/certbot_integration_tests/conftest.py

@@ -6,7 +6,6 @@ for a directory a specific configuration using built-in pytest hooks.
 
 See https://docs.pytest.org/en/latest/reference.html#hook-reference
 """
-from __future__ import print_function
 import contextlib
 import subprocess
 import sys
@@ -35,7 +34,7 @@ def pytest_configure(config):
     Standard pytest hook used to add a configuration logic for each node of a pytest run.
     :param config: the current pytest configuration
     """
-    if not hasattr(config, 'slaveinput'):  # If true, this is the primary node
+    if not hasattr(config, 'workerinput'):  # If true, this is the primary node
         with _print_on_err():
             _setup_primary_node(config)
 
@@ -45,8 +44,8 @@ def pytest_configure_node(node):
     Standard pytest-xdist hook used to configure a worker node.
     :param node: current worker node
     """
-    node.slaveinput['acme_xdist'] = node.config.acme_xdist
-    node.slaveinput['dns_xdist'] = node.config.dns_xdist
+    node.workerinput['acme_xdist'] = node.config.acme_xdist
+    node.workerinput['dns_xdist'] = node.config.dns_xdist
 
 
 @contextlib.contextmanager

certbot-ci/certbot_integration_tests/nginx_tests/test_main.py

@@ -1,8 +1,8 @@
 """Module executing integration tests against certbot with nginx plugin."""
 import os
 import ssl
-
 from typing import List
+
 import pytest
 
 from certbot_integration_tests.nginx_tests import context as nginx_context
@@ -32,8 +32,8 @@ def test_context(request):
         '--preferred-challenges', 'http'
     ], {'default_server': False}),
 ], indirect=['context'])
-def test_certificate_deployment(certname_pattern, params, context):
-    # type: (str, List[str], nginx_context.IntegrationTestsContext) -> None
+def test_certificate_deployment(certname_pattern: str, params: List[str],
+                                context: nginx_context.IntegrationTestsContext) -> None:
     """
     Test various scenarios to deploy a certificate to nginx using certbot.
     """

certbot-ci/certbot_integration_tests/rfc2136_tests/context.py

@@ -1,7 +1,7 @@
 """Module to handle the context of RFC2136 integration tests."""
 
-import tempfile
 from contextlib import contextmanager
+import tempfile
 
 from pkg_resources import resource_filename
 from pytest import skip
@@ -18,8 +18,8 @@ class IntegrationTestsContext(certbot_context.IntegrationTestsContext):
         self.request = request
 
         self._dns_xdist = None
-        if hasattr(request.config, 'slaveinput'):  # Worker node
-            self._dns_xdist = request.config.slaveinput['dns_xdist']
+        if hasattr(request.config, 'workerinput'):  # Worker node
+            self._dns_xdist = request.config.workerinput['dns_xdist']
         else:  # Primary node
             self._dns_xdist = request.config.dns_xdist
 

certbot-ci/certbot_integration_tests/utils/acme_server.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 """Module to setup an ACME CA server environment able to run multiple tests in parallel"""
-from __future__ import print_function
 
 import argparse
 import errno
@@ -12,14 +11,14 @@ import subprocess
 import sys
 import tempfile
 import time
-
 from typing import List
+
 import requests
 
+# pylint: disable=wildcard-import,unused-wildcard-import
 from certbot_integration_tests.utils import misc
 from certbot_integration_tests.utils import pebble_artifacts
 from certbot_integration_tests.utils import proxy
-# pylint: disable=wildcard-import,unused-wildcard-import
 from certbot_integration_tests.utils.constants import *
 
 
@@ -52,7 +51,7 @@ class ACMEServer:
         self._acme_type = 'pebble' if acme_server == 'pebble' else 'boulder'
         self._proxy = http_proxy
         self._workspace = tempfile.mkdtemp()
-        self._processes = []  # type: List[subprocess.Popen]
+        self._processes: List[subprocess.Popen] = []
         self._stdout = sys.stdout if stdout else open(os.devnull, 'w')
         self._dns_server = dns_server
         self._http_01_port = http_01_port

certbot-ci/certbot_integration_tests/utils/certbot_call.py

@@ -1,11 +1,10 @@
 #!/usr/bin/env python
 """Module to call certbot in test mode"""
-from __future__ import absolute_import
 
+from distutils.version import LooseVersion
 import os
 import subprocess
 import sys
-from distutils.version import LooseVersion
 
 import certbot_integration_tests
 # pylint: disable=wildcard-import,unused-wildcard-import

certbot-ci/certbot_integration_tests/utils/dns_server.py

@@ -1,7 +1,5 @@
 #!/usr/bin/env python
 """Module to setup an RFC2136-capable DNS server"""
-from __future__ import print_function
-
 import os
 import os.path
 import shutil
@@ -40,7 +38,7 @@ class DNSServer:
 
         self.bind_root = tempfile.mkdtemp()
 
-        self.process = None  # type: subprocess.Popen
+        self.process: subprocess.Popen = None
 
         self.dns_xdist = {"address": BIND_BIND_ADDRESS[0], "port": BIND_BIND_ADDRESS[1]}
 
@@ -113,8 +111,7 @@ class DNSServer:
             self.stop()
             raise
 
-    def _wait_until_ready(self, attempts=30):
-        # type: (int) -> None
+    def _wait_until_ready(self, attempts: int = 30) -> None:
         """
         Polls the DNS server over TCP until it gets a response, or until
         it runs out of attempts and raises a ValueError.

certbot-ci/certbot_integration_tests/utils/misc.py

@@ -26,8 +26,8 @@ from OpenSSL import crypto
 import pkg_resources
 import requests
 
-from certbot_integration_tests.utils.constants import \
-     PEBBLE_ALTERNATE_ROOTS, PEBBLE_MANAGEMENT_URL
+from certbot_integration_tests.utils.constants import PEBBLE_ALTERNATE_ROOTS
+from certbot_integration_tests.utils.constants import PEBBLE_MANAGEMENT_URL
 
 RSA_KEY_TYPE = 'rsa'
 ECDSA_KEY_TYPE = 'ecdsa'
@@ -311,7 +311,7 @@ def echo(keyword, path=None):
     if not re.match(r'^\w+$', keyword):
         raise ValueError('Error, keyword `{0}` is not a single keyword.'
                          .format(keyword))
-    return '{0} -c "from __future__ import print_function; print(\'{1}\')"{2}'.format(
+    return '{0} -c "print(\'{1}\')"{2}'.format(
         os.path.basename(sys.executable), keyword, ' >> "{0}"'.format(path) if path else '')
 
 

certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py

@@ -7,7 +7,8 @@ import stat
 import pkg_resources
 import requests
 
-from certbot_integration_tests.utils.constants import DEFAULT_HTTP_01_PORT, MOCK_OCSP_SERVER_PORT
+from certbot_integration_tests.utils.constants import DEFAULT_HTTP_01_PORT
+from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT
 
 PEBBLE_VERSION = 'v2.3.0'
 ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets')

certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py

@@ -29,10 +29,7 @@ class _ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
         request = requests.get(PEBBLE_MANAGEMENT_URL + '/intermediates/0', verify=False)
         issuer_cert = x509.load_pem_x509_certificate(request.content, default_backend())
 
-        try:
-            content_len = int(self.headers.getheader('content-length', 0))
-        except AttributeError:
-            content_len = int(self.headers.get('Content-Length'))
+        content_len = int(self.headers.get('Content-Length'))
 
         ocsp_request = ocsp.load_der_ocsp_request(self.rfile.read(content_len))
         response = requests.get('{0}/cert-status-by-serial/{1}'.format(

certbot-ci/setup.py

@@ -7,6 +7,13 @@ from setuptools import setup
 
 version = '0.32.0.dev0'
 
+# setuptools 36.2+ is needed for support for environment markers
+min_setuptools_version='36.2'
+# This conditional isn't necessary, but it provides better error messages to
+# people who try to install this package with older versions of setuptools.
+if LooseVersion(setuptools_version) < LooseVersion(min_setuptools_version):
+    raise RuntimeError(f'setuptools {min_setuptools_version}+ is required')
+
 install_requires = [
     'coverage',
     'cryptography',
@@ -14,23 +21,17 @@ install_requires = [
     'pyopenssl',
     'pytest',
     'pytest-cov',
-    'pytest-xdist',
+    # This version is needed for "worker" attributes we currently use like
+    # "workerinput".  See https://github.com/pytest-dev/pytest-xdist/pull/268.
+    'pytest-xdist>=1.22.1',
     'python-dateutil',
+    # This dependency needs to be added using environment markers to avoid its
+    # installation on Linux.
+    'pywin32>=300 ; sys_platform == "win32"',
     'pyyaml',
     'requests',
 ]
 
-# Add pywin32 on Windows platforms to handle low-level system calls.
-# This dependency needs to be added using environment markers to avoid its installation on Linux.
-# However environment markers are supported only with setuptools >= 36.2.
-# So this dependency is not added for old Linux distributions with old setuptools,
-# in order to allow these systems to build certbot from sources.
-if LooseVersion(setuptools_version) >= LooseVersion('36.2'):
-    install_requires.append("pywin32>=224 ; sys_platform == 'win32'")
-elif 'bdist_wheel' in sys.argv[1:]:
-    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
-                       'of setuptools. Version 36.2+ of setuptools is required.')
-
 setup(
     name='certbot-ci',
     version=version,

certbot-ci/snap_integration_tests/dns_tests/test_main.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
-import pytest
-import subprocess
 import glob
 import os
 import re
+import subprocess
+
+import pytest
 
 
 @pytest.fixture(autouse=True, scope="module")

certbot-ci/windows_installer_integration_tests/conftest.py

@@ -6,7 +6,7 @@ for a directory a specific configuration using built-in pytest hooks.
 
 See https://docs.pytest.org/en/latest/reference.html#hook-reference
 """
-from __future__ import print_function
+
 import os
 
 ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))

certbot-ci/windows_installer_integration_tests/test_main.py

@@ -1,8 +1,8 @@
 import os
+import re
+import subprocess
 import time
 import unittest
-import subprocess
-import re
 
 
 @unittest.skipIf(os.name != 'nt', reason='Windows installer tests must be run on Windows.')

certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py

@@ -2,10 +2,10 @@
 import os
 import shutil
 import subprocess
+from typing import Set
 
 import zope.interface
 
-from acme.magic_typing import Set
 from certbot._internal import configuration
 from certbot_compatibility_test import errors
 from certbot_compatibility_test import interfaces
@@ -68,7 +68,7 @@ def _get_server_root(config):
 
 def _get_names(config):
     """Returns all and testable domain names in config"""
-    all_names = set()  # type: Set[str]
+    all_names: Set[str] = set()
     for root, _dirs, files in os.walk(config):
         for this_file in files:
             update_names = _get_server_names(root, this_file)

certbot-compatibility-test/certbot_compatibility_test/test_driver.py

@@ -8,6 +8,8 @@ import shutil
 import sys
 import tempfile
 import time
+from typing import List
+from typing import Tuple
 
 import OpenSSL
 from urllib3.util import connection
@@ -15,8 +17,6 @@ from urllib3.util import connection
 from acme import challenges
 from acme import crypto_util
 from acme import messages
-from acme.magic_typing import List
-from acme.magic_typing import Tuple
 from certbot import achallenges
 from certbot import errors as le_errors
 from certbot.tests import acme_util
@@ -178,7 +178,7 @@ def test_enhancements(plugin, domains):
                      "enhancements")
         return False
 
-    domains_and_info = [(domain, []) for domain in domains]  # type: List[Tuple[str, List[bool]]]
+    domains_and_info: List[Tuple[str, List[bool]]] = [(domain, []) for domain in domains]
 
     for domain, info in domains_and_info:
         try:

certbot-compatibility-test/setup.py

@@ -3,7 +3,7 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 install_requires = [
     'certbot',

certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py

@@ -1,13 +1,12 @@
 """DNS Authenticator for Cloudflare."""
 import logging
+from typing import Any
+from typing import Dict
+from typing import List
 
 import CloudFlare
 import zope.interface
 
-from acme.magic_typing import Any
-from acme.magic_typing import Dict
-from acme.magic_typing import List
-
 from certbot import errors
 from certbot import interfaces
 from certbot.plugins import dns_common
@@ -173,7 +172,7 @@ class _CloudflareClient:
         """
 
         zone_name_guesses = dns_common.base_domain_name_guesses(domain)
-        zones = []  # type: List[Dict[str, Any]]
+        zones: List[Dict[str, Any]] = []
         code = msg = None
 
         for zone_name in zone_name_guesses:

certbot-dns-cloudflare/setup.py

@@ -4,7 +4,7 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-cloudxns/setup.py

@@ -4,7 +4,7 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py

@@ -21,6 +21,7 @@ class Authenticator(dns_common.DNSAuthenticator):
 
     description = 'Obtain certificates using a DNS TXT record (if you are ' + \
                   'using DigitalOcean for DNS).'
+    ttl = 30
 
     def __init__(self, *args, **kwargs):
         super(Authenticator, self).__init__(*args, **kwargs)
@@ -45,7 +46,8 @@ class Authenticator(dns_common.DNSAuthenticator):
         )
 
     def _perform(self, domain, validation_name, validation):
-        self._get_digitalocean_client().add_txt_record(domain, validation_name, validation)
+        self._get_digitalocean_client().add_txt_record(domain, validation_name, validation,
+                                                       self.ttl)
 
     def _cleanup(self, domain, validation_name, validation):
         self._get_digitalocean_client().del_txt_record(domain, validation_name, validation)
@@ -62,13 +64,15 @@ class _DigitalOceanClient:
     def __init__(self, token):
         self.manager = digitalocean.Manager(token=token)
 
-    def add_txt_record(self, domain_name, record_name, record_content):
+    def add_txt_record(self, domain_name: str, record_name: str, record_content: str,
+                       record_ttl: int):
         """
         Add a TXT record using the supplied information.
 
         :param str domain_name: The domain to use to associate the record with.
         :param str record_name: The record name (typically beginning with '_acme-challenge.').
         :param str record_content: The record content (typically the challenge validation).
+        :param int record_ttl: The record TTL.
         :raises certbot.errors.PluginError: if an error occurs communicating with the DigitalOcean
                                             API
         """
@@ -89,7 +93,8 @@ class _DigitalOceanClient:
             result = domain.create_new_domain_record(
                 type='TXT',
                 name=self._compute_record_name(domain, record_name),
-                data=record_content)
+                data=record_content,
+                ttl=record_ttl) # ttl kwarg is only effective starting python-digitalocean 1.15.0
 
             record_id = result['domain_record']['id']
 
@@ -99,7 +104,7 @@ class _DigitalOceanClient:
             raise errors.PluginError('Error adding TXT record using the DigitalOcean API: {0}'
                                      .format(e))
 
-    def del_txt_record(self, domain_name, record_name, record_content):
+    def del_txt_record(self, domain_name: str, record_name: str, record_content: str):
         """
         Delete a TXT record using the supplied information.
 

certbot-dns-digitalocean/setup.py

@@ -4,12 +4,12 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    'python-digitalocean>=1.11',
+    'python-digitalocean>=1.11', # 1.15.0 or newer is recommended for TTL support
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot-dns-digitalocean/tests/dns_digitalocean_test.py

@@ -40,7 +40,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic
     def test_perform(self):
         self.auth.perform([self.achall])
 
-        expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY)]
+        expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY, 30)]
         self.assertEqual(expected, self.mock_client.mock_calls)
 
     def test_cleanup(self):
@@ -58,6 +58,7 @@ class DigitalOceanClientTest(unittest.TestCase):
     record_prefix = "_acme-challenge"
     record_name = record_prefix + "." + DOMAIN
     record_content = "bar"
+    record_ttl = 60
 
     def setUp(self):
         from certbot_dns_digitalocean._internal.dns_digitalocean import _DigitalOceanClient
@@ -78,25 +79,27 @@ class DigitalOceanClientTest(unittest.TestCase):
 
         self.manager.get_all_domains.return_value = [wrong_domain_mock, domain_mock]
 
-        self.digitalocean_client.add_txt_record(DOMAIN, self.record_name, self.record_content)
+        self.digitalocean_client.add_txt_record(DOMAIN, self.record_name, self.record_content,
+                                                self.record_ttl)
 
         domain_mock.create_new_domain_record.assert_called_with(type='TXT',
                                                                 name=self.record_prefix,
-                                                                data=self.record_content)
+                                                                data=self.record_content,
+                                                                ttl=self.record_ttl)
 
     def test_add_txt_record_fail_to_find_domain(self):
         self.manager.get_all_domains.return_value = []
 
         self.assertRaises(errors.PluginError,
                           self.digitalocean_client.add_txt_record,
-                          DOMAIN, self.record_name, self.record_content)
+                          DOMAIN, self.record_name, self.record_content, self.record_ttl)
 
     def test_add_txt_record_error_finding_domain(self):
         self.manager.get_all_domains.side_effect = API_ERROR
 
         self.assertRaises(errors.PluginError,
                           self.digitalocean_client.add_txt_record,
-                          DOMAIN, self.record_name, self.record_content)
+                          DOMAIN, self.record_name, self.record_content, self.record_ttl)
 
     def test_add_txt_record_error_creating_record(self):
         domain_mock = mock.MagicMock()
@@ -107,7 +110,7 @@ class DigitalOceanClientTest(unittest.TestCase):
 
         self.assertRaises(errors.PluginError,
                           self.digitalocean_client.add_txt_record,
-                          DOMAIN, self.record_name, self.record_content)
+                          DOMAIN, self.record_name, self.record_content, self.record_ttl)
 
     def test_del_txt_record(self):
         first_record_mock = mock.MagicMock()

certbot-dns-dnsimple/setup.py

@@ -4,7 +4,7 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-dnsmadeeasy/setup.py

@@ -4,7 +4,7 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-gehirn/setup.py

@@ -4,7 +4,7 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-google/setup.py

@@ -4,7 +4,7 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-linode/setup.py

@@ -4,7 +4,7 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-luadns/setup.py

@@ -4,7 +4,7 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-nsone/setup.py

@@ -4,7 +4,7 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-ovh/setup.py

@@ -4,7 +4,7 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-rfc2136/setup.py

@@ -4,7 +4,7 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py

@@ -2,15 +2,15 @@
 import collections
 import logging
 import time
+from typing import DefaultDict
+from typing import Dict
+from typing import List
 
 import boto3
 from botocore.exceptions import ClientError
 from botocore.exceptions import NoCredentialsError
 import zope.interface
 
-from acme.magic_typing import DefaultDict
-from acme.magic_typing import Dict
-from acme.magic_typing import List
 from certbot import errors
 from certbot import interfaces
 from certbot.plugins import dns_common
@@ -39,7 +39,7 @@ class Authenticator(dns_common.DNSAuthenticator):
     def __init__(self, *args, **kwargs):
         super(Authenticator, self).__init__(*args, **kwargs)
         self.r53 = boto3.client("route53")
-        self._resource_records = collections.defaultdict(list) # type: DefaultDict[str, List[Dict[str, str]]]
+        self._resource_records: DefaultDict[str, List[Dict[str, str]]] = collections.defaultdict(list)
 
     def more_info(self):  # pylint: disable=missing-function-docstring
         return "Solve a DNS01 challenge using AWS Route53"

certbot-dns-route53/setup.py

@@ -4,7 +4,7 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-sakuracloud/setup.py

@@ -4,7 +4,7 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-nginx/certbot_nginx/_internal/configurator.py

@@ -7,6 +7,12 @@ import socket
 import subprocess
 import tempfile
 import time
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Set
+from typing import Text
+from typing import Tuple
 
 import OpenSSL
 import pkg_resources
@@ -14,12 +20,6 @@ import zope.interface
 
 from acme import challenges
 from acme import crypto_util as acme_crypto_util
-from acme.magic_typing import Dict
-from acme.magic_typing import List
-from acme.magic_typing import Optional
-from acme.magic_typing import Set
-from acme.magic_typing import Text
-from acme.magic_typing import Tuple
 from certbot import crypto_util
 from certbot import errors
 from certbot import interfaces
@@ -112,8 +112,8 @@ class NginxConfigurator(common.Installer):
 
         # List of vhosts configured per wildcard domain on this run.
         # used by deploy_cert() and enhance()
-        self._wildcard_vhosts = {} # type: Dict[str, List[obj.VirtualHost]]
-        self._wildcard_redirect_vhosts = {} # type: Dict[str, List[obj.VirtualHost]]
+        self._wildcard_vhosts: Dict[str, List[obj.VirtualHost]] = {}
+        self._wildcard_redirect_vhosts: Dict[str, List[obj.VirtualHost]] = {}
 
         # Add number of outstanding challenges
         self._chall_out = 0
@@ -641,7 +641,7 @@ class NginxConfigurator(common.Installer):
         :rtype: set
 
         """
-        all_names = set()  # type: Set[str]
+        all_names: Set[str] = set()
 
         for vhost in self.parser.get_vhosts():
             try:
@@ -1222,7 +1222,7 @@ def nginx_restart(nginx_ctl, nginx_conf, sleep_duration):
 
     """
     try:
-        reload_output = u"" # type: Text
+        reload_output: Text = u""
         with tempfile.TemporaryFile() as out:
             proc = subprocess.Popen([nginx_ctl, "-c", nginx_conf, "-s", "reload"],
                                     env=util.env_no_snap_for_external_calls(),

certbot-nginx/certbot_nginx/_internal/constants.py

@@ -1,8 +1,7 @@
 """nginx plugin constants."""
 import platform
-
-from acme.magic_typing import Any
-from acme.magic_typing import Dict
+from typing import Any
+from typing import Dict
 
 FREEBSD_DARWIN_SERVER_ROOT = "/usr/local/etc/nginx"
 LINUX_SERVER_ROOT = "/etc/nginx"
@@ -15,11 +14,11 @@ elif platform.system() in ('NetBSD',):
 else:
     server_root_tmp = LINUX_SERVER_ROOT
 
-CLI_DEFAULTS = dict(
+CLI_DEFAULTS: Dict[str, Any] = dict(
     server_root=server_root_tmp,
     ctl="nginx",
     sleep_seconds=1
-) # type: Dict[str, Any]
+)
 """CLI defaults."""
 
 

certbot-nginx/certbot_nginx/_internal/http_01.py

@@ -2,10 +2,10 @@
 
 import io
 import logging
+from typing import List
+from typing import Optional
 
 from acme import challenges
-from acme.magic_typing import List
-from acme.magic_typing import Optional
 from certbot import achallenges
 from certbot import errors
 from certbot.compat import os
@@ -113,7 +113,7 @@ class NginxHttp01(common.ChallengePerformer):
         :returns: list of :class:`certbot_nginx._internal.obj.Addr` to apply
         :rtype: list
         """
-        addresses = [] # type: List[obj.Addr]
+        addresses: List[obj.Addr] = []
         default_addr = "%s" % self.configurator.config.http01_port
         ipv6_addr = "[::]:{0}".format(
             self.configurator.config.http01_port)

certbot-nginx/certbot_nginx/_internal/nginxparser.py

@@ -2,6 +2,8 @@
 # Forked from https://github.com/fatiherikli/nginxparser (MIT Licensed)
 import copy
 import logging
+from typing import Any
+from typing import IO
 
 from pyparsing import Combine
 from pyparsing import Forward
@@ -15,10 +17,10 @@ from pyparsing import restOfLine
 from pyparsing import stringEnd
 from pyparsing import White
 from pyparsing import ZeroOrMore
-from acme.magic_typing import IO, Any # pylint: disable=unused-import
 
 logger = logging.getLogger(__name__)
 
+
 class RawNginxParser:
     # pylint: disable=pointless-statement
     """A class that parses nginx configuration with pyparsing."""
@@ -104,57 +106,9 @@ class RawNginxDumper:
         return ''.join(self)
 
 
-# Shortcut functions to respect Python's serialization interface
-# (like pyyaml, picker or json)
-
-def loads(source):
-    """Parses from a string.
-
-    :param str source: The string to parse
-    :returns: The parsed tree
-    :rtype: list
-
-    """
-    return UnspacedList(RawNginxParser(source).as_list())
-
-
-def load(_file):
-    """Parses from a file.
-
-    :param file _file: The file to parse
-    :returns: The parsed tree
-    :rtype: list
-
-    """
-    return loads(_file.read())
-
-
-def dumps(blocks):
-    # type: (UnspacedList) -> str
-    """Dump to a Unicode string.
-
-    :param UnspacedList block: The parsed tree
-    :rtype: str
-
-    """
-    return str(RawNginxDumper(blocks.spaced))
-
-
-def dump(blocks, _file):
-    # type: (UnspacedList, IO[Any]) -> None
-    """Dump to a file.
-
-    :param UnspacedList block: The parsed tree
-    :param IO[Any] _file: The file stream to dump to. It must be opened with
-                          Unicode encoding.
-    :rtype: None
-
-    """
-    _file.write(dumps(blocks))
-
-
 spacey = lambda x: (isinstance(x, str) and x.isspace()) or x == ''
 
+
 class UnspacedList(list):
     """Wrap a list [of lists], making any whitespace entries magically invisible"""
 
@@ -273,3 +227,50 @@ class UnspacedList(list):
                 idx -= 1
             pos += 1
         return idx0 + spaces
+
+
+# Shortcut functions to respect Python's serialization interface
+# (like pyyaml, picker or json)
+
+def loads(source):
+    """Parses from a string.
+
+    :param str source: The string to parse
+    :returns: The parsed tree
+    :rtype: list
+
+    """
+    return UnspacedList(RawNginxParser(source).as_list())
+
+
+def load(_file):
+    """Parses from a file.
+
+    :param file _file: The file to parse
+    :returns: The parsed tree
+    :rtype: list
+
+    """
+    return loads(_file.read())
+
+
+def dumps(blocks: UnspacedList) -> str:
+    """Dump to a Unicode string.
+
+    :param UnspacedList block: The parsed tree
+    :rtype: six.text_type
+
+    """
+    return str(RawNginxDumper(blocks.spaced))
+
+
+def dump(blocks: UnspacedList, _file: IO[Any]) -> None:
+    """Dump to a file.
+
+    :param UnspacedList block: The parsed tree
+    :param IO[Any] _file: The file stream to dump to. It must be opened with
+                          Unicode encoding.
+    :rtype: None
+
+    """
+    _file.write(dumps(blocks))

certbot-nginx/certbot_nginx/_internal/obj.py

@@ -1,7 +1,6 @@
 """Module contains classes used by the Nginx Configurator."""
 import re
 
-
 from certbot.plugins import common
 
 ADD_HEADER_DIRECTIVE = 'add_header'

certbot-nginx/certbot_nginx/_internal/parser.py

@@ -5,19 +5,20 @@ import glob
 import io
 import logging
 import re
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Set
+from typing import Tuple
+from typing import Union
 
 import pyparsing
 
-from acme.magic_typing import Dict
-from acme.magic_typing import List
-from acme.magic_typing import Optional
-from acme.magic_typing import Set
-from acme.magic_typing import Tuple
-from acme.magic_typing import Union
 from certbot import errors
 from certbot.compat import os
 from certbot_nginx._internal import nginxparser
 from certbot_nginx._internal import obj
+from certbot_nginx._internal.nginxparser import UnspacedList
 
 logger = logging.getLogger(__name__)
 
@@ -32,7 +33,7 @@ class NginxParser:
     """
 
     def __init__(self, root):
-        self.parsed = {} # type: Dict[str, Union[List, nginxparser.UnspacedList]]
+        self.parsed: Dict[str, Union[List, nginxparser.UnspacedList]] = {}
         self.root = os.path.abspath(root)
         self.config_root = self._find_config_root()
 
@@ -94,7 +95,7 @@ class NginxParser:
         """
         servers = self._get_raw_servers()
 
-        addr_to_ssl = {} # type: Dict[Tuple[str, str], bool]
+        addr_to_ssl: Dict[Tuple[str, str], bool] = {}
         for filename in servers:
             for server, _ in servers[filename]:
                 # Parse the server block to save addr info
@@ -106,12 +107,11 @@ class NginxParser:
                     addr_to_ssl[addr_tuple] = addr.ssl or addr_to_ssl[addr_tuple]
         return addr_to_ssl
 
-    def _get_raw_servers(self):
+    def _get_raw_servers(self) -> Dict:
         # pylint: disable=cell-var-from-loop
-        # type: () -> Dict
         """Get a map of unparsed all server blocks
         """
-        servers = {} # type: Dict[str, Union[List, nginxparser.UnspacedList]]
+        servers: Dict[str, Union[List, nginxparser.UnspacedList]] = {}
         for filename in self.parsed:
             tree = self.parsed[filename]
             servers[filename] = []
@@ -244,6 +244,8 @@ class NginxParser:
             tree = self.parsed[filename]
             if ext:
                 filename = filename + os.path.extsep + ext
+            if not isinstance(tree, UnspacedList):
+                raise ValueError(f"Error tree {tree} is not an UnspacedList")
             try:
                 if lazy and not tree.is_dirty():
                     continue
@@ -741,9 +743,9 @@ def _parse_server_raw(server):
     :rtype: dict
 
     """
-    addrs = set() # type: Set[obj.Addr]
-    ssl = False # type: bool
-    names = set() # type: Set[str]
+    addrs: Set[obj.Addr] = set()
+    ssl: bool = False
+    names: Set[str] = set()
 
     apply_ssl_to_all_addrs = False
 

certbot-nginx/certbot_nginx/_internal/parser_obj.py

@@ -1,11 +1,12 @@
+# type: ignore
+# This module is not used for now, so we just skip type check for the sake of simplicity.
 """ This file contains parsing routines and object classes to help derive meaning from
 raw lists of tokens from pyparsing. """
 
 import abc
 import logging
+from typing import List
 
-
-from acme.magic_typing import List
 from certbot import errors
 
 logger = logging.getLogger(__name__)
@@ -23,7 +24,7 @@ class Parsable:
     __metaclass__ = abc.ABCMeta
 
     def __init__(self, parent=None):
-        self._data = [] # type: List[object]
+        self._data: List[object] = []
         self._tabs = None
         self.parent = parent
 
@@ -182,7 +183,7 @@ class Statements(Parsable):
 
 def _space_list(list_):
     """ Inserts whitespace between adjacent non-whitespace tokens. """
-    spaced_statement = [] # type: List[str]
+    spaced_statement: List[str] = []
     for i in reversed(range(len(list_))):
         spaced_statement.insert(0, list_[i])
         if i > 0 and not list_[i].isspace() and not list_[i-1].isspace():
@@ -205,7 +206,7 @@ class Sentence(Parsable):
         :returns: whether this lists is parseable by `Sentence`.
         """
         return isinstance(lists, list) and len(lists) > 0 and \
-            all(isinstance(elem, str) for elem in lists)
+               all(isinstance(elem, str) for elem in lists)
 
     def parse(self, raw_list, add_spaces=False):
         """ Parses a list of string types into this object.
@@ -213,7 +214,7 @@ class Sentence(Parsable):
         if add_spaces:
             raw_list = _space_list(raw_list)
         if not isinstance(raw_list, list) or \
-                any(not isinstance(elem, str) for elem in raw_list):
+            any(not isinstance(elem, str) for elem in raw_list):
             raise errors.MisconfigurationError("Sentence parsing expects a list of string types.")
         self._data = raw_list
 
@@ -271,8 +272,8 @@ class Block(Parsable):
     """
     def __init__(self, parent=None):
         super(Block, self).__init__(parent)
-        self.names = None # type: Sentence
-        self.contents = None # type: Block
+        self.names: Sentence = None
+        self.contents: Block = None
 
     @staticmethod
     def should_parse(lists):
@@ -284,7 +285,7 @@ class Block(Parsable):
 
         :returns: whether this lists is parseable by `Block`. """
         return isinstance(lists, list) and len(lists) == 2 and \
-            Sentence.should_parse(lists[0]) and isinstance(lists[1], list)
+               Sentence.should_parse(lists[0]) and isinstance(lists[1], list)
 
     def set_tabs(self, tabs="    "):
         """ Sets tabs by setting equivalent tabbing on names, then adding tabbing

certbot-nginx/setup.py

@@ -1,7 +1,7 @@
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.13.0'
+version = '1.14.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-nginx/tests/parser_test.py

@@ -3,6 +3,7 @@ import glob
 import re
 import shutil
 import unittest
+from typing import List
 
 from certbot import errors
 from certbot.compat import os
@@ -112,7 +113,7 @@ class NginxParserTest(util.NginxTest):
                    ([[[0], [3], [4]], [[5], [3], [0]]], [])]
 
         for mylist, result in mylists:
-            paths = [] # type: List[List[int]]
+            paths: List[List[int]] = []
             parser._do_for_subarray(mylist,
                                     lambda x: isinstance(x, list) and
                                     len(x) >= 1 and

certbot/CHANGELOG.md

@@ -2,6 +2,26 @@
 
 Certbot adheres to [Semantic Versioning](https://semver.org/).
 
+## 1.14.0 - master
+
+### Added
+
+*
+
+### Changed
+
+* certbot-auto no longer checks for updates on any operating system.
+* The module `acme.magic_typing` is deprecated and will be removed in a future release.
+  Please use the built-in module `typing` instead.
+* The DigitalOcean plugin now creates TXT records for the DNS-01 challenge with a lower 30s TTL.
+
+### Fixed
+
+* Don't output an empty line for a hidden certificate when `certbot certificates` is being used
+  in combination with `--cert-name` or `-d`.
+
+More details about these changes can be found on our GitHub repo.
+
 ## 1.13.0 - 2021-03-02
 
 ### Added

certbot/certbot/__init__.py

@@ -1,3 +1,3 @@
 """Certbot client."""
 # version number like 1.2.3a0, must have at least 2 parts, like 1.2
-__version__ = '1.13.0'
+__version__ = '1.14.0.dev0'

certbot/certbot/_internal/account.py

@@ -18,8 +18,8 @@ from certbot import errors
 from certbot import interfaces
 from certbot import util
 from certbot._internal import constants
-from certbot.compat import os
 from certbot.compat import filesystem
+from certbot.compat import os
 
 logger = logging.getLogger(__name__)
 
@@ -229,8 +229,7 @@ class AccountFileStorage(interfaces.AccountStorage):
     def load(self, account_id):
         return self._load_for_server_path(account_id, self.config.server_path)
 
-    def save(self, account, client):
-        # type: (Account, ClientBase) -> None
+    def save(self, account: Account, client: ClientBase) -> None:
         """Create a new account.
 
         :param Account account: account to create
@@ -245,8 +244,7 @@ class AccountFileStorage(interfaces.AccountStorage):
         except IOError as error:
             raise errors.AccountStorageError(error)
 
-    def update_regr(self, account, client):
-        # type: (Account, ClientBase) -> None
+    def update_regr(self, account: Account, client: ClientBase) -> None:
         """Update the registration resource.
 
         :param Account account: account to update
@@ -259,8 +257,7 @@ class AccountFileStorage(interfaces.AccountStorage):
         except IOError as error:
             raise errors.AccountStorageError(error)
 
-    def update_meta(self, account):
-        # type: (Account) -> None
+    def update_meta(self, account: Account) -> None:
         """Update the meta resource.
 
         :param Account account: account to update
@@ -338,19 +335,16 @@ class AccountFileStorage(interfaces.AccountStorage):
 
         return dir_path
 
-    def _prepare(self, account):
-        # type: (Account) -> str
+    def _prepare(self, account: Account) -> str:
         account_dir_path = self._account_dir_path(account.id)
         util.make_or_verify_dir(account_dir_path, 0o700, self.config.strict_permissions)
         return account_dir_path
 
-    def _create(self, account, dir_path):
-        # type: (Account, str) -> None
+    def _create(self, account: Account, dir_path: str) -> None:
         with util.safe_open(self._key_path(dir_path), "w", chmod=0o400) as key_file:
             key_file.write(account.key.json_dumps())
 
-    def _update_regr(self, account, acme, dir_path):
-        # type: (Account, ClientBase, str) -> None
+    def _update_regr(self, account: Account, acme: ClientBase, dir_path: str) -> None:
         with open(self._regr_path(dir_path), "w") as regr_file:
             regr = account.regr
             # If we have a value for new-authz, save it for forwards

certbot/certbot/_internal/auth_handler.py

@@ -2,15 +2,15 @@
 import datetime
 import logging
 import time
+from typing import Dict
+from typing import List
+from typing import Tuple
 
 import zope.component
 
 from acme import challenges
 from acme import errors as acme_errors
 from acme import messages
-from acme.magic_typing import Dict
-from acme.magic_typing import List
-from acme.magic_typing import Tuple
 from certbot import achallenges
 from certbot import errors
 from certbot import interfaces
@@ -98,8 +98,7 @@ class AuthHandler:
 
             return authzrs_validated
 
-    def deactivate_valid_authorizations(self, orderr):
-        # type: (messages.OrderResource) -> Tuple[List, List]
+    def deactivate_valid_authorizations(self, orderr: messages.OrderResource) -> Tuple[List, List]:
         """
         Deactivate all `valid` authorizations in the order, so that they cannot be re-used
         in subsequent orders.
@@ -191,7 +190,7 @@ class AuthHandler:
         """
         pending_authzrs = [authzr for authzr in authzrs
                            if authzr.body.status != messages.STATUS_VALID]
-        achalls = []  # type: List[achallenges.AnnotatedChallenge]
+        achalls: List[achallenges.AnnotatedChallenge] = []
         if pending_authzrs:
             logger.info("Performing the following challenges:")
         for authzr in pending_authzrs:
@@ -428,7 +427,7 @@ _ERROR_HELP = {
 
 def _report_failed_authzrs(failed_authzrs, account_key):
     """Notifies the user about failed authorizations."""
-    problems = {}  # type: Dict[str, List[achallenges.AnnotatedChallenge]]
+    problems: Dict[str, List[achallenges.AnnotatedChallenge]] = {}
     failed_achalls = [challb_to_achall(challb, account_key, authzr.body.identifier.value)
                       for authzr in failed_authzrs for challb in authzr.body.challenges
                       if challb.error]

certbot/certbot/_internal/cert_manager.py

@@ -3,11 +3,11 @@ import datetime
 import logging
 import re
 import traceback
+from typing import List
 
 import pytz
 import zope.component
 
-from acme.magic_typing import List
 from certbot import crypto_util
 from certbot import errors
 from certbot import interfaces
@@ -241,7 +241,7 @@ def match_and_check_overlaps(cli_config, acceptable_matches, match_func, rv_func
     def find_matches(candidate_lineage, return_value, acceptable_matches):
         """Returns a list of matches using _search_lineages."""
         acceptable_matches = [func(candidate_lineage) for func in acceptable_matches]
-        acceptable_matches_rv = []  # type: List[str]
+        acceptable_matches_rv: List[str] = []
         for item in acceptable_matches:
             if isinstance(item, list):
                 acceptable_matches_rv += item
@@ -266,9 +266,9 @@ def human_readable_cert_info(config, cert, skip_filter_checks=False):
     checker = ocsp.RevocationChecker()
 
     if config.certname and cert.lineagename != config.certname and not skip_filter_checks:
-        return ""
+        return None
     if config.domains and not set(config.domains).issubset(cert.names()):
-        return ""
+        return None
     now = pytz.UTC.fromutc(datetime.datetime.utcnow())
 
     reasons = []
@@ -358,13 +358,15 @@ def _report_human_readable(config, parsed_certs):
     """Format a results report for a parsed cert"""
     certinfo = []
     for cert in parsed_certs:
-        certinfo.append(human_readable_cert_info(config, cert))
+        cert_info = human_readable_cert_info(config, cert)
+        if cert_info is not None:
+            certinfo.append(cert_info)
     return "\n".join(certinfo)
 
 
 def _describe_certs(config, parsed_certs, parse_failures):
     """Print information about the certs we know about"""
-    out = []  # type: List[str]
+    out: List[str] = []
 
     notify = out.append
 

certbot/certbot/_internal/cli/__init__.py

@@ -1,73 +1,54 @@
 """Certbot command line argument & config processing."""
 # pylint: disable=too-many-lines
-from __future__ import print_function
+import argparse
 import logging
 import logging.handlers
-import argparse
 import sys
-import certbot._internal.plugins.selection as plugin_selection
-from certbot._internal.plugins import disco as plugins_disco
+from typing import Optional
 
-from acme.magic_typing import Optional
-
-# pylint: disable=ungrouped-imports
 import certbot
 from certbot._internal import constants
-
-import certbot.plugins.enhancements as enhancements
-
-
-from certbot._internal.cli.cli_constants import (
-    LEAUTO,
-    old_path_fragment,
-    new_path_prefix,
-    cli_command,
-    SHORT_USAGE,
-    COMMAND_OVERVIEW,
-    HELP_AND_VERSION_USAGE,
-    ARGPARSE_PARAMS_TO_REMOVE,
-    EXIT_ACTIONS,
-    ZERO_ARG_ACTIONS,
-    VAR_MODIFIERS,
-    DEPRECATED_OPTIONS
-)
-
-from certbot._internal.cli.cli_utils import (
-    _Default,
-    read_file,
-    flag_default,
-    config_help,
-    HelpfulArgumentGroup,
-    CustomHelpFormatter,
-    _DomainsAction,
-    add_domains,
-    CaseInsensitiveList,
-    _user_agent_comment_type,
-    _EncodeReasonAction,
-    parse_preferred_challenges,
-    _PrefChallAction,
-    _DeployHookAction,
-    _RenewHookAction,
-    nonnegative_int
-)
-
-# These imports depend on cli_constants and cli_utils.
-from certbot._internal.cli.verb_help import VERB_HELP, VERB_HELP_MAP
+from certbot._internal.cli.cli_constants import ARGPARSE_PARAMS_TO_REMOVE
+from certbot._internal.cli.cli_constants import cli_command
+from certbot._internal.cli.cli_constants import COMMAND_OVERVIEW
+from certbot._internal.cli.cli_constants import DEPRECATED_OPTIONS
+from certbot._internal.cli.cli_constants import EXIT_ACTIONS
+from certbot._internal.cli.cli_constants import HELP_AND_VERSION_USAGE
+from certbot._internal.cli.cli_constants import SHORT_USAGE
+from certbot._internal.cli.cli_constants import VAR_MODIFIERS
+from certbot._internal.cli.cli_constants import ZERO_ARG_ACTIONS
+from certbot._internal.cli.cli_utils import _Default
+from certbot._internal.cli.cli_utils import _DeployHookAction
+from certbot._internal.cli.cli_utils import _DomainsAction
+from certbot._internal.cli.cli_utils import _EncodeReasonAction
+from certbot._internal.cli.cli_utils import _PrefChallAction
+from certbot._internal.cli.cli_utils import _RenewHookAction
+from certbot._internal.cli.cli_utils import _user_agent_comment_type
+from certbot._internal.cli.cli_utils import add_domains
+from certbot._internal.cli.cli_utils import CaseInsensitiveList
+from certbot._internal.cli.cli_utils import config_help
+from certbot._internal.cli.cli_utils import CustomHelpFormatter
+from certbot._internal.cli.cli_utils import flag_default
+from certbot._internal.cli.cli_utils import HelpfulArgumentGroup
+from certbot._internal.cli.cli_utils import nonnegative_int
+from certbot._internal.cli.cli_utils import parse_preferred_challenges
+from certbot._internal.cli.cli_utils import read_file
 from certbot._internal.cli.group_adder import _add_all_groups
-from certbot._internal.cli.subparsers import _create_subparsers
+from certbot._internal.cli.helpful import HelpfulArgumentParser
 from certbot._internal.cli.paths_parser import _paths_parser
 from certbot._internal.cli.plugins_parsing import _plugins_parsing
-
-# These imports depend on some or all of the submodules for cli.
-from certbot._internal.cli.helpful import HelpfulArgumentParser
-# pylint: enable=ungrouped-imports
-
+from certbot._internal.cli.subparsers import _create_subparsers
+from certbot._internal.cli.verb_help import VERB_HELP
+from certbot._internal.cli.verb_help import VERB_HELP_MAP
+from certbot._internal.plugins import disco as plugins_disco
+import certbot._internal.plugins.selection as plugin_selection
+import certbot.plugins.enhancements as enhancements
 
 logger = logging.getLogger(__name__)
 
 
 # Global, to save us from a lot of argument passing within the scope of this module
-helpful_parser = None  # type: Optional[HelpfulArgumentParser]
+helpful_parser: Optional[HelpfulArgumentParser] = None
 
 
 def prepare_and_parse_args(plugins, args, detect_defaults=False):

certbot/certbot/_internal/cli/cli_constants.py

@@ -1,29 +1,5 @@
 """Certbot command line constants"""
-import sys
-
-from certbot.compat import os
-
-# For help strings, figure out how the user ran us.
-# When invoked from letsencrypt-auto, sys.argv[0] is something like:
-# "/home/user/.local/share/certbot/bin/certbot"
-# Note that this won't work if the user set VENV_PATH or XDG_DATA_HOME before
-# running letsencrypt-auto (and sudo stops us from seeing if they did), so it
-# should only be used for purposes where inability to detect letsencrypt-auto
-# fails safely
-
-LEAUTO = "letsencrypt-auto"
-if "CERTBOT_AUTO" in os.environ:
-    # if we're here, this is probably going to be certbot-auto, unless the
-    # user saved the script under a different name
-    LEAUTO = os.path.basename(os.environ["CERTBOT_AUTO"])
-
-old_path_fragment = os.path.join(".local", "share", "letsencrypt")
-new_path_prefix = os.path.abspath(os.path.join(os.sep, "opt",
-                                               "eff.org", "certbot", "venv"))
-if old_path_fragment in sys.argv[0] or sys.argv[0].startswith(new_path_prefix):
-    cli_command = LEAUTO
-else:
-    cli_command = "certbot"
+cli_command = "certbot"
 
 
 # Argparse's help formatting has a lot of unhelpful peculiarities, so we want

certbot/certbot/_internal/cli/cli_utils.py

@@ -5,11 +5,11 @@ import copy
 import zope.interface.interface  # pylint: disable=unused-import
 
 from acme import challenges
+from certbot import errors
 from certbot import interfaces
 from certbot import util
-from certbot import errors
-from certbot.compat import os
 from certbot._internal import constants
+from certbot.compat import os
 
 
 class _Default:
@@ -62,7 +62,7 @@ def config_help(name, hidden=False):
     """Extract the help message for an `.IConfig` attribute."""
     if hidden:
         return argparse.SUPPRESS
-    field = interfaces.IConfig.__getitem__(name)  # type: zope.interface.interface.Attribute
+    field: zope.interface.interface.Attribute = interfaces.IConfig.__getitem__(name)
     return field.__doc__
 
 

certbot/certbot/_internal/cli/group_adder.py

@@ -1,6 +1,6 @@
 """This module contains a function to add the groups of arguments for the help
 display"""
-from certbot._internal.cli import VERB_HELP
+from certbot._internal.cli.verb_help import VERB_HELP
 
 
 def _add_all_groups(helpful):

certbot/certbot/_internal/cli/helpful.py

@@ -1,45 +1,40 @@
 """Certbot command line argument parser"""
-from __future__ import print_function
+
 import argparse
 import copy
 import functools
 import glob
 import sys
+from typing import Any
+from typing import Dict
 
 import configargparse
 import zope.component
 import zope.interface
-
 from zope.interface import interfaces as zope_interfaces
 
-from acme.magic_typing import Any, Dict
-
 from certbot import crypto_util
 from certbot import errors
 from certbot import interfaces
 from certbot import util
-from certbot.compat import os
 from certbot._internal import constants
 from certbot._internal import hooks
-
+from certbot._internal.cli.cli_constants import ARGPARSE_PARAMS_TO_REMOVE
+from certbot._internal.cli.cli_constants import COMMAND_OVERVIEW
+from certbot._internal.cli.cli_constants import EXIT_ACTIONS
+from certbot._internal.cli.cli_constants import HELP_AND_VERSION_USAGE
+from certbot._internal.cli.cli_constants import SHORT_USAGE
+from certbot._internal.cli.cli_constants import ZERO_ARG_ACTIONS
+from certbot._internal.cli.cli_utils import _Default
+from certbot._internal.cli.cli_utils import add_domains
+from certbot._internal.cli.cli_utils import CustomHelpFormatter
+from certbot._internal.cli.cli_utils import flag_default
+from certbot._internal.cli.cli_utils import HelpfulArgumentGroup
+from certbot._internal.cli.verb_help import VERB_HELP
+from certbot._internal.cli.verb_help import VERB_HELP_MAP
+from certbot.compat import os
 from certbot.display import util as display_util
 
-from certbot._internal.cli import (
-    SHORT_USAGE,
-    CustomHelpFormatter,
-    flag_default,
-    VERB_HELP,
-    VERB_HELP_MAP,
-    COMMAND_OVERVIEW,
-    HELP_AND_VERSION_USAGE,
-    _Default,
-    add_domains,
-    EXIT_ACTIONS,
-    ZERO_ARG_ACTIONS,
-    ARGPARSE_PARAMS_TO_REMOVE,
-    HelpfulArgumentGroup
-)
-
 
 class HelpfulArgumentParser:
     """Argparse Wrapper.
@@ -105,9 +100,9 @@ class HelpfulArgumentParser:
         self.visible_topics = self.determine_help_topics(self.help_arg)
 
         # elements are added by .add_group()
-        self.groups = {}  # type: Dict[str, argparse._ArgumentGroup]
+        self.groups: Dict[str, argparse._ArgumentGroup] = {}
         # elements are added by .parse_args()
-        self.defaults = {}  # type: Dict[str, Any]
+        self.defaults: Dict[str, Any] = {}
 
         self.parser = configargparse.ArgParser(
             prog="certbot",
@@ -121,6 +116,8 @@ class HelpfulArgumentParser:
         # This is the only way to turn off overly verbose config flag documentation
         self.parser._add_config_file_help = False
 
+        self.verb: str
+
     # Help that are synonyms for --help subcommands
     COMMANDS_TOPICS = ["command", "commands", "subcommand", "subcommands", "verbs"]
 

certbot/certbot/_internal/cli/paths_parser.py

@@ -1,10 +1,8 @@
 """This is a module that adds configuration to the argument parser regarding
 paths for certificates"""
+from certbot._internal.cli.cli_utils import config_help
+from certbot._internal.cli.cli_utils import flag_default
 from certbot.compat import os
-from certbot._internal.cli import (
-    flag_default,
-    config_help
-)
 
 
 def _paths_parser(helpful):

certbot/certbot/_internal/cli/plugins_parsing.py

@@ -1,5 +1,5 @@
 """This is a module that handles parsing of plugins for the argument parser"""
-from certbot._internal.cli import flag_default
+from certbot._internal.cli.cli_utils import flag_default
 
 
 def _plugins_parsing(helpful, plugins):

certbot/certbot/_internal/cli/subparsers.py

@@ -1,14 +1,11 @@
 """This module creates subparsers for the argument parser"""
 from certbot import interfaces
 from certbot._internal import constants
-
-from certbot._internal.cli import (
-    flag_default,
-    read_file,
-    CaseInsensitiveList,
-    _user_agent_comment_type,
-    _EncodeReasonAction
-)
+from certbot._internal.cli.cli_utils import _EncodeReasonAction
+from certbot._internal.cli.cli_utils import _user_agent_comment_type
+from certbot._internal.cli.cli_utils import CaseInsensitiveList
+from certbot._internal.cli.cli_utils import flag_default
+from certbot._internal.cli.cli_utils import read_file
 
 
 def _create_subparsers(helpful):

certbot/certbot/_internal/cli/verb_help.py

@@ -1,9 +1,7 @@
 """This module contain help information for verbs supported by certbot"""
+from certbot._internal.cli.cli_constants import SHORT_USAGE
+from certbot._internal.cli.cli_utils import flag_default
 from certbot.compat import os
-from certbot._internal.cli import (
-    SHORT_USAGE,
-    flag_default
-)
 
 # The attributes here are:
 # short: a string that will be displayed by "certbot -h commands"

certbot/certbot/_internal/client.py

@@ -2,6 +2,7 @@
 import datetime
 import logging
 import platform
+from typing import Optional
 
 from cryptography.hazmat.backends import default_backend
 # See https://github.com/pyca/cryptography/issues/4275
@@ -14,8 +15,6 @@ from acme import client as acme_client
 from acme import crypto_util as acme_crypto_util
 from acme import errors as acme_errors
 from acme import messages
-from acme.magic_typing import List
-from acme.magic_typing import Optional
 import certbot
 from certbot import crypto_util
 from certbot import errors
@@ -328,7 +327,7 @@ class Client:
             with open(old_keypath, "rb") as f:
                 keypath = old_keypath
                 keypem = f.read()
-            key = util.Key(file=keypath, pem=keypem) # type: Optional[util.Key]
+            key: Optional[util.Key] = util.Key(file=keypath, pem=keypem)
             logger.info("Reusing existing private key from %s.", old_keypath)
         else:
             # The key is set to None here but will be created below.
@@ -390,8 +389,8 @@ class Client:
             cert, chain = self.obtain_certificate_from_csr(csr, orderr)
             return cert, chain, key, csr
 
-    def _get_order_and_authorizations(self, csr_pem, best_effort):
-        # type: (str, bool) -> List[messages.OrderResource]
+    def _get_order_and_authorizations(self, csr_pem: str,
+                                      best_effort: bool) -> messages.OrderResource:
         """Request a new order and complete its authorizations.
 
         :param str csr_pem: A CSR in PEM format.

certbot/certbot/_internal/eff.py

@@ -1,23 +1,21 @@
 """Subscribes users to the EFF newsletter."""
 import logging
+from typing import Optional
 
 import requests
 import zope.component
 
-from acme.magic_typing import Optional  # pylint: disable=unused-import
-
 from certbot import interfaces
-from certbot.display import util as display_util
 from certbot._internal import constants
-from certbot._internal.account import Account  # pylint: disable=unused-import
+from certbot._internal.account import Account
 from certbot._internal.account import AccountFileStorage
-from certbot.interfaces import IConfig  # pylint: disable=unused-import
+from certbot.display import util as display_util
+from certbot.interfaces import IConfig
 
 logger = logging.getLogger(__name__)
 
 
-def prepare_subscription(config, acc):
-    # type: (IConfig, Account) -> None
+def prepare_subscription(config: IConfig, acc: Account) -> None:
     """High level function to store potential EFF newsletter subscriptions.
 
     The user may be asked if they want to sign up for the newsletter if
@@ -45,8 +43,7 @@ def prepare_subscription(config, acc):
         storage.update_meta(acc)
 
 
-def handle_subscription(config, acc):
-    # type: (IConfig, Account) -> None
+def handle_subscription(config: IConfig, acc: Account) -> None:
     """High level function to take care of EFF newsletter subscriptions.
 
     Once subscription is handled, it will not be handled again.
@@ -65,8 +62,7 @@ def handle_subscription(config, acc):
         storage.update_meta(acc)
 
 
-def _want_subscription():
-    # type: () -> bool
+def _want_subscription() -> bool:
     """Does the user want to be subscribed to the EFF newsletter?
 
     :returns: True if we should subscribe the user, otherwise, False
@@ -83,8 +79,7 @@ def _want_subscription():
     return display.yesno(prompt, default=False)
 
 
-def subscribe(email):
-    # type: (str) -> None
+def subscribe(email: str) -> None:
     """Subscribe the user to the EFF mailing list.
 
     :param str email: the e-mail address to subscribe
@@ -99,8 +94,7 @@ def subscribe(email):
     _check_response(requests.post(url, data=data))
 
 
-def _check_response(response):
-    # type: (requests.Response) -> None
+def _check_response(response: requests.Response) -> None:
     """Check for errors in the server's response.
 
     If an error occurred, it will be reported to the user.
@@ -120,8 +114,7 @@ def _check_response(response):
         _report_failure('there was a problem with the server response')
 
 
-def _report_failure(reason=None):
-    # type: (Optional[str]) -> None
+def _report_failure(reason: Optional[str] = None) -> None:
     """Notify the user of failing to sign them up for the newsletter.
 
     :param reason: a phrase describing what the problem was

certbot/certbot/_internal/error_handler.py

@@ -3,12 +3,12 @@ import functools
 import logging
 import signal
 import traceback
+from typing import Any
+from typing import Callable
+from typing import Dict
+from typing import List
+from typing import Union
 
-from acme.magic_typing import Any
-from acme.magic_typing import Callable
-from acme.magic_typing import Dict
-from acme.magic_typing import List
-from acme.magic_typing import Union
 from certbot import errors
 from certbot.compat import os
 
@@ -77,9 +77,9 @@ class ErrorHandler:
     def __init__(self, func, *args, **kwargs):
         self.call_on_regular_exit = False
         self.body_executed = False
-        self.funcs = []  # type: List[Callable[[], Any]]
-        self.prev_handlers = {}  # type: Dict[int, Union[int, None, Callable]]
-        self.received_signals = []  # type: List[int]
+        self.funcs: List[Callable[[], Any]] = []
+        self.prev_handlers: Dict[int, Union[int, None, Callable]] = {}
+        self.received_signals: List[int] = []
         if func is not None:
             self.register(func, *args, **kwargs)
 
@@ -108,8 +108,7 @@ class ErrorHandler:
         self._call_signals()
         return retval
 
-    def register(self, func, *args, **kwargs):
-        # type: (Callable, *Any, **Any) -> None
+    def register(self, func: Callable, *args: Any, **kwargs: Any) -> None:
         """Sets func to be run with the given arguments during cleanup.
 
         :param function func: function to be called in case of an error