revert dependency on new certbot

Fixes #7857.

* stop using urllib2 in test farm tests

* use six for urllib instead

* remove fabric lcd usage

* correct lcd removal

* remove fabric cd

* convert some remote calls to v2

* move more cxns to v2

* get run working with prefix

* get sudo commands working

* remove final fabric v1 references including local

* update requirements and README

* add new venv to gitignore

* update version used in travis

* remove deploy_script unused kwargs

* fix killboulder implementation so I can test creating a new boulder server

* hardcode the gopath due to broken env manamagement in fabric2

* Update letstest readme

* move the comment about hardcoding the ggopath

* catch BaseException instead of Exception

* work around fabric #2007

* use connections as context managers to ensure they're closed

* remove reference to virtualenv

.azure-pipelines/advanced-test.yml

@@ -0,0 +1,13 @@
+# Advanced pipeline for running our full test suite on demand.
+trigger:
+  # When changing these triggers, please ensure the documentation under
+  # "Running tests in CI" is still correct.
+  - azure-test-*
+  - test-*
+pr: none
+
+jobs:
+  # Any addition here should be reflected in the advanced and release pipelines.
+  # It is advised to declare all jobs here as templates to improve maintainability.
+  - template: templates/tests-suite.yml
+  - template: templates/installer-tests.yml

.azure-pipelines/advanced.yml

@@ -1,12 +1,7 @@
-# Advanced pipeline for isolated checks and release purpose
+# Advanced pipeline for running our full test suite on protected branches.
 trigger:
-  # When changing these triggers, please ensure the documentation under
-  # "Running tests in CI" is still correct.
-  - azure-test-*
-  - test-*
   - '*.x'
-pr:
-  - test-*
+pr: none
 # This pipeline is also nightly run on master
 schedules:
   - cron: "0 4 * * *"
@@ -17,7 +12,7 @@ schedules:
     always: true
 
 jobs:
-  # Any addition here should be reflected in the release pipeline.
+  # Any addition here should be reflected in the advanced-test and release pipelines.
   # It is advised to declare all jobs here as templates to improve maintainability.
   - template: templates/tests-suite.yml
   - template: templates/installer-tests.yml

.azure-pipelines/release.yml

@@ -6,7 +6,7 @@ trigger:
 pr: none
 
 jobs:
-  # Any addition here should be reflected in the advanced pipeline.
+  # Any addition here should be reflected in the advanced and advanced-test pipelines.
   # It is advised to declare all jobs here as templates to improve maintainability.
   - template: templates/tests-suite.yml
   - template: templates/installer-tests.yml

.azure-pipelines/templates/installer-tests.yml

@@ -28,15 +28,20 @@ jobs:
           imageName: windows-2019
         win2016:
           imageName: vs2017-win2016
-        win2012r2:
-          imageName: vs2015-win2012r2
     pool:
       vmImage: $(imageName)
     steps:
-      - powershell: Invoke-WebRequest https://www.python.org/ftp/python/3.8.1/python-3.8.1-amd64-webinstall.exe -OutFile C:\py3-setup.exe
-        displayName: Get Python
-      - script: C:\py3-setup.exe /quiet PrependPath=1 InstallAllUsers=1 Include_launcher=1 InstallLauncherAllUsers=1 Include_test=0 Include_doc=0 Include_dev=1 Include_debug=0 Include_tcltk=0 TargetDir=C:\py3
-        displayName: Install Python
+      - powershell: |
+          $currentVersion = $PSVersionTable.PSVersion
+          if ($currentVersion.Major -ne 5) {
+              throw "Powershell version is not 5.x"
+          }
+        condition: eq(variables['imageName'], 'vs2017-win2016')
+        displayName: Check Powershell 5.x is used in vs2017-win2016
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: 3.8
+          addToPath: true
       - task: DownloadPipelineArtifact@2
         inputs:
           artifact: windows-installer

.azure-pipelines/templates/tests-suite.yml

@@ -25,8 +25,6 @@ jobs:
           PYTEST_ADDOPTS: --numprocesses 4
     pool:
       vmImage: $(IMAGE_NAME)
-    variables:
-    - group: certbot-common
     steps:
     - bash: brew install augeas
       condition: startswith(variables['IMAGE_NAME'], 'macOS')
@@ -39,14 +37,3 @@ jobs:
       displayName: Install dependencies
     - script: python -m tox
       displayName: Run tox
-    # We do not require codecov report upload to succeed. So to avoid to break the pipeline if
-    # something goes wrong, each command is suffixed with a command that hides any non zero exit
-    # codes and echoes an informative message instead.
-    - bash: |
-        curl -s https://codecov.io/bash -o codecov-bash || echo "Failed to download codecov-bash"
-        chmod +x codecov-bash || echo "Failed to apply execute permissions on codecov-bash"
-        ./codecov-bash -F windows  || echo "Codecov did not collect coverage reports"
-      condition: in(variables['TOXENV'], 'py37-cover', 'integration-certbot')
-      env:
-        CODECOV_TOKEN: $(codecov_token)
-      displayName: Publish coverage

.codecov.yml

@@ -1,18 +0,0 @@
-coverage:
-  status:
-    project:
-      default: off
-      linux:
-        flags: linux
-        # Fixed target instead of auto set by #7173, can
-        # be removed when flags in Codecov are added back.
-        target: 97.4
-        threshold: 0.1
-        base: auto
-      windows:
-        flags: windows
-        # Fixed target instead of auto set by #7173, can
-        # be removed when flags in Codecov are added back.
-        target: 97.4
-        threshold: 0.1
-        base: auto

.gitignore

@@ -35,6 +35,7 @@ tags
 tests/letstest/letest-*/
 tests/letstest/*.pem
 tests/letstest/venv/
+tests/letstest/venv3/
 
 .venv
 

.travis.yml

@@ -90,24 +90,24 @@ matrix:
       before_install:
       addons:
       <<: *extended-test-suite
-    - python: "2.7"
+    - python: "3.7"
       env:
         - TOXENV=travis-test-farm-apache2
         - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
       <<: *extended-test-suite
-    - python: "2.7"
+    - python: "3.7"
       env:
         - TOXENV=travis-test-farm-leauto-upgrades
         - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
       git:
         depth: false  # This is needed to have the history to checkout old versions of certbot-auto.
       <<: *extended-test-suite
-    - python: "2.7"
+    - python: "3.7"
       env:
         - TOXENV=travis-test-farm-certonly-standalone
         - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
       <<: *extended-test-suite
-    - python: "2.7"
+    - python: "3.7"
       env:
         - TOXENV=travis-test-farm-sdists
         - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
@@ -247,15 +247,13 @@ addons:
 # version of virtualenv. The option "-I" is set so when CERTBOT_NO_PIN is also
 # set, pip updates dependencies it thinks are already satisfied to avoid some
 # problems with its lack of real dependency resolution.
-install: 'tools/pip_install.py -I codecov tox virtualenv'
+install: 'tools/pip_install.py -I tox virtualenv'
 # Most of the time TRAVIS_RETRY is an empty string, and has no effect on the
 # script command. It is set only to `travis_retry` during farm tests, in
 # order to trigger the Travis retry feature, and compensate the inherent
 # flakiness of these specific tests.
 script: '$TRAVIS_RETRY tox'
 
-after_success: '[ "$TOXENV" == "py27-cover" ] && codecov -F linux'
-
 notifications:
   email: false
   irc:

AUTHORS.md

@@ -103,6 +103,7 @@ Authors
 * [Henry Chen](https://github.com/henrychen95)
 * [Hugo van Kemenade](https://github.com/hugovk)
 * [Ingolf Becker](https://github.com/watercrossing)
+* [Ivan Nejgebauer](https://github.com/inejge)
 * [Jaap Eldering](https://github.com/eldering)
 * [Jacob Hoffman-Andrews](https://github.com/jsha)
 * [Jacob Sachs](https://github.com/jsachs)
@@ -266,5 +267,6 @@ Authors
 * [Yomna](https://github.com/ynasser)
 * [Yoni Jah](https://github.com/yonjah)
 * [YourDaddyIsHere](https://github.com/YourDaddyIsHere)
+* [Yuseong Cho](https://github.com/g6123)
 * [Zach Shepherd](https://github.com/zjs)
 * [陈三](https://github.com/chenxsan)

acme/acme/challenges.py

@@ -1,15 +1,22 @@
 """ACME Identifier Validation Challenges."""
 import abc
+import codecs
 import functools
 import hashlib
 import logging
+import socket
 
 from cryptography.hazmat.primitives import hashes  # type: ignore
 import josepy as jose
 import requests
 import six
+from OpenSSL import SSL  # type: ignore # https://github.com/python/typeshed/issues/2052
+from OpenSSL import crypto
 
+from acme import crypto_util
+from acme import errors
 from acme import fields
+from acme.mixins import ResourceMixin, TypeMixin
 
 logger = logging.getLogger(__name__)
 
@@ -28,7 +35,7 @@ class Challenge(jose.TypedJSONObjectWithFields):
             return UnrecognizedChallenge.from_json(jobj)
 
 
-class ChallengeResponse(jose.TypedJSONObjectWithFields):
+class ChallengeResponse(ResourceMixin, TypeMixin, jose.TypedJSONObjectWithFields):
     # _fields_to_partial_json
     """ACME challenge response."""
     TYPES = {}  # type: dict
@@ -362,29 +369,163 @@ class HTTP01(KeyAuthorizationChallenge):
 
 @ChallengeResponse.register
 class TLSALPN01Response(KeyAuthorizationChallengeResponse):
-    """ACME TLS-ALPN-01 challenge response.
-
-    This class only allows initiating a TLS-ALPN-01 challenge returned from the
-    CA. Full support for responding to TLS-ALPN-01 challenges by generating and
-    serving the expected response certificate is not currently provided.
-    """
+    """ACME tls-alpn-01 challenge response."""
     typ = "tls-alpn-01"
 
+    PORT = 443
+    """Verification port as defined by the protocol.
 
-@Challenge.register
+    You can override it (e.g. for testing) by passing ``port`` to
+    `simple_verify`.
+
+    """
+
+    ID_PE_ACME_IDENTIFIER_V1 = b"1.3.6.1.5.5.7.1.30.1"
+    ACME_TLS_1_PROTOCOL = "acme-tls/1"
+
+    @property
+    def h(self):
+        """Hash value stored in challenge certificate"""
+        return hashlib.sha256(self.key_authorization.encode('utf-8')).digest()
+
+    def gen_cert(self, domain, key=None, bits=2048):
+        """Generate tls-alpn-01 certificate.
+
+        :param unicode domain: Domain verified by the challenge.
+        :param OpenSSL.crypto.PKey key: Optional private key used in
+            certificate generation. If not provided (``None``), then
+            fresh key will be generated.
+        :param int bits: Number of bits for newly generated key.
+
+        :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey`
+
+        """
+        if key is None:
+            key = crypto.PKey()
+            key.generate_key(crypto.TYPE_RSA, bits)
+
+
+        der_value = b"DER:" + codecs.encode(self.h, 'hex')
+        acme_extension = crypto.X509Extension(self.ID_PE_ACME_IDENTIFIER_V1,
+                critical=True, value=der_value)
+
+        return crypto_util.gen_ss_cert(key, [domain], force_san=True,
+                extensions=[acme_extension]), key
+
+    def probe_cert(self, domain, host=None, port=None):
+        """Probe tls-alpn-01 challenge certificate.
+
+        :param unicode domain: domain being validated, required.
+        :param string host: IP address used to probe the certificate.
+        :param int port: Port used to probe the certificate.
+
+        """
+        if host is None:
+            host = socket.gethostbyname(domain)
+            logger.debug('%s resolved to %s', domain, host)
+        if port is None:
+            port = self.PORT
+
+        return crypto_util.probe_sni(host=host, port=port, name=domain,
+                alpn_protocols=[self.ACME_TLS_1_PROTOCOL])
+
+    def verify_cert(self, domain, cert):
+        """Verify tls-alpn-01 challenge certificate.
+
+        :param unicode domain: Domain name being validated.
+        :param OpensSSL.crypto.X509 cert: Challenge certificate.
+
+        :returns: Whether the certificate was successfully verified.
+        :rtype: bool
+
+        """
+        # pylint: disable=protected-access
+        names = crypto_util._pyopenssl_cert_or_req_all_names(cert)
+        logger.debug('Certificate %s. SANs: %s', cert.digest('sha256'), names)
+        if len(names) != 1 or names[0].lower() != domain.lower():
+            return False
+
+        for i in range(cert.get_extension_count()):
+            ext = cert.get_extension(i)
+            # FIXME: assume this is the ACME extension. Currently there is no
+            # way to get full OID of an unknown extension from pyopenssl.
+            if ext.get_short_name() == b'UNDEF':
+                data = ext.get_data()
+                return data == self.h
+
+        return False
+
+    # pylint: disable=too-many-arguments
+    def simple_verify(self, chall, domain, account_public_key,
+                      cert=None, host=None, port=None):
+        """Simple verify.
+
+        Verify ``validation`` using ``account_public_key``, optionally
+        probe tls-alpn-01 certificate and check using `verify_cert`.
+
+        :param .challenges.TLSALPN01 chall: Corresponding challenge.
+        :param str domain: Domain name being validated.
+        :param JWK account_public_key:
+        :param OpenSSL.crypto.X509 cert: Optional certificate. If not
+            provided (``None``) certificate will be retrieved using
+            `probe_cert`.
+        :param string host: IP address used to probe the certificate.
+        :param int port: Port used to probe the certificate.
+
+
+        :returns: ``True`` if and only if client's control of the domain has been verified.
+        :rtype: bool
+
+        """
+        if not self.verify(chall, account_public_key):
+            logger.debug("Verification of key authorization in response failed")
+            return False
+
+        if cert is None:
+            try:
+                cert = self.probe_cert(domain=domain, host=host, port=port)
+            except errors.Error as error:
+                logger.debug(str(error), exc_info=True)
+                return False
+
+        return self.verify_cert(domain, cert)
+
+
+@Challenge.register  # pylint: disable=too-many-ancestors
 class TLSALPN01(KeyAuthorizationChallenge):
-    """ACME tls-alpn-01 challenge.
-
-    This class simply allows parsing the TLS-ALPN-01 challenge returned from
-    the CA. Full TLS-ALPN-01 support is not currently provided.
-
-    """
-    typ = "tls-alpn-01"
+    """ACME tls-alpn-01 challenge."""
     response_cls = TLSALPN01Response
+    typ = response_cls.typ
 
     def validation(self, account_key, **kwargs):
-        """Generate validation for the challenge."""
-        raise NotImplementedError()
+        """Generate validation.
+
+        :param JWK account_key:
+        :param unicode domain: Domain verified by the challenge.
+        :param OpenSSL.crypto.PKey cert_key: Optional private key used
+            in certificate generation. If not provided (``None``), then
+            fresh key will be generated.
+
+        :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey`
+
+        """
+        return self.response(account_key).gen_cert(
+            key=kwargs.get('cert_key'),
+            domain=kwargs.get('domain'))
+
+    @staticmethod
+    def is_supported():
+        """
+        Check if TLS-ALPN-01 challenge is supported on this machine.
+        This implies that a recent version of OpenSSL is installed (>= 1.0.2),
+        or a recent cryptography version shipped with the OpenSSL library is installed.
+
+        :returns: ``True`` if TLS-ALPN-01 is supported on this machine, ``False`` otherwise.
+        :rtype: bool
+
+        """
+        return (hasattr(SSL.Connection, "set_alpn_protos")
+                and hasattr(SSL.Context, "set_alpn_select_callback"))
 
 
 @Challenge.register

acme/acme/client.py

@@ -25,6 +25,7 @@ from acme.magic_typing import Dict
 from acme.magic_typing import List
 from acme.magic_typing import Set
 from acme.magic_typing import Text
+from acme.mixins import VersionedLEACMEMixin
 
 logger = logging.getLogger(__name__)
 
@@ -987,6 +988,8 @@ class ClientNetwork(object):
         :rtype: `josepy.JWS`
 
         """
+        if isinstance(obj, VersionedLEACMEMixin):
+            obj.le_acme_version = acme_version
         jobj = obj.json_dumps(indent=2).encode() if obj else b''
         logger.debug('JWS payload:\n%s', jobj)
         kwargs = {

acme/acme/crypto_util.py

@@ -27,19 +27,41 @@ logger = logging.getLogger(__name__)
 _DEFAULT_SSL_METHOD = SSL.SSLv23_METHOD  # type: ignore
 
 
-class SSLSocket(object):
+class _DefaultCertSelection(object):
+    def __init__(self, certs):
+        self.certs = certs
+
+    def __call__(self, connection):
+        server_name = connection.get_servername()
+        return self.certs.get(server_name, None)
+
+
+class SSLSocket(object):  # pylint: disable=too-few-public-methods
     """SSL wrapper for sockets.
 
     :ivar socket sock: Original wrapped socket.
     :ivar dict certs: Mapping from domain names (`bytes`) to
         `OpenSSL.crypto.X509`.
     :ivar method: See `OpenSSL.SSL.Context` for allowed values.
+    :ivar alpn_selection: Hook to select negotiated ALPN protocol for
+        connection.
+    :ivar cert_selection: Hook to select certificate for connection. If given,
+        `certs` parameter would be ignored, and therefore must be empty.
 
     """
-    def __init__(self, sock, certs, method=_DEFAULT_SSL_METHOD):
+    def __init__(self, sock, certs=None,
+            method=_DEFAULT_SSL_METHOD, alpn_selection=None,
+            cert_selection=None):
         self.sock = sock
-        self.certs = certs
+        self.alpn_selection = alpn_selection
         self.method = method
+        if not cert_selection and not certs:
+            raise ValueError("Neither cert_selection or certs specified.")
+        if cert_selection and certs:
+            raise ValueError("Both cert_selection and certs specified.")
+        if cert_selection is None:
+            cert_selection = _DefaultCertSelection(certs)
+        self.cert_selection = cert_selection
 
     def __getattr__(self, name):
         return getattr(self.sock, name)
@@ -56,18 +78,19 @@ class SSLSocket(object):
         :type connection: :class:`OpenSSL.Connection`
 
         """
-        server_name = connection.get_servername()
-        try:
-            key, cert = self.certs[server_name]
-        except KeyError:
-            logger.debug("Server name (%s) not recognized, dropping SSL",
-                         server_name)
+        pair = self.cert_selection(connection)
+        if pair is None:
+            logger.debug("Certificate selection for server name %s failed, dropping SSL",
+                         connection.get_servername())
             return
+        key, cert = pair
         new_context = SSL.Context(self.method)
         new_context.set_options(SSL.OP_NO_SSLv2)
         new_context.set_options(SSL.OP_NO_SSLv3)
         new_context.use_privatekey(key)
         new_context.use_certificate(cert)
+        if self.alpn_selection is not None:
+            new_context.set_alpn_select_callback(self.alpn_selection)
         connection.set_context(new_context)
 
     class FakeConnection(object):
@@ -92,6 +115,8 @@ class SSLSocket(object):
         context.set_options(SSL.OP_NO_SSLv2)
         context.set_options(SSL.OP_NO_SSLv3)
         context.set_tlsext_servername_callback(self._pick_certificate_cb)
+        if self.alpn_selection is not None:
+            context.set_alpn_select_callback(self.alpn_selection)
 
         ssl_sock = self.FakeConnection(SSL.Connection(context, sock))
         ssl_sock.set_accept_state()
@@ -107,8 +132,9 @@ class SSLSocket(object):
         return ssl_sock, addr
 
 
-def probe_sni(name, host, port=443, timeout=300,
-              method=_DEFAULT_SSL_METHOD, source_address=('', 0)):
+def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-arguments
+              method=_DEFAULT_SSL_METHOD, source_address=('', 0),
+              alpn_protocols=None):
     """Probe SNI server for SSL certificate.
 
     :param bytes name: Byte string to send as the server name in the
@@ -120,6 +146,8 @@ def probe_sni(name, host, port=443, timeout=300,
     :param tuple source_address: Enables multi-path probing (selection
         of source interface). See `socket.creation_connection` for more
         info. Available only in Python 2.7+.
+    :param alpn_protocols: Protocols to request using ALPN.
+    :type alpn_protocols: `list` of `bytes`
 
     :raises acme.errors.Error: In case of any problems.
 
@@ -149,6 +177,8 @@ def probe_sni(name, host, port=443, timeout=300,
         client_ssl = SSL.Connection(context, client)
         client_ssl.set_connect_state()
         client_ssl.set_tlsext_host_name(name)  # pyOpenSSL>=0.13
+        if alpn_protocols is not None:
+            client_ssl.set_alpn_protos(alpn_protocols)
         try:
             client_ssl.do_handshake()
             client_ssl.shutdown()
@@ -239,12 +269,14 @@ def _pyopenssl_cert_or_req_san(cert_or_req):
 
 
 def gen_ss_cert(key, domains, not_before=None,
-                validity=(7 * 24 * 60 * 60), force_san=True):
+                validity=(7 * 24 * 60 * 60), force_san=True, extensions=None):
     """Generate new self-signed certificate.
 
     :type domains: `list` of `unicode`
     :param OpenSSL.crypto.PKey key:
     :param bool force_san:
+    :param extensions: List of additional extensions to include in the cert.
+    :type extensions: `list` of `OpenSSL.crypto.X509Extension`
 
     If more than one domain is provided, all of the domains are put into
     ``subjectAltName`` X.509 extension and first domain is set as the
@@ -257,10 +289,13 @@ def gen_ss_cert(key, domains, not_before=None,
     cert.set_serial_number(int(binascii.hexlify(os.urandom(16)), 16))
     cert.set_version(2)
 
-    extensions = [
+    if extensions is None:
+        extensions = []
+
+    extensions.append(
         crypto.X509Extension(
             b"basicConstraints", True, b"CA:TRUE, pathlen:0"),
-    ]
+    )
 
     cert.get_subject().CN = domains[0]
     # TODO: what to put into cert.get_subject()?

acme/acme/magic_typing.py

@@ -11,6 +11,5 @@ try:
     # mypy doesn't respect modifying sys.modules
     from typing import *  # pylint: disable=wildcard-import, unused-wildcard-import
     from typing import Collection, IO  # type: ignore
-    # pylint: enable=unused-import
 except ImportError:
     sys.modules[__name__] = TypingClass()

acme/acme/messages.py

@@ -9,6 +9,7 @@ from acme import errors
 from acme import fields
 from acme import jws
 from acme import util
+from acme.mixins import ResourceMixin
 
 try:
     from collections.abc import Hashable
@@ -356,13 +357,13 @@ class Registration(ResourceBody):
 
 
 @Directory.register
-class NewRegistration(Registration):
+class NewRegistration(ResourceMixin, Registration):
     """New registration."""
     resource_type = 'new-reg'
     resource = fields.Resource(resource_type)
 
 
-class UpdateRegistration(Registration):
+class UpdateRegistration(ResourceMixin, Registration):
     """Update registration."""
     resource_type = 'reg'
     resource = fields.Resource(resource_type)
@@ -498,13 +499,13 @@ class Authorization(ResourceBody):
 
 
 @Directory.register
-class NewAuthorization(Authorization):
+class NewAuthorization(ResourceMixin, Authorization):
     """New authorization."""
     resource_type = 'new-authz'
     resource = fields.Resource(resource_type)
 
 
-class UpdateAuthorization(Authorization):
+class UpdateAuthorization(ResourceMixin, Authorization):
     """Update authorization."""
     resource_type = 'authz'
     resource = fields.Resource(resource_type)
@@ -522,7 +523,7 @@ class AuthorizationResource(ResourceWithURI):
 
 
 @Directory.register
-class CertificateRequest(jose.JSONObjectWithFields):
+class CertificateRequest(ResourceMixin, jose.JSONObjectWithFields):
     """ACME new-cert request.
 
     :ivar josepy.util.ComparableX509 csr:
@@ -548,7 +549,7 @@ class CertificateResource(ResourceWithURI):
 
 
 @Directory.register
-class Revocation(jose.JSONObjectWithFields):
+class Revocation(ResourceMixin, jose.JSONObjectWithFields):
     """Revocation message.
 
     :ivar .ComparableX509 certificate: `OpenSSL.crypto.X509` wrapped in

acme/acme/mixins.py

@@ -0,0 +1,65 @@
+"""Useful mixins for Challenge and Resource objects"""
+
+
+class VersionedLEACMEMixin(object):
+    """This mixin stores the version of Let's Encrypt's endpoint being used."""
+    @property
+    def le_acme_version(self):
+        """Define the version of ACME protocol to use"""
+        return getattr(self, '_le_acme_version', 1)
+
+    @le_acme_version.setter
+    def le_acme_version(self, version):
+        # We need to use object.__setattr__ to not depend on the specific implementation of
+        # __setattr__  in current class (eg. jose.TypedJSONObjectWithFields raises AttributeError
+        # for any attempt to set an attribute to make objects immutable).
+        object.__setattr__(self, '_le_acme_version', version)
+
+    def __setattr__(self, key, value):
+        if key == 'le_acme_version':
+            # Required for @property to operate properly. See comment above.
+            object.__setattr__(self, key, value)
+        else:
+            super(VersionedLEACMEMixin, self).__setattr__(key, value)  # pragma: no cover
+
+
+class ResourceMixin(VersionedLEACMEMixin):
+    """
+    This mixin generates a RFC8555 compliant JWS payload
+    by removing the `resource` field if needed (eg. ACME v2 protocol).
+    """
+    def to_partial_json(self):
+        """See josepy.JSONDeserializable.to_partial_json()"""
+        return _safe_jobj_compliance(super(ResourceMixin, self),
+                                     'to_partial_json', 'resource')
+
+    def fields_to_partial_json(self):
+        """See josepy.JSONObjectWithFields.fields_to_partial_json()"""
+        return _safe_jobj_compliance(super(ResourceMixin, self),
+                                     'fields_to_partial_json', 'resource')
+
+
+class TypeMixin(VersionedLEACMEMixin):
+    """
+    This mixin allows generation of a RFC8555 compliant JWS payload
+    by removing the `type` field if needed (eg. ACME v2 protocol).
+    """
+    def to_partial_json(self):
+        """See josepy.JSONDeserializable.to_partial_json()"""
+        return _safe_jobj_compliance(super(TypeMixin, self),
+                                     'to_partial_json', 'type')
+
+    def fields_to_partial_json(self):
+        """See josepy.JSONObjectWithFields.fields_to_partial_json()"""
+        return _safe_jobj_compliance(super(TypeMixin, self),
+                                     'fields_to_partial_json', 'type')
+
+
+def _safe_jobj_compliance(instance, jobj_method, uncompliant_field):
+    if hasattr(instance, jobj_method):
+        jobj = getattr(instance, jobj_method)()
+        if instance.le_acme_version == 2:
+            jobj.pop(uncompliant_field, None)
+        return jobj
+
+    raise AttributeError('Method {0}() is not implemented.'.format(jobj_method))  # pragma: no cover

acme/acme/standalone.py

@@ -33,7 +33,14 @@ class TLSServer(socketserver.TCPServer):
 
     def _wrap_sock(self):
         self.socket = crypto_util.SSLSocket(
-            self.socket, certs=self.certs, method=self.method)
+            self.socket, cert_selection=self._cert_selection,
+            alpn_selection=getattr(self, '_alpn_selection', None),
+            method=self.method)
+
+    def _cert_selection(self, connection):  # pragma: no cover
+        """Callback selecting certificate for connection."""
+        server_name = connection.get_servername()
+        return self.certs.get(server_name, None)
 
     def server_bind(self):
         self._wrap_sock()
@@ -120,6 +127,40 @@ class BaseDualNetworkedServers(object):
         self.threads = []
 
 
+class TLSALPN01Server(TLSServer, ACMEServerMixin):
+    """TLSALPN01 Server."""
+
+    ACME_TLS_1_PROTOCOL = b"acme-tls/1"
+
+    def __init__(self, server_address, certs, challenge_certs, ipv6=False):
+        TLSServer.__init__(
+            self, server_address, _BaseRequestHandlerWithLogging, certs=certs,
+            ipv6=ipv6)
+        self.challenge_certs = challenge_certs
+
+    def _cert_selection(self, connection):
+        # TODO: We would like to serve challenge cert only if asked for it via
+        # ALPN. To do this, we need to retrieve the list of protos from client
+        # hello, but this is currently impossible with openssl [0], and ALPN
+        # negotiation is done after cert selection.
+        # Therefore, currently we always return challenge cert, and terminate
+        # handshake in alpn_selection() if ALPN protos are not what we expect.
+        # [0] https://github.com/openssl/openssl/issues/4952
+        server_name = connection.get_servername()
+        logger.debug("Serving challenge cert for server name %s", server_name)
+        return self.challenge_certs.get(server_name, None)
+
+    def _alpn_selection(self, _connection, alpn_protos):
+        """Callback to select alpn protocol."""
+        if len(alpn_protos) == 1 and alpn_protos[0] == self.ACME_TLS_1_PROTOCOL:
+            logger.debug("Agreed on %s ALPN", self.ACME_TLS_1_PROTOCOL)
+            return self.ACME_TLS_1_PROTOCOL
+        logger.debug("Cannot agree on ALPN proto. Got: %s", str(alpn_protos))
+        # Explicitly close the connection now, by returning an empty string.
+        # See https://www.pyopenssl.org/en/stable/api/ssl.html#OpenSSL.SSL.Context.set_alpn_select_callback  # pylint: disable=line-too-long
+        return b""
+
+
 class HTTPServer(BaseHTTPServer.HTTPServer):
     """Generic HTTP Server."""
 
@@ -135,10 +176,10 @@ class HTTPServer(BaseHTTPServer.HTTPServer):
 class HTTP01Server(HTTPServer, ACMEServerMixin):
     """HTTP01 Server."""
 
-    def __init__(self, server_address, resources, ipv6=False):
+    def __init__(self, server_address, resources, ipv6=False, timeout=30):
         HTTPServer.__init__(
             self, server_address, HTTP01RequestHandler.partial_init(
-                simple_http_resources=resources), ipv6=ipv6)
+                simple_http_resources=resources, timeout=timeout), ipv6=ipv6)
 
 
 class HTTP01DualNetworkedServers(BaseDualNetworkedServers):
@@ -163,6 +204,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 
     def __init__(self, *args, **kwargs):
         self.simple_http_resources = kwargs.pop("simple_http_resources", set())
+        self.timeout = kwargs.pop('timeout', 30)
         BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
 
     def log_message(self, format, *args):  # pylint: disable=redefined-builtin
@@ -212,7 +254,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
                          self.path)
 
     @classmethod
-    def partial_init(cls, simple_http_resources):
+    def partial_init(cls, simple_http_resources, timeout):
         """Partially initialize this handler.
 
         This is useful because `socketserver.BaseServer` takes
@@ -221,4 +263,18 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 
         """
         return functools.partial(
-            cls, simple_http_resources=simple_http_resources)
+            cls, simple_http_resources=simple_http_resources,
+            timeout=timeout)
+
+
+class _BaseRequestHandlerWithLogging(socketserver.BaseRequestHandler):
+    """BaseRequestHandler with logging."""
+
+    def log_message(self, format, *args):  # pylint: disable=redefined-builtin
+        """Log arbitrary message."""
+        logger.debug("%s - - %s", self.client_address[0], format % args)
+
+    def handle(self):
+        """Handle request."""
+        self.log_message("Incoming request")
+        socketserver.BaseRequestHandler.handle(self)

acme/setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

acme/tests/challenges_test.py

@@ -2,10 +2,13 @@
 import unittest
 
 import josepy as jose
+import OpenSSL
 import mock
 import requests
 from six.moves.urllib import parse as urllib_parse
 
+from acme import errors
+
 import test_util
 
 CERT = test_util.load_comparable_cert('cert.pem')
@@ -256,30 +259,87 @@ class HTTP01Test(unittest.TestCase):
 class TLSALPN01ResponseTest(unittest.TestCase):
 
     def setUp(self):
-        from acme.challenges import TLSALPN01Response
-        self.msg = TLSALPN01Response(key_authorization=u'foo')
+        from acme.challenges import TLSALPN01
+        self.chall = TLSALPN01(
+            token=jose.b64decode(b'a82d5ff8ef740d12881f6d3c2277ab2e'))
+        self.domain = u'example.com'
+        self.domain2 = u'example2.com'
+
+        self.response = self.chall.response(KEY)
         self.jmsg = {
             'resource': 'challenge',
             'type': 'tls-alpn-01',
-            'keyAuthorization': u'foo',
+            'keyAuthorization': self.response.key_authorization,
         }
 
-        from acme.challenges import TLSALPN01
-        self.chall = TLSALPN01(token=(b'x' * 16))
-        self.response = self.chall.response(KEY)
-
     def test_to_partial_json(self):
         self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'},
-                         self.msg.to_partial_json())
+                         self.response.to_partial_json())
 
     def test_from_json(self):
         from acme.challenges import TLSALPN01Response
-        self.assertEqual(self.msg, TLSALPN01Response.from_json(self.jmsg))
+        self.assertEqual(self.response, TLSALPN01Response.from_json(self.jmsg))
 
     def test_from_json_hashable(self):
         from acme.challenges import TLSALPN01Response
         hash(TLSALPN01Response.from_json(self.jmsg))
 
+    def test_gen_verify_cert(self):
+        key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem')
+        cert, key2 = self.response.gen_cert(self.domain, key1)
+        self.assertEqual(key1, key2)
+        self.assertTrue(self.response.verify_cert(self.domain, cert))
+
+    def test_gen_verify_cert_gen_key(self):
+        cert, key = self.response.gen_cert(self.domain)
+        self.assertTrue(isinstance(key, OpenSSL.crypto.PKey))
+        self.assertTrue(self.response.verify_cert(self.domain, cert))
+
+    def test_verify_bad_cert(self):
+        self.assertFalse(self.response.verify_cert(self.domain,
+            test_util.load_cert('cert.pem')))
+
+    def test_verify_bad_domain(self):
+        key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem')
+        cert, key2 = self.response.gen_cert(self.domain, key1)
+        self.assertEqual(key1, key2)
+        self.assertFalse(self.response.verify_cert(self.domain2, cert))
+
+    def test_simple_verify_bad_key_authorization(self):
+        key2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem'))
+        self.response.simple_verify(self.chall, "local", key2.public_key())
+
+    @mock.patch('acme.challenges.TLSALPN01Response.verify_cert', autospec=True)
+    def test_simple_verify(self, mock_verify_cert):
+        mock_verify_cert.return_value = mock.sentinel.verification
+        self.assertEqual(
+            mock.sentinel.verification, self.response.simple_verify(
+                self.chall, self.domain, KEY.public_key(),
+                cert=mock.sentinel.cert))
+        mock_verify_cert.assert_called_once_with(
+            self.response, self.domain, mock.sentinel.cert)
+
+    @mock.patch('acme.challenges.socket.gethostbyname')
+    @mock.patch('acme.challenges.crypto_util.probe_sni')
+    def test_probe_cert(self, mock_probe_sni, mock_gethostbyname):
+        mock_gethostbyname.return_value = '127.0.0.1'
+        self.response.probe_cert('foo.com')
+        mock_gethostbyname.assert_called_once_with('foo.com')
+        mock_probe_sni.assert_called_once_with(
+            host='127.0.0.1', port=self.response.PORT, name='foo.com',
+            alpn_protocols=['acme-tls/1'])
+
+        self.response.probe_cert('foo.com', host='8.8.8.8')
+        mock_probe_sni.assert_called_with(
+            host='8.8.8.8', port=mock.ANY, name='foo.com',
+            alpn_protocols=['acme-tls/1'])
+
+    @mock.patch('acme.challenges.TLSALPN01Response.probe_cert')
+    def test_simple_verify_false_on_probe_error(self, mock_probe_cert):
+        mock_probe_cert.side_effect = errors.Error
+        self.assertFalse(self.response.simple_verify(
+            self.chall, self.domain, KEY.public_key()))
+
 
 class TLSALPN01Test(unittest.TestCase):
 
@@ -309,8 +369,13 @@ class TLSALPN01Test(unittest.TestCase):
         self.assertRaises(
             jose.DeserializationError, TLSALPN01.from_json, self.jmsg)
 
-    def test_validation(self):
-        self.assertRaises(NotImplementedError, self.msg.validation, KEY)
+    @mock.patch('acme.challenges.TLSALPN01Response.gen_cert')
+    def test_validation(self, mock_gen_cert):
+        mock_gen_cert.return_value = ('cert', 'key')
+        self.assertEqual(('cert', 'key'), self.msg.validation(
+            KEY, cert_key=mock.sentinel.cert_key, domain=mock.sentinel.domain))
+        mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key,
+                                              domain=mock.sentinel.domain)
 
 
 class DNSTest(unittest.TestCase):
@@ -413,5 +478,18 @@ class DNSResponseTest(unittest.TestCase):
             self.msg.check_validation(self.chall, KEY.public_key()))
 
 
+class JWSPayloadRFC8555Compliant(unittest.TestCase):
+    """Test for RFC8555 compliance of JWS generated from resources/challenges"""
+    def test_challenge_payload(self):
+        from acme.challenges import HTTP01Response
+
+        challenge_body = HTTP01Response()
+        challenge_body.le_acme_version = 2
+
+        jobj = challenge_body.json_dumps(indent=2).encode()
+        # RFC8555 states that challenge responses must have an empty payload.
+        self.assertEqual(jobj, b'{}')
+
+
 if __name__ == '__main__':
     unittest.main()  # pragma: no cover

acme/tests/client_test.py

@@ -15,7 +15,7 @@ from acme import challenges
 from acme import errors
 from acme import jws as acme_jws
 from acme import messages
-from acme.magic_typing import Dict  # pylint: disable=unused-import, no-name-in-module
+from acme.mixins import VersionedLEACMEMixin
 import messages_test
 import test_util
 
@@ -886,7 +886,7 @@ class ClientV2Test(ClientTestBase):
             self.client.net.get.assert_not_called()
 
 
-class MockJSONDeSerializable(jose.JSONDeSerializable):
+class MockJSONDeSerializable(VersionedLEACMEMixin, jose.JSONDeSerializable):
     # pylint: disable=missing-docstring
     def __init__(self, value):
         self.value = value

acme/tests/crypto_util_test.py

@@ -11,14 +11,12 @@ import six
 from six.moves import socketserver  # type: ignore  # pylint: disable=import-error
 
 from acme import errors
-from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
 import test_util
 
 
 class SSLSocketAndProbeSNITest(unittest.TestCase):
     """Tests for acme.crypto_util.SSLSocket/probe_sni."""
 
-
     def setUp(self):
         self.cert = test_util.load_comparable_cert('rsa2048_cert.pem')
         key = test_util.load_pyopenssl_private_key('rsa2048_key.pem')
@@ -32,7 +30,8 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
             # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init
 
             def server_bind(self):  # pylint: disable=missing-docstring
-                self.socket = SSLSocket(socket.socket(), certs=certs)
+                self.socket = SSLSocket(socket.socket(),
+                        certs)
                 socketserver.TCPServer.server_bind(self)
 
         self.server = _TestServer(('', 0), socketserver.BaseRequestHandler)
@@ -73,6 +72,18 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
             socket.setdefaulttimeout(original_timeout)
 
 
+class SSLSocketTest(unittest.TestCase):
+    """Tests for acme.crypto_util.SSLSocket."""
+
+    def test_ssl_socket_invalid_arguments(self):
+        from acme.crypto_util import SSLSocket
+        with self.assertRaises(ValueError):
+            _ = SSLSocket(None, {'sni': ('key', 'cert')},
+                    cert_selection=lambda _: None)
+        with self.assertRaises(ValueError):
+            _ = SSLSocket(None)
+
+
 class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase):
     """Test for acme.crypto_util._pyopenssl_cert_or_req_all_names."""
 

acme/tests/magic_typing_test.py

@@ -18,7 +18,7 @@ class MagicTypingTest(unittest.TestCase):
         sys.modules['typing'] = typing_class_mock
         if 'acme.magic_typing' in sys.modules:
             del sys.modules['acme.magic_typing'] # pragma: no cover
-        from acme.magic_typing import Text # pylint: disable=no-name-in-module
+        from acme.magic_typing import Text
         self.assertEqual(Text, text_mock)
         del sys.modules['acme.magic_typing']
         sys.modules['typing'] = temp_typing
@@ -31,7 +31,7 @@ class MagicTypingTest(unittest.TestCase):
         sys.modules['typing'] = None
         if 'acme.magic_typing' in sys.modules:
             del sys.modules['acme.magic_typing'] # pragma: no cover
-        from acme.magic_typing import Text # pylint: disable=no-name-in-module
+        from acme.magic_typing import Text
         self.assertTrue(Text is None)
         del sys.modules['acme.magic_typing']
         sys.modules['typing'] = temp_typing

acme/tests/messages_test.py

@@ -5,7 +5,6 @@ import josepy as jose
 import mock
 
 from acme import challenges
-from acme.magic_typing import Dict  # pylint: disable=unused-import, no-name-in-module
 import test_util
 
 CERT = test_util.load_comparable_cert('cert.der')
@@ -453,6 +452,7 @@ class OrderResourceTest(unittest.TestCase):
             'authorizations': None,
         })
 
+
 class NewOrderTest(unittest.TestCase):
     """Tests for acme.messages.NewOrder."""
 
@@ -467,5 +467,18 @@ class NewOrderTest(unittest.TestCase):
         })
 
 
+class JWSPayloadRFC8555Compliant(unittest.TestCase):
+    """Test for RFC8555 compliance of JWS generated from resources/challenges"""
+    def test_message_payload(self):
+        from acme.messages import NewAuthorization
+
+        new_order = NewAuthorization()
+        new_order.le_acme_version = 2
+
+        jobj = new_order.json_dumps(indent=2).encode()
+        # RFC8555 states that JWS bodies must not have a resource field.
+        self.assertEqual(jobj, b'{}')
+
+
 if __name__ == '__main__':
     unittest.main()  # pragma: no cover

acme/tests/standalone_test.py

@@ -10,7 +10,9 @@ from six.moves import http_client  # pylint: disable=import-error
 from six.moves import socketserver  # type: ignore  # pylint: disable=import-error
 
 from acme import challenges
-from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
+from acme import crypto_util
+from acme import errors
+
 import test_util
 
 
@@ -83,6 +85,81 @@ class HTTP01ServerTest(unittest.TestCase):
     def test_http01_not_found(self):
         self.assertFalse(self._test_http01(add=False))
 
+    def test_timely_shutdown(self):
+        from acme.standalone import HTTP01Server
+        server = HTTP01Server(('', 0), resources=set(), timeout=0.05)
+        server_thread = threading.Thread(target=server.serve_forever)
+        server_thread.start()
+
+        client = socket.socket()
+        client.connect(('localhost', server.socket.getsockname()[1]))
+
+        stop_thread = threading.Thread(target=server.shutdown)
+        stop_thread.start()
+        server_thread.join(5.)
+
+        is_hung = server_thread.is_alive()
+        try:
+            client.shutdown(socket.SHUT_RDWR)
+        except: # pragma: no cover, pylint: disable=bare-except
+            # may raise error because socket could already be closed
+            pass
+
+        self.assertFalse(is_hung, msg='Server shutdown should not be hung')
+
+
+@unittest.skipIf(not challenges.TLSALPN01.is_supported(), "pyOpenSSL too old")
+class TLSALPN01ServerTest(unittest.TestCase):
+    """Test for acme.standalone.TLSALPN01Server."""
+
+    def setUp(self):
+        self.certs = {b'localhost': (
+            test_util.load_pyopenssl_private_key('rsa2048_key.pem'),
+            test_util.load_cert('rsa2048_cert.pem'),
+        )}
+        # Use different certificate for challenge.
+        self.challenge_certs = {b'localhost': (
+            test_util.load_pyopenssl_private_key('rsa1024_key.pem'),
+            test_util.load_cert('rsa1024_cert.pem'),
+        )}
+        from acme.standalone import TLSALPN01Server
+        self.server = TLSALPN01Server(("localhost", 0), certs=self.certs,
+                challenge_certs=self.challenge_certs)
+        # pylint: disable=no-member
+        self.thread = threading.Thread(target=self.server.serve_forever)
+        self.thread.start()
+
+    def tearDown(self):
+        self.server.shutdown()  # pylint: disable=no-member
+        self.thread.join()
+
+    # TODO: This is not implemented yet, see comments in standalone.py
+    # def test_certs(self):
+    #    host, port = self.server.socket.getsockname()[:2]
+    #    cert = crypto_util.probe_sni(
+    #        b'localhost', host=host, port=port, timeout=1)
+    #    # Expect normal cert when connecting without ALPN.
+    #    self.assertEqual(jose.ComparableX509(cert),
+    #                     jose.ComparableX509(self.certs[b'localhost'][1]))
+
+    def test_challenge_certs(self):
+        host, port = self.server.socket.getsockname()[:2]
+        cert = crypto_util.probe_sni(
+            b'localhost', host=host, port=port, timeout=1,
+            alpn_protocols=[b"acme-tls/1"])
+        #  Expect challenge cert when connecting with ALPN.
+        self.assertEqual(
+                jose.ComparableX509(cert),
+                jose.ComparableX509(self.challenge_certs[b'localhost'][1])
+        )
+
+    def test_bad_alpn(self):
+        host, port = self.server.socket.getsockname()[:2]
+        with self.assertRaises(errors.Error):
+            crypto_util.probe_sni(
+                b'localhost', host=host, port=port, timeout=1,
+                alpn_protocols=[b"bad-alpn"])
+
 
 class BaseDualNetworkedServersTest(unittest.TestCase):
     """Test for acme.standalone.BaseDualNetworkedServers."""
@@ -138,7 +215,6 @@ class BaseDualNetworkedServersTest(unittest.TestCase):
 class HTTP01DualNetworkedServersTest(unittest.TestCase):
     """Tests for acme.standalone.HTTP01DualNetworkedServers."""
 
-
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))

acme/tests/testdata/README

@@ -10,6 +10,8 @@ and for the CSR:
 
   openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -outform DER > csr.der
 
-and for the certificate:
+and for the certificates:
 
-  openssl req -key rsa2047_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der
+  openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der
+  openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 > rsa2048_cert.pem
+  openssl req -key rsa1024_key.pem -new -subj '/CN=example.com' -x509 > rsa1024_cert.pem

acme/tests/testdata/rsa1024_cert.pem

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIIB/TCCAWagAwIBAgIJAOyRIBs3QT8QMA0GCSqGSIb3DQEBCwUAMBYxFDASBgNV
+BAMMC2V4YW1wbGUuY29tMB4XDTE4MDQyMzEwMzE0NFoXDTE4MDUyMzEwMzE0NFow
+FjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
+AoGBAJqJ87R8aVwByONxgQA9hwgvQd/QqI1r1UInXhEF2VnEtZGtUWLi100IpIqr
+Mq4qusDwNZ3g8cUPtSkvJGs89djoajMDIJP7lQUEKUYnYrI0q755Tr/DgLWSk7iW
+l5ezym0VzWUD0/xXUz8yRbNMTjTac80rS5SZk2ja2wWkYlRJAgMBAAGjUzBRMB0G
+A1UdDgQWBBSsaX0IVZ4XXwdeffVAbG7gnxSYjTAfBgNVHSMEGDAWgBSsaX0IVZ4X
+XwdeffVAbG7gnxSYjTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GB
+ADe7SVmvGH2nkwVfONk8TauRUDkePN1CJZKFb2zW1uO9ANJ2v5Arm/OQp0BG/xnI
+Djw/aLTNVESF89oe15dkrUErtcaF413MC1Ld5lTCaJLHLGqDKY69e02YwRuxW7jY
+qarpt7k7aR5FbcfO5r4V/FK/Gvp4Dmoky8uap7SJIW6x
+-----END CERTIFICATE-----

certbot-apache/MANIFEST.in

@@ -1,7 +1,7 @@
 include LICENSE.txt
 include README.rst
 recursive-include tests *
-include certbot_apache/_internal/options-ssl-apache.conf
 recursive-include certbot_apache/_internal/augeas_lens *.aug
+recursive-include certbot_apache/_internal/tls_configs *.conf
 global-exclude __pycache__
 global-exclude *.py[cod]

certbot-apache/certbot_apache/_internal/apache_util.py

@@ -5,6 +5,8 @@ import logging
 import re
 import subprocess
 
+import pkg_resources
+
 from certbot import errors
 from certbot import util
 
@@ -241,3 +243,14 @@ def _get_runtime_cfg(command):
             "loaded because Apache is misconfigured.")
 
     return stdout
+
+def find_ssl_apache_conf(prefix):
+    """
+    Find a TLS Apache config file in the dedicated storage.
+    :param str prefix: prefix of the TLS Apache config file to find
+    :return: the path the TLS Apache config file
+    :rtype: str
+    """
+    return pkg_resources.resource_filename(
+        "certbot_apache",
+        os.path.join("_internal", "tls_configs", "{0}-options-ssl-apache.conf".format(prefix)))

certbot-apache/certbot_apache/_internal/apacheparser.py

@@ -73,7 +73,7 @@ class ApacheDirectiveNode(ApacheParserNode):
                     self.metadata == other.metadata)
         return False
 
-    def set_parameters(self, _parameters):
+    def set_parameters(self, _parameters):  # pragma: no cover
         """Sets the parameters for DirectiveNode"""
         return
 
@@ -97,7 +97,8 @@ class ApacheBlockNode(ApacheDirectiveNode):
                     self.metadata == other.metadata)
         return False
 
-    def add_child_block(self, name, parameters=None, position=None):  # pylint: disable=unused-argument
+    # pylint: disable=unused-argument
+    def add_child_block(self, name, parameters=None, position=None):  # pragma: no cover
         """Adds a new BlockNode to the sequence of children"""
         new_block = ApacheBlockNode(name=assertions.PASS,
                                     parameters=assertions.PASS,
@@ -107,7 +108,8 @@ class ApacheBlockNode(ApacheDirectiveNode):
         self.children += (new_block,)
         return new_block
 
-    def add_child_directive(self, name, parameters=None, position=None):  # pylint: disable=unused-argument
+    # pylint: disable=unused-argument
+    def add_child_directive(self, name, parameters=None, position=None):  # pragma: no cover
         """Adds a new DirectiveNode to the sequence of children"""
         new_dir = ApacheDirectiveNode(name=assertions.PASS,
                                       parameters=assertions.PASS,
@@ -144,7 +146,8 @@ class ApacheBlockNode(ApacheDirectiveNode):
                                     filepath=assertions.PASS,
                                     metadata=self.metadata)]
 
-    def find_comments(self, comment, exact=False): # pylint: disable=unused-argument
+    # pylint: disable=unused-argument
+    def find_comments(self, comment, exact=False):  # pragma: no cover
         """Recursive search of DirectiveNodes from the sequence of children"""
         return [ApacheCommentNode(comment=assertions.PASS,
                                   ancestor=self,

certbot-apache/certbot_apache/_internal/augeasparser.py

@@ -64,7 +64,7 @@ Translates over to:
     "/files/etc/apache2/apache2.conf/bLoCk[1]",
 ]
 """
-from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Set
 from certbot import errors
 from certbot.compat import os
 

certbot-apache/certbot_apache/_internal/configurator.py

@@ -1,6 +1,7 @@
 """Apache Configurator."""
 # pylint: disable=too-many-lines
 from collections import defaultdict
+from distutils.version import LooseVersion
 import copy
 import fnmatch
 import logging
@@ -8,10 +9,14 @@ import re
 import socket
 import time
 
-import pkg_resources
 import six
 import zope.component
 import zope.interface
+try:
+    import apacheconfig
+    HAS_APACHECONFIG = True
+except ImportError:  # pragma: no cover
+    HAS_APACHECONFIG = False
 
 from acme import challenges
 from acme.magic_typing import DefaultDict
@@ -110,14 +115,29 @@ class ApacheConfigurator(common.Installer):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2",
-        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
-            "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf"))
     )
 
     def option(self, key):
         """Get a value from options"""
         return self.options.get(key)
 
+    def pick_apache_config(self, warn_on_no_mod_ssl=True):
+        """
+        Pick the appropriate TLS Apache configuration file for current version of Apache and OS.
+
+        :param bool warn_on_no_mod_ssl: True if we should warn if mod_ssl is not found.
+
+        :return: the path to the TLS Apache configuration file to use
+        :rtype: str
+        """
+        # Disabling TLS session tickets is supported by Apache 2.4.11+ and OpenSSL 1.0.2l+.
+        # So for old versions of Apache we pick a configuration without this option.
+        openssl_version = self.openssl_version(warn_on_no_mod_ssl)
+        if self.version < (2, 4, 11) or not openssl_version or\
+            LooseVersion(openssl_version) < LooseVersion('1.0.2l'):
+            return apache_util.find_ssl_apache_conf("old")
+        return apache_util.find_ssl_apache_conf("current")
+
     def _prepare_options(self):
         """
         Set the values possibly changed by command line parameters to
@@ -184,6 +204,7 @@ class ApacheConfigurator(common.Installer):
         """
         version = kwargs.pop("version", None)
         use_parsernode = kwargs.pop("use_parsernode", False)
+        openssl_version = kwargs.pop("openssl_version", None)
         super(ApacheConfigurator, self).__init__(*args, **kwargs)
 
         # Add name_server association dict
@@ -209,6 +230,7 @@ class ApacheConfigurator(common.Installer):
         self.parser = None
         self.parser_root = None
         self.version = version
+        self._openssl_version = openssl_version
         self.vhosts = None
         self.options = copy.deepcopy(self.OS_DEFAULTS)
         self._enhance_func = {"redirect": self._enable_redirect,
@@ -225,6 +247,52 @@ class ApacheConfigurator(common.Installer):
         """Full absolute path to digest of updated SSL configuration file."""
         return os.path.join(self.config.config_dir, constants.UPDATED_MOD_SSL_CONF_DIGEST)
 
+    def _open_module_file(self, ssl_module_location):
+        """Extract the open lines of openssl_version for testing purposes"""
+        try:
+            with open(ssl_module_location, mode="rb") as f:
+                contents = f.read()
+        except IOError as error:
+            logger.debug(str(error), exc_info=True)
+            return None
+        return contents
+
+    def openssl_version(self, warn_on_no_mod_ssl=True):
+        """Lazily retrieve openssl version
+
+        :param bool warn_on_no_mod_ssl: `True` if we should warn if mod_ssl is not found. Set to
+            `False` when we know we'll try to enable mod_ssl later. This is currently debian/ubuntu,
+            when called from `prepare`.
+
+        :return: the OpenSSL version as a string, or None.
+        :rtype: str or None
+        """
+        if self._openssl_version:
+            return self._openssl_version
+        # Step 1. Check for LoadModule directive
+        try:
+            ssl_module_location = self.parser.modules['ssl_module']
+        except KeyError:
+            if warn_on_no_mod_ssl:
+                logger.warning("Could not find ssl_module; not disabling session tickets.")
+            return None
+        if not ssl_module_location:
+            logger.warning("Could not find ssl_module; not disabling session tickets.")
+            return None
+        ssl_module_location = self.parser.standard_path_from_server_root(ssl_module_location)
+        # Step 2. Grep in the .so for openssl version
+        contents = self._open_module_file(ssl_module_location)
+        if not contents:
+            logger.warning("Unable to read ssl_module file; not disabling session tickets.")
+            return None
+        # looks like: OpenSSL 1.0.2s  28 May 2019
+        matches = re.findall(br"OpenSSL ([0-9]\.[^ ]+) ", contents)
+        if not matches:
+            logger.warning("Could not find OpenSSL version; not disabling session tickets.")
+            return None
+        self._openssl_version = matches[0].decode('UTF-8')
+        return self._openssl_version
+
     def prepare(self):
         """Prepare the authenticator/installer.
 
@@ -271,8 +339,12 @@ class ApacheConfigurator(common.Installer):
         # Get all of the available vhosts
         self.vhosts = self.get_virtual_hosts()
 
+        # We may try to enable mod_ssl later. If so, we shouldn't warn if we can't find it now.
+        # This is currently only true for debian/ubuntu.
+        warn_on_no_mod_ssl = not self.option("handle_modules")
         self.install_ssl_options_conf(self.mod_ssl_conf,
-                                      self.updated_mod_ssl_conf_digest)
+                                      self.updated_mod_ssl_conf_digest,
+                                      warn_on_no_mod_ssl)
 
         # Prevent two Apache plugins from modifying a config at once
         try:
@@ -363,11 +435,17 @@ class ApacheConfigurator(common.Installer):
     def get_parsernode_root(self, metadata):
         """Initializes the ParserNode parser root instance."""
 
-        apache_vars = dict()
-        apache_vars["defines"] = apache_util.parse_defines(self.option("ctl"))
-        apache_vars["includes"] = apache_util.parse_includes(self.option("ctl"))
-        apache_vars["modules"] = apache_util.parse_modules(self.option("ctl"))
-        metadata["apache_vars"] = apache_vars
+        if HAS_APACHECONFIG:
+            apache_vars = dict()
+            apache_vars["defines"] = apache_util.parse_defines(self.option("ctl"))
+            apache_vars["includes"] = apache_util.parse_includes(self.option("ctl"))
+            apache_vars["modules"] = apache_util.parse_modules(self.option("ctl"))
+            metadata["apache_vars"] = apache_vars
+
+            with open(self.parser.loc["root"]) as f:
+                with apacheconfig.make_loader(writable=True,
+                      **apacheconfig.flavors.NATIVE_APACHE) as loader:
+                    metadata["ac_ast"] = loader.loads(f.read())
 
         return dualparser.DualBlockNode(
             name=assertions.PASS,
@@ -907,7 +985,7 @@ class ApacheConfigurator(common.Installer):
         """
 
         v1_vhosts = self.get_virtual_hosts_v1()
-        if self.USE_PARSERNODE:
+        if self.USE_PARSERNODE and HAS_APACHECONFIG:
             v2_vhosts = self.get_virtual_hosts_v2()
 
             for v1_vh in v1_vhosts:
@@ -1241,6 +1319,14 @@ class ApacheConfigurator(common.Installer):
                 self.enable_mod("socache_shmcb", temp=temp)
             if "ssl_module" not in self.parser.modules:
                 self.enable_mod("ssl", temp=temp)
+                # Make sure we're not throwing away any unwritten changes to the config
+                self.parser.ensure_augeas_state()
+                self.parser.aug.load()
+                self.parser.reset_modules() # Reset to load the new ssl_module path
+                # Call again because now we can gate on openssl version
+                self.install_ssl_options_conf(self.mod_ssl_conf,
+                                              self.updated_mod_ssl_conf_digest,
+                                              warn_on_no_mod_ssl=True)
 
     def make_vhost_ssl(self, nonssl_vhost):
         """Makes an ssl_vhost version of a nonssl_vhost.
@@ -2454,14 +2540,19 @@ class ApacheConfigurator(common.Installer):
             self.restart()
             self.parser.reset_modules()
 
-    def install_ssl_options_conf(self, options_ssl, options_ssl_digest):
-        """Copy Certbot's SSL options file into the system's config dir if required."""
+    def install_ssl_options_conf(self, options_ssl, options_ssl_digest, warn_on_no_mod_ssl=True):
+        """Copy Certbot's SSL options file into the system's config dir if required.
+
+        :param bool warn_on_no_mod_ssl: True if we should warn if mod_ssl is not found.
+        """
 
         # XXX if we ever try to enforce a local privilege boundary (eg, running
         # certbot for unprivileged users via setuid), this function will need
         # to be modified.
-        return common.install_version_controlled_file(options_ssl, options_ssl_digest,
-            self.option("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES)
+        apache_config_path = self.pick_apache_config(warn_on_no_mod_ssl)
+
+        return common.install_version_controlled_file(
+            options_ssl, options_ssl_digest, apache_config_path, constants.ALL_SSL_OPTIONS_HASHES)
 
     def enable_autohsts(self, _unused_lineage, domains):
         """

certbot-apache/certbot_apache/_internal/constants.py

@@ -26,6 +26,7 @@ ALL_SSL_OPTIONS_HASHES = [
     '06675349e457eae856120cdebb564efe546f0b87399f2264baeb41e442c724c7',
     '5cc003edd93fb9cd03d40c7686495f8f058f485f75b5e764b789245a386e6daf',
     '007cd497a56a3bb8b6a2c1aeb4997789e7e38992f74e44cc5d13a625a738ac73',
+    '34783b9e2210f5c4a23bced2dfd7ec289834716673354ed7c7abf69fe30192a3',
 ]
 """SHA256 hashes of the contents of previous versions of all versions of MOD_SSL_CONF_SRC"""
 

certbot-apache/certbot_apache/_internal/entrypoint.py

@@ -1,6 +1,4 @@
 """ Entry point for Apache Plugin """
-# Pylint does not like disutils.version when running inside a venv.
-# See: https://github.com/PyCQA/pylint/issues/73
 from distutils.version import LooseVersion
 
 from certbot import util

certbot-apache/certbot_apache/_internal/http_01.py

@@ -1,5 +1,6 @@
 """A class that performs HTTP-01 challenges for Apache"""
 import logging
+import errno
 
 from acme.magic_typing import List
 from acme.magic_typing import Set
@@ -168,7 +169,15 @@ class ApacheHttp01(common.ChallengePerformer):
 
     def _set_up_challenges(self):
         if not os.path.isdir(self.challenge_dir):
-            filesystem.makedirs(self.challenge_dir, 0o755)
+            old_umask = os.umask(0o022)
+            try:
+                filesystem.makedirs(self.challenge_dir, 0o755)
+            except OSError as exception:
+                if exception.errno not in (errno.EEXIST, errno.EISDIR):
+                    raise errors.PluginError(
+                        "Couldn't create root for http-01 challenge")
+            finally:
+                os.umask(old_umask)
 
         responses = []
         for achall in self.achalls:

certbot-apache/certbot_apache/_internal/interfaces.py

@@ -102,7 +102,6 @@ For this reason the internal representation of data should not ignore the case.
 import abc
 import six
 
-from acme.magic_typing import Any, Dict, Optional, Tuple  # pylint: disable=unused-import, no-name-in-module
 
 
 @six.add_metaclass(abc.ABCMeta)

certbot-apache/certbot_apache/_internal/override_arch.py

@@ -1,9 +1,7 @@
 """ Distribution specific override class for Arch Linux """
-import pkg_resources
 import zope.interface
 
 from certbot import interfaces
-from certbot.compat import os
 from certbot_apache._internal import configurator
 
 
@@ -26,6 +24,4 @@ class ArchConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/httpd/conf",
-        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
-            "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf"))
     )

certbot-apache/certbot_apache/_internal/override_centos.py

@@ -1,14 +1,12 @@
 """ Distribution specific override class for CentOS family (RHEL, Fedora) """
 import logging
 
-import pkg_resources
 import zope.interface
 
 from acme.magic_typing import List
 from certbot import errors
 from certbot import interfaces
 from certbot import util
-from certbot.compat import os
 from certbot.errors import MisconfigurationError
 from certbot_apache._internal import apache_util
 from certbot_apache._internal import configurator
@@ -37,8 +35,6 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/httpd/conf.d",
-        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
-            "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf"))
     )
 
     def config_test(self):

certbot-apache/certbot_apache/_internal/override_darwin.py

@@ -1,9 +1,7 @@
 """ Distribution specific override class for macOS """
-import pkg_resources
 import zope.interface
 
 from certbot import interfaces
-from certbot.compat import os
 from certbot_apache._internal import configurator
 
 
@@ -26,6 +24,4 @@ class DarwinConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/other",
-        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
-            "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf"))
     )

certbot-apache/certbot_apache/_internal/override_debian.py

@@ -1,7 +1,6 @@
 """ Distribution specific override class for Debian family (Ubuntu/Debian) """
 import logging
 
-import pkg_resources
 import zope.interface
 
 from certbot import errors
@@ -34,8 +33,6 @@ class DebianConfigurator(configurator.ApacheConfigurator):
         handle_modules=True,
         handle_sites=True,
         challenge_location="/etc/apache2",
-        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
-            "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf"))
     )
 
     def enable_site(self, vhost):

certbot-apache/certbot_apache/_internal/override_fedora.py

@@ -1,11 +1,9 @@
 """ Distribution specific override class for Fedora 29+ """
-import pkg_resources
 import zope.interface
 
 from certbot import errors
 from certbot import interfaces
 from certbot import util
-from certbot.compat import os
 from certbot_apache._internal import apache_util
 from certbot_apache._internal import configurator
 from certbot_apache._internal import parser
@@ -31,9 +29,6 @@ class FedoraConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/httpd/conf.d",
-        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
-            # TODO: eventually newest version of Fedora will need their own config
-            "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf"))
     )
 
     def config_test(self):

certbot-apache/certbot_apache/_internal/override_gentoo.py

@@ -1,9 +1,7 @@
 """ Distribution specific override class for Gentoo Linux """
-import pkg_resources
 import zope.interface
 
 from certbot import interfaces
-from certbot.compat import os
 from certbot_apache._internal import apache_util
 from certbot_apache._internal import configurator
 from certbot_apache._internal import parser
@@ -29,8 +27,6 @@ class GentooConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/vhosts.d",
-        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
-            "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf"))
     )
 
     def _prepare_options(self):

certbot-apache/certbot_apache/_internal/override_suse.py

@@ -1,9 +1,7 @@
 """ Distribution specific override class for OpenSUSE """
-import pkg_resources
 import zope.interface
 
 from certbot import interfaces
-from certbot.compat import os
 from certbot_apache._internal import configurator
 
 
@@ -26,6 +24,4 @@ class OpenSUSEConfigurator(configurator.ApacheConfigurator):
         handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/vhosts.d",
-        MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
-            "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf"))
     )

certbot-apache/certbot_apache/_internal/parser.py

@@ -9,7 +9,6 @@ import six
 
 from acme.magic_typing import Dict
 from acme.magic_typing import List
-from acme.magic_typing import Set
 from certbot import errors
 from certbot.compat import os
 from certbot_apache._internal import apache_util
@@ -52,7 +51,7 @@ class ApacheParser(object):
                 "version 1.2.0 or higher, please make sure you have you have "
                 "those installed.")
 
-        self.modules = set()  # type: Set[str]
+        self.modules = {}  # type: Dict[str, str]
         self.parser_paths = {}  # type: Dict[str, List[str]]
         self.variables = {}  # type: Dict[str, str]
 
@@ -249,14 +248,14 @@ class ApacheParser(object):
     def add_mod(self, mod_name):
         """Shortcut for updating parser modules."""
         if mod_name + "_module" not in self.modules:
-            self.modules.add(mod_name + "_module")
+            self.modules[mod_name + "_module"] = None
         if "mod_" + mod_name + ".c" not in self.modules:
-            self.modules.add("mod_" + mod_name + ".c")
+            self.modules["mod_" + mod_name + ".c"] = None
 
     def reset_modules(self):
         """Reset the loaded modules list. This is called from cleanup to clear
         temporarily loaded modules."""
-        self.modules = set()
+        self.modules = {}
         self.update_modules()
         self.parse_modules()
 
@@ -267,7 +266,7 @@ class ApacheParser(object):
             the iteration issue.  Else... parse and enable mods at same time.
 
         """
-        mods = set()  # type: Set[str]
+        mods = {}  # type: Dict[str, str]
         matches = self.find_dir("LoadModule")
         iterator = iter(matches)
         # Make sure prev_size != cur_size for do: while: iteration
@@ -281,8 +280,8 @@ class ApacheParser(object):
                 mod_name = self.get_arg(match_name)
                 mod_filename = self.get_arg(match_filename)
                 if mod_name and mod_filename:
-                    mods.add(mod_name)
-                    mods.add(os.path.basename(mod_filename)[:-2] + "c")
+                    mods[mod_name] = mod_filename
+                    mods[os.path.basename(mod_filename)[:-2] + "c"] = mod_filename
                 else:
                     logger.debug("Could not read LoadModule directive from Augeas path: %s",
                                  match_name[6:])
@@ -621,7 +620,7 @@ class ApacheParser(object):
 
     def exclude_dirs(self, matches):
         """Exclude directives that are not loaded into the configuration."""
-        filters = [("ifmodule", self.modules), ("ifdefine", self.variables)]
+        filters = [("ifmodule", self.modules.keys()), ("ifdefine", self.variables)]
 
         valid_matches = []
 
@@ -662,6 +661,25 @@ class ApacheParser(object):
 
         return True
 
+    def standard_path_from_server_root(self, arg):
+        """Ensure paths are consistent and absolute
+
+        :param str arg: Argument of directive
+
+        :returns: Standardized argument path
+        :rtype: str
+        """
+        # Remove beginning and ending quotes
+        arg = arg.strip("'\"")
+
+        # Standardize the include argument based on server root
+        if not arg.startswith("/"):
+            # Normpath will condense ../
+            arg = os.path.normpath(os.path.join(self.root, arg))
+        else:
+            arg = os.path.normpath(arg)
+        return arg
+
     def _get_include_path(self, arg):
         """Converts an Apache Include directive into Augeas path.
 
@@ -682,16 +700,7 @@ class ApacheParser(object):
         # if matchObj.group() != arg:
         #     logger.error("Error: Invalid regexp characters in %s", arg)
         #     return []
-
-        # Remove beginning and ending quotes
-        arg = arg.strip("'\"")
-
-        # Standardize the include argument based on server root
-        if not arg.startswith("/"):
-            # Normpath will condense ../
-            arg = os.path.normpath(os.path.join(self.root, arg))
-        else:
-            arg = os.path.normpath(arg)
+        arg = self.standard_path_from_server_root(arg)
 
         # Attempts to add a transform to the file if one does not already exist
         if os.path.isdir(arg):

certbot-apache/certbot_apache/_internal/tls_configs/current-options-ssl-apache.conf

@@ -0,0 +1,19 @@
+# This file contains important security parameters. If you modify this file
+# manually, Certbot will be unable to automatically provide future security
+# updates. Instead, Certbot will print and log an error message with a path to
+# the up-to-date file that you will need to refer to when manually updating
+# this file.
+
+SSLEngine on
+
+# Intermediate configuration, tweak to your needs
+SSLProtocol             all -SSLv2 -SSLv3 -TLSv1 -TLSv1.1
+SSLCipherSuite          ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+SSLHonorCipherOrder     off
+SSLSessionTickets       off
+
+SSLOptions +StrictRequire
+
+# Add vhost name to log entries:
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" vhost_combined
+LogFormat "%v %h %l %u %t \"%r\" %>s %b" vhost_common

certbot-apache/local-oldest-requirements.txt

@@ -1,3 +1,3 @@
 # Remember to update setup.py to match the package versions below.
 acme[dev]==0.29.0
-certbot[dev]==1.1.0
+certbot[dev]==1.1.0

certbot-apache/setup.py

@@ -1,25 +1,35 @@
+from distutils.version import StrictVersion
 import sys
 
+from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
     'acme>=0.29.0',
     'certbot>=1.1.0',
-    'mock',
     'python-augeas',
     'setuptools',
     'zope.component',
     'zope.interface',
 ]
 
+setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2'))
+if setuptools_known_environment_markers:
+    install_requires.append('mock ; python_version < "3.3"')
+elif sys.version_info < (3,3):
+    install_requires.append('mock')
+else:
+    raise RuntimeError('Error, you are trying to build certbot wheels using an old version '
+                       'of setuptools. Version 36.2+ of setuptools is required.')
+
 dev_extras = [
-    'apacheconfig>=0.3.1',
+    'apacheconfig>=0.3.2',
 ]
 
 class PyTest(TestCommand):

certbot-apache/tests/augeasnode_test.py

@@ -1,14 +1,28 @@
 """Tests for AugeasParserNode classes"""
-import mock
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock
 
+import os
+import unittest
 import util
 
-from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
 from certbot import errors
 
 from certbot_apache._internal import assertions
+from certbot_apache._internal import augeasparser
 
 
+def _get_augeasnode_mock(filepath):
+    """ Helper function for mocking out DualNode instance with an AugeasNode """
+    def augeasnode_mock(metadata):
+        return augeasparser.AugeasBlockNode(
+            name=assertions.PASS,
+            ancestor=None,
+            filepath=filepath,
+            metadata=metadata)
+    return augeasnode_mock
 
 class AugeasParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-methods
     """Test AugeasParserNode using available test configurations"""
@@ -16,8 +30,11 @@ class AugeasParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-
     def setUp(self):  # pylint: disable=arguments-differ
         super(AugeasParserNodeTest, self).setUp()
 
-        self.config = util.get_apache_configurator(
-            self.config_path, self.vhost_path, self.config_dir, self.work_dir, use_parsernode=True)
+        with mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.get_parsernode_root") as mock_parsernode:
+            mock_parsernode.side_effect = _get_augeasnode_mock(
+                                              os.path.join(self.config_path, "apache2.conf"))
+            self.config = util.get_apache_configurator(
+                self.config_path, self.vhost_path, self.config_dir, self.work_dir, use_parsernode=True)
         self.vh_truth = util.get_vh_truth(
             self.temp_dir, "debian_apache_2_4/multiple_vhosts")
 
@@ -111,7 +128,7 @@ class AugeasParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-
                 name=servernames[0].name,
                 parameters=["test", "setting", "these"],
                 ancestor=assertions.PASS,
-                metadata=servernames[0].primary.metadata
+                metadata=servernames[0].metadata
             )
             self.assertTrue(mock_set.called)
             self.assertEqual(
@@ -146,26 +163,26 @@ class AugeasParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-
         self.assertTrue(found)
 
     def test_add_child_comment(self):
-        newc = self.config.parser_root.primary.add_child_comment("The content")
+        newc = self.config.parser_root.add_child_comment("The content")
         comments = self.config.parser_root.find_comments("The content")
         self.assertEqual(len(comments), 1)
         self.assertEqual(
             newc.metadata["augeaspath"],
-            comments[0].primary.metadata["augeaspath"]
+            comments[0].metadata["augeaspath"]
         )
         self.assertEqual(newc.comment, comments[0].comment)
 
     def test_delete_child(self):
-        listens = self.config.parser_root.primary.find_directives("Listen")
+        listens = self.config.parser_root.find_directives("Listen")
         self.assertEqual(len(listens), 1)
-        self.config.parser_root.primary.delete_child(listens[0])
+        self.config.parser_root.delete_child(listens[0])
 
-        listens = self.config.parser_root.primary.find_directives("Listen")
+        listens = self.config.parser_root.find_directives("Listen")
         self.assertEqual(len(listens), 0)
 
     def test_delete_child_not_found(self):
         listen = self.config.parser_root.find_directives("Listen")[0]
-        listen.primary.metadata["augeaspath"] = "/files/something/nonexistent"
+        listen.metadata["augeaspath"] = "/files/something/nonexistent"
 
         self.assertRaises(
             errors.PluginError,
@@ -178,10 +195,10 @@ class AugeasParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-
             "NewBlock",
             ["first", "second"]
         )
-        rpath, _, directive = nb.primary.metadata["augeaspath"].rpartition("/")
+        rpath, _, directive = nb.metadata["augeaspath"].rpartition("/")
         self.assertEqual(
             rpath,
-            self.config.parser_root.primary.metadata["augeaspath"]
+            self.config.parser_root.metadata["augeaspath"]
         )
         self.assertTrue(directive.startswith("NewBlock"))
 
@@ -190,8 +207,8 @@ class AugeasParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-
             "Beginning",
             position=0
         )
-        parser = self.config.parser_root.primary.parser
-        root_path = self.config.parser_root.primary.metadata["augeaspath"]
+        parser = self.config.parser_root.parser
+        root_path = self.config.parser_root.metadata["augeaspath"]
         # Get first child
         first = parser.aug.match("{}/*[1]".format(root_path))
         self.assertTrue(first[0].endswith("Beginning"))
@@ -200,8 +217,8 @@ class AugeasParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-
         self.config.parser_root.add_child_block(
             "VeryLast",
         )
-        parser = self.config.parser_root.primary.parser
-        root_path = self.config.parser_root.primary.metadata["augeaspath"]
+        parser = self.config.parser_root.parser
+        root_path = self.config.parser_root.metadata["augeaspath"]
         # Get last child
         last = parser.aug.match("{}/*[last()]".format(root_path))
         self.assertTrue(last[0].endswith("VeryLast"))
@@ -211,8 +228,8 @@ class AugeasParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-
             "VeryLastAlt",
             position=99999
         )
-        parser = self.config.parser_root.primary.parser
-        root_path = self.config.parser_root.primary.metadata["augeaspath"]
+        parser = self.config.parser_root.parser
+        root_path = self.config.parser_root.metadata["augeaspath"]
         # Get last child
         last = parser.aug.match("{}/*[last()]".format(root_path))
         self.assertTrue(last[0].endswith("VeryLastAlt"))
@@ -222,15 +239,15 @@ class AugeasParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-
             "Middle",
             position=5
         )
-        parser = self.config.parser_root.primary.parser
-        root_path = self.config.parser_root.primary.metadata["augeaspath"]
+        parser = self.config.parser_root.parser
+        root_path = self.config.parser_root.metadata["augeaspath"]
         # Augeas indices start at 1 :(
         middle = parser.aug.match("{}/*[6]".format(root_path))
         self.assertTrue(middle[0].endswith("Middle"))
 
     def test_add_child_block_existing_name(self):
-        parser = self.config.parser_root.primary.parser
-        root_path = self.config.parser_root.primary.metadata["augeaspath"]
+        parser = self.config.parser_root.parser
+        root_path = self.config.parser_root.metadata["augeaspath"]
         # There already exists a single VirtualHost in the base config
         new_block = parser.aug.match("{}/VirtualHost[2]".format(root_path))
         self.assertEqual(len(new_block), 0)
@@ -239,7 +256,7 @@ class AugeasParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-
         )
         new_block = parser.aug.match("{}/VirtualHost[2]".format(root_path))
         self.assertEqual(len(new_block), 1)
-        self.assertTrue(vh.primary.metadata["augeaspath"].endswith("VirtualHost[2]"))
+        self.assertTrue(vh.metadata["augeaspath"].endswith("VirtualHost[2]"))
 
     def test_node_init_error_bad_augeaspath(self):
         from certbot_apache._internal.augeasparser import AugeasBlockNode
@@ -284,7 +301,7 @@ class AugeasParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-
         self.assertEqual(len(dirs), 1)
         self.assertEqual(dirs[0].parameters, ("with", "parameters"))
         # The new directive was added to the very first line of the config
-        self.assertTrue(dirs[0].primary.metadata["augeaspath"].endswith("[1]"))
+        self.assertTrue(dirs[0].metadata["augeaspath"].endswith("[1]"))
 
     def test_add_child_directive_exception(self):
         self.assertRaises(
@@ -314,6 +331,6 @@ class AugeasParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-
         self.assertTrue(nonmacro_test)
 
     def test_find_ancestors_bad_path(self):
-        self.config.parser_root.primary.metadata["augeaspath"] = ""
-        ancs = self.config.parser_root.primary.find_ancestors("Anything")
+        self.config.parser_root.metadata["augeaspath"] = ""
+        ancs = self.config.parser_root.find_ancestors("Anything")
         self.assertEqual(len(ancs), 0)

certbot-apache/tests/autohsts_test.py

@@ -3,7 +3,10 @@
 import re
 import unittest
 
-import mock
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock
 import six  # pylint: disable=unused-import  # six is used in mock.patch()
 
 from certbot import errors
@@ -20,10 +23,10 @@ class AutoHSTSTest(util.ApacheTest):
 
         self.config = util.get_apache_configurator(
             self.config_path, self.vhost_path, self.config_dir, self.work_dir)
-        self.config.parser.modules.add("headers_module")
-        self.config.parser.modules.add("mod_headers.c")
-        self.config.parser.modules.add("ssl_module")
-        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules["headers_module"] = None
+        self.config.parser.modules["mod_headers.c"] = None
+        self.config.parser.modules["ssl_module"] = None
+        self.config.parser.modules["mod_ssl.c"] = None
 
         self.vh_truth = util.get_vh_truth(
             self.temp_dir, "debian_apache_2_4/multiple_vhosts")
@@ -42,8 +45,8 @@ class AutoHSTSTest(util.ApacheTest):
     @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.restart")
     @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.enable_mod")
     def test_autohsts_enable_headers_mod(self, mock_enable, _restart):
-        self.config.parser.modules.discard("headers_module")
-        self.config.parser.modules.discard("mod_header.c")
+        self.config.parser.modules.pop("headers_module", None)
+        self.config.parser.modules.pop("mod_header.c", None)
         self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
         self.assertTrue(mock_enable.called)
 

certbot-apache/tests/centos_test.py

@@ -1,7 +1,10 @@
 """Test for certbot_apache._internal.configurator for Centos overrides"""
 import unittest
 
-import mock
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock
 
 from certbot import errors
 from certbot.compat import filesystem
@@ -126,7 +129,7 @@ class MultipleVhostsTestCentOS(util.ApacheTest):
                 return mod_val
             return ""
         mock_get.side_effect = mock_get_cfg
-        self.config.parser.modules = set()
+        self.config.parser.modules = {}
         self.config.parser.variables = {}
 
         with mock.patch("certbot.util.get_os_info") as mock_osi:

certbot-apache/tests/configurator_reverter_test.py

@@ -2,7 +2,10 @@
 import shutil
 import unittest
 
-import mock
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock
 
 from certbot import errors
 import util

certbot-apache/tests/configurator_test.py

@@ -6,7 +6,10 @@ import socket
 import tempfile
 import unittest
 
-import mock
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock
 import six  # pylint: disable=unused-import  # six is used in mock.patch()
 
 from acme import challenges
@@ -341,9 +344,9 @@ class MultipleVhostsTest(util.ApacheTest):
     def test_deploy_cert_enable_new_vhost(self):
         # Create
         ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])
-        self.config.parser.modules.add("ssl_module")
-        self.config.parser.modules.add("mod_ssl.c")
-        self.config.parser.modules.add("socache_shmcb_module")
+        self.config.parser.modules["ssl_module"] = None
+        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules["socache_shmcb_module"] = None
 
         self.assertFalse(ssl_vhost.enabled)
         self.config.deploy_cert(
@@ -377,9 +380,9 @@ class MultipleVhostsTest(util.ApacheTest):
                     # pragma: no cover
 
     def test_deploy_cert(self):
-        self.config.parser.modules.add("ssl_module")
-        self.config.parser.modules.add("mod_ssl.c")
-        self.config.parser.modules.add("socache_shmcb_module")
+        self.config.parser.modules["ssl_module"] = None
+        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules["socache_shmcb_module"] = None
         # Patch _add_dummy_ssl_directives to make sure we write them correctly
         # pylint: disable=protected-access
         orig_add_dummy = self.config._add_dummy_ssl_directives
@@ -459,9 +462,9 @@ class MultipleVhostsTest(util.ApacheTest):
         method is called with an invalid vhost parameter. Currently this tests
         that a PluginError is appropriately raised when important directives
         are missing in an SSL module."""
-        self.config.parser.modules.add("ssl_module")
-        self.config.parser.modules.add("mod_ssl.c")
-        self.config.parser.modules.add("socache_shmcb_module")
+        self.config.parser.modules["ssl_module"] = None
+        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules["socache_shmcb_module"] = None
 
         def side_effect(*args):
             """Mocks case where an SSLCertificateFile directive can be found
@@ -544,7 +547,8 @@ class MultipleVhostsTest(util.ApacheTest):
                 call_found = True
         self.assertTrue(call_found)
 
-    def test_prepare_server_https(self):
+    @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules")
+    def test_prepare_server_https(self, mock_reset):
         mock_enable = mock.Mock()
         self.config.enable_mod = mock_enable
 
@@ -570,7 +574,8 @@ class MultipleVhostsTest(util.ApacheTest):
 
         self.assertEqual(mock_add_dir.call_count, 2)
 
-    def test_prepare_server_https_named_listen(self):
+    @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules")
+    def test_prepare_server_https_named_listen(self, mock_reset):
         mock_find = mock.Mock()
         mock_find.return_value = ["test1", "test2", "test3"]
         mock_get = mock.Mock()
@@ -608,7 +613,8 @@ class MultipleVhostsTest(util.ApacheTest):
         # self.config.prepare_server_https("8080", temp=True)
         # self.assertEqual(self.listens, 0)
 
-    def test_prepare_server_https_needed_listen(self):
+    @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules")
+    def test_prepare_server_https_needed_listen(self, mock_reset):
         mock_find = mock.Mock()
         mock_find.return_value = ["test1", "test2"]
         mock_get = mock.Mock()
@@ -624,8 +630,8 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.prepare_server_https("443")
         self.assertEqual(mock_add_dir.call_count, 1)
 
-    def test_prepare_server_https_mixed_listen(self):
-
+    @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules")
+    def test_prepare_server_https_mixed_listen(self, mock_reset):
         mock_find = mock.Mock()
         mock_find.return_value = ["test1", "test2"]
         mock_get = mock.Mock()
@@ -904,7 +910,7 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot_apache._internal.display_ops.select_vhost")
     @mock.patch("certbot.util.exe_exists")
     def test_enhance_unknown_vhost(self, mock_exe, mock_sel_vhost, mock_get):
-        self.config.parser.modules.add("rewrite_module")
+        self.config.parser.modules["rewrite_module"] = None
         mock_exe.return_value = True
         ssl_vh1 = obj.VirtualHost(
             "fp1", "ap1", set([obj.Addr(("*", "443"))]),
@@ -942,8 +948,8 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot.util.exe_exists")
     def test_ocsp_stapling(self, mock_exe):
         self.config.parser.update_runtime_variables = mock.Mock()
-        self.config.parser.modules.add("mod_ssl.c")
-        self.config.parser.modules.add("socache_shmcb_module")
+        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules["socache_shmcb_module"] = None
         self.config.get_version = mock.Mock(return_value=(2, 4, 7))
         mock_exe.return_value = True
 
@@ -969,8 +975,8 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot.util.exe_exists")
     def test_ocsp_stapling_twice(self, mock_exe):
         self.config.parser.update_runtime_variables = mock.Mock()
-        self.config.parser.modules.add("mod_ssl.c")
-        self.config.parser.modules.add("socache_shmcb_module")
+        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules["socache_shmcb_module"] = None
         self.config.get_version = mock.Mock(return_value=(2, 4, 7))
         mock_exe.return_value = True
 
@@ -997,8 +1003,8 @@ class MultipleVhostsTest(util.ApacheTest):
     def test_ocsp_unsupported_apache_version(self, mock_exe):
         mock_exe.return_value = True
         self.config.parser.update_runtime_variables = mock.Mock()
-        self.config.parser.modules.add("mod_ssl.c")
-        self.config.parser.modules.add("socache_shmcb_module")
+        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules["socache_shmcb_module"] = None
         self.config.get_version = mock.Mock(return_value=(2, 2, 0))
         self.config.choose_vhost("certbot.demo")
 
@@ -1021,8 +1027,8 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot.util.exe_exists")
     def test_http_header_hsts(self, mock_exe, _):
         self.config.parser.update_runtime_variables = mock.Mock()
-        self.config.parser.modules.add("mod_ssl.c")
-        self.config.parser.modules.add("headers_module")
+        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules["headers_module"] = None
         mock_exe.return_value = True
 
         # This will create an ssl vhost for certbot.demo
@@ -1042,9 +1048,9 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertEqual(len(hsts_header), 4)
 
     def test_http_header_hsts_twice(self):
-        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules["mod_ssl.c"] = None
         # skip the enable mod
-        self.config.parser.modules.add("headers_module")
+        self.config.parser.modules["headers_module"] = None
 
         # This will create an ssl vhost for encryption-example.demo
         self.config.choose_vhost("encryption-example.demo")
@@ -1060,8 +1066,8 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot.util.exe_exists")
     def test_http_header_uir(self, mock_exe, _):
         self.config.parser.update_runtime_variables = mock.Mock()
-        self.config.parser.modules.add("mod_ssl.c")
-        self.config.parser.modules.add("headers_module")
+        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules["headers_module"] = None
 
         mock_exe.return_value = True
 
@@ -1084,9 +1090,9 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertEqual(len(uir_header), 4)
 
     def test_http_header_uir_twice(self):
-        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules["mod_ssl.c"] = None
         # skip the enable mod
-        self.config.parser.modules.add("headers_module")
+        self.config.parser.modules["headers_module"] = None
 
         # This will create an ssl vhost for encryption-example.demo
         self.config.choose_vhost("encryption-example.demo")
@@ -1101,7 +1107,7 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot.util.run_script")
     @mock.patch("certbot.util.exe_exists")
     def test_redirect_well_formed_http(self, mock_exe, _):
-        self.config.parser.modules.add("rewrite_module")
+        self.config.parser.modules["rewrite_module"] = None
         self.config.parser.update_runtime_variables = mock.Mock()
         mock_exe.return_value = True
         self.config.get_version = mock.Mock(return_value=(2, 2))
@@ -1127,7 +1133,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
     def test_rewrite_rule_exists(self):
         # Skip the enable mod
-        self.config.parser.modules.add("rewrite_module")
+        self.config.parser.modules["rewrite_module"] = None
         self.config.get_version = mock.Mock(return_value=(2, 3, 9))
         self.config.parser.add_dir(
             self.vh_truth[3].path, "RewriteRule", ["Unknown"])
@@ -1136,7 +1142,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
     def test_rewrite_engine_exists(self):
         # Skip the enable mod
-        self.config.parser.modules.add("rewrite_module")
+        self.config.parser.modules["rewrite_module"] = None
         self.config.get_version = mock.Mock(return_value=(2, 3, 9))
         self.config.parser.add_dir(
             self.vh_truth[3].path, "RewriteEngine", "on")
@@ -1146,7 +1152,7 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot.util.run_script")
     @mock.patch("certbot.util.exe_exists")
     def test_redirect_with_existing_rewrite(self, mock_exe, _):
-        self.config.parser.modules.add("rewrite_module")
+        self.config.parser.modules["rewrite_module"] = None
         self.config.parser.update_runtime_variables = mock.Mock()
         mock_exe.return_value = True
         self.config.get_version = mock.Mock(return_value=(2, 2, 0))
@@ -1180,7 +1186,7 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot.util.run_script")
     @mock.patch("certbot.util.exe_exists")
     def test_redirect_with_old_https_redirection(self, mock_exe, _):
-        self.config.parser.modules.add("rewrite_module")
+        self.config.parser.modules["rewrite_module"] = None
         self.config.parser.update_runtime_variables = mock.Mock()
         mock_exe.return_value = True
         self.config.get_version = mock.Mock(return_value=(2, 2, 0))
@@ -1209,7 +1215,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
 
     def test_redirect_with_conflict(self):
-        self.config.parser.modules.add("rewrite_module")
+        self.config.parser.modules["rewrite_module"] = None
         ssl_vh = obj.VirtualHost(
             "fp", "ap", set([obj.Addr(("*", "443")),
                              obj.Addr(("zombo.com",))]),
@@ -1222,7 +1228,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
     def test_redirect_two_domains_one_vhost(self):
         # Skip the enable mod
-        self.config.parser.modules.add("rewrite_module")
+        self.config.parser.modules["rewrite_module"] = None
         self.config.get_version = mock.Mock(return_value=(2, 3, 9))
 
         # Creates ssl vhost for the domain
@@ -1237,7 +1243,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
     def test_redirect_from_previous_run(self):
         # Skip the enable mod
-        self.config.parser.modules.add("rewrite_module")
+        self.config.parser.modules["rewrite_module"] = None
         self.config.get_version = mock.Mock(return_value=(2, 3, 9))
         self.config.choose_vhost("red.blue.purple.com")
         self.config.enhance("red.blue.purple.com", "redirect")
@@ -1250,7 +1256,7 @@ class MultipleVhostsTest(util.ApacheTest):
             self.config.enhance, "green.blue.purple.com", "redirect")
 
     def test_create_own_redirect(self):
-        self.config.parser.modules.add("rewrite_module")
+        self.config.parser.modules["rewrite_module"] = None
         self.config.get_version = mock.Mock(return_value=(2, 3, 9))
         # For full testing... give names...
         self.vh_truth[1].name = "default.com"
@@ -1261,7 +1267,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertEqual(len(self.config.vhosts), 13)
 
     def test_create_own_redirect_for_old_apache_version(self):
-        self.config.parser.modules.add("rewrite_module")
+        self.config.parser.modules["rewrite_module"] = None
         self.config.get_version = mock.Mock(return_value=(2, 2))
         # For full testing... give names...
         self.vh_truth[1].name = "default.com"
@@ -1326,9 +1332,9 @@ class MultipleVhostsTest(util.ApacheTest):
     def test_deploy_cert_not_parsed_path(self):
         # Make sure that we add include to root config for vhosts when
         # handle-sites is false
-        self.config.parser.modules.add("ssl_module")
-        self.config.parser.modules.add("mod_ssl.c")
-        self.config.parser.modules.add("socache_shmcb_module")
+        self.config.parser.modules["ssl_module"] = None
+        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules["socache_shmcb_module"] = None
         tmp_path = filesystem.realpath(tempfile.mkdtemp("vhostroot"))
         filesystem.chmod(tmp_path, 0o755)
         mock_p = "certbot_apache._internal.configurator.ApacheConfigurator._get_ssl_vhost_path"
@@ -1441,8 +1447,8 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator._choose_vhosts_wildcard")
     def test_enhance_wildcard_after_install(self, mock_choose):
         # pylint: disable=protected-access
-        self.config.parser.modules.add("mod_ssl.c")
-        self.config.parser.modules.add("headers_module")
+        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules["headers_module"] = None
         self.vh_truth[3].ssl = True
         self.config._wildcard_vhosts["*.certbot.demo"] = [self.vh_truth[3]]
         self.config.enhance("*.certbot.demo", "ensure-http-header",
@@ -1453,8 +1459,8 @@ class MultipleVhostsTest(util.ApacheTest):
     def test_enhance_wildcard_no_install(self, mock_choose):
         self.vh_truth[3].ssl = True
         mock_choose.return_value = [self.vh_truth[3]]
-        self.config.parser.modules.add("mod_ssl.c")
-        self.config.parser.modules.add("headers_module")
+        self.config.parser.modules["mod_ssl.c"] = None
+        self.config.parser.modules["headers_module"] = None
         self.config.enhance("*.certbot.demo", "ensure-http-header",
                             "Upgrade-Insecure-Requests")
         self.assertTrue(mock_choose.called)
@@ -1638,7 +1644,7 @@ class MultiVhostsTest(util.ApacheTest):
 
     @certbot_util.patch_get_utility()
     def test_make_vhost_ssl_with_existing_rewrite_rule(self, mock_get_utility):
-        self.config.parser.modules.add("rewrite_module")
+        self.config.parser.modules["rewrite_module"] = None
 
         ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[4])
 
@@ -1658,7 +1664,7 @@ class MultiVhostsTest(util.ApacheTest):
 
     @certbot_util.patch_get_utility()
     def test_make_vhost_ssl_with_existing_rewrite_conds(self, mock_get_utility):
-        self.config.parser.modules.add("rewrite_module")
+        self.config.parser.modules["rewrite_module"] = None
 
         ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[3])
 
@@ -1700,7 +1706,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
                                              self.config.updated_mod_ssl_conf_digest)
 
     def _current_ssl_options_hash(self):
-        return crypto_util.sha256sum(self.config.option("MOD_SSL_CONF_SRC"))
+        return crypto_util.sha256sum(self.config.pick_apache_config())
 
     def _assert_current_file(self):
         self.assertTrue(os.path.isfile(self.config.mod_ssl_conf))
@@ -1736,7 +1742,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
             self.assertFalse(mock_logger.warning.called)
         self.assertTrue(os.path.isfile(self.config.mod_ssl_conf))
         self.assertEqual(crypto_util.sha256sum(
-            self.config.option("MOD_SSL_CONF_SRC")),
+            self.config.pick_apache_config()),
             self._current_ssl_options_hash())
         self.assertNotEqual(crypto_util.sha256sum(self.config.mod_ssl_conf),
             self._current_ssl_options_hash())
@@ -1752,19 +1758,99 @@ class InstallSslOptionsConfTest(util.ApacheTest):
                 "%s has been manually modified; updated file "
                 "saved to %s. We recommend updating %s for security purposes.")
         self.assertEqual(crypto_util.sha256sum(
-            self.config.option("MOD_SSL_CONF_SRC")),
+            self.config.pick_apache_config()),
             self._current_ssl_options_hash())
         # only print warning once
         with mock.patch("certbot.plugins.common.logger") as mock_logger:
             self._call()
             self.assertFalse(mock_logger.warning.called)
 
-    def test_current_file_hash_in_all_hashes(self):
+    def test_ssl_config_files_hash_in_all_hashes(self):
+        """
+        It is really critical that all TLS Apache config files have their SHA256 hash registered in
+        constants.ALL_SSL_OPTIONS_HASHES. Otherwise Certbot will mistakenly assume that the config
+        file has been manually edited by the user, and will refuse to update it.
+        This test ensures that all necessary hashes are present.
+        """
         from certbot_apache._internal.constants import ALL_SSL_OPTIONS_HASHES
-        self.assertTrue(self._current_ssl_options_hash() in ALL_SSL_OPTIONS_HASHES,
-            "Constants.ALL_SSL_OPTIONS_HASHES must be appended"
-            " with the sha256 hash of self.config.mod_ssl_conf when it is updated.")
+        import pkg_resources
 
+        tls_configs_dir = pkg_resources.resource_filename(
+            "certbot_apache", os.path.join("_internal", "tls_configs"))
+        all_files = [os.path.join(tls_configs_dir, name) for name in os.listdir(tls_configs_dir)
+                     if name.endswith('options-ssl-apache.conf')]
+        self.assertTrue(all_files)
+        for one_file in all_files:
+            file_hash = crypto_util.sha256sum(one_file)
+            self.assertTrue(file_hash in ALL_SSL_OPTIONS_HASHES,
+                            "Constants.ALL_SSL_OPTIONS_HASHES must be appended with the sha256 "
+                            "hash of {0} when it is updated.".format(one_file))
+
+    def test_openssl_version(self):
+        self.config._openssl_version = None
+        some_string_contents = b"""
+            SSLOpenSSLConfCmd
+            OpenSSL configuration command
+            SSLv3 not supported by this version of OpenSSL
+            '%s': invalid OpenSSL configuration command
+            OpenSSL 1.0.2g  1 Mar 2016
+            OpenSSL
+            AH02407: "SSLOpenSSLConfCmd %s %s" failed for %s
+            AH02556: "SSLOpenSSLConfCmd %s %s" applied to %s
+            OpenSSL 1.0.2g  1 Mar 2016
+            """
+        self.config.parser.modules['ssl_module'] = '/fake/path'
+        with mock.patch("certbot_apache._internal.configurator."
+            "ApacheConfigurator._open_module_file") as mock_omf:
+            mock_omf.return_value = some_string_contents
+            self.assertEqual(self.config.openssl_version(), "1.0.2g")
+
+    def test_current_version(self):
+        self.config.version = (2, 4, 10)
+        self.config._openssl_version = '1.0.2m'
+        self.assertTrue('old' in self.config.pick_apache_config())
+
+        self.config.version = (2, 4, 11)
+        self.config._openssl_version = '1.0.2m'
+        self.assertTrue('current' in self.config.pick_apache_config())
+
+        self.config._openssl_version = '1.0.2a'
+        self.assertTrue('old' in self.config.pick_apache_config())
+
+    def test_openssl_version_warns(self):
+        self.config._openssl_version = '1.0.2a'
+        self.assertEqual(self.config.openssl_version(), '1.0.2a')
+
+        self.config._openssl_version = None
+        with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log:
+            self.assertEqual(self.config.openssl_version(), None)
+            self.assertTrue("Could not find ssl_module" in mock_log.call_args[0][0])
+
+        self.config._openssl_version = None
+        self.config.parser.modules['ssl_module'] = None
+        with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log:
+            self.assertEqual(self.config.openssl_version(), None)
+            self.assertTrue("Could not find ssl_module" in mock_log.call_args[0][0])
+
+        self.config.parser.modules['ssl_module'] = "/fake/path"
+        with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log:
+            # Check that correct logger.warning was printed
+            self.assertEqual(self.config.openssl_version(), None)
+            self.assertTrue("Unable to read" in mock_log.call_args[0][0])
+
+        contents_missing_openssl = b"these contents won't match the regex"
+        with mock.patch("certbot_apache._internal.configurator."
+            "ApacheConfigurator._open_module_file") as mock_omf:
+            mock_omf.return_value = contents_missing_openssl
+            with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log:
+                # Check that correct logger.warning was printed
+                self.assertEqual(self.config.openssl_version(), None)
+                self.assertTrue("Could not find OpenSSL" in mock_log.call_args[0][0])
+
+    def test_open_module_file(self):
+        mock_open = mock.mock_open(read_data="testing 12 3")
+        with mock.patch("six.moves.builtins.open", mock_open):
+            self.assertEqual(self.config._open_module_file("/nonsense/"), "testing 12 3")
 
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot-apache/tests/debian_test.py

@@ -2,7 +2,10 @@
 import shutil
 import unittest
 
-import mock
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock
 
 from certbot import errors
 from certbot.compat import os
@@ -61,8 +64,8 @@ class MultipleVhostsTestDebian(util.ApacheTest):
     def test_deploy_cert_enable_new_vhost(self):
         # Create
         ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])
-        self.config.parser.modules.add("ssl_module")
-        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules["ssl_module"] = None
+        self.config.parser.modules["mod_ssl.c"] = None
         self.assertFalse(ssl_vhost.enabled)
         self.config.deploy_cert(
             "encryption-example.demo", "example/cert.pem", "example/key.pem",
@@ -92,8 +95,8 @@ class MultipleVhostsTestDebian(util.ApacheTest):
             self.config_path, self.vhost_path, self.config_dir,
             self.work_dir, version=(2, 4, 16))
         self.config = self.mock_deploy_cert(self.config)
-        self.config.parser.modules.add("ssl_module")
-        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules["ssl_module"] = None
+        self.config.parser.modules["mod_ssl.c"] = None
 
         # Get the default 443 vhost
         self.config.assoc["random.demo"] = self.vh_truth[1]
@@ -128,8 +131,8 @@ class MultipleVhostsTestDebian(util.ApacheTest):
             self.config_path, self.vhost_path, self.config_dir,
             self.work_dir, version=(2, 4, 16))
         self.config = self.mock_deploy_cert(self.config)
-        self.config.parser.modules.add("ssl_module")
-        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules["ssl_module"] = None
+        self.config.parser.modules["mod_ssl.c"] = None
 
         # Get the default 443 vhost
         self.config.assoc["random.demo"] = self.vh_truth[1]
@@ -143,8 +146,8 @@ class MultipleVhostsTestDebian(util.ApacheTest):
             self.config_path, self.vhost_path, self.config_dir,
             self.work_dir, version=(2, 4, 7))
         self.config = self.mock_deploy_cert(self.config)
-        self.config.parser.modules.add("ssl_module")
-        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules["ssl_module"] = None
+        self.config.parser.modules["mod_ssl.c"] = None
 
         # Get the default 443 vhost
         self.config.assoc["random.demo"] = self.vh_truth[1]
@@ -157,7 +160,7 @@ class MultipleVhostsTestDebian(util.ApacheTest):
     @mock.patch("certbot.util.exe_exists")
     def test_ocsp_stapling_enable_mod(self, mock_exe, _):
         self.config.parser.update_runtime_variables = mock.Mock()
-        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules["mod_ssl.c"] = None
         self.config.get_version = mock.Mock(return_value=(2, 4, 7))
         mock_exe.return_value = True
         # This will create an ssl vhost for certbot.demo
@@ -169,7 +172,7 @@ class MultipleVhostsTestDebian(util.ApacheTest):
     @mock.patch("certbot.util.exe_exists")
     def test_ensure_http_header_enable_mod(self, mock_exe, _):
         self.config.parser.update_runtime_variables = mock.Mock()
-        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules["mod_ssl.c"] = None
         mock_exe.return_value = True
 
         # This will create an ssl vhost for certbot.demo

certbot-apache/tests/display_ops_test.py

@@ -1,7 +1,10 @@
 """Test certbot_apache._internal.display_ops."""
 import unittest
 
-import mock
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock
 
 from certbot import errors
 from certbot.display import util as display_util

certbot-apache/tests/dualnode_test.py

@@ -1,7 +1,10 @@
 """Tests for DualParserNode implementation"""
 import unittest
 
-import mock
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock
 
 from certbot_apache._internal import assertions
 from certbot_apache._internal import augeasparser

certbot-apache/tests/entrypoint_test.py

@@ -1,7 +1,10 @@
 """Test for certbot_apache._internal.entrypoint for override class resolution"""
 import unittest
 
-import mock
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock
 
 from certbot_apache._internal import configurator
 from certbot_apache._internal import entrypoint

certbot-apache/tests/fedora_test.py

@@ -1,7 +1,10 @@
 """Test for certbot_apache._internal.configurator for Fedora 29+ overrides"""
 import unittest
 
-import mock
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock
 
 from certbot import errors
 from certbot.compat import filesystem
@@ -120,7 +123,7 @@ class MultipleVhostsTestFedora(util.ApacheTest):
                 return mod_val
             return ""
         mock_get.side_effect = mock_get_cfg
-        self.config.parser.modules = set()
+        self.config.parser.modules = {}
         self.config.parser.variables = {}
 
         with mock.patch("certbot.util.get_os_info") as mock_osi:

certbot-apache/tests/gentoo_test.py

@@ -1,7 +1,10 @@
 """Test for certbot_apache._internal.configurator for Gentoo overrides"""
 import unittest
 
-import mock
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock
 
 from certbot import errors
 from certbot.compat import filesystem
@@ -117,7 +120,7 @@ class MultipleVhostsTestGentoo(util.ApacheTest):
                 return mod_val
             return None  # pragma: no cover
         mock_get.side_effect = mock_get_cfg
-        self.config.parser.modules = set()
+        self.config.parser.modules = {}
 
         with mock.patch("certbot.util.get_os_info") as mock_osi:
             # Make sure we have the have the Gentoo httpd constants

certbot-apache/tests/http_01_test.py

@@ -1,10 +1,13 @@
 """Test for certbot_apache._internal.http_01."""
 import unittest
+import errno
 
-import mock
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock
 
 from acme import challenges
-from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
 from certbot import achallenges
 from certbot import errors
 from certbot.compat import filesystem
@@ -40,8 +43,8 @@ class ApacheHttp01Test(util.ApacheTest):
 
         modules = ["ssl", "rewrite", "authz_core", "authz_host"]
         for mod in modules:
-            self.config.parser.modules.add("mod_{0}.c".format(mod))
-            self.config.parser.modules.add(mod + "_module")
+            self.config.parser.modules["mod_{0}.c".format(mod)] = None
+            self.config.parser.modules[mod + "_module"] = None
 
         from certbot_apache._internal.http_01 import ApacheHttp01
         self.http = ApacheHttp01(self.config)
@@ -52,24 +55,24 @@ class ApacheHttp01Test(util.ApacheTest):
     @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.enable_mod")
     def test_enable_modules_apache_2_2(self, mock_enmod):
         self.config.version = (2, 2)
-        self.config.parser.modules.remove("authz_host_module")
-        self.config.parser.modules.remove("mod_authz_host.c")
+        del self.config.parser.modules["authz_host_module"]
+        del self.config.parser.modules["mod_authz_host.c"]
 
         enmod_calls = self.common_enable_modules_test(mock_enmod)
         self.assertEqual(enmod_calls[0][0][0], "authz_host")
 
     @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.enable_mod")
     def test_enable_modules_apache_2_4(self, mock_enmod):
-        self.config.parser.modules.remove("authz_core_module")
-        self.config.parser.modules.remove("mod_authz_core.c")
+        del self.config.parser.modules["authz_core_module"]
+        del self.config.parser.modules["mod_authz_host.c"]
 
         enmod_calls = self.common_enable_modules_test(mock_enmod)
         self.assertEqual(enmod_calls[0][0][0], "authz_core")
 
     def common_enable_modules_test(self, mock_enmod):
         """Tests enabling mod_rewrite and other modules."""
-        self.config.parser.modules.remove("rewrite_module")
-        self.config.parser.modules.remove("mod_rewrite.c")
+        del self.config.parser.modules["rewrite_module"]
+        del self.config.parser.modules["mod_rewrite.c"]
 
         self.http.prepare_http01_modules()
 
@@ -197,6 +200,12 @@ class ApacheHttp01Test(util.ApacheTest):
 
         self.assertTrue(os.path.exists(challenge_dir))
 
+    @mock.patch("certbot_apache._internal.http_01.filesystem.makedirs")
+    def test_failed_makedirs(self, mock_makedirs):
+        mock_makedirs.side_effect = OSError(errno.EACCES, "msg")
+        self.http.add_chall(self.achalls[0])
+        self.assertRaises(errors.PluginError, self.http.perform)
+
     def _test_challenge_conf(self):
         with open(self.http.challenge_conf_pre) as f:
             pre_conf_contents = f.read()

certbot-apache/tests/parser_test.py

@@ -2,7 +2,10 @@
 import shutil
 import unittest
 
-import mock
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock
 
 from certbot import errors
 from certbot.compat import os
@@ -114,7 +117,7 @@ class BasicParserTest(util.ParserTest):
         """
         from certbot_apache._internal.parser import get_aug_path
         # This makes sure that find_dir will work
-        self.parser.modules.add("mod_ssl.c")
+        self.parser.modules["mod_ssl.c"] = "/fake/path"
 
         self.parser.add_dir_to_ifmodssl(
             get_aug_path(self.parser.loc["default"]),
@@ -128,7 +131,7 @@ class BasicParserTest(util.ParserTest):
     def test_add_dir_to_ifmodssl_multiple(self):
         from certbot_apache._internal.parser import get_aug_path
         # This makes sure that find_dir will work
-        self.parser.modules.add("mod_ssl.c")
+        self.parser.modules["mod_ssl.c"] = "/fake/path"
 
         self.parser.add_dir_to_ifmodssl(
             get_aug_path(self.parser.loc["default"]),
@@ -260,7 +263,7 @@ class BasicParserTest(util.ParserTest):
         expected_vars = {"TEST": "", "U_MICH": "", "TLS": "443",
                          "example_path": "Documents/path"}
 
-        self.parser.modules = set()
+        self.parser.modules = {}
         with mock.patch(
             "certbot_apache._internal.parser.ApacheParser.parse_file") as mock_parse:
             self.parser.update_runtime_variables()
@@ -282,7 +285,7 @@ class BasicParserTest(util.ParserTest):
                  os.path.dirname(self.parser.loc["root"]))
 
         mock_cfg.return_value = inc_val
-        self.parser.modules = set()
+        self.parser.modules = {}
 
         with mock.patch(
             "certbot_apache._internal.parser.ApacheParser.parse_file") as mock_parse:

certbot-apache/tests/parsernode_configurator_test.py

@@ -1,11 +1,21 @@
 """Tests for ApacheConfigurator for AugeasParserNode classes"""
 import unittest
 
-import mock
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock
 
 import util
 
+try:
+    import apacheconfig
+    HAS_APACHECONFIG = True
+except ImportError:  # pragma: no cover
+    HAS_APACHECONFIG = False
 
+
+@unittest.skipIf(not HAS_APACHECONFIG, reason='Tests require apacheconfig dependency')
 class ConfiguratorParserNodeTest(util.ApacheTest):  # pylint: disable=too-many-public-methods
     """Test AugeasParserNode using available test configurations"""
 

certbot-apache/tests/util.py

@@ -5,7 +5,10 @@ import unittest
 
 import augeas
 import josepy as jose
-import mock
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock
 import zope.component
 
 from certbot.compat import os
@@ -85,7 +88,8 @@ def get_apache_configurator(
         config_dir, work_dir, version=(2, 4, 7),
         os_info="generic",
         conf_vhost_path=None,
-        use_parsernode=False):
+        use_parsernode=False,
+        openssl_version="1.1.1a"):
     """Create an Apache Configurator with the specified options.
 
     :param conf: Function that returns binary paths. self.conf in Configurator
@@ -118,7 +122,8 @@ def get_apache_configurator(
                     except KeyError:
                         config_class = configurator.ApacheConfigurator
                     config = config_class(config=mock_le_config, name="apache",
-                                          version=version, use_parsernode=use_parsernode)
+                                          version=version, use_parsernode=use_parsernode,
+                                          openssl_version=openssl_version)
                     if not conf_vhost_path:
                         config_class.OS_DEFAULTS["vhost_root"] = vhost_path
                     else:

certbot-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="1.2.0"
+LE_AUTO_VERSION="1.3.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -1540,18 +1540,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==1.2.0 \
-    --hash=sha256:e25c17125c00b3398c8e9b9d54ef473c0e8f5aff53389f313a51b06cf472d335 \
-    --hash=sha256:95dcbae085f8e4eb18442fe7b12994b08964a9a6e8e352e556cdb4a8a625373c
-acme==1.2.0 \
-    --hash=sha256:284d22fde75687a8ea72d737cac6bcbdc91f3c796221aa25378b8732ba6f6875 \
-    --hash=sha256:0630c740d49bda945e97bd35fc8d6f02d082c8cb9e18f8fec0dbb3d395ac26ab
-certbot-apache==1.2.0 \
-    --hash=sha256:3f7493918353d3bd6067d446a2cf263e03831c4c10ec685b83d644b47767090d \
-    --hash=sha256:b46e9def272103a68108e48bf7e410ea46801529b1ea6954f6506b14dd9df9b3
-certbot-nginx==1.2.0 \
-    --hash=sha256:efd32a2b32f2439279da446b6bf67684f591f289323c5f494ebfd86a566a28fd \
-    --hash=sha256:6fd7cf4f2545ad66e57000343227df9ccccaf04420e835e05cb3250fac1fa6db
+certbot==1.3.0 \
+    --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \
+    --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8
+acme==1.3.0 \
+    --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \
+    --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d
+certbot-apache==1.3.0 \
+    --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \
+    --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52
+certbot-nginx==1.3.0 \
+    --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \
+    --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------

certbot-ci/certbot_integration_tests/certbot_tests/test_main.py

@@ -595,6 +595,23 @@ def test_ocsp_status_live(context):
     assert output.count('REVOKED') == 1, 'Expected {0} to be REVOKED'.format(cert)
 
 
+def test_ocsp_renew(context):
+    """Test that revoked certificates are renewed."""
+    # Obtain a certificate
+    certname = context.get_domain('ocsp-renew')
+    context.certbot(['--domains', certname])
+
+    # Test that "certbot renew" does not renew the certificate
+    assert_cert_count_for_lineage(context.config_dir, certname, 1)
+    context.certbot(['renew'], force_renew=False)
+    assert_cert_count_for_lineage(context.config_dir, certname, 1)
+
+    # Revoke the certificate and test that it does renew the certificate
+    context.certbot(['revoke', '--cert-name', certname, '--no-delete-after-revoke'])
+    context.certbot(['renew'], force_renew=False)
+    assert_cert_count_for_lineage(context.config_dir, certname, 2)
+
+
 def test_dry_run_deactivate_authzs(context):
     """Test that Certbot deactivates authorizations when performing a dry run"""
 

certbot-ci/certbot_integration_tests/utils/acme_server.py

@@ -38,7 +38,7 @@ class ACMEServer(object):
         :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble)
         :param list nodes: list of node names that will be setup by pytest xdist
         :param bool http_proxy: if False do not start the HTTP proxy
-        :param bool stdout: if True stream subprocesses stdout to standard stdout
+        :param bool stdout: if True stream all subprocesses stdout to standard stdout
         """
         self._construct_acme_xdist(acme_server, nodes)
 
@@ -131,7 +131,7 @@ class ACMEServer(object):
         environ['PEBBLE_AUTHZREUSE'] = '100'
 
         self._launch_process(
-            [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053'],
+            [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053', '-strict'],
             env=environ)
 
         self._launch_process(
@@ -165,17 +165,24 @@ class ACMEServer(object):
         os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'),
                   join(instance_path, 'test/rate-limit-policies.yml'))
 
-        # Launch the Boulder server
-        self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path)
+        try:
+            # Launch the Boulder server
+            self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path)
 
-        # Wait for the ACME CA server to be up.
-        print('=> Waiting for boulder instance to respond...')
-        misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=240)
+            # Wait for the ACME CA server to be up.
+            print('=> Waiting for boulder instance to respond...')
+            misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=300)
 
-        # Configure challtestsrv to answer any A record request with ip of the docker host.
-        response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT),
-                                 json={'ip': '10.77.77.1'})
-        response.raise_for_status()
+            # Configure challtestsrv to answer any A record request with ip of the docker host.
+            response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT),
+                                     json={'ip': '10.77.77.1'})
+            response.raise_for_status()
+        except BaseException:
+            # If we failed to set up boulder, print its logs.
+            print('=> Boulder setup failed. Boulder logs are:')
+            process = self._launch_process(['docker-compose', 'logs'], cwd=instance_path, force_stderr=True)
+            process.wait()
+            raise
 
         print('=> Finished boulder instance deployment.')
 
@@ -188,11 +195,12 @@ class ACMEServer(object):
         self._launch_process(command)
         print('=> Finished configuring the HTTP proxy.')
 
-    def _launch_process(self, command, cwd=os.getcwd(), env=None):
+    def _launch_process(self, command, cwd=os.getcwd(), env=None, force_stderr=False):
         """Launch silently a subprocess OS command"""
         if not env:
             env = os.environ
-        process = subprocess.Popen(command, stdout=self._stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env)
+        stdout = sys.stderr if force_stderr else self._stdout
+        process = subprocess.Popen(command, stdout=stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env)
         self._processes.append(process)
         return process
 

certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py

@@ -7,7 +7,7 @@ import requests
 
 from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT
 
-PEBBLE_VERSION = 'v2.2.1'
+PEBBLE_VERSION = 'v2.3.0'
 ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets')
 
 

certbot-compatibility-test/Dockerfile

@@ -31,7 +31,7 @@ COPY certbot-nginx /opt/certbot/src/certbot-nginx/
 COPY certbot-compatibility-test /opt/certbot/src/certbot-compatibility-test/
 COPY tools /opt/certbot/src/tools
 
-RUN VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages -p python2 /opt/certbot/venv && \
+RUN VIRTUALENV_NO_DOWNLOAD=1 virtualenv -p python2 /opt/certbot/venv && \
     /opt/certbot/venv/bin/pip install -U setuptools && \
     /opt/certbot/venv/bin/pip install -U pip
 ENV PATH /opt/certbot/venv/bin:$PATH

certbot-compatibility-test/setup.py

@@ -3,7 +3,7 @@ import sys
 from setuptools import find_packages
 from setuptools import setup
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 install_requires = [
     'certbot',

certbot-dns-cloudflare/setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-cloudxns/setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-digitalocean/setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-dnsimple/setup.py

@@ -5,7 +5,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-dnsmadeeasy/setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-gehirn/setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-google/setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-linode/setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-luadns/setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-nsone/setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-ovh/setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-rfc2136/setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-route53/setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-sakuracloud/setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-nginx/certbot_nginx/_internal/configurator.py

@@ -1,5 +1,4 @@
 """Nginx Configuration"""
-# https://github.com/PyCQA/pylint/issues/73
 from distutils.version import LooseVersion
 import logging
 import re

certbot-nginx/certbot_nginx/_internal/parser.py

@@ -2,6 +2,7 @@
 import copy
 import functools
 import glob
+import io
 import logging
 import re
 
@@ -205,12 +206,16 @@ class NginxParser(object):
             if item in self.parsed and not override:
                 continue
             try:
-                with open(item) as _file:
+                with io.open(item, "r", encoding="utf-8") as _file:
                     parsed = nginxparser.load(_file)
                     self.parsed[item] = parsed
                     trees.append(parsed)
             except IOError:
                 logger.warning("Could not open file: %s", item)
+            except UnicodeDecodeError:
+                logger.warning("Could not read file: %s due to invalid "
+                               "character. Only UTF-8 encoding is "
+                               "supported.", item)
             except pyparsing.ParseException as err:
                 logger.debug("Could not parse file: %s due to %s", item, err)
         return trees
@@ -414,10 +419,13 @@ class NginxParser(object):
 def _parse_ssl_options(ssl_options):
     if ssl_options is not None:
         try:
-            with open(ssl_options) as _file:
+            with io.open(ssl_options, "r", encoding="utf-8") as _file:
                 return nginxparser.load(_file)
         except IOError:
             logger.warning("Missing NGINX TLS options file: %s", ssl_options)
+        except UnicodeDecodeError:
+            logger.warning("Could not read file: %s due to invalid character. "
+                           "Only UTF-8 encoding is supported.", ssl_options)
         except pyparsing.ParseBaseException as err:
             logger.debug("Could not parse file: %s due to %s", ssl_options, err)
     return []

certbot-nginx/local-oldest-requirements.txt

@@ -1,3 +1,3 @@
 # Remember to update setup.py to match the package versions below.
-acme[dev]==1.0.0
-certbot[dev]==1.1.0
+-e acme[dev]
+-e certbot[dev]

certbot-nginx/setup.py

@@ -4,13 +4,13 @@ from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
 
-version = '1.3.0.dev0'
+version = '1.4.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    'acme>=1.0.0',
-    'certbot>=1.1.0',
+    'acme>=1.4.0.dev0',
+    'certbot>=1.4.0.dev0',
     'mock',
     'PyOpenSSL',
     'pyparsing>=1.5.5',  # Python3 support

certbot-nginx/tests/parser_test.py

@@ -4,7 +4,6 @@ import re
 import shutil
 import unittest
 
-from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
 from certbot import errors
 from certbot.compat import os
 from certbot_nginx._internal import nginxparser
@@ -482,7 +481,43 @@ class NginxParserTest(util.NginxTest):
                 called = True
         self.assertTrue(called)
 
+    def test_valid_unicode_characters(self):
+        nparser = parser.NginxParser(self.config_path)
+        path = nparser.abs_path('valid_unicode_comments.conf')
+        parsed = nparser._parse_files(path)  # pylint: disable=protected-access
+        self.assertEqual(['server'], parsed[0][2][0])
+        self.assertEqual(['listen', '80'], parsed[0][2][1][3])
 
+    def test_invalid_unicode_characters(self):
+        with self.assertLogs() as log:
+            nparser = parser.NginxParser(self.config_path)
+            path = nparser.abs_path('invalid_unicode_comments.conf')
+            parsed = nparser._parse_files(path)  # pylint: disable=protected-access
+
+        self.assertEqual([], parsed)
+        self.assertTrue(any(
+            ('invalid character' in output) and ('UTF-8' in output)
+            for output in log.output
+        ))
+
+    def test_valid_unicode_characters_in_ssl_options(self):
+        nparser = parser.NginxParser(self.config_path)
+        path = nparser.abs_path('valid_unicode_comments.conf')
+        parsed = parser._parse_ssl_options(path)  # pylint: disable=protected-access
+        self.assertEqual(['server'], parsed[2][0])
+        self.assertEqual(['listen', '80'], parsed[2][1][3])
+
+    def test_invalid_unicode_characters_in_ssl_options(self):
+        with self.assertLogs() as log:
+            nparser = parser.NginxParser(self.config_path)
+            path = nparser.abs_path('invalid_unicode_comments.conf')
+            parsed = parser._parse_ssl_options(path)  # pylint: disable=protected-access
+
+        self.assertEqual([], parsed)
+        self.assertTrue(any(
+            ('invalid character' in output) and ('UTF-8' in output)
+            for output in log.output
+        ))
 
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot-nginx/tests/test_log_util.py

@@ -0,0 +1,125 @@
+"""Backport for `TestCase.assertLogs()`.
+
+Most of the idea and code are from CPython implementation.
+https://github.com/python/cpython/blob/b76518d43fb82ed9e5d27025d18c90a23d525c90/Lib/unittest/case.py
+"""
+import logging
+import collections
+
+__all__ = ['AssertLogsMixin']
+
+LoggingWatcher = collections.namedtuple('LoggingWatcher', ['records', 'output'])
+
+
+class CapturingHandler(logging.Handler):
+    """
+    A logging handler capturing all (raw and formatted) logging output.
+    """
+
+    def __init__(self):
+        super(CapturingHandler, self).__init__()
+        self.watcher = LoggingWatcher([], [])
+
+    def flush(self):
+        pass
+
+    def emit(self, record):
+        self.watcher.records.append(record)
+        self.watcher.output.append(self.format(record))
+
+
+
+class AssertLogsContext(object):
+    """
+    A context manager used to implement `TestCase.assertLogs()`.
+    """
+
+    LOGGING_FORMAT = '%(levelname)s:%(name)s:%(message)s'
+
+    def __init__(self, test_case, logger_name, level):
+        self.test_case = test_case
+
+        self.logger_name = logger_name
+        self.logger_states = None
+        self.logger = None
+
+        if level:
+            # pylint: disable=protected-access,no-member
+            try:
+                # In Python 3.x
+                name_to_level = logging._nameToLevel  # type: ignore
+            except AttributeError:
+                # In Python 2.7
+                name_to_level = logging._levelNames  # type: ignore
+
+            self.level = name_to_level.get(level, level)
+        else:
+            self.level = logging.INFO
+
+        self.watcher = None
+
+    def _save_logger_states(self):
+        self.logger_states = (self.logger.handlers[:], self.logger.level, self.logger.propagate)
+
+    def _restore_logger_states(self):
+        self.logger.handlers, self.logger.level, self.logger.propagate = self.logger_states
+
+    def __enter__(self):
+        if isinstance(self.logger_name, logging.Logger):
+            self.logger = self.logger_name
+        else:
+            self.logger = logging.getLogger(self.logger_name)
+
+        formatter = logging.Formatter(self.LOGGING_FORMAT)
+
+        handler = CapturingHandler()
+        handler.setFormatter(formatter)
+
+        self._save_logger_states()
+        self.logger.handlers = [handler]
+        self.logger.setLevel(self.level)
+        self.logger.propagate = False
+
+        self.watcher = handler.watcher
+        return handler.watcher
+
+    def __exit__(self, exc_type, exc_value, tb):
+        self._restore_logger_states()
+
+        if exc_type is not None:
+            # let unexpected exceptions pass through
+            return
+
+        if not self.watcher.records:
+            self._raiseFailure(
+                "no logs of level {} or higher triggered on {}"
+                .format(logging.getLevelName(self.level), self.logger.name))
+
+    def _raiseFailure(self, message):
+        message = self.test_case._formatMessage(None, message)  # pylint: disable=protected-access
+        raise self.test_case.failureException(message)
+
+
+class AssertLogsMixin(object):
+    """
+    A mixin that implements `TestCase.assertLogs()`.
+    """
+
+    def assertLogs(self, logger=None, level=None):
+        """Fail unless a log message of level *level* or higher is emitted
+        on *logger_name* or its children.  If omitted, *level* defaults to
+        INFO and *logger* defaults to the root logger.
+        This method must be used as a context manager, and will yield
+        a recording object with two attributes: `output` and `records`.
+        At the end of the context manager, the `output` attribute will
+        be a list of the matching formatted log messages and the
+        `records` attribute will be a list of the corresponding LogRecord
+        objects.
+        Example::
+            with self.assertLogs('foo', level='INFO') as cm:
+                logging.getLogger('foo').info('first message')
+                logging.getLogger('foo.bar').error('second message')
+            self.assertEqual(cm.output, ['INFO:foo:first message',
+                                         'ERROR:foo.bar:second message'])
+        """
+        return AssertLogsContext(self, logger, level)

certbot-nginx/tests/test_util.py

@@ -14,9 +14,10 @@ from certbot.plugins import common
 from certbot.tests import util as test_util
 from certbot_nginx._internal import configurator
 from certbot_nginx._internal import nginxparser
+import test_log_util
 
 
-class NginxTest(test_util.ConfigTestCase):
+class NginxTest(test_log_util.AssertLogsMixin, test_util.ConfigTestCase):
 
     def setUp(self):
         super(NginxTest, self).setUp()

certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf

@@ -0,0 +1,7 @@
+# This configuration file is saved with EUC-KR (a.k.a. cp949) encoding,
+# including some Korean letters.
+
+server {
+    # <20>ȳ<EFBFBD><C8B3>ϼ<EFBFBD><CFBC><EFBFBD>. 80<38><30> <20><>Ʈ<EFBFBD><C6AE><EFBFBD><EFBFBD> <20><>û<EFBFBD><C3BB> <20><><EFBFBD>ٸ<EFBFBD><D9B8><EFBFBD>.
+    listen	80;
+}

certbot-nginx/tests/testdata/etc_nginx/valid_unicode_comments.conf

@@ -0,0 +1,9 @@
+# This configuration file is saved with valid UTF-8 encoding,
+# including some CJK alphabets.
+
+server {
+    # 안녕하세요. 80번 포트에서 요청을 기다린다.
+    # こんにちは。８０番ポートからリクエストを待つ。
+    # 你好。等待端口80上的请求。
+    listen	80;
+}

certbot/CHANGELOG.md

@@ -2,24 +2,59 @@
 
 Certbot adheres to [Semantic Versioning](https://semver.org/).
 
-## 1.3.0 - master
+## 1.4.0 - master
 
 ### Added
 
-* Added certbot.ocsp Certbot's API. The certbot.ocsp module can be used to 
-  determine the OCSP status of certificates.
-* Don't verify the existing certificate in HTTP01Response.simple_verify, for 
-  compatibility with the real-world ACME challenge checks.
+* Turn off session tickets for apache plugin by default when appropriate.
+* Added serial number of certificate to the output of `certbot certificates`
+* Expose two new environment variables in the authenticator and cleanup scripts used by
+  the `manual` plugin: `CERTBOT_REMAINING_CHALLENGES` is equal to the number of challenges
+  remaining after the current challenge, `CERTBOT_ALL_DOMAINS` is a comma-separated list
+  of all domains challenged for the current certificate.
+* Added TLS-ALPN-01 challenge support in the `acme` library. Support of this
+  challenge in the Certbot client is planned to be added in a future release.
+* Added minimal proxy support for OCSP verification.
 
 ### Changed
 
+* Stop asking interactively if the user would like to add a redirect.
+
+### Fixed
+
+* When using an RFC 8555 compliant endpoint, the `acme` library no longer sends the
+  `resource` field in any requests or the `type` field when responding to challenges.
+* Fix nginx plugin crash when non-ASCII configuration file is being read (instead,
+  the user will be warned that UTF-8 must be used).
+* Fix hanging OCSP queries during revocation checking - added a 10 second timeout.
+* Standalone servers now have a default socket timeout of 30 seconds, fixing
+  cases where an idle connection can cause the standalone plugin to hang.
+* Parsing of the RFC 8555 application/pem-certificate-chain now tolerates CRLF line
+  endings. This should fix interoperability with Buypass' services.
+
+More details about these changes can be found on our GitHub repo.
+
+## 1.3.0 - 2020-03-03
+
+### Added
+
+* Added certbot.ocsp Certbot's API. The certbot.ocsp module can be used to
+  determine the OCSP status of certificates.
+* Don't verify the existing certificate in HTTP01Response.simple_verify, for
+  compatibility with the real-world ACME challenge checks.
+* Added support for `$hostname` in nginx `server_name` directive
+
+### Changed
+
+* Certbot will now renew certificates early if they have been revoked according
+  to OCSP.
 * Fix acme module warnings when response Content-Type includes params (e.g. charset).
-* Fixed issue where webroot plugin would incorrectly raise `Read-only file system` 
+* Fixed issue where webroot plugin would incorrectly raise `Read-only file system`
   error when creating challenge directories (issue #7165).
 
 ### Fixed
 
-*
+* Fix Apache plugin to use less restrictive umask for making the challenge directory when a restrictive umask was set when certbot was started.
 
 More details about these changes can be found on our GitHub repo.
 
@@ -28,7 +63,6 @@ More details about these changes can be found on our GitHub repo.
 ### Added
 
 * Added support for Cloudflare's limited-scope API Tokens
-* Added support for `$hostname` in nginx `server_name` directive
 
 ### Changed
 

certbot/README.rst

@@ -71,16 +71,12 @@ ACME spec: http://ietf-wg-acme.github.io/acme/
 
 ACME working area in github: https://github.com/ietf-wg-acme/acme
 
-|build-status| |coverage| |container|
+|build-status| |container|
 
 .. |build-status| image:: https://travis-ci.com/certbot/certbot.svg?branch=master
    :target: https://travis-ci.com/certbot/certbot
    :alt: Travis CI status
 
-.. |coverage| image:: https://codecov.io/gh/certbot/certbot/branch/master/graph/badge.svg
-   :target: https://codecov.io/gh/certbot/certbot
-   :alt: Coverage status
-
 .. |container| image:: https://quay.io/repository/letsencrypt/letsencrypt/status
    :target: https://quay.io/repository/letsencrypt/letsencrypt
    :alt: Docker Repository on Quay.io

certbot/certbot/__init__.py

@@ -1,4 +1,4 @@
 """Certbot client."""
 
 # version number like 1.2.3a0, must have at least 2 parts, like 1.2
-__version__ = '1.3.0.dev0'
+__version__ = '1.4.0.dev0'

certbot/certbot/_internal/cert_manager.py

@@ -276,12 +276,15 @@ def human_readable_cert_info(config, cert, skip_filter_checks=False):
             status = "VALID: {0} days".format(diff.days)
 
     valid_string = "{0} ({1})".format(cert.target_expiry, status)
+    serial = format(crypto_util.get_serial_from_cert(cert.cert_path), 'x')
     certinfo.append("  Certificate Name: {0}\n"
-                    "    Domains: {1}\n"
-                    "    Expiry Date: {2}\n"
-                    "    Certificate Path: {3}\n"
-                    "    Private Key Path: {4}".format(
+                    "    Serial Number: {1}\n"
+                    "    Domains: {2}\n"
+                    "    Expiry Date: {3}\n"
+                    "    Certificate Path: {4}\n"
+                    "    Private Key Path: {5}".format(
                          cert.lineagename,
+                         serial,
                          " ".join(cert.names()),
                          valid_string,
                          cert.fullchain,

certbot/certbot/_internal/cli/__init__.py

@@ -321,12 +321,14 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False):
         "--redirect", action="store_true", dest="redirect",
         default=flag_default("redirect"),
         help="Automatically redirect all HTTP traffic to HTTPS for the newly "
-             "authenticated vhost. (default: Ask)")
+             "authenticated vhost. (default: redirect enabled for install and run, "
+             "disabled for enhance)")
     helpful.add(
         "security", "--no-redirect", action="store_false", dest="redirect",
         default=flag_default("redirect"),
         help="Do not automatically redirect all HTTP traffic to HTTPS for the newly "
-             "authenticated vhost. (default: Ask)")
+             "authenticated vhost. (default: redirect enabled for install and run, "
+             "disabled for enhance)")
     helpful.add(
         ["security", "enhance"],
         "--hsts", action="store_true", dest="hsts", default=flag_default("hsts"),

certbot/certbot/_internal/cli/helpful.py

@@ -11,9 +11,7 @@ import zope.interface
 
 from zope.interface import interfaces as zope_interfaces
 
-# pylint: disable=unused-import, no-name-in-module
-from acme.magic_typing import Any, Dict, Optional
-# pylint: enable=unused-import, no-name-in-module
+from acme.magic_typing import Any, Dict
 
 from certbot import crypto_util
 from certbot import errors

certbot/certbot/_internal/client.py

@@ -28,7 +28,6 @@ from certbot._internal import constants
 from certbot._internal import eff
 from certbot._internal import error_handler
 from certbot._internal import storage
-from certbot._internal.display import enhancements
 from certbot._internal.plugins import selection as plugin_selection
 from certbot.compat import os
 from certbot.display import ops as display_ops
@@ -521,12 +520,13 @@ class Client(object):
             # sites may have been enabled / final cleanup
             self.installer.restart()
 
-    def enhance_config(self, domains, chain_path, ask_redirect=True):
+    def enhance_config(self, domains, chain_path, redirect_default=True):
         """Enhance the configuration.
 
         :param list domains: list of domains to configure
         :param chain_path: chain file path
         :type chain_path: `str` or `None`
+        :param redirect_default: boolean value that the "redirect" flag should default to
 
         :raises .errors.Error: if no installer is specified in the
             client.
@@ -548,14 +548,8 @@ class Client(object):
         for config_name, enhancement_name, option in enhancement_info:
             config_value = getattr(self.config, config_name)
             if enhancement_name in supported:
-                if ask_redirect:
-                    if config_name == "redirect" and config_value is None:
-                        config_value = enhancements.ask(enhancement_name)
-                        if not config_value:
-                            logger.warning("Future versions of Certbot will automatically "
-                                "configure the webserver so that all requests redirect to secure "
-                                "HTTPS access. You can control this behavior and disable this "
-                                "warning with the --redirect and --no-redirect flags.")
+                if config_name == "redirect" and config_value is None:
+                    config_value = redirect_default
                 if config_value:
                     self.apply_enhancement(domains, enhancement_name, option)
                     enhanced = True

certbot/certbot/_internal/display/enhancements.py

@@ -1,63 +0,0 @@
-"""Certbot Enhancement Display"""
-import logging
-
-import zope.component
-
-from certbot import errors
-from certbot import interfaces
-from certbot.display import util as display_util
-
-logger = logging.getLogger(__name__)
-
-# Define a helper function to avoid verbose code
-util = zope.component.getUtility
-
-
-def ask(enhancement):
-    """Display the enhancement to the user.
-
-    :param str enhancement: One of the
-        :const:`~certbot.plugins.enhancements.ENHANCEMENTS` enhancements
-
-    :returns: True if feature is desired, False otherwise
-    :rtype: bool
-
-    :raises .errors.Error: if the enhancement provided is not supported
-
-    """
-    try:
-        # Call the appropriate function based on the enhancement
-        return DISPATCH[enhancement]()
-    except KeyError:
-        logger.error("Unsupported enhancement given to ask(): %s", enhancement)
-        raise errors.Error("Unsupported Enhancement")
-
-
-def redirect_by_default():
-    """Determines whether the user would like to redirect to HTTPS.
-
-    :returns: True if redirect is desired, False otherwise
-    :rtype: bool
-
-    """
-    choices = [
-        ("No redirect", "Make no further changes to the webserver configuration."),
-        ("Redirect", "Make all requests redirect to secure HTTPS access. "
-            "Choose this for new sites, or if you're confident your site works on HTTPS. "
-            "You can undo this change by editing your web server's configuration."),
-    ]
-
-    code, selection = util(interfaces.IDisplay).menu(
-        "Please choose whether or not to redirect HTTP traffic to HTTPS, removing HTTP access.",
-        choices, default=1,
-        cli_flag="--redirect / --no-redirect", force_interactive=True)
-
-    if code != display_util.OK:
-        return False
-
-    return selection == 1
-
-
-DISPATCH = {
-    "redirect": redirect_by_default
-}

certbot/certbot/_internal/main.py

@@ -394,7 +394,7 @@ def _find_domains_or_certname(config, installer, question=None):
     :param installer: Installer object
     :type installer: interfaces.IInstaller
 
-    :param `str` question: Overriding dialog question to ask the user if asked
+    :param `str` question: Overriding default question to ask the user if asked
         to choose from domain names.
 
     :returns: Two-part tuple of domains and certname
@@ -927,7 +927,7 @@ def enhance(config, plugins):
         config.chain_path = lineage.chain_path
     if oldstyle_enh:
         le_client = _init_le_client(config, authenticator=None, installer=installer)
-        le_client.enhance_config(domains, config.chain_path, ask_redirect=False)
+        le_client.enhance_config(domains, config.chain_path, redirect_default=False)
     if enhancements.are_requested(config):
         enhancements.enable(lineage, domains, installer, config)
 

certbot/certbot/_internal/notify.py

@@ -1,34 +0,0 @@
-"""Send e-mail notification to system administrators."""
-
-import email
-import smtplib
-import socket
-import subprocess
-
-
-def notify(subject, whom, what):
-    """Send email notification.
-
-    Try to notify the addressee (``whom``) by e-mail, with Subject:
-    defined by ``subject`` and message body by ``what``.
-
-    """
-    msg = email.message_from_string(what)
-    msg.add_header("From", "Certbot renewal agent <root>")
-    msg.add_header("To", whom)
-    msg.add_header("Subject", subject)
-    msg = msg.as_string()
-    try:
-        lmtp = smtplib.LMTP()
-        lmtp.connect()
-        lmtp.sendmail("root", [whom], msg)
-    except (smtplib.SMTPHeloError, smtplib.SMTPRecipientsRefused,
-            smtplib.SMTPSenderRefused, smtplib.SMTPDataError, socket.error):
-        # We should try using /usr/sbin/sendmail in this case
-        try:
-            proc = subprocess.Popen(["/usr/sbin/sendmail", "-t"],
-                                    stdin=subprocess.PIPE)
-            proc.communicate(msg)
-        except OSError:
-            return False
-    return True