Revert "stop nginx"

This reverts commit 0dc3f78339.

.pylintrc

@@ -41,7 +41,7 @@ load-plugins=linter_plugin
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=fixme,locally-disabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes,cyclic-import,duplicate-code
+disable=fixme,locally-disabled,locally-enabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes,cyclic-import,duplicate-code
 # abstract-class-not-used cannot be disabled locally (at least in
 # pylint 1.4.1), same for abstract-class-little-used
 

.travis.yml

@@ -29,6 +29,8 @@ matrix:
       addons:
     - python: "2.7"
       env: TOXENV=lint
+    - python: "3.4"
+      env: TOXENV=mypy
     - python: "3.5"
       env: TOXENV=mypy
     - python: "2.7"
@@ -39,8 +41,9 @@ matrix:
       env: TOXENV=py34
       sudo: required
       services: docker
-    - python: "3.6"
-      env: TOXENV=py36
+    - python: "3.7"
+      dist: xenial
+      env: TOXENV=py37
       sudo: required
       services: docker
     - sudo: required
@@ -75,8 +78,6 @@ sudo: false
 
 addons:
   apt:
-    sources:
-    - augeas
     packages:  # Keep in sync with letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh and Boulder.
     - python-dev
     - python-virtualenv
@@ -88,10 +89,6 @@ addons:
     # For certbot-nginx integration testing
     - nginx-light
     - openssl
-    # for apacheconftest
-    - apache2
-    - libapache2-mod-wsgi
-    - libapache2-mod-macro
 
 install: "travis_retry $(command -v pip || command -v pip3) install tox coveralls"
 script:
@@ -102,9 +99,3 @@ after_success: '[ "$TOXENV" == "cover" ] && coveralls'
 
 notifications:
   email: false
-  irc:
-    channels:
-      - secure: "SGWZl3ownKx9xKVV2VnGt7DqkTmutJ89oJV9tjKhSs84kLijU6EYdPnllqISpfHMTxXflNZuxtGo0wTDYHXBuZL47w1O32W6nzuXdra5zC+i4sYQwYULUsyfOv9gJX8zWAULiK0Z3r0oho45U+FR5ZN6TPCidi8/eGU+EEPwaAw="
-    on_success: never
-    on_failure: always
-    use_notice: true

CHANGELOG.md

@@ -2,6 +2,135 @@
 
 Certbot adheres to [Semantic Versioning](http://semver.org/).
 
+## 0.25.1 - 2018-06-13
+
+### Fixed
+
+* TLS-ALPN-01 support has been removed from our acme library. Using our current
+  dependencies, we are unable to provide a correct implementation of this
+  challenge so we decided to remove it from the library until we can provide
+  proper support.
+* Issues causing test failures when running the tests in the acme package with
+  pytest<3.0 has been resolved.
+* certbot-nginx now correctly depends on acme>=0.25.0.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with changes other than their version number were:
+
+* acme
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/56?closed=1
+
+## 0.25.0 - 2018-06-06
+
+### Added
+
+* Support for the ready status type was added to acme. Without this change,
+  Certbot and acme users will begin encountering errors when using Let's
+  Encrypt's ACMEv2 API starting on June 19th for the staging environment and
+  July 5th for production. See
+  https://community.letsencrypt.org/t/acmev2-order-ready-status/62866 for more
+  information.
+* Certbot now accepts the flag --reuse-key which will cause the same key to be
+  used in the certificate when the lineage is renewed rather than generating a
+  new key.
+* You can now add multiple email addresses to your ACME account with Certbot by
+  providing a comma separated list of emails to the --email flag.
+* Support for Let's Encrypt's upcoming TLS-ALPN-01 challenge was added to acme.
+  For more information, see
+  https://community.letsencrypt.org/t/tls-alpn-validation-method/63814/1.
+* acme now supports specifying the source address to bind to when sending
+  outgoing connections. You still cannot specify this address using Certbot.
+* If you run Certbot against Let's Encrypt's ACMEv2 staging server but don't
+  already have an account registered at that server URL, Certbot will
+  automatically reuse your staging account from Let's Encrypt's ACMEv1 endpoint
+  if it exists.
+* Interfaces were added to Certbot allowing plugins to be called at additional
+  points. The `GenericUpdater` interface allows plugins to perform actions
+  every time `certbot renew` is run, regardless of whether any certificates are
+  due for renewal, and the `RenewDeployer` interface allows plugins to perform
+  actions when a certificate is renewed. See `certbot.interfaces` for more
+  information.
+
+### Changed
+
+* When running Certbot with --dry-run and you don't already have a staging
+  account, the created account does not contain an email address even if one
+  was provided to avoid expiration emails from Let's Encrypt's staging server.
+* certbot-nginx does a better job of automatically detecting the location of
+  Nginx's configuration files when run on BSD based systems.
+* acme now requires and uses pytest when running tests with setuptools with
+  `python setup.py test`.
+* `certbot config_changes` no longer waits for user input before exiting.
+
+### Fixed
+
+* Misleading log output that caused users to think that Certbot's standalone
+  plugin failed to bind to a port when performing a challenge has been
+  corrected.
+* An issue where certbot-nginx would fail to enable HSTS if the server block
+  already had an `add_header` directive has been resolved.
+* certbot-nginx now does a better job detecting the server block to base the
+  configuration for TLS-SNI challenges on.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with functional changes were:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/54?closed=1
+
+## 0.24.0 - 2018-05-02
+
+### Added
+
+* certbot now has an enhance subcommand which allows you to configure security
+  enhancements like HTTP to HTTPS redirects, OCSP stapling, and HSTS without
+  reinstalling a certificate.
+* certbot-dns-rfc2136 now allows the user to specify the port to use to reach
+  the DNS server in its credentials file.
+* acme now parses the wildcard field included in authorizations so it can be
+  used by users of the library.
+
+### Changed
+
+* certbot-dns-route53 used to wait for each DNS update to propagate before
+  sending the next one, but now it sends all updates before waiting which
+  speeds up issuance for multiple domains dramatically.
+* Certbot's official Docker images are now based on Alpine Linux 3.7 rather
+  than 3.4 because 3.4 has reached its end-of-life.
+* We've doubled the time Certbot will spend polling authorizations before
+  timing out.
+* The level of the message logged when Certbot is being used with
+  non-standard paths warning that crontabs for renewal included in Certbot
+  packages from OS package managers may not work has been reduced. This stops
+  the message from being written to stderr every time `certbot renew` runs.
+
+### Fixed
+
+* certbot-auto now works with Python 3.6.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with changes other than their version number were:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-dns-digitalocean (only style improvements to tests)
+* certbot-dns-rfc2136
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/52?closed=1
+
 ## 0.23.0 - 2018-04-04
 
 ### Added

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:2-alpine
+FROM python:2-alpine3.7
 
 ENTRYPOINT [ "certbot" ]
 EXPOSE 80 443

acme/MANIFEST.in

@@ -1,5 +1,6 @@
 include LICENSE.txt
 include README.rst
+include pytest.ini
 recursive-include docs *
 recursive-include examples *
 recursive-include acme/testdata *

acme/acme/challenges.py

@@ -9,6 +9,7 @@ from cryptography.hazmat.primitives import hashes  # type: ignore
 import josepy as jose
 import OpenSSL
 import requests
+import six
 
 from acme import errors
 from acme import crypto_util
@@ -139,16 +140,16 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse):
         return True
 
 
+@six.add_metaclass(abc.ABCMeta)
 class KeyAuthorizationChallenge(_TokenChallenge):
     # pylint: disable=abstract-class-little-used,too-many-ancestors
     """Challenge based on Key Authorization.
 
     :param response_cls: Subclass of `KeyAuthorizationChallengeResponse`
         that will be used to generate `response`.
-
+    :param str typ: type of the challenge
     """
-    __metaclass__ = abc.ABCMeta
-
+    typ = NotImplemented
     response_cls = NotImplemented
     thumbprint_hash_function = (
         KeyAuthorizationChallengeResponse.thumbprint_hash_function)
@@ -477,7 +478,7 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse):
             try:
                 cert = self.probe_cert(domain=domain, **kwargs)
             except errors.Error as error:
-                logger.debug(error, exc_info=True)
+                logger.debug(str(error), exc_info=True)
                 return False
 
         return self.verify_cert(cert)
@@ -506,6 +507,21 @@ class TLSSNI01(KeyAuthorizationChallenge):
         return self.response(account_key).gen_cert(key=kwargs.get('cert_key'))
 
 
+@Challenge.register  # pylint: disable=too-many-ancestors
+class TLSALPN01(KeyAuthorizationChallenge):
+    """ACME tls-alpn-01 challenge.
+
+    This class simply allows parsing the TLS-ALPN-01 challenge returned from
+    the CA. Full TLS-ALPN-01 support is not currently provided.
+
+    """
+    typ = "tls-alpn-01"
+
+    def validation(self, account_key, **kwargs):
+        """Generate validation for the challenge."""
+        raise NotImplementedError()
+
+
 @Challenge.register  # pylint: disable=too-many-ancestors
 class DNS(_TokenChallenge):
     """ACME "dns" challenge."""

acme/acme/challenges_test.py

@@ -393,6 +393,38 @@ class TLSSNI01Test(unittest.TestCase):
         mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key)
 
 
+class TLSALPN01Test(unittest.TestCase):
+
+    def setUp(self):
+        from acme.challenges import TLSALPN01
+        self.msg = TLSALPN01(
+            token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e'))
+        self.jmsg = {
+            'type': 'tls-alpn-01',
+            'token': 'a82d5ff8ef740d12881f6d3c2277ab2e',
+        }
+
+    def test_to_partial_json(self):
+        self.assertEqual(self.jmsg, self.msg.to_partial_json())
+
+    def test_from_json(self):
+        from acme.challenges import TLSALPN01
+        self.assertEqual(self.msg, TLSALPN01.from_json(self.jmsg))
+
+    def test_from_json_hashable(self):
+        from acme.challenges import TLSALPN01
+        hash(TLSALPN01.from_json(self.jmsg))
+
+    def test_from_json_invalid_token_length(self):
+        from acme.challenges import TLSALPN01
+        self.jmsg['token'] = jose.encode_b64jose(b'abcd')
+        self.assertRaises(
+            jose.DeserializationError, TLSALPN01.from_json, self.jmsg)
+
+    def test_validation(self):
+        self.assertRaises(NotImplementedError, self.msg.validation, KEY)
+
+
 class DNSTest(unittest.TestCase):
 
     def setUp(self):

acme/acme/client.py

@@ -9,17 +9,20 @@ import time
 
 import six
 from six.moves import http_client  # pylint: disable=import-error
-
 import josepy as jose
 import OpenSSL
 import re
+from requests_toolbelt.adapters.source import SourceAddressAdapter
 import requests
+from requests.adapters import HTTPAdapter
 import sys
 
 from acme import crypto_util
 from acme import errors
 from acme import jws
 from acme import messages
+# pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Dict, List, Set, Text
 
 
 logger = logging.getLogger(__name__)
@@ -415,7 +418,7 @@ class Client(ClientBase):
         """
         # pylint: disable=too-many-locals
         assert max_attempts > 0
-        attempts = collections.defaultdict(int)
+        attempts = collections.defaultdict(int) # type: Dict[messages.AuthorizationResource, int]
         exhausted = set()
 
         # priority queue with datetime.datetime (based on Retry-After) as key,
@@ -529,7 +532,7 @@ class Client(ClientBase):
         :rtype: `list` of `OpenSSL.crypto.X509` wrapped in `.ComparableX509`
 
         """
-        chain = []
+        chain = [] # type: List[jose.ComparableX509]
         uri = certr.cert_chain_uri
         while uri is not None and len(chain) < max_length:
             response, cert = self._get_cert(uri)
@@ -856,18 +859,28 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
     :param bool verify_ssl: Whether to verify certificates on SSL connections.
     :param str user_agent: String to send as User-Agent header.
     :param float timeout: Timeout for requests.
+    :param source_address: Optional source address to bind to when making requests.
+    :type source_address: str or tuple(str, int)
     """
     def __init__(self, key, account=None, alg=jose.RS256, verify_ssl=True,
-                 user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT):
+                 user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT,
+                 source_address=None):
         # pylint: disable=too-many-arguments
         self.key = key
         self.account = account
         self.alg = alg
         self.verify_ssl = verify_ssl
-        self._nonces = set()
+        self._nonces = set() # type: Set[Text]
         self.user_agent = user_agent
         self.session = requests.Session()
         self._default_timeout = timeout
+        adapter = HTTPAdapter()
+
+        if source_address is not None:
+            adapter = SourceAddressAdapter(source_address)
+
+        self.session.mount("http://", adapter)
+        self.session.mount("https://", adapter)
 
     def __del__(self):
         # Try to close the session, but don't show exceptions to the
@@ -1017,7 +1030,7 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
         if response.headers.get("Content-Type") == DER_CONTENT_TYPE:
             debug_content = base64.b64encode(response.content)
         else:
-            debug_content = response.content
+            debug_content = response.content.decode("utf-8")
         logger.debug('Received response:\nHTTP %d\n%s\n\n%s',
                      response.status_code,
                      "\n".join(["{0}: {1}".format(k, v)

acme/acme/client_test.py

@@ -17,6 +17,7 @@ from acme import jws as acme_jws
 from acme import messages
 from acme import messages_test
 from acme import test_util
+from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module
 
 
 CERT_DER = test_util.load_vector('cert.der')
@@ -61,7 +62,8 @@ class ClientTestBase(unittest.TestCase):
         self.contact = ('mailto:cert-admin@example.com', 'tel:+12025551212')
         reg = messages.Registration(
             contact=self.contact, key=KEY.public_key())
-        self.new_reg = messages.NewRegistration(**dict(reg))
+        the_arg = dict(reg) # type: Dict
+        self.new_reg = messages.NewRegistration(**the_arg) # pylint: disable=star-args
         self.regr = messages.RegistrationResource(
             body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1')
 
@@ -1127,6 +1129,31 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
         self.assertRaises(requests.exceptions.RequestException,
                           self.net.post, 'uri', obj=self.obj)
 
+class ClientNetworkSourceAddressBindingTest(unittest.TestCase):
+    """Tests that if ClientNetwork has a source IP set manually, the underlying library has
+    used the provided source address."""
+
+    def setUp(self):
+        self.source_address = "8.8.8.8"
+
+    def test_source_address_set(self):
+        from acme.client import ClientNetwork
+        net = ClientNetwork(key=None, alg=None, source_address=self.source_address)
+        for adapter in net.session.adapters.values():
+            self.assertTrue(self.source_address in adapter.source_address)
+
+    def test_behavior_assumption(self):
+        """This is a test that guardrails the HTTPAdapter behavior so that if the default for
+        a Session() changes, the assumptions here aren't violated silently."""
+        from acme.client import ClientNetwork
+        # Source address not specified, so the default adapter type should be bound -- this
+        # test should fail if the default adapter type is changed by requests
+        net = ClientNetwork(key=None, alg=None)
+        session = requests.Session()
+        for scheme in session.adapters.keys():
+            client_network_adapter = net.session.adapters.get(scheme)
+            default_adapter = session.adapters.get(scheme)
+            self.assertEqual(client_network_adapter.__class__, default_adapter.__class__)
 
 if __name__ == '__main__':
     unittest.main()  # pragma: no cover

acme/acme/crypto_util.py

@@ -6,11 +6,14 @@ import os
 import re
 import socket
 
-import OpenSSL
+from OpenSSL import crypto
+from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052
 import josepy as jose
 
-
 from acme import errors
+# pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Callable, Union, Tuple, Optional
+# pylint: enable=unused-import, no-name-in-module
 
 
 logger = logging.getLogger(__name__)
@@ -25,7 +28,7 @@ logger = logging.getLogger(__name__)
 # https://www.openssl.org/docs/ssl/SSLv23_method.html). _serve_sni
 # should be changed to use "set_options" to disable SSLv2 and SSLv3,
 # in case it's used for things other than probing/serving!
-_DEFAULT_TLSSNI01_SSL_METHOD = OpenSSL.SSL.SSLv23_METHOD  # type: ignore
+_DEFAULT_TLSSNI01_SSL_METHOD = SSL.SSLv23_METHOD  # type: ignore
 
 
 class SSLSocket(object):  # pylint: disable=too-few-public-methods
@@ -64,9 +67,9 @@ class SSLSocket(object):  # pylint: disable=too-few-public-methods
             logger.debug("Server name (%s) not recognized, dropping SSL",
                          server_name)
             return
-        new_context = OpenSSL.SSL.Context(self.method)
-        new_context.set_options(OpenSSL.SSL.OP_NO_SSLv2)
-        new_context.set_options(OpenSSL.SSL.OP_NO_SSLv3)
+        new_context = SSL.Context(self.method)
+        new_context.set_options(SSL.OP_NO_SSLv2)
+        new_context.set_options(SSL.OP_NO_SSLv3)
         new_context.use_privatekey(key)
         new_context.use_certificate(cert)
         connection.set_context(new_context)
@@ -89,18 +92,18 @@ class SSLSocket(object):  # pylint: disable=too-few-public-methods
     def accept(self):  # pylint: disable=missing-docstring
         sock, addr = self.sock.accept()
 
-        context = OpenSSL.SSL.Context(self.method)
-        context.set_options(OpenSSL.SSL.OP_NO_SSLv2)
-        context.set_options(OpenSSL.SSL.OP_NO_SSLv3)
+        context = SSL.Context(self.method)
+        context.set_options(SSL.OP_NO_SSLv2)
+        context.set_options(SSL.OP_NO_SSLv3)
         context.set_tlsext_servername_callback(self._pick_certificate_cb)
 
-        ssl_sock = self.FakeConnection(OpenSSL.SSL.Connection(context, sock))
+        ssl_sock = self.FakeConnection(SSL.Connection(context, sock))
         ssl_sock.set_accept_state()
 
         logger.debug("Performing handshake with %s", addr)
         try:
             ssl_sock.do_handshake()
-        except OpenSSL.SSL.Error as error:
+        except SSL.Error as error:
             # _pick_certificate_cb might have returned without
             # creating SSL context (wrong server name)
             raise socket.error(error)
@@ -128,30 +131,39 @@ def probe_sni(name, host, port=443, timeout=300,
     :rtype: OpenSSL.crypto.X509
 
     """
-    context = OpenSSL.SSL.Context(method)
+    context = SSL.Context(method)
     context.set_timeout(timeout)
 
     socket_kwargs = {'source_address': source_address}
 
-    host_protocol_agnostic = None if host == '::' or host == '0' else host
+    host_protocol_agnostic = host
+    if host == '::' or host == '0':
+        # https://github.com/python/typeshed/pull/2136
+        # while PR is not merged, we need to ignore
+        host_protocol_agnostic = None
 
     try:
         # pylint: disable=star-args
-        logger.debug("Attempting to connect to %s:%d%s.", host_protocol_agnostic, port,
-            " from {0}:{1}".format(source_address[0], source_address[1]) if \
-            socket_kwargs else "")
-        sock = socket.create_connection((host_protocol_agnostic, port), **socket_kwargs)
+        logger.debug(
+            "Attempting to connect to %s:%d%s.", host_protocol_agnostic, port,
+            " from {0}:{1}".format(
+                source_address[0],
+                source_address[1]
+            ) if socket_kwargs else ""
+        )
+        socket_tuple = (host_protocol_agnostic, port)  # type: Tuple[Optional[str], int]
+        sock = socket.create_connection(socket_tuple, **socket_kwargs)  # type: ignore
     except socket.error as error:
         raise errors.Error(error)
 
     with contextlib.closing(sock) as client:
-        client_ssl = OpenSSL.SSL.Connection(context, client)
+        client_ssl = SSL.Connection(context, client)
         client_ssl.set_connect_state()
         client_ssl.set_tlsext_host_name(name)  # pyOpenSSL>=0.13
         try:
             client_ssl.do_handshake()
             client_ssl.shutdown()
-        except OpenSSL.SSL.Error as error:
+        except SSL.Error as error:
             raise errors.Error(error)
     return client_ssl.get_peer_certificate()
 
@@ -164,18 +176,18 @@ def make_csr(private_key_pem, domains, must_staple=False):
         OCSP Must Staple: https://tools.ietf.org/html/rfc7633).
     :returns: buffer PEM-encoded Certificate Signing Request.
     """
-    private_key = OpenSSL.crypto.load_privatekey(
-        OpenSSL.crypto.FILETYPE_PEM, private_key_pem)
-    csr = OpenSSL.crypto.X509Req()
+    private_key = crypto.load_privatekey(
+        crypto.FILETYPE_PEM, private_key_pem)
+    csr = crypto.X509Req()
     extensions = [
-        OpenSSL.crypto.X509Extension(
+        crypto.X509Extension(
             b'subjectAltName',
             critical=False,
             value=', '.join('DNS:' + d for d in domains).encode('ascii')
         ),
     ]
     if must_staple:
-        extensions.append(OpenSSL.crypto.X509Extension(
+        extensions.append(crypto.X509Extension(
             b"1.3.6.1.5.5.7.1.24",
             critical=False,
             value=b"DER:30:03:02:01:05"))
@@ -183,8 +195,8 @@ def make_csr(private_key_pem, domains, must_staple=False):
     csr.set_pubkey(private_key)
     csr.set_version(2)
     csr.sign(private_key, 'sha256')
-    return OpenSSL.crypto.dump_certificate_request(
-        OpenSSL.crypto.FILETYPE_PEM, csr)
+    return crypto.dump_certificate_request(
+        crypto.FILETYPE_PEM, csr)
 
 def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req):
     common_name = loaded_cert_or_req.get_subject().CN
@@ -221,11 +233,12 @@ def _pyopenssl_cert_or_req_san(cert_or_req):
     parts_separator = ", "
     prefix = "DNS" + part_separator
 
-    if isinstance(cert_or_req, OpenSSL.crypto.X509):
-        func = OpenSSL.crypto.dump_certificate
+    if isinstance(cert_or_req, crypto.X509):
+        # pylint: disable=line-too-long
+        func = crypto.dump_certificate # type: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]]
     else:
-        func = OpenSSL.crypto.dump_certificate_request
-    text = func(OpenSSL.crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8")
+        func = crypto.dump_certificate_request
+    text = func(crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8")
     # WARNING: this function does not support multiple SANs extensions.
     # Multiple X509v3 extensions of the same type is disallowed by RFC 5280.
     match = re.search(r"X509v3 Subject Alternative Name:(?: critical)?\s*(.*)", text)
@@ -252,12 +265,12 @@ def gen_ss_cert(key, domains, not_before=None,
 
     """
     assert domains, "Must provide one or more hostnames for the cert."
-    cert = OpenSSL.crypto.X509()
+    cert = crypto.X509()
     cert.set_serial_number(int(binascii.hexlify(os.urandom(16)), 16))
     cert.set_version(2)
 
     extensions = [
-        OpenSSL.crypto.X509Extension(
+        crypto.X509Extension(
             b"basicConstraints", True, b"CA:TRUE, pathlen:0"),
     ]
 
@@ -266,7 +279,7 @@ def gen_ss_cert(key, domains, not_before=None,
     cert.set_issuer(cert.get_subject())
 
     if force_san or len(domains) > 1:
-        extensions.append(OpenSSL.crypto.X509Extension(
+        extensions.append(crypto.X509Extension(
             b"subjectAltName",
             critical=False,
             value=b", ".join(b"DNS:" + d.encode() for d in domains)
@@ -281,7 +294,7 @@ def gen_ss_cert(key, domains, not_before=None,
     cert.sign(key, "sha256")
     return cert
 
-def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM):
+def dump_pyopenssl_chain(chain, filetype=crypto.FILETYPE_PEM):
     """Dump certificate chain into a bundle.
 
     :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in
@@ -298,7 +311,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM):
         if isinstance(cert, jose.ComparableX509):
             # pylint: disable=protected-access
             cert = cert.wrapped
-        return OpenSSL.crypto.dump_certificate(filetype, cert)
+        return crypto.dump_certificate(filetype, cert)
 
     # assumes that OpenSSL.crypto.dump_certificate includes ending
     # newline character

acme/acme/crypto_util_test.py

@@ -13,6 +13,7 @@ import OpenSSL
 
 from acme import errors
 from acme import test_util
+from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module
 
 
 class SSLSocketAndProbeSNITest(unittest.TestCase):
@@ -41,28 +42,38 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
         self.server_thread = threading.Thread(
             # pylint: disable=no-member
             target=self.server.handle_request)
-        self.server_thread.start()
-        time.sleep(1)  # TODO: avoid race conditions in other way
 
     def tearDown(self):
-        self.server_thread.join()
+        if self.server_thread.is_alive():
+            # The thread may have already terminated.
+            self.server_thread.join()  # pragma: no cover
 
     def _probe(self, name):
         from acme.crypto_util import probe_sni
         return jose.ComparableX509(probe_sni(
             name, host='127.0.0.1', port=self.port))
 
+    def _start_server(self):
+        self.server_thread.start()
+        time.sleep(1)  # TODO: avoid race conditions in other way
+
     def test_probe_ok(self):
+        self._start_server()
         self.assertEqual(self.cert, self._probe(b'foo'))
 
     def test_probe_not_recognized_name(self):
+        self._start_server()
         self.assertRaises(errors.Error, self._probe, b'bar')
 
-    # TODO: py33/py34 tox hangs forever on do_handshake in second probe
-    #def probe_connection_error(self):
-    #    self._probe(b'foo')
-    #    #time.sleep(1)  # TODO: avoid race conditions in other way
-    #    self.assertRaises(errors.Error, self._probe, b'bar')
+    def test_probe_connection_error(self):
+        # pylint has a hard time with six
+        self.server.server_close()  # pylint: disable=no-member
+        original_timeout = socket.getdefaulttimeout()
+        try:
+            socket.setdefaulttimeout(1)
+            self.assertRaises(errors.Error, self._probe, b'bar')
+        finally:
+            socket.setdefaulttimeout(original_timeout)
 
 
 class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase):
@@ -165,7 +176,7 @@ class RandomSnTest(unittest.TestCase):
 
     def setUp(self):
         self.cert_count = 5
-        self.serial_num = []
+        self.serial_num = [] # type: List[int]
         self.key = OpenSSL.crypto.PKey()
         self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
 

acme/acme/magic_typing.py

@@ -0,0 +1,16 @@
+"""Shim class to not have to depend on typing module in prod."""
+import sys
+
+class TypingClass(object):
+    """Ignore import errors by getting anything"""
+    def __getattr__(self, name):
+        return None
+
+try:
+    # mypy doesn't respect modifying sys.modules
+    from typing import *  # pylint: disable=wildcard-import, unused-wildcard-import
+    # pylint: disable=unused-import
+    from typing import Collection, IO  # type: ignore
+    # pylint: enable=unused-import
+except ImportError:
+    sys.modules[__name__] = TypingClass()

acme/acme/magic_typing_test.py

@@ -0,0 +1,41 @@
+"""Tests for acme.magic_typing."""
+import sys
+import unittest
+
+import mock
+
+
+class MagicTypingTest(unittest.TestCase):
+    """Tests for acme.magic_typing."""
+    def test_import_success(self):
+        try:
+            import typing as temp_typing
+        except ImportError: # pragma: no cover
+            temp_typing = None # pragma: no cover
+        typing_class_mock = mock.MagicMock()
+        text_mock = mock.MagicMock()
+        typing_class_mock.Text = text_mock
+        sys.modules['typing'] = typing_class_mock
+        if 'acme.magic_typing' in sys.modules:
+            del sys.modules['acme.magic_typing'] # pragma: no cover
+        from acme.magic_typing import Text # pylint: disable=no-name-in-module
+        self.assertEqual(Text, text_mock)
+        del sys.modules['acme.magic_typing']
+        sys.modules['typing'] = temp_typing
+
+    def test_import_failure(self):
+        try:
+            import typing as temp_typing
+        except ImportError: # pragma: no cover
+            temp_typing = None # pragma: no cover
+        sys.modules['typing'] = None
+        if 'acme.magic_typing' in sys.modules:
+            del sys.modules['acme.magic_typing'] # pragma: no cover
+        from acme.magic_typing import Text # pylint: disable=no-name-in-module
+        self.assertTrue(Text is None)
+        del sys.modules['acme.magic_typing']
+        sys.modules['typing'] = temp_typing
+
+
+if __name__ == '__main__':
+    unittest.main()  # pragma: no cover

acme/acme/messages.py

@@ -145,6 +145,7 @@ STATUS_PROCESSING = Status('processing')
 STATUS_VALID = Status('valid')
 STATUS_INVALID = Status('invalid')
 STATUS_REVOKED = Status('revoked')
+STATUS_READY = Status('ready')
 
 
 class IdentifierType(_Constant):
@@ -284,7 +285,7 @@ class Registration(ResourceBody):
         if phone is not None:
             details.append(cls.phone_prefix + phone)
         if email is not None:
-            details.append(cls.email_prefix + email)
+            details.extend([cls.email_prefix + mail for mail in email.split(',')])
         kwargs['contact'] = tuple(details)
         return cls(**kwargs)
 

acme/acme/messages_test.py

@@ -6,6 +6,7 @@ import mock
 
 from acme import challenges
 from acme import test_util
+from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module
 
 
 CERT = test_util.load_comparable_cert('cert.der')
@@ -85,7 +86,7 @@ class ConstantTest(unittest.TestCase):
         from acme.messages import _Constant
 
         class MockConstant(_Constant):  # pylint: disable=missing-docstring
-            POSSIBLE_NAMES = {}
+            POSSIBLE_NAMES = {} # type: Dict
 
         self.MockConstant = MockConstant  # pylint: disable=invalid-name
         self.const_a = MockConstant('a')

acme/acme/standalone.py

@@ -16,6 +16,7 @@ import OpenSSL
 
 from acme import challenges
 from acme import crypto_util
+from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module
 
 
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class BaseDualNetworkedServers(object):
 
     def __init__(self, ServerClass, server_address, *remaining_args, **kwargs):
         port = server_address[1]
-        self.threads = []
-        self.servers = []
+        self.threads = [] # type: List[threading.Thread]
+        self.servers = [] # type: List[ACMEServerMixin]
 
         # Must try True first.
         # Ubuntu, for example, will fail to bind to IPv4 if we've already bound
@@ -82,9 +83,22 @@ class BaseDualNetworkedServers(object):
                 new_address = (server_address[0],) + (port,) + server_address[2:]
                 new_args = (new_address,) + remaining_args
                 server = ServerClass(*new_args, **kwargs) # pylint: disable=star-args
-            except socket.error:
-                logger.debug("Failed to bind to %s:%s using %s", new_address[0],
+                logger.debug(
+                    "Successfully bound to %s:%s using %s", new_address[0],
                     new_address[1], "IPv6" if ip_version else "IPv4")
+            except socket.error:
+                if self.servers:
+                    # Already bound using IPv6.
+                    logger.debug(
+                        "Certbot wasn't able to bind to %s:%s using %s, this " +
+                        "is often expected due to the dual stack nature of " +
+                        "IPv6 socket implementations.",
+                        new_address[0], new_address[1],
+                        "IPv6" if ip_version else "IPv4")
+                else:
+                    logger.debug(
+                        "Failed to bind to %s:%s using %s", new_address[0],
+                        new_address[1], "IPv6" if ip_version else "IPv4")
             else:
                 self.servers.append(server)
                 # If two servers are set up and port 0 was passed in, ensure we always
@@ -189,7 +203,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 
     def __init__(self, *args, **kwargs):
         self.simple_http_resources = kwargs.pop("simple_http_resources", set())
-        socketserver.BaseRequestHandler.__init__(self, *args, **kwargs)
+        BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
 
     def log_message(self, format, *args):  # pylint: disable=redefined-builtin
         """Log arbitrary message."""
@@ -262,7 +276,7 @@ def simple_tls_sni_01_server(cli_args, forever=True):
 
     certs = {}
 
-    _, hosts, _ = next(os.walk('.'))
+    _, hosts, _ = next(os.walk('.')) # type: ignore # https://github.com/python/mypy/issues/465
     for host in hosts:
         with open(os.path.join(host, "cert.pem")) as cert_file:
             cert_contents = cert_file.read()

acme/acme/standalone_test.py

@@ -4,10 +4,10 @@ import shutil
 import socket
 import threading
 import tempfile
-import time
 import unittest
 
 from six.moves import http_client  # pylint: disable=import-error
+from six.moves import queue  # pylint: disable=import-error
 from six.moves import socketserver  # type: ignore  # pylint: disable=import-error
 
 import josepy as jose
@@ -16,8 +16,8 @@ import requests
 
 from acme import challenges
 from acme import crypto_util
-from acme import errors
 from acme import test_util
+from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module
 
 
 class TLSServerTest(unittest.TestCase):
@@ -72,7 +72,7 @@ class HTTP01ServerTest(unittest.TestCase):
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))
-        self.resources = set()
+        self.resources = set() # type: Set
 
         from acme.standalone import HTTP01Server
         self.server = HTTP01Server(('', 0), resources=self.resources)
@@ -201,7 +201,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase):
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))
-        self.resources = set()
+        self.resources = set() # type: Set
 
         from acme.standalone import HTTP01DualNetworkedServers
         self.servers = HTTP01DualNetworkedServers(('', 0), resources=self.resources)
@@ -260,10 +260,9 @@ class TestSimpleTLSSNI01Server(unittest.TestCase):
                     os.path.join(localhost_dir, 'key.pem'))
 
         from acme.standalone import simple_tls_sni_01_server
-        self.port = 1234
         self.thread = threading.Thread(
             target=simple_tls_sni_01_server, kwargs={
-                'cli_args': ('xxx', '--port', str(self.port)),
+                'cli_args': ('filename',),
                 'forever': False,
             },
         )
@@ -275,25 +274,20 @@ class TestSimpleTLSSNI01Server(unittest.TestCase):
         self.thread.join()
         shutil.rmtree(self.test_cwd)
 
-    def test_it(self):
-        max_attempts = 5
-        for attempt in range(max_attempts):
-            try:
-                cert = crypto_util.probe_sni(
-                    b'localhost', b'0.0.0.0', self.port)
-            except errors.Error:
-                self.assertTrue(attempt + 1 < max_attempts, "Timeout!")
-                time.sleep(1)  # wait until thread starts
-            else:
-                self.assertEqual(jose.ComparableX509(cert),
-                                 test_util.load_comparable_cert(
-                                     'rsa2048_cert.pem'))
-                break
+    @mock.patch('acme.standalone.logger')
+    def test_it(self, mock_logger):
+        # Use a Queue because mock objects aren't thread safe.
+        q = queue.Queue()  # type: queue.Queue[int]
+        # Add port number to the queue.
+        mock_logger.info.side_effect = lambda *args: q.put(args[-1])
+        self.thread.start()
 
-            if attempt == 0:
-                # the first attempt is always meant to fail, so we can test
-                # the socket failure code-path for probe_sni, as well
-                self.thread.start()
+        # After the timeout, an exception is raised if the queue is empty.
+        port = q.get(timeout=5)
+        cert = crypto_util.probe_sni(b'localhost', b'0.0.0.0', port)
+        self.assertEqual(jose.ComparableX509(cert),
+                         test_util.load_comparable_cert(
+                             'rsa2048_cert.pem'))
 
 
 if __name__ == "__main__":

acme/acme/test_util.py

@@ -10,7 +10,7 @@ import unittest
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 import josepy as jose
-import OpenSSL
+from OpenSSL import crypto
 
 
 def vector_path(*names):
@@ -39,8 +39,8 @@ def _guess_loader(filename, loader_pem, loader_der):
 def load_cert(*names):
     """Load certificate."""
     loader = _guess_loader(
-        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
-    return OpenSSL.crypto.load_certificate(loader, load_vector(*names))
+        names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1)
+    return crypto.load_certificate(loader, load_vector(*names))
 
 
 def load_comparable_cert(*names):
@@ -51,8 +51,8 @@ def load_comparable_cert(*names):
 def load_csr(*names):
     """Load certificate request."""
     loader = _guess_loader(
-        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
-    return OpenSSL.crypto.load_certificate_request(loader, load_vector(*names))
+        names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1)
+    return crypto.load_certificate_request(loader, load_vector(*names))
 
 
 def load_comparable_csr(*names):
@@ -71,8 +71,8 @@ def load_rsa_private_key(*names):
 def load_pyopenssl_private_key(*names):
     """Load pyOpenSSL private key."""
     loader = _guess_loader(
-        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
-    return OpenSSL.crypto.load_privatekey(loader, load_vector(*names))
+        names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1)
+    return crypto.load_privatekey(loader, load_vector(*names))
 
 
 def skip_unless(condition, reason):  # pragma: no cover

acme/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+norecursedirs = .* build dist CVS _darcs {arch} *.egg

acme/setup.py

@@ -1,10 +1,9 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
-
-version = '0.24.0.dev0'
+version = '0.26.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
@@ -19,6 +18,7 @@ install_requires = [
     'pyrfc3339',
     'pytz',
     'requests[security]>=2.4.1',  # security extras added in 2.4.1
+    'requests-toolbelt>=0.3.0',
     'setuptools',
     'six>=1.9.0',  # needed for python_2_unicode_compatible
 ]
@@ -34,6 +34,19 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
 
 setup(
     name='acme',
@@ -66,5 +79,7 @@ setup(
         'dev': dev_extras,
         'docs': docs_extras,
     },
+    tests_require=["pytest"],
     test_suite='acme',
+    cmdclass={"test": PyTest},
 )

certbot-apache/certbot_apache/apache_util.py

@@ -1,4 +1,5 @@
 """ Utility functions for certbot-apache plugin """
+import binascii
 import os
 
 from certbot import util
@@ -98,3 +99,8 @@ def parse_define_file(filepath, varname):
             var_parts = v[2:].partition("=")
             return_vars[var_parts[0]] = var_parts[2]
     return return_vars
+
+
+def unique_id():
+    """ Returns an unique id to be used as a VirtualHost identifier"""
+    return binascii.hexlify(os.urandom(16)).decode("utf-8")

certbot-apache/certbot_apache/configurator.py

@@ -13,13 +13,16 @@ import zope.component
 import zope.interface
 
 from acme import challenges
+from acme.magic_typing import Any, DefaultDict, Dict, List, Set, Union  # pylint: disable=unused-import, no-name-in-module
 
 from certbot import errors
 from certbot import interfaces
 from certbot import util
 
+from certbot.achallenges import KeyAuthorizationAnnotatedChallenge  # pylint: disable=unused-import
 from certbot.plugins import common
 from certbot.plugins.util import path_surgery
+from certbot.plugins.enhancements import AutoHSTSEnhancement
 
 from certbot_apache import apache_util
 from certbot_apache import augeas_configurator
@@ -130,10 +133,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             default=cls.OS_DEFAULTS["challenge_location"],
             help="Directory path for challenge configuration.")
         add("handle-modules", default=cls.OS_DEFAULTS["handle_mods"],
-            help="Let installer handle enabling required modules for you." +
+            help="Let installer handle enabling required modules for you. " +
                  "(Only Ubuntu/Debian currently)")
         add("handle-sites", default=cls.OS_DEFAULTS["handle_sites"],
-            help="Let installer handle enabling sites for you." +
+            help="Let installer handle enabling sites for you. " +
                  "(Only Ubuntu/Debian currently)")
         util.add_deprecated_argument(add, argument_name="ctl", nargs=1)
         util.add_deprecated_argument(
@@ -150,14 +153,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         super(ApacheConfigurator, self).__init__(*args, **kwargs)
 
         # Add name_server association dict
-        self.assoc = dict()
+        self.assoc = dict()  # type: Dict[str, obj.VirtualHost]
         # Outstanding challenges
-        self._chall_out = set()
+        self._chall_out = set()  # type: Set[KeyAuthorizationAnnotatedChallenge]
         # List of vhosts configured per wildcard domain on this run.
         # used by deploy_cert() and enhance()
-        self._wildcard_vhosts = dict()
+        self._wildcard_vhosts = dict()  # type: Dict[str, List[obj.VirtualHost]]
         # Maps enhancements to vhosts we've enabled the enhancement for
-        self._enhanced_vhosts = defaultdict(set)
+        self._enhanced_vhosts = defaultdict(set)  # type: DefaultDict[str, Set[obj.VirtualHost]]
+        # Temporary state for AutoHSTS enhancement
+        self._autohsts = {}  # type: Dict[str, Dict[str, Union[int, float]]]
 
         # These will be set in the prepare function
         self.parser = None
@@ -659,7 +664,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         :rtype: set
 
         """
-        all_names = set()
+        all_names = set()  # type: Set[str]
 
         vhost_macro = []
 
@@ -800,8 +805,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         """
         # Search base config, and all included paths for VirtualHosts
-        file_paths = {}
-        internal_paths = defaultdict(set)
+        file_paths = {}  # type: Dict[str, str]
+        internal_paths = defaultdict(set)  # type: DefaultDict[str, Set[str]]
         vhs = []
         # Make a list of parser paths because the parser_paths
         # dictionary may be modified during the loop.
@@ -1239,7 +1244,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             if not self.parser.parsed_in_current(ssl_fp):
                 self.parser.parse_file(ssl_fp)
         except IOError:
-            logger.fatal("Error writing/reading to file in make_vhost_ssl")
+            logger.critical("Error writing/reading to file in make_vhost_ssl", exc_info=True)
             raise errors.PluginError("Unable to write/read in make_vhost_ssl")
 
         if sift:
@@ -1327,7 +1332,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         try:
             span_val = self.aug.span(vhost.path)
         except ValueError:
-            logger.fatal("Error while reading the VirtualHost %s from "
+            logger.critical("Error while reading the VirtualHost %s from "
                          "file %s", vhost.name, vhost.filep, exc_info=True)
             raise errors.PluginError("Unable to read VirtualHost from file")
         span_filep = span_val[0]
@@ -1470,6 +1475,67 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         if need_to_save:
             self.save()
 
+    def find_vhost_by_id(self, id_str):
+        """
+        Searches through VirtualHosts and tries to match the id in a comment
+
+        :param str id_str: Id string for matching
+
+        :returns: The matched VirtualHost or None
+        :rtype: :class:`~certbot_apache.obj.VirtualHost` or None
+
+        :raises .errors.PluginError: If no VirtualHost is found
+        """
+
+        for vh in self.vhosts:
+            if self._find_vhost_id(vh) == id_str:
+                return vh
+        msg = "No VirtualHost with ID {} was found.".format(id_str)
+        logger.warning(msg)
+        raise errors.PluginError(msg)
+
+    def _find_vhost_id(self, vhost):
+        """Tries to find the unique ID from the VirtualHost comments. This is
+        used for keeping track of VirtualHost directive over time.
+
+        :param vhost: Virtual host to add the id
+        :type vhost: :class:`~certbot_apache.obj.VirtualHost`
+
+        :returns: The unique ID or None
+        :rtype: str or None
+        """
+
+        # Strip the {} off from the format string
+        search_comment = constants.MANAGED_COMMENT_ID.format("")
+
+        id_comment = self.parser.find_comments(search_comment, vhost.path)
+        if id_comment:
+            # Use the first value, multiple ones shouldn't exist
+            comment = self.parser.get_arg(id_comment[0])
+            return comment.split(" ")[-1]
+        return None
+
+    def add_vhost_id(self, vhost):
+        """Adds an unique ID to the VirtualHost as a comment for mapping back
+        to it on later invocations, as the config file order might have changed.
+        If ID already exists, returns that instead.
+
+        :param vhost: Virtual host to add or find the id
+        :type vhost: :class:`~certbot_apache.obj.VirtualHost`
+
+        :returns: The unique ID for vhost
+        :rtype: str or None
+        """
+
+        vh_id = self._find_vhost_id(vhost)
+        if vh_id:
+            return vh_id
+
+        id_string = apache_util.unique_id()
+        comment = constants.MANAGED_COMMENT_ID.format(id_string)
+        self.parser.add_comment(vhost.path, comment)
+        return id_string
+
     def _escape(self, fp):
         fp = fp.replace(",", "\\,")
         fp = fp.replace("[", "\\[")
@@ -1529,6 +1595,78 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             logger.warning("Failed %s for %s", enhancement, domain)
             raise
 
+    def _autohsts_increase(self, vhost, id_str, nextstep):
+        """Increase the AutoHSTS max-age value
+
+        :param vhost: Virtual host object to modify
+        :type vhost: :class:`~certbot_apache.obj.VirtualHost`
+
+        :param str id_str: The unique ID string of VirtualHost
+
+        :param int nextstep: Next AutoHSTS max-age value index
+
+        """
+        nextstep_value = constants.AUTOHSTS_STEPS[nextstep]
+        self._autohsts_write(vhost, nextstep_value)
+        self._autohsts[id_str] = {"laststep": nextstep, "timestamp": time.time()}
+
+    def _autohsts_write(self, vhost, nextstep_value):
+        """
+        Write the new HSTS max-age value to the VirtualHost file
+        """
+
+        hsts_dirpath = None
+        header_path = self.parser.find_dir("Header", None, vhost.path)
+        if header_path:
+            pat = '(?:[ "]|^)(strict-transport-security)(?:[ "]|$)'
+            for match in header_path:
+                if re.search(pat, self.aug.get(match).lower()):
+                    hsts_dirpath = match
+        if not hsts_dirpath:
+            err_msg = ("Certbot was unable to find the existing HSTS header "
+                       "from the VirtualHost at path {0}.").format(vhost.filep)
+            raise errors.PluginError(err_msg)
+
+        # Prepare the HSTS header value
+        hsts_maxage = "\"max-age={0}\"".format(nextstep_value)
+
+        # Update the header
+        # Our match statement was for string strict-transport-security, but
+        # we need to update the value instead. The next index is for the value
+        hsts_dirpath = hsts_dirpath.replace("arg[3]", "arg[4]")
+        self.aug.set(hsts_dirpath, hsts_maxage)
+        note_msg = ("Increasing HSTS max-age value to {0} for VirtualHost "
+                    "in {1}\n".format(nextstep_value, vhost.filep))
+        logger.debug(note_msg)
+        self.save_notes += note_msg
+        self.save(note_msg)
+
+    def _autohsts_fetch_state(self):
+        """
+        Populates the AutoHSTS state from the pluginstorage
+        """
+        try:
+            self._autohsts = self.storage.fetch("autohsts")
+        except KeyError:
+            self._autohsts = dict()
+
+    def _autohsts_save_state(self):
+        """
+        Saves the state of AutoHSTS object to pluginstorage
+        """
+        self.storage.put("autohsts", self._autohsts)
+        self.storage.save()
+
+    def _autohsts_vhost_in_lineage(self, vhost, lineage):
+        """
+        Searches AutoHSTS managed VirtualHosts that belong to the lineage.
+        Matches the private key path.
+        """
+
+        return bool(
+            self.parser.find_dir("SSLCertificateKeyFile",
+                                 lineage.key_path, vhost.path))
+
     def _enable_ocsp_stapling(self, ssl_vhost, unused_options):
         """Enables OCSP Stapling
 
@@ -1770,7 +1908,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         # There can be other RewriteRule directive lines in vhost config.
         # rewrite_args_dict keys are directive ids and the corresponding value
         # for each is a list of arguments to that directive.
-        rewrite_args_dict = defaultdict(list)
+        rewrite_args_dict = defaultdict(list)  # type: DefaultDict[str, List[str]]
         pat = r'(.*directive\[\d+\]).*'
         for match in rewrite_path:
             m = re.match(pat, match)
@@ -1864,7 +2002,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         if ssl_vhost.aliases:
             serveralias = "ServerAlias " + " ".join(ssl_vhost.aliases)
 
-        rewrite_rule_args = []
+        rewrite_rule_args = []  # type: List[str]
         if self.get_version() >= (2, 3, 9):
             rewrite_rule_args = constants.REWRITE_HTTPS_ARGS_WITH_END
         else:
@@ -2156,3 +2294,177 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         # to be modified.
         return common.install_version_controlled_file(options_ssl, options_ssl_digest,
             self.constant("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES)
+
+    def enable_autohsts(self, _unused_lineage, domains):
+        """
+        Enable the AutoHSTS enhancement for defined domains
+
+        :param _unused_lineage: Certificate lineage object, unused
+        :type _unused_lineage: certbot.storage.RenewableCert
+
+        :param domains: List of domains in certificate to enhance
+        :type domains: str
+        """
+
+        self._autohsts_fetch_state()
+        _enhanced_vhosts = []
+        for d in domains:
+            matched_vhosts = self.choose_vhosts(d, create_if_no_ssl=False)
+            # We should be handling only SSL vhosts for AutoHSTS
+            vhosts = [vhost for vhost in matched_vhosts if vhost.ssl]
+
+            if not vhosts:
+                msg_tmpl = ("Certbot was not able to find SSL VirtualHost for a "
+                            "domain {0} for enabling AutoHSTS enhancement.")
+                msg = msg_tmpl.format(d)
+                logger.warning(msg)
+                raise errors.PluginError(msg)
+            for vh in vhosts:
+                try:
+                    self._enable_autohsts_domain(vh)
+                    _enhanced_vhosts.append(vh)
+                except errors.PluginEnhancementAlreadyPresent:
+                    if vh in _enhanced_vhosts:
+                        continue
+                    msg = ("VirtualHost for domain {0} in file {1} has a " +
+                           "String-Transport-Security header present, exiting.")
+                    raise errors.PluginEnhancementAlreadyPresent(
+                        msg.format(d, vh.filep))
+        if _enhanced_vhosts:
+            note_msg = "Enabling AutoHSTS"
+            self.save(note_msg)
+            logger.info(note_msg)
+            self.restart()
+
+        # Save the current state to pluginstorage
+        self._autohsts_save_state()
+
+    def _enable_autohsts_domain(self, ssl_vhost):
+        """Do the initial AutoHSTS deployment to a vhost
+
+        :param ssl_vhost: The VirtualHost object to deploy the AutoHSTS
+        :type ssl_vhost: :class:`~certbot_apache.obj.VirtualHost` or None
+
+        :raises errors.PluginEnhancementAlreadyPresent: When already enhanced
+
+        """
+        # This raises the exception
+        self._verify_no_matching_http_header(ssl_vhost,
+                                             "Strict-Transport-Security")
+
+        if "headers_module" not in self.parser.modules:
+            self.enable_mod("headers")
+        # Prepare the HSTS header value
+        hsts_header = constants.HEADER_ARGS["Strict-Transport-Security"][:-1]
+        initial_maxage = constants.AUTOHSTS_STEPS[0]
+        hsts_header.append("\"max-age={0}\"".format(initial_maxage))
+
+        # Add ID to the VirtualHost for mapping back to it later
+        uniq_id = self.add_vhost_id(ssl_vhost)
+        self.save_notes += "Adding unique ID {0} to VirtualHost in {1}\n".format(
+            uniq_id, ssl_vhost.filep)
+        # Add the actual HSTS header
+        self.parser.add_dir(ssl_vhost.path, "Header", hsts_header)
+        note_msg = ("Adding gradually increasing HSTS header with initial value "
+                    "of {0} to VirtualHost in {1}\n".format(
+                        initial_maxage, ssl_vhost.filep))
+        self.save_notes += note_msg
+
+        # Save the current state to pluginstorage
+        self._autohsts[uniq_id] = {"laststep": 0, "timestamp": time.time()}
+
+    def update_autohsts(self, _unused_domain):
+        """
+        Increase the AutoHSTS values of VirtualHosts that the user has enabled
+        this enhancement for.
+
+        :param _unused_domain: Not currently used
+        :type _unused_domain: Not Available
+
+        """
+        self._autohsts_fetch_state()
+        if not self._autohsts:
+            # No AutoHSTS enabled for any domain
+            return
+        curtime = time.time()
+        save_and_restart = False
+        for id_str, config in list(self._autohsts.items()):
+            if config["timestamp"] + constants.AUTOHSTS_FREQ > curtime:
+                # Skip if last increase was < AUTOHSTS_FREQ ago
+                continue
+            nextstep = config["laststep"] + 1
+            if nextstep < len(constants.AUTOHSTS_STEPS):
+                # Have not reached the max value yet
+                try:
+                    vhost = self.find_vhost_by_id(id_str)
+                except errors.PluginError:
+                    msg = ("Could not find VirtualHost with ID {0}, disabling "
+                           "AutoHSTS for this VirtualHost").format(id_str)
+                    logger.warning(msg)
+                    # Remove the orphaned AutoHSTS entry from pluginstorage
+                    self._autohsts.pop(id_str)
+                    continue
+                self._autohsts_increase(vhost, id_str, nextstep)
+                msg = ("Increasing HSTS max-age value for VirtualHost with id "
+                       "{0}").format(id_str)
+                self.save_notes += msg
+                save_and_restart = True
+
+        if save_and_restart:
+            self.save("Increased HSTS max-age values")
+            self.restart()
+
+        self._autohsts_save_state()
+
+    def deploy_autohsts(self, lineage):
+        """
+        Checks if autohsts vhost has reached maximum auto-increased value
+        and changes the HSTS max-age to a high value.
+
+        :param lineage: Certificate lineage object
+        :type lineage: certbot.storage.RenewableCert
+        """
+        self._autohsts_fetch_state()
+        if not self._autohsts:
+            # No autohsts enabled for any vhost
+            return
+
+        vhosts = []
+        affected_ids = []
+        # Copy, as we are removing from the dict inside the loop
+        for id_str, config in list(self._autohsts.items()):
+            if config["laststep"]+1 >= len(constants.AUTOHSTS_STEPS):
+                # max value reached, try to make permanent
+                try:
+                    vhost = self.find_vhost_by_id(id_str)
+                except errors.PluginError:
+                    msg = ("VirtualHost with id {} was not found, unable to "
+                           "make HSTS max-age permanent.").format(id_str)
+                    logger.warning(msg)
+                    self._autohsts.pop(id_str)
+                    continue
+                if self._autohsts_vhost_in_lineage(vhost, lineage):
+                    vhosts.append(vhost)
+                    affected_ids.append(id_str)
+
+        save_and_restart = False
+        for vhost in vhosts:
+            self._autohsts_write(vhost, constants.AUTOHSTS_PERMANENT)
+            msg = ("Strict-Transport-Security max-age value for "
+                   "VirtualHost in {0} was made permanent.").format(vhost.filep)
+            logger.debug(msg)
+            self.save_notes += msg+"\n"
+            save_and_restart = True
+
+        if save_and_restart:
+            self.save("Made HSTS max-age permanent")
+            self.restart()
+
+        for id_str in affected_ids:
+            self._autohsts.pop(id_str)
+
+        # Update AutoHSTS storage (We potentially removed vhosts from managed)
+        self._autohsts_save_state()
+
+
+AutoHSTSEnhancement.register(ApacheConfigurator)  # pylint: disable=no-member

certbot-apache/certbot_apache/constants.py

@@ -48,3 +48,16 @@ UIR_ARGS = ["always", "set", "Content-Security-Policy",
 
 HEADER_ARGS = {"Strict-Transport-Security": HSTS_ARGS,
                "Upgrade-Insecure-Requests": UIR_ARGS}
+
+AUTOHSTS_STEPS = [60, 300, 900, 3600, 21600, 43200, 86400]
+"""AutoHSTS increase steps: 1min, 5min, 15min, 1h, 6h, 12h, 24h"""
+
+AUTOHSTS_PERMANENT = 31536000
+"""Value for the last max-age of HSTS"""
+
+AUTOHSTS_FREQ = 172800
+"""Minimum time since last increase to perform a new one: 48h"""
+
+MANAGED_COMMENT = "DO NOT REMOVE - Managed by Certbot"
+MANAGED_COMMENT_ID = MANAGED_COMMENT+", VirtualHost id: {0}"
+"""Managed by Certbot comments and the VirtualHost identification template"""

certbot-apache/certbot_apache/http_01.py

@@ -2,9 +2,10 @@
 import logging
 import os
 
+from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
 from certbot import errors
-
 from certbot.plugins import common
+from certbot_apache.obj import VirtualHost  # pylint: disable=unused-import
 
 logger = logging.getLogger(__name__)
 
@@ -51,7 +52,7 @@ class ApacheHttp01(common.TLSSNI01):
         self.challenge_dir = os.path.join(
             self.configurator.config.work_dir,
             "http_challenges")
-        self.moded_vhosts = set()
+        self.moded_vhosts = set()  # type: Set[VirtualHost]
 
     def perform(self):
         """Perform all HTTP-01 challenges."""

certbot-apache/certbot_apache/obj.py

@@ -1,6 +1,7 @@
 """Module contains classes used by the Apache Configurator."""
 import re
 
+from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module
 from certbot.plugins import common
 
 
@@ -140,7 +141,7 @@ class VirtualHost(object):  # pylint: disable=too-few-public-methods
 
     def get_names(self):
         """Return a set of all names."""
-        all_names = set()
+        all_names = set()  # type: Set[str]
         all_names.update(self.aliases)
         # Strip out any scheme:// and <port> field from servername
         if self.name is not None:
@@ -251,7 +252,7 @@ class VirtualHost(object):  # pylint: disable=too-few-public-methods
 
         # already_found acts to keep everything very conservative.
         # Don't allow multiple ip:ports in same set.
-        already_found = set()
+        already_found = set()  # type: Set[str]
 
         for addr in vhost.addrs:
             for local_addr in self.addrs:

certbot-apache/certbot_apache/override_centos.py

@@ -47,10 +47,10 @@ class CentOSParser(parser.ApacheParser):
         self.sysconfig_filep = "/etc/sysconfig/httpd"
         super(CentOSParser, self).__init__(*args, **kwargs)
 
-    def update_runtime_variables(self, *args, **kwargs):
+    def update_runtime_variables(self):
         """ Override for update_runtime_variables for custom parsing """
         # Opportunistic, works if SELinux not enforced
-        super(CentOSParser, self).update_runtime_variables(*args, **kwargs)
+        super(CentOSParser, self).update_runtime_variables()
         self.parse_sysconfig_var()
 
     def parse_sysconfig_var(self):

certbot-apache/certbot_apache/parser.py

@@ -9,12 +9,14 @@ import sys
 
 import six
 
+from acme.magic_typing import Dict, List, Set  # pylint: disable=unused-import, no-name-in-module
 from certbot import errors
 
 logger = logging.getLogger(__name__)
 
 
 class ApacheParser(object):
+    # pylint: disable=too-many-public-methods
     """Class handles the fine details of parsing the Apache Configuration.
 
     .. todo:: Make parsing general... remove sites-available etc...
@@ -38,9 +40,9 @@ class ApacheParser(object):
         # issues with aug.load() after adding new files / defines to parse tree
         self.configurator = configurator
 
-        self.modules = set()
-        self.parser_paths = {}
-        self.variables = {}
+        self.modules = set()  # type: Set[str]
+        self.parser_paths = {}  # type: Dict[str, List[str]]
+        self.variables = {}  # type: Dict[str, str]
 
         self.aug = aug
         # Find configuration root and make sure augeas can parse it.
@@ -119,7 +121,7 @@ class ApacheParser(object):
             the iteration issue.  Else... parse and enable mods at same time.
 
         """
-        mods = set()
+        mods = set()  # type: Set[str]
         matches = self.find_dir("LoadModule")
         iterator = iter(matches)
         # Make sure prev_size != cur_size for do: while: iteration
@@ -349,6 +351,37 @@ class ApacheParser(object):
         else:
             self.aug.set(first_dir + "/arg", args)
 
+    def add_comment(self, aug_conf_path, comment):
+        """Adds the comment to the augeas path
+
+        :param str aug_conf_path: Augeas configuration path to add directive
+        :param str comment: Comment content
+
+        """
+        self.aug.set(aug_conf_path + "/#comment[last() + 1]", comment)
+
+    def find_comments(self, arg, start=None):
+        """Finds a comment with specified content from the provided DOM path
+
+        :param str arg: Comment content to search
+        :param str start: Beginning Augeas path to begin looking
+
+        :returns: List of augeas paths containing the comment content
+        :rtype: list
+
+        """
+        if not start:
+            start = get_aug_path(self.root)
+
+        comments = self.aug.match("%s//*[label() = '#comment']" % start)
+
+        results = []
+        for comment in comments:
+            c_content = self.aug.get(comment)
+            if c_content and arg in c_content:
+                results.append(comment)
+        return results
+
     def find_dir(self, directive, arg=None, start=None, exclude=True):
         """Finds directive in the configuration.
 
@@ -408,7 +441,7 @@ class ApacheParser(object):
         else:
             arg_suffix = "/*[self::arg=~regexp('%s')]" % case_i(arg)
 
-        ordered_matches = []
+        ordered_matches = []  # type: List[str]
 
         # TODO: Wildcards should be included in alphabetical order
         # https://httpd.apache.org/docs/2.4/mod/core.html#include

certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test

@@ -46,6 +46,7 @@ function Cleanup() {
 
 # if our environment asks us to enable modules, do our best!
 if [ "$1" = --debian-modules ] ; then
+    sudo apt-get install -y apache2
     sudo apt-get install -y libapache2-mod-wsgi
     sudo apt-get install -y libapache2-mod-macro
 

certbot-apache/certbot_apache/tests/autohsts_test.py

@@ -0,0 +1,181 @@
+# pylint: disable=too-many-public-methods,too-many-lines
+"""Test for certbot_apache.configurator AutoHSTS functionality"""
+import re
+import unittest
+import mock
+# six is used in mock.patch()
+import six  # pylint: disable=unused-import
+
+from certbot import errors
+from certbot_apache import constants
+from certbot_apache.tests import util
+
+
+class AutoHSTSTest(util.ApacheTest):
+    """Tests for AutoHSTS feature"""
+    # pylint: disable=protected-access
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        super(AutoHSTSTest, self).setUp()
+
+        self.config = util.get_apache_configurator(
+            self.config_path, self.vhost_path, self.config_dir, self.work_dir)
+        self.config.parser.modules.add("headers_module")
+        self.config.parser.modules.add("mod_headers.c")
+        self.config.parser.modules.add("ssl_module")
+        self.config.parser.modules.add("mod_ssl.c")
+
+        self.vh_truth = util.get_vh_truth(
+            self.temp_dir, "debian_apache_2_4/multiple_vhosts")
+
+    def get_autohsts_value(self, vh_path):
+        """ Get value from Strict-Transport-Security header """
+        header_path = self.config.parser.find_dir("Header", None, vh_path)
+        if header_path:
+            pat = '(?:[ "]|^)(strict-transport-security)(?:[ "]|$)'
+            for head in header_path:
+                if re.search(pat, self.config.parser.aug.get(head).lower()):
+                    return self.config.parser.aug.get(head.replace("arg[3]",
+                                                                   "arg[4]"))
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod")
+    def test_autohsts_enable_headers_mod(self, mock_enable, _restart):
+        self.config.parser.modules.discard("headers_module")
+        self.config.parser.modules.discard("mod_header.c")
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        self.assertTrue(mock_enable.called)
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    def test_autohsts_deploy_already_exists(self, _restart):
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        self.assertRaises(errors.PluginEnhancementAlreadyPresent,
+                          self.config.enable_autohsts,
+                          mock.MagicMock(), ["ocspvhost.com"])
+
+    @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0)
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    def test_autohsts_increase(self, _mock_restart):
+        maxage = "\"max-age={0}\""
+        initial_val = maxage.format(constants.AUTOHSTS_STEPS[0])
+        inc_val = maxage.format(constants.AUTOHSTS_STEPS[1])
+
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        # Verify initial value
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          initial_val)
+        # Increase
+        self.config.update_autohsts(mock.MagicMock())
+        # Verify increased value
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          inc_val)
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator._autohsts_increase")
+    def test_autohsts_increase_noop(self, mock_increase, _restart):
+        maxage = "\"max-age={0}\""
+        initial_val = maxage.format(constants.AUTOHSTS_STEPS[0])
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        # Verify initial value
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          initial_val)
+
+        self.config.update_autohsts(mock.MagicMock())
+        # Freq not patched, so value shouldn't increase
+        self.assertFalse(mock_increase.called)
+
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0)
+    def test_autohsts_increase_no_header(self, _restart):
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        # Remove the header
+        dir_locs = self.config.parser.find_dir("Header", None,
+                                              self.vh_truth[7].path)
+        dir_loc = "/".join(dir_locs[0].split("/")[:-1])
+        self.config.parser.aug.remove(dir_loc)
+        self.assertRaises(errors.PluginError,
+                          self.config.update_autohsts,
+                          mock.MagicMock())
+
+    @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0)
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    def test_autohsts_increase_and_make_permanent(self, _mock_restart):
+        maxage = "\"max-age={0}\""
+        max_val = maxage.format(constants.AUTOHSTS_PERMANENT)
+        mock_lineage = mock.MagicMock()
+        mock_lineage.key_path = "/etc/apache2/ssl/key-certbot_15.pem"
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        for i in range(len(constants.AUTOHSTS_STEPS)-1):
+            # Ensure that value is not made permanent prematurely
+            self.config.deploy_autohsts(mock_lineage)
+            self.assertNotEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                                 max_val)
+            self.config.update_autohsts(mock.MagicMock())
+            # Value should match pre-permanent increment step
+            cur_val = maxage.format(constants.AUTOHSTS_STEPS[i+1])
+            self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                              cur_val)
+        # Make permanent
+        self.config.deploy_autohsts(mock_lineage)
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          max_val)
+
+    def test_autohsts_update_noop(self):
+        with mock.patch("time.time") as mock_time:
+            # Time mock is used to make sure that the execution does not
+            # continue when no autohsts entries exist in pluginstorage
+            self.config.update_autohsts(mock.MagicMock())
+            self.assertFalse(mock_time.called)
+
+    def test_autohsts_make_permanent_noop(self):
+        self.config.storage.put = mock.MagicMock()
+        self.config.deploy_autohsts(mock.MagicMock())
+        # Make sure that the execution does not continue when no entries in store
+        self.assertFalse(self.config.storage.put.called)
+
+    @mock.patch("certbot_apache.display_ops.select_vhost")
+    def test_autohsts_no_ssl_vhost(self, mock_select):
+        mock_select.return_value = self.vh_truth[0]
+        with mock.patch("certbot_apache.configurator.logger.warning") as mock_log:
+            self.assertRaises(errors.PluginError,
+                              self.config.enable_autohsts,
+                              mock.MagicMock(), "invalid.example.com")
+            self.assertTrue(
+                "Certbot was not able to find SSL" in mock_log.call_args[0][0])
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.add_vhost_id")
+    def test_autohsts_dont_enhance_twice(self, mock_id, _restart):
+        mock_id.return_value = "1234567"
+        self.config.enable_autohsts(mock.MagicMock(),
+                                    ["ocspvhost.com", "ocspvhost.com"])
+        self.assertEquals(mock_id.call_count, 1)
+
+    def test_autohsts_remove_orphaned(self):
+        # pylint: disable=protected-access
+        self.config._autohsts_fetch_state()
+        self.config._autohsts["orphan_id"] = {"laststep": 0, "timestamp": 0}
+
+        self.config._autohsts_save_state()
+        self.config.update_autohsts(mock.MagicMock())
+        self.assertFalse("orphan_id" in self.config._autohsts)
+        # Make sure it's removed from the pluginstorage file as well
+        self.config._autohsts = None
+        self.config._autohsts_fetch_state()
+        self.assertFalse(self.config._autohsts)
+
+    def test_autohsts_make_permanent_vhost_not_found(self):
+        # pylint: disable=protected-access
+        self.config._autohsts_fetch_state()
+        self.config._autohsts["orphan_id"] = {"laststep": 999, "timestamp": 0}
+        self.config._autohsts_save_state()
+        with mock.patch("certbot_apache.configurator.logger.warning") as mock_log:
+            self.config.deploy_autohsts(mock.MagicMock())
+            self.assertTrue(mock_log.called)
+            self.assertTrue(
+                "VirtualHost with id orphan_id was not" in mock_log.call_args[0][0])
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/configurator_test.py

@@ -353,14 +353,11 @@ class MultipleVhostsTest(util.ApacheTest):
 
         self.config.parser.find_dir = mock_find_dir
         mock_add.reset_mock()
-
         self.config._add_dummy_ssl_directives(self.vh_truth[0])  # pylint: disable=protected-access
-        tried_to_add = []
         for a in mock_add.call_args_list:
-            tried_to_add.append(a[0][1] == "Include" and
-                                a[0][2] == self.config.mod_ssl_conf)
-        # Include shouldn't be added, as patched find_dir "finds" existing one
-        self.assertFalse(any(tried_to_add))
+            if a[0][1] == "Include" and a[0][2] == self.config.mod_ssl_conf:
+                self.fail("Include shouldn't be added, as patched find_dir 'finds' existing one") \
+                    # pragma: no cover
 
     def test_deploy_cert(self):
         self.config.parser.modules.add("ssl_module")
@@ -1490,6 +1487,21 @@ class MultipleVhostsTest(util.ApacheTest):
                             "Upgrade-Insecure-Requests")
         self.assertTrue(mock_choose.called)
 
+    def test_add_vhost_id(self):
+        for vh in [self.vh_truth[0], self.vh_truth[1], self.vh_truth[2]]:
+            vh_id = self.config.add_vhost_id(vh)
+            self.assertEqual(vh, self.config.find_vhost_by_id(vh_id))
+
+    def test_find_vhost_by_id_404(self):
+        self.assertRaises(errors.PluginError,
+                          self.config.find_vhost_by_id,
+                          "nonexistent")
+
+    def test_add_vhost_id_already_exists(self):
+        first_id = self.config.add_vhost_id(self.vh_truth[0])
+        second_id = self.config.add_vhost_id(self.vh_truth[0])
+        self.assertEqual(first_id, second_id)
+
 
 class AugeasVhostsTest(util.ApacheTest):
     """Test vhosts with illegal names dependent on augeas version."""

certbot-apache/certbot_apache/tests/http_01_test.py

@@ -4,12 +4,12 @@ import os
 import unittest
 
 from acme import challenges
+from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
 
 from certbot import achallenges
 from certbot import errors
 
 from certbot.tests import acme_util
-
 from certbot_apache.tests import util
 
 
@@ -23,7 +23,7 @@ class ApacheHttp01Test(util.ApacheTest):
         super(ApacheHttp01Test, self).setUp(*args, **kwargs)
 
         self.account_key = self.rsa512jwk
-        self.achalls = []
+        self.achalls = []  # type: List[achallenges.KeyAuthorizationAnnotatedChallenge]
         vh_truth = util.get_vh_truth(
             self.temp_dir, "debian_apache_2_4/multiple_vhosts")
         # Takes the vhosts for encryption-example.demo, certbot.demo, and

certbot-apache/certbot_apache/tests/parser_test.py

@@ -299,6 +299,13 @@ class BasicParserTest(util.ParserTest):
             errors.MisconfigurationError,
             self.parser.update_runtime_variables)
 
+    def test_add_comment(self):
+        from certbot_apache.parser import get_aug_path
+        self.parser.add_comment(get_aug_path(self.parser.loc["name"]), "123456")
+        comm = self.parser.find_comments("123456")
+        self.assertEquals(len(comm), 1)
+        self.assertTrue(self.parser.loc["name"] in comm[0])
+
 
 class ParserInitTest(util.ApacheTest):
     def setUp(self):  # pylint: disable=arguments-differ

certbot-apache/certbot_apache/tests/util.py

@@ -87,7 +87,6 @@ class ParserTest(ApacheTest):
 def get_apache_configurator(  # pylint: disable=too-many-arguments, too-many-locals
         config_path, vhost_path,
         config_dir, work_dir, version=(2, 4, 7),
-        conf=None,
         os_info="generic",
         conf_vhost_path=None):
     """Create an Apache Configurator with the specified options.
@@ -133,10 +132,6 @@ def get_apache_configurator(  # pylint: disable=too-many-arguments, too-many-loc
                         config_class = configurator.ApacheConfigurator
                     config = config_class(config=mock_le_config, name="apache",
                         version=version)
-                    # This allows testing scripts to set it a bit more
-                    # quickly
-                    if conf is not None:
-                        config.conf = conf  # pragma: no cover
 
                     config.prepare()
     return config

certbot-apache/certbot_apache/tls_sni_01.py

@@ -3,6 +3,7 @@
 import os
 import logging
 
+from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
 from certbot.plugins import common
 from certbot.errors import PluginError, MissingCommandlineFlag
 
@@ -93,7 +94,7 @@ class ApacheTlsSni01(common.TLSSNI01):
         :rtype: set
 
         """
-        addrs = set()
+        addrs = set()  # type: Set[obj.Addr]
         config_text = "<IfModule mod_ssl.c>\n"
 
         for achall in self.achalls:

certbot-apache/local-oldest-requirements.txt

@@ -1,2 +1,2 @@
-acme[dev]==0.21.1
-certbot[dev]==0.21.1
+acme[dev]==0.25.0
+-e .[dev]

certbot-apache/setup.py

@@ -1,16 +1,14 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.24.0.dev0'
+version = '0.26.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    'acme>=0.21.1',
-    'certbot>=0.21.1',
+    'acme>=0.25.0',
+    'certbot>=0.26.0.dev0',
     'mock',
     'python-augeas',
     'setuptools',

certbot-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.23.0"
+LE_AUTO_VERSION="0.25.1"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -1055,9 +1055,11 @@ cffi==1.10.0 \
     --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \
     --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5
 ConfigArgParse==0.12.0 \
-    --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339
+    --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \
+    --no-binary ConfigArgParse
 configobj==5.0.6 \
-    --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902
+    --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \
+    --no-binary configobj
 cryptography==2.0.2 \
     --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \
     --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \
@@ -1089,7 +1091,7 @@ cryptography==2.0.2 \
     --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \
     --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \
     --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab
-enum34==1.1.2 \
+enum34==1.1.2 ; python_version < '3.4' \
     --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \
     --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501
 funcsigs==1.0.2 \
@@ -1112,7 +1114,8 @@ mock==1.3.0 \
     --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \
     --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6
 ordereddict==1.1 \
-    --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f
+    --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \
+    --no-binary ordereddict
 packaging==16.8 \
     --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \
     --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e
@@ -1138,7 +1141,8 @@ pyRFC3339==1.0 \
     --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \
     --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535
 python-augeas==0.5.0 \
-    --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2
+    --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \
+    --no-binary python-augeas
 pytz==2015.7 \
     --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \
     --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \
@@ -1166,9 +1170,11 @@ unittest2==1.1.0 \
     --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \
     --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579
 zope.component==4.2.2 \
-    --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a
+    --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \
+    --no-binary zope.component
 zope.event==4.1.0 \
-    --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786
+    --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \
+    --no-binary zope.event
 zope.interface==4.1.3 \
     --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \
     --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \
@@ -1187,6 +1193,9 @@ zope.interface==4.1.3 \
     --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \
     --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \
     --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392
+requests-toolbelt==0.8.0 \
+    --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \
+    --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5
 
 # Contains the requirements for the letsencrypt package.
 #
@@ -1199,18 +1208,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.23.0 \
-    --hash=sha256:66c42cf780ddbf582ecc52aa6a61242450a2650227b436ad0d260685c4ef8a49 \
-    --hash=sha256:6cff4c5da1228661ccaf95195064cb29e6cdf80913193bdb2eb20e164c76053e
-acme==0.23.0 \
-    --hash=sha256:02e9b596bd3bf8f0733d6d43ec2464ac8185a000acb58d2b4fd9e19223bbbf0b \
-    --hash=sha256:08c16635578507f526c338b3418c1147a9f015bf2d366abd51f38918703b4550
-certbot-apache==0.23.0 \
-    --hash=sha256:50077742d2763b7600dfda618eb89c870aeea5e6a4c00f60157877f7a7d81f7c \
-    --hash=sha256:6b7acec243e224de5268d46c2597277586dffa55e838c252b6931c30d549028e
-certbot-nginx==0.23.0 \
-    --hash=sha256:f12c21bbe3eb955ca533f1da96d28c6310378b138e844d83253562e18b6cbb32 \
-    --hash=sha256:cadf14e4bd504d9ce5987a5ec6dbd8e136638e55303ad5dc81dcb723ddd64324
+certbot==0.25.1 \
+    --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \
+    --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e
+acme==0.25.1 \
+    --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \
+    --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734
+certbot-apache==0.25.1 \
+    --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \
+    --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a
+certbot-nginx==0.25.1 \
+    --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \
+    --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------

certbot-compatibility-test/certbot_compatibility_test/test_driver.py

@@ -15,6 +15,7 @@ from six.moves import xrange  # pylint: disable=import-error,redefined-builtin
 from acme import challenges
 from acme import crypto_util
 from acme import messages
+from acme.magic_typing import List, Tuple  # pylint: disable=unused-import, no-name-in-module
 from certbot import achallenges
 from certbot import errors as le_errors
 from certbot.tests import acme_util
@@ -52,9 +53,8 @@ def test_authenticator(plugin, config, temp_dir):
 
     try:
         responses = plugin.perform(achalls)
-    except le_errors.Error as error:
-        logger.error("Performing challenges on %s caused an error:", config)
-        logger.exception(error)
+    except le_errors.Error:
+        logger.error("Performing challenges on %s caused an error:", config, exc_info=True)
         return False
 
     success = True
@@ -82,9 +82,8 @@ def test_authenticator(plugin, config, temp_dir):
     if success:
         try:
             plugin.cleanup(achalls)
-        except le_errors.Error as error:
-            logger.error("Challenge cleanup for %s caused an error:", config)
-            logger.exception(error)
+        except le_errors.Error:
+            logger.error("Challenge cleanup for %s caused an error:", config, exc_info=True)
             success = False
 
         if _dirs_are_unequal(config, backup):
@@ -147,9 +146,8 @@ def test_deploy_cert(plugin, temp_dir, domains):
         try:
             plugin.deploy_cert(domain, cert_path, util.KEY_PATH, cert_path, cert_path)
             plugin.save()  # Needed by the Apache plugin
-        except le_errors.Error as error:
-            logger.error("**** Plugin failed to deploy certificate for %s:", domain)
-            logger.exception(error)
+        except le_errors.Error:
+            logger.error("**** Plugin failed to deploy certificate for %s:", domain, exc_info=True)
             return False
 
     if not _save_and_restart(plugin, "deployed"):
@@ -179,7 +177,7 @@ def test_enhancements(plugin, domains):
                      "enhancements")
         return False
 
-    domains_and_info = [(domain, []) for domain in domains]
+    domains_and_info = [(domain, []) for domain in domains]  # type: List[Tuple[str, List[bool]]]
 
     for domain, info in domains_and_info:
         try:
@@ -192,10 +190,9 @@ def test_enhancements(plugin, domains):
             # Don't immediately fail because a redirect may already be enabled
             logger.warning("*** Plugin failed to enable redirect for %s:", domain)
             logger.warning("%s", error)
-        except le_errors.Error as error:
+        except le_errors.Error:
             logger.error("*** An error occurred while enabling redirect for %s:",
-                         domain)
-            logger.exception(error)
+                         domain, exc_info=True)
 
     if not _save_and_restart(plugin, "enhanced"):
         return False
@@ -222,9 +219,8 @@ def _save_and_restart(plugin, title=None):
         plugin.save(title)
         plugin.restart()
         return True
-    except le_errors.Error as error:
-        logger.error("*** Plugin failed to save and restart server:")
-        logger.exception(error)
+    except le_errors.Error:
+        logger.error("*** Plugin failed to save and restart server:", exc_info=True)
         return False
 
 
@@ -232,9 +228,8 @@ def test_rollback(plugin, config, backup):
     """Tests the rollback checkpoints function"""
     try:
         plugin.rollback_checkpoints(1337)
-    except le_errors.Error as error:
-        logger.error("*** Plugin raised an exception during rollback:")
-        logger.exception(error)
+    except le_errors.Error:
+        logger.error("*** Plugin raised an exception during rollback:", exc_info=True)
         return False
 
     if _dirs_are_unequal(config, backup):
@@ -263,21 +258,21 @@ def _dirs_are_unequal(dir1, dir2):
             logger.error("The following files and directories are only "
                          "present in one directory")
             if dircmp.left_only:
-                logger.error(dircmp.left_only)
+                logger.error(str(dircmp.left_only))
             else:
-                logger.error(dircmp.right_only)
+                logger.error(str(dircmp.right_only))
             return True
         elif dircmp.common_funny or dircmp.funny_files:
             logger.error("The following files and directories could not be "
                          "compared:")
             if dircmp.common_funny:
-                logger.error(dircmp.common_funny)
+                logger.error(str(dircmp.common_funny))
             else:
-                logger.error(dircmp.funny_files)
+                logger.error(str(dircmp.funny_files))
             return True
         elif dircmp.diff_files:
             logger.error("The following files differ:")
-            logger.error(dircmp.diff_files)
+            logger.error(str(dircmp.diff_files))
             return True
 
         for subdir in dircmp.subdirs.itervalues():
@@ -354,9 +349,8 @@ def main():
                     success = test_authenticator(plugin, config, temp_dir)
                 if success and args.install:
                     success = test_installer(args, plugin, config, temp_dir)
-            except errors.Error as error:
-                logger.error("Tests on %s raised:", config)
-                logger.exception(error)
+            except errors.Error:
+                logger.error("Tests on %s raised:", config, exc_info=True)
                 success = False
 
             if success:

certbot-compatibility-test/certbot_compatibility_test/util.py

@@ -26,12 +26,12 @@ def create_le_config(parent_dir):
     config = copy.deepcopy(constants.CLI_DEFAULTS)
 
     le_dir = os.path.join(parent_dir, "certbot")
-    config["config_dir"] = os.path.join(le_dir, "config")
-    config["work_dir"] = os.path.join(le_dir, "work")
-    config["logs_dir"] = os.path.join(le_dir, "logs_dir")
-    os.makedirs(config["config_dir"])
-    os.mkdir(config["work_dir"])
-    os.mkdir(config["logs_dir"])
+    os.mkdir(le_dir)
+    for dir_name in ("config", "logs", "work"):
+        full_path = os.path.join(le_dir, dir_name)
+        os.mkdir(full_path)
+        full_name = dir_name + "_dir"
+        config[full_name] = full_path
 
     config["domains"] = None
 

certbot-compatibility-test/certbot_compatibility_test/validator.py

@@ -33,7 +33,7 @@ class Validator(object):
         try:
             presented_cert = crypto_util.probe_sni(name, host, port)
         except acme_errors.Error as error:
-            logger.exception(error)
+            logger.exception(str(error))
             return False
 
         return presented_cert.digest("sha256") == cert.digest("sha256")
@@ -86,8 +86,7 @@ class Validator(object):
             return False
 
         try:
-            _, max_age_value = max_age[0]
-            max_age_value = int(max_age_value)
+            max_age_value = int(max_age[0][1])
         except ValueError:
             logger.error("Server responded with invalid HSTS header field")
             return False

certbot-compatibility-test/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.24.0.dev0'
+version = '0.26.0.dev0'
 
 install_requires = [
     'certbot',

certbot-dns-cloudflare/setup.py

@@ -1,10 +1,8 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.24.0.dev0'
+version = '0.26.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-cloudxns/setup.py

@@ -1,10 +1,8 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.24.0.dev0'
+version = '0.26.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-digitalocean/setup.py

@@ -1,10 +1,8 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.24.0.dev0'
+version = '0.26.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-dnsimple/setup.py

@@ -1,10 +1,8 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.24.0.dev0'
+version = '0.26.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-dnsmadeeasy/setup.py

@@ -1,10 +1,8 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.24.0.dev0'
+version = '0.26.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-google/setup.py

@@ -1,10 +1,8 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.24.0.dev0'
+version = '0.26.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-luadns/setup.py

@@ -1,10 +1,8 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.24.0.dev0'
+version = '0.26.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-nsone/setup.py

@@ -1,10 +1,8 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.24.0.dev0'
+version = '0.26.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-rfc2136/setup.py

@@ -1,10 +1,8 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.24.0.dev0'
+version = '0.26.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-route53/certbot_dns_route53/dns_route53.py

@@ -11,6 +11,8 @@ from certbot import errors
 from certbot import interfaces
 from certbot.plugins import dns_common
 
+from acme.magic_typing import DefaultDict, List, Dict # pylint: disable=unused-import, no-name-in-module
+
 logger = logging.getLogger(__name__)
 
 INSTRUCTIONS = (
@@ -34,7 +36,7 @@ class Authenticator(dns_common.DNSAuthenticator):
     def __init__(self, *args, **kwargs):
         super(Authenticator, self).__init__(*args, **kwargs)
         self.r53 = boto3.client("route53")
-        self._resource_records = collections.defaultdict(list)
+        self._resource_records = collections.defaultdict(list) # type: DefaultDict[str, List[Dict[str, str]]]
 
     def more_info(self):  # pylint: disable=missing-docstring,no-self-use
         return "Solve a DNS01 challenge using AWS Route53"
@@ -42,14 +44,26 @@ class Authenticator(dns_common.DNSAuthenticator):
     def _setup_credentials(self):
         pass
 
-    def _perform(self, domain, validation_domain_name, validation):
-        try:
-            change_id = self._change_txt_record("UPSERT", validation_domain_name, validation)
+    def _perform(self, domain, validation_domain_name, validation): # pylint: disable=missing-docstring
+        pass
 
-            self._wait_for_change(change_id)
+    def perform(self, achalls):
+        self._attempt_cleanup = True
+
+        try:
+            change_ids = [
+                self._change_txt_record("UPSERT",
+                  achall.validation_domain_name(achall.domain),
+                  achall.validation(achall.account_key))
+                for achall in achalls
+            ]
+
+            for change_id in change_ids:
+                self._wait_for_change(change_id)
         except (NoCredentialsError, ClientError) as e:
             logger.debug('Encountered error during perform: %s', e, exc_info=True)
             raise errors.PluginError("\n".join([str(e), INSTRUCTIONS]))
+        return [achall.response(achall.account_key) for achall in achalls]
 
     def _cleanup(self, domain, validation_domain_name, validation):
         try:

certbot-dns-route53/local-oldest-requirements.txt

@@ -1,2 +1,2 @@
-acme[dev]==0.21.1
+acme[dev]==0.25.0
 certbot[dev]==0.21.1

certbot-dns-route53/setup.py

@@ -1,14 +1,12 @@
-import sys
-
-from distutils.core import setup
+from setuptools import setup
 from setuptools import find_packages
 
-version = '0.24.0.dev0'
+version = '0.26.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    'acme>=0.21.1',
+    'acme>=0.25.0',
     'certbot>=0.21.1',
     'boto3',
     'mock',

certbot-nginx/certbot_nginx/configurator.py

@@ -28,6 +28,9 @@ from certbot_nginx import nginxparser
 from certbot_nginx import parser
 from certbot_nginx import tls_sni_01
 from certbot_nginx import http_01
+from certbot_nginx import obj # pylint: disable=unused-import
+from acme.magic_typing import List, Dict, Set # pylint: disable=unused-import, no-name-in-module
+
 
 
 logger = logging.getLogger(__name__)
@@ -66,8 +69,9 @@ class NginxConfigurator(common.Installer):
 
     @classmethod
     def add_parser_arguments(cls, add):
+        default_server_root = _determine_default_server_root()
         add("server-root", default=constants.CLI_DEFAULTS["server_root"],
-            help="Nginx server root directory.")
+            help="Nginx server root directory. (default: %s)" % default_server_root)
         add("ctl", default=constants.CLI_DEFAULTS["ctl"], help="Path to the "
             "'nginx' binary, used for 'configtest' and retrieving nginx "
             "version number.")
@@ -98,8 +102,8 @@ class NginxConfigurator(common.Installer):
 
         # List of vhosts configured per wildcard domain on this run.
         # used by deploy_cert() and enhance()
-        self._wildcard_vhosts = {}
-        self._wildcard_redirect_vhosts = {}
+        self._wildcard_vhosts = {} # type: Dict[str, List[obj.VirtualHost]]
+        self._wildcard_redirect_vhosts = {} # type: Dict[str, List[obj.VirtualHost]]
 
         # Add number of outstanding challenges
         self._chall_out = 0
@@ -286,14 +290,15 @@ class NginxConfigurator(common.Installer):
         if not vhosts:
             if create_if_no_match:
                 # result will not be [None] because it errors on failure
-                vhosts = [self._vhost_from_duplicated_default(target_name)]
+                vhosts = [self._vhost_from_duplicated_default(target_name, True,
+                    str(self.config.tls_sni_01_port))]
             else:
                 # No matches. Raise a misconfiguration error.
                 raise errors.MisconfigurationError(
                             ("Cannot find a VirtualHost matching domain %s. "
                              "In order for Certbot to correctly perform the challenge "
                              "please add a corresponding server_name directive to your "
-                             "nginx configuration: "
+                             "nginx configuration for every domain on your certificate: "
                              "https://nginx.org/en/docs/http/server_names.html") % (target_name))
         # Note: if we are enhancing with ocsp, vhost should already be ssl.
         for vhost in vhosts:
@@ -329,9 +334,12 @@ class NginxConfigurator(common.Installer):
                     ipv6only_present = True
         return (ipv6_active, ipv6only_present)
 
-    def _vhost_from_duplicated_default(self, domain, port=None):
+    def _vhost_from_duplicated_default(self, domain, allow_port_mismatch, port):
+        """if allow_port_mismatch is False, only server blocks with matching ports will be
+           used as a default server block template.
+        """
         if self.new_vhost is None:
-            default_vhost = self._get_default_vhost(port)
+            default_vhost = self._get_default_vhost(domain, allow_port_mismatch, port)
             self.new_vhost = self.parser.duplicate_vhost(default_vhost,
                 remove_singleton_listen_params=True)
             self.new_vhost.names = set()
@@ -347,24 +355,29 @@ class NginxConfigurator(common.Installer):
             name_block[0].append(name)
         self.parser.update_or_add_server_directives(vhost, name_block)
 
-    def _get_default_vhost(self, port):
+    def _get_default_vhost(self, domain, allow_port_mismatch, port):
+        """Helper method for _vhost_from_duplicated_default; see argument documentation there"""
         vhost_list = self.parser.get_vhosts()
         # if one has default_server set, return that one
-        default_vhosts = []
+        all_default_vhosts = []
+        port_matching_vhosts = []
         for vhost in vhost_list:
             for addr in vhost.addrs:
                 if addr.default:
-                    if port is None or self._port_matches(port, addr.get_port()):
-                        default_vhosts.append(vhost)
-                        break
+                    all_default_vhosts.append(vhost)
+                    if self._port_matches(port, addr.get_port()):
+                        port_matching_vhosts.append(vhost)
+                    break
 
-        if len(default_vhosts) == 1:
-            return default_vhosts[0]
+        if len(port_matching_vhosts) == 1:
+            return port_matching_vhosts[0]
+        elif len(all_default_vhosts) == 1 and allow_port_mismatch:
+            return all_default_vhosts[0]
 
         # TODO: present a list of vhosts for user to choose from
 
         raise errors.MisconfigurationError("Could not automatically find a matching server"
-            " block. Set the `server_name` directive to use the Nginx installer.")
+            " block for %s. Set the `server_name` directive to use the Nginx installer." % domain)
 
     def _get_ranked_matches(self, target_name):
         """Returns a ranked list of vhosts that match target_name.
@@ -468,7 +481,7 @@ class NginxConfigurator(common.Installer):
             matches = self._get_redirect_ranked_matches(target_name, port)
             vhosts = [x for x in [self._select_best_name_match(matches)]if x is not None]
         if not vhosts and create_if_no_match:
-            vhosts = [self._vhost_from_duplicated_default(target_name, port=port)]
+            vhosts = [self._vhost_from_duplicated_default(target_name, False, port)]
         return vhosts
 
     def _port_matches(self, test_port, matching_port):
@@ -528,7 +541,7 @@ class NginxConfigurator(common.Installer):
         :rtype: set
 
         """
-        all_names = set()
+        all_names = set() # type: Set[str]
 
         for vhost in self.parser.get_vhosts():
             all_names.update(vhost.names)
@@ -824,7 +837,7 @@ class NginxConfigurator(common.Installer):
             self.parser.add_server_directives(vhost,
                                               stapling_directives)
         except errors.MisconfigurationError as error:
-            logger.debug(error)
+            logger.debug(str(error))
             raise errors.PluginError("An error occurred while enabling OCSP "
                                      "stapling for {0}.".format(vhost.names))
 
@@ -892,7 +905,7 @@ class NginxConfigurator(common.Installer):
                 universal_newlines=True)
             text = proc.communicate()[1]  # nginx prints output to stderr
         except (OSError, ValueError) as error:
-            logger.debug(error, exc_info=True)
+            logger.debug(str(error), exc_info=True)
             raise errors.PluginError(
                 "Unable to run %s -V" % self.conf('ctl'))
 
@@ -1117,3 +1130,11 @@ def install_ssl_options_conf(options_ssl, options_ssl_digest):
     """Copy Certbot's SSL options file into the system's config dir if required."""
     return common.install_version_controlled_file(options_ssl, options_ssl_digest,
         constants.MOD_SSL_CONF_SRC, constants.ALL_SSL_OPTIONS_HASHES)
+
+def _determine_default_server_root():
+    if os.environ.get("CERTBOT_DOCS") == "1":
+        default_server_root = "%s or %s" % (constants.LINUX_SERVER_ROOT,
+            constants.FREEBSD_DARWIN_SERVER_ROOT)
+    else:
+        default_server_root = constants.CLI_DEFAULTS["server_root"]
+    return default_server_root

certbot-nginx/certbot_nginx/constants.py

@@ -1,9 +1,17 @@
 """nginx plugin constants."""
 import pkg_resources
+import platform
 
+FREEBSD_DARWIN_SERVER_ROOT = "/usr/local/etc/nginx"
+LINUX_SERVER_ROOT = "/etc/nginx"
+
+if platform.system() in ('FreeBSD', 'Darwin'):
+    server_root_tmp = FREEBSD_DARWIN_SERVER_ROOT
+else:
+    server_root_tmp = LINUX_SERVER_ROOT
 
 CLI_DEFAULTS = dict(
-    server_root="/etc/nginx",
+    server_root=server_root_tmp,
     ctl="nginx",
 )
 """CLI defaults."""

certbot-nginx/certbot_nginx/http_01.py

@@ -10,6 +10,7 @@ from certbot.plugins import common
 
 from certbot_nginx import obj
 from certbot_nginx import nginxparser
+from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module
 
 
 logger = logging.getLogger(__name__)
@@ -113,7 +114,7 @@ class NginxHttp01(common.ChallengePerformer):
         :returns: list of :class:`certbot_nginx.obj.Addr` to apply
         :rtype: list
         """
-        addresses = []
+        addresses = [] # type: List[obj.Addr]
         default_addr = "%s" % self.configurator.config.http01_port
         ipv6_addr = "[::]:{0}".format(
             self.configurator.config.http01_port)

certbot-nginx/certbot_nginx/nginxparser.py

@@ -248,7 +248,7 @@ class UnspacedList(list):
         """Recurse through the parse tree to figure out if any sublists are dirty"""
         if self.dirty:
             return True
-        return any((isinstance(x, list) and x.is_dirty() for x in self))
+        return any((isinstance(x, UnspacedList) and x.is_dirty() for x in self))
 
     def _spaced_position(self, idx):
         "Convert from indexes in the unspaced list to positions in the spaced one"

certbot-nginx/certbot_nginx/parser.py

@@ -13,7 +13,7 @@ from certbot import errors
 
 from certbot_nginx import obj
 from certbot_nginx import nginxparser
-
+from acme.magic_typing import Union, Dict, Set, Any, List, Tuple # pylint: disable=unused-import, no-name-in-module
 
 logger = logging.getLogger(__name__)
 
@@ -28,7 +28,7 @@ class NginxParser(object):
     """
 
     def __init__(self, root):
-        self.parsed = {}
+        self.parsed = {} # type: Dict[str, Union[List, nginxparser.UnspacedList]]
         self.root = os.path.abspath(root)
         self.config_root = self._find_config_root()
 
@@ -90,7 +90,7 @@ class NginxParser(object):
         """
         servers = self._get_raw_servers()
 
-        addr_to_ssl = {}
+        addr_to_ssl = {} # type: Dict[Tuple[str, str], bool]
         for filename in servers:
             for server, _ in servers[filename]:
                 # Parse the server block to save addr info
@@ -104,9 +104,10 @@ class NginxParser(object):
 
     def _get_raw_servers(self):
         # pylint: disable=cell-var-from-loop
+        # type: () -> Dict
         """Get a map of unparsed all server blocks
         """
-        servers = {}
+        servers = {} # type: Dict[str, Union[List, nginxparser.UnspacedList]]
         for filename in self.parsed:
             tree = self.parsed[filename]
             servers[filename] = []
@@ -565,7 +566,7 @@ def _update_or_add_directives(directives, insert_at_top, block):
 
 
 INCLUDE = 'include'
-REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'rewrite'])
+REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'rewrite', 'add_header'])
 COMMENT = ' managed by Certbot'
 COMMENT_BLOCK = [' ', '#', COMMENT]
 
@@ -727,9 +728,9 @@ def _parse_server_raw(server):
     :rtype: dict
 
     """
-    parsed_server = {'addrs': set(),
-                     'ssl': False,
-                     'names': set()}
+    addrs = set() # type: Set[obj.Addr]
+    ssl = False # type: bool
+    names = set() # type: Set[str]
 
     apply_ssl_to_all_addrs = False
 
@@ -739,17 +740,21 @@ def _parse_server_raw(server):
         if directive[0] == 'listen':
             addr = obj.Addr.fromstring(" ".join(directive[1:]))
             if addr:
-                parsed_server['addrs'].add(addr)
+                addrs.add(addr)
                 if addr.ssl:
-                    parsed_server['ssl'] = True
+                    ssl = True
         elif directive[0] == 'server_name':
-            parsed_server['names'].update(x.strip('"\'') for x in directive[1:])
+            names.update(x.strip('"\'') for x in directive[1:])
         elif _is_ssl_on_directive(directive):
-            parsed_server['ssl'] = True
+            ssl = True
             apply_ssl_to_all_addrs = True
 
     if apply_ssl_to_all_addrs:
-        for addr in parsed_server['addrs']:
+        for addr in addrs:
             addr.ssl = True
 
-    return parsed_server
+    return {
+        'addrs': addrs,
+        'ssl': ssl,
+        'names': names
+    }

certbot-nginx/certbot_nginx/tests/configurator_test.py

@@ -47,7 +47,7 @@ class NginxConfiguratorTest(util.NginxTest):
 
     def test_prepare(self):
         self.assertEqual((1, 6, 2), self.config.version)
-        self.assertEqual(10, len(self.config.parser.parsed))
+        self.assertEqual(11, len(self.config.parser.parsed))
 
     @mock.patch("certbot_nginx.configurator.util.exe_exists")
     @mock.patch("certbot_nginx.configurator.subprocess.Popen")
@@ -91,7 +91,8 @@ class NginxConfiguratorTest(util.NginxTest):
         self.assertEqual(names, set(
             ["155.225.50.69.nephoscale.net", "www.example.org", "another.alias",
              "migration.com", "summer.com", "geese.com", "sslon.com",
-             "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com"]))
+             "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com",
+             "headers.com"]))
 
     def test_supported_enhancements(self):
         self.assertEqual(['redirect', 'ensure-http-header', 'staple-ocsp'],
@@ -548,6 +549,14 @@ class NginxConfiguratorTest(util.NginxTest):
         generated_conf = self.config.parser.parsed[example_conf]
         self.assertTrue(util.contains_at_depth(generated_conf, expected, 2))
 
+    def test_multiple_headers_hsts(self):
+        headers_conf = self.config.parser.abs_path('sites-enabled/headers.com')
+        self.config.enhance("headers.com", "ensure-http-header",
+                            "Strict-Transport-Security")
+        expected = ['add_header', 'Strict-Transport-Security', '"max-age=31536000"', 'always']
+        generated_conf = self.config.parser.parsed[headers_conf]
+        self.assertTrue(util.contains_at_depth(generated_conf, expected, 2))
+
     def test_http_header_hsts_twice(self):
         self.config.enhance("www.example.com", "ensure-http-header",
                             "Strict-Transport-Security")
@@ -722,6 +731,13 @@ class NginxConfiguratorTest(util.NginxTest):
             "www.nomatch.com", "example/cert.pem", "example/key.pem",
             "example/chain.pem", "example/fullchain.pem")
 
+    def test_deploy_no_match_multiple_defaults_ok(self):
+        foo_conf = self.config.parser.abs_path('foo.conf')
+        self.config.parser.parsed[foo_conf][2][1][0][1][0][1] = '*:5001'
+        self.config.version = (1, 3, 1)
+        self.config.deploy_cert("www.nomatch.com", "example/cert.pem", "example/key.pem",
+            "example/chain.pem", "example/fullchain.pem")
+
     def test_deploy_no_match_add_redirect(self):
         default_conf = self.config.parser.abs_path('sites-enabled/default')
         foo_conf = self.config.parser.abs_path('foo.conf')
@@ -852,7 +868,7 @@ class NginxConfiguratorTest(util.NginxTest):
                                                 prefer_ssl=False,
                                                 no_ssl_filter_port='80')
             # Check that the dialog was called with only port 80 vhosts
-            self.assertEqual(len(mock_select_vhs.call_args[0][0]), 4)
+            self.assertEqual(len(mock_select_vhs.call_args[0][0]), 5)
 
 
 class InstallSslOptionsConfTest(util.NginxTest):
@@ -933,5 +949,30 @@ class InstallSslOptionsConfTest(util.NginxTest):
             " with the sha256 hash of self.config.mod_ssl_conf when it is updated.")
 
 
+class DetermineDefaultServerRootTest(certbot_test_util.ConfigTestCase):
+    """Tests for certbot_nginx.configurator._determine_default_server_root."""
+
+    def _call(self):
+        from certbot_nginx.configurator import _determine_default_server_root
+        return _determine_default_server_root()
+
+    @mock.patch.dict(os.environ, {"CERTBOT_DOCS": "1"})
+    def test_docs_value(self):
+        self._test(expect_both_values=True)
+
+    @mock.patch.dict(os.environ, {})
+    def test_real_values(self):
+        self._test(expect_both_values=False)
+
+    def _test(self, expect_both_values):
+        server_root = self._call()
+
+        if expect_both_values:
+            self.assertIn("/usr/local/etc/nginx", server_root)
+            self.assertIn("/etc/nginx", server_root)
+        else:
+            self.assertTrue(server_root == "/etc/nginx" or server_root == "/usr/local/etc/nginx")
+
+
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot-nginx/certbot_nginx/tests/parser_test.py

@@ -11,6 +11,7 @@ from certbot_nginx import nginxparser
 from certbot_nginx import obj
 from certbot_nginx import parser
 from certbot_nginx.tests import util
+from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module
 
 
 class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods
@@ -48,6 +49,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods
                               ['foo.conf', 'nginx.conf', 'server.conf',
                                'sites-enabled/default',
                                'sites-enabled/example.com',
+                               'sites-enabled/headers.com',
                                'sites-enabled/migration.com',
                                'sites-enabled/sslon.com',
                                'sites-enabled/globalssl.com',
@@ -76,7 +78,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods
         parsed = nparser._parse_files(nparser.abs_path(
             'sites-enabled/example.com.test'))
         self.assertEqual(3, len(glob.glob(nparser.abs_path('*.test'))))
-        self.assertEqual(7, len(
+        self.assertEqual(8, len(
             glob.glob(nparser.abs_path('sites-enabled/*.test'))))
         self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'],
                                         ['listen', '127.0.0.1'],
@@ -99,7 +101,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods
                    ([[[0], [3], [4]], [[5], [3], [0]]], [])]
 
         for mylist, result in mylists:
-            paths = []
+            paths = [] # type: List[List[int]]
             parser._do_for_subarray(mylist,
                                     lambda x: isinstance(x, list) and
                                     len(x) >= 1 and
@@ -159,7 +161,7 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods
                                                   '*.www.example.com']),
                                  [], [2, 1, 0])
 
-        self.assertEqual(12, len(vhosts))
+        self.assertEqual(13, len(vhosts))
         example_com = [x for x in vhosts if 'example.com' in x.filep][0]
         self.assertEqual(vhost3, example_com)
         default = [x for x in vhosts if 'default' in x.filep][0]

certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/headers.com

@@ -0,0 +1,4 @@
+server {
+    server_name headers.com;
+    add_header X-Content-Type-Options nosniff;
+}

certbot-nginx/local-oldest-requirements.txt

@@ -1,2 +1,2 @@
--e acme[dev]
+acme[dev]==0.25.0
 -e .[dev]

certbot-nginx/setup.py

@@ -1,19 +1,14 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.24.0.dev0'
+version = '0.26.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    # This plugin works with an older version of acme, but Certbot does not.
-    # 0.22.0 is specified here to work around
-    # https://github.com/pypa/pip/issues/988.
-    'acme>0.21.1',
-    'certbot>0.21.1',
+    'acme>=0.25.0',
+    'certbot>=0.22.0',
     'mock',
     'PyOpenSSL',
     'pyparsing>=1.5.5',  # Python3 support; perhaps unnecessary?

certbot-nginx/tests/boulder-integration.sh

@@ -62,3 +62,5 @@ test_deployment_and_rollback nginx6.wtf
 # note: not reached if anything above fails, hence "killall" at the
 # top
 nginx -c $nginx_root/nginx.conf -s stop
+
+coverage report --fail-under 75 --include 'certbot-nginx/*' --show-missing

certbot-postfix/LICENSE.txt

@@ -0,0 +1,190 @@
+   Copyright 2017 Electronic Frontier Foundation and others
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS

certbot-postfix/MANIFEST.in

@@ -0,0 +1,4 @@
+include LICENSE.txt
+include README.rst
+recursive-include certbot_postfix/testdata *
+recursive-include certbot_postfix/docs *

certbot-postfix/README.rst

@@ -0,0 +1,23 @@
+==========================
+Postfix plugin for Certbot
+==========================
+
+Note: this MTA installer is in **developer beta**-- we appreciate any testing, feedback, or
+feature requests for this plugin.
+
+To install this plugin, in the root of this repo, run::
+
+    ./tools/venv.sh
+    source venv/bin/activate
+
+You can use this installer with any `authenticator plugin
+<https://certbot.eff.org/docs/using.html#getting-certificates-and-choosing-plugins>`_.
+For instance, with the `standalone authenticator
+<https://certbot.eff.org/docs/using.html#standalone>`_, which requires no extra server
+software, you might run::
+
+    sudo ./venv/bin/certbot run --standalone -i postfix -d <domain name>
+
+To just install existing certs with this plugin, run::
+
+    sudo ./venv/bin/certbot install -i postfix --cert-path <path to cert> --key-path <path to key> -d <domain name>

certbot-postfix/certbot_postfix/__init__.py

@@ -0,0 +1,3 @@
+"""Certbot Postfix plugin."""
+
+from certbot_postfix.installer import Installer

certbot-postfix/certbot_postfix/constants.py

@@ -0,0 +1,63 @@
+"""Postfix plugin constants."""
+
+# pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Dict, Tuple, Union
+# pylint: enable=unused-import, no-name-in-module
+
+MINIMUM_VERSION = (2, 11,)
+
+# If the value of a default VAR is a tuple, then the values which
+# come LATER in the tuple are more strict/more secure.
+# Certbot will default to the first value in the tuple, but will
+# not override "more secure" settings.
+
+ACCEPTABLE_SERVER_SECURITY_LEVELS = ("may", "encrypt")
+ACCEPTABLE_CLIENT_SECURITY_LEVELS = ("may", "encrypt",
+                                     "dane", "dane-only",
+                                     "fingerprint",
+                                     "verify", "secure")
+ACCEPTABLE_CIPHER_LEVELS = ("medium", "high")
+
+# Exporting certain ciphers to prevent logjam: https://weakdh.org/sysadmin.html
+EXCLUDE_CIPHERS = ("aNULL, eNULL, EXPORT, DES, RC4, MD5, PSK, aECDH, "
+                   "EDH-DSS-DES-CBC3-SHA, EDH-RSA-DES-CBC3-SHA, KRB5-DES, CBC3-SHA",)
+
+
+TLS_VERSIONS = ("SSLv2", "SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2")
+# Should NOT use SSLv2/3.
+ACCEPTABLE_TLS_VERSIONS = ("TLSv1", "TLSv1.1", "TLSv1.2")
+
+# Variables associated with enabling opportunistic TLS.
+TLS_SERVER_VARS = {
+    "smtpd_tls_security_level": ACCEPTABLE_SERVER_SECURITY_LEVELS,
+} # type:Dict[str, Tuple[str, ...]]
+TLS_CLIENT_VARS = {
+    "smtp_tls_security_level": ACCEPTABLE_CLIENT_SECURITY_LEVELS,
+} # type:Dict[str, Tuple[str, ...]]
+# Default variables for a secure MTA server [receiver].
+DEFAULT_SERVER_VARS = {
+    "smtpd_tls_auth_only": ("yes",),
+    "smtpd_tls_mandatory_protocols": ("!SSLv2, !SSLv3",),
+    "smtpd_tls_protocols": ("!SSLv2, !SSLv3",),
+    "smtpd_tls_ciphers": ACCEPTABLE_CIPHER_LEVELS,
+    "smtpd_tls_mandatory_ciphers": ACCEPTABLE_CIPHER_LEVELS,
+    "smtpd_tls_exclude_ciphers": EXCLUDE_CIPHERS,
+    "smtpd_tls_eecdh_grade": ("strong",),
+} # type:Dict[str, Tuple[str, ...]]
+
+# Default variables for a secure MTA client [sender].
+DEFAULT_CLIENT_VARS = {
+    "smtp_tls_ciphers": ACCEPTABLE_CIPHER_LEVELS,
+    "smtp_tls_exclude_ciphers": EXCLUDE_CIPHERS,
+    "smtp_tls_mandatory_ciphers": ACCEPTABLE_CIPHER_LEVELS,
+} # type:Dict[str, Tuple[str, ...]]
+
+CLI_DEFAULTS = dict(
+    config_dir="/etc/postfix",
+    ctl="postfix",
+    config_utility="postconf",
+    tls_only=False,
+    ignore_master_overrides=False,
+    server_only=False,
+)
+"""CLI defaults."""

certbot-postfix/certbot_postfix/installer.py

@@ -0,0 +1,288 @@
+"""certbot installer plugin for postfix."""
+import logging
+import os
+
+import zope.interface
+import zope.component
+import six
+
+from certbot import errors
+from certbot import interfaces
+from certbot import util as certbot_util
+from certbot.plugins import common as plugins_common
+
+from certbot_postfix import constants
+from certbot_postfix import postconf
+from certbot_postfix import util
+
+# pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Callable, Dict, List
+# pylint: enable=unused-import, no-name-in-module
+
+logger = logging.getLogger(__name__)
+
+@zope.interface.implementer(interfaces.IInstaller)
+@zope.interface.provider(interfaces.IPluginFactory)
+class Installer(plugins_common.Installer):
+    """Certbot installer plugin for Postfix.
+
+    :ivar str config_dir: Postfix configuration directory to modify
+    :ivar list save_notes: documentation for proposed changes. This is
+        cleared and stored in Certbot checkpoints when save() is called
+
+    :ivar postconf: Wrapper for Postfix configuration command-line tool.
+    :type postconf: :class: `certbot_postfix.postconf.ConfigMain`
+    :ivar postfix: Wrapper for Postfix command-line tool.
+    :type postfix: :class: `certbot_postfix.util.PostfixUtil`
+    """
+
+    description = "Configure TLS with the Postfix MTA"
+
+    @classmethod
+    def add_parser_arguments(cls, add):
+        add("ctl", default=constants.CLI_DEFAULTS["ctl"],
+            help="Path to the 'postfix' control program.")
+        # This directory points to Postfix's configuration directory.
+        add("config-dir", default=constants.CLI_DEFAULTS["config_dir"],
+            help="Path to the directory containing the "
+            "Postfix main.cf file to modify instead of using the "
+            "default configuration paths.")
+        add("config-utility", default=constants.CLI_DEFAULTS["config_utility"],
+            help="Path to the 'postconf' executable.")
+        add("tls-only", action="store_true", default=constants.CLI_DEFAULTS["tls_only"],
+            help="Only set params to enable opportunistic TLS and install certificates.")
+        add("server-only", action="store_true", default=constants.CLI_DEFAULTS["server_only"],
+            help="Only set server params (prefixed with smtpd*)")
+        add("ignore-master-overrides", action="store_true",
+            default=constants.CLI_DEFAULTS["ignore_master_overrides"],
+            help="Ignore errors reporting overridden TLS parameters in master.cf.")
+
+    def __init__(self, *args, **kwargs):
+        super(Installer, self).__init__(*args, **kwargs)
+        # Wrapper around postconf commands
+        self.postfix = None
+        self.postconf = None
+
+        # Files to save
+        self.save_notes = [] # type: List[str]
+
+        self._enhance_func = {} # type: Dict[str, Callable[[str, str], None]]
+        # Since we only need to enable TLS once for all domains,
+        # keep track of whether this enhancement was already called.
+        self._tls_enabled = False
+
+    def prepare(self):
+        """Prepare the installer.
+
+        :raises errors.PluginError: when an unexpected error occurs
+        :raises errors.MisconfigurationError: when the config is invalid
+        :raises errors.NoInstallationError: when can't find installation
+        :raises errors.NotSupportedError: when version is not supported
+        """
+        # Verify postfix and postconf are installed
+        for param in ("ctl", "config_utility",):
+            util.verify_exe_exists(self.conf(param),
+                    "Cannot find executable '{0}'. You can provide the "
+                    "path to this command with --{1}".format(
+                        self.conf(param),
+                        self.option_name(param)))
+
+        # Set up CLI tools
+        self.postfix = util.PostfixUtil(self.conf('config-dir'))
+        self.postconf = postconf.ConfigMain(self.conf('config-utility'),
+                                            self.conf('ignore-master-overrides'),
+                                            self.conf('config-dir'))
+
+        # Ensure current configuration is valid.
+        self.config_test()
+
+        # Check Postfix version
+        self._check_version()
+        self._lock_config_dir()
+        self.install_ssl_dhparams()
+
+    def config_test(self):
+        """Test to see that the current Postfix configuration is valid.
+
+        :raises errors.MisconfigurationError: If the configuration is invalid.
+        """
+        self.postfix.test()
+
+    def _check_version(self):
+        """Verifies that the installed Postfix version is supported.
+
+        :raises errors.NotSupportedError: if the version is unsupported
+        """
+        if self._get_version() < constants.MINIMUM_VERSION:
+            version_string = '.'.join([str(n) for n in constants.MINIMUM_VERSION])
+            raise errors.NotSupportedError('Postfix version must be at least %s' % version_string)
+
+    def _lock_config_dir(self):
+        """Stop two Postfix plugins from modifying the config at once.
+
+        :raises .PluginError: if unable to acquire the lock
+        """
+        try:
+            certbot_util.lock_dir_until_exit(self.conf('config-dir'))
+        except (OSError, errors.LockError):
+            logger.debug("Encountered error:", exc_info=True)
+            raise errors.PluginError(
+                "Unable to lock %s" % self.conf('config-dir'))
+
+    def more_info(self):
+        """Human-readable string to help the user. Describes steps taken and any relevant
+        info to help the user decide which plugin to use.
+
+        :rtype: str
+        """
+        return (
+            "Configures Postfix to try to authenticate mail servers, use "
+            "installed certificates and disable weak ciphers and protocols.{0}"
+            "Server root: {root}{0}"
+            "Version: {version}".format(
+                os.linesep,
+                root=self.conf('config-dir'),
+                version='.'.join([str(i) for i in self._get_version()]))
+        )
+
+    def _get_version(self):
+        """Return the version of Postfix, as a tuple. (e.g. '2.11.3' is (2, 11, 3))
+
+        :returns: version
+        :rtype: tuple
+
+        :raises errors.PluginError: Unable to find Postfix version.
+        """
+        mail_version = self.postconf.get_default("mail_version")
+        return tuple(int(i) for i in mail_version.split('.'))
+
+    def get_all_names(self):
+        """Returns all names that may be authenticated.
+
+        :rtype: `set` of `str`
+
+        """
+        return certbot_util.get_filtered_names(self.postconf.get(var)
+                   for var in ('mydomain', 'myhostname', 'myorigin',))
+
+    def _set_vars(self, var_dict):
+        """Sets all parameters in var_dict to config file. If current value is already set
+        as more secure (acceptable), then don't set/overwrite it.
+        """
+        for param, acceptable in six.iteritems(var_dict):
+            if not util.is_acceptable_value(param, self.postconf.get(param), acceptable):
+                self.postconf.set(param, acceptable[0], acceptable)
+
+    def _confirm_changes(self):
+        """Confirming outstanding updates for configuration parameters.
+
+        :raises errors.PluginError: when user rejects the configuration changes.
+        """
+        updates = self.postconf.get_changes()
+        output_string = "Postfix TLS configuration parameters to update in main.cf:\n"
+        for name, value in six.iteritems(updates):
+            output_string += "{0} = {1}\n".format(name, value)
+        output_string += "Is this okay?\n"
+        if not zope.component.getUtility(interfaces.IDisplay).yesno(output_string,
+            force_interactive=True, default=True):
+            raise errors.PluginError(
+                "Manually rejected configuration changes.\n"
+                "Try using --tls-only or --server-only to change a particular"
+                "subset of configuration parameters.")
+
+    def deploy_cert(self, domain, cert_path,
+                    key_path, chain_path, fullchain_path):
+        """Configure the Postfix SMTP server to use the given TLS cert.
+
+        :param str domain: domain to deploy certificate file
+        :param str cert_path: absolute path to the certificate file
+        :param str key_path: absolute path to the private key file
+        :param str chain_path: absolute path to the certificate chain file
+        :param str fullchain_path: absolute path to the certificate fullchain
+            file (cert plus chain)
+
+        :raises .PluginError: when cert cannot be deployed
+
+        """
+        # pylint: disable=unused-argument
+        if self._tls_enabled:
+            return
+        self._tls_enabled = True
+        self.save_notes.append("Configuring TLS for {0}".format(domain))
+        self.postconf.set("smtpd_tls_cert_file", cert_path)
+        self.postconf.set("smtpd_tls_key_file", key_path)
+        self._set_vars(constants.TLS_SERVER_VARS)
+        if not self.conf('server_only'):
+            self._set_vars(constants.TLS_CLIENT_VARS)
+        if not self.conf('tls_only'):
+            self._set_vars(constants.DEFAULT_SERVER_VARS)
+            if not self.conf('server_only'):
+                self._set_vars(constants.DEFAULT_CLIENT_VARS)
+            # Despite the name, this option also supports 2048-bit DH params.
+            # http://www.postfix.org/FORWARD_SECRECY_README.html#server_fs
+            self.postconf.set("smtpd_tls_dh1024_param_file", self.ssl_dhparams)
+        self._confirm_changes()
+
+    def enhance(self, domain, enhancement, options=None):
+        """Raises an exception since this installer doesn't support any enhancements.
+        """
+        # pylint: disable=unused-argument
+        raise errors.PluginError(
+            "Unsupported enhancement: {0}".format(enhancement))
+
+    def supported_enhancements(self):
+        """Returns a list of supported enhancements.
+
+        :rtype: list
+
+        """
+        return []
+
+    def save(self, title=None, temporary=False):
+        """Creates backups and writes changes to configuration files.
+
+        :param str title: The title of the save. If a title is given, the
+            configuration will be saved as a new checkpoint and put in a
+            timestamped directory. `title` has no effect if temporary is true.
+
+        :param bool temporary: Indicates whether the changes made will
+            be quickly reversed in the future (challenges)
+
+        :raises errors.PluginError: when save is unsuccessful
+        """
+        save_files = set((os.path.join(self.conf('config-dir'), "main.cf"),))
+        self.add_to_checkpoint(save_files,
+                               "\n".join(self.save_notes), temporary)
+        self.postconf.flush()
+
+        del self.save_notes[:]
+
+        if title and not temporary:
+            self.finalize_checkpoint(title)
+
+    def recovery_routine(self):
+        super(Installer, self).recovery_routine()
+        self.postconf = postconf.ConfigMain(self.conf('config-utility'),
+                                            self.conf('ignore-master-overrides'),
+                                            self.conf('config-dir'))
+
+    def rollback_checkpoints(self, rollback=1):
+        """Rollback saved checkpoints.
+
+        :param int rollback: Number of checkpoints to revert
+
+        :raises .errors.PluginError: If there is a problem with the input or
+            the function is unable to correctly revert the configuration
+        """
+        super(Installer, self).rollback_checkpoints(rollback)
+        self.postconf = postconf.ConfigMain(self.conf('config-utility'),
+                                            self.conf('ignore-master-overrides'),
+                                            self.conf('config-dir'))
+
+    def restart(self):
+        """Restart or refresh the server content.
+
+        :raises .PluginError: when server cannot be restarted
+        """
+        self.postfix.restart()
+

certbot-postfix/certbot_postfix/postconf.py

@@ -0,0 +1,152 @@
+"""Classes that wrap the postconf command line utility.
+"""
+import six
+from certbot import errors
+from certbot_postfix import util
+
+# pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Dict, List, Tuple
+# pylint: enable=unused-import, no-name-in-module
+
+class ConfigMain(util.PostfixUtilBase):
+    """A parser for Postfix's main.cf file."""
+
+    def __init__(self, executable, ignore_master_overrides=False, config_dir=None):
+        super(ConfigMain, self).__init__(executable, config_dir)
+        # Whether to ignore overrides from master.
+        self._ignore_master_overrides = ignore_master_overrides
+        # List of all current Postfix parameters, from `postconf` command.
+        self._db = {} # type: Dict[str, str]
+        # List of current master.cf overrides from Postfix config. Dictionary
+        # of parameter name => list of tuples (service name, paramter value)
+        # Note: We should never modify master without explicit permission.
+        self._master_db = {} # type: Dict[str, List[Tuple[str, str]]]
+        # List of all changes requested to the Postfix parameters as they are now
+        # in _db. These changes are flushed to `postconf` on `flush`.
+        self._updated = {} # type: Dict[str, str]
+        self._read_from_conf()
+
+    def _read_from_conf(self):
+        """Reads initial parameter state from `main.cf` into this object.
+        """
+        out = self._get_output()
+        for name, value in _parse_main_output(out):
+            self._db[name] = value
+        out = self._get_output_master()
+        for name, value in _parse_main_output(out):
+            service, param_name = name.rsplit("/", 1)
+            if param_name not in self._master_db:
+                self._master_db[param_name] = []
+            self._master_db[param_name].append((service, value))
+
+    def _get_output_master(self):
+        """Retrieves output for `master.cf` parameters."""
+        return self._get_output('-P')
+
+    def get_default(self, name):
+        """Retrieves default value of parameter `name` from postfix parameters.
+
+        :param str name: The name of the parameter to fetch.
+        :returns: The default value of parameter `name`.
+        :rtype: str
+        """
+        out = self._get_output(['-d', name])
+        _, value = next(_parse_main_output(out), (None, None))
+        return value
+
+    def get(self, name):
+        """Retrieves working value of parameter `name` from postfix parameters.
+
+        :param str name: The name of the parameter to fetch.
+        :returns: The value of parameter `name`.
+        :rtype: str
+        """
+        if name in self._updated:
+            return self._updated[name]
+        return self._db[name]
+
+    def get_master_overrides(self, name):
+        """Retrieves list of overrides for parameter `name` in postfix's Master config
+        file.
+
+        :returns: List of tuples (service, value), meaning that parameter `name`
+            is overridden as `value` for `service`.
+        :rtype: `list` of `tuple` of `str`
+        """
+        if name in self._master_db:
+            return self._master_db[name]
+        return None
+
+    def set(self, name, value, acceptable_overrides=None):
+        """Sets parameter `name` to `value`. If `name` is overridden by a particular service in
+        `master.cf`, reports any of these parameter conflicts as long as
+        `ignore_master_overrides` was not set.
+
+        .. note:: that this function does not flush these parameter values to main.cf;
+            To do that, use `flush`.
+
+        :param str name: The name of the parameter to set.
+        :param str value: The value of the parameter.
+        :param tuple acceptable_overrides: If the master configuration file overrides `value`
+            with a value in acceptable_overrides.
+        """
+        if name not in self._db:
+            raise KeyError("Parameter name %s is not a valid Postfix parameter name.", name)
+        # Check to see if this parameter is overridden by master.
+        overrides = self.get_master_overrides(name)
+        if not self._ignore_master_overrides and overrides is not None:
+            util.report_master_overrides(name, overrides, acceptable_overrides)
+        if value != self._db[name]:
+        # _db contains the "original" state of parameters. We only care about
+        # writes if they cause a delta from the original state.
+            self._updated[name] = value
+        elif name in self._updated:
+        # If this write reverts a previously updated parameter back to the
+        # original DB's state, we don't have to keep track of it in _updated.
+            del self._updated[name]
+
+    def flush(self):
+        """Flushes all parameter changes made using `self.set`, to `main.cf`
+
+        :raises error.PluginError: When flush to main.cf fails for some reason.
+        """
+        if len(self._updated) == 0:
+            return
+        args = ['-e']
+        for name, value in six.iteritems(self._updated):
+            args.append('{0}={1}'.format(name, value))
+        try:
+            self._get_output(args)
+        except IOError as e:
+            raise errors.PluginError("Unable to save to Postfix config: %v", e)
+        for name, value in six.iteritems(self._updated):
+            self._db[name] = value
+        self._updated = {}
+
+    def get_changes(self):
+        """ Return queued changes to main.cf.
+
+        :rtype: dict[str, str]
+        """
+        return self._updated
+
+def _parse_main_output(output):
+    """Parses the raw output from Postconf about main.cf.
+
+    Expects the output to look like:
+
+    .. code-block:: none
+
+       name1 = value1
+       name2 = value2
+
+    :param str output: data postconf wrote to stdout about main.cf
+
+    :returns: generator providing key-value pairs from main.cf
+    :rtype: Iterator[tuple(str, str)]
+    """
+    for line in output.splitlines():
+        name, _, value = line.partition(" =")
+        yield name, value.strip()
+
+

certbot-postfix/certbot_postfix/tests/__init__.py

@@ -0,0 +1 @@
+""" Certbot Postfix Tests """

certbot-postfix/certbot_postfix/tests/installer_test.py

@@ -0,0 +1,314 @@
+"""Tests for certbot_postfix.installer."""
+from contextlib import contextmanager
+import copy
+import functools
+import os
+import pkg_resources
+import six
+import unittest
+
+import mock
+
+from certbot import errors
+from certbot.tests import util as certbot_test_util
+
+# pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Dict, Tuple, Union
+# pylint: enable=unused-import, no-name-in-module
+
+DEFAULT_MAIN_CF = {
+    "smtpd_tls_cert_file": "",
+    "smtpd_tls_key_file": "",
+    "smtpd_tls_dh1024_param_file": "",
+    "smtpd_tls_security_level": "none",
+    "smtpd_tls_auth_only": "",
+    "smtpd_tls_mandatory_protocols": "",
+    "smtpd_tls_protocols": "",
+    "smtpd_tls_ciphers": "",
+    "smtpd_tls_exclude_ciphers": "",
+    "smtpd_tls_mandatory_ciphers": "",
+    "smtpd_tls_eecdh_grade": "medium",
+    "smtp_tls_security_level": "",
+    "smtp_tls_ciphers": "",
+    "smtp_tls_exclude_ciphers": "",
+    "smtp_tls_mandatory_ciphers": "",
+    "mail_version": "3.2.3"
+}
+
+def _main_cf_with(obj):
+    main_cf = copy.copy(DEFAULT_MAIN_CF)
+    main_cf.update(obj)
+    return main_cf
+
+class InstallerTest(certbot_test_util.ConfigTestCase):
+    # pylint: disable=too-many-public-methods
+
+    def setUp(self):
+        super(InstallerTest, self).setUp()
+        _config_file = pkg_resources.resource_filename("certbot_postfix.tests",
+                           os.path.join("testdata", "config.json"))
+        self.config.postfix_ctl = "postfix"
+        self.config.postfix_config_dir = self.tempdir
+        self.config.postfix_config_utility = "postconf"
+        self.config.postfix_tls_only = False
+        self.config.postfix_server_only = False
+        self.config.config_dir = self.tempdir
+
+    @mock.patch("certbot_postfix.installer.util.is_acceptable_value")
+    def test_set_vars(self, mock_is_acceptable_value):
+        mock_is_acceptable_value.return_value = True
+        with create_installer(self.config) as installer:
+            installer.prepare()
+            mock_is_acceptable_value.return_value = False
+
+    @mock.patch("certbot_postfix.installer.util.is_acceptable_value")
+    def test_acceptable_value(self, mock_is_acceptable_value):
+        mock_is_acceptable_value.return_value = True
+        with create_installer(self.config) as installer:
+            installer.prepare()
+            mock_is_acceptable_value.return_value = False
+
+    @certbot_test_util.patch_get_utility()
+    def test_confirm_changes_no_raises_error(self, mock_util):
+        mock_util().yesno.return_value = False
+        with create_installer(self.config) as installer:
+            installer.prepare()
+            self.assertRaises(errors.PluginError, installer.deploy_cert,
+                              "example.com", "cert_path", "key_path",
+                              "chain_path", "fullchain_path")
+
+    @certbot_test_util.patch_get_utility()
+    def test_save(self, mock_util):
+        mock_util().yesno.return_value = True
+        with create_installer(self.config) as installer:
+            installer.prepare()
+            installer.postconf.flush = mock.Mock()
+            installer.reverter = mock.Mock()
+            installer.deploy_cert("example.com", "cert_path", "key_path",
+                                  "chain_path", "fullchain_path")
+            installer.save()
+            self.assertEqual(installer.save_notes, [])
+            self.assertEqual(installer.postconf.flush.call_count, 1)
+            self.assertEqual(installer.reverter.add_to_checkpoint.call_count, 1)
+
+    @certbot_test_util.patch_get_utility()
+    def test_save_with_title(self, mock_util):
+        mock_util().yesno.return_value = True
+        with create_installer(self.config) as installer:
+            installer.prepare()
+            installer.postconf.flush = mock.Mock()
+            installer.reverter = mock.Mock()
+            installer.deploy_cert("example.com", "cert_path", "key_path",
+                                  "chain_path", "fullchain_path")
+            installer.save(title="new_file!")
+            self.assertEqual(installer.reverter.finalize_checkpoint.call_count, 1)
+
+    @certbot_test_util.patch_get_utility()
+    def test_rollback_checkpoints_resets_postconf(self, mock_util):
+        mock_util().yesno.return_value = True
+        with create_installer(self.config) as installer:
+            installer.prepare()
+            installer.deploy_cert("example.com", "cert_path", "key_path",
+                                  "chain_path", "fullchain_path")
+            installer.rollback_checkpoints()
+            self.assertEqual(installer.postconf.get_changes(), {})
+
+    @certbot_test_util.patch_get_utility()
+    def test_recovery_routine_resets_postconf(self, mock_util):
+        mock_util().yesno.return_value = True
+        with create_installer(self.config) as installer:
+            installer.prepare()
+            installer.deploy_cert("example.com", "cert_path", "key_path",
+                                  "chain_path", "fullchain_path")
+            installer.recovery_routine()
+            self.assertEqual(installer.postconf.get_changes(), {})
+
+    def test_restart(self):
+        with create_installer(self.config) as installer:
+            installer.prepare()
+            installer.restart()
+            self.assertEqual(installer.postfix.restart.call_count, 1)
+
+    def test_add_parser_arguments(self):
+        options = set(("ctl", "config-dir", "config-utility",
+                       "tls-only", "server-only", "ignore-master-overrides"))
+        mock_add = mock.MagicMock()
+
+        from certbot_postfix import installer
+        installer.Installer.add_parser_arguments(mock_add)
+
+        for call in mock_add.call_args_list:
+            self.assertTrue(call[0][0] in options)
+
+    def test_no_postconf_prepare(self):
+        with create_installer(self.config) as installer:
+            installer_path = "certbot_postfix.installer"
+            exe_exists_path = installer_path + ".certbot_util.exe_exists"
+            path_surgery_path = "certbot_postfix.util.plugins_util.path_surgery"
+            with mock.patch(path_surgery_path, return_value=False):
+                with mock.patch(exe_exists_path, return_value=False):
+                    self.assertRaises(errors.NoInstallationError,
+                                      installer.prepare)
+
+    def test_old_version(self):
+        with create_installer(self.config, main_cf=_main_cf_with({"mail_version": "0.0.1"}))\
+                as installer:
+            self.assertRaises(errors.NotSupportedError, installer.prepare)
+
+    def test_lock_error(self):
+        with create_installer(self.config) as installer:
+            assert_raises = functools.partial(self.assertRaises,
+                                              errors.PluginError,
+                                              installer.prepare)
+            certbot_test_util.lock_and_call(assert_raises, self.tempdir)
+
+
+    @mock.patch('certbot.util.lock_dir_until_exit')
+    def test_dir_locked(self, lock_dir):
+        with create_installer(self.config) as installer:
+            lock_dir.side_effect = errors.LockError
+            self.assertRaises(errors.PluginError, installer.prepare)
+
+    def test_more_info(self):
+        with create_installer(self.config) as installer:
+            installer.prepare()
+            output = installer.more_info()
+            self.assertTrue("Postfix" in output)
+            self.assertTrue(self.tempdir in output)
+            self.assertTrue(DEFAULT_MAIN_CF["mail_version"] in output)
+
+    def test_get_all_names(self):
+        config = {"mydomain": "example.org",
+                  "myhostname": "mail.example.org",
+                  "myorigin": "example.org"}
+        with create_installer(self.config, main_cf=_main_cf_with(config)) as installer:
+            installer.prepare()
+            result = installer.get_all_names()
+            self.assertEqual(result, set(config.values()))
+
+    @certbot_test_util.patch_get_utility()
+    def test_deploy(self, mock_util):
+        mock_util().yesno.return_value = True
+        from certbot_postfix import constants
+        with create_installer(self.config) as installer:
+            installer.prepare()
+
+            # pylint: disable=protected-access
+            installer.deploy_cert("example.com", "cert_path", "key_path",
+                                  "chain_path", "fullchain_path")
+            changes = installer.postconf.get_changes()
+            expected = {} # type: Dict[str, Tuple[str, ...]]
+            expected.update(constants.TLS_SERVER_VARS)
+            expected.update(constants.DEFAULT_SERVER_VARS)
+            expected.update(constants.DEFAULT_CLIENT_VARS)
+            self.assertEqual(changes["smtpd_tls_key_file"], "key_path")
+            self.assertEqual(changes["smtpd_tls_cert_file"], "cert_path")
+            for name, value in six.iteritems(expected):
+                self.assertEqual(changes[name], value[0])
+
+    @certbot_test_util.patch_get_utility()
+    def test_tls_only(self, mock_util):
+        mock_util().yesno.return_value = True
+        with create_installer(self.config) as installer:
+            installer.prepare()
+            installer.conf = lambda x: x == "tls_only"
+            installer.postconf.set = mock.Mock()
+            installer.deploy_cert("example.com", "cert_path", "key_path",
+                                  "chain_path", "fullchain_path")
+            self.assertEqual(installer.postconf.set.call_count, 4)
+
+    @certbot_test_util.patch_get_utility()
+    def test_server_only(self, mock_util):
+        mock_util().yesno.return_value = True
+        with create_installer(self.config) as installer:
+            installer.prepare()
+            installer.conf = lambda x: x == "server_only"
+            installer.postconf.set = mock.Mock()
+            installer.deploy_cert("example.com", "cert_path", "key_path",
+                                  "chain_path", "fullchain_path")
+            self.assertEqual(installer.postconf.set.call_count, 11)
+
+    @certbot_test_util.patch_get_utility()
+    def test_tls_and_server_only(self, mock_util):
+        mock_util().yesno.return_value = True
+        with create_installer(self.config) as installer:
+            installer.prepare()
+            installer.conf = lambda x: True
+            installer.postconf.set = mock.Mock()
+            installer.deploy_cert("example.com", "cert_path", "key_path",
+                                  "chain_path", "fullchain_path")
+            self.assertEqual(installer.postconf.set.call_count, 3)
+
+    @certbot_test_util.patch_get_utility()
+    def test_deploy_twice(self, mock_util):
+        # Deploying twice on the same installer shouldn't do anything!
+        mock_util().yesno.return_value = True
+        with create_installer(self.config) as installer:
+            installer.prepare()
+            from certbot_postfix.postconf import ConfigMain
+            with mock.patch.object(ConfigMain, "set", wraps=installer.postconf.set) as fake_set:
+                installer.deploy_cert("example.com", "cert_path", "key_path",
+                                      "chain_path", "fullchain_path")
+                self.assertEqual(fake_set.call_count, 15)
+                fake_set.reset_mock()
+                installer.deploy_cert("example.com", "cert_path", "key_path",
+                                      "chain_path", "fullchain_path")
+                fake_set.assert_not_called()
+
+    @certbot_test_util.patch_get_utility()
+    def test_deploy_already_secure(self, mock_util):
+        # Should not overwrite "more-secure" parameters
+        mock_util().yesno.return_value = True
+        more_secure = {
+            "smtpd_tls_security_level": "encrypt",
+            "smtpd_tls_protocols": "!SSLv3, !SSLv2, !TLSv1",
+            "smtpd_tls_eecdh_grade": "strong"
+        }
+        with create_installer(self.config,\
+            main_cf=_main_cf_with(more_secure)) as installer:
+            installer.prepare()
+            installer.deploy_cert("example.com", "cert_path", "key_path",
+                                  "chain_path", "fullchain_path")
+            for param in more_secure.keys():
+                self.assertFalse(param in installer.postconf.get_changes())
+
+    def test_enhance(self):
+        with create_installer(self.config) as installer:
+            installer.prepare()
+            self.assertRaises(errors.PluginError,
+                              installer.enhance,
+                              "example.org", "redirect")
+
+    def test_supported_enhancements(self):
+        with create_installer(self.config) as installer:
+            installer.prepare()
+            self.assertEqual(installer.supported_enhancements(), [])
+
+@contextmanager
+def create_installer(config, main_cf=DEFAULT_MAIN_CF):
+# pylint: disable=dangerous-default-value
+    """Creates a Postfix installer with calls to `postconf` and `postfix` mocked out.
+
+    In particular, creates a ConfigMain object that does regular things, but seeds it
+    with values from `main_cf` and `master_cf` dicts.
+    """
+    from certbot_postfix.postconf import ConfigMain
+    from certbot_postfix import installer
+    def _mock_init_postconf(postconf, executable, ignore_master_overrides=False, config_dir=None):
+        # pylint: disable=protected-access,unused-argument
+        postconf._ignore_master_overrides = ignore_master_overrides
+        postconf._db = main_cf
+        postconf._master_db = {}
+        postconf._updated = {}
+        # override get_default to get from main
+        postconf.get_default = lambda name: main_cf[name]
+    with mock.patch.object(ConfigMain, "__init__", _mock_init_postconf):
+        exe_exists_path = "certbot_postfix.installer.certbot_util.exe_exists"
+        with mock.patch(exe_exists_path, return_value=True):
+            with mock.patch("certbot_postfix.installer.util.PostfixUtil",
+                             return_value=mock.Mock()):
+                yield installer.Installer(config, "postfix")
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover
+

certbot-postfix/certbot_postfix/tests/postconf_test.py

@@ -0,0 +1,107 @@
+"""Tests for certbot_postfix.postconf."""
+
+import mock
+import unittest
+
+from certbot import errors
+
+class PostConfTest(unittest.TestCase):
+    """Tests for certbot_postfix.util.PostConf."""
+    def setUp(self):
+        from certbot_postfix.postconf import ConfigMain
+        super(PostConfTest, self).setUp()
+        with mock.patch('certbot_postfix.util.PostfixUtilBase._get_output') as mock_call:
+            with mock.patch('certbot_postfix.postconf.ConfigMain._get_output_master') as \
+                    mock_master_call:
+                with mock.patch('certbot_postfix.postconf.util.verify_exe_exists') as verify_exe:
+                    verify_exe.return_value = True
+                    mock_call.return_value = ('default_parameter = value\n'
+                                             'extra_param =\n'
+                                             'overridden_by_master = default\n')
+                    mock_master_call.return_value = (
+                        'service/type/overridden_by_master = master_value\n'
+                        'service2/type/overridden_by_master = master_value2\n'
+                    )
+                    self.config = ConfigMain('postconf', False)
+
+    @mock.patch('certbot_postfix.util.PostfixUtilBase._get_output')
+    @mock.patch('certbot_postfix.postconf.util.verify_exe_exists')
+    def test_get_output_master(self, mock_verify_exe, mock_get_output):
+        from certbot_postfix.postconf import ConfigMain
+        mock_verify_exe.return_value = True
+        ConfigMain('postconf', lambda x, y, z: None)
+        mock_get_output.assert_called_with('-P')
+
+    @mock.patch('certbot_postfix.util.PostfixUtilBase._get_output')
+    def test_read_default(self, mock_get_output):
+        mock_get_output.return_value = 'param = default_value'
+        self.assertEqual(self.config.get_default('param'), 'default_value')
+
+    @mock.patch('certbot_postfix.util.PostfixUtilBase._call')
+    def test_set(self, mock_call):
+        self.config.set('extra_param', 'other_value')
+        self.assertEqual(self.config.get('extra_param'), 'other_value')
+        self.config.flush()
+        mock_call.assert_called_with(['-e', 'extra_param=other_value'])
+
+    def test_set_bad_param_name(self):
+        self.assertRaises(KeyError, self.config.set, 'nonexistent_param', 'some_value')
+
+    @mock.patch('certbot_postfix.util.PostfixUtilBase._call')
+    def test_write_revert(self, mock_call):
+        self.config.set('default_parameter', 'fake_news')
+        # revert config set
+        self.config.set('default_parameter', 'value')
+        self.config.flush()
+        mock_call.assert_not_called()
+
+    @mock.patch('certbot_postfix.util.PostfixUtilBase._call')
+    def test_write_default(self, mock_call):
+        self.config.set('default_parameter', 'value')
+        self.config.flush()
+        mock_call.assert_not_called()
+
+    def test_master_overrides(self):
+        self.assertEqual(self.config.get_master_overrides('overridden_by_master'),
+                         [('service/type', 'master_value'),
+                          ('service2/type', 'master_value2')])
+
+    def test_set_check_override(self):
+        self.assertRaises(errors.PluginError, self.config.set,
+            'overridden_by_master', 'new_value')
+
+    def test_ignore_check_override(self):
+        # pylint: disable=protected-access
+        self.config._ignore_master_overrides = True
+        self.config.set('overridden_by_master', 'new_value')
+
+    def test_check_acceptable_overrides(self):
+        self.config.set('overridden_by_master', 'new_value',
+                        ('master_value', 'master_value2'))
+
+    @mock.patch('certbot_postfix.util.PostfixUtilBase._get_output')
+    def test_flush(self, mock_out):
+        self.config.set('default_parameter', 'new_value')
+        self.config.set('extra_param', 'another_value')
+        self.config.flush()
+        arguments = mock_out.call_args_list[-1][0][0]
+        self.assertEquals('-e', arguments[0])
+        self.assertTrue('default_parameter=new_value' in arguments)
+        self.assertTrue('extra_param=another_value' in arguments)
+
+    @mock.patch('certbot_postfix.util.PostfixUtilBase._get_output')
+    def test_flush_updates_object(self, mock_out):
+        self.config.set('default_parameter', 'new_value')
+        self.config.flush()
+        mock_out.reset_mock()
+        self.config.set('default_parameter', 'new_value')
+        mock_out.assert_not_called()
+
+    @mock.patch('certbot_postfix.util.PostfixUtilBase._get_output')
+    def test_flush_throws_error_on_fail(self, mock_out):
+        mock_out.side_effect = [IOError("oh no!")]
+        self.config.set('default_parameter', 'new_value')
+        self.assertRaises(errors.PluginError, self.config.flush)
+
+if __name__ == '__main__':  # pragma: no cover
+    unittest.main()

certbot-postfix/certbot_postfix/tests/util_test.py

@@ -0,0 +1,205 @@
+"""Tests for certbot_postfix.util."""
+
+import subprocess
+import unittest
+
+import mock
+
+from certbot import errors
+
+
+class PostfixUtilBaseTest(unittest.TestCase):
+    """Tests for certbot_postfix.util.PostfixUtilBase."""
+
+    @classmethod
+    def _create_object(cls, *args, **kwargs):
+        from certbot_postfix.util import PostfixUtilBase
+        return PostfixUtilBase(*args, **kwargs)
+
+    @mock.patch('certbot_postfix.util.verify_exe_exists')
+    def test_no_exe(self, mock_verify):
+        expected_error = errors.NoInstallationError
+        mock_verify.side_effect = expected_error
+        self.assertRaises(expected_error, self._create_object, 'nonexistent')
+
+    def test_object_creation(self):
+        with mock.patch('certbot_postfix.util.verify_exe_exists'):
+            self._create_object('existent')
+
+    @mock.patch('certbot_postfix.util.check_all_output')
+    def test_call_extends_args(self, mock_output):
+        # pylint: disable=protected-access
+        with mock.patch('certbot_postfix.util.verify_exe_exists'):
+            mock_output.return_value = 'expected'
+            postfix = self._create_object('executable')
+            postfix._call(['many', 'extra', 'args'])
+            mock_output.assert_called_with(['executable', 'many', 'extra', 'args'])
+            postfix._call()
+            mock_output.assert_called_with(['executable'])
+
+    def test_create_with_config(self):
+        # pylint: disable=protected-access
+        with mock.patch('certbot_postfix.util.verify_exe_exists'):
+            postfix = self._create_object('exec', 'config_dir')
+            self.assertEqual(postfix._base_command, ['exec', '-c', 'config_dir'])
+
+class PostfixUtilTest(unittest.TestCase):
+    def setUp(self):
+        # pylint: disable=protected-access
+        from certbot_postfix.util import PostfixUtil
+        with mock.patch('certbot_postfix.util.verify_exe_exists'):
+            self.postfix = PostfixUtil()
+            self.postfix._call = mock.Mock()
+            self.mock_call = self.postfix._call
+
+    def test_test(self):
+        self.postfix.test()
+        self.mock_call.assert_called_with(['check'])
+
+    def test_test_raises_error_when_check_fails(self):
+        self.mock_call.side_effect = [subprocess.CalledProcessError(1, "")]
+        self.assertRaises(errors.MisconfigurationError, self.postfix.test)
+        self.mock_call.assert_called_with(['check'])
+
+    def test_restart_while_running(self):
+        self.mock_call.side_effect = [subprocess.CalledProcessError(1, ""), None]
+        self.postfix.restart()
+        self.mock_call.assert_called_with(['start'])
+
+    def test_restart_while_not_running(self):
+        self.postfix.restart()
+        self.mock_call.assert_called_with(['reload'])
+
+    def test_restart_raises_error_when_reload_fails(self):
+        self.mock_call.side_effect = [None, subprocess.CalledProcessError(1, "")]
+        self.assertRaises(errors.PluginError, self.postfix.restart)
+        self.mock_call.assert_called_with(['reload'])
+
+    def test_restart_raises_error_when_start_fails(self):
+        self.mock_call.side_effect = [
+             subprocess.CalledProcessError(1, ""),
+             subprocess.CalledProcessError(1, "")]
+        self.assertRaises(errors.PluginError, self.postfix.restart)
+        self.mock_call.assert_called_with(['start'])
+
+class CheckAllOutputTest(unittest.TestCase):
+    """Tests for certbot_postfix.util.check_all_output."""
+
+    @classmethod
+    def _call(cls, *args, **kwargs):
+        from certbot_postfix.util import check_all_output
+        return check_all_output(*args, **kwargs)
+
+    @mock.patch('certbot_postfix.util.logger')
+    @mock.patch('certbot_postfix.util.subprocess.Popen')
+    def test_command_error(self, mock_popen, mock_logger):
+        command = 'foo'
+        retcode = 42
+        output = 'bar'
+        err = 'baz'
+
+        mock_popen().communicate.return_value = (output, err)
+        mock_popen().poll.return_value = 42
+
+        self.assertRaises(subprocess.CalledProcessError, self._call, command)
+        log_args = mock_logger.debug.call_args[0]
+        for value in (command, retcode, output, err,):
+            self.assertTrue(value in log_args)
+
+    @mock.patch('certbot_postfix.util.subprocess.Popen')
+    def test_success(self, mock_popen):
+        command = 'foo'
+        expected = ('bar', '')
+        mock_popen().communicate.return_value = expected
+        mock_popen().poll.return_value = 0
+
+        self.assertEqual(self._call(command), expected)
+
+    def test_stdout_error(self):
+        self.assertRaises(ValueError, self._call, stdout=None)
+
+    def test_stderr_error(self):
+        self.assertRaises(ValueError, self._call, stderr=None)
+
+    def test_universal_newlines_error(self):
+        self.assertRaises(ValueError, self._call, universal_newlines=False)
+
+
+class VerifyExeExistsTest(unittest.TestCase):
+    """Tests for certbot_postfix.util.verify_exe_exists."""
+
+    @classmethod
+    def _call(cls, *args, **kwargs):
+        from certbot_postfix.util import verify_exe_exists
+        return verify_exe_exists(*args, **kwargs)
+
+    @mock.patch('certbot_postfix.util.certbot_util.exe_exists')
+    @mock.patch('certbot_postfix.util.plugins_util.path_surgery')
+    def test_failure(self, mock_exe_exists, mock_path_surgery):
+        mock_exe_exists.return_value = mock_path_surgery.return_value = False
+        self.assertRaises(errors.NoInstallationError, self._call, 'foo')
+
+    @mock.patch('certbot_postfix.util.certbot_util.exe_exists')
+    def test_simple_success(self, mock_exe_exists):
+        mock_exe_exists.return_value = True
+        self._call('foo')
+
+    @mock.patch('certbot_postfix.util.certbot_util.exe_exists')
+    @mock.patch('certbot_postfix.util.plugins_util.path_surgery')
+    def test_successful_surgery(self, mock_exe_exists, mock_path_surgery):
+        mock_exe_exists.return_value = False
+        mock_path_surgery.return_value = True
+        self._call('foo')
+
+class TestUtils(unittest.TestCase):
+    """ Testing random utility functions in util.py
+    """
+    def test_report_master_overrides(self):
+        from certbot_postfix.util import report_master_overrides
+        self.assertRaises(errors.PluginError, report_master_overrides, 'name',
+                          [('service/type', 'value')])
+        # Shouldn't raise error
+        report_master_overrides('name', [('service/type', 'value')],
+                                acceptable_overrides=('value',))
+
+    def test_no_acceptable_value(self):
+        from certbot_postfix.util import is_acceptable_value
+        self.assertFalse(is_acceptable_value('name', 'value', None))
+
+    def test_is_acceptable_value(self):
+        from certbot_postfix.util import is_acceptable_value
+        self.assertTrue(is_acceptable_value('name', 'value', ('value',)))
+        self.assertFalse(is_acceptable_value('name', 'bad', ('value',)))
+
+    def test_is_acceptable_tuples(self):
+        from certbot_postfix.util import is_acceptable_value
+        self.assertTrue(is_acceptable_value('name', 'value', ('value', 'value1')))
+        self.assertFalse(is_acceptable_value('name', 'bad', ('value', 'value1')))
+
+    def test_is_acceptable_protocols(self):
+        from certbot_postfix.util import is_acceptable_value
+        # SSLv2 and SSLv3 are both not supported, unambiguously
+        self.assertFalse(is_acceptable_value('tls_mandatory_protocols_lol',
+            'SSLv2, SSLv3', None))
+        self.assertFalse(is_acceptable_value('tls_protocols_lol',
+            'SSLv2, SSLv3', None))
+        self.assertFalse(is_acceptable_value('tls_protocols_lol',
+            '!SSLv2, !TLSv1', None))
+        self.assertFalse(is_acceptable_value('tls_protocols_lol',
+            '!SSLv2, SSLv3, !SSLv3, ', None))
+        self.assertTrue(is_acceptable_value('tls_protocols_lol',
+            '!SSLv2, !SSLv3', None))
+        self.assertTrue(is_acceptable_value('tls_protocols_lol',
+            '!SSLv3, !TLSv1, !SSLv2', None))
+        # TLSv1.2 is supported unambiguously
+        self.assertFalse(is_acceptable_value('tls_protocols_lol',
+            'TLSv1, TLSv1.1,', None))
+        self.assertFalse(is_acceptable_value('tls_protocols_lol',
+            'TLSv1.2, !TLSv1.2,', None))
+        self.assertTrue(is_acceptable_value('tls_protocols_lol',
+            'TLSv1.2, ', None))
+        self.assertTrue(is_acceptable_value('tls_protocols_lol',
+            'TLSv1, TLSv1.1, TLSv1.2', None))
+
+if __name__ == '__main__': # pragma: no cover
+    unittest.main()

certbot-postfix/certbot_postfix/util.py

@@ -0,0 +1,292 @@
+"""Utility functions for use in the Postfix installer."""
+import logging
+import re
+import subprocess
+
+from certbot import errors
+from certbot import util as certbot_util
+from certbot.plugins import util as plugins_util
+from certbot_postfix import constants
+
+logger = logging.getLogger(__name__)
+
+COMMAND = "postfix"
+
+class PostfixUtilBase(object):
+    """A base class for wrapping Postfix command line utilities."""
+
+    def __init__(self, executable, config_dir=None):
+        """Sets up the Postfix utility class.
+
+        :param str executable: name or path of the Postfix utility
+        :param str config_dir: path to an alternative Postfix config
+
+        :raises .NoInstallationError: when the executable isn't found
+
+        """
+        self.executable = executable
+        verify_exe_exists(executable)
+        self._set_base_command(config_dir)
+        self.config_dir = None
+
+    def _set_base_command(self, config_dir):
+        self._base_command = [self.executable]
+        if config_dir is not None:
+            self._base_command.extend(('-c', config_dir,))
+
+    def _call(self, extra_args=None):
+        """Runs the Postfix utility and returns the result.
+
+        :param list extra_args: additional arguments for the command
+
+        :returns: data written to stdout and stderr
+        :rtype: `tuple` of `str`
+
+        :raises subprocess.CalledProcessError: if the command fails
+
+        """
+        args = list(self._base_command)
+        if extra_args is not None:
+            args.extend(extra_args)
+        return check_all_output(args)
+
+    def _get_output(self, extra_args=None):
+        """Runs the Postfix utility and returns only stdout output.
+
+        This function relies on self._call for running the utility.
+
+        :param list extra_args: additional arguments for the command
+
+        :returns: data written to stdout
+        :rtype: str
+
+        :raises subprocess.CalledProcessError: if the command fails
+
+        """
+        return self._call(extra_args)[0]
+
+class PostfixUtil(PostfixUtilBase):
+    """Wrapper around Postfix CLI tool.
+    """
+
+    def __init__(self, config_dir=None):
+        super(PostfixUtil, self).__init__(COMMAND, config_dir)
+
+    def test(self):
+        """Make sure the configuration is valid.
+
+        :raises .MisconfigurationError: if the config is invalid
+        """
+        try:
+            self._call(["check"])
+        except subprocess.CalledProcessError as e:
+            logger.debug("Could not check postfix configuration:\n%s", e)
+            raise errors.MisconfigurationError(
+                "Postfix failed internal configuration check.")
+
+    def restart(self):
+        """Restart or refresh the server content.
+
+        :raises .PluginError: when server cannot be restarted
+
+        """
+        logger.info("Reloading Postfix configuration...")
+        if self._is_running():
+            self._reload()
+        else:
+            self._start()
+
+
+    def _is_running(self):
+        """Is Postfix currently running?
+
+        Uses the 'postfix status' command to determine if Postfix is
+        currently running using the specified configuration files.
+
+        :returns: True if Postfix is running, otherwise, False
+        :rtype: bool
+
+        """
+        try:
+            self._call(["status"])
+        except subprocess.CalledProcessError:
+            return False
+        return True
+
+    def _start(self):
+        """Instructions Postfix to start running.
+
+        :raises .PluginError: when Postfix cannot start
+
+        """
+        try:
+            self._call(["start"])
+        except subprocess.CalledProcessError:
+            raise errors.PluginError("Postfix failed to start")
+
+    def _reload(self):
+        """Instructs Postfix to reload its configuration.
+
+        If Postfix isn't currently running, this method will fail.
+
+        :raises .PluginError: when Postfix cannot reload
+        """
+        try:
+            self._call(["reload"])
+        except subprocess.CalledProcessError:
+            raise errors.PluginError(
+                "Postfix failed to reload its configuration")
+
+def check_all_output(*args, **kwargs):
+    """A version of subprocess.check_output that also captures stderr.
+
+    This is the same as :func:`subprocess.check_output` except output
+    written to stderr is also captured and returned to the caller. The
+    return value is a tuple of two strings (rather than byte strings).
+    To accomplish this, the caller cannot set the stdout, stderr, or
+    universal_newlines parameters to :class:`subprocess.Popen`.
+
+    Additionally, if the command exits with a nonzero status, output is
+    not included in the raised :class:`subprocess.CalledProcessError`
+    because Python 2.6 does not support this. Instead, the failure
+    including the output is logged.
+
+    :param tuple args: positional arguments for Popen
+    :param dict kwargs: keyword arguments for Popen
+
+    :returns: data written to stdout and stderr
+    :rtype: `tuple` of `str`
+
+    :raises ValueError: if arguments are invalid
+    :raises subprocess.CalledProcessError: if the command fails
+
+    """
+    for keyword in ('stdout', 'stderr', 'universal_newlines',):
+        if keyword in kwargs:
+            raise ValueError(
+                keyword + ' argument not allowed, it will be overridden.')
+
+    kwargs['stdout'] = subprocess.PIPE
+    kwargs['stderr'] = subprocess.PIPE
+    kwargs['universal_newlines'] = True
+
+    process = subprocess.Popen(*args, **kwargs)
+    output, err = process.communicate()
+    retcode = process.poll()
+    if retcode:
+        cmd = kwargs.get('args')
+        if cmd is None:
+            cmd = args[0]
+        logger.debug(
+            "'%s' exited with %d. stdout output was:\n%s\nstderr output was:\n%s",
+            cmd, retcode, output, err)
+        raise subprocess.CalledProcessError(retcode, cmd)
+    return (output, err)
+
+
+def verify_exe_exists(exe, message=None):
+    """Ensures an executable with the given name is available.
+
+    If an executable isn't found for the given path or name, extra
+    directories are added to the user's PATH to help find system
+    utilities that may not be available in the default cron PATH.
+
+    :param str exe: executable path or name
+    :param str message: Error message to print.
+
+    :raises .NoInstallationError: when the executable isn't found
+
+    """
+    if message is None:
+        message = "Cannot find executable '{0}'.".format(exe)
+    if not (certbot_util.exe_exists(exe) or plugins_util.path_surgery(exe)):
+        raise errors.NoInstallationError(message)
+
+def report_master_overrides(name, overrides, acceptable_overrides=None):
+    """If the value for a parameter `name` is overridden by other services,
+    report a warning to notify the user. If `parameter` is a TLS version parameter
+    (i.e., `parameter` contains 'tls_protocols' or 'tls_mandatory_protocols'), then
+    `acceptable_overrides` isn't used each value in overrides is inspected for secure TLS
+    versions.
+
+    :param str name: The name of the parameter that is being overridden.
+    :param list overrides: The values that other services are setting for `name`.
+        Each override is a tuple: (service name, value)
+    :param tuple acceptable_overrides: Override values that are acceptable. For instance, if
+        another service is overriding our parameter with a more secure option, we don't have
+        to warn. If this is set to None, errors are raised for *any* overrides of `name`!
+    """
+    error_string = ""
+    for override in overrides:
+        service, value = override
+        # If this override is acceptable:
+        if acceptable_overrides is not None and \
+            is_acceptable_value(name, value, acceptable_overrides):
+            continue
+        error_string += "  {0}: {1}\n".format(service, value)
+    if error_string:
+        raise errors.PluginError("{0} is overridden with less secure options by the "
+             "following services in master.cf:\n".format(name) + error_string)
+
+
+def is_acceptable_value(parameter, value, acceptable=None):
+    """ Returns whether the `value` for this `parameter` is acceptable,
+    given a tuple of `acceptable` values. If `parameter` is a TLS version parameter
+    (i.e., `parameter` contains 'tls_protocols' or 'tls_mandatory_protocols'), then
+    `acceptable` isn't used and `value` is inspected for secure TLS versions.
+
+    :param str parameter:       The name of the parameter being set.
+    :param str value:           Proposed new value for parameter.
+    :param tuple acceptable:    List of acceptable values for parameter.
+    """
+    # Check if param value is a comma-separated list of protocols.
+    # Otherwise, just check whether the value is in the acceptable list.
+    if 'tls_protocols' in parameter or 'tls_mandatory_protocols' in parameter:
+        return _has_acceptable_tls_versions(value)
+    if acceptable is not None:
+        return value in acceptable
+    return False
+
+
+def _has_acceptable_tls_versions(parameter_string):
+    """
+    Checks to see if the list of TLS protocols is acceptable.
+    This requires that TLSv1.2 is supported, and neither SSLv2 nor SSLv3 are supported.
+
+    Should be a string of protocol names delimited by commas, spaces, or colons.
+
+    Postfix's documents suggest listing protocols to exclude, like "!SSLv2, !SSLv3".
+    Listing the protocols to include, like "TLSv1, TLSv1.1, TLSv1.2" is okay as well,
+    though not recommended
+
+    When these two modes are interspersed, the presence of a single non-negated protocol name
+    (i.e. "TLSv1" rather than "!TLSv1") automatically excludes all other unnamed protocols.
+
+    In addition, the presence of both a protocol name inclusion and exclusion isn't explicitly
+    documented, so this method should return False if it encounters contradicting statements
+    about TLSv1.2, SSLv2, or SSLv3. (for instance, "SSLv3, !SSLv3").
+    """
+    if not parameter_string:
+        return False
+    bad_versions = list(constants.TLS_VERSIONS)
+    for version in constants.ACCEPTABLE_TLS_VERSIONS:
+        del bad_versions[bad_versions.index(version)]
+    supported_version_list = re.split("[, :]+", parameter_string)
+    # The presence of any non-"!" protocol listing excludes the others by default.
+    inclusion_list = False
+    for version in supported_version_list:
+        if not version:
+            continue
+        if version in bad_versions: # short-circuit if we recognize any bad version
+            return False
+        if version[0] != "!":
+            inclusion_list = True
+    if inclusion_list: # For any inclusion list, we still require TLS 1.2.
+        if "TLSv1.2" not in supported_version_list or "!TLSv1.2" in supported_version_list:
+            return False
+    else:
+        for bad_version in bad_versions:
+            if "!" + bad_version not in supported_version_list:
+                return False
+    return True
+

certbot-postfix/docs/.gitignore

@@ -0,0 +1 @@
+/_build/

certbot-postfix/docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = certbot-postfix
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

certbot-postfix/docs/api.rst

@@ -0,0 +1,8 @@
+=================
+API Documentation
+=================
+
+.. toctree::
+   :glob:
+
+   api/**

certbot-postfix/docs/api/installer.rst

@@ -0,0 +1,5 @@
+:mod:`certbot_postfix.installer`
+--------------------------------------
+
+.. automodule:: certbot_postfix.installer
+   :members:

certbot-postfix/docs/api/postconf.rst

@@ -0,0 +1,5 @@
+:mod:`certbot_postfix.postconf`
+-------------------------------
+
+.. automodule:: certbot_postfix.postconf
+   :members:

certbot-postfix/docs/conf.py

@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = u'certbot-postfix'
+copyright = u'2018, Certbot Project'
+author = u'Certbot Project'
+
+# The short X.Y version
+version = u'0'
+# The full version, including alpha/beta/rc tags
+release = u'0'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'sphinx.ext.coverage',
+    'sphinx.ext.viewcode',
+]
+
+autodoc_member_order = 'bysource'
+autodoc_default_flags = ['show-inheritance', 'private-members']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = u'en'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path .
+exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store']
+
+default_role = 'py:obj'
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+
+# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs
+# on_rtd is whether we are on readthedocs.org
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+if not on_rtd:  # only import and set the theme if we're building docs locally
+    import sphinx_rtd_theme
+    html_theme = 'sphinx_rtd_theme'
+    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+# otherwise, readthedocs.org uses their theme by default, so no need to specify it
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'certbot-postfixdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'certbot-postfix.tex', u'certbot-postfix Documentation',
+     u'Certbot Project', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'certbot-postfix', u'certbot-postfix Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'certbot-postfix', u'certbot-postfix Documentation',
+     author, 'certbot-postfix', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Extension configuration -------------------------------------------------
+
+# -- Options for intersphinx extension ---------------------------------------
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/', None),
+    'acme': ('https://acme-python.readthedocs.org/en/latest/', None),
+    'certbot': ('https://certbot.eff.org/docs/', None),
+}
+
+# -- Options for todo extension ----------------------------------------------
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True

certbot-postfix/docs/index.rst

@@ -0,0 +1,28 @@
+.. certbot-postfix documentation master file, created by
+   sphinx-quickstart on Wed May  2 16:01:06 2018.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to certbot-postfix's documentation!
+===========================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+.. automodule:: certbot_postfix
+   :members:
+
+.. toctree::
+   :maxdepth: 1
+
+   api
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

certbot-postfix/docs/make.bat

@@ -0,0 +1,36 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+set SPHINXPROJ=certbot-postfix
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

certbot-postfix/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.25.0
+certbot[dev]==0.23.0

certbot-postfix/setup.cfg

@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal = 1

certbot-postfix/setup.py

@@ -0,0 +1,63 @@
+from setuptools import setup
+from setuptools import find_packages
+
+
+version = '0.26.0.dev0'
+
+install_requires = [
+    'acme>=0.25.0',
+    'certbot>=0.23.0',
+    'setuptools',
+    'six',
+    'zope.component',
+    'zope.interface',
+]
+
+docs_extras = [
+    'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
+    'sphinx_rtd_theme',
+]
+
+setup(
+    name='certbot-postfix',
+    version=version,
+    description="Postfix plugin for Certbot",
+    url='https://github.com/certbot/certbot',
+    author="Certbot Project",
+    author_email='client-dev@letsencrypt.org',
+    license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Environment :: Plugins',
+        'Intended Audience :: System Administrators',
+        'License :: OSI Approved :: Apache Software License',
+        'Operating System :: POSIX :: Linux',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Topic :: Communications :: Email :: Mail Transport Agents',
+        'Topic :: Security',
+        'Topic :: System :: Installation/Setup',
+        'Topic :: System :: Networking',
+        'Topic :: System :: Systems Administration',
+        'Topic :: Utilities',
+    ],
+
+    packages=find_packages(),
+    include_package_data=True,
+    install_requires=install_requires,
+    extras_require={
+        'docs': docs_extras,
+    },
+    entry_points={
+        'certbot.plugins': [
+            'postfix = certbot_postfix:Installer',
+        ],
+    },
+    test_suite='certbot_postfix',
+)

certbot/__init__.py

@@ -1,4 +1,4 @@
 """Certbot client."""
 
 # version number like 1.2.3a0, must have at least 2 parts, like 1.2
-__version__ = '0.24.0.dev0'
+__version__ = '0.26.0.dev0'

certbot/account.py

@@ -16,6 +16,7 @@ import zope.component
 from acme import fields as acme_fields
 from acme import messages
 
+from certbot import constants
 from certbot import errors
 from certbot import interfaces
 from certbot import util
@@ -142,7 +143,11 @@ class AccountFileStorage(interfaces.AccountStorage):
                                    self.config.strict_permissions)
 
     def _account_dir_path(self, account_id):
-        return os.path.join(self.config.accounts_dir, account_id)
+        return self._account_dir_path_for_server_path(account_id, self.config.server_path)
+
+    def _account_dir_path_for_server_path(self, account_id, server_path):
+        accounts_dir = self.config.accounts_dir_for_server_path(server_path)
+        return os.path.join(accounts_dir, account_id)
 
     @classmethod
     def _regr_path(cls, account_dir_path):
@@ -156,25 +161,57 @@ class AccountFileStorage(interfaces.AccountStorage):
     def _metadata_path(cls, account_dir_path):
         return os.path.join(account_dir_path, "meta.json")
 
-    def find_all(self):
+    def _find_all_for_server_path(self, server_path):
+        accounts_dir = self.config.accounts_dir_for_server_path(server_path)
         try:
-            candidates = os.listdir(self.config.accounts_dir)
+            candidates = os.listdir(accounts_dir)
         except OSError:
             return []
 
         accounts = []
         for account_id in candidates:
             try:
-                accounts.append(self.load(account_id))
+                accounts.append(self._load_for_server_path(account_id, server_path))
             except errors.AccountStorageError:
                 logger.debug("Account loading problem", exc_info=True)
+
+        if not accounts and server_path in constants.LE_REUSE_SERVERS:
+            # find all for the next link down
+            prev_server_path = constants.LE_REUSE_SERVERS[server_path]
+            prev_accounts = self._find_all_for_server_path(prev_server_path)
+            # if we found something, link to that
+            if prev_accounts:
+                try:
+                    self._symlink_to_accounts_dir(prev_server_path, server_path)
+                except OSError:
+                    return []
+            accounts = prev_accounts
         return accounts
 
-    def load(self, account_id):
-        account_dir_path = self._account_dir_path(account_id)
-        if not os.path.isdir(account_dir_path):
-            raise errors.AccountNotFound(
-                "Account at %s does not exist" % account_dir_path)
+    def find_all(self):
+        return self._find_all_for_server_path(self.config.server_path)
+
+    def _symlink_to_accounts_dir(self, prev_server_path, server_path):
+        accounts_dir = self.config.accounts_dir_for_server_path(server_path)
+        if os.path.islink(accounts_dir):
+            os.unlink(accounts_dir)
+        else:
+            os.rmdir(accounts_dir)
+        prev_account_dir = self.config.accounts_dir_for_server_path(prev_server_path)
+        os.symlink(prev_account_dir, accounts_dir)
+
+    def _load_for_server_path(self, account_id, server_path):
+        account_dir_path = self._account_dir_path_for_server_path(account_id, server_path)
+        if not os.path.isdir(account_dir_path): # isdir is also true for symlinks
+            if server_path in constants.LE_REUSE_SERVERS:
+                prev_server_path = constants.LE_REUSE_SERVERS[server_path]
+                prev_loaded_account = self._load_for_server_path(account_id, prev_server_path)
+                # we didn't error so we found something, so create a symlink to that
+                self._symlink_to_accounts_dir(prev_server_path, server_path)
+                return prev_loaded_account
+            else:
+                raise errors.AccountNotFound(
+                    "Account at %s does not exist" % account_dir_path)
 
         try:
             with open(self._regr_path(account_dir_path)) as regr_file:
@@ -193,6 +230,9 @@ class AccountFileStorage(interfaces.AccountStorage):
                     account_id, acc.id))
         return acc
 
+    def load(self, account_id):
+        return self._load_for_server_path(account_id, self.config.server_path)
+
     def save(self, account, acme):
         self._save(account, acme, regr_only=False)
 

certbot/auth_handler.py

@@ -8,7 +8,9 @@ import zope.component
 
 from acme import challenges
 from acme import messages
-
+# pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import DefaultDict, Dict, List, Set, Collection
+# pylint: enable=unused-import, no-name-in-module
 from certbot import achallenges
 from certbot import errors
 from certbot import error_handler
@@ -117,7 +119,7 @@ class AuthHandler(object):
 
     def _solve_challenges(self, aauthzrs):
         """Get Responses for challenges from authenticators."""
-        resp = []
+        resp = []  # type: Collection[acme.challenges.ChallengeResponse]
         all_achalls = self._get_all_achalls(aauthzrs)
         try:
             if all_achalls:
@@ -133,10 +135,9 @@ class AuthHandler(object):
 
     def _get_all_achalls(self, aauthzrs):
         """Return all active challenges."""
-        all_achalls = []
+        all_achalls = []  # type: Collection[challenges.ChallengeResponse]
         for aauthzr in aauthzrs:
             all_achalls.extend(aauthzr.achalls)
-
         return all_achalls
 
     def _respond(self, aauthzrs, resp, best_effort):
@@ -146,7 +147,8 @@ class AuthHandler(object):
 
         """
         # TODO: chall_update is a dirty hack to get around acme-spec #105
-        chall_update = dict()
+        chall_update = dict() \
+        # type: Dict[int, List[achallenges.KeyAuthorizationAnnotatedChallenge]]
         self._send_responses(aauthzrs, resp, chall_update)
 
         # Check for updated status...
@@ -198,7 +200,7 @@ class AuthHandler(object):
         while indices_to_check and rounds < max_rounds:
             # TODO: Use retry-after...
             time.sleep(min_sleep)
-            all_failed_achalls = set()
+            all_failed_achalls = set()  # type: Set[achallenges.KeyAuthorizationAnnotatedChallenge]
             for index in indices_to_check:
                 comp_achalls, failed_achalls = self._handle_check(
                     aauthzrs, index, chall_update[index])
@@ -424,7 +426,7 @@ def _find_smart_path(challbs, preferences, combinations):
 
     # max_cost is now equal to sum(indices) + 1
 
-    best_combo = []
+    best_combo = None
     # Set above completing all of the available challenges
     best_combo_cost = max_cost
 
@@ -479,7 +481,7 @@ def _report_no_chall_path(challbs):
         msg += (
             " You may need to use an authenticator "
             "plugin that can do challenges over DNS.")
-    logger.fatal(msg)
+    logger.critical(msg)
     raise errors.AuthorizationError(msg)
 
 
@@ -522,11 +524,11 @@ def _report_failed_challs(failed_achalls):
         :class:`certbot.achallenges.AnnotatedChallenge`.
 
     """
-    problems = dict()
+    problems = collections.defaultdict(list)\
+    # type: DefaultDict[str, List[achallenges.KeyAuthorizationAnnotatedChallenge]]
     for achall in failed_achalls:
         if achall.error:
-            problems.setdefault(achall.error.typ, []).append(achall)
-
+            problems[achall.error.typ].append(achall)
     reporter = zope.component.getUtility(interfaces.IReporter)
     for achalls in six.itervalues(problems):
         reporter.add_message(

certbot/cert_manager.py

@@ -7,6 +7,7 @@ import re
 import traceback
 import zope.component
 
+from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
 from certbot import crypto_util
 from certbot import errors
 from certbot import interfaces
@@ -226,7 +227,7 @@ def match_and_check_overlaps(cli_config, acceptable_matches, match_func, rv_func
     def find_matches(candidate_lineage, return_value, acceptable_matches):
         """Returns a list of matches using _search_lineages."""
         acceptable_matches = [func(candidate_lineage) for func in acceptable_matches]
-        acceptable_matches_rv = []
+        acceptable_matches_rv = []  # type: List[str]
         for item in acceptable_matches:
             if isinstance(item, list):
                 acceptable_matches_rv += item
@@ -340,7 +341,7 @@ def _report_human_readable(config, parsed_certs):
 
 def _describe_certs(config, parsed_certs, parse_failures):
     """Print information about the certs we know about"""
-    out = []
+    out = []  # type: List[str]
 
     notify = out.append
 

certbot/cli.py

@@ -12,10 +12,14 @@ import sys
 import configargparse
 import six
 import zope.component
+import zope.interface
 
 from zope.interface import interfaces as zope_interfaces
 
 from acme import challenges
+# pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Any, Dict, Optional
+# pylint: enable=unused-import, no-name-in-module
 
 import certbot
 
@@ -28,12 +32,13 @@ from certbot import util
 
 from certbot.display import util as display_util
 from certbot.plugins import disco as plugins_disco
+import certbot.plugins.enhancements as enhancements
 import certbot.plugins.selection as plugin_selection
 
 logger = logging.getLogger(__name__)
 
 # Global, to save us from a lot of argument passing within the scope of this module
-helpful_parser = None
+helpful_parser = None  # type: Optional[HelpfulArgumentParser]
 
 # For help strings, figure out how the user ran us.
 # When invoked from letsencrypt-auto, sys.argv[0] is something like:
@@ -196,17 +201,17 @@ def set_by_cli(var):
     (CLI or config file) including if the user explicitly set it to the
     default.  Returns False if the variable was assigned a default value.
     """
-    detector = set_by_cli.detector
-    if detector is None:
+    detector = set_by_cli.detector  # type: ignore
+    if detector is None and helpful_parser is not None:
         # Setup on first run: `detector` is a weird version of config in which
         # the default value of every attribute is wrangled to be boolean-false
         plugins = plugins_disco.PluginsRegistry.find_all()
         # reconstructed_args == sys.argv[1:], or whatever was passed to main()
         reconstructed_args = helpful_parser.args + [helpful_parser.verb]
-        detector = set_by_cli.detector = prepare_and_parse_args(
+        detector = set_by_cli.detector = prepare_and_parse_args(  # type: ignore
             plugins, reconstructed_args, detect_defaults=True)
         # propagate plugin requests: eg --standalone modifies config.authenticator
-        detector.authenticator, detector.installer = (
+        detector.authenticator, detector.installer = (  # type: ignore
             plugin_selection.cli_plugin_requests(detector))
 
     if not isinstance(getattr(detector, var), _Default):
@@ -220,7 +225,10 @@ def set_by_cli(var):
             return True
 
     return False
+
 # static housekeeping var
+# functions attributed are not supported by mypy
+# https://github.com/python/mypy/issues/2087
 set_by_cli.detector = None  # type: ignore
 
 
@@ -236,8 +244,10 @@ def has_default_value(option, value):
     :rtype: bool
 
     """
-    return (option in helpful_parser.defaults and
-            helpful_parser.defaults[option] == value)
+    if helpful_parser is not None:
+        return (option in helpful_parser.defaults and
+                helpful_parser.defaults[option] == value)
+    return False
 
 
 def option_was_set(option, value):
@@ -254,11 +264,12 @@ def option_was_set(option, value):
 
 
 def argparse_type(variable):
-    "Return our argparse type function for a config variable (default: str)"
+    """Return our argparse type function for a config variable (default: str)"""
     # pylint: disable=protected-access
-    for action in helpful_parser.parser._actions:
-        if action.type is not None and action.dest == variable:
-            return action.type
+    if helpful_parser is not None:
+        for action in helpful_parser.parser._actions:
+            if action.type is not None and action.dest == variable:
+                return action.type
     return str
 
 def read_file(filename, mode="rb"):
@@ -291,10 +302,12 @@ def flag_default(name):
 
 def config_help(name, hidden=False):
     """Extract the help message for an `.IConfig` attribute."""
+    # pylint: disable=no-member
     if hidden:
         return argparse.SUPPRESS
     else:
-        return interfaces.IConfig[name].__doc__
+        field = interfaces.IConfig.__getitem__(name) # type: zope.interface.interface.Attribute
+        return field.__doc__
 
 
 class HelpfulArgumentGroup(object):
@@ -418,7 +431,7 @@ VERB_HELP = [
     }),
     ("enhance", {
         "short": "Add security enhancements to your existing configuration",
-        "opts": ("Helps to harden the TLS configration by adding security enhancements "
+        "opts": ("Helps to harden the TLS configuration by adding security enhancements "
                  "to already existing configuration."),
         "usage": "\n\n  certbot enhance [options]\n\n"
     }),
@@ -473,7 +486,7 @@ class HelpfulArgumentParser(object):
         HELP_TOPICS += list(self.VERBS) + self.COMMANDS_TOPICS + ["manage"]
 
         plugin_names = list(plugins)
-        self.help_topics = HELP_TOPICS + plugin_names + [None]
+        self.help_topics = HELP_TOPICS + plugin_names + [None]  # type: ignore
 
         self.detect_defaults = detect_defaults
         self.args = args
@@ -492,8 +505,11 @@ class HelpfulArgumentParser(object):
         short_usage = self._usage_string(plugins, self.help_arg)
 
         self.visible_topics = self.determine_help_topics(self.help_arg)
-        self.groups = {}       # elements are added by .add_group()
-        self.defaults = {}     # elements are added by .parse_args()
+
+        # elements are added by .add_group()
+        self.groups = {}  # type: Dict[str, argparse._ArgumentGroup]
+        # elements are added by .parse_args()
+        self.defaults = {}  # type: Dict[str, Any]
 
         self.parser = configargparse.ArgParser(
             prog="certbot",
@@ -612,6 +628,10 @@ class HelpfulArgumentParser(object):
                 raise errors.Error("Using --allow-subset-of-names with a"
                                    " wildcard domain is not supported.")
 
+        if parsed_args.hsts and parsed_args.auto_hsts:
+            raise errors.Error(
+                "Parameters --hsts and --auto-hsts cannot be used simultaneously.")
+
         possible_deprecation_warning(parsed_args)
 
         return parsed_args
@@ -805,7 +825,6 @@ class HelpfulArgumentParser(object):
             if self.help_arg:
                 for v in verbs:
                     self.groups[topic].add_argument(v, help=VERB_HELP_MAP[v]["short"])
-
         return HelpfulArgumentGroup(self, topic)
 
     def add_plugin_args(self, plugins):
@@ -1003,6 +1022,12 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False):  # pylint: dis
              "certificate already exists for the requested certificate name "
              "but does not match the requested domains, renew it now, "
              "regardless of whether it is near expiry.")
+    helpful.add(
+        "automation", "--reuse-key", dest="reuse_key",
+        action="store_true", default=flag_default("reuse_key"),
+        help="When renewing, use the same private key as the existing "
+             "certificate.")
+
     helpful.add(
         ["automation", "renew", "certonly"],
         "--allow-subset-of-names", action="store_true",
@@ -1057,7 +1082,7 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False):  # pylint: dis
         help="Show tracebacks in case of errors, and allow certbot-auto "
              "execution on experimental platforms")
     helpful.add(
-        [None, "certonly", "renew", "run"], "--debug-challenges", action="store_true",
+        [None, "certonly", "run"], "--debug-challenges", action="store_true",
         default=flag_default("debug_challenges"),
         help="After setting up challenges, wait for user input before "
              "submitting to CA")
@@ -1192,10 +1217,25 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False):  # pylint: dis
         default=flag_default("directory_hooks"), dest="directory_hooks",
         help="Disable running executables found in Certbot's hook directories"
         " during renewal. (default: False)")
+    helpful.add(
+        "renew", "--disable-renew-updates", action="store_true",
+        default=flag_default("disable_renew_updates"), dest="disable_renew_updates",
+        help="Disable automatic updates to your server configuration that"
+        " would otherwise be done by the selected installer plugin, and triggered"
+        " when the user executes \"certbot renew\", regardless of if the certificate"
+        " is renewed. This setting does not apply to important TLS configuration"
+        " updates.")
+    helpful.add(
+        "renew", "--no-autorenew", action="store_false",
+        default=flag_default("autorenew"), dest="autorenew",
+        help="Disable auto renewal of certificates.")
 
     helpful.add_deprecated_argument("--agree-dev-preview", 0)
     helpful.add_deprecated_argument("--dialog", 0)
 
+    # Populate the command line parameters for new style enhancements
+    enhancements.populate_cli(helpful.add)
+
     _create_subparsers(helpful)
     _paths_parser(helpful)
     # _plugins_parsing should be the last thing to act upon the main
@@ -1288,14 +1328,14 @@ def _paths_parser(helpful):
         verb = helpful.help_arg
 
     cph = "Path to where certificate is saved (with auth --csr), installed from, or revoked."
-    section = ["paths", "install", "revoke", "certonly", "manage"]
+    sections = ["paths", "install", "revoke", "certonly", "manage"]
     if verb == "certonly":
-        add(section, "--cert-path", type=os.path.abspath,
+        add(sections, "--cert-path", type=os.path.abspath,
             default=flag_default("auth_cert_path"), help=cph)
     elif verb == "revoke":
-        add(section, "--cert-path", type=read_file, required=True, help=cph)
+        add(sections, "--cert-path", type=read_file, required=True, help=cph)
     else:
-        add(section, "--cert-path", type=os.path.abspath, help=cph)
+        add(sections, "--cert-path", type=os.path.abspath, help=cph)
 
     section = "paths"
     if verb in ("install", "revoke"):

certbot/client.py

@@ -4,8 +4,11 @@ import logging
 import os
 import platform
 
+
 from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives.asymmetric import rsa
+# https://github.com/python/typeshed/blob/master/third_party/
+# 2/cryptography/hazmat/primitives/asymmetric/rsa.pyi
+from cryptography.hazmat.primitives.asymmetric.rsa import generate_private_key  # type: ignore
 import josepy as jose
 import OpenSSL
 import zope.component
@@ -14,6 +17,7 @@ from acme import client as acme_client
 from acme import crypto_util as acme_crypto_util
 from acme import errors as acme_errors
 from acme import messages
+from acme.magic_typing import Optional  # pylint: disable=unused-import,no-name-in-module
 
 import certbot
 
@@ -61,9 +65,17 @@ def determine_user_agent(config):
     if config.user_agent is None:
         ua = ("CertbotACMEClient/{0} ({1}; {2}{8}) Authenticator/{3} Installer/{4} "
               "({5}; flags: {6}) Py/{7}")
-        ua = ua.format(certbot.__version__, cli.cli_command, util.get_os_info_ua(),
+        if os.environ.get("CERTBOT_DOCS") == "1":
+            cli_command = "certbot(-auto)"
+            os_info = "OS_NAME OS_VERSION"
+            python_version = "major.minor.patchlevel"
+        else:
+            cli_command = cli.cli_command
+            os_info = util.get_os_info_ua()
+            python_version = platform.python_version()
+        ua = ua.format(certbot.__version__, cli_command, os_info,
                        config.authenticator, config.installer, config.verb,
-                       ua_flags(config), platform.python_version(),
+                       ua_flags(config), python_version,
                        "; " + config.user_agent_comment if config.user_agent_comment else "")
     else:
         ua = config.user_agent
@@ -155,12 +167,16 @@ def register(config, account_storage, tos_cb=None):
         if not config.dry_run:
             logger.info("Registering without email!")
 
+    # If --dry-run is used, and there is no staging account, create one with no email.
+    if config.dry_run:
+        config.email = None
+
     # Each new registration shall use a fresh new key
-    key = jose.JWKRSA(key=jose.ComparableRSAKey(
-        rsa.generate_private_key(
+    rsa_key = generate_private_key(
             public_exponent=65537,
             key_size=config.rsa_key_size,
-            backend=default_backend())))
+            backend=default_backend())
+    key = jose.JWKRSA(key=jose.ComparableRSAKey(rsa_key))
     acme = acme_from_config_key(config, key)
     # TODO: add phone?
     regr = perform_registration(acme, config, tos_cb)
@@ -179,8 +195,9 @@ def perform_registration(acme, config, tos_cb):
     Actually register new account, trying repeatedly if there are email
     problems
 
-    :param .IConfig config: Client configuration.
     :param acme.client.Client client: ACME client object.
+    :param .IConfig config: Client configuration.
+    :param Callable tos_cb: a callback to handle Term of Service agreement.
 
     :returns: Registration Resource.
     :rtype: `acme.messages.RegistrationResource`
@@ -266,7 +283,7 @@ class Client(object):
         cert, chain = crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem)
         return cert.encode(), chain.encode()
 
-    def obtain_certificate(self, domains):
+    def obtain_certificate(self, domains, old_keypath=None):
         """Obtains a certificate from the ACME server.
 
         `.register` must be called before `.obtain_certificate`
@@ -279,16 +296,39 @@ class Client(object):
         :rtype: tuple
 
         """
+
+        # We need to determine the key path, key PEM data, CSR path,
+        # and CSR PEM data.  For a dry run, the paths are None because
+        # they aren't permanently saved to disk.  For a lineage with
+        # --reuse-key, the key path and PEM data are derived from an
+        # existing file.
+
+        if old_keypath is not None:
+            # We've been asked to reuse a specific existing private key.
+            # Therefore, we'll read it now and not generate a new one in
+            # either case below.
+            #
+            # We read in bytes here because the type of `key.pem`
+            # created below is also bytes.
+            with open(old_keypath, "rb") as f:
+                keypath = old_keypath
+                keypem = f.read()
+            key = util.Key(file=keypath, pem=keypem) # type: Optional[util.Key]
+            logger.info("Reusing existing private key from %s.", old_keypath)
+        else:
+            # The key is set to None here but will be created below.
+            key = None
+
         # Create CSR from names
         if self.config.dry_run:
-            key = util.Key(file=None,
-                           pem=crypto_util.make_key(self.config.rsa_key_size))
+            key = key or util.Key(file=None,
+                                  pem=crypto_util.make_key(self.config.rsa_key_size))
             csr = util.CSR(file=None, form="pem",
                            data=acme_crypto_util.make_csr(
                                key.pem, domains, self.config.must_staple))
         else:
-            key = crypto_util.init_save_key(
-                self.config.rsa_key_size, self.config.key_dir)
+            key = key or crypto_util.init_save_key(self.config.rsa_key_size,
+                                                   self.config.key_dir)
             csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir)
 
         orderr = self._get_order_and_authorizations(csr.data, self.config.allow_subset_of_names)
@@ -605,8 +645,10 @@ def validate_key_csr(privkey, csr=None):
         if csr.form == "der":
             csr_obj = OpenSSL.crypto.load_certificate_request(
                 OpenSSL.crypto.FILETYPE_ASN1, csr.data)
-            csr = util.CSR(csr.file, OpenSSL.crypto.dump_certificate(
-                OpenSSL.crypto.FILETYPE_PEM, csr_obj), "pem")
+            cert_buffer = OpenSSL.crypto.dump_certificate_request(
+                OpenSSL.crypto.FILETYPE_PEM, csr_obj
+            )
+            csr = util.CSR(csr.file, cert_buffer, "pem")
 
         # If CSR is provided, it must be readable and valid.
         if csr.data and not crypto_util.valid_csr(csr.data):

certbot/configuration.py

@@ -65,8 +65,12 @@ class NamespaceConfig(object):
 
     @property
     def accounts_dir(self):  # pylint: disable=missing-docstring
+        return self.accounts_dir_for_server_path(self.server_path)
+
+    def accounts_dir_for_server_path(self, server_path):
+        """Path to accounts directory based on server_path"""
         return os.path.join(
-            self.namespace.config_dir, constants.ACCOUNTS_DIR, self.server_path)
+            self.namespace.config_dir, constants.ACCOUNTS_DIR, server_path)
 
     @property
     def backup_dir(self):  # pylint: disable=missing-docstring

certbot/constants.py

@@ -37,6 +37,7 @@ CLI_DEFAULTS = dict(
     expand=False,
     renew_by_default=False,
     renew_with_new_domains=False,
+    autorenew=True,
     allow_subset_of_names=False,
     tos=False,
     account=None,
@@ -57,6 +58,7 @@ CLI_DEFAULTS = dict(
     rsa_key_size=2048,
     must_staple=False,
     redirect=None,
+    auto_hsts=False,
     hsts=None,
     uir=None,
     staple=None,
@@ -64,6 +66,8 @@ CLI_DEFAULTS = dict(
     pref_challs=[],
     validate_hooks=True,
     directory_hooks=True,
+    reuse_key=False,
+    disable_renew_updates=False,
 
     # Subparsers
     num=None,
@@ -156,6 +160,13 @@ CONFIG_DIRS_MODE = 0o755
 ACCOUNTS_DIR = "accounts"
 """Directory where all accounts are saved."""
 
+LE_REUSE_SERVERS = {
+    'acme-v02.api.letsencrypt.org/directory': 'acme-v01.api.letsencrypt.org/directory',
+    'acme-staging-v02.api.letsencrypt.org/directory':
+        'acme-staging.api.letsencrypt.org/directory'
+}
+"""Servers that can reuse accounts from other servers."""
+
 BACKUP_DIR = "backups"
 """Directory (relative to `IConfig.work_dir`) where backups are kept."""
 

certbot/crypto_util.py

@@ -7,16 +7,24 @@
 import hashlib
 import logging
 import os
+import warnings
 
-import OpenSSL
 import pyrfc3339
 import six
 import zope.component
+from cryptography.exceptions import InvalidSignature
 from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.asymmetric.ec import ECDSA
+from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey
+from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15
+from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey
+# https://github.com/python/typeshed/tree/master/third_party/2/cryptography
 from cryptography import x509 # type: ignore
+from OpenSSL import crypto
+from OpenSSL import SSL  # type: ignore
 
 from acme import crypto_util as acme_crypto_util
-
+from acme.magic_typing import IO  # pylint: disable=unused-import, no-name-in-module
 from certbot import errors
 from certbot import interfaces
 from certbot import util
@@ -47,7 +55,7 @@ def init_save_key(key_size, key_dir, keyname="key-certbot.pem"):
     try:
         key_pem = make_key(key_size)
     except ValueError as err:
-        logger.exception(err)
+        logger.error("", exc_info=True)
         raise err
 
     config = zope.component.getUtility(interfaces.IConfig)
@@ -111,11 +119,11 @@ def valid_csr(csr):
 
     """
     try:
-        req = OpenSSL.crypto.load_certificate_request(
-            OpenSSL.crypto.FILETYPE_PEM, csr)
+        req = crypto.load_certificate_request(
+            crypto.FILETYPE_PEM, csr)
         return req.verify(req.get_pubkey())
-    except OpenSSL.crypto.Error as error:
-        logger.debug(error, exc_info=True)
+    except crypto.Error:
+        logger.debug("", exc_info=True)
         return False
 
 
@@ -129,13 +137,13 @@ def csr_matches_pubkey(csr, privkey):
     :rtype: bool
 
     """
-    req = OpenSSL.crypto.load_certificate_request(
-        OpenSSL.crypto.FILETYPE_PEM, csr)
-    pkey = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, privkey)
+    req = crypto.load_certificate_request(
+        crypto.FILETYPE_PEM, csr)
+    pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, privkey)
     try:
         return req.verify(pkey)
-    except OpenSSL.crypto.Error as error:
-        logger.debug(error, exc_info=True)
+    except crypto.Error:
+        logger.debug("", exc_info=True)
         return False
 
 
@@ -145,26 +153,26 @@ def import_csr_file(csrfile, data):
     :param str csrfile: CSR filename
     :param str data: contents of the CSR file
 
-    :returns: (`OpenSSL.crypto.FILETYPE_PEM`,
+    :returns: (`crypto.FILETYPE_PEM`,
                util.CSR object representing the CSR,
                list of domains requested in the CSR)
     :rtype: tuple
 
     """
-    PEM = OpenSSL.crypto.FILETYPE_PEM
-    load = OpenSSL.crypto.load_certificate_request
+    PEM = crypto.FILETYPE_PEM
+    load = crypto.load_certificate_request
     try:
         # Try to parse as DER first, then fall back to PEM.
-        csr = load(OpenSSL.crypto.FILETYPE_ASN1, data)
-    except OpenSSL.crypto.Error:
+        csr = load(crypto.FILETYPE_ASN1, data)
+    except crypto.Error:
         try:
             csr = load(PEM, data)
-        except OpenSSL.crypto.Error:
+        except crypto.Error:
             raise errors.Error("Failed to parse CSR file: {0}".format(csrfile))
 
     domains = _get_names_from_loaded_cert_or_req(csr)
     # Internally we always use PEM, so re-encode as PEM before returning.
-    data_pem = OpenSSL.crypto.dump_certificate_request(PEM, csr)
+    data_pem = crypto.dump_certificate_request(PEM, csr)
     return PEM, util.CSR(file=csrfile, data=data_pem, form="pem"), domains
 
 
@@ -178,9 +186,9 @@ def make_key(bits):
 
     """
     assert bits >= 1024  # XXX
-    key = OpenSSL.crypto.PKey()
-    key.generate_key(OpenSSL.crypto.TYPE_RSA, bits)
-    return OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key)
+    key = crypto.PKey()
+    key.generate_key(crypto.TYPE_RSA, bits)
+    return crypto.dump_privatekey(crypto.FILETYPE_PEM, key)
 
 
 def valid_privkey(privkey):
@@ -193,9 +201,9 @@ def valid_privkey(privkey):
 
     """
     try:
-        return OpenSSL.crypto.load_privatekey(
-            OpenSSL.crypto.FILETYPE_PEM, privkey).check()
-    except (TypeError, OpenSSL.crypto.Error):
+        return crypto.load_privatekey(
+            crypto.FILETYPE_PEM, privkey).check()
+    except (TypeError, crypto.Error):
         return False
 
 
@@ -224,13 +232,29 @@ def verify_renewable_cert_sig(renewable_cert):
     :raises errors.Error: If signature verification fails.
     """
     try:
-        with open(renewable_cert.chain, 'rb') as chain:
-            chain, _ = pyopenssl_load_certificate(chain.read())
-        with open(renewable_cert.cert, 'rb') as cert:
-            cert = x509.load_pem_x509_certificate(cert.read(), default_backend())
-        hash_name = cert.signature_hash_algorithm.name
-        OpenSSL.crypto.verify(chain, cert.signature, cert.tbs_certificate_bytes, hash_name)
-    except (IOError, ValueError, OpenSSL.crypto.Error) as e:
+        with open(renewable_cert.chain, 'rb') as chain_file:  # type: IO[bytes]
+            chain = x509.load_pem_x509_certificate(chain_file.read(), default_backend())
+        with open(renewable_cert.cert, 'rb') as cert_file:  # type: IO[bytes]
+            cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend())
+        pk = chain.public_key()
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            if isinstance(pk, RSAPublicKey):
+                # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi
+                verifier = pk.verifier(  # type: ignore
+                    cert.signature, PKCS1v15(), cert.signature_hash_algorithm
+                )
+                verifier.update(cert.tbs_certificate_bytes)
+                verifier.verify()
+            elif isinstance(pk, EllipticCurvePublicKey):
+                verifier = pk.verifier(
+                    cert.signature, ECDSA(cert.signature_hash_algorithm)
+                )
+                verifier.update(cert.tbs_certificate_bytes)
+                verifier.verify()
+            else:
+                raise errors.Error("Unsupported public key type")
+    except (IOError, ValueError, InvalidSignature) as e:
         error_str = "verifying the signature of the cert located at {0} has failed. \
                 Details: {1}".format(renewable_cert.cert, e)
         logger.exception(error_str)
@@ -246,11 +270,11 @@ def verify_cert_matches_priv_key(cert_path, key_path):
     :raises errors.Error: If they don't match.
     """
     try:
-        context = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
+        context = SSL.Context(SSL.SSLv23_METHOD)
         context.use_certificate_file(cert_path)
         context.use_privatekey_file(key_path)
         context.check_privatekey()
-    except (IOError, OpenSSL.SSL.Error) as e:
+    except (IOError, SSL.Error) as e:
         error_str = "verifying the cert located at {0} matches the \
                 private key located at {1} has failed. \
                 Details: {2}".format(cert_path,
@@ -267,12 +291,12 @@ def verify_fullchain(renewable_cert):
     :raises errors.Error: If cert and chain do not combine to fullchain.
     """
     try:
-        with open(renewable_cert.chain) as chain:
-            chain = chain.read()
-        with open(renewable_cert.cert) as cert:
-            cert = cert.read()
-        with open(renewable_cert.fullchain) as fullchain:
-            fullchain = fullchain.read()
+        with open(renewable_cert.chain) as chain_file:  # type: IO[str]
+            chain = chain_file.read()
+        with open(renewable_cert.cert) as cert_file:  # type: IO[str]
+            cert = cert_file.read()
+        with open(renewable_cert.fullchain) as fullchain_file:  # type: IO[str]
+            fullchain = fullchain_file.read()
         if (cert + chain) != fullchain:
             error_str = "fullchain does not match cert + chain for {0}!"
             error_str = error_str.format(renewable_cert.lineagename)
@@ -294,43 +318,43 @@ def pyopenssl_load_certificate(data):
 
     openssl_errors = []
 
-    for file_type in (OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1):
+    for file_type in (crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1):
         try:
-            return OpenSSL.crypto.load_certificate(file_type, data), file_type
-        except OpenSSL.crypto.Error as error:  # TODO: other errors?
+            return crypto.load_certificate(file_type, data), file_type
+        except crypto.Error as error:  # TODO: other errors?
             openssl_errors.append(error)
     raise errors.Error("Unable to load: {0}".format(",".join(
         str(error) for error in openssl_errors)))
 
 
 def _load_cert_or_req(cert_or_req_str, load_func,
-                      typ=OpenSSL.crypto.FILETYPE_PEM):
+                      typ=crypto.FILETYPE_PEM):
     try:
         return load_func(typ, cert_or_req_str)
-    except OpenSSL.crypto.Error as error:
-        logger.exception(error)
+    except crypto.Error:
+        logger.error("", exc_info=True)
         raise
 
 
 def _get_sans_from_cert_or_req(cert_or_req_str, load_func,
-                               typ=OpenSSL.crypto.FILETYPE_PEM):
+                               typ=crypto.FILETYPE_PEM):
     # pylint: disable=protected-access
     return acme_crypto_util._pyopenssl_cert_or_req_san(_load_cert_or_req(
         cert_or_req_str, load_func, typ))
 
 
-def get_sans_from_cert(cert, typ=OpenSSL.crypto.FILETYPE_PEM):
+def get_sans_from_cert(cert, typ=crypto.FILETYPE_PEM):
     """Get a list of Subject Alternative Names from a certificate.
 
     :param str cert: Certificate (encoded).
-    :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1`
+    :param typ: `crypto.FILETYPE_PEM` or `crypto.FILETYPE_ASN1`
 
     :returns: A list of Subject Alternative Names.
     :rtype: list
 
     """
     return _get_sans_from_cert_or_req(
-        cert, OpenSSL.crypto.load_certificate, typ)
+        cert, crypto.load_certificate, typ)
 
 
 def _get_names_from_cert_or_req(cert_or_req, load_func, typ):
@@ -343,24 +367,24 @@ def _get_names_from_loaded_cert_or_req(loaded_cert_or_req):
     return acme_crypto_util._pyopenssl_cert_or_req_all_names(loaded_cert_or_req)
 
 
-def get_names_from_cert(csr, typ=OpenSSL.crypto.FILETYPE_PEM):
+def get_names_from_cert(csr, typ=crypto.FILETYPE_PEM):
     """Get a list of domains from a cert, including the CN if it is set.
 
     :param str cert: Certificate (encoded).
-    :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1`
+    :param typ: `crypto.FILETYPE_PEM` or `crypto.FILETYPE_ASN1`
 
     :returns: A list of domain names.
     :rtype: list
 
     """
     return _get_names_from_cert_or_req(
-        csr, OpenSSL.crypto.load_certificate, typ)
+        csr, crypto.load_certificate, typ)
 
 
-def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM):
+def dump_pyopenssl_chain(chain, filetype=crypto.FILETYPE_PEM):
     """Dump certificate chain into a bundle.
 
-    :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in
+    :param list chain: List of `crypto.X509` (or wrapped in
         :class:`josepy.util.ComparableX509`).
 
     """
@@ -378,7 +402,7 @@ def notBefore(cert_path):
     :rtype: :class:`datetime.datetime`
 
     """
-    return _notAfterBefore(cert_path, OpenSSL.crypto.X509.get_notBefore)
+    return _notAfterBefore(cert_path, crypto.X509.get_notBefore)
 
 
 def notAfter(cert_path):
@@ -390,15 +414,15 @@ def notAfter(cert_path):
     :rtype: :class:`datetime.datetime`
 
     """
-    return _notAfterBefore(cert_path, OpenSSL.crypto.X509.get_notAfter)
+    return _notAfterBefore(cert_path, crypto.X509.get_notAfter)
 
 
 def _notAfterBefore(cert_path, method):
     """Internal helper function for finding notbefore/notafter.
 
     :param str cert_path: path to a cert in PEM format
-    :param function method: one of ``OpenSSL.crypto.X509.get_notBefore``
-        or ``OpenSSL.crypto.X509.get_notAfter``
+    :param function method: one of ``crypto.X509.get_notBefore``
+        or ``crypto.X509.get_notAfter``
 
     :returns: the notBefore or notAfter value from the cert at cert_path
     :rtype: :class:`datetime.datetime`
@@ -406,7 +430,7 @@ def _notAfterBefore(cert_path, method):
     """
     # pylint: disable=redefined-outer-name
     with open(cert_path) as f:
-        x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
+        x509 = crypto.load_certificate(crypto.FILETYPE_PEM,
                                                f.read())
     # pyopenssl always returns bytes
     timestamp = method(x509)
@@ -443,7 +467,7 @@ def cert_and_chain_from_fullchain(fullchain_pem):
     :rtype: tuple
 
     """
-    cert = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM,
-        OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)).decode()
+    cert = crypto.dump_certificate(crypto.FILETYPE_PEM,
+        crypto.load_certificate(crypto.FILETYPE_PEM, fullchain_pem)).decode()
     chain = fullchain_pem[len(cert):].lstrip()
     return (cert, chain)