let apacheconftest handle deps

This reverts commit e5d35099a7.

.gitignore

@@ -38,6 +38,7 @@ tests/letstest/venv/
 
 # pytest cache
 .cache
+.mypy_cache/
 
 # docker files
 .docker

.pylintrc

@@ -41,7 +41,7 @@ load-plugins=linter_plugin
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=fixme,locally-disabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes,cyclic-import,duplicate-code
+disable=fixme,locally-disabled,locally-enabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes,cyclic-import,duplicate-code
 # abstract-class-not-used cannot be disabled locally (at least in
 # pylint 1.4.1), same for abstract-class-little-used
 

.travis.yml

@@ -29,16 +29,21 @@ matrix:
       addons:
     - python: "2.7"
       env: TOXENV=lint
+    - python: "3.4"
+      env: TOXENV=mypy
+    - python: "3.5"
+      env: TOXENV=mypy
     - python: "2.7"
-      env: TOXENV=py27-oldest
+      env: TOXENV='py27-{acme,apache,certbot,dns,nginx}-oldest'
       sudo: required
       services: docker
     - python: "3.4"
       env: TOXENV=py34
       sudo: required
       services: docker
-    - python: "3.6"
-      env: TOXENV=py36
+    - python: "3.7"
+      dist: xenial
+      env: TOXENV=py37
       sudo: required
       services: docker
     - sudo: required
@@ -73,8 +78,6 @@ sudo: false
 
 addons:
   apt:
-    sources:
-    - augeas
     packages:  # Keep in sync with letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh and Boulder.
     - python-dev
     - python-virtualenv
@@ -86,10 +89,6 @@ addons:
     # For certbot-nginx integration testing
     - nginx-light
     - openssl
-    # for apacheconftest
-    - apache2
-    - libapache2-mod-wsgi
-    - libapache2-mod-macro
 
 install: "travis_retry $(command -v pip || command -v pip3) install tox coveralls"
 script:
@@ -103,6 +102,7 @@ notifications:
   irc:
     channels:
       - secure: "SGWZl3ownKx9xKVV2VnGt7DqkTmutJ89oJV9tjKhSs84kLijU6EYdPnllqISpfHMTxXflNZuxtGo0wTDYHXBuZL47w1O32W6nzuXdra5zC+i4sYQwYULUsyfOv9gJX8zWAULiK0Z3r0oho45U+FR5ZN6TPCidi8/eGU+EEPwaAw="
+    on_cancel: never
     on_success: never
     on_failure: always
     use_notice: true

CHANGELOG.md

@@ -2,6 +2,282 @@
 
 Certbot adheres to [Semantic Versioning](http://semver.org/).
 
+## 0.25.1 - 2018-06-13
+
+### Fixed
+
+* TLS-ALPN-01 support has been removed from our acme library. Using our current
+  dependencies, we are unable to provide a correct implementation of this
+  challenge so we decided to remove it from the library until we can provide
+  proper support.
+* Issues causing test failures when running the tests in the acme package with
+  pytest<3.0 has been resolved.
+* certbot-nginx now correctly depends on acme>=0.25.0.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with changes other than their version number were:
+
+* acme
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/56?closed=1
+
+## 0.25.0 - 2018-06-06
+
+### Added
+
+* Support for the ready status type was added to acme. Without this change,
+  Certbot and acme users will begin encountering errors when using Let's
+  Encrypt's ACMEv2 API starting on June 19th for the staging environment and
+  July 5th for production. See
+  https://community.letsencrypt.org/t/acmev2-order-ready-status/62866 for more
+  information.
+* Certbot now accepts the flag --reuse-key which will cause the same key to be
+  used in the certificate when the lineage is renewed rather than generating a
+  new key.
+* You can now add multiple email addresses to your ACME account with Certbot by
+  providing a comma separated list of emails to the --email flag.
+* Support for Let's Encrypt's upcoming TLS-ALPN-01 challenge was added to acme.
+  For more information, see
+  https://community.letsencrypt.org/t/tls-alpn-validation-method/63814/1.
+* acme now supports specifying the source address to bind to when sending
+  outgoing connections. You still cannot specify this address using Certbot.
+* If you run Certbot against Let's Encrypt's ACMEv2 staging server but don't
+  already have an account registered at that server URL, Certbot will
+  automatically reuse your staging account from Let's Encrypt's ACMEv1 endpoint
+  if it exists.
+* Interfaces were added to Certbot allowing plugins to be called at additional
+  points. The `GenericUpdater` interface allows plugins to perform actions
+  every time `certbot renew` is run, regardless of whether any certificates are
+  due for renewal, and the `RenewDeployer` interface allows plugins to perform
+  actions when a certificate is renewed. See `certbot.interfaces` for more
+  information.
+
+### Changed
+
+* When running Certbot with --dry-run and you don't already have a staging
+  account, the created account does not contain an email address even if one
+  was provided to avoid expiration emails from Let's Encrypt's staging server.
+* certbot-nginx does a better job of automatically detecting the location of
+  Nginx's configuration files when run on BSD based systems.
+* acme now requires and uses pytest when running tests with setuptools with
+  `python setup.py test`.
+* `certbot config_changes` no longer waits for user input before exiting.
+
+### Fixed
+
+* Misleading log output that caused users to think that Certbot's standalone
+  plugin failed to bind to a port when performing a challenge has been
+  corrected.
+* An issue where certbot-nginx would fail to enable HSTS if the server block
+  already had an `add_header` directive has been resolved.
+* certbot-nginx now does a better job detecting the server block to base the
+  configuration for TLS-SNI challenges on.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with functional changes were:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/54?closed=1
+
+## 0.24.0 - 2018-05-02
+
+### Added
+
+* certbot now has an enhance subcommand which allows you to configure security
+  enhancements like HTTP to HTTPS redirects, OCSP stapling, and HSTS without
+  reinstalling a certificate.
+* certbot-dns-rfc2136 now allows the user to specify the port to use to reach
+  the DNS server in its credentials file.
+* acme now parses the wildcard field included in authorizations so it can be
+  used by users of the library.
+
+### Changed
+
+* certbot-dns-route53 used to wait for each DNS update to propagate before
+  sending the next one, but now it sends all updates before waiting which
+  speeds up issuance for multiple domains dramatically.
+* Certbot's official Docker images are now based on Alpine Linux 3.7 rather
+  than 3.4 because 3.4 has reached its end-of-life.
+* We've doubled the time Certbot will spend polling authorizations before
+  timing out.
+* The level of the message logged when Certbot is being used with
+  non-standard paths warning that crontabs for renewal included in Certbot
+  packages from OS package managers may not work has been reduced. This stops
+  the message from being written to stderr every time `certbot renew` runs.
+
+### Fixed
+
+* certbot-auto now works with Python 3.6.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with changes other than their version number were:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-dns-digitalocean (only style improvements to tests)
+* certbot-dns-rfc2136
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/52?closed=1
+
+## 0.23.0 - 2018-04-04
+
+### Added
+
+* Support for OpenResty was added to the Nginx plugin.
+
+### Changed
+
+* The timestamps in Certbot's logfiles now use the system's local time zone
+  rather than UTC.
+* Certbot's DNS plugins that use Lexicon now rely on Lexicon>=2.2.1 to be able
+  to create and delete multiple TXT records on a single domain.
+* certbot-dns-google's test suite now works without an internet connection.
+
+### Fixed
+
+* Removed a small window that if during which an error occurred, Certbot
+  wouldn't clean up performed challenges.
+* The parameters `default` and `ipv6only` are now removed from `listen`
+  directives when creating a new server block in the Nginx plugin.
+* `server_name` directives enclosed in quotation marks in Nginx are now properly
+  supported.
+* Resolved an issue preventing the Apache plugin from starting Apache when it's
+  not currently running on RHEL and Gentoo based systems.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with changes other than their version number were:
+
+* certbot
+* certbot-apache
+* certbot-dns-cloudxns
+* certbot-dns-dnsimple
+* certbot-dns-dnsmadeeasy
+* certbot-dns-google
+* certbot-dns-luadns
+* certbot-dns-nsone
+* certbot-dns-rfc2136
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/50?closed=1
+
+## 0.22.2 - 2018-03-19
+
+### Fixed
+
+* A type error introduced in 0.22.1 that would occur during challenge cleanup
+  when a Certbot plugin raises an exception while trying to complete the
+  challenge was fixed.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with changes other than their version number were:
+
+* certbot
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/53?closed=1
+
+## 0.22.1 - 2018-03-19
+
+### Changed
+
+* The ACME server used with Certbot's --dry-run and --staging flags is now
+  Let's Encrypt's ACMEv2 staging server which allows people to also test ACMEv2
+  features with these flags.
+
+### Fixed
+
+* The HTTP Content-Type header is now set to the correct value during
+  certificate revocation with new versions of the ACME protocol.
+* When using Certbot with Let's Encrypt's ACMEv2 server, it would add a blank
+  line to the top of chain.pem and between the certificates in fullchain.pem
+  for each lineage. These blank lines have been removed.
+* Resolved a bug that caused Certbot's --allow-subset-of-names flag not to
+  work.
+* Fixed a regression in acme.client.Client that caused the class to not work
+  when it was initialized without a ClientNetwork which is done by some of the
+  other projects using our ACME library.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with changes other than their version number were:
+
+* acme
+* certbot
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/51?closed=1
+
+## 0.22.0 - 2018-03-07
+
+### Added
+
+* Support for obtaining wildcard certificates and a newer version of the ACME
+  protocol such as the one implemented by Let's Encrypt's upcoming ACMEv2
+  endpoint was added to Certbot and its ACME library. Certbot still works with
+  older ACME versions and will automatically change the version of the protocol
+  used based on the version the ACME CA implements.
+* The Apache and Nginx plugins are now able to automatically install a wildcard
+  certificate to multiple virtual hosts that you select from your server
+  configuration.
+* The `certbot install` command now accepts the `--cert-name` flag for
+  selecting a certificate.
+* `acme.client.BackwardsCompatibleClientV2` was added to Certbot's ACME library
+  which automatically handles most of the differences between new and old ACME
+  versions. `acme.client.ClientV2` is also available for people who only want
+  to support one version of the protocol or want to handle the differences
+  between versions themselves.
+* certbot-auto now supports the flag --install-only which has the script
+  install Certbot and its dependencies and exit without invoking Certbot.
+* Support for issuing a single certificate for a wildcard and base domain was
+  added to our Google Cloud DNS plugin. To do this, we now require your API
+  credentials have additional permissions, however, your credentials will
+  already have these permissions unless you defined a custom role with fewer
+  permissions than the standard DNS administrator role provided by Google.
+  These permissions are also only needed for the case described above so it
+  will continue to work for existing users. For more information about the
+  permissions changes, see the documentation in the plugin.
+
+### Changed
+
+* We have broken lockstep between our ACME library, Certbot, and its plugins.
+  This means that the different components do not need to be the same version
+  to work together like they did previously. This makes packaging easier
+  because not every piece of Certbot needs to be repackaged to ship a change to
+  a subset of its components.
+* Support for Python 2.6 and Python 3.3 has been removed from ACME, Certbot,
+  Certbot's plugins, and certbot-auto. If you are using certbot-auto on a RHEL
+  6 based system, it will walk you through the process of installing Certbot
+  with Python 3 and refuse to upgrade to a newer version of Certbot until you
+  have done so.
+* Certbot's components now work with older versions of setuptools to simplify
+  packaging for EPEL 7.
+
+### Fixed
+
+* Issues caused by Certbot's Nginx plugin adding multiple ipv6only directives
+  has been resolved.
+* A problem where Certbot's Apache plugin would add redundant include
+  directives for the TLS configuration managed by Certbot has been fixed.
+* Certbot's webroot plugin now properly deletes any directories it creates.
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/48?closed=1
+
 ## 0.21.1 - 2018-01-25
 
 ### Fixed

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:2-alpine
+FROM python:2-alpine3.7
 
 ENTRYPOINT [ "certbot" ]
 EXPOSE 80 443

acme/MANIFEST.in

@@ -1,5 +1,6 @@
 include LICENSE.txt
 include README.rst
+include pytest.ini
 recursive-include docs *
 recursive-include examples *
 recursive-include acme/testdata *

acme/acme/challenges.py

@@ -9,6 +9,7 @@ from cryptography.hazmat.primitives import hashes  # type: ignore
 import josepy as jose
 import OpenSSL
 import requests
+import six
 
 from acme import errors
 from acme import crypto_util
@@ -139,16 +140,16 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse):
         return True
 
 
+@six.add_metaclass(abc.ABCMeta)
 class KeyAuthorizationChallenge(_TokenChallenge):
     # pylint: disable=abstract-class-little-used,too-many-ancestors
     """Challenge based on Key Authorization.
 
     :param response_cls: Subclass of `KeyAuthorizationChallengeResponse`
         that will be used to generate `response`.
-
+    :param str typ: type of the challenge
     """
-    __metaclass__ = abc.ABCMeta
-
+    typ = NotImplemented
     response_cls = NotImplemented
     thumbprint_hash_function = (
         KeyAuthorizationChallengeResponse.thumbprint_hash_function)
@@ -477,7 +478,7 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse):
             try:
                 cert = self.probe_cert(domain=domain, **kwargs)
             except errors.Error as error:
-                logger.debug(error, exc_info=True)
+                logger.debug(str(error), exc_info=True)
                 return False
 
         return self.verify_cert(cert)
@@ -506,6 +507,21 @@ class TLSSNI01(KeyAuthorizationChallenge):
         return self.response(account_key).gen_cert(key=kwargs.get('cert_key'))
 
 
+@Challenge.register  # pylint: disable=too-many-ancestors
+class TLSALPN01(KeyAuthorizationChallenge):
+    """ACME tls-alpn-01 challenge.
+
+    This class simply allows parsing the TLS-ALPN-01 challenge returned from
+    the CA. Full TLS-ALPN-01 support is not currently provided.
+
+    """
+    typ = "tls-alpn-01"
+
+    def validation(self, account_key, **kwargs):
+        """Generate validation for the challenge."""
+        raise NotImplementedError()
+
+
 @Challenge.register  # pylint: disable=too-many-ancestors
 class DNS(_TokenChallenge):
     """ACME "dns" challenge."""

acme/acme/challenges_test.py

@@ -393,6 +393,38 @@ class TLSSNI01Test(unittest.TestCase):
         mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key)
 
 
+class TLSALPN01Test(unittest.TestCase):
+
+    def setUp(self):
+        from acme.challenges import TLSALPN01
+        self.msg = TLSALPN01(
+            token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e'))
+        self.jmsg = {
+            'type': 'tls-alpn-01',
+            'token': 'a82d5ff8ef740d12881f6d3c2277ab2e',
+        }
+
+    def test_to_partial_json(self):
+        self.assertEqual(self.jmsg, self.msg.to_partial_json())
+
+    def test_from_json(self):
+        from acme.challenges import TLSALPN01
+        self.assertEqual(self.msg, TLSALPN01.from_json(self.jmsg))
+
+    def test_from_json_hashable(self):
+        from acme.challenges import TLSALPN01
+        hash(TLSALPN01.from_json(self.jmsg))
+
+    def test_from_json_invalid_token_length(self):
+        from acme.challenges import TLSALPN01
+        self.jmsg['token'] = jose.encode_b64jose(b'abcd')
+        self.assertRaises(
+            jose.DeserializationError, TLSALPN01.from_json, self.jmsg)
+
+    def test_validation(self):
+        self.assertRaises(NotImplementedError, self.msg.validation, KEY)
+
+
 class DNSTest(unittest.TestCase):
 
     def setUp(self):

acme/acme/client.py

@@ -9,17 +9,20 @@ import time
 
 import six
 from six.moves import http_client  # pylint: disable=import-error
-
 import josepy as jose
 import OpenSSL
 import re
+from requests_toolbelt.adapters.source import SourceAddressAdapter
 import requests
+from requests.adapters import HTTPAdapter
 import sys
 
 from acme import crypto_util
 from acme import errors
 from acme import jws
 from acme import messages
+# pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Dict, List, Set, Text
 
 
 logger = logging.getLogger(__name__)
@@ -227,8 +230,7 @@ class ClientBase(object):  # pylint: disable=too-many-instance-attributes
         response = self._post(url,
                               messages.Revocation(
                                 certificate=cert,
-                                reason=rsn),
-                                content_type=None)
+                                reason=rsn))
         if response.status_code != http_client.OK:
             raise errors.ClientError(
                 'Successful revocation must return HTTP OK status')
@@ -260,11 +262,12 @@ class Client(ClientBase):
         """
         # pylint: disable=too-many-arguments
         self.key = key
-        self.net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl) if net is None else net
+        if net is None:
+            net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl)
 
         if isinstance(directory, six.string_types):
             directory = messages.Directory.from_json(
-                self.net.get(directory).json())
+                net.get(directory).json())
         super(Client, self).__init__(directory=directory,
             net=net, acme_version=1)
 
@@ -415,7 +418,7 @@ class Client(ClientBase):
         """
         # pylint: disable=too-many-locals
         assert max_attempts > 0
-        attempts = collections.defaultdict(int)
+        attempts = collections.defaultdict(int) # type: Dict[messages.AuthorizationResource, int]
         exhausted = set()
 
         # priority queue with datetime.datetime (based on Retry-After) as key,
@@ -529,7 +532,7 @@ class Client(ClientBase):
         :rtype: `list` of `OpenSSL.crypto.X509` wrapped in `.ComparableX509`
 
         """
-        chain = []
+        chain = [] # type: List[jose.ComparableX509]
         uri = certr.cert_chain_uri
         while uri is not None and len(chain) < max_length:
             response, cert = self._get_cert(uri)
@@ -809,8 +812,8 @@ class BackwardsCompatibleClientV2(object):
                     'certificate, please rerun the command for a new one.')
 
             cert = OpenSSL.crypto.dump_certificate(
-                    OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped)
-            chain = crypto_util.dump_pyopenssl_chain(chain)
+                    OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped).decode()
+            chain = crypto_util.dump_pyopenssl_chain(chain).decode()
 
             return orderr.update(fullchain_pem=(cert + chain))
         else:
@@ -856,18 +859,28 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
     :param bool verify_ssl: Whether to verify certificates on SSL connections.
     :param str user_agent: String to send as User-Agent header.
     :param float timeout: Timeout for requests.
+    :param source_address: Optional source address to bind to when making requests.
+    :type source_address: str or tuple(str, int)
     """
     def __init__(self, key, account=None, alg=jose.RS256, verify_ssl=True,
-                 user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT):
+                 user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT,
+                 source_address=None):
         # pylint: disable=too-many-arguments
         self.key = key
         self.account = account
         self.alg = alg
         self.verify_ssl = verify_ssl
-        self._nonces = set()
+        self._nonces = set() # type: Set[Text]
         self.user_agent = user_agent
         self.session = requests.Session()
         self._default_timeout = timeout
+        adapter = HTTPAdapter()
+
+        if source_address is not None:
+            adapter = SourceAddressAdapter(source_address)
+
+        self.session.mount("http://", adapter)
+        self.session.mount("https://", adapter)
 
     def __del__(self):
         # Try to close the session, but don't show exceptions to the
@@ -1017,7 +1030,7 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
         if response.headers.get("Content-Type") == DER_CONTENT_TYPE:
             debug_content = base64.b64encode(response.content)
         else:
-            debug_content = response.content
+            debug_content = response.content.decode("utf-8")
         logger.debug('Received response:\nHTTP %d\n%s\n\n%s',
                      response.status_code,
                      "\n".join(["{0}: {1}".format(k, v)

acme/acme/client_test.py

@@ -17,6 +17,7 @@ from acme import jws as acme_jws
 from acme import messages
 from acme import messages_test
 from acme import test_util
+from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module
 
 
 CERT_DER = test_util.load_vector('cert.der')
@@ -61,7 +62,8 @@ class ClientTestBase(unittest.TestCase):
         self.contact = ('mailto:cert-admin@example.com', 'tel:+12025551212')
         reg = messages.Registration(
             contact=self.contact, key=KEY.public_key())
-        self.new_reg = messages.NewRegistration(**dict(reg))
+        the_arg = dict(reg) # type: Dict
+        self.new_reg = messages.NewRegistration(**the_arg) # pylint: disable=star-args
         self.regr = messages.RegistrationResource(
             body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1')
 
@@ -99,10 +101,10 @@ class BackwardsCompatibleClientV2Test(ClientTestBase):
         self.chain = [wrapped, wrapped]
 
         self.cert_pem = OpenSSL.crypto.dump_certificate(
-            OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped)
+            OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped).decode()
 
         single_chain = OpenSSL.crypto.dump_certificate(
-            OpenSSL.crypto.FILETYPE_PEM, loaded)
+            OpenSSL.crypto.FILETYPE_PEM, loaded).decode()
         self.chain_pem = single_chain + single_chain
 
         self.fullchain_pem = self.cert_pem + self.chain_pem
@@ -299,6 +301,16 @@ class ClientTest(ClientTestBase):
             directory=uri, key=KEY, alg=jose.RS256, net=self.net)
         self.net.get.assert_called_once_with(uri)
 
+    @mock.patch('acme.client.ClientNetwork')
+    def test_init_without_net(self, mock_net):
+        mock_net.return_value = mock.sentinel.net
+        alg = jose.RS256
+        from acme.client import Client
+        self.client = Client(
+            directory=self.directory, key=KEY, alg=alg)
+        mock_net.called_once_with(KEY, alg=alg, verify_ssl=True)
+        self.assertEqual(self.client.net, mock.sentinel.net)
+
     def test_register(self):
         # "Instance of 'Field' has no to_json/update member" bug:
         # pylint: disable=no-member
@@ -635,8 +647,7 @@ class ClientTest(ClientTestBase):
     def test_revoke(self):
         self.client.revoke(self.certr.body, self.rsn)
         self.net.post.assert_called_once_with(
-            self.directory[messages.Revocation], mock.ANY, content_type=None,
-            acme_version=1)
+            self.directory[messages.Revocation], mock.ANY, acme_version=1)
 
     def test_revocation_payload(self):
         obj = messages.Revocation(certificate=self.certr.body, reason=self.rsn)
@@ -776,8 +787,7 @@ class ClientV2Test(ClientTestBase):
     def test_revoke(self):
         self.client.revoke(messages_test.CERT, self.rsn)
         self.net.post.assert_called_once_with(
-            self.directory["revokeCert"], mock.ANY, content_type=None,
-            acme_version=2)
+            self.directory["revokeCert"], mock.ANY, acme_version=2)
 
 
 class MockJSONDeSerializable(jose.JSONDeSerializable):
@@ -1119,6 +1129,31 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
         self.assertRaises(requests.exceptions.RequestException,
                           self.net.post, 'uri', obj=self.obj)
 
+class ClientNetworkSourceAddressBindingTest(unittest.TestCase):
+    """Tests that if ClientNetwork has a source IP set manually, the underlying library has
+    used the provided source address."""
+
+    def setUp(self):
+        self.source_address = "8.8.8.8"
+
+    def test_source_address_set(self):
+        from acme.client import ClientNetwork
+        net = ClientNetwork(key=None, alg=None, source_address=self.source_address)
+        for adapter in net.session.adapters.values():
+            self.assertTrue(self.source_address in adapter.source_address)
+
+    def test_behavior_assumption(self):
+        """This is a test that guardrails the HTTPAdapter behavior so that if the default for
+        a Session() changes, the assumptions here aren't violated silently."""
+        from acme.client import ClientNetwork
+        # Source address not specified, so the default adapter type should be bound -- this
+        # test should fail if the default adapter type is changed by requests
+        net = ClientNetwork(key=None, alg=None)
+        session = requests.Session()
+        for scheme in session.adapters.keys():
+            client_network_adapter = net.session.adapters.get(scheme)
+            default_adapter = session.adapters.get(scheme)
+            self.assertEqual(client_network_adapter.__class__, default_adapter.__class__)
 
 if __name__ == '__main__':
     unittest.main()  # pragma: no cover

acme/acme/crypto_util.py

@@ -6,11 +6,14 @@ import os
 import re
 import socket
 
-import OpenSSL
+from OpenSSL import crypto
+from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052
 import josepy as jose
 
-
 from acme import errors
+# pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Callable, Union, Tuple, Optional
+# pylint: enable=unused-import, no-name-in-module
 
 
 logger = logging.getLogger(__name__)
@@ -25,7 +28,7 @@ logger = logging.getLogger(__name__)
 # https://www.openssl.org/docs/ssl/SSLv23_method.html). _serve_sni
 # should be changed to use "set_options" to disable SSLv2 and SSLv3,
 # in case it's used for things other than probing/serving!
-_DEFAULT_TLSSNI01_SSL_METHOD = OpenSSL.SSL.SSLv23_METHOD  # type: ignore
+_DEFAULT_TLSSNI01_SSL_METHOD = SSL.SSLv23_METHOD  # type: ignore
 
 
 class SSLSocket(object):  # pylint: disable=too-few-public-methods
@@ -64,9 +67,9 @@ class SSLSocket(object):  # pylint: disable=too-few-public-methods
             logger.debug("Server name (%s) not recognized, dropping SSL",
                          server_name)
             return
-        new_context = OpenSSL.SSL.Context(self.method)
-        new_context.set_options(OpenSSL.SSL.OP_NO_SSLv2)
-        new_context.set_options(OpenSSL.SSL.OP_NO_SSLv3)
+        new_context = SSL.Context(self.method)
+        new_context.set_options(SSL.OP_NO_SSLv2)
+        new_context.set_options(SSL.OP_NO_SSLv3)
         new_context.use_privatekey(key)
         new_context.use_certificate(cert)
         connection.set_context(new_context)
@@ -89,18 +92,18 @@ class SSLSocket(object):  # pylint: disable=too-few-public-methods
     def accept(self):  # pylint: disable=missing-docstring
         sock, addr = self.sock.accept()
 
-        context = OpenSSL.SSL.Context(self.method)
-        context.set_options(OpenSSL.SSL.OP_NO_SSLv2)
-        context.set_options(OpenSSL.SSL.OP_NO_SSLv3)
+        context = SSL.Context(self.method)
+        context.set_options(SSL.OP_NO_SSLv2)
+        context.set_options(SSL.OP_NO_SSLv3)
         context.set_tlsext_servername_callback(self._pick_certificate_cb)
 
-        ssl_sock = self.FakeConnection(OpenSSL.SSL.Connection(context, sock))
+        ssl_sock = self.FakeConnection(SSL.Connection(context, sock))
         ssl_sock.set_accept_state()
 
         logger.debug("Performing handshake with %s", addr)
         try:
             ssl_sock.do_handshake()
-        except OpenSSL.SSL.Error as error:
+        except SSL.Error as error:
             # _pick_certificate_cb might have returned without
             # creating SSL context (wrong server name)
             raise socket.error(error)
@@ -128,30 +131,39 @@ def probe_sni(name, host, port=443, timeout=300,
     :rtype: OpenSSL.crypto.X509
 
     """
-    context = OpenSSL.SSL.Context(method)
+    context = SSL.Context(method)
     context.set_timeout(timeout)
 
     socket_kwargs = {'source_address': source_address}
 
-    host_protocol_agnostic = None if host == '::' or host == '0' else host
+    host_protocol_agnostic = host
+    if host == '::' or host == '0':
+        # https://github.com/python/typeshed/pull/2136
+        # while PR is not merged, we need to ignore
+        host_protocol_agnostic = None
 
     try:
         # pylint: disable=star-args
-        logger.debug("Attempting to connect to %s:%d%s.", host_protocol_agnostic, port,
-            " from {0}:{1}".format(source_address[0], source_address[1]) if \
-            socket_kwargs else "")
-        sock = socket.create_connection((host_protocol_agnostic, port), **socket_kwargs)
+        logger.debug(
+            "Attempting to connect to %s:%d%s.", host_protocol_agnostic, port,
+            " from {0}:{1}".format(
+                source_address[0],
+                source_address[1]
+            ) if socket_kwargs else ""
+        )
+        socket_tuple = (host_protocol_agnostic, port)  # type: Tuple[Optional[str], int]
+        sock = socket.create_connection(socket_tuple, **socket_kwargs)  # type: ignore
     except socket.error as error:
         raise errors.Error(error)
 
     with contextlib.closing(sock) as client:
-        client_ssl = OpenSSL.SSL.Connection(context, client)
+        client_ssl = SSL.Connection(context, client)
         client_ssl.set_connect_state()
         client_ssl.set_tlsext_host_name(name)  # pyOpenSSL>=0.13
         try:
             client_ssl.do_handshake()
             client_ssl.shutdown()
-        except OpenSSL.SSL.Error as error:
+        except SSL.Error as error:
             raise errors.Error(error)
     return client_ssl.get_peer_certificate()
 
@@ -164,18 +176,18 @@ def make_csr(private_key_pem, domains, must_staple=False):
         OCSP Must Staple: https://tools.ietf.org/html/rfc7633).
     :returns: buffer PEM-encoded Certificate Signing Request.
     """
-    private_key = OpenSSL.crypto.load_privatekey(
-        OpenSSL.crypto.FILETYPE_PEM, private_key_pem)
-    csr = OpenSSL.crypto.X509Req()
+    private_key = crypto.load_privatekey(
+        crypto.FILETYPE_PEM, private_key_pem)
+    csr = crypto.X509Req()
     extensions = [
-        OpenSSL.crypto.X509Extension(
+        crypto.X509Extension(
             b'subjectAltName',
             critical=False,
             value=', '.join('DNS:' + d for d in domains).encode('ascii')
         ),
     ]
     if must_staple:
-        extensions.append(OpenSSL.crypto.X509Extension(
+        extensions.append(crypto.X509Extension(
             b"1.3.6.1.5.5.7.1.24",
             critical=False,
             value=b"DER:30:03:02:01:05"))
@@ -183,8 +195,8 @@ def make_csr(private_key_pem, domains, must_staple=False):
     csr.set_pubkey(private_key)
     csr.set_version(2)
     csr.sign(private_key, 'sha256')
-    return OpenSSL.crypto.dump_certificate_request(
-        OpenSSL.crypto.FILETYPE_PEM, csr)
+    return crypto.dump_certificate_request(
+        crypto.FILETYPE_PEM, csr)
 
 def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req):
     common_name = loaded_cert_or_req.get_subject().CN
@@ -221,11 +233,12 @@ def _pyopenssl_cert_or_req_san(cert_or_req):
     parts_separator = ", "
     prefix = "DNS" + part_separator
 
-    if isinstance(cert_or_req, OpenSSL.crypto.X509):
-        func = OpenSSL.crypto.dump_certificate
+    if isinstance(cert_or_req, crypto.X509):
+        # pylint: disable=line-too-long
+        func = crypto.dump_certificate # type: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]]
     else:
-        func = OpenSSL.crypto.dump_certificate_request
-    text = func(OpenSSL.crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8")
+        func = crypto.dump_certificate_request
+    text = func(crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8")
     # WARNING: this function does not support multiple SANs extensions.
     # Multiple X509v3 extensions of the same type is disallowed by RFC 5280.
     match = re.search(r"X509v3 Subject Alternative Name:(?: critical)?\s*(.*)", text)
@@ -252,12 +265,12 @@ def gen_ss_cert(key, domains, not_before=None,
 
     """
     assert domains, "Must provide one or more hostnames for the cert."
-    cert = OpenSSL.crypto.X509()
+    cert = crypto.X509()
     cert.set_serial_number(int(binascii.hexlify(os.urandom(16)), 16))
     cert.set_version(2)
 
     extensions = [
-        OpenSSL.crypto.X509Extension(
+        crypto.X509Extension(
             b"basicConstraints", True, b"CA:TRUE, pathlen:0"),
     ]
 
@@ -266,7 +279,7 @@ def gen_ss_cert(key, domains, not_before=None,
     cert.set_issuer(cert.get_subject())
 
     if force_san or len(domains) > 1:
-        extensions.append(OpenSSL.crypto.X509Extension(
+        extensions.append(crypto.X509Extension(
             b"subjectAltName",
             critical=False,
             value=b", ".join(b"DNS:" + d.encode() for d in domains)
@@ -281,12 +294,15 @@ def gen_ss_cert(key, domains, not_before=None,
     cert.sign(key, "sha256")
     return cert
 
-def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM):
+def dump_pyopenssl_chain(chain, filetype=crypto.FILETYPE_PEM):
     """Dump certificate chain into a bundle.
 
     :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in
         :class:`josepy.util.ComparableX509`).
 
+    :returns: certificate chain bundle
+    :rtype: bytes
+
     """
     # XXX: returns empty string when no chain is available, which
     # shuts up RenewableCert, but might not be the best solution...
@@ -295,7 +311,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM):
         if isinstance(cert, jose.ComparableX509):
             # pylint: disable=protected-access
             cert = cert.wrapped
-        return OpenSSL.crypto.dump_certificate(filetype, cert)
+        return crypto.dump_certificate(filetype, cert)
 
     # assumes that OpenSSL.crypto.dump_certificate includes ending
     # newline character

acme/acme/crypto_util_test.py

@@ -13,6 +13,7 @@ import OpenSSL
 
 from acme import errors
 from acme import test_util
+from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module
 
 
 class SSLSocketAndProbeSNITest(unittest.TestCase):
@@ -41,28 +42,38 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
         self.server_thread = threading.Thread(
             # pylint: disable=no-member
             target=self.server.handle_request)
-        self.server_thread.start()
-        time.sleep(1)  # TODO: avoid race conditions in other way
 
     def tearDown(self):
-        self.server_thread.join()
+        if self.server_thread.is_alive():
+            # The thread may have already terminated.
+            self.server_thread.join()  # pragma: no cover
 
     def _probe(self, name):
         from acme.crypto_util import probe_sni
         return jose.ComparableX509(probe_sni(
             name, host='127.0.0.1', port=self.port))
 
+    def _start_server(self):
+        self.server_thread.start()
+        time.sleep(1)  # TODO: avoid race conditions in other way
+
     def test_probe_ok(self):
+        self._start_server()
         self.assertEqual(self.cert, self._probe(b'foo'))
 
     def test_probe_not_recognized_name(self):
+        self._start_server()
         self.assertRaises(errors.Error, self._probe, b'bar')
 
-    # TODO: py33/py34 tox hangs forever on do_handshake in second probe
-    #def probe_connection_error(self):
-    #    self._probe(b'foo')
-    #    #time.sleep(1)  # TODO: avoid race conditions in other way
-    #    self.assertRaises(errors.Error, self._probe, b'bar')
+    def test_probe_connection_error(self):
+        # pylint has a hard time with six
+        self.server.server_close()  # pylint: disable=no-member
+        original_timeout = socket.getdefaulttimeout()
+        try:
+            socket.setdefaulttimeout(1)
+            self.assertRaises(errors.Error, self._probe, b'bar')
+        finally:
+            socket.setdefaulttimeout(original_timeout)
 
 
 class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase):
@@ -165,7 +176,7 @@ class RandomSnTest(unittest.TestCase):
 
     def setUp(self):
         self.cert_count = 5
-        self.serial_num = []
+        self.serial_num = [] # type: List[int]
         self.key = OpenSSL.crypto.PKey()
         self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
 

acme/acme/magic_typing.py

@@ -0,0 +1,16 @@
+"""Shim class to not have to depend on typing module in prod."""
+import sys
+
+class TypingClass(object):
+    """Ignore import errors by getting anything"""
+    def __getattr__(self, name):
+        return None
+
+try:
+    # mypy doesn't respect modifying sys.modules
+    from typing import *  # pylint: disable=wildcard-import, unused-wildcard-import
+    # pylint: disable=unused-import
+    from typing import Collection, IO  # type: ignore
+    # pylint: enable=unused-import
+except ImportError:
+    sys.modules[__name__] = TypingClass()

acme/acme/magic_typing_test.py

@@ -0,0 +1,41 @@
+"""Tests for acme.magic_typing."""
+import sys
+import unittest
+
+import mock
+
+
+class MagicTypingTest(unittest.TestCase):
+    """Tests for acme.magic_typing."""
+    def test_import_success(self):
+        try:
+            import typing as temp_typing
+        except ImportError: # pragma: no cover
+            temp_typing = None # pragma: no cover
+        typing_class_mock = mock.MagicMock()
+        text_mock = mock.MagicMock()
+        typing_class_mock.Text = text_mock
+        sys.modules['typing'] = typing_class_mock
+        if 'acme.magic_typing' in sys.modules:
+            del sys.modules['acme.magic_typing'] # pragma: no cover
+        from acme.magic_typing import Text # pylint: disable=no-name-in-module
+        self.assertEqual(Text, text_mock)
+        del sys.modules['acme.magic_typing']
+        sys.modules['typing'] = temp_typing
+
+    def test_import_failure(self):
+        try:
+            import typing as temp_typing
+        except ImportError: # pragma: no cover
+            temp_typing = None # pragma: no cover
+        sys.modules['typing'] = None
+        if 'acme.magic_typing' in sys.modules:
+            del sys.modules['acme.magic_typing'] # pragma: no cover
+        from acme.magic_typing import Text # pylint: disable=no-name-in-module
+        self.assertTrue(Text is None)
+        del sys.modules['acme.magic_typing']
+        sys.modules['typing'] = temp_typing
+
+
+if __name__ == '__main__':
+    unittest.main()  # pragma: no cover

acme/acme/messages.py

@@ -145,6 +145,7 @@ STATUS_PROCESSING = Status('processing')
 STATUS_VALID = Status('valid')
 STATUS_INVALID = Status('invalid')
 STATUS_REVOKED = Status('revoked')
+STATUS_READY = Status('ready')
 
 
 class IdentifierType(_Constant):
@@ -284,7 +285,7 @@ class Registration(ResourceBody):
         if phone is not None:
             details.append(cls.phone_prefix + phone)
         if email is not None:
-            details.append(cls.email_prefix + email)
+            details.extend([cls.email_prefix + mail for mail in email.split(',')])
         kwargs['contact'] = tuple(details)
         return cls(**kwargs)
 
@@ -435,6 +436,7 @@ class Authorization(ResourceBody):
     # be absent'... then acme-spec gives example with 'expires'
     # present... That's confusing!
     expires = fields.RFC3339Field('expires', omitempty=True)
+    wildcard = jose.Field('wildcard', omitempty=True)
 
     @challenges.decoder
     def challenges(value):  # pylint: disable=missing-docstring,no-self-argument

acme/acme/messages_test.py

@@ -6,6 +6,7 @@ import mock
 
 from acme import challenges
 from acme import test_util
+from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module
 
 
 CERT = test_util.load_comparable_cert('cert.der')
@@ -85,7 +86,7 @@ class ConstantTest(unittest.TestCase):
         from acme.messages import _Constant
 
         class MockConstant(_Constant):  # pylint: disable=missing-docstring
-            POSSIBLE_NAMES = {}
+            POSSIBLE_NAMES = {} # type: Dict
 
         self.MockConstant = MockConstant  # pylint: disable=invalid-name
         self.const_a = MockConstant('a')

acme/acme/standalone.py

@@ -16,6 +16,7 @@ import OpenSSL
 
 from acme import challenges
 from acme import crypto_util
+from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module
 
 
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class BaseDualNetworkedServers(object):
 
     def __init__(self, ServerClass, server_address, *remaining_args, **kwargs):
         port = server_address[1]
-        self.threads = []
-        self.servers = []
+        self.threads = [] # type: List[threading.Thread]
+        self.servers = [] # type: List[ACMEServerMixin]
 
         # Must try True first.
         # Ubuntu, for example, will fail to bind to IPv4 if we've already bound
@@ -82,9 +83,22 @@ class BaseDualNetworkedServers(object):
                 new_address = (server_address[0],) + (port,) + server_address[2:]
                 new_args = (new_address,) + remaining_args
                 server = ServerClass(*new_args, **kwargs) # pylint: disable=star-args
-            except socket.error:
-                logger.debug("Failed to bind to %s:%s using %s", new_address[0],
+                logger.debug(
+                    "Successfully bound to %s:%s using %s", new_address[0],
                     new_address[1], "IPv6" if ip_version else "IPv4")
+            except socket.error:
+                if self.servers:
+                    # Already bound using IPv6.
+                    logger.debug(
+                        "Certbot wasn't able to bind to %s:%s using %s, this " +
+                        "is often expected due to the dual stack nature of " +
+                        "IPv6 socket implementations.",
+                        new_address[0], new_address[1],
+                        "IPv6" if ip_version else "IPv4")
+                else:
+                    logger.debug(
+                        "Failed to bind to %s:%s using %s", new_address[0],
+                        new_address[1], "IPv6" if ip_version else "IPv4")
             else:
                 self.servers.append(server)
                 # If two servers are set up and port 0 was passed in, ensure we always
@@ -189,7 +203,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 
     def __init__(self, *args, **kwargs):
         self.simple_http_resources = kwargs.pop("simple_http_resources", set())
-        socketserver.BaseRequestHandler.__init__(self, *args, **kwargs)
+        BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
 
     def log_message(self, format, *args):  # pylint: disable=redefined-builtin
         """Log arbitrary message."""
@@ -262,7 +276,7 @@ def simple_tls_sni_01_server(cli_args, forever=True):
 
     certs = {}
 
-    _, hosts, _ = next(os.walk('.'))
+    _, hosts, _ = next(os.walk('.')) # type: ignore # https://github.com/python/mypy/issues/465
     for host in hosts:
         with open(os.path.join(host, "cert.pem")) as cert_file:
             cert_contents = cert_file.read()

acme/acme/standalone_test.py

@@ -4,10 +4,10 @@ import shutil
 import socket
 import threading
 import tempfile
-import time
 import unittest
 
 from six.moves import http_client  # pylint: disable=import-error
+from six.moves import queue  # pylint: disable=import-error
 from six.moves import socketserver  # type: ignore  # pylint: disable=import-error
 
 import josepy as jose
@@ -16,8 +16,8 @@ import requests
 
 from acme import challenges
 from acme import crypto_util
-from acme import errors
 from acme import test_util
+from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module
 
 
 class TLSServerTest(unittest.TestCase):
@@ -72,7 +72,7 @@ class HTTP01ServerTest(unittest.TestCase):
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))
-        self.resources = set()
+        self.resources = set() # type: Set
 
         from acme.standalone import HTTP01Server
         self.server = HTTP01Server(('', 0), resources=self.resources)
@@ -201,7 +201,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase):
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))
-        self.resources = set()
+        self.resources = set() # type: Set
 
         from acme.standalone import HTTP01DualNetworkedServers
         self.servers = HTTP01DualNetworkedServers(('', 0), resources=self.resources)
@@ -260,10 +260,9 @@ class TestSimpleTLSSNI01Server(unittest.TestCase):
                     os.path.join(localhost_dir, 'key.pem'))
 
         from acme.standalone import simple_tls_sni_01_server
-        self.port = 1234
         self.thread = threading.Thread(
             target=simple_tls_sni_01_server, kwargs={
-                'cli_args': ('xxx', '--port', str(self.port)),
+                'cli_args': ('filename',),
                 'forever': False,
             },
         )
@@ -275,25 +274,20 @@ class TestSimpleTLSSNI01Server(unittest.TestCase):
         self.thread.join()
         shutil.rmtree(self.test_cwd)
 
-    def test_it(self):
-        max_attempts = 5
-        for attempt in range(max_attempts):
-            try:
-                cert = crypto_util.probe_sni(
-                    b'localhost', b'0.0.0.0', self.port)
-            except errors.Error:
-                self.assertTrue(attempt + 1 < max_attempts, "Timeout!")
-                time.sleep(1)  # wait until thread starts
-            else:
-                self.assertEqual(jose.ComparableX509(cert),
-                                 test_util.load_comparable_cert(
-                                     'rsa2048_cert.pem'))
-                break
+    @mock.patch('acme.standalone.logger')
+    def test_it(self, mock_logger):
+        # Use a Queue because mock objects aren't thread safe.
+        q = queue.Queue()  # type: queue.Queue[int]
+        # Add port number to the queue.
+        mock_logger.info.side_effect = lambda *args: q.put(args[-1])
+        self.thread.start()
 
-            if attempt == 0:
-                # the first attempt is always meant to fail, so we can test
-                # the socket failure code-path for probe_sni, as well
-                self.thread.start()
+        # After the timeout, an exception is raised if the queue is empty.
+        port = q.get(timeout=5)
+        cert = crypto_util.probe_sni(b'localhost', b'0.0.0.0', port)
+        self.assertEqual(jose.ComparableX509(cert),
+                         test_util.load_comparable_cert(
+                             'rsa2048_cert.pem'))
 
 
 if __name__ == "__main__":

acme/acme/test_util.py

@@ -10,7 +10,7 @@ import unittest
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 import josepy as jose
-import OpenSSL
+from OpenSSL import crypto
 
 
 def vector_path(*names):
@@ -39,8 +39,8 @@ def _guess_loader(filename, loader_pem, loader_der):
 def load_cert(*names):
     """Load certificate."""
     loader = _guess_loader(
-        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
-    return OpenSSL.crypto.load_certificate(loader, load_vector(*names))
+        names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1)
+    return crypto.load_certificate(loader, load_vector(*names))
 
 
 def load_comparable_cert(*names):
@@ -51,8 +51,8 @@ def load_comparable_cert(*names):
 def load_csr(*names):
     """Load certificate request."""
     loader = _guess_loader(
-        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
-    return OpenSSL.crypto.load_certificate_request(loader, load_vector(*names))
+        names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1)
+    return crypto.load_certificate_request(loader, load_vector(*names))
 
 
 def load_comparable_csr(*names):
@@ -71,8 +71,8 @@ def load_rsa_private_key(*names):
 def load_pyopenssl_private_key(*names):
     """Load pyOpenSSL private key."""
     loader = _guess_loader(
-        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
-    return OpenSSL.crypto.load_privatekey(loader, load_vector(*names))
+        names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1)
+    return crypto.load_privatekey(loader, load_vector(*names))
 
 
 def skip_unless(condition, reason):  # pragma: no cover

acme/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+norecursedirs = .* build dist CVS _darcs {arch} *.egg

acme/setup.py

@@ -1,10 +1,9 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
-
-version = '0.22.0.dev0'
+version = '0.26.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
@@ -19,6 +18,7 @@ install_requires = [
     'pyrfc3339',
     'pytz',
     'requests[security]>=2.4.1',  # security extras added in 2.4.1
+    'requests-toolbelt>=0.3.0',
     'setuptools',
     'six>=1.9.0',  # needed for python_2_unicode_compatible
 ]
@@ -34,6 +34,19 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
 
 setup(
     name='acme',
@@ -66,5 +79,7 @@ setup(
         'dev': dev_extras,
         'docs': docs_extras,
     },
+    tests_require=["pytest"],
     test_suite='acme',
+    cmdclass={"test": PyTest},
 )

certbot-apache/certbot_apache/apache_util.py

@@ -1,4 +1,5 @@
 """ Utility functions for certbot-apache plugin """
+import binascii
 import os
 
 from certbot import util
@@ -98,3 +99,8 @@ def parse_define_file(filepath, varname):
             var_parts = v[2:].partition("=")
             return_vars[var_parts[0]] = var_parts[2]
     return return_vars
+
+
+def unique_id():
+    """ Returns an unique id to be used as a VirtualHost identifier"""
+    return binascii.hexlify(os.urandom(16)).decode("utf-8")

certbot-apache/certbot_apache/configurator.py

@@ -13,13 +13,16 @@ import zope.component
 import zope.interface
 
 from acme import challenges
+from acme.magic_typing import Any, DefaultDict, Dict, List, Set, Union  # pylint: disable=unused-import, no-name-in-module
 
 from certbot import errors
 from certbot import interfaces
 from certbot import util
 
+from certbot.achallenges import KeyAuthorizationAnnotatedChallenge  # pylint: disable=unused-import
 from certbot.plugins import common
 from certbot.plugins.util import path_surgery
+from certbot.plugins.enhancements import AutoHSTSEnhancement
 
 from certbot_apache import apache_util
 from certbot_apache import augeas_configurator
@@ -130,10 +133,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             default=cls.OS_DEFAULTS["challenge_location"],
             help="Directory path for challenge configuration.")
         add("handle-modules", default=cls.OS_DEFAULTS["handle_mods"],
-            help="Let installer handle enabling required modules for you." +
+            help="Let installer handle enabling required modules for you. " +
                  "(Only Ubuntu/Debian currently)")
         add("handle-sites", default=cls.OS_DEFAULTS["handle_sites"],
-            help="Let installer handle enabling sites for you." +
+            help="Let installer handle enabling sites for you. " +
                  "(Only Ubuntu/Debian currently)")
         util.add_deprecated_argument(add, argument_name="ctl", nargs=1)
         util.add_deprecated_argument(
@@ -150,14 +153,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         super(ApacheConfigurator, self).__init__(*args, **kwargs)
 
         # Add name_server association dict
-        self.assoc = dict()
+        self.assoc = dict()  # type: Dict[str, obj.VirtualHost]
         # Outstanding challenges
-        self._chall_out = set()
+        self._chall_out = set()  # type: Set[KeyAuthorizationAnnotatedChallenge]
         # List of vhosts configured per wildcard domain on this run.
         # used by deploy_cert() and enhance()
-        self._wildcard_vhosts = dict()
+        self._wildcard_vhosts = dict()  # type: Dict[str, List[obj.VirtualHost]]
         # Maps enhancements to vhosts we've enabled the enhancement for
-        self._enhanced_vhosts = defaultdict(set)
+        self._enhanced_vhosts = defaultdict(set)  # type: DefaultDict[str, Set[obj.VirtualHost]]
+        # Temporary state for AutoHSTS enhancement
+        self._autohsts = {}  # type: Dict[str, Dict[str, Union[int, float]]]
 
         # These will be set in the prepare function
         self.parser = None
@@ -285,8 +290,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                     chain_path=None, fullchain_path=None):
         """Deploys certificate to specified virtual host.
 
-        Currently tries to find the last directives to deploy the cert in
-        the VHost associated with the given domain. If it can't find the
+        Currently tries to find the last directives to deploy the certificate
+        in the VHost associated with the given domain. If it can't find the
         directives, it searches the "included" confs. The function verifies
         that it has located the three directives and finally modifies them
         to point to the correct destination. After the certificate is
@@ -323,7 +328,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             # Returned objects are guaranteed to be ssl vhosts
             return self._choose_vhosts_wildcard(domain, create_if_no_ssl)
         else:
-            return [self.choose_vhost(domain)]
+            return [self.choose_vhost(domain, create_if_no_ssl)]
 
     def _vhosts_for_wildcard(self, domain):
         """
@@ -424,14 +429,20 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             path["chain_path"] = self.parser.find_dir(
                 "SSLCertificateChainFile", None, vhost.path)
 
-        if not path["cert_path"] or not path["cert_key"]:
-            # Throw some can't find all of the directives error"
+        # Handle errors when certificate/key directives cannot be found
+        if not path["cert_path"]:
             logger.warning(
-                "Cannot find a cert or key directive in %s. "
+                "Cannot find an SSLCertificateFile directive in %s. "
                 "VirtualHost was not modified", vhost.path)
-            # Presumably break here so that the virtualhost is not modified
             raise errors.PluginError(
-                "Unable to find cert and/or key directives")
+                "Unable to find an SSLCertificateFile directive")
+        elif not path["cert_key"]:
+            logger.warning(
+                "Cannot find an SSLCertificateKeyFile directive for "
+                "certificate in %s. VirtualHost was not modified", vhost.path)
+            raise errors.PluginError(
+                "Unable to find an SSLCertificateKeyFile directive for "
+                "certificate")
 
         logger.info("Deploying Certificate to VirtualHost %s", vhost.filep)
 
@@ -469,20 +480,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         if chain_path is not None:
             self.save_notes += "\tSSLCertificateChainFile %s\n" % chain_path
 
-    def choose_vhost(self, target_name, temp=False):
+    def choose_vhost(self, target_name, create_if_no_ssl=True):
         """Chooses a virtual host based on the given domain name.
 
         If there is no clear virtual host to be selected, the user is prompted
         with all available choices.
 
-        The returned vhost is guaranteed to have TLS enabled unless temp is
-        True. If temp is True, there is no such guarantee and the result is
-        not cached.
+        The returned vhost is guaranteed to have TLS enabled unless
+        create_if_no_ssl is set to False, in which case there is no such guarantee
+        and the result is not cached.
 
         :param str target_name: domain name
-        :param bool temp: whether the vhost is only used temporarily
+        :param bool create_if_no_ssl: If found VirtualHost doesn't have a HTTPS
+            counterpart, should one get created
 
-        :returns: ssl vhost associated with name
+        :returns: vhost associated with name
         :rtype: :class:`~certbot_apache.obj.VirtualHost`
 
         :raises .errors.PluginError: If no vhost is available or chosen
@@ -495,7 +507,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         # Try to find a reasonable vhost
         vhost = self._find_best_vhost(target_name)
         if vhost is not None:
-            if temp:
+            if not create_if_no_ssl:
                 return vhost
             if not vhost.ssl:
                 vhost = self.make_vhost_ssl(vhost)
@@ -504,7 +516,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             self.assoc[target_name] = vhost
             return vhost
 
-        return self._choose_vhost_from_list(target_name, temp)
+        # Negate create_if_no_ssl value to indicate if we want a SSL vhost
+        # to get created if a non-ssl vhost is selected.
+        return self._choose_vhost_from_list(target_name, temp=not create_if_no_ssl)
 
     def _choose_vhost_from_list(self, target_name, temp=False):
         # Select a vhost from a list
@@ -650,7 +664,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         :rtype: set
 
         """
-        all_names = set()
+        all_names = set()  # type: Set[str]
 
         vhost_macro = []
 
@@ -791,8 +805,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         """
         # Search base config, and all included paths for VirtualHosts
-        file_paths = {}
-        internal_paths = defaultdict(set)
+        file_paths = {}  # type: Dict[str, str]
+        internal_paths = defaultdict(set)  # type: DefaultDict[str, Set[str]]
         vhs = []
         # Make a list of parser paths because the parser_paths
         # dictionary may be modified during the loop.
@@ -1230,7 +1244,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             if not self.parser.parsed_in_current(ssl_fp):
                 self.parser.parse_file(ssl_fp)
         except IOError:
-            logger.fatal("Error writing/reading to file in make_vhost_ssl")
+            logger.critical("Error writing/reading to file in make_vhost_ssl", exc_info=True)
             raise errors.PluginError("Unable to write/read in make_vhost_ssl")
 
         if sift:
@@ -1318,7 +1332,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         try:
             span_val = self.aug.span(vhost.path)
         except ValueError:
-            logger.fatal("Error while reading the VirtualHost %s from "
+            logger.critical("Error while reading the VirtualHost %s from "
                          "file %s", vhost.name, vhost.filep, exc_info=True)
             raise errors.PluginError("Unable to read VirtualHost from file")
         span_filep = span_val[0]
@@ -1461,6 +1475,67 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         if need_to_save:
             self.save()
 
+    def find_vhost_by_id(self, id_str):
+        """
+        Searches through VirtualHosts and tries to match the id in a comment
+
+        :param str id_str: Id string for matching
+
+        :returns: The matched VirtualHost or None
+        :rtype: :class:`~certbot_apache.obj.VirtualHost` or None
+
+        :raises .errors.PluginError: If no VirtualHost is found
+        """
+
+        for vh in self.vhosts:
+            if self._find_vhost_id(vh) == id_str:
+                return vh
+        msg = "No VirtualHost with ID {} was found.".format(id_str)
+        logger.warning(msg)
+        raise errors.PluginError(msg)
+
+    def _find_vhost_id(self, vhost):
+        """Tries to find the unique ID from the VirtualHost comments. This is
+        used for keeping track of VirtualHost directive over time.
+
+        :param vhost: Virtual host to add the id
+        :type vhost: :class:`~certbot_apache.obj.VirtualHost`
+
+        :returns: The unique ID or None
+        :rtype: str or None
+        """
+
+        # Strip the {} off from the format string
+        search_comment = constants.MANAGED_COMMENT_ID.format("")
+
+        id_comment = self.parser.find_comments(search_comment, vhost.path)
+        if id_comment:
+            # Use the first value, multiple ones shouldn't exist
+            comment = self.parser.get_arg(id_comment[0])
+            return comment.split(" ")[-1]
+        return None
+
+    def add_vhost_id(self, vhost):
+        """Adds an unique ID to the VirtualHost as a comment for mapping back
+        to it on later invocations, as the config file order might have changed.
+        If ID already exists, returns that instead.
+
+        :param vhost: Virtual host to add or find the id
+        :type vhost: :class:`~certbot_apache.obj.VirtualHost`
+
+        :returns: The unique ID for vhost
+        :rtype: str or None
+        """
+
+        vh_id = self._find_vhost_id(vhost)
+        if vh_id:
+            return vh_id
+
+        id_string = apache_util.unique_id()
+        comment = constants.MANAGED_COMMENT_ID.format(id_string)
+        self.parser.add_comment(vhost.path, comment)
+        return id_string
+
     def _escape(self, fp):
         fp = fp.replace(",", "\\,")
         fp = fp.replace("[", "\\[")
@@ -1499,7 +1574,20 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             raise errors.PluginError(
                 "Unsupported enhancement: {0}".format(enhancement))
 
-        vhosts = self.choose_vhosts(domain, create_if_no_ssl=False)
+        matched_vhosts = self.choose_vhosts(domain, create_if_no_ssl=False)
+        # We should be handling only SSL vhosts for enhancements
+        vhosts = [vhost for vhost in matched_vhosts if vhost.ssl]
+
+        if not vhosts:
+            msg_tmpl = ("Certbot was not able to find SSL VirtualHost for a "
+                        "domain {0} for enabling enhancement \"{1}\". The requested "
+                        "enhancement was not configured.")
+            msg_enhancement = enhancement
+            if options:
+                msg_enhancement += ": " + options
+            msg = msg_tmpl.format(domain, msg_enhancement)
+            logger.warning(msg)
+            raise errors.PluginError(msg)
         try:
             for vhost in vhosts:
                 func(vhost, options)
@@ -1507,6 +1595,78 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             logger.warning("Failed %s for %s", enhancement, domain)
             raise
 
+    def _autohsts_increase(self, vhost, id_str, nextstep):
+        """Increase the AutoHSTS max-age value
+
+        :param vhost: Virtual host object to modify
+        :type vhost: :class:`~certbot_apache.obj.VirtualHost`
+
+        :param str id_str: The unique ID string of VirtualHost
+
+        :param int nextstep: Next AutoHSTS max-age value index
+
+        """
+        nextstep_value = constants.AUTOHSTS_STEPS[nextstep]
+        self._autohsts_write(vhost, nextstep_value)
+        self._autohsts[id_str] = {"laststep": nextstep, "timestamp": time.time()}
+
+    def _autohsts_write(self, vhost, nextstep_value):
+        """
+        Write the new HSTS max-age value to the VirtualHost file
+        """
+
+        hsts_dirpath = None
+        header_path = self.parser.find_dir("Header", None, vhost.path)
+        if header_path:
+            pat = '(?:[ "]|^)(strict-transport-security)(?:[ "]|$)'
+            for match in header_path:
+                if re.search(pat, self.aug.get(match).lower()):
+                    hsts_dirpath = match
+        if not hsts_dirpath:
+            err_msg = ("Certbot was unable to find the existing HSTS header "
+                       "from the VirtualHost at path {0}.").format(vhost.filep)
+            raise errors.PluginError(err_msg)
+
+        # Prepare the HSTS header value
+        hsts_maxage = "\"max-age={0}\"".format(nextstep_value)
+
+        # Update the header
+        # Our match statement was for string strict-transport-security, but
+        # we need to update the value instead. The next index is for the value
+        hsts_dirpath = hsts_dirpath.replace("arg[3]", "arg[4]")
+        self.aug.set(hsts_dirpath, hsts_maxage)
+        note_msg = ("Increasing HSTS max-age value to {0} for VirtualHost "
+                    "in {1}\n".format(nextstep_value, vhost.filep))
+        logger.debug(note_msg)
+        self.save_notes += note_msg
+        self.save(note_msg)
+
+    def _autohsts_fetch_state(self):
+        """
+        Populates the AutoHSTS state from the pluginstorage
+        """
+        try:
+            self._autohsts = self.storage.fetch("autohsts")
+        except KeyError:
+            self._autohsts = dict()
+
+    def _autohsts_save_state(self):
+        """
+        Saves the state of AutoHSTS object to pluginstorage
+        """
+        self.storage.put("autohsts", self._autohsts)
+        self.storage.save()
+
+    def _autohsts_vhost_in_lineage(self, vhost, lineage):
+        """
+        Searches AutoHSTS managed VirtualHosts that belong to the lineage.
+        Matches the private key path.
+        """
+
+        return bool(
+            self.parser.find_dir("SSLCertificateKeyFile",
+                                 lineage.key_path, vhost.path))
+
     def _enable_ocsp_stapling(self, ssl_vhost, unused_options):
         """Enables OCSP Stapling
 
@@ -1748,7 +1908,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         # There can be other RewriteRule directive lines in vhost config.
         # rewrite_args_dict keys are directive ids and the corresponding value
         # for each is a list of arguments to that directive.
-        rewrite_args_dict = defaultdict(list)
+        rewrite_args_dict = defaultdict(list)  # type: DefaultDict[str, List[str]]
         pat = r'(.*directive\[\d+\]).*'
         for match in rewrite_path:
             m = re.match(pat, match)
@@ -1842,7 +2002,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         if ssl_vhost.aliases:
             serveralias = "ServerAlias " + " ".join(ssl_vhost.aliases)
 
-        rewrite_rule_args = []
+        rewrite_rule_args = []  # type: List[str]
         if self.get_version() >= (2, 3, 9):
             rewrite_rule_args = constants.REWRITE_HTTPS_ARGS_WITH_END
         else:
@@ -1994,10 +2154,27 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         :raises .errors.MisconfigurationError: If reload fails
 
         """
+        error = ""
         try:
             util.run_script(self.constant("restart_cmd"))
         except errors.SubprocessError as err:
-            raise errors.MisconfigurationError(str(err))
+            logger.info("Unable to restart apache using %s",
+                        self.constant("restart_cmd"))
+            alt_restart = self.constant("restart_cmd_alt")
+            if alt_restart:
+                logger.debug("Trying alternative restart command: %s",
+                             alt_restart)
+                # There is an alternative restart command available
+                # This usually is "restart" verb while original is "graceful"
+                try:
+                    util.run_script(self.constant(
+                        "restart_cmd_alt"))
+                    return
+                except errors.SubprocessError as secerr:
+                    error = str(secerr)
+            else:
+                error = str(err)
+            raise errors.MisconfigurationError(error)
 
     def config_test(self):  # pylint: disable=no-self-use
         """Check the configuration of Apache for errors.
@@ -2118,4 +2295,176 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         return common.install_version_controlled_file(options_ssl, options_ssl_digest,
             self.constant("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES)
 
+    def enable_autohsts(self, _unused_lineage, domains):
+        """
+        Enable the AutoHSTS enhancement for defined domains
 
+        :param _unused_lineage: Certificate lineage object, unused
+        :type _unused_lineage: certbot.storage.RenewableCert
+
+        :param domains: List of domains in certificate to enhance
+        :type domains: str
+        """
+
+        self._autohsts_fetch_state()
+        _enhanced_vhosts = []
+        for d in domains:
+            matched_vhosts = self.choose_vhosts(d, create_if_no_ssl=False)
+            # We should be handling only SSL vhosts for AutoHSTS
+            vhosts = [vhost for vhost in matched_vhosts if vhost.ssl]
+
+            if not vhosts:
+                msg_tmpl = ("Certbot was not able to find SSL VirtualHost for a "
+                            "domain {0} for enabling AutoHSTS enhancement.")
+                msg = msg_tmpl.format(d)
+                logger.warning(msg)
+                raise errors.PluginError(msg)
+            for vh in vhosts:
+                try:
+                    self._enable_autohsts_domain(vh)
+                    _enhanced_vhosts.append(vh)
+                except errors.PluginEnhancementAlreadyPresent:
+                    if vh in _enhanced_vhosts:
+                        continue
+                    msg = ("VirtualHost for domain {0} in file {1} has a " +
+                           "String-Transport-Security header present, exiting.")
+                    raise errors.PluginEnhancementAlreadyPresent(
+                        msg.format(d, vh.filep))
+        if _enhanced_vhosts:
+            note_msg = "Enabling AutoHSTS"
+            self.save(note_msg)
+            logger.info(note_msg)
+            self.restart()
+
+        # Save the current state to pluginstorage
+        self._autohsts_save_state()
+
+    def _enable_autohsts_domain(self, ssl_vhost):
+        """Do the initial AutoHSTS deployment to a vhost
+
+        :param ssl_vhost: The VirtualHost object to deploy the AutoHSTS
+        :type ssl_vhost: :class:`~certbot_apache.obj.VirtualHost` or None
+
+        :raises errors.PluginEnhancementAlreadyPresent: When already enhanced
+
+        """
+        # This raises the exception
+        self._verify_no_matching_http_header(ssl_vhost,
+                                             "Strict-Transport-Security")
+
+        if "headers_module" not in self.parser.modules:
+            self.enable_mod("headers")
+        # Prepare the HSTS header value
+        hsts_header = constants.HEADER_ARGS["Strict-Transport-Security"][:-1]
+        initial_maxage = constants.AUTOHSTS_STEPS[0]
+        hsts_header.append("\"max-age={0}\"".format(initial_maxage))
+
+        # Add ID to the VirtualHost for mapping back to it later
+        uniq_id = self.add_vhost_id(ssl_vhost)
+        self.save_notes += "Adding unique ID {0} to VirtualHost in {1}\n".format(
+            uniq_id, ssl_vhost.filep)
+        # Add the actual HSTS header
+        self.parser.add_dir(ssl_vhost.path, "Header", hsts_header)
+        note_msg = ("Adding gradually increasing HSTS header with initial value "
+                    "of {0} to VirtualHost in {1}\n".format(
+                        initial_maxage, ssl_vhost.filep))
+        self.save_notes += note_msg
+
+        # Save the current state to pluginstorage
+        self._autohsts[uniq_id] = {"laststep": 0, "timestamp": time.time()}
+
+    def update_autohsts(self, _unused_domain):
+        """
+        Increase the AutoHSTS values of VirtualHosts that the user has enabled
+        this enhancement for.
+
+        :param _unused_domain: Not currently used
+        :type _unused_domain: Not Available
+
+        """
+        self._autohsts_fetch_state()
+        if not self._autohsts:
+            # No AutoHSTS enabled for any domain
+            return
+        curtime = time.time()
+        save_and_restart = False
+        for id_str, config in list(self._autohsts.items()):
+            if config["timestamp"] + constants.AUTOHSTS_FREQ > curtime:
+                # Skip if last increase was < AUTOHSTS_FREQ ago
+                continue
+            nextstep = config["laststep"] + 1
+            if nextstep < len(constants.AUTOHSTS_STEPS):
+                # Have not reached the max value yet
+                try:
+                    vhost = self.find_vhost_by_id(id_str)
+                except errors.PluginError:
+                    msg = ("Could not find VirtualHost with ID {0}, disabling "
+                           "AutoHSTS for this VirtualHost").format(id_str)
+                    logger.warning(msg)
+                    # Remove the orphaned AutoHSTS entry from pluginstorage
+                    self._autohsts.pop(id_str)
+                    continue
+                self._autohsts_increase(vhost, id_str, nextstep)
+                msg = ("Increasing HSTS max-age value for VirtualHost with id "
+                       "{0}").format(id_str)
+                self.save_notes += msg
+                save_and_restart = True
+
+        if save_and_restart:
+            self.save("Increased HSTS max-age values")
+            self.restart()
+
+        self._autohsts_save_state()
+
+    def deploy_autohsts(self, lineage):
+        """
+        Checks if autohsts vhost has reached maximum auto-increased value
+        and changes the HSTS max-age to a high value.
+
+        :param lineage: Certificate lineage object
+        :type lineage: certbot.storage.RenewableCert
+        """
+        self._autohsts_fetch_state()
+        if not self._autohsts:
+            # No autohsts enabled for any vhost
+            return
+
+        vhosts = []
+        affected_ids = []
+        # Copy, as we are removing from the dict inside the loop
+        for id_str, config in list(self._autohsts.items()):
+            if config["laststep"]+1 >= len(constants.AUTOHSTS_STEPS):
+                # max value reached, try to make permanent
+                try:
+                    vhost = self.find_vhost_by_id(id_str)
+                except errors.PluginError:
+                    msg = ("VirtualHost with id {} was not found, unable to "
+                           "make HSTS max-age permanent.").format(id_str)
+                    logger.warning(msg)
+                    self._autohsts.pop(id_str)
+                    continue
+                if self._autohsts_vhost_in_lineage(vhost, lineage):
+                    vhosts.append(vhost)
+                    affected_ids.append(id_str)
+
+        save_and_restart = False
+        for vhost in vhosts:
+            self._autohsts_write(vhost, constants.AUTOHSTS_PERMANENT)
+            msg = ("Strict-Transport-Security max-age value for "
+                   "VirtualHost in {0} was made permanent.").format(vhost.filep)
+            logger.debug(msg)
+            self.save_notes += msg+"\n"
+            save_and_restart = True
+
+        if save_and_restart:
+            self.save("Made HSTS max-age permanent")
+            self.restart()
+
+        for id_str in affected_ids:
+            self._autohsts.pop(id_str)
+
+        # Update AutoHSTS storage (We potentially removed vhosts from managed)
+        self._autohsts_save_state()
+
+
+AutoHSTSEnhancement.register(ApacheConfigurator)  # pylint: disable=no-member

certbot-apache/certbot_apache/constants.py

@@ -48,3 +48,16 @@ UIR_ARGS = ["always", "set", "Content-Security-Policy",
 
 HEADER_ARGS = {"Strict-Transport-Security": HSTS_ARGS,
                "Upgrade-Insecure-Requests": UIR_ARGS}
+
+AUTOHSTS_STEPS = [60, 300, 900, 3600, 21600, 43200, 86400]
+"""AutoHSTS increase steps: 1min, 5min, 15min, 1h, 6h, 12h, 24h"""
+
+AUTOHSTS_PERMANENT = 31536000
+"""Value for the last max-age of HSTS"""
+
+AUTOHSTS_FREQ = 172800
+"""Minimum time since last increase to perform a new one: 48h"""
+
+MANAGED_COMMENT = "DO NOT REMOVE - Managed by Certbot"
+MANAGED_COMMENT_ID = MANAGED_COMMENT+", VirtualHost id: {0}"
+"""Managed by Certbot comments and the VirtualHost identification template"""

certbot-apache/certbot_apache/http_01.py

@@ -2,9 +2,10 @@
 import logging
 import os
 
+from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
 from certbot import errors
-
 from certbot.plugins import common
+from certbot_apache.obj import VirtualHost  # pylint: disable=unused-import
 
 logger = logging.getLogger(__name__)
 
@@ -51,7 +52,7 @@ class ApacheHttp01(common.TLSSNI01):
         self.challenge_dir = os.path.join(
             self.configurator.config.work_dir,
             "http_challenges")
-        self.moded_vhosts = set()
+        self.moded_vhosts = set()  # type: Set[VirtualHost]
 
     def perform(self):
         """Perform all HTTP-01 challenges."""

certbot-apache/certbot_apache/obj.py

@@ -1,6 +1,7 @@
 """Module contains classes used by the Apache Configurator."""
 import re
 
+from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module
 from certbot.plugins import common
 
 
@@ -140,7 +141,7 @@ class VirtualHost(object):  # pylint: disable=too-few-public-methods
 
     def get_names(self):
         """Return a set of all names."""
-        all_names = set()
+        all_names = set()  # type: Set[str]
         all_names.update(self.aliases)
         # Strip out any scheme:// and <port> field from servername
         if self.name is not None:
@@ -251,7 +252,7 @@ class VirtualHost(object):  # pylint: disable=too-few-public-methods
 
         # already_found acts to keep everything very conservative.
         # Don't allow multiple ip:ports in same set.
-        already_found = set()
+        already_found = set()  # type: Set[str]
 
         for addr in vhost.addrs:
             for local_addr in self.addrs:

certbot-apache/certbot_apache/override_centos.py

@@ -21,6 +21,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
         version_cmd=['apachectl', '-v'],
         apache_cmd="apachectl",
         restart_cmd=['apachectl', 'graceful'],
+        restart_cmd_alt=['apachectl', 'restart'],
         conftest_cmd=['apachectl', 'configtest'],
         enmod=None,
         dismod=None,
@@ -46,10 +47,10 @@ class CentOSParser(parser.ApacheParser):
         self.sysconfig_filep = "/etc/sysconfig/httpd"
         super(CentOSParser, self).__init__(*args, **kwargs)
 
-    def update_runtime_variables(self, *args, **kwargs):
+    def update_runtime_variables(self):
         """ Override for update_runtime_variables for custom parsing """
         # Opportunistic, works if SELinux not enforced
-        super(CentOSParser, self).update_runtime_variables(*args, **kwargs)
+        super(CentOSParser, self).update_runtime_variables()
         self.parse_sysconfig_var()
 
     def parse_sysconfig_var(self):

certbot-apache/certbot_apache/override_gentoo.py

@@ -21,6 +21,7 @@ class GentooConfigurator(configurator.ApacheConfigurator):
         version_cmd=['/usr/sbin/apache2', '-v'],
         apache_cmd="apache2ctl",
         restart_cmd=['apache2ctl', 'graceful'],
+        restart_cmd_alt=['apache2ctl', 'restart'],
         conftest_cmd=['apache2ctl', 'configtest'],
         enmod=None,
         dismod=None,

certbot-apache/certbot_apache/parser.py

@@ -9,12 +9,14 @@ import sys
 
 import six
 
+from acme.magic_typing import Dict, List, Set  # pylint: disable=unused-import, no-name-in-module
 from certbot import errors
 
 logger = logging.getLogger(__name__)
 
 
 class ApacheParser(object):
+    # pylint: disable=too-many-public-methods
     """Class handles the fine details of parsing the Apache Configuration.
 
     .. todo:: Make parsing general... remove sites-available etc...
@@ -38,9 +40,9 @@ class ApacheParser(object):
         # issues with aug.load() after adding new files / defines to parse tree
         self.configurator = configurator
 
-        self.modules = set()
-        self.parser_paths = {}
-        self.variables = {}
+        self.modules = set()  # type: Set[str]
+        self.parser_paths = {}  # type: Dict[str, List[str]]
+        self.variables = {}  # type: Dict[str, str]
 
         self.aug = aug
         # Find configuration root and make sure augeas can parse it.
@@ -119,7 +121,7 @@ class ApacheParser(object):
             the iteration issue.  Else... parse and enable mods at same time.
 
         """
-        mods = set()
+        mods = set()  # type: Set[str]
         matches = self.find_dir("LoadModule")
         iterator = iter(matches)
         # Make sure prev_size != cur_size for do: while: iteration
@@ -349,6 +351,37 @@ class ApacheParser(object):
         else:
             self.aug.set(first_dir + "/arg", args)
 
+    def add_comment(self, aug_conf_path, comment):
+        """Adds the comment to the augeas path
+
+        :param str aug_conf_path: Augeas configuration path to add directive
+        :param str comment: Comment content
+
+        """
+        self.aug.set(aug_conf_path + "/#comment[last() + 1]", comment)
+
+    def find_comments(self, arg, start=None):
+        """Finds a comment with specified content from the provided DOM path
+
+        :param str arg: Comment content to search
+        :param str start: Beginning Augeas path to begin looking
+
+        :returns: List of augeas paths containing the comment content
+        :rtype: list
+
+        """
+        if not start:
+            start = get_aug_path(self.root)
+
+        comments = self.aug.match("%s//*[label() = '#comment']" % start)
+
+        results = []
+        for comment in comments:
+            c_content = self.aug.get(comment)
+            if c_content and arg in c_content:
+                results.append(comment)
+        return results
+
     def find_dir(self, directive, arg=None, start=None, exclude=True):
         """Finds directive in the configuration.
 
@@ -408,7 +441,7 @@ class ApacheParser(object):
         else:
             arg_suffix = "/*[self::arg=~regexp('%s')]" % case_i(arg)
 
-        ordered_matches = []
+        ordered_matches = []  # type: List[str]
 
         # TODO: Wildcards should be included in alphabetical order
         # https://httpd.apache.org/docs/2.4/mod/core.html#include

certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test

@@ -46,6 +46,7 @@ function Cleanup() {
 
 # if our environment asks us to enable modules, do our best!
 if [ "$1" = --debian-modules ] ; then
+    sudo apt-get install -y apache2
     sudo apt-get install -y libapache2-mod-wsgi
     sudo apt-get install -y libapache2-mod-macro
 

certbot-apache/certbot_apache/tests/autohsts_test.py

@@ -0,0 +1,181 @@
+# pylint: disable=too-many-public-methods,too-many-lines
+"""Test for certbot_apache.configurator AutoHSTS functionality"""
+import re
+import unittest
+import mock
+# six is used in mock.patch()
+import six  # pylint: disable=unused-import
+
+from certbot import errors
+from certbot_apache import constants
+from certbot_apache.tests import util
+
+
+class AutoHSTSTest(util.ApacheTest):
+    """Tests for AutoHSTS feature"""
+    # pylint: disable=protected-access
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        super(AutoHSTSTest, self).setUp()
+
+        self.config = util.get_apache_configurator(
+            self.config_path, self.vhost_path, self.config_dir, self.work_dir)
+        self.config.parser.modules.add("headers_module")
+        self.config.parser.modules.add("mod_headers.c")
+        self.config.parser.modules.add("ssl_module")
+        self.config.parser.modules.add("mod_ssl.c")
+
+        self.vh_truth = util.get_vh_truth(
+            self.temp_dir, "debian_apache_2_4/multiple_vhosts")
+
+    def get_autohsts_value(self, vh_path):
+        """ Get value from Strict-Transport-Security header """
+        header_path = self.config.parser.find_dir("Header", None, vh_path)
+        if header_path:
+            pat = '(?:[ "]|^)(strict-transport-security)(?:[ "]|$)'
+            for head in header_path:
+                if re.search(pat, self.config.parser.aug.get(head).lower()):
+                    return self.config.parser.aug.get(head.replace("arg[3]",
+                                                                   "arg[4]"))
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod")
+    def test_autohsts_enable_headers_mod(self, mock_enable, _restart):
+        self.config.parser.modules.discard("headers_module")
+        self.config.parser.modules.discard("mod_header.c")
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        self.assertTrue(mock_enable.called)
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    def test_autohsts_deploy_already_exists(self, _restart):
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        self.assertRaises(errors.PluginEnhancementAlreadyPresent,
+                          self.config.enable_autohsts,
+                          mock.MagicMock(), ["ocspvhost.com"])
+
+    @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0)
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    def test_autohsts_increase(self, _mock_restart):
+        maxage = "\"max-age={0}\""
+        initial_val = maxage.format(constants.AUTOHSTS_STEPS[0])
+        inc_val = maxage.format(constants.AUTOHSTS_STEPS[1])
+
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        # Verify initial value
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          initial_val)
+        # Increase
+        self.config.update_autohsts(mock.MagicMock())
+        # Verify increased value
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          inc_val)
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator._autohsts_increase")
+    def test_autohsts_increase_noop(self, mock_increase, _restart):
+        maxage = "\"max-age={0}\""
+        initial_val = maxage.format(constants.AUTOHSTS_STEPS[0])
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        # Verify initial value
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          initial_val)
+
+        self.config.update_autohsts(mock.MagicMock())
+        # Freq not patched, so value shouldn't increase
+        self.assertFalse(mock_increase.called)
+
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0)
+    def test_autohsts_increase_no_header(self, _restart):
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        # Remove the header
+        dir_locs = self.config.parser.find_dir("Header", None,
+                                              self.vh_truth[7].path)
+        dir_loc = "/".join(dir_locs[0].split("/")[:-1])
+        self.config.parser.aug.remove(dir_loc)
+        self.assertRaises(errors.PluginError,
+                          self.config.update_autohsts,
+                          mock.MagicMock())
+
+    @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0)
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    def test_autohsts_increase_and_make_permanent(self, _mock_restart):
+        maxage = "\"max-age={0}\""
+        max_val = maxage.format(constants.AUTOHSTS_PERMANENT)
+        mock_lineage = mock.MagicMock()
+        mock_lineage.key_path = "/etc/apache2/ssl/key-certbot_15.pem"
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        for i in range(len(constants.AUTOHSTS_STEPS)-1):
+            # Ensure that value is not made permanent prematurely
+            self.config.deploy_autohsts(mock_lineage)
+            self.assertNotEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                                 max_val)
+            self.config.update_autohsts(mock.MagicMock())
+            # Value should match pre-permanent increment step
+            cur_val = maxage.format(constants.AUTOHSTS_STEPS[i+1])
+            self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                              cur_val)
+        # Make permanent
+        self.config.deploy_autohsts(mock_lineage)
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          max_val)
+
+    def test_autohsts_update_noop(self):
+        with mock.patch("time.time") as mock_time:
+            # Time mock is used to make sure that the execution does not
+            # continue when no autohsts entries exist in pluginstorage
+            self.config.update_autohsts(mock.MagicMock())
+            self.assertFalse(mock_time.called)
+
+    def test_autohsts_make_permanent_noop(self):
+        self.config.storage.put = mock.MagicMock()
+        self.config.deploy_autohsts(mock.MagicMock())
+        # Make sure that the execution does not continue when no entries in store
+        self.assertFalse(self.config.storage.put.called)
+
+    @mock.patch("certbot_apache.display_ops.select_vhost")
+    def test_autohsts_no_ssl_vhost(self, mock_select):
+        mock_select.return_value = self.vh_truth[0]
+        with mock.patch("certbot_apache.configurator.logger.warning") as mock_log:
+            self.assertRaises(errors.PluginError,
+                              self.config.enable_autohsts,
+                              mock.MagicMock(), "invalid.example.com")
+            self.assertTrue(
+                "Certbot was not able to find SSL" in mock_log.call_args[0][0])
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.add_vhost_id")
+    def test_autohsts_dont_enhance_twice(self, mock_id, _restart):
+        mock_id.return_value = "1234567"
+        self.config.enable_autohsts(mock.MagicMock(),
+                                    ["ocspvhost.com", "ocspvhost.com"])
+        self.assertEquals(mock_id.call_count, 1)
+
+    def test_autohsts_remove_orphaned(self):
+        # pylint: disable=protected-access
+        self.config._autohsts_fetch_state()
+        self.config._autohsts["orphan_id"] = {"laststep": 0, "timestamp": 0}
+
+        self.config._autohsts_save_state()
+        self.config.update_autohsts(mock.MagicMock())
+        self.assertFalse("orphan_id" in self.config._autohsts)
+        # Make sure it's removed from the pluginstorage file as well
+        self.config._autohsts = None
+        self.config._autohsts_fetch_state()
+        self.assertFalse(self.config._autohsts)
+
+    def test_autohsts_make_permanent_vhost_not_found(self):
+        # pylint: disable=protected-access
+        self.config._autohsts_fetch_state()
+        self.config._autohsts["orphan_id"] = {"laststep": 999, "timestamp": 0}
+        self.config._autohsts_save_state()
+        with mock.patch("certbot_apache.configurator.logger.warning") as mock_log:
+            self.config.deploy_autohsts(mock.MagicMock())
+            self.assertTrue(mock_log.called)
+            self.assertTrue(
+                "VirtualHost with id orphan_id was not" in mock_log.call_args[0][0])
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/centos_test.py

@@ -4,6 +4,8 @@ import unittest
 
 import mock
 
+from certbot import errors
+
 from certbot_apache import obj
 from certbot_apache import override_centos
 from certbot_apache.tests import util
@@ -121,5 +123,17 @@ class MultipleVhostsTestCentOS(util.ApacheTest):
         self.assertTrue("MOCK_NOSEP" in self.config.parser.variables.keys())
         self.assertEqual("NOSEP_VAL", self.config.parser.variables["NOSEP_TWO"])
 
+    @mock.patch("certbot_apache.configurator.util.run_script")
+    def test_alt_restart_works(self, mock_run_script):
+        mock_run_script.side_effect = [None, errors.SubprocessError, None]
+        self.config.restart()
+        self.assertEquals(mock_run_script.call_count, 3)
+
+    @mock.patch("certbot_apache.configurator.util.run_script")
+    def test_alt_restart_errors(self, mock_run_script):
+        mock_run_script.side_effect = [None,
+                                       errors.SubprocessError,
+                                       errors.SubprocessError]
+        self.assertRaises(errors.MisconfigurationError, self.config.restart)
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/configurator_test.py

@@ -246,7 +246,7 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot_apache.display_ops.select_vhost")
     def test_choose_vhost_select_vhost_with_temp(self, mock_select):
         mock_select.return_value = self.vh_truth[0]
-        chosen_vhost = self.config.choose_vhost("none.com", temp=True)
+        chosen_vhost = self.config.choose_vhost("none.com", create_if_no_ssl=False)
         self.assertEqual(self.vh_truth[0], chosen_vhost)
 
     @mock.patch("certbot_apache.display_ops.select_vhost")
@@ -353,14 +353,11 @@ class MultipleVhostsTest(util.ApacheTest):
 
         self.config.parser.find_dir = mock_find_dir
         mock_add.reset_mock()
-
         self.config._add_dummy_ssl_directives(self.vh_truth[0])  # pylint: disable=protected-access
-        tried_to_add = []
         for a in mock_add.call_args_list:
-            tried_to_add.append(a[0][1] == "Include" and
-                                a[0][2] == self.config.mod_ssl_conf)
-        # Include shouldn't be added, as patched find_dir "finds" existing one
-        self.assertFalse(any(tried_to_add))
+            if a[0][1] == "Include" and a[0][2] == self.config.mod_ssl_conf:
+                self.fail("Include shouldn't be added, as patched find_dir 'finds' existing one") \
+                    # pragma: no cover
 
     def test_deploy_cert(self):
         self.config.parser.modules.add("ssl_module")
@@ -441,13 +438,37 @@ class MultipleVhostsTest(util.ApacheTest):
             self.vh_truth[1].path))
 
     def test_deploy_cert_invalid_vhost(self):
+        """For test cases where the `ApacheConfigurator` class' `_deploy_cert`
+        method is called with an invalid vhost parameter. Currently this tests
+        that a PluginError is appropriately raised when important directives
+        are missing in an SSL module."""
         self.config.parser.modules.add("ssl_module")
-        mock_find = mock.MagicMock()
-        mock_find.return_value = []
-        self.config.parser.find_dir = mock_find
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("socache_shmcb_module")
+
+        def side_effect(*args):
+            """Mocks case where an SSLCertificateFile directive can be found
+            but an SSLCertificateKeyFile directive is missing."""
+            if "SSLCertificateFile" in args:
+                return ["example/cert.pem"]
+            else:
+                return []
+
+        mock_find_dir = mock.MagicMock(return_value=[])
+        mock_find_dir.side_effect = side_effect
+
+        self.config.parser.find_dir = mock_find_dir
 
         # Get the default 443 vhost
         self.config.assoc["random.demo"] = self.vh_truth[1]
+
+        self.assertRaises(
+            errors.PluginError, self.config.deploy_cert, "random.demo",
+            "example/cert.pem", "example/key.pem", "example/cert_chain.pem")
+
+        # Remove side_effect to mock case where both SSLCertificateFile
+        # and SSLCertificateKeyFile directives are missing
+        self.config.parser.find_dir.side_effect = None
         self.assertRaises(
             errors.PluginError, self.config.deploy_cert, "random.demo",
             "example/cert.pem", "example/key.pem", "example/cert_chain.pem")
@@ -912,6 +933,22 @@ class MultipleVhostsTest(util.ApacheTest):
             errors.PluginError,
             self.config.enhance, "certbot.demo", "unknown_enhancement")
 
+    def test_enhance_no_ssl_vhost(self):
+        with mock.patch("certbot_apache.configurator.logger.warning") as mock_log:
+            self.assertRaises(errors.PluginError, self.config.enhance,
+                              "certbot.demo", "redirect")
+            # Check that correct logger.warning was printed
+            self.assertTrue("not able to find" in mock_log.call_args[0][0])
+            self.assertTrue("\"redirect\"" in mock_log.call_args[0][0])
+
+            mock_log.reset_mock()
+
+            self.assertRaises(errors.PluginError, self.config.enhance,
+                              "certbot.demo", "ensure-http-header", "Test")
+            # Check that correct logger.warning was printed
+            self.assertTrue("not able to find" in mock_log.call_args[0][0])
+            self.assertTrue("Test" in mock_log.call_args[0][0])
+
     @mock.patch("certbot.util.exe_exists")
     def test_ocsp_stapling(self, mock_exe):
         self.config.parser.update_runtime_variables = mock.Mock()
@@ -921,6 +958,7 @@ class MultipleVhostsTest(util.ApacheTest):
         mock_exe.return_value = True
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "staple-ocsp")
 
         # Get the ssl vhost for certbot.demo
@@ -947,6 +985,7 @@ class MultipleVhostsTest(util.ApacheTest):
         mock_exe.return_value = True
 
         # Checking the case with already enabled ocsp stapling configuration
+        self.config.choose_vhost("ocspvhost.com")
         self.config.enhance("ocspvhost.com", "staple-ocsp")
 
         # Get the ssl vhost for letsencrypt.demo
@@ -971,6 +1010,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.parser.modules.add("mod_ssl.c")
         self.config.parser.modules.add("socache_shmcb_module")
         self.config.get_version = mock.Mock(return_value=(2, 2, 0))
+        self.config.choose_vhost("certbot.demo")
 
         self.assertRaises(errors.PluginError,
                 self.config.enhance, "certbot.demo", "staple-ocsp")
@@ -996,6 +1036,7 @@ class MultipleVhostsTest(util.ApacheTest):
         mock_exe.return_value = True
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "ensure-http-header",
                             "Strict-Transport-Security")
 
@@ -1015,7 +1056,8 @@ class MultipleVhostsTest(util.ApacheTest):
         # skip the enable mod
         self.config.parser.modules.add("headers_module")
 
-        # This will create an ssl vhost for certbot.demo
+        # This will create an ssl vhost for encryption-example.demo
+        self.config.choose_vhost("encryption-example.demo")
         self.config.enhance("encryption-example.demo", "ensure-http-header",
                             "Strict-Transport-Security")
 
@@ -1034,6 +1076,7 @@ class MultipleVhostsTest(util.ApacheTest):
         mock_exe.return_value = True
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "ensure-http-header",
                             "Upgrade-Insecure-Requests")
 
@@ -1055,7 +1098,8 @@ class MultipleVhostsTest(util.ApacheTest):
         # skip the enable mod
         self.config.parser.modules.add("headers_module")
 
-        # This will create an ssl vhost for certbot.demo
+        # This will create an ssl vhost for encryption-example.demo
+        self.config.choose_vhost("encryption-example.demo")
         self.config.enhance("encryption-example.demo", "ensure-http-header",
                             "Upgrade-Insecure-Requests")
 
@@ -1073,6 +1117,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.get_version = mock.Mock(return_value=(2, 2))
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "redirect")
 
         # These are not immediately available in find_dir even with save() and
@@ -1123,6 +1168,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.save()
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "redirect")
 
         # These are not immediately available in find_dir even with save() and
@@ -1189,6 +1235,9 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.parser.modules.add("rewrite_module")
         self.config.get_version = mock.Mock(return_value=(2, 3, 9))
 
+        # Creates ssl vhost for the domain
+        self.config.choose_vhost("red.blue.purple.com")
+
         self.config.enhance("red.blue.purple.com", "redirect")
         verify_no_redirect = ("certbot_apache.configurator."
                               "ApacheConfigurator._verify_no_certbot_redirect")
@@ -1200,7 +1249,7 @@ class MultipleVhostsTest(util.ApacheTest):
         # Skip the enable mod
         self.config.parser.modules.add("rewrite_module")
         self.config.get_version = mock.Mock(return_value=(2, 3, 9))
-
+        self.config.choose_vhost("red.blue.purple.com")
         self.config.enhance("red.blue.purple.com", "redirect")
         # Clear state about enabling redirect on this run
         # pylint: disable=protected-access
@@ -1422,6 +1471,7 @@ class MultipleVhostsTest(util.ApacheTest):
         # pylint: disable=protected-access
         self.config.parser.modules.add("mod_ssl.c")
         self.config.parser.modules.add("headers_module")
+        self.vh_truth[3].ssl = True
         self.config._wildcard_vhosts["*.certbot.demo"] = [self.vh_truth[3]]
         self.config.enhance("*.certbot.demo", "ensure-http-header",
                             "Upgrade-Insecure-Requests")
@@ -1429,6 +1479,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
     @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard")
     def test_enhance_wildcard_no_install(self, mock_choose):
+        self.vh_truth[3].ssl = True
         mock_choose.return_value = [self.vh_truth[3]]
         self.config.parser.modules.add("mod_ssl.c")
         self.config.parser.modules.add("headers_module")
@@ -1436,6 +1487,21 @@ class MultipleVhostsTest(util.ApacheTest):
                             "Upgrade-Insecure-Requests")
         self.assertTrue(mock_choose.called)
 
+    def test_add_vhost_id(self):
+        for vh in [self.vh_truth[0], self.vh_truth[1], self.vh_truth[2]]:
+            vh_id = self.config.add_vhost_id(vh)
+            self.assertEqual(vh, self.config.find_vhost_by_id(vh_id))
+
+    def test_find_vhost_by_id_404(self):
+        self.assertRaises(errors.PluginError,
+                          self.config.find_vhost_by_id,
+                          "nonexistent")
+
+    def test_add_vhost_id_already_exists(self):
+        first_id = self.config.add_vhost_id(self.vh_truth[0])
+        second_id = self.config.add_vhost_id(self.vh_truth[0])
+        self.assertEqual(first_id, second_id)
+
 
 class AugeasVhostsTest(util.ApacheTest):
     """Test vhosts with illegal names dependent on augeas version."""

certbot-apache/certbot_apache/tests/debian_test.py

@@ -161,6 +161,8 @@ class MultipleVhostsTestDebian(util.ApacheTest):
         self.config.parser.modules.add("mod_ssl.c")
         self.config.get_version = mock.Mock(return_value=(2, 4, 7))
         mock_exe.return_value = True
+        # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "staple-ocsp")
         self.assertTrue("socache_shmcb_module" in self.config.parser.modules)
 
@@ -172,6 +174,7 @@ class MultipleVhostsTestDebian(util.ApacheTest):
         mock_exe.return_value = True
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "ensure-http-header",
                             "Strict-Transport-Security")
         self.assertTrue("headers_module" in self.config.parser.modules)
@@ -183,6 +186,7 @@ class MultipleVhostsTestDebian(util.ApacheTest):
         mock_exe.return_value = True
         self.config.get_version = mock.Mock(return_value=(2, 2))
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "redirect")
         self.assertTrue("rewrite_module" in self.config.parser.modules)
 

certbot-apache/certbot_apache/tests/gentoo_test.py

@@ -4,6 +4,8 @@ import unittest
 
 import mock
 
+from certbot import errors
+
 from certbot_apache import override_gentoo
 from certbot_apache import obj
 from certbot_apache.tests import util
@@ -123,5 +125,11 @@ class MultipleVhostsTestGentoo(util.ApacheTest):
         self.assertEquals(len(self.config.parser.modules), 4)
         self.assertTrue("mod_another.c" in self.config.parser.modules)
 
+    @mock.patch("certbot_apache.configurator.util.run_script")
+    def test_alt_restart_works(self, mock_run_script):
+        mock_run_script.side_effect = [None, errors.SubprocessError, None]
+        self.config.restart()
+        self.assertEquals(mock_run_script.call_count, 3)
+
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/http_01_test.py

@@ -4,47 +4,26 @@ import os
 import unittest
 
 from acme import challenges
+from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
 
 from certbot import achallenges
 from certbot import errors
 
 from certbot.tests import acme_util
-
 from certbot_apache.tests import util
 
 
 NUM_ACHALLS = 3
 
 
-class ApacheHttp01TestMeta(type):
-    """Generates parmeterized tests for testing perform."""
-    def __new__(mcs, name, bases, class_dict):
-
-        def _gen_test(num_achalls, minor_version):
-            def _test(self):
-                achalls = self.achalls[:num_achalls]
-                vhosts = self.vhosts[:num_achalls]
-                self.config.version = (2, minor_version)
-                self.common_perform_test(achalls, vhosts)
-            return _test
-
-        for i in range(1, NUM_ACHALLS + 1):
-            for j in (2, 4):
-                test_name = "test_perform_{0}_{1}".format(i, j)
-                class_dict[test_name] = _gen_test(i, j)
-        return type.__new__(mcs, name, bases, class_dict)
-
-
 class ApacheHttp01Test(util.ApacheTest):
     """Test for certbot_apache.http_01.ApacheHttp01."""
 
-    __metaclass__ = ApacheHttp01TestMeta
-
     def setUp(self, *args, **kwargs):
         super(ApacheHttp01Test, self).setUp(*args, **kwargs)
 
         self.account_key = self.rsa512jwk
-        self.achalls = []
+        self.achalls = []  # type: List[achallenges.KeyAuthorizationAnnotatedChallenge]
         vh_truth = util.get_vh_truth(
             self.temp_dir, "debian_apache_2_4/multiple_vhosts")
         # Takes the vhosts for encryption-example.demo, certbot.demo, and
@@ -71,7 +50,7 @@ class ApacheHttp01Test(util.ApacheTest):
         self.assertFalse(self.http.perform())
 
     @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod")
-    def test_enable_modules_22(self, mock_enmod):
+    def test_enable_modules_apache_2_2(self, mock_enmod):
         self.config.version = (2, 2)
         self.config.parser.modules.remove("authz_host_module")
         self.config.parser.modules.remove("mod_authz_host.c")
@@ -80,7 +59,7 @@ class ApacheHttp01Test(util.ApacheTest):
         self.assertEqual(enmod_calls[0][0][0], "authz_host")
 
     @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod")
-    def test_enable_modules_24(self, mock_enmod):
+    def test_enable_modules_apache_2_4(self, mock_enmod):
         self.config.parser.modules.remove("authz_core_module")
         self.config.parser.modules.remove("mod_authz_core.c")
 
@@ -137,6 +116,31 @@ class ApacheHttp01Test(util.ApacheTest):
         self.config.config.http01_port = 12345
         self.assertRaises(errors.PluginError, self.http.perform)
 
+    def test_perform_1_achall_apache_2_2(self):
+        self.combinations_perform_test(num_achalls=1, minor_version=2)
+
+    def test_perform_1_achall_apache_2_4(self):
+        self.combinations_perform_test(num_achalls=1, minor_version=4)
+
+    def test_perform_2_achall_apache_2_2(self):
+        self.combinations_perform_test(num_achalls=2, minor_version=2)
+
+    def test_perform_2_achall_apache_2_4(self):
+        self.combinations_perform_test(num_achalls=2, minor_version=4)
+
+    def test_perform_3_achall_apache_2_2(self):
+        self.combinations_perform_test(num_achalls=3, minor_version=2)
+
+    def test_perform_3_achall_apache_2_4(self):
+        self.combinations_perform_test(num_achalls=3, minor_version=4)
+
+    def combinations_perform_test(self, num_achalls, minor_version):
+        """Test perform with the given achall count and Apache version."""
+        achalls = self.achalls[:num_achalls]
+        vhosts = self.vhosts[:num_achalls]
+        self.config.version = (2, minor_version)
+        self.common_perform_test(achalls, vhosts)
+
     def common_perform_test(self, achalls, vhosts):
         """Tests perform with the given achalls."""
         challenge_dir = self.http.challenge_dir

certbot-apache/certbot_apache/tests/parser_test.py

@@ -299,6 +299,13 @@ class BasicParserTest(util.ParserTest):
             errors.MisconfigurationError,
             self.parser.update_runtime_variables)
 
+    def test_add_comment(self):
+        from certbot_apache.parser import get_aug_path
+        self.parser.add_comment(get_aug_path(self.parser.loc["name"]), "123456")
+        comm = self.parser.find_comments("123456")
+        self.assertEquals(len(comm), 1)
+        self.assertTrue(self.parser.loc["name"] in comm[0])
+
 
 class ParserInitTest(util.ApacheTest):
     def setUp(self):  # pylint: disable=arguments-differ

certbot-apache/certbot_apache/tests/util.py

@@ -87,7 +87,6 @@ class ParserTest(ApacheTest):
 def get_apache_configurator(  # pylint: disable=too-many-arguments, too-many-locals
         config_path, vhost_path,
         config_dir, work_dir, version=(2, 4, 7),
-        conf=None,
         os_info="generic",
         conf_vhost_path=None):
     """Create an Apache Configurator with the specified options.
@@ -133,10 +132,6 @@ def get_apache_configurator(  # pylint: disable=too-many-arguments, too-many-loc
                         config_class = configurator.ApacheConfigurator
                     config = config_class(config=mock_le_config, name="apache",
                         version=version)
-                    # This allows testing scripts to set it a bit more
-                    # quickly
-                    if conf is not None:
-                        config.conf = conf  # pragma: no cover
 
                     config.prepare()
     return config

certbot-apache/certbot_apache/tls_sni_01.py

@@ -3,6 +3,7 @@
 import os
 import logging
 
+from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
 from certbot.plugins import common
 from certbot.errors import PluginError, MissingCommandlineFlag
 
@@ -93,7 +94,7 @@ class ApacheTlsSni01(common.TLSSNI01):
         :rtype: set
 
         """
-        addrs = set()
+        addrs = set()  # type: Set[obj.Addr]
         config_text = "<IfModule mod_ssl.c>\n"
 
         for achall in self.achalls:
@@ -123,7 +124,8 @@ class ApacheTlsSni01(common.TLSSNI01):
             self.configurator.config.tls_sni_01_port)))
 
         try:
-            vhost = self.configurator.choose_vhost(achall.domain, temp=True)
+            vhost = self.configurator.choose_vhost(achall.domain,
+                                                   create_if_no_ssl=False)
         except (PluginError, MissingCommandlineFlag):
             # We couldn't find the virtualhost for this domain, possibly
             # because it's a new vhost that's not configured yet

certbot-apache/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.25.0
+-e .[dev]

certbot-apache/setup.py

@@ -1,15 +1,14 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.22.0.dev0'
+version = '0.26.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.25.0',
+    'certbot>=0.26.0.dev0',
     'mock',
     'python-augeas',
     'setuptools',

certbot-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.21.1"
+LE_AUTO_VERSION="0.25.1"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -47,6 +47,7 @@ Help for certbot itself cannot be provided until it is installed.
   --no-bootstrap                            do not install OS dependencies
   --no-self-upgrade                         do not download updates
   --os-packages-only                        install OS dependencies and exit
+  --install-only                            install certbot, upgrade if needed, and exit
   -v, --verbose                             provide more output
   -q, --quiet                               provide only update/error output;
                                             implies --non-interactive
@@ -60,6 +61,8 @@ for arg in "$@" ; do
       DEBUG=1;;
     --os-packages-only)
       OS_PACKAGES_ONLY=1;;
+    --install-only)
+      INSTALL_ONLY=1;;
     --no-self-upgrade)
       # Do not upgrade this script (also prevents client upgrades, because each
       # copy of the script pins a hash of the python client)
@@ -246,7 +249,7 @@ DeprecationBootstrap() {
   fi
 }
 
-MIN_PYTHON_VERSION="2.6"
+MIN_PYTHON_VERSION="2.7"
 MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//')
 # Sets LE_PYTHON to Python version string and PYVER to the first two
 # digits of the python version
@@ -1052,9 +1055,11 @@ cffi==1.10.0 \
     --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \
     --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5
 ConfigArgParse==0.12.0 \
-    --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339
+    --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \
+    --no-binary ConfigArgParse
 configobj==5.0.6 \
-    --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902
+    --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \
+    --no-binary configobj
 cryptography==2.0.2 \
     --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \
     --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \
@@ -1086,7 +1091,7 @@ cryptography==2.0.2 \
     --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \
     --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \
     --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab
-enum34==1.1.2 \
+enum34==1.1.2 ; python_version < '3.4' \
     --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \
     --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501
 funcsigs==1.0.2 \
@@ -1109,7 +1114,8 @@ mock==1.3.0 \
     --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \
     --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6
 ordereddict==1.1 \
-    --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f
+    --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \
+    --no-binary ordereddict
 packaging==16.8 \
     --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \
     --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e
@@ -1135,7 +1141,8 @@ pyRFC3339==1.0 \
     --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \
     --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535
 python-augeas==0.5.0 \
-    --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2
+    --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \
+    --no-binary python-augeas
 pytz==2015.7 \
     --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \
     --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \
@@ -1163,9 +1170,11 @@ unittest2==1.1.0 \
     --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \
     --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579
 zope.component==4.2.2 \
-    --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a
+    --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \
+    --no-binary zope.component
 zope.event==4.1.0 \
-    --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786
+    --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \
+    --no-binary zope.event
 zope.interface==4.1.3 \
     --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \
     --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \
@@ -1184,6 +1193,9 @@ zope.interface==4.1.3 \
     --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \
     --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \
     --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392
+requests-toolbelt==0.8.0 \
+    --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \
+    --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5
 
 # Contains the requirements for the letsencrypt package.
 #
@@ -1196,24 +1208,24 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.21.1 \
-    --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \
-    --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea
-acme==0.21.1 \
-    --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \
-    --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b
-certbot-apache==0.21.1 \
-    --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \
-    --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5
-certbot-nginx==0.21.1 \
-    --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \
-    --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64
+certbot==0.25.1 \
+    --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \
+    --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e
+acme==0.25.1 \
+    --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \
+    --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734
+certbot-apache==0.25.1 \
+    --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \
+    --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a
+certbot-nginx==0.25.1 \
+    --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \
+    --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
     cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py"
 #!/usr/bin/env python
-"""A small script that can act as a trust root for installing pip 8
+"""A small script that can act as a trust root for installing pip >=8
 
 Embed this in your project, and your VCS checkout is all you have to trust. In
 a post-peep era, this lets you claw your way to a hash-checking version of pip,
@@ -1237,6 +1249,7 @@ anything goes wrong, it will exit with a non-zero status code.
 from __future__ import print_function
 from distutils.version import StrictVersion
 from hashlib import sha256
+from os import environ
 from os.path import join
 from pipes import quote
 from shutil import rmtree
@@ -1270,14 +1283,14 @@ except ImportError:
     from urllib.parse import urlparse  # 3.4
 
 
-__version__ = 1, 3, 0
+__version__ = 1, 5, 1
 PIP_VERSION = '9.0.1'
+DEFAULT_INDEX_BASE = 'https://pypi.python.org'
 
 
 # wheel has a conditional dependency on argparse:
 maybe_argparse = (
-    [('https://pypi.python.org/packages/18/dd/'
-      'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/'
+    [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/'
       'argparse-1.4.0.tar.gz',
       '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')]
     if version_info < (2, 7, 0) else [])
@@ -1285,18 +1298,14 @@ maybe_argparse = (
 
 PACKAGES = maybe_argparse + [
     # Pip has no dependencies, as it vendors everything:
-    ('https://pypi.python.org/packages/11/b6/'
-     'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/'
-     'pip-{0}.tar.gz'
-     .format(PIP_VERSION),
+    ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/'
+     'pip-{0}.tar.gz'.format(PIP_VERSION),
      '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'),
     # This version of setuptools has only optional dependencies:
-    ('https://pypi.python.org/packages/69/65/'
-     '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/'
-     'setuptools-20.2.2.tar.gz',
-     '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'),
-    ('https://pypi.python.org/packages/c9/1d/'
-     'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/'
+    ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/'
+     'setuptools-29.0.1.tar.gz',
+     'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'),
+    ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/'
      'wheel-0.29.0.tar.gz',
      '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648')
 ]
@@ -1317,12 +1326,13 @@ def hashed_download(url, temp, digest):
     # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert
     # authenticity has only privacy (not arbitrary code execution)
     # implications, since we're checking hashes.
-    def opener():
+    def opener(using_https=True):
         opener = build_opener(HTTPSHandler())
-        # Strip out HTTPHandler to prevent MITM spoof:
-        for handler in opener.handlers:
-            if isinstance(handler, HTTPHandler):
-                opener.handlers.remove(handler)
+        if using_https:
+            # Strip out HTTPHandler to prevent MITM spoof:
+            for handler in opener.handlers:
+                if isinstance(handler, HTTPHandler):
+                    opener.handlers.remove(handler)
         return opener
 
     def read_chunks(response, chunk_size):
@@ -1332,8 +1342,9 @@ def hashed_download(url, temp, digest):
                 break
             yield chunk
 
-    response = opener().open(url)
-    path = join(temp, urlparse(url).path.split('/')[-1])
+    parsed_url = urlparse(url)
+    response = opener(using_https=parsed_url.scheme == 'https').open(url)
+    path = join(temp, parsed_url.path.split('/')[-1])
     actual_hash = sha256()
     with open(path, 'wb') as file:
         for chunk in read_chunks(response, 4096):
@@ -1346,6 +1357,24 @@ def hashed_download(url, temp, digest):
     return path
 
 
+def get_index_base():
+    """Return the URL to the dir containing the "packages" folder.
+
+    Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the
+    end if it's there; that is likely to give us the right dir.
+
+    """
+    env_var = environ.get('PIP_INDEX_URL', '').rstrip('/')
+    if env_var:
+        SIMPLE = '/simple'
+        if env_var.endswith(SIMPLE):
+            return env_var[:-len(SIMPLE)]
+        else:
+            return env_var
+    else:
+        return DEFAULT_INDEX_BASE
+
+
 def main():
     pip_version = StrictVersion(check_output(['pip', '--version'])
                                 .decode('utf-8').split()[1])
@@ -1353,11 +1382,13 @@ def main():
     if pip_version >= min_pip_version:
         return 0
     has_pip_cache = pip_version >= StrictVersion('6.0')
-
+    index_base = get_index_base()
     temp = mkdtemp(prefix='pipstrap-')
     try:
-        downloads = [hashed_download(url, temp, digest)
-                     for url, digest in PACKAGES]
+        downloads = [hashed_download(index_base + '/packages/' + path,
+                                     temp,
+                                     digest)
+                     for path, digest in PACKAGES]
         check_output('pip install --no-index --no-deps -U ' +
                      # Disable cache since we're not using it and it otherwise
                      # sometimes throws permission warnings:
@@ -1428,6 +1459,12 @@ UNLIKELY_EOF
 
     say "Installation succeeded."
   fi
+
+  if [ "$INSTALL_ONLY" = 1 ]; then
+    say "Certbot is installed."
+    exit 0
+  fi
+
   "$VENV_BIN/letsencrypt" "$@"
 
 else

certbot-compatibility-test/certbot_compatibility_test/test_driver.py

@@ -10,9 +10,12 @@ import sys
 
 import OpenSSL
 
+from six.moves import xrange  # pylint: disable=import-error,redefined-builtin
+
 from acme import challenges
 from acme import crypto_util
 from acme import messages
+from acme.magic_typing import List, Tuple  # pylint: disable=unused-import, no-name-in-module
 from certbot import achallenges
 from certbot import errors as le_errors
 from certbot.tests import acme_util
@@ -50,9 +53,8 @@ def test_authenticator(plugin, config, temp_dir):
 
     try:
         responses = plugin.perform(achalls)
-    except le_errors.Error as error:
-        logger.error("Performing challenges on %s caused an error:", config)
-        logger.exception(error)
+    except le_errors.Error:
+        logger.error("Performing challenges on %s caused an error:", config, exc_info=True)
         return False
 
     success = True
@@ -80,9 +82,8 @@ def test_authenticator(plugin, config, temp_dir):
     if success:
         try:
             plugin.cleanup(achalls)
-        except le_errors.Error as error:
-            logger.error("Challenge cleanup for %s caused an error:", config)
-            logger.exception(error)
+        except le_errors.Error:
+            logger.error("Challenge cleanup for %s caused an error:", config, exc_info=True)
             success = False
 
         if _dirs_are_unequal(config, backup):
@@ -145,9 +146,8 @@ def test_deploy_cert(plugin, temp_dir, domains):
         try:
             plugin.deploy_cert(domain, cert_path, util.KEY_PATH, cert_path, cert_path)
             plugin.save()  # Needed by the Apache plugin
-        except le_errors.Error as error:
-            logger.error("**** Plugin failed to deploy certificate for %s:", domain)
-            logger.exception(error)
+        except le_errors.Error:
+            logger.error("**** Plugin failed to deploy certificate for %s:", domain, exc_info=True)
             return False
 
     if not _save_and_restart(plugin, "deployed"):
@@ -177,7 +177,7 @@ def test_enhancements(plugin, domains):
                      "enhancements")
         return False
 
-    domains_and_info = [(domain, []) for domain in domains]
+    domains_and_info = [(domain, []) for domain in domains]  # type: List[Tuple[str, List[bool]]]
 
     for domain, info in domains_and_info:
         try:
@@ -190,10 +190,9 @@ def test_enhancements(plugin, domains):
             # Don't immediately fail because a redirect may already be enabled
             logger.warning("*** Plugin failed to enable redirect for %s:", domain)
             logger.warning("%s", error)
-        except le_errors.Error as error:
+        except le_errors.Error:
             logger.error("*** An error occurred while enabling redirect for %s:",
-                         domain)
-            logger.exception(error)
+                         domain, exc_info=True)
 
     if not _save_and_restart(plugin, "enhanced"):
         return False
@@ -220,9 +219,8 @@ def _save_and_restart(plugin, title=None):
         plugin.save(title)
         plugin.restart()
         return True
-    except le_errors.Error as error:
-        logger.error("*** Plugin failed to save and restart server:")
-        logger.exception(error)
+    except le_errors.Error:
+        logger.error("*** Plugin failed to save and restart server:", exc_info=True)
         return False
 
 
@@ -230,9 +228,8 @@ def test_rollback(plugin, config, backup):
     """Tests the rollback checkpoints function"""
     try:
         plugin.rollback_checkpoints(1337)
-    except le_errors.Error as error:
-        logger.error("*** Plugin raised an exception during rollback:")
-        logger.exception(error)
+    except le_errors.Error:
+        logger.error("*** Plugin raised an exception during rollback:", exc_info=True)
         return False
 
     if _dirs_are_unequal(config, backup):
@@ -261,21 +258,21 @@ def _dirs_are_unequal(dir1, dir2):
             logger.error("The following files and directories are only "
                          "present in one directory")
             if dircmp.left_only:
-                logger.error(dircmp.left_only)
+                logger.error(str(dircmp.left_only))
             else:
-                logger.error(dircmp.right_only)
+                logger.error(str(dircmp.right_only))
             return True
         elif dircmp.common_funny or dircmp.funny_files:
             logger.error("The following files and directories could not be "
                          "compared:")
             if dircmp.common_funny:
-                logger.error(dircmp.common_funny)
+                logger.error(str(dircmp.common_funny))
             else:
-                logger.error(dircmp.funny_files)
+                logger.error(str(dircmp.funny_files))
             return True
         elif dircmp.diff_files:
             logger.error("The following files differ:")
-            logger.error(dircmp.diff_files)
+            logger.error(str(dircmp.diff_files))
             return True
 
         for subdir in dircmp.subdirs.itervalues():
@@ -352,9 +349,8 @@ def main():
                     success = test_authenticator(plugin, config, temp_dir)
                 if success and args.install:
                     success = test_installer(args, plugin, config, temp_dir)
-            except errors.Error as error:
-                logger.error("Tests on %s raised:", config)
-                logger.exception(error)
+            except errors.Error:
+                logger.error("Tests on %s raised:", config, exc_info=True)
                 success = False
 
             if success:

certbot-compatibility-test/certbot_compatibility_test/util.py

@@ -26,12 +26,12 @@ def create_le_config(parent_dir):
     config = copy.deepcopy(constants.CLI_DEFAULTS)
 
     le_dir = os.path.join(parent_dir, "certbot")
-    config["config_dir"] = os.path.join(le_dir, "config")
-    config["work_dir"] = os.path.join(le_dir, "work")
-    config["logs_dir"] = os.path.join(le_dir, "logs_dir")
-    os.makedirs(config["config_dir"])
-    os.mkdir(config["work_dir"])
-    os.mkdir(config["logs_dir"])
+    os.mkdir(le_dir)
+    for dir_name in ("config", "logs", "work"):
+        full_path = os.path.join(le_dir, dir_name)
+        os.mkdir(full_path)
+        full_name = dir_name + "_dir"
+        config[full_name] = full_path
 
     config["domains"] = None
 

certbot-compatibility-test/certbot_compatibility_test/validator.py

@@ -5,6 +5,7 @@ import requests
 import zope.interface
 
 import six
+from six.moves import xrange  # pylint: disable=import-error,redefined-builtin
 
 from acme import crypto_util
 from acme import errors as acme_errors
@@ -32,7 +33,7 @@ class Validator(object):
         try:
             presented_cert = crypto_util.probe_sni(name, host, port)
         except acme_errors.Error as error:
-            logger.exception(error)
+            logger.exception(str(error))
             return False
 
         return presented_cert.digest("sha256") == cert.digest("sha256")
@@ -85,8 +86,7 @@ class Validator(object):
             return False
 
         try:
-            _, max_age_value = max_age[0]
-            max_age_value = int(max_age_value)
+            max_age_value = int(max_age[0][1])
         except ValueError:
             logger.error("Server responded with invalid HSTS header field")
             return False

certbot-compatibility-test/nginx/roundtrip.py

@@ -8,7 +8,7 @@ from certbot_nginx import nginxparser
 def roundtrip(stuff):
     success = True
     for t in stuff:
-        print t
+        print(t)
         if not os.path.isfile(t):
             continue
         with open(t, "r") as f:

certbot-compatibility-test/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.22.0.dev0'
+version = '0.26.0.dev0'
 
 install_requires = [
     'certbot',

certbot-dns-cloudflare/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-cloudflare
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-cloudflare

certbot-dns-cloudflare/docs/index.rst

@@ -10,14 +10,14 @@ Welcome to certbot-dns-cloudflare's documentation!
    :maxdepth: 2
    :caption: Contents:
 
+.. automodule:: certbot_dns_cloudflare
+   :members:
+
 .. toctree::
    :maxdepth: 1
 
    api
 
-.. automodule:: certbot_dns_cloudflare
-   :members:
-
 
 Indices and tables
 ==================

certbot-dns-cloudflare/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-cloudflare/readthedocs.org.requirements.txt

@@ -0,0 +1,12 @@
+# readthedocs.org gives no way to change the install command to "pip
+# install -e .[docs]" (that would in turn install documentation
+# dependencies), but it allows to specify a requirements.txt file at
+# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259)
+
+# Although ReadTheDocs certainly doesn't need to install the project
+# in --editable mode (-e), just "pip install .[docs]" does not work as
+# expected and "pip install -e .[docs]" must be used instead
+
+-e acme
+-e .
+-e certbot-dns-cloudflare[docs]

certbot-dns-cloudflare/setup.py

@@ -1,15 +1,14 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.22.0.dev0'
+version = '0.26.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
     'cloudflare>=1.5.1',
     'mock',
     'setuptools',

certbot-dns-cloudxns/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-cloudxns
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-cloudxns

certbot-dns-cloudxns/docs/index.rst

@@ -10,14 +10,14 @@ Welcome to certbot-dns-cloudxns's documentation!
    :maxdepth: 2
    :caption: Contents:
 
+.. automodule:: certbot_dns_cloudxns
+   :members:
+
 .. toctree::
    :maxdepth: 1
 
    api
 
-.. automodule:: certbot_dns_cloudxns
-   :members:
-
 
 
 Indices and tables

certbot-dns-cloudxns/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-cloudxns/readthedocs.org.requirements.txt

@@ -0,0 +1,12 @@
+# readthedocs.org gives no way to change the install command to "pip
+# install -e .[docs]" (that would in turn install documentation
+# dependencies), but it allows to specify a requirements.txt file at
+# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259)
+
+# Although ReadTheDocs certainly doesn't need to install the project
+# in --editable mode (-e), just "pip install .[docs]" does not work as
+# expected and "pip install -e .[docs]" must be used instead
+
+-e acme
+-e .
+-e certbot-dns-cloudxns[docs]

certbot-dns-cloudxns/setup.py

@@ -1,16 +1,15 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.22.0.dev0'
+version = '0.26.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
-    'dns-lexicon',
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
+    'dns-lexicon>=2.2.1', # Support for >1 TXT record per name
     'mock',
     'setuptools',
     'zope.interface',

certbot-dns-digitalocean/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-digitalocean
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-digitalocean

certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py

@@ -50,7 +50,8 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic
 
 
 class DigitalOceanClientTest(unittest.TestCase):
-    id = 1
+
+    id_num = 1
     record_prefix = "_acme-challenge"
     record_name = record_prefix + "." + DOMAIN
     record_content = "bar"
@@ -70,7 +71,7 @@ class DigitalOceanClientTest(unittest.TestCase):
 
         domain_mock = mock.MagicMock()
         domain_mock.name = DOMAIN
-        domain_mock.create_new_domain_record.return_value = {'domain_record': {'id': self.id}}
+        domain_mock.create_new_domain_record.return_value = {'domain_record': {'id': self.id_num}}
 
         self.manager.get_all_domains.return_value = [wrong_domain_mock, domain_mock]
 

certbot-dns-digitalocean/docs/index.rst

@@ -10,14 +10,14 @@ Welcome to certbot-dns-digitalocean's documentation!
    :maxdepth: 2
    :caption: Contents:
 
+.. automodule:: certbot_dns_digitalocean
+   :members:
+
 .. toctree::
    :maxdepth: 1
 
    api
 
-.. automodule:: certbot_dns_digitalocean
-   :members:
-
 
 
 Indices and tables

certbot-dns-digitalocean/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-digitalocean/readthedocs.org.requirements.txt

@@ -0,0 +1,12 @@
+# readthedocs.org gives no way to change the install command to "pip
+# install -e .[docs]" (that would in turn install documentation
+# dependencies), but it allows to specify a requirements.txt file at
+# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259)
+
+# Although ReadTheDocs certainly doesn't need to install the project
+# in --editable mode (-e), just "pip install .[docs]" does not work as
+# expected and "pip install -e .[docs]" must be used instead
+
+-e acme
+-e .
+-e certbot-dns-digitalocean[docs]

certbot-dns-digitalocean/setup.py

@@ -1,15 +1,14 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.22.0.dev0'
+version = '0.26.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
     'mock',
     'python-digitalocean>=1.11',
     'setuptools',

certbot-dns-dnsimple/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-dnsimple
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-dnsimple

certbot-dns-dnsimple/docs/index.rst

@@ -10,14 +10,14 @@ Welcome to certbot-dns-dnsimple's documentation!
    :maxdepth: 2
    :caption: Contents:
 
+.. automodule:: certbot_dns_dnsimple
+   :members:
+
 .. toctree::
    :maxdepth: 1
 
    api
 
-.. automodule:: certbot_dns_dnsimple
-   :members:
-
 
 
 Indices and tables

certbot-dns-dnsimple/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-dnsimple/readthedocs.org.requirements.txt

@@ -0,0 +1,12 @@
+# readthedocs.org gives no way to change the install command to "pip
+# install -e .[docs]" (that would in turn install documentation
+# dependencies), but it allows to specify a requirements.txt file at
+# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259)
+
+# Although ReadTheDocs certainly doesn't need to install the project
+# in --editable mode (-e), just "pip install .[docs]" does not work as
+# expected and "pip install -e .[docs]" must be used instead
+
+-e acme
+-e .
+-e certbot-dns-dnsimple[docs]

certbot-dns-dnsimple/setup.py

@@ -1,16 +1,15 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.22.0.dev0'
+version = '0.26.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
-    'dns-lexicon',
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
+    'dns-lexicon>=2.2.1', # Support for >1 TXT record per name
     'mock',
     'setuptools',
     'zope.interface',

certbot-dns-dnsmadeeasy/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-dnsmadeeasy
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-dnsmadeeasy

certbot-dns-dnsmadeeasy/docs/index.rst

@@ -10,14 +10,14 @@ Welcome to certbot-dns-dnsmadeeasy's documentation!
    :maxdepth: 2
    :caption: Contents:
 
+.. automodule:: certbot_dns_dnsmadeeasy
+   :members:
+
 .. toctree::
    :maxdepth: 1
 
    api
 
-.. automodule:: certbot_dns_dnsmadeeasy
-   :members:
-
 
 
 Indices and tables

certbot-dns-dnsmadeeasy/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-dnsmadeeasy/readthedocs.org.requirements.txt

@@ -0,0 +1,12 @@
+# readthedocs.org gives no way to change the install command to "pip
+# install -e .[docs]" (that would in turn install documentation
+# dependencies), but it allows to specify a requirements.txt file at
+# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259)
+
+# Although ReadTheDocs certainly doesn't need to install the project
+# in --editable mode (-e), just "pip install .[docs]" does not work as
+# expected and "pip install -e .[docs]" must be used instead
+
+-e acme
+-e .
+-e certbot-dns-dnsmadeeasy[docs]

certbot-dns-dnsmadeeasy/setup.py

@@ -1,16 +1,15 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.22.0.dev0'
+version = '0.26.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
-    'dns-lexicon',
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
+    'dns-lexicon>=2.2.1', # Support for >1 TXT record per name
     'mock',
     'setuptools',
     'zope.interface',

certbot-dns-google/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-google
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-google

certbot-dns-google/MANIFEST.in

@@ -1,3 +1,4 @@
 include LICENSE.txt
 include README.rst
 recursive-include docs *
+recursive-include certbot_dns_google/testdata *

certbot-dns-google/certbot_dns_google/__init__.py

@@ -29,6 +29,8 @@ for an account with the following permissions:
 * ``dns.managedZones.list``
 * ``dns.resourceRecordSets.create``
 * ``dns.resourceRecordSets.delete``
+* ``dns.resourceRecordSets.list``
+* ``dns.resourceRecordSets.update``
 
 Google provides instructions for `creating a service account <https://developers
 .google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount>`_ and

certbot-dns-google/certbot_dns_google/dns_google.py

@@ -81,7 +81,7 @@ class _GoogleClient(object):
     Encapsulates all communication with the Google Cloud DNS API.
     """
 
-    def __init__(self, account_json=None):
+    def __init__(self, account_json=None, dns_api=None):
 
         scopes = ['https://www.googleapis.com/auth/ndev.clouddns.readwrite']
         if account_json is not None:
@@ -92,7 +92,12 @@ class _GoogleClient(object):
             credentials = None
             self.project_id = self.get_project_id()
 
-        self.dns = discovery.build('dns', 'v1', credentials=credentials, cache_discovery=False)
+        if not dns_api:
+            self.dns = discovery.build('dns', 'v1',
+                                       credentials=credentials,
+                                       cache_discovery=False)
+        else:
+            self.dns = dns_api
 
     def add_txt_record(self, domain, record_name, record_content, record_ttl):
         """
@@ -107,6 +112,17 @@ class _GoogleClient(object):
 
         zone_id = self._find_managed_zone_id(domain)
 
+        record_contents = self.get_existing_txt_rrset(zone_id, record_name)
+        if record_contents is None:
+            record_contents = []
+        add_records = record_contents[:]
+
+        if "\""+record_content+"\"" in record_contents:
+            # The process was interrupted previously and validation token exists
+            return
+
+        add_records.append(record_content)
+
         data = {
             "kind": "dns#change",
             "additions": [
@@ -114,12 +130,24 @@ class _GoogleClient(object):
                     "kind": "dns#resourceRecordSet",
                     "type": "TXT",
                     "name": record_name + ".",
-                    "rrdatas": [record_content, ],
+                    "rrdatas": add_records,
                     "ttl": record_ttl,
                 },
             ],
         }
 
+        if record_contents:
+            # We need to remove old records in the same request
+            data["deletions"] = [
+                {
+                    "kind": "dns#resourceRecordSet",
+                    "type": "TXT",
+                    "name": record_name + ".",
+                    "rrdatas": record_contents,
+                    "ttl": record_ttl,
+                },
+            ]
+
         changes = self.dns.changes()  # changes | pylint: disable=no-member
 
         try:
@@ -154,6 +182,10 @@ class _GoogleClient(object):
             logger.warn('Error finding zone. Skipping cleanup.')
             return
 
+        record_contents = self.get_existing_txt_rrset(zone_id, record_name)
+        if record_contents is None:
+            record_contents = ["\"" + record_content + "\""]
+
         data = {
             "kind": "dns#change",
             "deletions": [
@@ -161,12 +193,26 @@ class _GoogleClient(object):
                     "kind": "dns#resourceRecordSet",
                     "type": "TXT",
                     "name": record_name + ".",
-                    "rrdatas": [record_content, ],
+                    "rrdatas": record_contents,
                     "ttl": record_ttl,
                 },
             ],
         }
 
+        # Remove the record being deleted from the list
+        readd_contents = [r for r in record_contents if r != "\"" + record_content + "\""]
+        if readd_contents:
+            # We need to remove old records in the same request
+            data["additions"] = [
+                {
+                    "kind": "dns#resourceRecordSet",
+                    "type": "TXT",
+                    "name": record_name + ".",
+                    "rrdatas": readd_contents,
+                    "ttl": record_ttl,
+                },
+            ]
+
         changes = self.dns.changes()  # changes | pylint: disable=no-member
 
         try:
@@ -175,6 +221,37 @@ class _GoogleClient(object):
         except googleapiclient_errors.Error as e:
             logger.warn('Encountered error deleting TXT record: %s', e)
 
+    def get_existing_txt_rrset(self, zone_id, record_name):
+        """
+        Get existing TXT records from the RRset for the record name.
+
+        If an error occurs while requesting the record set, it is suppressed
+        and None is returned.
+
+        :param str zone_id: The ID of the managed zone.
+        :param str record_name: The record name (typically beginning with '_acme-challenge.').
+
+        :returns: List of TXT record values or None
+        :rtype: `list` of `string` or `None`
+
+        """
+        rrs_request = self.dns.resourceRecordSets()  # pylint: disable=no-member
+        request = rrs_request.list(managedZone=zone_id, project=self.project_id)
+        # Add dot as the API returns absolute domains
+        record_name += "."
+        try:
+            response = request.execute()
+        except googleapiclient_errors.Error:
+            logger.info("Unable to list existing records. If you're "
+                        "requesting a wildcard certificate, this might not work.")
+            logger.debug("Error was:", exc_info=True)
+        else:
+            if response:
+                for rr in response["rrsets"]:
+                    if rr["name"] == record_name and rr["type"] == "TXT":
+                        return rr["rrdatas"]
+        return None
+
     def _find_managed_zone_id(self, domain):
         """
         Find the managed zone for a given domain.

certbot-dns-google/certbot_dns_google/dns_google_test.py

@@ -4,7 +4,9 @@ import os
 import unittest
 
 import mock
+from googleapiclient import discovery
 from googleapiclient.errors import Error
+from googleapiclient.http import HttpMock
 from httplib2 import ServerNotFoundError
 
 from certbot import errors
@@ -68,16 +70,27 @@ class GoogleClientTest(unittest.TestCase):
     def _setUp_client_with_mock(self, zone_request_side_effect):
         from certbot_dns_google.dns_google import _GoogleClient
 
-        client = _GoogleClient(ACCOUNT_JSON_PATH)
+        pwd = os.path.dirname(__file__)
+        rel_path = 'testdata/discovery.json'
+        discovery_file = os.path.join(pwd, rel_path)
+        http_mock = HttpMock(discovery_file, {'status': '200'})
+        dns_api = discovery.build('dns', 'v1', http=http_mock)
+
+        client = _GoogleClient(ACCOUNT_JSON_PATH, dns_api)
 
         # Setup
         mock_mz = mock.MagicMock()
         mock_mz.list.return_value.execute.side_effect = zone_request_side_effect
 
+        mock_rrs = mock.MagicMock()
+        rrsets = {"rrsets": [{"name": "_acme-challenge.example.org.", "type": "TXT",
+                              "rrdatas": ["\"example-txt-contents\""]}]}
+        mock_rrs.list.return_value.execute.return_value = rrsets
         mock_changes = mock.MagicMock()
 
         client.dns.managedZones = mock.MagicMock(return_value=mock_mz)
         client.dns.changes = mock.MagicMock(return_value=mock_changes)
+        client.dns.resourceRecordSets = mock.MagicMock(return_value=mock_rrs)
 
         return client, mock_changes
 
@@ -137,6 +150,30 @@ class GoogleClientTest(unittest.TestCase):
                                             managedZone=self.zone,
                                             project=PROJECT_ID)
 
+    @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name')
+    @mock.patch('certbot_dns_google.dns_google.open',
+                mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True)
+    def test_add_txt_record_delete_old(self, unused_credential_mock):
+        client, changes = self._setUp_client_with_mock(
+            [{'managedZones': [{'id': self.zone}]}])
+        mock_get_rrs = "certbot_dns_google.dns_google._GoogleClient.get_existing_txt_rrset"
+        with mock.patch(mock_get_rrs) as mock_rrs:
+            mock_rrs.return_value = ["sample-txt-contents"]
+            client.add_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl)
+            self.assertTrue(changes.create.called)
+            self.assertTrue("sample-txt-contents" in
+                changes.create.call_args_list[0][1]["body"]["deletions"][0]["rrdatas"])
+
+    @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name')
+    @mock.patch('certbot_dns_google.dns_google.open',
+                mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True)
+    def test_add_txt_record_noop(self, unused_credential_mock):
+        client, changes = self._setUp_client_with_mock(
+            [{'managedZones': [{'id': self.zone}]}])
+        client.add_txt_record(DOMAIN, "_acme-challenge.example.org",
+                              "example-txt-contents", self.record_ttl)
+        self.assertFalse(changes.create.called)
+
     @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name')
     @mock.patch('certbot_dns_google.dns_google.open',
                 mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True)
@@ -172,7 +209,12 @@ class GoogleClientTest(unittest.TestCase):
     def test_del_txt_record(self, unused_credential_mock):
         client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}])
 
-        client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl)
+        mock_get_rrs = "certbot_dns_google.dns_google._GoogleClient.get_existing_txt_rrset"
+        with mock.patch(mock_get_rrs) as mock_rrs:
+            mock_rrs.return_value = ["\"sample-txt-contents\"",
+                                     "\"example-txt-contents\""]
+            client.del_txt_record(DOMAIN, "_acme-challenge.example.org",
+                                "example-txt-contents", self.record_ttl)
 
         expected_body = {
             "kind": "dns#change",
@@ -180,8 +222,17 @@ class GoogleClientTest(unittest.TestCase):
                 {
                     "kind": "dns#resourceRecordSet",
                     "type": "TXT",
-                    "name": self.record_name + ".",
-                    "rrdatas": [self.record_content, ],
+                    "name": "_acme-challenge.example.org.",
+                    "rrdatas": ["\"sample-txt-contents\"", "\"example-txt-contents\""],
+                    "ttl": self.record_ttl,
+                },
+            ],
+            "additions": [
+                {
+                    "kind": "dns#resourceRecordSet",
+                    "type": "TXT",
+                    "name": "_acme-challenge.example.org.",
+                    "rrdatas": ["\"sample-txt-contents\"", ],
                     "ttl": self.record_ttl,
                 },
             ],
@@ -217,6 +268,31 @@ class GoogleClientTest(unittest.TestCase):
 
         client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl)
 
+    @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name')
+    @mock.patch('certbot_dns_google.dns_google.open',
+                mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True)
+    def test_get_existing(self, unused_credential_mock):
+        client, unused_changes = self._setUp_client_with_mock(
+            [{'managedZones': [{'id': self.zone}]}])
+        # Record name mocked in setUp
+        found = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org")
+        self.assertEquals(found, ["\"example-txt-contents\""])
+        not_found = client.get_existing_txt_rrset(self.zone, "nonexistent.tld")
+        self.assertEquals(not_found, None)
+
+    @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name')
+    @mock.patch('certbot_dns_google.dns_google.open',
+                mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True)
+    def test_get_existing_fallback(self, unused_credential_mock):
+        client, unused_changes = self._setUp_client_with_mock(
+            [{'managedZones': [{'id': self.zone}]}])
+        # pylint: disable=no-member
+        mock_execute = client.dns.resourceRecordSets.return_value.list.return_value.execute
+        mock_execute.side_effect = API_ERROR
+
+        rrset = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org")
+        self.assertFalse(rrset)
+
     def test_get_project_id(self):
         from certbot_dns_google.dns_google import _GoogleClient
 

certbot-dns-google/docs/index.rst

@@ -10,14 +10,14 @@ Welcome to certbot-dns-google's documentation!
    :maxdepth: 2
    :caption: Contents:
 
+.. automodule:: certbot_dns_google
+   :members:
+
 .. toctree::
    :maxdepth: 1
 
    api
 
-.. automodule:: certbot_dns_google
-   :members:
-
 
 
 Indices and tables

certbot-dns-google/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-google/readthedocs.org.requirements.txt

@@ -0,0 +1,12 @@
+# readthedocs.org gives no way to change the install command to "pip
+# install -e .[docs]" (that would in turn install documentation
+# dependencies), but it allows to specify a requirements.txt file at
+# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259)
+
+# Although ReadTheDocs certainly doesn't need to install the project
+# in --editable mode (-e), just "pip install .[docs]" does not work as
+# expected and "pip install -e .[docs]" must be used instead
+
+-e acme
+-e .
+-e certbot-dns-google[docs]

certbot-dns-google/setup.py

@@ -1,15 +1,14 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.22.0.dev0'
+version = '0.26.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
     # 1.5 is the first version that supports oauth2client>=2.0
     'google-api-python-client>=1.5',
     'mock',

certbot-dns-luadns/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-luadns
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-luadns

certbot-dns-luadns/docs/index.rst

@@ -10,14 +10,14 @@ Welcome to certbot-dns-luadns's documentation!
    :maxdepth: 2
    :caption: Contents:
 
+.. automodule:: certbot_dns_luadns
+   :members:
+
 .. toctree::
    :maxdepth: 1
 
    api
 
-.. automodule:: certbot_dns_luadns
-   :members:
-
 
 
 Indices and tables

certbot-dns-luadns/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-luadns/readthedocs.org.requirements.txt

@@ -0,0 +1,12 @@
+# readthedocs.org gives no way to change the install command to "pip
+# install -e .[docs]" (that would in turn install documentation
+# dependencies), but it allows to specify a requirements.txt file at
+# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259)
+
+# Although ReadTheDocs certainly doesn't need to install the project
+# in --editable mode (-e), just "pip install .[docs]" does not work as
+# expected and "pip install -e .[docs]" must be used instead
+
+-e acme
+-e .
+-e certbot-dns-luadns[docs]

certbot-dns-luadns/setup.py

@@ -1,16 +1,15 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.22.0.dev0'
+version = '0.26.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
-    'dns-lexicon',
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
+    'dns-lexicon>=2.2.1', # Support for >1 TXT record per name
     'mock',
     'setuptools',
     'zope.interface',

certbot-dns-nsone/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-nsone
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-nsone

certbot-dns-nsone/docs/index.rst

@@ -10,14 +10,14 @@ Welcome to certbot-dns-nsone's documentation!
    :maxdepth: 2
    :caption: Contents:
 
+.. automodule:: certbot_dns_nsone
+   :members:
+
 .. toctree::
    :maxdepth: 1
 
    api
 
-.. automodule:: certbot_dns_nsone
-   :members:
-
 
 
 Indices and tables

certbot-dns-nsone/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-nsone/readthedocs.org.requirements.txt

@@ -0,0 +1,12 @@
+# readthedocs.org gives no way to change the install command to "pip
+# install -e .[docs]" (that would in turn install documentation
+# dependencies), but it allows to specify a requirements.txt file at
+# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259)
+
+# Although ReadTheDocs certainly doesn't need to install the project
+# in --editable mode (-e), just "pip install .[docs]" does not work as
+# expected and "pip install -e .[docs]" must be used instead
+
+-e acme
+-e .
+-e certbot-dns-nsone[docs]

certbot-dns-nsone/setup.py

@@ -1,16 +1,15 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.22.0.dev0'
+version = '0.26.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
-    'dns-lexicon',
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
+    'dns-lexicon>=2.2.1', # Support for >1 TXT record per name
     'mock',
     'setuptools',
     'zope.interface',

certbot-dns-rfc2136/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-rfc2136
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-rfc2136

certbot-dns-rfc2136/certbot_dns_rfc2136/__init__.py

@@ -21,8 +21,9 @@ Credentials
 -----------
 
 Use of this plugin requires a configuration file containing the target DNS
-server that supports RFC 2136 Dynamic Updates, the name of the TSIG key, the
-TSIG key secret itself and the algorithm used if it's different to HMAC-MD5.
+server and optional port that supports RFC 2136 Dynamic Updates, the name
+of the TSIG key, the TSIG key secret itself and the algorithm used if it's
+different to HMAC-MD5.
 
 .. code-block:: ini
    :name: credentials.ini
@@ -30,6 +31,8 @@ TSIG key secret itself and the algorithm used if it's different to HMAC-MD5.
 
    # Target DNS server
    dns_rfc2136_server = 192.0.2.1
+   # Target DNS port
+   dns_rfc2136_port = 53
    # TSIG key name
    dns_rfc2136_name = keyname.
    # TSIG key secret

certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136.py

@@ -36,6 +36,8 @@ class Authenticator(dns_common.DNSAuthenticator):
       'HMAC-SHA512': dns.tsig.HMAC_SHA512
     }
 
+    PORT = 53
+
     description = 'Obtain certificates using a DNS TXT record (if you are using BIND for DNS).'
     ttl = 120
 
@@ -70,14 +72,15 @@ class Authenticator(dns_common.DNSAuthenticator):
             self._validate_algorithm
         )
 
-    def _perform(self, domain, validation_name, validation):
-        self._get_rfc2136_client().add_txt_record(domain, validation_name, validation, self.ttl)
+    def _perform(self, _domain, validation_name, validation):
+        self._get_rfc2136_client().add_txt_record(validation_name, validation, self.ttl)
 
-    def _cleanup(self, domain, validation_name, validation):
-        self._get_rfc2136_client().del_txt_record(domain, validation_name, validation)
+    def _cleanup(self, _domain, validation_name, validation):
+        self._get_rfc2136_client().del_txt_record(validation_name, validation)
 
     def _get_rfc2136_client(self):
         return _RFC2136Client(self.credentials.conf('server'),
+                              int(self.credentials.conf('port') or self.PORT),
                               self.credentials.conf('name'),
                               self.credentials.conf('secret'),
                               self.ALGORITHMS.get(self.credentials.conf('algorithm'),
@@ -88,25 +91,25 @@ class _RFC2136Client(object):
     """
     Encapsulates all communication with the target DNS server.
     """
-    def __init__(self, server, key_name, key_secret, key_algorithm):
+    def __init__(self, server, port, key_name, key_secret, key_algorithm):
         self.server = server
+        self.port = port
         self.keyring = dns.tsigkeyring.from_text({
             key_name: key_secret
         })
         self.algorithm = key_algorithm
 
-    def add_txt_record(self, domain_name, record_name, record_content, record_ttl):
+    def add_txt_record(self, record_name, record_content, record_ttl):
         """
         Add a TXT record using the supplied information.
 
-        :param str domain: The domain to use to find the closest SOA.
         :param str record_name: The record name (typically beginning with '_acme-challenge.').
         :param str record_content: The record content (typically the challenge validation).
         :param int record_ttl: The record TTL (number of seconds that the record may be cached).
         :raises certbot.errors.PluginError: if an error occurs communicating with the DNS server
         """
 
-        domain = self._find_domain(domain_name)
+        domain = self._find_domain(record_name)
 
         n = dns.name.from_text(record_name)
         o = dns.name.from_text(domain)
@@ -119,7 +122,7 @@ class _RFC2136Client(object):
         update.add(rel, record_ttl, dns.rdatatype.TXT, record_content)
 
         try:
-            response = dns.query.tcp(update, self.server)
+            response = dns.query.tcp(update, self.server, port=self.port)
         except Exception as e:
             raise errors.PluginError('Encountered error adding TXT record: {0}'
                                      .format(e))
@@ -131,18 +134,17 @@ class _RFC2136Client(object):
             raise errors.PluginError('Received response from server: {0}'
                                      .format(dns.rcode.to_text(rcode)))
 
-    def del_txt_record(self, domain_name, record_name, record_content):
+    def del_txt_record(self, record_name, record_content):
         """
         Delete a TXT record using the supplied information.
 
-        :param str domain: The domain to use to find the closest SOA.
         :param str record_name: The record name (typically beginning with '_acme-challenge.').
         :param str record_content: The record content (typically the challenge validation).
         :param int record_ttl: The record TTL (number of seconds that the record may be cached).
         :raises certbot.errors.PluginError: if an error occurs communicating with the DNS server
         """
 
-        domain = self._find_domain(domain_name)
+        domain = self._find_domain(record_name)
 
         n = dns.name.from_text(record_name)
         o = dns.name.from_text(domain)
@@ -155,7 +157,7 @@ class _RFC2136Client(object):
         update.delete(rel, dns.rdatatype.TXT, record_content)
 
         try:
-            response = dns.query.tcp(update, self.server)
+            response = dns.query.tcp(update, self.server, port=self.port)
         except Exception as e:
             raise errors.PluginError('Encountered error deleting TXT record: {0}'
                                      .format(e))
@@ -167,17 +169,17 @@ class _RFC2136Client(object):
             raise errors.PluginError('Received response from server: {0}'
                                      .format(dns.rcode.to_text(rcode)))
 
-    def _find_domain(self, domain_name):
+    def _find_domain(self, record_name):
         """
         Find the closest domain with an SOA record for a given domain name.
 
-        :param str domain_name: The domain name for which to find the closest SOA record.
+        :param str record_name: The record name for which to find the closest SOA record.
         :returns: The domain, if found.
         :rtype: str
         :raises certbot.errors.PluginError: if no SOA record can be found.
         """
 
-        domain_name_guesses = dns_common.base_domain_name_guesses(domain_name)
+        domain_name_guesses = dns_common.base_domain_name_guesses(record_name)
 
         # Loop through until we find an authoritative SOA record
         for guess in domain_name_guesses:
@@ -185,7 +187,7 @@ class _RFC2136Client(object):
                 return guess
 
         raise errors.PluginError('Unable to determine base domain for {0} using names: {1}.'
-                                 .format(domain_name, domain_name_guesses))
+                                 .format(record_name, domain_name_guesses))
 
     def _query_soa(self, domain_name):
         """
@@ -204,7 +206,7 @@ class _RFC2136Client(object):
         request.flags ^= dns.flags.RD
 
         try:
-            response = dns.query.udp(request, self.server)
+            response = dns.query.udp(request, self.server, port=self.port)
             rcode = response.rcode()
 
             # Authoritative Answer bit should be set