Add acme-v2 branches to Travis.

This will be required in order to implement the latest ACME spec, which uses
these protected header fields.

.travis.yml

@@ -67,6 +67,8 @@ branches:
     - master
     - /^\d+\.\d+\.x$/
     - /^test-.*$/
+    - acme-v2
+    - acme-v2-integration
 
 # container-based infrastructure
 sudo: false

acme/acme/client.py

@@ -10,6 +10,7 @@ import time
 import six
 from six.moves import http_client  # pylint: disable=import-error
 
+import crypto_util
 import josepy as jose
 import OpenSSL
 import re
@@ -48,6 +49,8 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
 
     :ivar messages.Directory directory:
     :ivar key: `.JWK` (private)
+    :ivar account: `.Account` (private)
+    :ivar acme_version: `int` (private) 1 for ACMEv1, or 2 for ACMEv2.
     :ivar alg: `.JWASignature`
     :ivar bool verify_ssl: Verify SSL certificates?
     :ivar .ClientNetwork net: Client network. Useful for testing. If not
@@ -56,8 +59,8 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
 
     """
 
-    def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True,
-                 net=None):
+    def __init__(self, directory, key, account=None, acme_version=1, alg=jose.RS256,
+                 verify_ssl=True, net=None):
         """Initialize.
 
         :param directory: Directory Resource (`.messages.Directory`) or
@@ -65,7 +68,10 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
 
         """
         self.key = key
-        self.net = ClientNetwork(key, alg, verify_ssl) if net is None else net
+        self.account = account
+        self.acme_version = acme_version
+        self.net = ClientNetwork(key, account=account, acme_version=acme_version,
+            alg=alg, verify_ssl=verify_ssl) if net is None else net
 
         if isinstance(directory, six.string_types):
             self.directory = messages.Directory.from_json(
@@ -93,9 +99,12 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
 
         """
         new_reg = messages.NewRegistration() if new_reg is None else new_reg
-        assert isinstance(new_reg, messages.NewRegistration)
+        if hasattr(self.directory, 'new_account'):
+            url = self.directory.new_account
+        else:
+            url = self.directory.new_reg
 
-        response = self.net.post(self.directory[new_reg], new_reg)
+        response = self.net.post(url, new_reg)
         # TODO: handle errors
         assert response.status_code == http_client.CREATED
 
@@ -168,14 +177,48 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
         return self.update_registration(
             regr.update(body=regr.body.update(agreement=regr.terms_of_service)))
 
-    def _authzr_from_response(self, response, identifier, uri=None):
+    def _authzr_from_response(self, response, identifier=None, uri=None):
         authzr = messages.AuthorizationResource(
             body=messages.Authorization.from_json(response.json()),
             uri=response.headers.get('Location', uri))
-        if authzr.body.identifier != identifier:
+        if identifier is not None and authzr.body.identifier != identifier:
             raise errors.UnexpectedUpdate(authzr)
         return authzr
 
+    def _order_resource_from_response(self, response, uri=None, csr=None):
+        body = messages.Order.from_json(response.json())
+        authorizations = []
+        for url in body.authorizations:
+            authorizations.append(self._authzr_from_response(self.net.get(url)))
+        fullchain_pem = None
+        if body.certificate is not None:
+            certificate_response = self.net.get(body.certificate, content_type=None)
+            if certificate_response.ok:
+                fullchain_pem = certificate_response.text
+        return messages.OrderResource(
+            body=body,
+            uri=response.headers.get('Location', uri),
+            fullchain_pem=fullchain_pem,
+            authorizations=authorizations,
+            csr=csr)
+
+    def new_order(self, csr_pem):
+        """Request challenges.
+
+        :returns: List of Authorization Resources.
+        :rtype: `list` of `.AuthorizationResource`
+        """
+        csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)
+        wrapped_csr = jose.ComparableX509(csr)
+        identifiers = []
+        for name in crypto_util._pyopenssl_cert_or_req_san(csr):
+            identifiers.append(messages.Identifier(typ=messages.IDENTIFIER_FQDN,
+                value=name))
+        order = messages.NewOrder(identifiers=identifiers)
+        response = self.net.post(self.directory.new_order, order)
+        order_response = self._order_resource_from_response(response, csr=wrapped_csr)
+        return order_response
+
     def request_challenges(self, identifier, new_authzr_uri=None):
         """Request challenges.
 
@@ -227,18 +270,15 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
         :raises .UnexpectedUpdate:
 
         """
-        response = self.net.post(challb.uri, response)
+        challr = messages.ChallengeResource(body=challb)
+        response = self.net.post(challr.uri, response)
         try:
             authzr_uri = response.links['up']['url']
         except KeyError:
             raise errors.ClientError('"up" Link header missing')
-        challr = messages.ChallengeResource(
+        return messages.ChallengeResource(
             authzr_uri=authzr_uri,
             body=messages.ChallengeBody.from_json(response.json()))
-        # TODO: check that challr.uri == response.headers['Location']?
-        if challr.uri != challb.uri:
-            raise errors.UnexpectedUpdate(challr.uri)
-        return challr
 
     @classmethod
     def retry_after(cls, response, default):
@@ -288,6 +328,43 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
             response, authzr.body.identifier, authzr.uri)
         return updated_authzr, response
 
+    def poll_order_and_request_issuance(self, orderr, max_time=datetime.timedelta(seconds=90)):
+        """Poll Order Resource for status.
+
+        :param orderr: Order Resource
+        :type orderr: `.OrderResource`
+
+        :returns: Updated Order Resource
+
+        :rtype: (`.OrderResource`)
+
+        """
+        responses = []
+        deadline = datetime.datetime.now() + max_time
+        for url in orderr.body.authorizations:
+            while datetime.datetime.now() < deadline:
+                time.sleep(1)
+                authzr = self._authzr_from_response(self.net.get(url), uri=url)
+                if authzr.body.status != messages.STATUS_PENDING:
+                    responses.append(authzr)
+                    break
+        for authzr in responses:
+            if authzr.body.status != messages.STATUS_VALID:
+                for chall in authzr.body.challenges:
+                    if chall.error != None:
+                        raise Exception("failed challenge for %s: %s" %
+                            (authzr.body.identifier.value, chall.error))
+                raise Exception("failed authorization: %s" % authzr.body)
+        latest = self._order_resource_from_response(self.net.get(orderr.uri), uri=orderr.uri)
+        self.net.post(latest.body.finalize, messages.CertificateRequest(csr=orderr.csr))
+        while datetime.datetime.now() < deadline:
+            time.sleep(1)
+            latest = self._order_resource_from_response(self.net.get(orderr.uri), uri=orderr.uri)
+            if latest.fullchain_pem is not None:
+               return latest
+        return None
+
+
     def request_issuance(self, csr, authzrs):
         """Request issuance.
 
@@ -509,15 +586,18 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
     JSON_ERROR_CONTENT_TYPE = 'application/problem+json'
     REPLAY_NONCE_HEADER = 'Replay-Nonce'
 
-    def __init__(self, key, alg=jose.RS256, verify_ssl=True,
-                 user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT):
+    def __init__(self, key, account=None, acme_version=1, alg=jose.RS256,
+                 verify_ssl=True, user_agent='acme-python',
+                 timeout=DEFAULT_NETWORK_TIMEOUT):
         self.key = key
+        self.account = account
         self.alg = alg
         self.verify_ssl = verify_ssl
         self._nonces = set()
         self.user_agent = user_agent
         self.session = requests.Session()
         self._default_timeout = timeout
+        self.acme_version = acme_version
 
     def __del__(self):
         # Try to close the session, but don't show exceptions to the
@@ -527,7 +607,7 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
         except Exception:  # pylint: disable=broad-except
             pass
 
-    def _wrap_in_jws(self, obj, nonce):
+    def _wrap_in_jws(self, obj, nonce, url):
         """Wrap `JSONDeSerializable` object in JWS.
 
         .. todo:: Implement ``acmePath``.
@@ -539,9 +619,17 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
         """
         jobj = obj.json_dumps(indent=2).encode()
         logger.debug('JWS payload:\n%s', jobj)
-        return jws.JWS.sign(
-            payload=jobj, key=self.key, alg=self.alg,
-            nonce=nonce).json_dumps(indent=2)
+        kwargs = {
+            "alg": self.alg,
+            "nonce": nonce
+        }
+        if self.acme_version is 2:
+            # new ACME spec
+            kwargs["url"] = url
+            if self.account is not None:
+                kwargs["kid"] = self.account["uri"]
+        kwargs["key"] = self.key
+        return jws.JWS.sign(jobj, **kwargs).json_dumps(indent=2)
 
     @classmethod
     def _check_response(cls, response, content_type=None):
@@ -715,7 +803,7 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
                 raise
 
     def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, **kwargs):
-        data = self._wrap_in_jws(obj, self._get_nonce(url))
+        data = self._wrap_in_jws(obj, self._get_nonce(url), url)
         kwargs.setdefault('headers', {'Content-Type': content_type})
         response = self._send_request('POST', url, data=data, **kwargs)
         self._add_nonce(response)

acme/acme/client_test.py

@@ -467,11 +467,21 @@ class ClientNetworkTest(unittest.TestCase):
 
         # pylint: disable=protected-access
         jws_dump = self.net._wrap_in_jws(
-            MockJSONDeSerializable('foo'), nonce=b'Tg')
+            MockJSONDeSerializable('foo'), nonce=b'Tg', url="url")
         jws = acme_jws.JWS.json_loads(jws_dump)
         self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'})
         self.assertEqual(jws.signature.combined.nonce, b'Tg')
 
+        self.net.account = {'uri': 'acct-uri'}
+        jws_dump = self.net._wrap_in_jws(
+            MockJSONDeSerializable('foo'), nonce=b'Tg', url="url")
+        jws = acme_jws.JWS.json_loads(jws_dump)
+        self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'})
+        self.assertEqual(jws.signature.combined.nonce, b'Tg')
+        self.assertEqual(jws.signature.combined.kid, u'acct-uri')
+        self.assertEqual(jws.signature.combined.url, u'url')
+
+
     def test_check_response_not_ok_jobj_no_error(self):
         self.response.ok = False
         self.response.json.return_value = {}
@@ -701,13 +711,13 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
         self.assertEqual(self.checked_response, self.net.post(
             'uri', self.obj, content_type=self.content_type))
         self.net._wrap_in_jws.assert_called_once_with(
-            self.obj, jose.b64decode(self.all_nonces.pop()))
+            self.obj, jose.b64decode(self.all_nonces.pop()), "uri")
 
         self.available_nonces = []
         self.assertRaises(errors.MissingNonce, self.net.post,
                           'uri', self.obj, content_type=self.content_type)
         self.net._wrap_in_jws.assert_called_with(
-            self.obj, jose.b64decode(self.all_nonces.pop()))
+            self.obj, jose.b64decode(self.all_nonces.pop()), "uri")
 
     def test_post_wrong_initial_nonce(self):  # HEAD
         self.available_nonces = [b'f', jose.b64encode(b'good')]

acme/acme/messages.py

@@ -1,5 +1,6 @@
 """ACME protocol messages."""
 import collections
+import re
 import six
 
 import josepy as jose
@@ -171,9 +172,9 @@ class Directory(jose.JSONDeSerializable):
 
     class Meta(jose.JSONObjectWithFields):
         """Directory Meta."""
-        terms_of_service = jose.Field('terms-of-service', omitempty=True)
+        terms_of_service = jose.Field('termsOfService', omitempty=True)
         website = jose.Field('website', omitempty=True)
-        caa_identities = jose.Field('caa-identities', omitempty=True)
+        caa_identities = jose.Field('caaIdentities', omitempty=True)
 
     @classmethod
     def _canon_key(cls, key):
@@ -193,11 +194,19 @@ class Directory(jose.JSONDeSerializable):
         # not clear on that
         self._jobj = canon_jobj
 
+    def _camelCase(self, name):
+        """Convert a snake_case name to camelCase."""
+        return re.sub('_([a-z])', lambda x: x.group(1).upper(),  name)
+
     def __getattr__(self, name):
         try:
             return self[name.replace('_', '-')]
         except KeyError as error:
-            raise AttributeError(str(error) + ': ' + name)
+            try:
+                print self._camelCase(name)
+                return self[self._camelCase(name)]
+            except KeyError as error:
+                raise AttributeError(str(error) + ': ' + name)
 
     def __getitem__(self, name):
         try:
@@ -251,6 +260,7 @@ class Registration(ResourceBody):
     contact = jose.Field('contact', omitempty=True, default=())
     agreement = jose.Field('agreement', omitempty=True)
     status = jose.Field('status', omitempty=True)
+    terms_of_service_agreed = jose.Field('termsOfServiceAgreed', omitempty=True)
 
     phone_prefix = 'tel:'
     email_prefix = 'mailto:'
@@ -299,12 +309,10 @@ class RegistrationResource(ResourceWithURI):
     """Registration Resource.
 
     :ivar acme.messages.Registration body:
-    :ivar unicode new_authzr_uri: Deprecated. Do not use.
     :ivar unicode terms_of_service: URL for the CA TOS.
 
     """
     body = jose.Field('body', decoder=Registration.from_json)
-    new_authzr_uri = jose.Field('new_authzr_uri', omitempty=True)
     terms_of_service = jose.Field('terms_of_service', omitempty=True)
 
 
@@ -482,3 +490,37 @@ class Revocation(jose.JSONObjectWithFields):
     certificate = jose.Field(
         'certificate', decoder=jose.decode_cert, encoder=jose.encode_cert)
     reason = jose.Field('reason')
+
+
+class Order(ResourceBody):
+    """Order Resource Body.
+
+    :ivar buffer csr: CSR in pem format.
+    :ivar string status:
+    :ivar list of string authorizations:
+    :ivar datetime.datetime expires:
+
+    """
+    identifiers = jose.Field('identifiers', omitempty=True)
+    status = jose.Field('status', omitempty=True, default=None)
+    authorizations = jose.Field('authorizations', omitempty=True)
+    certificate = jose.Field('certificate', omitempty=True)
+    finalize = jose.Field('finalize', omitempty=True)
+    expires = fields.RFC3339Field('expires', omitempty=True)
+
+class OrderResource(ResourceWithURI):
+    """Order Resource.
+
+    :ivar acme.messages.Order body:
+
+    """
+    body = jose.Field('body', decoder=Order.from_json)
+    csr = jose.Field('csr', omitempty=True)
+    authorizations = jose.Field('authorizations')
+    fullchain_pem = jose.Field('fullchain_pem', omitempty=True)
+
+@Directory.register
+class NewOrder(Order):
+    """New order."""
+    resource_type = 'new-order'
+    resource = fields.Resource(resource_type)

acme/acme/messages_test.py

@@ -292,6 +292,11 @@ class ChallengeBodyTest(unittest.TestCase):
     def test_from_json(self):
         from acme.messages import ChallengeBody
         self.assertEqual(self.challb, ChallengeBody.from_json(self.jobj_from))
+        jobj_from_cpy = self.jobj_from
+        jobj_from_cpy["url"] = "http://challb"
+        del(jobj_from_cpy["uri"])
+        self.assertEqual(self.challb, ChallengeBody.from_json(jobj_from_cpy))
+
 
     def test_from_json_hashable(self):
         from acme.messages import ChallengeBody

chisel2.py

@@ -0,0 +1,216 @@
+"""
+A simple client that uses the Python ACME library to run a test issuance against
+a local Boulder server. Unlike chisel.py this version implements the most recent
+version of the ACME specification. Usage:
+
+$ virtualenv venv
+$ . venv/bin/activate
+$ pip install -r requirements.txt
+$ python chisel.py foo.com bar.com
+"""
+import json
+import logging
+import os
+import sys
+import signal
+import threading
+import time
+import urllib2
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography import x509
+from cryptography.hazmat.primitives import hashes
+
+import OpenSSL
+import josepy
+
+from acme import challenges
+from acme import client as acme_client
+from acme import crypto_util as acme_crypto_util
+from acme import errors as acme_errors
+from acme import messages
+from acme import standalone
+
+logging.basicConfig()
+logger = logging.getLogger()
+logger.setLevel(int(os.getenv('LOGLEVEL', 0)))
+
+DIRECTORY = os.getenv('DIRECTORY', 'http://localhost:4001/directory')
+
+def make_client(email=None):
+    """Build an acme.Client and register a new account with a random key."""
+    key = josepy.JWKRSA(key=rsa.generate_private_key(65537, 2048, default_backend()))
+
+    net = acme_client.ClientNetwork(key, acme_version=2,
+                                    user_agent="Boulder integration tester")
+
+    client = acme_client.Client(DIRECTORY, key=key, net=net, acme_version=2)
+    tos = client.directory.meta.terms_of_service
+    if tos is not None and "Do%20what%20thou%20wilt" in tos:
+        net.account = client.register(messages.NewRegistration.from_data(email=email,
+            terms_of_service_agreed=True))
+    else:
+        raise Exception("Unrecognized terms of service URL %s" % tos)
+    return client
+
+def get_chall(authz, typ):
+    for chall_body in authz.body.challenges:
+        if isinstance(chall_body.chall, typ):
+            return chall_body
+    raise "No %s challenge found" % typ
+
+class ValidationError(Exception):
+    """An error that occurs during challenge validation."""
+    def __init__(self, domain, problem_type, detail, *args, **kwargs):
+        self.domain = domain
+        self.problem_type = problem_type
+        self.detail = detail
+
+    def __str__(self):
+        return "%s: %s: %s" % (self.domain, self.problem_type, self.detail)
+
+def make_csr(domains):
+    key = OpenSSL.crypto.PKey()
+    key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
+    pem = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key)
+    return acme_crypto_util.make_csr(pem, domains, False)
+
+def issue(client, authzs, cert_output=None):
+    """Given a list of authzs that are being processed by the server,
+       wait for them to be ready, then request issuance of a cert with a random
+       key for the given domains.
+
+       If cert_output is provided, write the cert as a PEM file to that path."""
+    csr = make_csr([authz.body.identifier.value for authz in authzs])
+
+    cert_resource = None
+    try:
+        cert_resource, _ = client.poll_and_request_issuance(jose.ComparableX509(csr), authzs)
+    except acme_errors.PollError as error:
+        # If we get a PollError, pick the first failed authz and turn it into a more
+        # useful ValidationError that contains details we can look for in tests.
+        for authz in error.updated:
+            updated_authz = json.loads(urllib2.urlopen(authz.uri).read())
+            domain = authz.body.identifier.value,
+            for c in updated_authz['challenges']:
+                if 'error' in c:
+                    err = c['error']
+                    raise ValidationError(domain, err['type'], err['detail'])
+        # If none of the authz's had an error, just re-raise.
+        raise
+    if cert_output is not None:
+        pem = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM,
+                                              cert_resource.body)
+        with open(cert_output, 'w') as f:
+            f.write(pem)
+    return cert_resource
+
+def http_01_answer(client, chall_body):
+    """Return an HTTP01Resource to server in response to the given challenge."""
+    response, validation = chall_body.response_and_validation(client.key)
+    return standalone.HTTP01RequestHandler.HTTP01Resource(
+          chall=chall_body.chall, response=response,
+          validation=validation)
+
+def auth_and_issue(domains, chall_type="http-01", email=None, cert_output=None, client=None):
+    """Make authzs for each of the given domains, set up a server to answer the
+       challenges in those authzs, tell the ACME server to validate the challenges,
+       then poll for the authzs to be ready and issue a cert."""
+    if client is None:
+        client = make_client(email)
+
+    csr_pem = make_csr(domains)
+    order = client.new_order(csr_pem)
+    authzs = order.authorizations
+
+    if chall_type == "http-01":
+        cleanup = do_http_challenges(client, authzs)
+    else:
+        raise Exception("invalid challenge type %s" % chall_type)
+
+    try:
+        order = client.poll_order_and_request_issuance(order)
+        print(order.fullchain_pem)
+    finally:
+        cleanup()
+
+def do_dns_challenges(client, authzs):
+    for a in authzs:
+        c = get_chall(a, challenges.DNS01)
+        name, value = (c.validation_domain_name(a.body.identifier.value),
+            c.validation(client.key))
+        urllib2.urlopen("http://localhost:8055/set-txt",
+            data=json.dumps({
+                "host": name + ".",
+                "value": value,
+            })).read()
+        client.answer_challenge(c, c.response(client.key))
+    def cleanup():
+        pass
+    return cleanup
+
+def do_http_challenges(client, authzs):
+    port = 5002
+    challs = [get_chall(a, challenges.HTTP01) for a in authzs]
+    answers = set([http_01_answer(client, c) for c in challs])
+    server = standalone.HTTP01Server(("", port), answers)
+    thread = threading.Thread(target=server.serve_forever)
+    thread.start()
+
+    # cleanup has to be called on any exception, or when validation is done.
+    # Otherwise the process won't terminate.
+    def cleanup():
+        server.shutdown()
+        server.server_close()
+        thread.join()
+
+    try:
+        # Loop until the HTTP01Server is ready.
+        while True:
+            try:
+                urllib2.urlopen("http://localhost:%d" % port)
+                break
+            except urllib2.URLError:
+                time.sleep(0.1)
+
+        for chall_body in challs:
+            client.answer_challenge(chall_body, chall_body.response(client.key))
+    except Exception:
+        cleanup()
+        raise
+
+    return cleanup
+
+def expect_problem(problem_type, func):
+    """Run a function. If it raises a ValidationError or messages.Error that
+       contains the given problem_type, return. If it raises no error or the wrong
+       error, raise an exception."""
+    ok = False
+    try:
+        func()
+    except ValidationError as e:
+        if e.problem_type == problem_type:
+            ok = True
+        else:
+            raise
+    except messages.Error as e:
+        if problem_type in e.__str__():
+            ok = True
+        else:
+            raise
+    if not ok:
+        raise Exception('Expected %s, got no error' % problem_type)
+
+if __name__ == "__main__":
+    # Die on SIGINT
+    signal.signal(signal.SIGINT, signal.SIG_DFL)
+    domains = sys.argv[1:]
+    if len(domains) == 0:
+        print __doc__
+        sys.exit(0)
+    try:
+        auth_and_issue(domains)
+    except messages.Error, e:
+        print e
+        sys.exit(1)

tools/pip_install.sh

@@ -14,4 +14,4 @@ dev_constraints="$(dirname $my_path)/pip_constraints.txt"
 set -x
 
 # install the requested packages using the pinned requirements as constraints
-pip install -q --constraint $certbot_auto_constraints --constraint $dev_constraints "$@"
+pip install --constraint $certbot_auto_constraints --constraint $dev_constraints "$@"