disable not-callable for iter_entry_points

* use entire achall as key

* Add manual cleanup hook

* use manual cleanup hook

.travis.yml

@@ -30,7 +30,7 @@ matrix:
     - python: "2.7"
       env: TOXENV=lint
     - python: "2.7"
-      env: TOXENV=py27-oldest
+      env: TOXENV='py27-{acme,apache,certbot,dns,nginx}-oldest'
       sudo: required
       services: docker
     - python: "3.4"

acme/acme/client.py

@@ -809,8 +809,8 @@ class BackwardsCompatibleClientV2(object):
                     'certificate, please rerun the command for a new one.')
 
             cert = OpenSSL.crypto.dump_certificate(
-                    OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped)
-            chain = crypto_util.dump_pyopenssl_chain(chain)
+                    OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped).decode()
+            chain = crypto_util.dump_pyopenssl_chain(chain).decode()
 
             return orderr.update(fullchain_pem=(cert + chain))
         else:

acme/acme/client_test.py

@@ -99,10 +99,10 @@ class BackwardsCompatibleClientV2Test(ClientTestBase):
         self.chain = [wrapped, wrapped]
 
         self.cert_pem = OpenSSL.crypto.dump_certificate(
-            OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped)
+            OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped).decode()
 
         single_chain = OpenSSL.crypto.dump_certificate(
-            OpenSSL.crypto.FILETYPE_PEM, loaded)
+            OpenSSL.crypto.FILETYPE_PEM, loaded).decode()
         self.chain_pem = single_chain + single_chain
 
         self.fullchain_pem = self.cert_pem + self.chain_pem

acme/acme/crypto_util.py

@@ -287,6 +287,9 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM):
     :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in
         :class:`josepy.util.ComparableX509`).
 
+    :returns: certificate chain bundle
+    :rtype: bytes
+
     """
     # XXX: returns empty string when no chain is available, which
     # shuts up RenewableCert, but might not be the best solution...

certbot-apache/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-apache/setup.py

@@ -6,10 +6,11 @@ from setuptools import find_packages
 
 version = '0.22.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
     'mock',
     'python-augeas',
     'setuptools',

certbot-dns-cloudflare/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-cloudflare
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-cloudflare

certbot-dns-cloudflare/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-cloudflare/setup.py

@@ -6,10 +6,11 @@ from setuptools import find_packages
 
 version = '0.22.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
     'cloudflare>=1.5.1',
     'mock',
     'setuptools',

certbot-dns-cloudxns/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-cloudxns
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-cloudxns

certbot-dns-cloudxns/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-cloudxns/setup.py

@@ -6,10 +6,11 @@ from setuptools import find_packages
 
 version = '0.22.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
     'dns-lexicon',
     'mock',
     'setuptools',

certbot-dns-digitalocean/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-digitalocean
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-digitalocean

certbot-dns-digitalocean/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-digitalocean/setup.py

@@ -6,10 +6,11 @@ from setuptools import find_packages
 
 version = '0.22.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
     'mock',
     'python-digitalocean>=1.11',
     'setuptools',

certbot-dns-dnsimple/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-dnsimple
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-dnsimple

certbot-dns-dnsimple/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-dnsimple/setup.py

@@ -6,10 +6,11 @@ from setuptools import find_packages
 
 version = '0.22.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
     'dns-lexicon',
     'mock',
     'setuptools',

certbot-dns-dnsmadeeasy/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-dnsmadeeasy
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-dnsmadeeasy

certbot-dns-dnsmadeeasy/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-dnsmadeeasy/setup.py

@@ -6,10 +6,11 @@ from setuptools import find_packages
 
 version = '0.22.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
     'dns-lexicon',
     'mock',
     'setuptools',

certbot-dns-google/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-google
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-google

certbot-dns-google/certbot_dns_google/dns_google.py

@@ -107,6 +107,15 @@ class _GoogleClient(object):
 
         zone_id = self._find_managed_zone_id(domain)
 
+        record_contents = self.get_existing_txt_rrset(zone_id, record_name)
+        add_records = record_contents[:]
+
+        if "\""+record_content+"\"" in record_contents:
+            # The process was interrupted previously and validation token exists
+            return
+
+        add_records.append(record_content)
+
         data = {
             "kind": "dns#change",
             "additions": [
@@ -114,12 +123,24 @@ class _GoogleClient(object):
                     "kind": "dns#resourceRecordSet",
                     "type": "TXT",
                     "name": record_name + ".",
-                    "rrdatas": [record_content, ],
+                    "rrdatas": add_records,
                     "ttl": record_ttl,
                 },
             ],
         }
 
+        if record_contents:
+            # We need to remove old records in the same request
+            data["deletions"] = [
+                {
+                    "kind": "dns#resourceRecordSet",
+                    "type": "TXT",
+                    "name": record_name + ".",
+                    "rrdatas": record_contents,
+                    "ttl": record_ttl,
+                },
+            ]
+
         changes = self.dns.changes()  # changes | pylint: disable=no-member
 
         try:
@@ -154,6 +175,8 @@ class _GoogleClient(object):
             logger.warn('Error finding zone. Skipping cleanup.')
             return
 
+        record_contents = self.get_existing_txt_rrset(zone_id, record_name)
+
         data = {
             "kind": "dns#change",
             "deletions": [
@@ -161,12 +184,26 @@ class _GoogleClient(object):
                     "kind": "dns#resourceRecordSet",
                     "type": "TXT",
                     "name": record_name + ".",
-                    "rrdatas": [record_content, ],
+                    "rrdatas": record_contents,
                     "ttl": record_ttl,
                 },
             ],
         }
 
+        # Remove the record being deleted from the list
+        readd_contents = [r for r in record_contents if r != "\"" + record_content + "\""]
+        if readd_contents:
+            # We need to remove old records in the same request
+            data["additions"] = [
+                {
+                    "kind": "dns#resourceRecordSet",
+                    "type": "TXT",
+                    "name": record_name + ".",
+                    "rrdatas": readd_contents,
+                    "ttl": record_ttl,
+                },
+            ]
+
         changes = self.dns.changes()  # changes | pylint: disable=no-member
 
         try:
@@ -175,6 +212,28 @@ class _GoogleClient(object):
         except googleapiclient_errors.Error as e:
             logger.warn('Encountered error deleting TXT record: %s', e)
 
+    def get_existing_txt_rrset(self, zone_id, record_name):
+        """
+        Get existing TXT records from the RRset for the record name.
+
+        :param str zone_id: The ID of the managed zone.
+        :param str record_name: The record name (typically beginning with '_acme-challenge.').
+
+        :returns: List of TXT record values
+        :rtype: `list` of `string`
+
+        """
+        rrs_request = self.dns.resourceRecordSets()  # pylint: disable=no-member
+        request = rrs_request.list(managedZone=zone_id, project=self.project_id)
+        response = request.execute()
+        # Add dot as the API returns absolute domains
+        record_name += "."
+        if response:
+            for rr in response["rrsets"]:
+                if rr["name"] == record_name and rr["type"] == "TXT":
+                    return rr["rrdatas"]
+        return []
+
     def _find_managed_zone_id(self, domain):
         """
         Find the managed zone for a given domain.

certbot-dns-google/certbot_dns_google/dns_google_test.py

@@ -74,10 +74,15 @@ class GoogleClientTest(unittest.TestCase):
         mock_mz = mock.MagicMock()
         mock_mz.list.return_value.execute.side_effect = zone_request_side_effect
 
+        mock_rrs = mock.MagicMock()
+        rrsets = {"rrsets": [{"name": "_acme-challenge.example.org.", "type": "TXT",
+                              "rrdatas": ["\"example-txt-contents\""]}]}
+        mock_rrs.list.return_value.execute.return_value = rrsets
         mock_changes = mock.MagicMock()
 
         client.dns.managedZones = mock.MagicMock(return_value=mock_mz)
         client.dns.changes = mock.MagicMock(return_value=mock_changes)
+        client.dns.resourceRecordSets = mock.MagicMock(return_value=mock_rrs)
 
         return client, mock_changes
 
@@ -137,6 +142,30 @@ class GoogleClientTest(unittest.TestCase):
                                             managedZone=self.zone,
                                             project=PROJECT_ID)
 
+    @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name')
+    @mock.patch('certbot_dns_google.dns_google.open',
+                mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True)
+    def test_add_txt_record_delete_old(self, unused_credential_mock):
+        client, changes = self._setUp_client_with_mock(
+            [{'managedZones': [{'id': self.zone}]}])
+        mock_get_rrs = "certbot_dns_google.dns_google._GoogleClient.get_existing_txt_rrset"
+        with mock.patch(mock_get_rrs) as mock_rrs:
+            mock_rrs.return_value = ["sample-txt-contents"]
+            client.add_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl)
+            self.assertTrue(changes.create.called)
+            self.assertTrue("sample-txt-contents" in
+                changes.create.call_args_list[0][1]["body"]["deletions"][0]["rrdatas"])
+
+    @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name')
+    @mock.patch('certbot_dns_google.dns_google.open',
+                mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True)
+    def test_add_txt_record_noop(self, unused_credential_mock):
+        client, changes = self._setUp_client_with_mock(
+            [{'managedZones': [{'id': self.zone}]}])
+        client.add_txt_record(DOMAIN, "_acme-challenge.example.org",
+                              "example-txt-contents", self.record_ttl)
+        self.assertFalse(changes.create.called)
+
     @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name')
     @mock.patch('certbot_dns_google.dns_google.open',
                 mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True)
@@ -172,7 +201,12 @@ class GoogleClientTest(unittest.TestCase):
     def test_del_txt_record(self, unused_credential_mock):
         client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}])
 
-        client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl)
+        mock_get_rrs = "certbot_dns_google.dns_google._GoogleClient.get_existing_txt_rrset"
+        with mock.patch(mock_get_rrs) as mock_rrs:
+            mock_rrs.return_value = ["\"sample-txt-contents\"",
+                                     "\"example-txt-contents\""]
+            client.del_txt_record(DOMAIN, "_acme-challenge.example.org",
+                                "example-txt-contents", self.record_ttl)
 
         expected_body = {
             "kind": "dns#change",
@@ -180,8 +214,17 @@ class GoogleClientTest(unittest.TestCase):
                 {
                     "kind": "dns#resourceRecordSet",
                     "type": "TXT",
-                    "name": self.record_name + ".",
-                    "rrdatas": [self.record_content, ],
+                    "name": "_acme-challenge.example.org.",
+                    "rrdatas": ["\"sample-txt-contents\"", "\"example-txt-contents\""],
+                    "ttl": self.record_ttl,
+                },
+            ],
+            "additions": [
+                {
+                    "kind": "dns#resourceRecordSet",
+                    "type": "TXT",
+                    "name": "_acme-challenge.example.org.",
+                    "rrdatas": ["\"sample-txt-contents\"", ],
                     "ttl": self.record_ttl,
                 },
             ],
@@ -217,6 +260,18 @@ class GoogleClientTest(unittest.TestCase):
 
         client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl)
 
+    @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name')
+    @mock.patch('certbot_dns_google.dns_google.open',
+                mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True)
+    def test_get_existing(self, unused_credential_mock):
+        client, unused_changes = self._setUp_client_with_mock(
+            [{'managedZones': [{'id': self.zone}]}])
+        # Record name mocked in setUp
+        found = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org")
+        self.assertEquals(found, ["\"example-txt-contents\""])
+        not_found = client.get_existing_txt_rrset(self.zone, "nonexistent.tld")
+        self.assertEquals(not_found, [])
+
     def test_get_project_id(self):
         from certbot_dns_google.dns_google import _GoogleClient
 

certbot-dns-google/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-google/setup.py

@@ -6,10 +6,11 @@ from setuptools import find_packages
 
 version = '0.22.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
     # 1.5 is the first version that supports oauth2client>=2.0
     'google-api-python-client>=1.5',
     'mock',

certbot-dns-luadns/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-luadns
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-luadns

certbot-dns-luadns/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-luadns/setup.py

@@ -6,10 +6,11 @@ from setuptools import find_packages
 
 version = '0.22.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
     'dns-lexicon',
     'mock',
     'setuptools',

certbot-dns-nsone/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-nsone
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-nsone

certbot-dns-nsone/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-nsone/setup.py

@@ -6,10 +6,11 @@ from setuptools import find_packages
 
 version = '0.22.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
     'dns-lexicon',
     'mock',
     'setuptools',

certbot-dns-rfc2136/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-rfc2136
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-rfc2136

certbot-dns-rfc2136/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-rfc2136/setup.py

@@ -6,10 +6,11 @@ from setuptools import find_packages
 
 version = '0.22.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
     'dnspython',
     'mock',
     'setuptools',

certbot-dns-route53/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-route53
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-route53

certbot-dns-route53/certbot_dns_route53/dns_route53.py

@@ -1,4 +1,5 @@
 """Certbot Route53 authenticator plugin."""
+import collections
 import logging
 import time
 
@@ -33,6 +34,7 @@ class Authenticator(dns_common.DNSAuthenticator):
     def __init__(self, *args, **kwargs):
         super(Authenticator, self).__init__(*args, **kwargs)
         self.r53 = boto3.client("route53")
+        self._resource_records = collections.defaultdict(list)
 
     def more_info(self):  # pylint: disable=missing-docstring,no-self-use
         return "Solve a DNS01 challenge using AWS Route53"
@@ -88,6 +90,20 @@ class Authenticator(dns_common.DNSAuthenticator):
     def _change_txt_record(self, action, validation_domain_name, validation):
         zone_id = self._find_zone_id_for_domain(validation_domain_name)
 
+        rrecords = self._resource_records[validation_domain_name]
+        challenge = {"Value": '"{0}"'.format(validation)}
+        if action == "DELETE":
+            # Remove the record being deleted from the list of tracked records
+            rrecords.remove(challenge)
+            if rrecords:
+                # Need to update instead, as we're not deleting the rrset
+                action = "UPSERT"
+            else:
+                # Create a new list containing the record to use with DELETE
+                rrecords = [challenge]
+        else:
+            rrecords.append(challenge)
+
         response = self.r53.change_resource_record_sets(
             HostedZoneId=zone_id,
             ChangeBatch={
@@ -99,11 +115,7 @@ class Authenticator(dns_common.DNSAuthenticator):
                             "Name": validation_domain_name,
                             "Type": "TXT",
                             "TTL": self.ttl,
-                            "ResourceRecords": [
-                                # For some reason TXT records need to be
-                                # manually quoted.
-                                {"Value": '"{0}"'.format(validation)}
-                            ],
+                            "ResourceRecords": rrecords,
                         }
                     }
                 ]

certbot-dns-route53/certbot_dns_route53/dns_route53_test.py

@@ -186,6 +186,48 @@ class ClientTest(unittest.TestCase):
         call_count = self.client.r53.change_resource_record_sets.call_count
         self.assertEqual(call_count, 1)
 
+    def test_change_txt_record_delete(self):
+        self.client._find_zone_id_for_domain = mock.MagicMock()
+        self.client.r53.change_resource_record_sets = mock.MagicMock(
+            return_value={"ChangeInfo": {"Id": 1}})
+
+        validation = "some-value"
+        validation_record = {"Value": '"{0}"'.format(validation)}
+        self.client._resource_records[DOMAIN] = [validation_record]
+
+        self.client._change_txt_record("DELETE", DOMAIN, validation)
+
+        call_count = self.client.r53.change_resource_record_sets.call_count
+        self.assertEqual(call_count, 1)
+        call_args = self.client.r53.change_resource_record_sets.call_args_list[0][1]
+        call_args_batch = call_args["ChangeBatch"]["Changes"][0]
+        self.assertEqual(call_args_batch["Action"], "DELETE")
+        self.assertEqual(
+            call_args_batch["ResourceRecordSet"]["ResourceRecords"],
+            [validation_record])
+
+    def test_change_txt_record_multirecord(self):
+        self.client._find_zone_id_for_domain = mock.MagicMock()
+        self.client._get_validation_rrset = mock.MagicMock()
+        self.client._resource_records[DOMAIN] = [
+            {"Value": "\"pre-existing-value\""},
+            {"Value": "\"pre-existing-value-two\""},
+        ]
+        self.client.r53.change_resource_record_sets = mock.MagicMock(
+            return_value={"ChangeInfo": {"Id": 1}})
+
+        self.client._change_txt_record("DELETE", DOMAIN, "pre-existing-value")
+
+        call_count = self.client.r53.change_resource_record_sets.call_count
+        call_args = self.client.r53.change_resource_record_sets.call_args_list[0][1]
+        call_args_batch = call_args["ChangeBatch"]["Changes"][0]
+        self.assertEqual(call_args_batch["Action"], "UPSERT")
+        self.assertEqual(
+            call_args_batch["ResourceRecordSet"]["ResourceRecords"],
+            [{"Value": "\"pre-existing-value-two\""}])
+
+        self.assertEqual(call_count, 1)
+
     def test_wait_for_change(self):
         self.client.r53.get_change = mock.MagicMock(
             side_effect=[{"ChangeInfo": {"Status": "PENDING"}},

certbot-dns-route53/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-route53/setup.py

@@ -5,9 +5,11 @@ from setuptools import find_packages
 
 version = '0.22.0.dev0'
 
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
     'boto3',
     'mock',
     'setuptools',

certbot-nginx/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+-e acme[dev]
+-e .[dev]

certbot-nginx/setup.py

@@ -6,10 +6,14 @@ from setuptools import find_packages
 
 version = '0.22.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    # This plugin works with an older version of acme, but Certbot does not.
+    # 0.22.0 is specified here to work around
+    # https://github.com/pypa/pip/issues/988.
+    'acme>0.21.1',
+    'certbot>0.21.1',
     'mock',
     'PyOpenSSL',
     'pyparsing>=1.5.5',  # Python3 support; perhaps unnecessary?

certbot/client.py

@@ -244,7 +244,7 @@ class Client(object):
             than `authkey`.
         :param acme.messages.OrderResource orderr: contains authzrs
 
-        :returns: certificate and chain as PEM strings
+        :returns: certificate and chain as PEM byte strings
         :rtype: tuple
 
         """
@@ -263,7 +263,8 @@ class Client(object):
 
         deadline = datetime.datetime.now() + datetime.timedelta(seconds=90)
         orderr = self.acme.finalize_order(orderr, deadline)
-        return crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem)
+        cert, chain = crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem)
+        return cert.encode(), chain.encode()
 
     def obtain_certificate(self, domains):
         """Obtains a certificate from the ACME server.

certbot/crypto_util.py

@@ -441,8 +441,9 @@ def cert_and_chain_from_fullchain(fullchain_pem):
 
     :returns: tuple of string cert_pem and chain_pem
     :rtype: tuple
+
     """
     cert = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM,
-        OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem))
+        OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)).decode()
     chain = fullchain_pem[len(cert):]
     return (cert, chain)

certbot/plugins/disco.py

@@ -190,6 +190,7 @@ class PluginsRegistry(collections.Mapping):
     def find_all(cls):
         """Find plugins using setuptools entry points."""
         plugins = {}
+        # pylint: disable=not-callable
         entry_points = itertools.chain(
             pkg_resources.iter_entry_points(
                 constants.SETUPTOOLS_PLUGINS_ENTRY_POINT),

certbot/plugins/manual.py

@@ -189,7 +189,7 @@ when it receives a TLS ClientHello with the SNI extension set to
         os.environ.update(env)
         _, out = hooks.execute(self.conf('auth-hook'))
         env['CERTBOT_AUTH_OUTPUT'] = out.strip()
-        self.env[achall.domain] = env
+        self.env[achall] = env
 
     def _perform_achall_manually(self, achall):
         validation = achall.validation(achall.account_key)
@@ -215,7 +215,7 @@ when it receives a TLS ClientHello with the SNI extension set to
     def cleanup(self, achalls):  # pylint: disable=missing-docstring
         if self.conf('cleanup-hook'):
             for achall in achalls:
-                env = self.env.pop(achall.domain)
+                env = self.env.pop(achall)
                 if 'CERTBOT_TOKEN' not in env:
                     os.environ.pop('CERTBOT_TOKEN', None)
                 os.environ.update(env)

certbot/plugins/manual_test.py

@@ -93,10 +93,10 @@ class AuthenticatorTest(test_util.TempDirTestCase):
             self.auth.perform(self.achalls),
             [achall.response(achall.account_key) for achall in self.achalls])
         self.assertEqual(
-            self.auth.env[self.dns_achall.domain]['CERTBOT_AUTH_OUTPUT'],
+            self.auth.env[self.dns_achall]['CERTBOT_AUTH_OUTPUT'],
             dns_expected)
         self.assertEqual(
-            self.auth.env[self.http_achall.domain]['CERTBOT_AUTH_OUTPUT'],
+            self.auth.env[self.http_achall]['CERTBOT_AUTH_OUTPUT'],
             http_expected)
         # tls_sni_01 challenge must be perform()ed above before we can
         # get the cert_path and key_path.
@@ -107,7 +107,7 @@ class AuthenticatorTest(test_util.TempDirTestCase):
             self.auth.tls_sni_01.get_z_domain(self.tls_sni_achall),
             'novalidation')
         self.assertEqual(
-            self.auth.env[self.tls_sni_achall.domain]['CERTBOT_AUTH_OUTPUT'],
+            self.auth.env[self.tls_sni_achall]['CERTBOT_AUTH_OUTPUT'],
             tls_sni_expected)
 
     @test_util.patch_get_utility()

certbot/tests/client_test.py

@@ -132,7 +132,6 @@ class ClientTest(ClientTestCommon):
         self.eg_domains = ["example.com", "www.example.com"]
         self.eg_order = mock.MagicMock(
             authorizations=[None],
-            fullchain_pem=mock.sentinel.fullchain_pem,
             csr_pem=mock.sentinel.csr_pem)
 
     def test_init_acme_verify_ssl(self):
@@ -165,8 +164,7 @@ class ClientTest(ClientTestCommon):
         self._mock_obtain_certificate()
         test_csr = util.CSR(form="pem", file=None, data=CSR_SAN)
         auth_handler = self.client.auth_handler
-        mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert,
-            mock.sentinel.chain)
+        self._set_mock_from_fullchain(mock_crypto_util.cert_and_chain_from_fullchain)
 
         orderr = self.acme.new_order(test_csr.data)
         auth_handler.handle_authorizations(orderr, False)
@@ -199,8 +197,7 @@ class ClientTest(ClientTestCommon):
         csr = util.CSR(form="pem", file=None, data=CSR_SAN)
         mock_crypto_util.init_save_csr.return_value = csr
         mock_crypto_util.init_save_key.return_value = mock.sentinel.key
-        mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert,
-            mock.sentinel.chain)
+        self._set_mock_from_fullchain(mock_crypto_util.cert_and_chain_from_fullchain)
 
         self._test_obtain_certificate_common(mock.sentinel.key, csr)
 
@@ -209,7 +206,7 @@ class ClientTest(ClientTestCommon):
         mock_crypto_util.init_save_csr.assert_called_once_with(
             mock.sentinel.key, self.eg_domains, self.config.csr_dir)
         mock_crypto_util.cert_and_chain_from_fullchain.assert_called_once_with(
-            mock.sentinel.fullchain_pem)
+            self.eg_order.fullchain_pem)
 
     @mock.patch("certbot.client.crypto_util")
     @mock.patch("os.remove")
@@ -218,8 +215,7 @@ class ClientTest(ClientTestCommon):
         key = util.CSR(form="pem", file=mock.sentinel.key_file, data=CSR_SAN)
         mock_crypto_util.init_save_csr.return_value = csr
         mock_crypto_util.init_save_key.return_value = key
-        mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert,
-            mock.sentinel.chain)
+        self._set_mock_from_fullchain(mock_crypto_util.cert_and_chain_from_fullchain)
 
         authzr = self._authzr_from_domains(["example.com"])
         self.config.allow_subset_of_names = True
@@ -237,8 +233,7 @@ class ClientTest(ClientTestCommon):
         mock_acme_crypto.make_csr.return_value = CSR_SAN
         mock_crypto.make_key.return_value = mock.sentinel.key_pem
         key = util.Key(file=None, pem=mock.sentinel.key_pem)
-        mock_crypto.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert,
-            mock.sentinel.chain)
+        self._set_mock_from_fullchain(mock_crypto.cert_and_chain_from_fullchain)
 
         self.client.config.dry_run = True
         self._test_obtain_certificate_common(key, csr)
@@ -250,6 +245,13 @@ class ClientTest(ClientTestCommon):
         mock_crypto.init_save_csr.assert_not_called()
         self.assertEqual(mock_crypto.cert_and_chain_from_fullchain.call_count, 1)
 
+    def _set_mock_from_fullchain(self, mock_from_fullchain):
+        mock_cert = mock.Mock()
+        mock_cert.encode.return_value = mock.sentinel.cert
+        mock_chain = mock.Mock()
+        mock_chain.encode.return_value = mock.sentinel.chain
+        mock_from_fullchain.return_value = (mock_cert, mock_chain)
+
     def _authzr_from_domains(self, domains):
         authzr = []
 

certbot/tests/crypto_util_test.py

@@ -377,8 +377,8 @@ class CertAndChainFromFullchainTest(unittest.TestCase):
     """Tests for certbot.crypto_util.cert_and_chain_from_fullchain"""
 
     def test_cert_and_chain_from_fullchain(self):
-        cert_pem = CERT
-        chain_pem = CERT + SS_CERT
+        cert_pem = CERT.decode()
+        chain_pem = cert_pem + SS_CERT.decode()
         fullchain_pem = cert_pem + chain_pem
         from certbot.crypto_util import cert_and_chain_from_fullchain
         cert_out, chain_out = cert_and_chain_from_fullchain(fullchain_pem)

local-oldest-requirements.txt

@@ -0,0 +1 @@
+-e acme[dev]

setup.py

@@ -34,7 +34,9 @@ version = meta['version']
 # specified here to avoid masking the more specific request requirements in
 # acme. See https://github.com/pypa/pip/issues/988 for more info.
 install_requires = [
-    'acme=={0}'.format(version),
+    # Remember to update local-oldest-requirements.txt when changing the
+    # minimum acme version.
+    'acme>0.21.1',
     # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but
     # saying so here causes a runtime error against our temporary fork of 0.9.3
     # in which we added 2.6 support (see #2243), so we relax the requirement.

tests/boulder-integration.sh

@@ -233,6 +233,7 @@ certname="dns.le.wtf"
 common -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni run \
     --cert-name $certname \
     --manual-auth-hook ./tests/manual-dns-auth.sh \
+    --manual-cleanup-hook ./tests/manual-dns-cleanup.sh \
     --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \
     --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \
     --renew-hook 'echo deploy >> "$HOOK_TEST"'
@@ -433,7 +434,8 @@ done
 # Test ACMEv2-only features
 if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then
     common -a manual -d '*.le4.wtf,le4.wtf' --preferred-challenges dns \
-        --manual-auth-hook ./tests/manual-dns-auth.sh
+        --manual-auth-hook ./tests/manual-dns-auth.sh \
+        --manual-cleanup-hook ./tests/manual-dns-cleanup.sh
 fi
 
 # Most CI systems set this variable to true.
@@ -443,4 +445,4 @@ then
     . ./certbot-nginx/tests/boulder-integration.sh
 fi
 
-coverage report --fail-under 63 -m
+coverage report --fail-under 67 -m

tests/integration/_common.sh

@@ -23,7 +23,7 @@ fi
 
 certbot_test_no_force_renew () {
     omit_patterns="*/*.egg-info/*,*/dns_common*,*/setup.py,*/test_*,*/tests/*"
-    omit_patterns="$omit_patterns,*_test.py,*_test_*,"
+    omit_patterns="$omit_patterns,*_test.py,*_test_*,certbot-apache/*"
     omit_patterns="$omit_patterns,certbot-compatibility-test/*,certbot-dns*/"
     coverage run \
         --append \

tests/manual-dns-cleanup.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+curl -X POST 'http://localhost:8055/clear-txt' -d \
+    "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\"}"

tools/pip_install.sh

@@ -1,18 +1,30 @@
 #!/bin/bash -e
 # pip installs packages using pinned package versions. If CERTBOT_OLDEST is set
-# to 1, a combination of tools/oldest_constraints.txt and
-# tools/dev_constraints.txt is used, otherwise, a combination of certbot-auto's
-# requirements file and tools/dev_constraints.txt is used. The other file
-# always takes precedence over tools/dev_constraints.txt.
+# to 1, a combination of tools/oldest_constraints.txt,
+# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the
+# top level of the package's directory is used, otherwise, a combination of
+# certbot-auto's requirements file and tools/dev_constraints.txt is used. The
+# other file always takes precedence over tools/dev_constraints.txt. If
+# CERTBOT_OLDEST is set, this script must be run with `-e <package-name>` and
+# no other arguments.
 
 # get the root of the Certbot repo
 tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0))
-dev_constraints="$tools_dir/dev_constraints.txt"
-merge_reqs="$tools_dir/merge_requirements.py"
+all_constraints=$(mktemp)
 test_constraints=$(mktemp)
-trap "rm -f $test_constraints" EXIT
+trap "rm -f $all_constraints $test_constraints" EXIT
 
 if [ "$CERTBOT_OLDEST" = 1 ]; then
+    if [ "$1" != "-e" -o "$#" -ne "2" ]; then
+        echo "When CERTBOT_OLDEST is set, this script must be run with a single -e <path> argument."
+        exit 1
+    fi
+    pkg_dir=$(echo $2 | cut -f1 -d\[)  # remove any extras such as [dev]
+    requirements="$pkg_dir/local-oldest-requirements.txt"
+    # packages like acme don't have any local oldest requirements
+    if [ ! -f "$requirements" ]; then
+        unset requirements
+    fi
     cp "$tools_dir/oldest_constraints.txt" "$test_constraints"
 else
     repo_root=$(dirname "$tools_dir")
@@ -20,7 +32,13 @@ else
     sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints"
 fi
 
+"$tools_dir/merge_requirements.py" "$tools_dir/dev_constraints.txt" \
+                                   "$test_constraints" > "$all_constraints"
+
 set -x
 
 # install the requested packages using the pinned requirements as constraints
-pip install -q --constraint <("$merge_reqs" "$dev_constraints" "$test_constraints") "$@"
+if [ -n "$requirements" ]; then
+    pip install -q --constraint "$all_constraints" --requirement "$requirements"
+fi
+pip install -q --constraint "$all_constraints" "$@"

tox.ini

@@ -14,10 +14,7 @@ pip_install = {toxinidir}/tools/pip_install_editable.sh
 # before the script moves on to the next package. All dependencies are pinned
 # to a specific version for increased stability for developers.
 install_and_test = {toxinidir}/tools/install_and_test.sh
-all_packages =
-    acme[dev] \
-    .[dev] \
-    certbot-apache \
+dns_packages =
     certbot-dns-cloudflare \
     certbot-dns-cloudxns \
     certbot-dns-digitalocean \
@@ -27,7 +24,12 @@ all_packages =
     certbot-dns-luadns \
     certbot-dns-nsone \
     certbot-dns-rfc2136 \
-    certbot-dns-route53 \
+    certbot-dns-route53
+all_packages =
+    acme[dev] \
+    .[dev] \
+    certbot-apache \
+    {[base]dns_packages} \
     certbot-nginx \
     letshelp-certbot
 install_packages =
@@ -70,6 +72,47 @@ setenv =
 passenv =
     {[testenv]passenv}
 
+[testenv:py27-acme-oldest]
+commands =
+    {[base]install_and_test} acme[dev]
+setenv =
+    {[testenv:py27-oldest]setenv}
+passenv =
+    {[testenv:py27-oldest]passenv}
+
+[testenv:py27-apache-oldest]
+commands =
+    {[base]install_and_test} certbot-apache
+setenv =
+    {[testenv:py27-oldest]setenv}
+passenv =
+    {[testenv:py27-oldest]passenv}
+
+[testenv:py27-certbot-oldest]
+commands =
+    {[base]install_and_test} .[dev]
+setenv =
+    {[testenv:py27-oldest]setenv}
+passenv =
+    {[testenv:py27-oldest]passenv}
+
+[testenv:py27-dns-oldest]
+commands =
+    {[base]install_and_test} {[base]dns_packages}
+setenv =
+    {[testenv:py27-oldest]setenv}
+passenv =
+    {[testenv:py27-oldest]passenv}
+
+[testenv:py27-nginx-oldest]
+commands =
+    {[base]install_and_test} certbot-nginx
+    python tests/lock_test.py
+setenv =
+    {[testenv:py27-oldest]setenv}
+passenv =
+    {[testenv:py27-oldest]passenv}
+
 [testenv:py27_install]
 basepython = python2.7
 commands =