Make Py3 happy

* Warn users not to remove/undo previous challenges

* Even more specific DNS challenge message

* Fix spacing and variable names

* Create a second test DNS challenge for UI testing

* Changelog for subsequent manual challenge behavior

.gitignore

@@ -6,7 +6,8 @@ dist*/
 /venv*/
 /kgs/
 /.tox/
-/releases/
+/releases*/
+/log*
 letsencrypt.log
 certbot.log
 letsencrypt-auto-source/letsencrypt-auto.sig.lzma.base64
@@ -38,6 +39,8 @@ tests/letstest/venv/
 
 # pytest cache
 .cache
+.mypy_cache/
+.pytest_cache/
 
 # docker files
 .docker

.pylintrc

@@ -41,7 +41,7 @@ load-plugins=linter_plugin
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=fixme,locally-disabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes,cyclic-import,duplicate-code
+disable=fixme,locally-disabled,locally-enabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes,cyclic-import,duplicate-code
 # abstract-class-not-used cannot be disabled locally (at least in
 # pylint 1.4.1), same for abstract-class-little-used
 

.travis.yml

@@ -13,11 +13,11 @@ before_script:
 matrix:
   include:
     - python: "2.7"
-      env: TOXENV=py27_install BOULDER_INTEGRATION=v1
+      env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=all TOXENV=py27_install
       sudo: required
       services: docker
     - python: "2.7"
-      env: TOXENV=py27_install BOULDER_INTEGRATION=v2
+      env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=all TOXENV=py27_install
       sudo: required
       services: docker
     - python: "2.7"
@@ -29,16 +29,21 @@ matrix:
       addons:
     - python: "2.7"
       env: TOXENV=lint
+    - python: "3.4"
+      env: TOXENV=mypy
+    - python: "3.5"
+      env: TOXENV=mypy
     - python: "2.7"
-      env: TOXENV='py27-{acme,apache,certbot,dns,nginx}-oldest'
+      env: TOXENV='py27-{acme,apache,certbot,dns,nginx,postfix}-oldest'
       sudo: required
       services: docker
     - python: "3.4"
       env: TOXENV=py34
       sudo: required
       services: docker
-    - python: "3.6"
-      env: TOXENV=py36
+    - python: "3.7"
+      dist: xenial
+      env: TOXENV=py37
       sudo: required
       services: docker
     - sudo: required
@@ -73,8 +78,6 @@ sudo: false
 
 addons:
   apt:
-    sources:
-    - augeas
     packages:  # Keep in sync with letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh and Boulder.
     - python-dev
     - python-virtualenv
@@ -86,23 +89,20 @@ addons:
     # For certbot-nginx integration testing
     - nginx-light
     - openssl
-    # for apacheconftest
-    - apache2
-    - libapache2-mod-wsgi
-    - libapache2-mod-macro
 
-install: "travis_retry $(command -v pip || command -v pip3) install tox coveralls"
+install: "travis_retry $(command -v pip || command -v pip3) install codecov tox"
 script:
     - travis_retry tox
     - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)'
 
-after_success: '[ "$TOXENV" == "cover" ] && coveralls'
+after_success: '[ "$TOXENV" == "cover" ] && codecov'
 
 notifications:
   email: false
   irc:
     channels:
       - secure: "SGWZl3ownKx9xKVV2VnGt7DqkTmutJ89oJV9tjKhSs84kLijU6EYdPnllqISpfHMTxXflNZuxtGo0wTDYHXBuZL47w1O32W6nzuXdra5zC+i4sYQwYULUsyfOv9gJX8zWAULiK0Z3r0oho45U+FR5ZN6TPCidi8/eGU+EEPwaAw="
+    on_cancel: never
     on_success: never
     on_failure: always
     use_notice: true

CHANGELOG.md

@@ -2,6 +2,318 @@
 
 Certbot adheres to [Semantic Versioning](http://semver.org/).
 
+## 0.28.0 - master
+
+### Added
+
+* `revoke` accepts `--cert-name`, and doesn't accept both `--cert-name` and `--cert-path`.
+* Added OCSP prefetching functionality for Apache plugin that attempts to refresh the OCSP
+  response cache for managed certificates when scheduled Certbot renew is being run.
+
+### Changed
+
+*  `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges.
+
+### Fixed
+
+* Match Nginx parser update in allowing variable names to start with `${`.
+* Correct OVH integration tests on machines without internet access.
+* Stop caching the results of ipv6_info in http01.py
+
+## 0.27.1 - 2018-09-06
+
+### Fixed
+
+* Fixed parameter name in OpenSUSE overrides for default parameters in the
+  Apache plugin. Certbot on OpenSUSE works again.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+package with changes other than its version number was:
+
+* certbot-apache
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/60?closed=1
+
+## 0.27.0 - 2018-09-05
+
+### Added
+
+* The Apache plugin now accepts the parameter --apache-ctl which can be
+  used to configure the path to the Apache control script.
+
+### Changed
+
+* When using `acme.client.ClientV2` (or
+ `acme.client.BackwardsCompatibleClientV2` with an ACME server that supports a
+ newer version of the ACME protocol), an `acme.errors.ConflictError` will be
+ raised if you try to create an ACME account with a key that has already been
+ used. Previously, a JSON parsing error was raised in this scenario when using
+ the library with Let's Encrypt's ACMEv2 endpoint.
+
+### Fixed
+
+* When Apache is not installed, Certbot's Apache plugin no longer prints
+  messages about being unable to find apachectl to the terminal when the plugin
+  is not selected.
+* If you're using the Apache plugin with the --apache-vhost-root flag set to a
+  directory containing a disabled virtual host for the domain you're requesting
+  a certificate for, the virtual host will now be temporarily enabled if
+  necessary to pass the HTTP challenge.
+* The documentation for the Certbot package can now be built using Sphinx 1.6+.
+* You can now call `query_registration` without having to first call
+  `new_account` on `acme.client.ClientV2` objects.
+* The requirement of `setuptools>=1.0` has been removed from `certbot-dns-ovh`.
+* Names in certbot-dns-sakuracloud's tests have been updated to refer to Sakura
+  Cloud rather than NS1 whose plugin certbot-dns-sakuracloud was based on.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+package with changes other than its version number was:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-dns-ovh
+* certbot-dns-sakuracloud
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/57?closed=1
+
+## 0.26.1 - 2018-07-17
+
+### Fixed
+
+* Fix a bug that was triggered when users who had previously manually set `--server` to get ACMEv2 certs tried to renew ACMEv1 certs.
+
+Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was:
+
+* certbot
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/58?closed=1
+
+## 0.26.0 - 2018-07-11
+
+### Added
+
+* A new security enhancement which we're calling AutoHSTS has been added to
+  Certbot's Apache plugin. This enhancement configures your webserver to send a
+  HTTP Strict Transport Security header with a low max-age value that is slowly
+  increased over time. The max-age value is not increased to a large value
+  until you've successfully managed to renew your certificate. This enhancement
+  can be requested with the --auto-hsts flag.
+* New official DNS plugins have been created for Gehirn Infrastracture Service,
+  Linode, OVH, and Sakura Cloud. These plugins can be found on our Docker Hub
+  page at https://hub.docker.com/u/certbot and on PyPI.
+* The ability to reuse ACME accounts from Let's Encrypt's ACMEv1 endpoint on
+  Let's Encrypt's ACMEv2 endpoint has been added.
+* Certbot and its components now support Python 3.7.
+* Certbot's install subcommand now allows you to interactively choose which
+  certificate to install from the list of certificates managed by Certbot.
+* Certbot now accepts the flag `--no-autorenew` which causes any obtained
+  certificates to not be automatically renewed when it approaches expiration.
+* Support for parsing the TLS-ALPN-01 challenge has been added back to the acme
+  library.
+
+### Changed
+
+* Certbot's default ACME server has been changed to Let's Encrypt's ACMEv2
+  endpoint. By default, this server will now be used for both new certificate
+  lineages and renewals.
+* The Nginx plugin is no longer marked labeled as an "Alpha" version.
+* The `prepare` method of Certbot's plugins is no longer called before running
+  "Updater" enhancements that are run on every invocation of `certbot renew`.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with functional changes were:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-dns-gehirn
+* certbot-dns-linode
+* certbot-dns-ovh
+* certbot-dns-sakuracloud
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/55?closed=1
+
+## 0.25.1 - 2018-06-13
+
+### Fixed
+
+* TLS-ALPN-01 support has been removed from our acme library. Using our current
+  dependencies, we are unable to provide a correct implementation of this
+  challenge so we decided to remove it from the library until we can provide
+  proper support.
+* Issues causing test failures when running the tests in the acme package with
+  pytest<3.0 has been resolved.
+* certbot-nginx now correctly depends on acme>=0.25.0.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with changes other than their version number were:
+
+* acme
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/56?closed=1
+
+## 0.25.0 - 2018-06-06
+
+### Added
+
+* Support for the ready status type was added to acme. Without this change,
+  Certbot and acme users will begin encountering errors when using Let's
+  Encrypt's ACMEv2 API starting on June 19th for the staging environment and
+  July 5th for production. See
+  https://community.letsencrypt.org/t/acmev2-order-ready-status/62866 for more
+  information.
+* Certbot now accepts the flag --reuse-key which will cause the same key to be
+  used in the certificate when the lineage is renewed rather than generating a
+  new key.
+* You can now add multiple email addresses to your ACME account with Certbot by
+  providing a comma separated list of emails to the --email flag.
+* Support for Let's Encrypt's upcoming TLS-ALPN-01 challenge was added to acme.
+  For more information, see
+  https://community.letsencrypt.org/t/tls-alpn-validation-method/63814/1.
+* acme now supports specifying the source address to bind to when sending
+  outgoing connections. You still cannot specify this address using Certbot.
+* If you run Certbot against Let's Encrypt's ACMEv2 staging server but don't
+  already have an account registered at that server URL, Certbot will
+  automatically reuse your staging account from Let's Encrypt's ACMEv1 endpoint
+  if it exists.
+* Interfaces were added to Certbot allowing plugins to be called at additional
+  points. The `GenericUpdater` interface allows plugins to perform actions
+  every time `certbot renew` is run, regardless of whether any certificates are
+  due for renewal, and the `RenewDeployer` interface allows plugins to perform
+  actions when a certificate is renewed. See `certbot.interfaces` for more
+  information.
+
+### Changed
+
+* When running Certbot with --dry-run and you don't already have a staging
+  account, the created account does not contain an email address even if one
+  was provided to avoid expiration emails from Let's Encrypt's staging server.
+* certbot-nginx does a better job of automatically detecting the location of
+  Nginx's configuration files when run on BSD based systems.
+* acme now requires and uses pytest when running tests with setuptools with
+  `python setup.py test`.
+* `certbot config_changes` no longer waits for user input before exiting.
+
+### Fixed
+
+* Misleading log output that caused users to think that Certbot's standalone
+  plugin failed to bind to a port when performing a challenge has been
+  corrected.
+* An issue where certbot-nginx would fail to enable HSTS if the server block
+  already had an `add_header` directive has been resolved.
+* certbot-nginx now does a better job detecting the server block to base the
+  configuration for TLS-SNI challenges on.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with functional changes were:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/54?closed=1
+
+## 0.24.0 - 2018-05-02
+
+### Added
+
+* certbot now has an enhance subcommand which allows you to configure security
+  enhancements like HTTP to HTTPS redirects, OCSP stapling, and HSTS without
+  reinstalling a certificate.
+* certbot-dns-rfc2136 now allows the user to specify the port to use to reach
+  the DNS server in its credentials file.
+* acme now parses the wildcard field included in authorizations so it can be
+  used by users of the library.
+
+### Changed
+
+* certbot-dns-route53 used to wait for each DNS update to propagate before
+  sending the next one, but now it sends all updates before waiting which
+  speeds up issuance for multiple domains dramatically.
+* Certbot's official Docker images are now based on Alpine Linux 3.7 rather
+  than 3.4 because 3.4 has reached its end-of-life.
+* We've doubled the time Certbot will spend polling authorizations before
+  timing out.
+* The level of the message logged when Certbot is being used with
+  non-standard paths warning that crontabs for renewal included in Certbot
+  packages from OS package managers may not work has been reduced. This stops
+  the message from being written to stderr every time `certbot renew` runs.
+
+### Fixed
+
+* certbot-auto now works with Python 3.6.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with changes other than their version number were:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-dns-digitalocean (only style improvements to tests)
+* certbot-dns-rfc2136
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/52?closed=1
+
+## 0.23.0 - 2018-04-04
+
+### Added
+
+* Support for OpenResty was added to the Nginx plugin.
+
+### Changed
+
+* The timestamps in Certbot's logfiles now use the system's local time zone
+  rather than UTC.
+* Certbot's DNS plugins that use Lexicon now rely on Lexicon>=2.2.1 to be able
+  to create and delete multiple TXT records on a single domain.
+* certbot-dns-google's test suite now works without an internet connection.
+
+### Fixed
+
+* Removed a small window that if during which an error occurred, Certbot
+  wouldn't clean up performed challenges.
+* The parameters `default` and `ipv6only` are now removed from `listen`
+  directives when creating a new server block in the Nginx plugin.
+* `server_name` directives enclosed in quotation marks in Nginx are now properly
+  supported.
+* Resolved an issue preventing the Apache plugin from starting Apache when it's
+  not currently running on RHEL and Gentoo based systems.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with changes other than their version number were:
+
+* certbot
+* certbot-apache
+* certbot-dns-cloudxns
+* certbot-dns-dnsimple
+* certbot-dns-dnsmadeeasy
+* certbot-dns-google
+* certbot-dns-luadns
+* certbot-dns-nsone
+* certbot-dns-rfc2136
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/50?closed=1
+
 ## 0.22.2 - 2018-03-19
 
 ### Fixed
@@ -563,7 +875,7 @@ https://github.com/certbot/certbot/pulls?q=is%3Apr%20milestone%3A0.11.1%20is%3Ac
 
 ### Added
 
-* When using the standalone plugin while running Certbot interactively 
+* When using the standalone plugin while running Certbot interactively
 and a required port is bound by another process, Certbot will give you
 the option to retry to grab the port rather than immediately exiting.
 * You are now able to deactivate your account with the Let's Encrypt

CHANGES.rst

@@ -1,8 +0,0 @@
-ChangeLog
-=========
-
-To see the changes in a given release, view the issues closed in a given
-release's GitHub milestone:
-
- - `Past releases <https://github.com/certbot/certbot/milestones?state=closed>`_
- - `Upcoming releases <https://github.com/certbot/certbot/milestones>`_

Dockerfile

@@ -1,11 +1,11 @@
-FROM python:2-alpine
+FROM python:2-alpine3.7
 
 ENTRYPOINT [ "certbot" ]
 EXPOSE 80 443
 VOLUME /etc/letsencrypt /var/lib/letsencrypt
 WORKDIR /opt/certbot
 
-COPY CHANGES.rst README.rst setup.py src/
+COPY CHANGELOG.md README.rst setup.py src/
 COPY acme src/acme
 COPY certbot src/certbot
 

Dockerfile-old

@@ -34,7 +34,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only
 # Dockerfile we make sure we cache as much as possible
 
 
-COPY setup.py README.rst CHANGES.rst MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/
+COPY setup.py README.rst CHANGELOG.md MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/
 
 # all above files are necessary for setup.py and venv setup, however,
 # package source code directory has to be copied separately to a

MANIFEST.in

@@ -1,5 +1,5 @@
 include README.rst
-include CHANGES.rst
+include CHANGELOG.md
 include CONTRIBUTING.md
 include LICENSE.txt
 include linter_plugin.py

README.rst

@@ -6,7 +6,7 @@ Anyone who has gone through the trouble of setting up a secure website knows wha
 
 How you use Certbot depends on the configuration of your web server. The best way to get started is to use our `interactive guide <https://certbot.eff.org>`_. It generates instructions based on your configuration settings. In most cases, you’ll need `root or administrator access <https://certbot.eff.org/faq/#does-certbot-require-root-administrator-privileges>`_ to your web server to run Certbot.
 
-If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issued by Let’s Encrypt.
+Certbot is meant to be run directly on your web server, not on your personal computer. If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issued by Let’s Encrypt.
 
 Certbot is a fully-featured, extensible client for the Let's
 Encrypt CA (or any other CA that speaks the `ACME
@@ -107,8 +107,8 @@ ACME working area in github: https://github.com/ietf-wg-acme/acme
    :target: https://travis-ci.org/certbot/certbot
    :alt: Travis CI status
 
-.. |coverage| image:: https://coveralls.io/repos/certbot/certbot/badge.svg?branch=master
-   :target: https://coveralls.io/r/certbot/certbot
+.. |coverage| image:: https://codecov.io/gh/certbot/certbot/branch/master/graph/badge.svg
+   :target: https://codecov.io/gh/certbot/certbot
    :alt: Coverage status
 
 .. |docs| image:: https://readthedocs.org/projects/letsencrypt/badge/

acme/MANIFEST.in

@@ -1,5 +1,6 @@
 include LICENSE.txt
 include README.rst
+include pytest.ini
 recursive-include docs *
 recursive-include examples *
 recursive-include acme/testdata *

acme/acme/challenges.py

@@ -9,6 +9,7 @@ from cryptography.hazmat.primitives import hashes  # type: ignore
 import josepy as jose
 import OpenSSL
 import requests
+import six
 
 from acme import errors
 from acme import crypto_util
@@ -139,16 +140,16 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse):
         return True
 
 
+@six.add_metaclass(abc.ABCMeta)
 class KeyAuthorizationChallenge(_TokenChallenge):
     # pylint: disable=abstract-class-little-used,too-many-ancestors
     """Challenge based on Key Authorization.
 
     :param response_cls: Subclass of `KeyAuthorizationChallengeResponse`
         that will be used to generate `response`.
-
+    :param str typ: type of the challenge
     """
-    __metaclass__ = abc.ABCMeta
-
+    typ = NotImplemented
     response_cls = NotImplemented
     thumbprint_hash_function = (
         KeyAuthorizationChallengeResponse.thumbprint_hash_function)
@@ -477,7 +478,7 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse):
             try:
                 cert = self.probe_cert(domain=domain, **kwargs)
             except errors.Error as error:
-                logger.debug(error, exc_info=True)
+                logger.debug(str(error), exc_info=True)
                 return False
 
         return self.verify_cert(cert)
@@ -506,6 +507,21 @@ class TLSSNI01(KeyAuthorizationChallenge):
         return self.response(account_key).gen_cert(key=kwargs.get('cert_key'))
 
 
+@Challenge.register  # pylint: disable=too-many-ancestors
+class TLSALPN01(KeyAuthorizationChallenge):
+    """ACME tls-alpn-01 challenge.
+
+    This class simply allows parsing the TLS-ALPN-01 challenge returned from
+    the CA. Full TLS-ALPN-01 support is not currently provided.
+
+    """
+    typ = "tls-alpn-01"
+
+    def validation(self, account_key, **kwargs):
+        """Generate validation for the challenge."""
+        raise NotImplementedError()
+
+
 @Challenge.register  # pylint: disable=too-many-ancestors
 class DNS(_TokenChallenge):
     """ACME "dns" challenge."""

acme/acme/challenges_test.py

@@ -393,6 +393,38 @@ class TLSSNI01Test(unittest.TestCase):
         mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key)
 
 
+class TLSALPN01Test(unittest.TestCase):
+
+    def setUp(self):
+        from acme.challenges import TLSALPN01
+        self.msg = TLSALPN01(
+            token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e'))
+        self.jmsg = {
+            'type': 'tls-alpn-01',
+            'token': 'a82d5ff8ef740d12881f6d3c2277ab2e',
+        }
+
+    def test_to_partial_json(self):
+        self.assertEqual(self.jmsg, self.msg.to_partial_json())
+
+    def test_from_json(self):
+        from acme.challenges import TLSALPN01
+        self.assertEqual(self.msg, TLSALPN01.from_json(self.jmsg))
+
+    def test_from_json_hashable(self):
+        from acme.challenges import TLSALPN01
+        hash(TLSALPN01.from_json(self.jmsg))
+
+    def test_from_json_invalid_token_length(self):
+        from acme.challenges import TLSALPN01
+        self.jmsg['token'] = jose.encode_b64jose(b'abcd')
+        self.assertRaises(
+            jose.DeserializationError, TLSALPN01.from_json, self.jmsg)
+
+    def test_validation(self):
+        self.assertRaises(NotImplementedError, self.msg.validation, KEY)
+
+
 class DNSTest(unittest.TestCase):
 
     def setUp(self):

acme/acme/client.py

@@ -9,17 +9,20 @@ import time
 
 import six
 from six.moves import http_client  # pylint: disable=import-error
-
 import josepy as jose
 import OpenSSL
 import re
+from requests_toolbelt.adapters.source import SourceAddressAdapter
 import requests
+from requests.adapters import HTTPAdapter
 import sys
 
 from acme import crypto_util
 from acme import errors
 from acme import jws
 from acme import messages
+# pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Dict, List, Set, Text
 
 
 logger = logging.getLogger(__name__)
@@ -47,7 +50,6 @@ class ClientBase(object):  # pylint: disable=too-many-instance-attributes
     :ivar .ClientNetwork net: Client network.
     :ivar int acme_version: ACME protocol version. 1 or 2.
     """
-
     def __init__(self, directory, net, acme_version):
         """Initialize.
 
@@ -415,7 +417,7 @@ class Client(ClientBase):
         """
         # pylint: disable=too-many-locals
         assert max_attempts > 0
-        attempts = collections.defaultdict(int)
+        attempts = collections.defaultdict(int) # type: Dict[messages.AuthorizationResource, int]
         exhausted = set()
 
         # priority queue with datetime.datetime (based on Retry-After) as key,
@@ -529,7 +531,7 @@ class Client(ClientBase):
         :rtype: `list` of `OpenSSL.crypto.X509` wrapped in `.ComparableX509`
 
         """
-        chain = []
+        chain = [] # type: List[jose.ComparableX509]
         uri = certr.cert_chain_uri
         while uri is not None and len(chain) < max_length:
             response, cert = self._get_cert(uri)
@@ -575,16 +577,57 @@ class ClientV2(ClientBase):
 
         :param .NewRegistration new_account:
 
+        :raises .ConflictError: in case the account already exists
+
         :returns: Registration Resource.
         :rtype: `.RegistrationResource`
         """
         response = self._post(self.directory['newAccount'], new_account)
+        # if account already exists
+        if response.status_code == 200 and 'Location' in response.headers:
+            raise errors.ConflictError(response.headers.get('Location'))
         # "Instance of 'Field' has no key/contact member" bug:
         # pylint: disable=no-member
         regr = self._regr_from_response(response)
         self.net.account = regr
         return regr
 
+    def query_registration(self, regr):
+        """Query server about registration.
+
+        :param messages.RegistrationResource: Existing Registration
+            Resource.
+
+        """
+        self.net.account = regr
+        updated_regr = super(ClientV2, self).query_registration(regr)
+        self.net.account = updated_regr
+        return updated_regr
+
+    def update_registration(self, regr, update=None):
+        """Update registration.
+
+        :param messages.RegistrationResource regr: Registration Resource.
+        :param messages.Registration update: Updated body of the
+            resource. If not provided, body will be taken from `regr`.
+
+        :returns: Updated Registration Resource.
+        :rtype: `.RegistrationResource`
+
+        """
+        # https://github.com/certbot/certbot/issues/6155
+        new_regr = self._get_v2_account(regr)
+        return super(ClientV2, self).update_registration(new_regr, update)
+
+    def _get_v2_account(self, regr):
+        self.net.account = None
+        only_existing_reg = regr.body.update(only_return_existing=True)
+        response = self._post(self.directory['newAccount'], only_existing_reg)
+        updated_uri = response.headers['Location']
+        new_regr = regr.update(uri=updated_uri)
+        self.net.account = new_regr
+        return new_regr
+
     def new_order(self, csr_pem):
         """Request a new Order object from the server.
 
@@ -856,18 +899,28 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
     :param bool verify_ssl: Whether to verify certificates on SSL connections.
     :param str user_agent: String to send as User-Agent header.
     :param float timeout: Timeout for requests.
+    :param source_address: Optional source address to bind to when making requests.
+    :type source_address: str or tuple(str, int)
     """
     def __init__(self, key, account=None, alg=jose.RS256, verify_ssl=True,
-                 user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT):
+                 user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT,
+                 source_address=None):
         # pylint: disable=too-many-arguments
         self.key = key
         self.account = account
         self.alg = alg
         self.verify_ssl = verify_ssl
-        self._nonces = set()
+        self._nonces = set() # type: Set[Text]
         self.user_agent = user_agent
         self.session = requests.Session()
         self._default_timeout = timeout
+        adapter = HTTPAdapter()
+
+        if source_address is not None:
+            adapter = SourceAddressAdapter(source_address)
+
+        self.session.mount("http://", adapter)
+        self.session.mount("https://", adapter)
 
     def __del__(self):
         # Try to close the session, but don't show exceptions to the
@@ -897,6 +950,7 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
         if acme_version == 2:
             kwargs["url"] = url
             # newAccount and revokeCert work without the kid
+            # newAccount must not have kid
             if self.account is not None:
                 kwargs["kid"] = self.account["uri"]
         kwargs["key"] = self.key
@@ -1017,7 +1071,7 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
         if response.headers.get("Content-Type") == DER_CONTENT_TYPE:
             debug_content = base64.b64encode(response.content)
         else:
-            debug_content = response.content
+            debug_content = response.content.decode("utf-8")
         logger.debug('Received response:\nHTTP %d\n%s\n\n%s',
                      response.status_code,
                      "\n".join(["{0}: {1}".format(k, v)

acme/acme/client_test.py

@@ -17,6 +17,7 @@ from acme import jws as acme_jws
 from acme import messages
 from acme import messages_test
 from acme import test_util
+from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module
 
 
 CERT_DER = test_util.load_vector('cert.der')
@@ -61,7 +62,8 @@ class ClientTestBase(unittest.TestCase):
         self.contact = ('mailto:cert-admin@example.com', 'tel:+12025551212')
         reg = messages.Registration(
             contact=self.contact, key=KEY.public_key())
-        self.new_reg = messages.NewRegistration(**dict(reg))
+        the_arg = dict(reg) # type: Dict
+        self.new_reg = messages.NewRegistration(**the_arg) # pylint: disable=star-args
         self.regr = messages.RegistrationResource(
             body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1')
 
@@ -132,12 +134,18 @@ class BackwardsCompatibleClientV2Test(ClientTestBase):
         client = self._init()
         self.assertEqual(client.acme_version, 2)
 
+    def test_query_registration_client_v2(self):
+        self.response.json.return_value = DIRECTORY_V2.to_json()
+        client = self._init()
+        self.response.json.return_value = self.regr.body.to_json()
+        self.assertEqual(self.regr, client.query_registration(self.regr))
+
     def test_forwarding(self):
         self.response.json.return_value = DIRECTORY_V1.to_json()
         client = self._init()
         self.assertEqual(client.directory, client.client.directory)
         self.assertEqual(client.key, KEY)
-        self.assertEqual(client.update_registration, client.client.update_registration)
+        self.assertEqual(client.deactivate_registration, client.client.deactivate_registration)
         self.assertRaises(AttributeError, client.__getattr__, 'nonexistent')
         self.assertRaises(AttributeError, client.__getattr__, 'new_account_and_tos')
         self.assertRaises(AttributeError, client.__getattr__, 'new_account')
@@ -268,6 +276,13 @@ class BackwardsCompatibleClientV2Test(ClientTestBase):
             client.revoke(messages_test.CERT, self.rsn)
         mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn)
 
+    def test_update_registration(self):
+        self.response.json.return_value = DIRECTORY_V1.to_json()
+        with mock.patch('acme.client.Client') as mock_client:
+            client = self._init()
+            client.update_registration(mock.sentinel.regr, None)
+        mock_client().update_registration.assert_called_once_with(mock.sentinel.regr, None)
+
 
 class ClientTest(ClientTestBase):
     """Tests for acme.client.Client."""
@@ -697,6 +712,11 @@ class ClientV2Test(ClientTestBase):
 
         self.assertEqual(self.regr, self.client.new_account(self.new_reg))
 
+    def test_new_account_conflict(self):
+        self.response.status_code = http_client.OK
+        self.response.headers['Location'] = self.regr.uri
+        self.assertRaises(errors.ConflictError, self.client.new_account, self.new_reg)
+
     def test_new_order(self):
         order_response = copy.deepcopy(self.response)
         order_response.status_code = http_client.CREATED
@@ -787,6 +807,19 @@ class ClientV2Test(ClientTestBase):
         self.net.post.assert_called_once_with(
             self.directory["revokeCert"], mock.ANY, acme_version=2)
 
+    def test_update_registration(self):
+        # "Instance of 'Field' has no to_json/update member" bug:
+        # pylint: disable=no-member
+        self.response.headers['Location'] = self.regr.uri
+        self.response.json.return_value = self.regr.body.to_json()
+        self.assertEqual(self.regr, self.client.update_registration(self.regr))
+        self.assertNotEqual(self.client.net.account, None)
+        self.assertEqual(self.client.net.post.call_count, 2)
+        self.assertTrue(DIRECTORY_V2.newAccount in self.net.post.call_args_list[0][0])
+
+        self.response.json.return_value = self.regr.body.update(
+            contact=()).to_json()
+
 
 class MockJSONDeSerializable(jose.JSONDeSerializable):
     # pylint: disable=missing-docstring
@@ -1127,6 +1160,31 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
         self.assertRaises(requests.exceptions.RequestException,
                           self.net.post, 'uri', obj=self.obj)
 
+class ClientNetworkSourceAddressBindingTest(unittest.TestCase):
+    """Tests that if ClientNetwork has a source IP set manually, the underlying library has
+    used the provided source address."""
+
+    def setUp(self):
+        self.source_address = "8.8.8.8"
+
+    def test_source_address_set(self):
+        from acme.client import ClientNetwork
+        net = ClientNetwork(key=None, alg=None, source_address=self.source_address)
+        for adapter in net.session.adapters.values():
+            self.assertTrue(self.source_address in adapter.source_address)
+
+    def test_behavior_assumption(self):
+        """This is a test that guardrails the HTTPAdapter behavior so that if the default for
+        a Session() changes, the assumptions here aren't violated silently."""
+        from acme.client import ClientNetwork
+        # Source address not specified, so the default adapter type should be bound -- this
+        # test should fail if the default adapter type is changed by requests
+        net = ClientNetwork(key=None, alg=None)
+        session = requests.Session()
+        for scheme in session.adapters.keys():
+            client_network_adapter = net.session.adapters.get(scheme)
+            default_adapter = session.adapters.get(scheme)
+            self.assertEqual(client_network_adapter.__class__, default_adapter.__class__)
 
 if __name__ == '__main__':
     unittest.main()  # pragma: no cover

acme/acme/crypto_util.py

@@ -6,11 +6,14 @@ import os
 import re
 import socket
 
-import OpenSSL
+from OpenSSL import crypto
+from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052
 import josepy as jose
 
-
 from acme import errors
+# pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Callable, Union, Tuple, Optional
+# pylint: enable=unused-import, no-name-in-module
 
 
 logger = logging.getLogger(__name__)
@@ -25,7 +28,7 @@ logger = logging.getLogger(__name__)
 # https://www.openssl.org/docs/ssl/SSLv23_method.html). _serve_sni
 # should be changed to use "set_options" to disable SSLv2 and SSLv3,
 # in case it's used for things other than probing/serving!
-_DEFAULT_TLSSNI01_SSL_METHOD = OpenSSL.SSL.SSLv23_METHOD  # type: ignore
+_DEFAULT_TLSSNI01_SSL_METHOD = SSL.SSLv23_METHOD  # type: ignore
 
 
 class SSLSocket(object):  # pylint: disable=too-few-public-methods
@@ -64,9 +67,9 @@ class SSLSocket(object):  # pylint: disable=too-few-public-methods
             logger.debug("Server name (%s) not recognized, dropping SSL",
                          server_name)
             return
-        new_context = OpenSSL.SSL.Context(self.method)
-        new_context.set_options(OpenSSL.SSL.OP_NO_SSLv2)
-        new_context.set_options(OpenSSL.SSL.OP_NO_SSLv3)
+        new_context = SSL.Context(self.method)
+        new_context.set_options(SSL.OP_NO_SSLv2)
+        new_context.set_options(SSL.OP_NO_SSLv3)
         new_context.use_privatekey(key)
         new_context.use_certificate(cert)
         connection.set_context(new_context)
@@ -89,18 +92,18 @@ class SSLSocket(object):  # pylint: disable=too-few-public-methods
     def accept(self):  # pylint: disable=missing-docstring
         sock, addr = self.sock.accept()
 
-        context = OpenSSL.SSL.Context(self.method)
-        context.set_options(OpenSSL.SSL.OP_NO_SSLv2)
-        context.set_options(OpenSSL.SSL.OP_NO_SSLv3)
+        context = SSL.Context(self.method)
+        context.set_options(SSL.OP_NO_SSLv2)
+        context.set_options(SSL.OP_NO_SSLv3)
         context.set_tlsext_servername_callback(self._pick_certificate_cb)
 
-        ssl_sock = self.FakeConnection(OpenSSL.SSL.Connection(context, sock))
+        ssl_sock = self.FakeConnection(SSL.Connection(context, sock))
         ssl_sock.set_accept_state()
 
         logger.debug("Performing handshake with %s", addr)
         try:
             ssl_sock.do_handshake()
-        except OpenSSL.SSL.Error as error:
+        except SSL.Error as error:
             # _pick_certificate_cb might have returned without
             # creating SSL context (wrong server name)
             raise socket.error(error)
@@ -128,30 +131,39 @@ def probe_sni(name, host, port=443, timeout=300,
     :rtype: OpenSSL.crypto.X509
 
     """
-    context = OpenSSL.SSL.Context(method)
+    context = SSL.Context(method)
     context.set_timeout(timeout)
 
     socket_kwargs = {'source_address': source_address}
 
-    host_protocol_agnostic = None if host == '::' or host == '0' else host
+    host_protocol_agnostic = host
+    if host == '::' or host == '0':
+        # https://github.com/python/typeshed/pull/2136
+        # while PR is not merged, we need to ignore
+        host_protocol_agnostic = None
 
     try:
         # pylint: disable=star-args
-        logger.debug("Attempting to connect to %s:%d%s.", host_protocol_agnostic, port,
-            " from {0}:{1}".format(source_address[0], source_address[1]) if \
-            socket_kwargs else "")
-        sock = socket.create_connection((host_protocol_agnostic, port), **socket_kwargs)
+        logger.debug(
+            "Attempting to connect to %s:%d%s.", host_protocol_agnostic, port,
+            " from {0}:{1}".format(
+                source_address[0],
+                source_address[1]
+            ) if socket_kwargs else ""
+        )
+        socket_tuple = (host_protocol_agnostic, port)  # type: Tuple[Optional[str], int]
+        sock = socket.create_connection(socket_tuple, **socket_kwargs)  # type: ignore
     except socket.error as error:
         raise errors.Error(error)
 
     with contextlib.closing(sock) as client:
-        client_ssl = OpenSSL.SSL.Connection(context, client)
+        client_ssl = SSL.Connection(context, client)
         client_ssl.set_connect_state()
         client_ssl.set_tlsext_host_name(name)  # pyOpenSSL>=0.13
         try:
             client_ssl.do_handshake()
             client_ssl.shutdown()
-        except OpenSSL.SSL.Error as error:
+        except SSL.Error as error:
             raise errors.Error(error)
     return client_ssl.get_peer_certificate()
 
@@ -164,18 +176,18 @@ def make_csr(private_key_pem, domains, must_staple=False):
         OCSP Must Staple: https://tools.ietf.org/html/rfc7633).
     :returns: buffer PEM-encoded Certificate Signing Request.
     """
-    private_key = OpenSSL.crypto.load_privatekey(
-        OpenSSL.crypto.FILETYPE_PEM, private_key_pem)
-    csr = OpenSSL.crypto.X509Req()
+    private_key = crypto.load_privatekey(
+        crypto.FILETYPE_PEM, private_key_pem)
+    csr = crypto.X509Req()
     extensions = [
-        OpenSSL.crypto.X509Extension(
+        crypto.X509Extension(
             b'subjectAltName',
             critical=False,
             value=', '.join('DNS:' + d for d in domains).encode('ascii')
         ),
     ]
     if must_staple:
-        extensions.append(OpenSSL.crypto.X509Extension(
+        extensions.append(crypto.X509Extension(
             b"1.3.6.1.5.5.7.1.24",
             critical=False,
             value=b"DER:30:03:02:01:05"))
@@ -183,8 +195,8 @@ def make_csr(private_key_pem, domains, must_staple=False):
     csr.set_pubkey(private_key)
     csr.set_version(2)
     csr.sign(private_key, 'sha256')
-    return OpenSSL.crypto.dump_certificate_request(
-        OpenSSL.crypto.FILETYPE_PEM, csr)
+    return crypto.dump_certificate_request(
+        crypto.FILETYPE_PEM, csr)
 
 def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req):
     common_name = loaded_cert_or_req.get_subject().CN
@@ -221,11 +233,12 @@ def _pyopenssl_cert_or_req_san(cert_or_req):
     parts_separator = ", "
     prefix = "DNS" + part_separator
 
-    if isinstance(cert_or_req, OpenSSL.crypto.X509):
-        func = OpenSSL.crypto.dump_certificate
+    if isinstance(cert_or_req, crypto.X509):
+        # pylint: disable=line-too-long
+        func = crypto.dump_certificate # type: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]]
     else:
-        func = OpenSSL.crypto.dump_certificate_request
-    text = func(OpenSSL.crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8")
+        func = crypto.dump_certificate_request
+    text = func(crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8")
     # WARNING: this function does not support multiple SANs extensions.
     # Multiple X509v3 extensions of the same type is disallowed by RFC 5280.
     match = re.search(r"X509v3 Subject Alternative Name:(?: critical)?\s*(.*)", text)
@@ -252,12 +265,12 @@ def gen_ss_cert(key, domains, not_before=None,
 
     """
     assert domains, "Must provide one or more hostnames for the cert."
-    cert = OpenSSL.crypto.X509()
+    cert = crypto.X509()
     cert.set_serial_number(int(binascii.hexlify(os.urandom(16)), 16))
     cert.set_version(2)
 
     extensions = [
-        OpenSSL.crypto.X509Extension(
+        crypto.X509Extension(
             b"basicConstraints", True, b"CA:TRUE, pathlen:0"),
     ]
 
@@ -266,7 +279,7 @@ def gen_ss_cert(key, domains, not_before=None,
     cert.set_issuer(cert.get_subject())
 
     if force_san or len(domains) > 1:
-        extensions.append(OpenSSL.crypto.X509Extension(
+        extensions.append(crypto.X509Extension(
             b"subjectAltName",
             critical=False,
             value=b", ".join(b"DNS:" + d.encode() for d in domains)
@@ -281,7 +294,7 @@ def gen_ss_cert(key, domains, not_before=None,
     cert.sign(key, "sha256")
     return cert
 
-def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM):
+def dump_pyopenssl_chain(chain, filetype=crypto.FILETYPE_PEM):
     """Dump certificate chain into a bundle.
 
     :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in
@@ -298,7 +311,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM):
         if isinstance(cert, jose.ComparableX509):
             # pylint: disable=protected-access
             cert = cert.wrapped
-        return OpenSSL.crypto.dump_certificate(filetype, cert)
+        return crypto.dump_certificate(filetype, cert)
 
     # assumes that OpenSSL.crypto.dump_certificate includes ending
     # newline character

acme/acme/crypto_util_test.py

@@ -13,6 +13,7 @@ import OpenSSL
 
 from acme import errors
 from acme import test_util
+from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module
 
 
 class SSLSocketAndProbeSNITest(unittest.TestCase):
@@ -41,28 +42,38 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
         self.server_thread = threading.Thread(
             # pylint: disable=no-member
             target=self.server.handle_request)
-        self.server_thread.start()
-        time.sleep(1)  # TODO: avoid race conditions in other way
 
     def tearDown(self):
-        self.server_thread.join()
+        if self.server_thread.is_alive():
+            # The thread may have already terminated.
+            self.server_thread.join()  # pragma: no cover
 
     def _probe(self, name):
         from acme.crypto_util import probe_sni
         return jose.ComparableX509(probe_sni(
             name, host='127.0.0.1', port=self.port))
 
+    def _start_server(self):
+        self.server_thread.start()
+        time.sleep(1)  # TODO: avoid race conditions in other way
+
     def test_probe_ok(self):
+        self._start_server()
         self.assertEqual(self.cert, self._probe(b'foo'))
 
     def test_probe_not_recognized_name(self):
+        self._start_server()
         self.assertRaises(errors.Error, self._probe, b'bar')
 
-    # TODO: py33/py34 tox hangs forever on do_handshake in second probe
-    #def probe_connection_error(self):
-    #    self._probe(b'foo')
-    #    #time.sleep(1)  # TODO: avoid race conditions in other way
-    #    self.assertRaises(errors.Error, self._probe, b'bar')
+    def test_probe_connection_error(self):
+        # pylint has a hard time with six
+        self.server.server_close()  # pylint: disable=no-member
+        original_timeout = socket.getdefaulttimeout()
+        try:
+            socket.setdefaulttimeout(1)
+            self.assertRaises(errors.Error, self._probe, b'bar')
+        finally:
+            socket.setdefaulttimeout(original_timeout)
 
 
 class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase):
@@ -165,7 +176,7 @@ class RandomSnTest(unittest.TestCase):
 
     def setUp(self):
         self.cert_count = 5
-        self.serial_num = []
+        self.serial_num = [] # type: List[int]
         self.key = OpenSSL.crypto.PKey()
         self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
 

acme/acme/errors.py

@@ -110,6 +110,8 @@ class ConflictError(ClientError):
 
     In the version of ACME implemented by Boulder, this is used to find an
     account if you only have the private key, but don't know the account URL.
+
+    Also used in V2 of the ACME client for the same purpose.
     """
     def __init__(self, location):
         self.location = location

acme/acme/magic_typing.py

@@ -0,0 +1,16 @@
+"""Shim class to not have to depend on typing module in prod."""
+import sys
+
+class TypingClass(object):
+    """Ignore import errors by getting anything"""
+    def __getattr__(self, name):
+        return None
+
+try:
+    # mypy doesn't respect modifying sys.modules
+    from typing import *  # pylint: disable=wildcard-import, unused-wildcard-import
+    # pylint: disable=unused-import
+    from typing import Collection, IO  # type: ignore
+    # pylint: enable=unused-import
+except ImportError:
+    sys.modules[__name__] = TypingClass()

acme/acme/magic_typing_test.py

@@ -0,0 +1,41 @@
+"""Tests for acme.magic_typing."""
+import sys
+import unittest
+
+import mock
+
+
+class MagicTypingTest(unittest.TestCase):
+    """Tests for acme.magic_typing."""
+    def test_import_success(self):
+        try:
+            import typing as temp_typing
+        except ImportError: # pragma: no cover
+            temp_typing = None # pragma: no cover
+        typing_class_mock = mock.MagicMock()
+        text_mock = mock.MagicMock()
+        typing_class_mock.Text = text_mock
+        sys.modules['typing'] = typing_class_mock
+        if 'acme.magic_typing' in sys.modules:
+            del sys.modules['acme.magic_typing'] # pragma: no cover
+        from acme.magic_typing import Text # pylint: disable=no-name-in-module
+        self.assertEqual(Text, text_mock)
+        del sys.modules['acme.magic_typing']
+        sys.modules['typing'] = temp_typing
+
+    def test_import_failure(self):
+        try:
+            import typing as temp_typing
+        except ImportError: # pragma: no cover
+            temp_typing = None # pragma: no cover
+        sys.modules['typing'] = None
+        if 'acme.magic_typing' in sys.modules:
+            del sys.modules['acme.magic_typing'] # pragma: no cover
+        from acme.magic_typing import Text # pylint: disable=no-name-in-module
+        self.assertTrue(Text is None)
+        del sys.modules['acme.magic_typing']
+        sys.modules['typing'] = temp_typing
+
+
+if __name__ == '__main__':
+    unittest.main()  # pragma: no cover

acme/acme/messages.py

@@ -145,6 +145,7 @@ STATUS_PROCESSING = Status('processing')
 STATUS_VALID = Status('valid')
 STATUS_INVALID = Status('invalid')
 STATUS_REVOKED = Status('revoked')
+STATUS_READY = Status('ready')
 
 
 class IdentifierType(_Constant):
@@ -273,6 +274,7 @@ class Registration(ResourceBody):
     agreement = jose.Field('agreement', omitempty=True)
     status = jose.Field('status', omitempty=True)
     terms_of_service_agreed = jose.Field('termsOfServiceAgreed', omitempty=True)
+    only_return_existing = jose.Field('onlyReturnExisting', omitempty=True)
 
     phone_prefix = 'tel:'
     email_prefix = 'mailto:'
@@ -284,7 +286,7 @@ class Registration(ResourceBody):
         if phone is not None:
             details.append(cls.phone_prefix + phone)
         if email is not None:
-            details.append(cls.email_prefix + email)
+            details.extend([cls.email_prefix + mail for mail in email.split(',')])
         kwargs['contact'] = tuple(details)
         return cls(**kwargs)
 
@@ -435,6 +437,7 @@ class Authorization(ResourceBody):
     # be absent'... then acme-spec gives example with 'expires'
     # present... That's confusing!
     expires = fields.RFC3339Field('expires', omitempty=True)
+    wildcard = jose.Field('wildcard', omitempty=True)
 
     @challenges.decoder
     def challenges(value):  # pylint: disable=missing-docstring,no-self-argument

acme/acme/messages_test.py

@@ -6,6 +6,7 @@ import mock
 
 from acme import challenges
 from acme import test_util
+from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module
 
 
 CERT = test_util.load_comparable_cert('cert.der')
@@ -85,7 +86,7 @@ class ConstantTest(unittest.TestCase):
         from acme.messages import _Constant
 
         class MockConstant(_Constant):  # pylint: disable=missing-docstring
-            POSSIBLE_NAMES = {}
+            POSSIBLE_NAMES = {} # type: Dict
 
         self.MockConstant = MockConstant  # pylint: disable=invalid-name
         self.const_a = MockConstant('a')

acme/acme/standalone.py

@@ -16,6 +16,7 @@ import OpenSSL
 
 from acme import challenges
 from acme import crypto_util
+from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module
 
 
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class BaseDualNetworkedServers(object):
 
     def __init__(self, ServerClass, server_address, *remaining_args, **kwargs):
         port = server_address[1]
-        self.threads = []
-        self.servers = []
+        self.threads = [] # type: List[threading.Thread]
+        self.servers = [] # type: List[ACMEServerMixin]
 
         # Must try True first.
         # Ubuntu, for example, will fail to bind to IPv4 if we've already bound
@@ -82,9 +83,22 @@ class BaseDualNetworkedServers(object):
                 new_address = (server_address[0],) + (port,) + server_address[2:]
                 new_args = (new_address,) + remaining_args
                 server = ServerClass(*new_args, **kwargs) # pylint: disable=star-args
-            except socket.error:
-                logger.debug("Failed to bind to %s:%s using %s", new_address[0],
+                logger.debug(
+                    "Successfully bound to %s:%s using %s", new_address[0],
                     new_address[1], "IPv6" if ip_version else "IPv4")
+            except socket.error:
+                if self.servers:
+                    # Already bound using IPv6.
+                    logger.debug(
+                        "Certbot wasn't able to bind to %s:%s using %s, this " +
+                        "is often expected due to the dual stack nature of " +
+                        "IPv6 socket implementations.",
+                        new_address[0], new_address[1],
+                        "IPv6" if ip_version else "IPv4")
+                else:
+                    logger.debug(
+                        "Failed to bind to %s:%s using %s", new_address[0],
+                        new_address[1], "IPv6" if ip_version else "IPv4")
             else:
                 self.servers.append(server)
                 # If two servers are set up and port 0 was passed in, ensure we always
@@ -189,7 +203,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 
     def __init__(self, *args, **kwargs):
         self.simple_http_resources = kwargs.pop("simple_http_resources", set())
-        socketserver.BaseRequestHandler.__init__(self, *args, **kwargs)
+        BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
 
     def log_message(self, format, *args):  # pylint: disable=redefined-builtin
         """Log arbitrary message."""
@@ -262,7 +276,7 @@ def simple_tls_sni_01_server(cli_args, forever=True):
 
     certs = {}
 
-    _, hosts, _ = next(os.walk('.'))
+    _, hosts, _ = next(os.walk('.')) # type: ignore # https://github.com/python/mypy/issues/465
     for host in hosts:
         with open(os.path.join(host, "cert.pem")) as cert_file:
             cert_contents = cert_file.read()

acme/acme/standalone_test.py

@@ -4,10 +4,10 @@ import shutil
 import socket
 import threading
 import tempfile
-import time
 import unittest
 
 from six.moves import http_client  # pylint: disable=import-error
+from six.moves import queue  # pylint: disable=import-error
 from six.moves import socketserver  # type: ignore  # pylint: disable=import-error
 
 import josepy as jose
@@ -16,8 +16,8 @@ import requests
 
 from acme import challenges
 from acme import crypto_util
-from acme import errors
 from acme import test_util
+from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module
 
 
 class TLSServerTest(unittest.TestCase):
@@ -72,7 +72,7 @@ class HTTP01ServerTest(unittest.TestCase):
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))
-        self.resources = set()
+        self.resources = set() # type: Set
 
         from acme.standalone import HTTP01Server
         self.server = HTTP01Server(('', 0), resources=self.resources)
@@ -201,7 +201,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase):
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))
-        self.resources = set()
+        self.resources = set() # type: Set
 
         from acme.standalone import HTTP01DualNetworkedServers
         self.servers = HTTP01DualNetworkedServers(('', 0), resources=self.resources)
@@ -260,10 +260,9 @@ class TestSimpleTLSSNI01Server(unittest.TestCase):
                     os.path.join(localhost_dir, 'key.pem'))
 
         from acme.standalone import simple_tls_sni_01_server
-        self.port = 1234
         self.thread = threading.Thread(
             target=simple_tls_sni_01_server, kwargs={
-                'cli_args': ('xxx', '--port', str(self.port)),
+                'cli_args': ('filename',),
                 'forever': False,
             },
         )
@@ -275,25 +274,20 @@ class TestSimpleTLSSNI01Server(unittest.TestCase):
         self.thread.join()
         shutil.rmtree(self.test_cwd)
 
-    def test_it(self):
-        max_attempts = 5
-        for attempt in range(max_attempts):
-            try:
-                cert = crypto_util.probe_sni(
-                    b'localhost', b'0.0.0.0', self.port)
-            except errors.Error:
-                self.assertTrue(attempt + 1 < max_attempts, "Timeout!")
-                time.sleep(1)  # wait until thread starts
-            else:
-                self.assertEqual(jose.ComparableX509(cert),
-                                 test_util.load_comparable_cert(
-                                     'rsa2048_cert.pem'))
-                break
+    @mock.patch('acme.standalone.logger')
+    def test_it(self, mock_logger):
+        # Use a Queue because mock objects aren't thread safe.
+        q = queue.Queue()  # type: queue.Queue[int]
+        # Add port number to the queue.
+        mock_logger.info.side_effect = lambda *args: q.put(args[-1])
+        self.thread.start()
 
-            if attempt == 0:
-                # the first attempt is always meant to fail, so we can test
-                # the socket failure code-path for probe_sni, as well
-                self.thread.start()
+        # After the timeout, an exception is raised if the queue is empty.
+        port = q.get(timeout=5)
+        cert = crypto_util.probe_sni(b'localhost', b'0.0.0.0', port)
+        self.assertEqual(jose.ComparableX509(cert),
+                         test_util.load_comparable_cert(
+                             'rsa2048_cert.pem'))
 
 
 if __name__ == "__main__":

acme/acme/test_util.py

@@ -10,7 +10,7 @@ import unittest
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 import josepy as jose
-import OpenSSL
+from OpenSSL import crypto
 
 
 def vector_path(*names):
@@ -39,8 +39,8 @@ def _guess_loader(filename, loader_pem, loader_der):
 def load_cert(*names):
     """Load certificate."""
     loader = _guess_loader(
-        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
-    return OpenSSL.crypto.load_certificate(loader, load_vector(*names))
+        names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1)
+    return crypto.load_certificate(loader, load_vector(*names))
 
 
 def load_comparable_cert(*names):
@@ -51,8 +51,8 @@ def load_comparable_cert(*names):
 def load_csr(*names):
     """Load certificate request."""
     loader = _guess_loader(
-        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
-    return OpenSSL.crypto.load_certificate_request(loader, load_vector(*names))
+        names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1)
+    return crypto.load_certificate_request(loader, load_vector(*names))
 
 
 def load_comparable_csr(*names):
@@ -71,8 +71,8 @@ def load_rsa_private_key(*names):
 def load_pyopenssl_private_key(*names):
     """Load pyOpenSSL private key."""
     loader = _guess_loader(
-        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
-    return OpenSSL.crypto.load_privatekey(loader, load_vector(*names))
+        names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1)
+    return crypto.load_privatekey(loader, load_vector(*names))
 
 
 def skip_unless(condition, reason):  # pragma: no cover

acme/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+norecursedirs = .* build dist CVS _darcs {arch} *.egg

acme/setup.py

@@ -1,10 +1,9 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
-
-version = '0.23.0.dev0'
+version = '0.28.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
@@ -19,6 +18,7 @@ install_requires = [
     'pyrfc3339',
     'pytz',
     'requests[security]>=2.4.1',  # security extras added in 2.4.1
+    'requests-toolbelt>=0.3.0',
     'setuptools',
     'six>=1.9.0',  # needed for python_2_unicode_compatible
 ]
@@ -34,6 +34,19 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
 
 setup(
     name='acme',
@@ -45,7 +58,7 @@ setup(
     license='Apache License 2.0',
     python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
-        'Development Status :: 3 - Alpha',
+        'Development Status :: 5 - Production/Stable',
         'Intended Audience :: Developers',
         'License :: OSI Approved :: Apache Software License',
         'Programming Language :: Python',
@@ -55,6 +68,7 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
     ],
@@ -66,5 +80,7 @@ setup(
         'dev': dev_extras,
         'docs': docs_extras,
     },
+    tests_require=["pytest"],
     test_suite='acme',
+    cmdclass={"test": PyTest},
 )

appveyor.yml

@@ -0,0 +1,12 @@
+# AppVeyor CI pipeline, executed on Windows Server 2016/2012 R2
+branches:
+  only:
+    - master
+    - /^\d+\.\d+\.x$/ # version branches like X.X.X
+    - /^test-.*$/
+
+build: off
+
+test_script:
+  - ps: Write-Host "Hello, world!"
+  

certbot-apache/certbot_apache/apache_util.py

@@ -1,6 +1,11 @@
 """ Utility functions for certbot-apache plugin """
+import binascii
 import os
+import six
+import struct
+import time
 
+from certbot import crypto_util
 from certbot import util
 
 def get_mod_deps(mod_name):
@@ -98,3 +103,52 @@ def parse_define_file(filepath, varname):
             var_parts = v[2:].partition("=")
             return_vars[var_parts[0]] = var_parts[2]
     return return_vars
+
+
+def unique_id():
+    """ Returns an unique id to be used as a VirtualHost identifier"""
+    return binascii.hexlify(os.urandom(16)).decode("utf-8")
+
+
+def get_apache_ocsp_struct(ttl, ocsp_response):
+    """Create Apache OCSP response structure to be used in response cache
+
+    :param int ttl: Time-To-Live in seocnds
+    :param str ocsp_response: OCSP response data
+
+    :returns: Apache OCSP structure
+    :rtype: `str`
+
+    """
+
+    ttl = time.time() + ttl
+    # As microseconds
+    ttl_struct = struct.pack('l', int(ttl*1000000))
+    return b'\x01'.join([ttl_struct, ocsp_response])
+
+
+def certid_sha1_hex(cert_path):
+    """Hex representation of certificate SHA1 fingerprint
+
+    :param str cert_path: File path to certificate
+
+    :returns: Hex representation SHA1 fingerprint of certificate
+    :rtype: `str`
+
+    """
+    sha1_hex = binascii.hexlify(certid_sha1(cert_path))
+    if isinstance(sha1_hex, six.binary_type):
+        return sha1_hex.decode('utf-8')  # pragma: no cover
+    return sha1_hex  # pragma: no cover
+
+
+def certid_sha1(cert_path):
+    """SHA1 fingerprint of certificate
+
+    :param str cert_path: File path to certificate
+
+    :returns: SHA1 fingerprint bytestring
+    :rtype: `str`
+
+    """
+    return crypto_util.cert_sha1_fingerprint(cert_path)

certbot-apache/certbot_apache/constants.py

@@ -48,3 +48,22 @@ UIR_ARGS = ["always", "set", "Content-Security-Policy",
 
 HEADER_ARGS = {"Strict-Transport-Security": HSTS_ARGS,
                "Upgrade-Insecure-Requests": UIR_ARGS}
+
+AUTOHSTS_STEPS = [60, 300, 900, 3600, 21600, 43200, 86400]
+"""AutoHSTS increase steps: 1min, 5min, 15min, 1h, 6h, 12h, 24h"""
+
+AUTOHSTS_PERMANENT = 31536000
+"""Value for the last max-age of HSTS"""
+
+AUTOHSTS_FREQ = 172800
+"""Minimum time since last increase to perform a new one: 48h"""
+
+MANAGED_COMMENT = "DO NOT REMOVE - Managed by Certbot"
+MANAGED_COMMENT_ID = MANAGED_COMMENT+", VirtualHost id: {0}"
+"""Managed by Certbot comments and the VirtualHost identification template"""
+
+OCSP_APACHE_TTL = 432000
+"""Apache TTL for OCSP response: 5 days"""
+
+OCSP_INTERNAL_TTL = 86400
+"""Internal TTL for OCSP response: 1 day"""

certbot-apache/certbot_apache/display_ops.py

@@ -113,8 +113,7 @@ def _vhost_menu(domain, vhosts):
         code, tag = zope.component.getUtility(interfaces.IDisplay).menu(
             "We were unable to find a vhost with a ServerName "
             "or Address of {0}.{1}Which virtual host would you "
-            "like to choose?\n(note: conf files with multiple "
-            "vhosts are not yet supported)".format(domain, os.linesep),
+            "like to choose?".format(domain, os.linesep),
             choices, force_interactive=True)
     except errors.MissingCommandlineFlag:
         msg = (

certbot-apache/certbot_apache/http_01.py

@@ -2,9 +2,11 @@
 import logging
 import os
 
+from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
 from certbot import errors
-
 from certbot.plugins import common
+from certbot_apache.obj import VirtualHost  # pylint: disable=unused-import
+from certbot_apache.parser import get_aug_path
 
 logger = logging.getLogger(__name__)
 
@@ -51,7 +53,7 @@ class ApacheHttp01(common.TLSSNI01):
         self.challenge_dir = os.path.join(
             self.configurator.config.work_dir,
             "http_challenges")
-        self.moded_vhosts = set()
+        self.moded_vhosts = set()  # type: Set[VirtualHost]
 
     def perform(self):
         """Perform all HTTP-01 challenges."""
@@ -171,4 +173,9 @@ class ApacheHttp01(common.TLSSNI01):
             self.configurator.parser.add_dir(
                 vhost.path, "Include", self.challenge_conf_post)
 
+            if not vhost.enabled:
+                self.configurator.parser.add_dir(
+                    get_aug_path(self.configurator.parser.loc["default"]),
+                    "Include", vhost.filep)
+
             self.moded_vhosts.add(vhost)

certbot-apache/certbot_apache/obj.py

@@ -1,6 +1,7 @@
 """Module contains classes used by the Apache Configurator."""
 import re
 
+from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module
 from certbot.plugins import common
 
 
@@ -140,7 +141,7 @@ class VirtualHost(object):  # pylint: disable=too-few-public-methods
 
     def get_names(self):
         """Return a set of all names."""
-        all_names = set()
+        all_names = set()  # type: Set[str]
         all_names.update(self.aliases)
         # Strip out any scheme:// and <port> field from servername
         if self.name is not None:
@@ -251,7 +252,7 @@ class VirtualHost(object):  # pylint: disable=too-few-public-methods
 
         # already_found acts to keep everything very conservative.
         # Don't allow multiple ip:ports in same set.
-        already_found = set()
+        already_found = set()  # type: Set[str]
 
         for addr in vhost.addrs:
             for local_addr in self.addrs:

certbot-apache/certbot_apache/override_arch.py

@@ -16,14 +16,14 @@ class ArchConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/httpd/conf",
         vhost_files="*.conf",
         logs_root="/var/log/httpd",
+        ctl="apachectl",
         version_cmd=['apachectl', '-v'],
-        apache_cmd="apachectl",
         restart_cmd=['apachectl', 'graceful'],
         conftest_cmd=['apachectl', 'configtest'],
         enmod=None,
         dismod=None,
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=False,
+        handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/httpd/conf",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(

certbot-apache/certbot_apache/override_centos.py

@@ -18,24 +18,33 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/httpd/conf.d",
         vhost_files="*.conf",
         logs_root="/var/log/httpd",
+        ctl="apachectl",
         version_cmd=['apachectl', '-v'],
-        apache_cmd="apachectl",
         restart_cmd=['apachectl', 'graceful'],
+        restart_cmd_alt=['apachectl', 'restart'],
         conftest_cmd=['apachectl', 'configtest'],
         enmod=None,
         dismod=None,
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=False,
+        handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/httpd/conf.d",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
             "certbot_apache", "centos-options-ssl-apache.conf")
     )
 
+    def _prepare_options(self):
+        """
+        Override the options dictionary initialization in order to support
+        alternative restart cmd used in CentOS.
+        """
+        super(CentOSConfigurator, self)._prepare_options()
+        self.options["restart_cmd_alt"][0] = self.option("ctl")
+
     def get_parser(self):
         """Initializes the ApacheParser"""
         return CentOSParser(
-            self.aug, self.conf("server-root"), self.conf("vhost-root"),
+            self.aug, self.option("server_root"), self.option("vhost_root"),
             self.version, configurator=self)
 
 
@@ -46,10 +55,10 @@ class CentOSParser(parser.ApacheParser):
         self.sysconfig_filep = "/etc/sysconfig/httpd"
         super(CentOSParser, self).__init__(*args, **kwargs)
 
-    def update_runtime_variables(self, *args, **kwargs):
+    def update_runtime_variables(self):
         """ Override for update_runtime_variables for custom parsing """
         # Opportunistic, works if SELinux not enforced
-        super(CentOSParser, self).update_runtime_variables(*args, **kwargs)
+        super(CentOSParser, self).update_runtime_variables()
         self.parse_sysconfig_var()
 
     def parse_sysconfig_var(self):

certbot-apache/certbot_apache/override_darwin.py

@@ -16,14 +16,14 @@ class DarwinConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/apache2/other",
         vhost_files="*.conf",
         logs_root="/var/log/apache2",
-        version_cmd=['/usr/sbin/httpd', '-v'],
-        apache_cmd="/usr/sbin/httpd",
+        ctl="apachectl",
+        version_cmd=['apachectl', '-v'],
         restart_cmd=['apachectl', 'graceful'],
         conftest_cmd=['apachectl', 'configtest'],
         enmod=None,
         dismod=None,
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=False,
+        handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/other",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(

certbot-apache/certbot_apache/override_debian.py

@@ -12,6 +12,8 @@ from certbot import util
 from certbot_apache import apache_util
 from certbot_apache import configurator
 
+from certbot.plugins.enhancements import OCSPPrefetchEnhancement
+
 logger = logging.getLogger(__name__)
 
 @zope.interface.provider(interfaces.IPluginFactory)
@@ -23,14 +25,14 @@ class DebianConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/apache2/sites-available",
         vhost_files="*",
         logs_root="/var/log/apache2",
+        ctl="apache2ctl",
         version_cmd=['apache2ctl', '-v'],
-        apache_cmd="apache2ctl",
         restart_cmd=['apache2ctl', 'graceful'],
         conftest_cmd=['apache2ctl', 'configtest'],
         enmod="a2enmod",
         dismod="a2dismod",
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=True,
+        handle_modules=True,
         handle_sites=True,
         challenge_location="/etc/apache2",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
@@ -134,11 +136,14 @@ class DebianConfigurator(configurator.ApacheConfigurator):
         # Generate reversal command.
         # Try to be safe here... check that we can probably reverse before
         # applying enmod command
-        if not util.exe_exists(self.conf("dismod")):
+        if not util.exe_exists(self.option("dismod")):
             raise errors.MisconfigurationError(
                 "Unable to find a2dismod, please make sure a2enmod and "
                 "a2dismod are configured correctly for certbot.")
 
         self.reverter.register_undo_command(
-            temp, [self.conf("dismod"), "-f", mod_name])
-        util.run_script([self.conf("enmod"), mod_name])
+            temp, [self.option("dismod"), "-f", mod_name])
+        util.run_script([self.option("enmod"), mod_name])
+
+
+OCSPPrefetchEnhancement.register(DebianConfigurator)  # pylint: disable=no-member

certbot-apache/certbot_apache/override_gentoo.py

@@ -18,24 +18,33 @@ class GentooConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/apache2/vhosts.d",
         vhost_files="*.conf",
         logs_root="/var/log/apache2",
-        version_cmd=['/usr/sbin/apache2', '-v'],
-        apache_cmd="apache2ctl",
+        ctl="apache2ctl",
+        version_cmd=['apache2ctl', '-v'],
         restart_cmd=['apache2ctl', 'graceful'],
+        restart_cmd_alt=['apache2ctl', 'restart'],
         conftest_cmd=['apache2ctl', 'configtest'],
         enmod=None,
         dismod=None,
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=False,
+        handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/vhosts.d",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
             "certbot_apache", "options-ssl-apache.conf")
     )
 
+    def _prepare_options(self):
+        """
+        Override the options dictionary initialization in order to support
+        alternative restart cmd used in Gentoo.
+        """
+        super(GentooConfigurator, self)._prepare_options()
+        self.options["restart_cmd_alt"][0] = self.option("ctl")
+
     def get_parser(self):
         """Initializes the ApacheParser"""
         return GentooParser(
-            self.aug, self.conf("server-root"), self.conf("vhost-root"),
+            self.aug, self.option("server_root"), self.option("vhost_root"),
             self.version, configurator=self)
 
 
@@ -60,7 +69,7 @@ class GentooParser(parser.ApacheParser):
 
     def update_modules(self):
         """Get loaded modules from httpd process, and add them to DOM"""
-        mod_cmd = [self.configurator.constant("apache_cmd"), "modules"]
+        mod_cmd = [self.configurator.option("ctl"), "modules"]
         matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module")
         for mod in matches:
             self.add_mod(mod.strip())

certbot-apache/certbot_apache/override_suse.py

@@ -16,14 +16,14 @@ class OpenSUSEConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/apache2/vhosts.d",
         vhost_files="*.conf",
         logs_root="/var/log/apache2",
+        ctl="apache2ctl",
         version_cmd=['apache2ctl', '-v'],
-        apache_cmd="apache2ctl",
         restart_cmd=['apache2ctl', 'graceful'],
         conftest_cmd=['apache2ctl', 'configtest'],
         enmod="a2enmod",
         dismod="a2dismod",
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=False,
+        handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/vhosts.d",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(

certbot-apache/certbot_apache/parser.py

@@ -9,12 +9,14 @@ import sys
 
 import six
 
+from acme.magic_typing import Dict, List, Set  # pylint: disable=unused-import, no-name-in-module
 from certbot import errors
 
 logger = logging.getLogger(__name__)
 
 
 class ApacheParser(object):
+    # pylint: disable=too-many-public-methods
     """Class handles the fine details of parsing the Apache Configuration.
 
     .. todo:: Make parsing general... remove sites-available etc...
@@ -38,9 +40,9 @@ class ApacheParser(object):
         # issues with aug.load() after adding new files / defines to parse tree
         self.configurator = configurator
 
-        self.modules = set()
-        self.parser_paths = {}
-        self.variables = {}
+        self.modules = set()  # type: Set[str]
+        self.parser_paths = {}  # type: Dict[str, List[str]]
+        self.variables = {}  # type: Dict[str, str]
 
         self.aug = aug
         # Find configuration root and make sure augeas can parse it.
@@ -67,7 +69,7 @@ class ApacheParser(object):
         # Must also attempt to parse additional virtual host root
         if vhostroot:
             self.parse_file(os.path.abspath(vhostroot) + "/" +
-                            self.configurator.constant("vhost_files"))
+                            self.configurator.option("vhost_files"))
 
         # check to see if there were unparsed define statements
         if version < (2, 4):
@@ -119,7 +121,7 @@ class ApacheParser(object):
             the iteration issue.  Else... parse and enable mods at same time.
 
         """
-        mods = set()
+        mods = set()  # type: Set[str]
         matches = self.find_dir("LoadModule")
         iterator = iter(matches)
         # Make sure prev_size != cur_size for do: while: iteration
@@ -150,7 +152,7 @@ class ApacheParser(object):
         """Get Defines from httpd process"""
 
         variables = dict()
-        define_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D",
+        define_cmd = [self.configurator.option("ctl"), "-t", "-D",
                       "DUMP_RUN_CFG"]
         matches = self.parse_from_subprocess(define_cmd, r"Define: ([^ \n]*)")
         try:
@@ -177,7 +179,7 @@ class ApacheParser(object):
         # configuration files
         _ = self.find_dir("Include")
 
-        inc_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D",
+        inc_cmd = [self.configurator.option("ctl"), "-t", "-D",
                    "DUMP_INCLUDES"]
         matches = self.parse_from_subprocess(inc_cmd, r"\(.*\) (.*)")
         if matches:
@@ -188,7 +190,7 @@ class ApacheParser(object):
     def update_modules(self):
         """Get loaded modules from httpd process, and add them to DOM"""
 
-        mod_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D",
+        mod_cmd = [self.configurator.option("ctl"), "-t", "-D",
                        "DUMP_MODULES"]
         matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module")
         for mod in matches:
@@ -349,6 +351,37 @@ class ApacheParser(object):
         else:
             self.aug.set(first_dir + "/arg", args)
 
+    def add_comment(self, aug_conf_path, comment):
+        """Adds the comment to the augeas path
+
+        :param str aug_conf_path: Augeas configuration path to add directive
+        :param str comment: Comment content
+
+        """
+        self.aug.set(aug_conf_path + "/#comment[last() + 1]", comment)
+
+    def find_comments(self, arg, start=None):
+        """Finds a comment with specified content from the provided DOM path
+
+        :param str arg: Comment content to search
+        :param str start: Beginning Augeas path to begin looking
+
+        :returns: List of augeas paths containing the comment content
+        :rtype: list
+
+        """
+        if not start:
+            start = get_aug_path(self.root)
+
+        comments = self.aug.match("%s//*[label() = '#comment']" % start)
+
+        results = []
+        for comment in comments:
+            c_content = self.aug.get(comment)
+            if c_content and arg in c_content:
+                results.append(comment)
+        return results
+
     def find_dir(self, directive, arg=None, start=None, exclude=True):
         """Finds directive in the configuration.
 
@@ -408,7 +441,7 @@ class ApacheParser(object):
         else:
             arg_suffix = "/*[self::arg=~regexp('%s')]" % case_i(arg)
 
-        ordered_matches = []
+        ordered_matches = []  # type: List[str]
 
         # TODO: Wildcards should be included in alphabetical order
         # https://httpd.apache.org/docs/2.4/mod/core.html#include

certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test

@@ -46,6 +46,7 @@ function Cleanup() {
 
 # if our environment asks us to enable modules, do our best!
 if [ "$1" = --debian-modules ] ; then
+    sudo apt-get install -y apache2
     sudo apt-get install -y libapache2-mod-wsgi
     sudo apt-get install -y libapache2-mod-macro
 

certbot-apache/certbot_apache/tests/autohsts_test.py

@@ -0,0 +1,187 @@
+# pylint: disable=too-many-public-methods,too-many-lines
+"""Test for certbot_apache.configurator AutoHSTS functionality"""
+import re
+import unittest
+import mock
+# six is used in mock.patch()
+import six  # pylint: disable=unused-import
+
+from certbot import errors
+from certbot_apache import constants
+from certbot_apache.tests import util
+
+
+class AutoHSTSTest(util.ApacheTest):
+    """Tests for AutoHSTS feature"""
+    # pylint: disable=protected-access
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        super(AutoHSTSTest, self).setUp()
+
+        self.config = util.get_apache_configurator(
+            self.config_path, self.vhost_path, self.config_dir, self.work_dir)
+        self.config.parser.modules.add("headers_module")
+        self.config.parser.modules.add("mod_headers.c")
+        self.config.parser.modules.add("ssl_module")
+        self.config.parser.modules.add("mod_ssl.c")
+
+        self.vh_truth = util.get_vh_truth(
+            self.temp_dir, "debian_apache_2_4/multiple_vhosts")
+
+    def get_autohsts_value(self, vh_path):
+        """ Get value from Strict-Transport-Security header """
+        header_path = self.config.parser.find_dir("Header", None, vh_path)
+        if header_path:
+            pat = '(?:[ "]|^)(strict-transport-security)(?:[ "]|$)'
+            for head in header_path:
+                if re.search(pat, self.config.parser.aug.get(head).lower()):
+                    return self.config.parser.aug.get(head.replace("arg[3]",
+                                                                   "arg[4]"))
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod")
+    def test_autohsts_enable_headers_mod(self, mock_enable, _restart):
+        self.config.parser.modules.discard("headers_module")
+        self.config.parser.modules.discard("mod_header.c")
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        self.assertTrue(mock_enable.called)
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    def test_autohsts_deploy_already_exists(self, _restart):
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        self.assertRaises(errors.PluginEnhancementAlreadyPresent,
+                          self.config.enable_autohsts,
+                          mock.MagicMock(), ["ocspvhost.com"])
+
+    @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0)
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.prepare")
+    def test_autohsts_increase(self, mock_prepare, _mock_restart):
+        self.config._prepared = False
+        maxage = "\"max-age={0}\""
+        initial_val = maxage.format(constants.AUTOHSTS_STEPS[0])
+        inc_val = maxage.format(constants.AUTOHSTS_STEPS[1])
+
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        # Verify initial value
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          initial_val)
+        # Increase
+        self.config.update_autohsts(mock.MagicMock())
+        # Verify increased value
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          inc_val)
+        self.assertTrue(mock_prepare.called)
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator._autohsts_increase")
+    def test_autohsts_increase_noop(self, mock_increase, _restart):
+        maxage = "\"max-age={0}\""
+        initial_val = maxage.format(constants.AUTOHSTS_STEPS[0])
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        # Verify initial value
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          initial_val)
+
+        self.config.update_autohsts(mock.MagicMock())
+        # Freq not patched, so value shouldn't increase
+        self.assertFalse(mock_increase.called)
+
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0)
+    def test_autohsts_increase_no_header(self, _restart):
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        # Remove the header
+        dir_locs = self.config.parser.find_dir("Header", None,
+                                              self.vh_truth[7].path)
+        dir_loc = "/".join(dir_locs[0].split("/")[:-1])
+        self.config.parser.aug.remove(dir_loc)
+        self.assertRaises(errors.PluginError,
+                          self.config.update_autohsts,
+                          mock.MagicMock())
+
+    @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0)
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    def test_autohsts_increase_and_make_permanent(self, _mock_restart):
+        maxage = "\"max-age={0}\""
+        max_val = maxage.format(constants.AUTOHSTS_PERMANENT)
+        mock_lineage = mock.MagicMock()
+        mock_lineage.key_path = "/etc/apache2/ssl/key-certbot_15.pem"
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        for i in range(len(constants.AUTOHSTS_STEPS)-1):
+            # Ensure that value is not made permanent prematurely
+            self.config.deploy_autohsts(mock_lineage)
+            self.assertNotEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                                 max_val)
+            self.config.update_autohsts(mock.MagicMock())
+            # Value should match pre-permanent increment step
+            cur_val = maxage.format(constants.AUTOHSTS_STEPS[i+1])
+            self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                              cur_val)
+        # Ensure that the value is raised to max
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          maxage.format(constants.AUTOHSTS_STEPS[-1]))
+        # Make permanent
+        self.config.deploy_autohsts(mock_lineage)
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          max_val)
+
+    def test_autohsts_update_noop(self):
+        with mock.patch("time.time") as mock_time:
+            # Time mock is used to make sure that the execution does not
+            # continue when no autohsts entries exist in pluginstorage
+            self.config.update_autohsts(mock.MagicMock())
+            self.assertFalse(mock_time.called)
+
+    def test_autohsts_make_permanent_noop(self):
+        self.config.storage.put = mock.MagicMock()
+        self.config.deploy_autohsts(mock.MagicMock())
+        # Make sure that the execution does not continue when no entries in store
+        self.assertFalse(self.config.storage.put.called)
+
+    @mock.patch("certbot_apache.display_ops.select_vhost")
+    def test_autohsts_no_ssl_vhost(self, mock_select):
+        mock_select.return_value = self.vh_truth[0]
+        with mock.patch("certbot_apache.configurator.logger.warning") as mock_log:
+            self.assertRaises(errors.PluginError,
+                              self.config.enable_autohsts,
+                              mock.MagicMock(), "invalid.example.com")
+            self.assertTrue(
+                "Certbot was not able to find SSL" in mock_log.call_args[0][0])
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.add_vhost_id")
+    def test_autohsts_dont_enhance_twice(self, mock_id, _restart):
+        mock_id.return_value = "1234567"
+        self.config.enable_autohsts(mock.MagicMock(),
+                                    ["ocspvhost.com", "ocspvhost.com"])
+        self.assertEquals(mock_id.call_count, 1)
+
+    def test_autohsts_remove_orphaned(self):
+        # pylint: disable=protected-access
+        self.config._autohsts_fetch_state()
+        self.config._autohsts["orphan_id"] = {"laststep": 0, "timestamp": 0}
+
+        self.config._autohsts_save_state()
+        self.config.update_autohsts(mock.MagicMock())
+        self.assertFalse("orphan_id" in self.config._autohsts)
+        # Make sure it's removed from the pluginstorage file as well
+        self.config._autohsts = None
+        self.config._autohsts_fetch_state()
+        self.assertFalse(self.config._autohsts)
+
+    def test_autohsts_make_permanent_vhost_not_found(self):
+        # pylint: disable=protected-access
+        self.config._autohsts_fetch_state()
+        self.config._autohsts["orphan_id"] = {"laststep": 999, "timestamp": 0}
+        self.config._autohsts_save_state()
+        with mock.patch("certbot_apache.configurator.logger.warning") as mock_log:
+            self.config.deploy_autohsts(mock.MagicMock())
+            self.assertTrue(mock_log.called)
+            self.assertTrue(
+                "VirtualHost with id orphan_id was not" in mock_log.call_args[0][0])
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/centos_test.py

@@ -4,6 +4,8 @@ import unittest
 
 import mock
 
+from certbot import errors
+
 from certbot_apache import obj
 from certbot_apache import override_centos
 from certbot_apache.tests import util
@@ -121,5 +123,19 @@ class MultipleVhostsTestCentOS(util.ApacheTest):
         self.assertTrue("MOCK_NOSEP" in self.config.parser.variables.keys())
         self.assertEqual("NOSEP_VAL", self.config.parser.variables["NOSEP_TWO"])
 
+    @mock.patch("certbot_apache.configurator.util.run_script")
+    def test_alt_restart_works(self, mock_run_script):
+        mock_run_script.side_effect = [None, errors.SubprocessError, None]
+        self.config.restart()
+        self.assertEquals(mock_run_script.call_count, 3)
+
+    @mock.patch("certbot_apache.configurator.util.run_script")
+    def test_alt_restart_errors(self, mock_run_script):
+        mock_run_script.side_effect = [None,
+                                       errors.SubprocessError,
+                                       errors.SubprocessError]
+        self.assertRaises(errors.MisconfigurationError, self.config.restart)
+
+
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/configurator_test.py

@@ -115,9 +115,22 @@ class MultipleVhostsTest(util.ApacheTest):
         # Weak test..
         ApacheConfigurator.add_parser_arguments(mock.MagicMock())
 
+    def test_add_parser_arguments_all_configurators(self):  # pylint: disable=no-self-use
+        from certbot_apache.entrypoint import OVERRIDE_CLASSES
+        for cls in OVERRIDE_CLASSES.values():
+            cls.add_parser_arguments(mock.MagicMock())
+
+    def test_all_configurators_defaults_defined(self):
+        from certbot_apache.entrypoint import OVERRIDE_CLASSES
+        from certbot_apache.configurator import ApacheConfigurator
+        parameters = set(ApacheConfigurator.OS_DEFAULTS.keys())
+        for cls in OVERRIDE_CLASSES.values():
+            self.assertTrue(parameters.issubset(set(cls.OS_DEFAULTS.keys())))
+
     def test_constant(self):
-        self.assertEqual(self.config.constant("server_root"), "/etc/apache2")
-        self.assertEqual(self.config.constant("nonexistent"), None)
+        self.assertTrue("debian_apache_2_4/multiple_vhosts/apache" in
+                        self.config.option("server_root"))
+        self.assertEqual(self.config.option("nonexistent"), None)
 
     @certbot_util.patch_get_utility()
     def test_get_all_names(self, mock_getutility):
@@ -246,7 +259,7 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot_apache.display_ops.select_vhost")
     def test_choose_vhost_select_vhost_with_temp(self, mock_select):
         mock_select.return_value = self.vh_truth[0]
-        chosen_vhost = self.config.choose_vhost("none.com", temp=True)
+        chosen_vhost = self.config.choose_vhost("none.com", create_if_no_ssl=False)
         self.assertEqual(self.vh_truth[0], chosen_vhost)
 
     @mock.patch("certbot_apache.display_ops.select_vhost")
@@ -353,14 +366,11 @@ class MultipleVhostsTest(util.ApacheTest):
 
         self.config.parser.find_dir = mock_find_dir
         mock_add.reset_mock()
-
         self.config._add_dummy_ssl_directives(self.vh_truth[0])  # pylint: disable=protected-access
-        tried_to_add = []
         for a in mock_add.call_args_list:
-            tried_to_add.append(a[0][1] == "Include" and
-                                a[0][2] == self.config.mod_ssl_conf)
-        # Include shouldn't be added, as patched find_dir "finds" existing one
-        self.assertFalse(any(tried_to_add))
+            if a[0][1] == "Include" and a[0][2] == self.config.mod_ssl_conf:
+                self.fail("Include shouldn't be added, as patched find_dir 'finds' existing one") \
+                    # pragma: no cover
 
     def test_deploy_cert(self):
         self.config.parser.modules.add("ssl_module")
@@ -654,22 +664,10 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertEqual(ssl_vhost_slink.name, "nonsym.link")
 
     def test_make_vhost_ssl_nonexistent_vhost_path(self):
-        def conf_side_effect(arg):
-            """ Mock function for ApacheConfigurator.conf """
-            confvars = {
-                "vhost-root": "/tmp/nonexistent",
-                "le_vhost_ext": "-le-ssl.conf",
-                "handle-sites": True}
-            return confvars[arg]
-
-        with mock.patch(
-                "certbot_apache.configurator.ApacheConfigurator.conf"
-        ) as mock_conf:
-            mock_conf.side_effect = conf_side_effect
-            ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[1])
-            self.assertEqual(os.path.dirname(ssl_vhost.filep),
-                             os.path.dirname(os.path.realpath(
-                                 self.vh_truth[1].filep)))
+        ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[1])
+        self.assertEqual(os.path.dirname(ssl_vhost.filep),
+                            os.path.dirname(os.path.realpath(
+                                self.vh_truth[1].filep)))
 
     def test_make_vhost_ssl(self):
         ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])
@@ -936,6 +934,22 @@ class MultipleVhostsTest(util.ApacheTest):
             errors.PluginError,
             self.config.enhance, "certbot.demo", "unknown_enhancement")
 
+    def test_enhance_no_ssl_vhost(self):
+        with mock.patch("certbot_apache.configurator.logger.warning") as mock_log:
+            self.assertRaises(errors.PluginError, self.config.enhance,
+                              "certbot.demo", "redirect")
+            # Check that correct logger.warning was printed
+            self.assertTrue("not able to find" in mock_log.call_args[0][0])
+            self.assertTrue("\"redirect\"" in mock_log.call_args[0][0])
+
+            mock_log.reset_mock()
+
+            self.assertRaises(errors.PluginError, self.config.enhance,
+                              "certbot.demo", "ensure-http-header", "Test")
+            # Check that correct logger.warning was printed
+            self.assertTrue("not able to find" in mock_log.call_args[0][0])
+            self.assertTrue("Test" in mock_log.call_args[0][0])
+
     @mock.patch("certbot.util.exe_exists")
     def test_ocsp_stapling(self, mock_exe):
         self.config.parser.update_runtime_variables = mock.Mock()
@@ -945,6 +959,7 @@ class MultipleVhostsTest(util.ApacheTest):
         mock_exe.return_value = True
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "staple-ocsp")
 
         # Get the ssl vhost for certbot.demo
@@ -971,6 +986,7 @@ class MultipleVhostsTest(util.ApacheTest):
         mock_exe.return_value = True
 
         # Checking the case with already enabled ocsp stapling configuration
+        self.config.choose_vhost("ocspvhost.com")
         self.config.enhance("ocspvhost.com", "staple-ocsp")
 
         # Get the ssl vhost for letsencrypt.demo
@@ -995,6 +1011,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.parser.modules.add("mod_ssl.c")
         self.config.parser.modules.add("socache_shmcb_module")
         self.config.get_version = mock.Mock(return_value=(2, 2, 0))
+        self.config.choose_vhost("certbot.demo")
 
         self.assertRaises(errors.PluginError,
                 self.config.enhance, "certbot.demo", "staple-ocsp")
@@ -1020,6 +1037,7 @@ class MultipleVhostsTest(util.ApacheTest):
         mock_exe.return_value = True
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "ensure-http-header",
                             "Strict-Transport-Security")
 
@@ -1039,7 +1057,8 @@ class MultipleVhostsTest(util.ApacheTest):
         # skip the enable mod
         self.config.parser.modules.add("headers_module")
 
-        # This will create an ssl vhost for certbot.demo
+        # This will create an ssl vhost for encryption-example.demo
+        self.config.choose_vhost("encryption-example.demo")
         self.config.enhance("encryption-example.demo", "ensure-http-header",
                             "Strict-Transport-Security")
 
@@ -1058,6 +1077,7 @@ class MultipleVhostsTest(util.ApacheTest):
         mock_exe.return_value = True
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "ensure-http-header",
                             "Upgrade-Insecure-Requests")
 
@@ -1079,7 +1099,8 @@ class MultipleVhostsTest(util.ApacheTest):
         # skip the enable mod
         self.config.parser.modules.add("headers_module")
 
-        # This will create an ssl vhost for certbot.demo
+        # This will create an ssl vhost for encryption-example.demo
+        self.config.choose_vhost("encryption-example.demo")
         self.config.enhance("encryption-example.demo", "ensure-http-header",
                             "Upgrade-Insecure-Requests")
 
@@ -1097,6 +1118,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.get_version = mock.Mock(return_value=(2, 2))
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "redirect")
 
         # These are not immediately available in find_dir even with save() and
@@ -1147,6 +1169,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.save()
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "redirect")
 
         # These are not immediately available in find_dir even with save() and
@@ -1213,6 +1236,9 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.parser.modules.add("rewrite_module")
         self.config.get_version = mock.Mock(return_value=(2, 3, 9))
 
+        # Creates ssl vhost for the domain
+        self.config.choose_vhost("red.blue.purple.com")
+
         self.config.enhance("red.blue.purple.com", "redirect")
         verify_no_redirect = ("certbot_apache.configurator."
                               "ApacheConfigurator._verify_no_certbot_redirect")
@@ -1224,7 +1250,7 @@ class MultipleVhostsTest(util.ApacheTest):
         # Skip the enable mod
         self.config.parser.modules.add("rewrite_module")
         self.config.get_version = mock.Mock(return_value=(2, 3, 9))
-
+        self.config.choose_vhost("red.blue.purple.com")
         self.config.enhance("red.blue.purple.com", "redirect")
         # Clear state about enabling redirect on this run
         # pylint: disable=protected-access
@@ -1446,6 +1472,7 @@ class MultipleVhostsTest(util.ApacheTest):
         # pylint: disable=protected-access
         self.config.parser.modules.add("mod_ssl.c")
         self.config.parser.modules.add("headers_module")
+        self.vh_truth[3].ssl = True
         self.config._wildcard_vhosts["*.certbot.demo"] = [self.vh_truth[3]]
         self.config.enhance("*.certbot.demo", "ensure-http-header",
                             "Upgrade-Insecure-Requests")
@@ -1453,6 +1480,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
     @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard")
     def test_enhance_wildcard_no_install(self, mock_choose):
+        self.vh_truth[3].ssl = True
         mock_choose.return_value = [self.vh_truth[3]]
         self.config.parser.modules.add("mod_ssl.c")
         self.config.parser.modules.add("headers_module")
@@ -1460,6 +1488,21 @@ class MultipleVhostsTest(util.ApacheTest):
                             "Upgrade-Insecure-Requests")
         self.assertTrue(mock_choose.called)
 
+    def test_add_vhost_id(self):
+        for vh in [self.vh_truth[0], self.vh_truth[1], self.vh_truth[2]]:
+            vh_id = self.config.add_vhost_id(vh)
+            self.assertEqual(vh, self.config.find_vhost_by_id(vh_id))
+
+    def test_find_vhost_by_id_404(self):
+        self.assertRaises(errors.PluginError,
+                          self.config.find_vhost_by_id,
+                          "nonexistent")
+
+    def test_add_vhost_id_already_exists(self):
+        first_id = self.config.add_vhost_id(self.vh_truth[0])
+        second_id = self.config.add_vhost_id(self.vh_truth[0])
+        self.assertEqual(first_id, second_id)
+
 
 class AugeasVhostsTest(util.ApacheTest):
     """Test vhosts with illegal names dependent on augeas version."""
@@ -1541,7 +1584,7 @@ class AugeasVhostsTest(util.ApacheTest):
                           broken_vhost)
 
 class MultiVhostsTest(util.ApacheTest):
-    """Test vhosts with illegal names dependent on augeas version."""
+    """Test configuration with multiple virtualhosts in a single file."""
     # pylint: disable=protected-access
 
     def setUp(self):  # pylint: disable=arguments-differ
@@ -1661,7 +1704,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
                                              self.config.updated_mod_ssl_conf_digest)
 
     def _current_ssl_options_hash(self):
-        return crypto_util.sha256sum(self.config.constant("MOD_SSL_CONF_SRC"))
+        return crypto_util.sha256sum(self.config.option("MOD_SSL_CONF_SRC"))
 
     def _assert_current_file(self):
         self.assertTrue(os.path.isfile(self.config.mod_ssl_conf))
@@ -1697,7 +1740,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
             self.assertFalse(mock_logger.warning.called)
         self.assertTrue(os.path.isfile(self.config.mod_ssl_conf))
         self.assertEqual(crypto_util.sha256sum(
-            self.config.constant("MOD_SSL_CONF_SRC")),
+            self.config.option("MOD_SSL_CONF_SRC")),
             self._current_ssl_options_hash())
         self.assertNotEqual(crypto_util.sha256sum(self.config.mod_ssl_conf),
             self._current_ssl_options_hash())
@@ -1713,7 +1756,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
                 "%s has been manually modified; updated file "
                 "saved to %s. We recommend updating %s for security purposes.")
         self.assertEqual(crypto_util.sha256sum(
-            self.config.constant("MOD_SSL_CONF_SRC")),
+            self.config.option("MOD_SSL_CONF_SRC")),
             self._current_ssl_options_hash())
         # only print warning once
         with mock.patch("certbot.plugins.common.logger") as mock_logger:

certbot-apache/certbot_apache/tests/debian_test.py

@@ -20,7 +20,7 @@ class MultipleVhostsTestDebian(util.ApacheTest):
     def setUp(self):  # pylint: disable=arguments-differ
         super(MultipleVhostsTestDebian, self).setUp()
         self.config = util.get_apache_configurator(
-            self.config_path, None, self.config_dir, self.work_dir,
+            self.config_path, self.vhost_path, self.config_dir, self.work_dir,
             os_info="debian")
         self.config = self.mock_deploy_cert(self.config)
         self.vh_truth = util.get_vh_truth(self.temp_dir,
@@ -161,6 +161,8 @@ class MultipleVhostsTestDebian(util.ApacheTest):
         self.config.parser.modules.add("mod_ssl.c")
         self.config.get_version = mock.Mock(return_value=(2, 4, 7))
         mock_exe.return_value = True
+        # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "staple-ocsp")
         self.assertTrue("socache_shmcb_module" in self.config.parser.modules)
 
@@ -172,6 +174,7 @@ class MultipleVhostsTestDebian(util.ApacheTest):
         mock_exe.return_value = True
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "ensure-http-header",
                             "Strict-Transport-Security")
         self.assertTrue("headers_module" in self.config.parser.modules)
@@ -183,6 +186,7 @@ class MultipleVhostsTestDebian(util.ApacheTest):
         mock_exe.return_value = True
         self.config.get_version = mock.Mock(return_value=(2, 2))
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "redirect")
         self.assertTrue("rewrite_module" in self.config.parser.modules)
 

certbot-apache/certbot_apache/tests/gentoo_test.py

@@ -4,6 +4,8 @@ import unittest
 
 import mock
 
+from certbot import errors
+
 from certbot_apache import override_gentoo
 from certbot_apache import obj
 from certbot_apache.tests import util
@@ -115,7 +117,7 @@ class MultipleVhostsTestGentoo(util.ApacheTest):
         self.config.parser.modules = set()
 
         with mock.patch("certbot.util.get_os_info") as mock_osi:
-            # Make sure we have the have the CentOS httpd constants
+            # Make sure we have the have the Gentoo httpd constants
             mock_osi.return_value = ("gentoo", "123")
             self.config.parser.update_runtime_variables()
 
@@ -123,5 +125,11 @@ class MultipleVhostsTestGentoo(util.ApacheTest):
         self.assertEquals(len(self.config.parser.modules), 4)
         self.assertTrue("mod_another.c" in self.config.parser.modules)
 
+    @mock.patch("certbot_apache.configurator.util.run_script")
+    def test_alt_restart_works(self, mock_run_script):
+        mock_run_script.side_effect = [None, errors.SubprocessError, None]
+        self.config.restart()
+        self.assertEquals(mock_run_script.call_count, 3)
+
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/http_01_test.py

@@ -4,47 +4,27 @@ import os
 import unittest
 
 from acme import challenges
+from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
 
 from certbot import achallenges
 from certbot import errors
 
 from certbot.tests import acme_util
-
+from certbot_apache.parser import get_aug_path
 from certbot_apache.tests import util
 
 
 NUM_ACHALLS = 3
 
 
-class ApacheHttp01TestMeta(type):
-    """Generates parmeterized tests for testing perform."""
-    def __new__(mcs, name, bases, class_dict):
-
-        def _gen_test(num_achalls, minor_version):
-            def _test(self):
-                achalls = self.achalls[:num_achalls]
-                vhosts = self.vhosts[:num_achalls]
-                self.config.version = (2, minor_version)
-                self.common_perform_test(achalls, vhosts)
-            return _test
-
-        for i in range(1, NUM_ACHALLS + 1):
-            for j in (2, 4):
-                test_name = "test_perform_{0}_{1}".format(i, j)
-                class_dict[test_name] = _gen_test(i, j)
-        return type.__new__(mcs, name, bases, class_dict)
-
-
 class ApacheHttp01Test(util.ApacheTest):
     """Test for certbot_apache.http_01.ApacheHttp01."""
 
-    __metaclass__ = ApacheHttp01TestMeta
-
     def setUp(self, *args, **kwargs):
         super(ApacheHttp01Test, self).setUp(*args, **kwargs)
 
         self.account_key = self.rsa512jwk
-        self.achalls = []
+        self.achalls = []  # type: List[achallenges.KeyAuthorizationAnnotatedChallenge]
         vh_truth = util.get_vh_truth(
             self.temp_dir, "debian_apache_2_4/multiple_vhosts")
         # Takes the vhosts for encryption-example.demo, certbot.demo, and
@@ -71,7 +51,7 @@ class ApacheHttp01Test(util.ApacheTest):
         self.assertFalse(self.http.perform())
 
     @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod")
-    def test_enable_modules_22(self, mock_enmod):
+    def test_enable_modules_apache_2_2(self, mock_enmod):
         self.config.version = (2, 2)
         self.config.parser.modules.remove("authz_host_module")
         self.config.parser.modules.remove("mod_authz_host.c")
@@ -80,7 +60,7 @@ class ApacheHttp01Test(util.ApacheTest):
         self.assertEqual(enmod_calls[0][0][0], "authz_host")
 
     @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod")
-    def test_enable_modules_24(self, mock_enmod):
+    def test_enable_modules_apache_2_4(self, mock_enmod):
         self.config.parser.modules.remove("authz_core_module")
         self.config.parser.modules.remove("mod_authz_core.c")
 
@@ -137,6 +117,46 @@ class ApacheHttp01Test(util.ApacheTest):
         self.config.config.http01_port = 12345
         self.assertRaises(errors.PluginError, self.http.perform)
 
+    def test_perform_1_achall_apache_2_2(self):
+        self.combinations_perform_test(num_achalls=1, minor_version=2)
+
+    def test_perform_1_achall_apache_2_4(self):
+        self.combinations_perform_test(num_achalls=1, minor_version=4)
+
+    def test_perform_2_achall_apache_2_2(self):
+        self.combinations_perform_test(num_achalls=2, minor_version=2)
+
+    def test_perform_2_achall_apache_2_4(self):
+        self.combinations_perform_test(num_achalls=2, minor_version=4)
+
+    def test_perform_3_achall_apache_2_2(self):
+        self.combinations_perform_test(num_achalls=3, minor_version=2)
+
+    def test_perform_3_achall_apache_2_4(self):
+        self.combinations_perform_test(num_achalls=3, minor_version=4)
+
+    def test_activate_disabled_vhost(self):
+        vhosts = [v for v in self.config.vhosts if v.name == "certbot.demo"]
+        achalls = [
+            achallenges.KeyAuthorizationAnnotatedChallenge(
+                challb=acme_util.chall_to_challb(
+                    challenges.HTTP01(token=((b'a' * 16))),
+                    "pending"),
+                domain="certbot.demo", account_key=self.account_key)]
+        vhosts[0].enabled = False
+        self.common_perform_test(achalls, vhosts)
+        matches = self.config.parser.find_dir(
+            "Include", vhosts[0].filep,
+            get_aug_path(self.config.parser.loc["default"]))
+        self.assertEqual(len(matches), 1)
+
+    def combinations_perform_test(self, num_achalls, minor_version):
+        """Test perform with the given achall count and Apache version."""
+        achalls = self.achalls[:num_achalls]
+        vhosts = self.vhosts[:num_achalls]
+        self.config.version = (2, minor_version)
+        self.common_perform_test(achalls, vhosts)
+
     def common_perform_test(self, achalls, vhosts):
         """Tests perform with the given achalls."""
         challenge_dir = self.http.challenge_dir

certbot-apache/certbot_apache/tests/ocsp_prefetch_test.py

@@ -0,0 +1,327 @@
+"""Test for certbot_apache.configurator OCSP Prefetching functionality"""
+import base64
+import json
+import os
+import unittest
+import mock
+# six is used in mock.patch()
+import six  # pylint: disable=unused-import
+import sys
+
+from acme.magic_typing import Dict, List, Set, Union  # pylint: disable=unused-import, no-name-in-module
+
+from certbot import errors
+from certbot_apache.tests import util
+
+
+class MockDBM(object):
+    # pylint: disable=missing-docstring
+    """Main mock DBM class for Py3 dbm module"""
+    def __init__(self):
+        self.ndbm = Mockdbm_impl()
+
+
+class Mockdbm_impl(object):
+    """Mock dbm implementation that satisfies both bsddb and dbm interfaces"""
+    # pylint: disable=missing-docstring
+
+    def __init__(self):
+        self.library = 'Berkeley DB'
+        self.name = 'ndbm'
+
+    def open(self, path, mode):
+        return Mockdb(path, mode)
+
+    def hashopen(self, path, mode):
+        return Mockdb(path, mode)
+
+
+class Mockdb(object):
+    """Mock dbm.db for both bsddb and dbm databases"""
+    # pylint: disable=missing-docstring
+    def __init__(self, path, mode):
+        self._data = dict()  # type: Dict[str, str]
+        if mode == "r" or mode == "w":
+            if not path.endswith(".db"):
+                path = path+".db"
+            with open(path, 'r') as fh:
+                try:
+                    self._data = json.loads(fh.read())
+                except Exception:  # pylint: disable=broad-except
+                    self._data = dict()
+        self.path = path
+        self.mode = mode
+
+    def __setitem__(self, key, item):
+        bkey = base64.b64encode(key)
+        bitem = base64.b64encode(item)
+        self._data[bkey.decode()] = bitem.decode()
+
+    def __getitem__(self, key):
+        bkey = base64.b64encode(key)
+        return base64.b64decode(self._data[bkey.decode()])
+
+    def keys(self):
+        return [base64.b64decode(k) for k in self._data.keys()]
+
+    def sync(self):
+        return
+
+    def close(self):
+        with open(self.path, 'w') as fh:
+            fh.write(json.dumps(self._data))
+
+
+class OCSPPrefetchTest(util.ApacheTest):
+    """Tests for OCSP Prefetch feature"""
+    # pylint: disable=protected-access
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        super(OCSPPrefetchTest, self).setUp()
+
+        self.config = util.get_apache_configurator(
+            self.config_path, self.vhost_path, self.config_dir, self.work_dir,
+            os_info="debian")
+
+        self.lineage = mock.MagicMock(cert_path="cert", chain_path="chain")
+        self.config.parser.modules.add("headers_module")
+        self.config.parser.modules.add("mod_headers.c")
+        self.config.parser.modules.add("ssl_module")
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("socache_dbm_module")
+        self.config.parser.modules.add("mod_socache_dbm.c")
+
+        self.vh_truth = util.get_vh_truth(
+            self.temp_dir, "debian_apache_2_4/multiple_vhosts")
+        self.config._ensure_ocsp_dirs()
+        self.db_path = os.path.join(self.config_dir, "ocsp", "ocsp_cache")
+        self.db_fullpath = self.db_path + ".db"
+
+    def _call_mocked(self, func, *args, **kwargs):
+        """Helper method to call functins with mock stack"""
+
+        db_fullpath = self.db_path + ".db"
+        def mock_restart():
+            """Mock ApacheConfigurator.restart that creates the dbm file"""
+            # Mock the Apache dbm file creation
+            open(db_fullpath, 'a').close()
+
+        ver_path = "certbot_apache.configurator.ApacheConfigurator.get_version"
+        res_path = "certbot_apache.configurator.ApacheConfigurator.restart"
+        cry_path = "certbot.crypto_util.cert_sha1_fingerprint"
+
+        with mock.patch(ver_path) as mock_ver:
+            mock_ver.return_value = (2, 4, 10)
+            with mock.patch(cry_path) as mock_cry:
+                mock_cry.return_value = b'j\x056\x1f\xfa\x08B\xe8D\xa1Bn\xeb*A\xebWx\xdd\xfe'
+                with mock.patch(res_path, side_effect=mock_restart):
+                    return func(*args, **kwargs)
+
+    def call_mocked_py2(self, func, *args, **kwargs):
+        """Calls methods with imports mocked to suit Py2 environment"""
+        if 'dbm' in sys.modules.keys():
+            sys.modules['dbm'] = None
+        sys.modules['bsddb'] = Mockdbm_impl()
+        return self._call_mocked(func, *args, **kwargs)
+
+    def call_mocked_py3(self, func, *args, **kwargs):
+        """Calls methods with imports mocked to suit Py3 environment"""
+        real_import = six.moves.builtins.__import__
+
+        def mock_import(*args, **kwargs):
+            """Mock import to raise ImportError for Py2 specific module to make
+            ApacheConfigurator pick the correct one for Python3 regardless of the
+            python version the tests are running under."""
+            if args[0] == "bsddb":
+                raise ImportError
+            return real_import(*args, **kwargs)
+
+        with mock.patch('six.moves.builtins.__import__', side_effect=mock_import):
+            sys.modules['dbm'] = MockDBM()
+            return self._call_mocked(func, *args, **kwargs)
+
+    @mock.patch("certbot_apache.override_debian.DebianConfigurator.enable_mod")
+    def test_ocsp_prefetch_enable_mods(self, mock_enable):
+        self.config.parser.modules.discard("socache_dbm_module")
+        self.config.parser.modules.discard("mod_socache_dbm.c")
+        self.config.parser.modules.discard("headers_module")
+        self.config.parser.modules.discard("mod_header.c")
+
+        ref_path = "certbot_apache.configurator.ApacheConfigurator._ocsp_refresh"
+        with mock.patch(ref_path):
+            self.call_mocked_py2(self.config.enable_ocsp_prefetch,
+                             self.lineage,
+                             ["ocspvhost.com"])
+        self.assertTrue(mock_enable.called)
+        self.assertEquals(len(self.config._ocsp_prefetch), 1)
+
+    @mock.patch("certbot_apache.override_debian.DebianConfigurator.enable_mod")
+    def test_ocsp_prefetch_enable_error(self, _mock_enable):
+        ref_path = "certbot_apache.configurator.ApacheConfigurator._ocsp_refresh"
+        self.config.recovery_routine = mock.MagicMock()
+        with mock.patch(ref_path, side_effect=errors.PluginError("failed")):
+            self.assertRaises(errors.PluginError,
+                              self.call_mocked_py2,
+                              self.config.enable_ocsp_prefetch,
+                              self.lineage,
+                              ["ocspvhost.com"])
+        self.assertTrue(self.config.recovery_routine.called)
+
+    @mock.patch("certbot_apache.constants.OCSP_INTERNAL_TTL", 0)
+    def test_ocsp_prefetch_refresh(self):
+        def ocsp_req_mock(workfile):
+            """Method to mock the OCSP request and write response to file"""
+            with open(workfile, 'w') as fh:
+                fh.write("MOCKRESPONSE")
+            return True
+
+        ocsp_path = "certbot.ocsp.OCSPResponseHandler.ocsp_request_to_file"
+        with mock.patch(ocsp_path, side_effect=ocsp_req_mock):
+            self.call_mocked_py2(self.config.enable_ocsp_prefetch,
+                                self.lineage,
+                                ["ocspvhost.com"])
+        odbm = self.config._ocsp_dbm_open(self.db_path)
+        self.assertEquals(len(odbm.keys()), 1)
+        # The actual response data is prepended by Apache timestamp
+        self.assertTrue(odbm[list(odbm.keys())[0]].endswith(b'MOCKRESPONSE'))
+        self.config._ocsp_dbm_close(odbm)
+
+        with mock.patch(ocsp_path, side_effect=ocsp_req_mock) as mock_ocsp:
+            self.call_mocked_py2(self.config.update_ocsp_prefetch, None)
+            self.assertTrue(mock_ocsp.called)
+
+    def test_ocsp_prefetch_refresh_noop(self):
+        def ocsp_req_mock(workfile):
+            """Method to mock the OCSP request and write response to file"""
+            with open(workfile, 'w') as fh:
+                fh.write("MOCKRESPONSE")
+            return True
+
+        ocsp_path = "certbot.ocsp.OCSPResponseHandler.ocsp_request_to_file"
+        with mock.patch(ocsp_path, side_effect=ocsp_req_mock):
+            self.call_mocked_py2(self.config.enable_ocsp_prefetch,
+                                self.lineage,
+                                ["ocspvhost.com"])
+        self.assertEquals(len(self.config._ocsp_prefetch), 1)
+        refresh_path = "certbot_apache.configurator.ApacheConfigurator._ocsp_refresh"
+        with mock.patch(refresh_path) as mock_refresh:
+            self.call_mocked_py2(self.config.update_ocsp_prefetch, None)
+            self.assertFalse(mock_refresh.called)
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.config_test")
+    def test_ocsp_prefetch_backup_db(self, _mock_test):
+        def ocsp_del_db():
+            """Side effect of _reload() that deletes the DBM file, like Apache
+            does when restarting"""
+            os.remove(self.db_fullpath)
+            self.assertFalse(os.path.isfile(self.db_fullpath))
+
+        # Make sure that the db file exists
+        open(self.db_fullpath, 'a').close()
+        odbm = self.call_mocked_py2(self.config._ocsp_dbm_open, self.db_path)
+        odbm[b'mock_key'] = b'mock_value'
+        self.config._ocsp_dbm_close(odbm)
+
+        # Mock OCSP prefetch dict to signify that there should be a db
+        self.config._ocsp_prefetch = {"mock": "value"}
+        rel_path = "certbot_apache.configurator.ApacheConfigurator._reload"
+        with mock.patch(rel_path, side_effect=ocsp_del_db):
+            self.config.restart()
+
+        odbm = self.config._ocsp_dbm_open(self.db_path)
+        self.assertEquals(odbm[b'mock_key'], b'mock_value')
+        self.config._ocsp_dbm_close(odbm)
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.config_test")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator._reload")
+    def test_ocsp_prefetch_backup_db_error(self, _mock_reload, _mock_test):
+        log_path = "certbot_apache.configurator.logger.debug"
+        log_string = "Encountered an issue while trying to backup OCSP dbm file"
+        log_string2 = "Encountered an issue when trying to restore OCSP dbm file"
+        self.config._ocsp_prefetch = {"mock": "value"}
+        with mock.patch("shutil.copy2", side_effect=IOError):
+            with mock.patch(log_path) as mock_log:
+                self.config.restart()
+                self.assertTrue(mock_log.called)
+                self.assertEquals(mock_log.call_count, 2)
+                self.assertTrue(log_string in mock_log.call_args_list[0][0][0])
+                self.assertTrue(log_string2 in mock_log.call_args_list[1][0][0])
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    def test_ocsp_prefetch_refresh_fail(self, _mock_restart):
+        ocsp_path = "certbot.ocsp.OCSPResponseHandler.ocsp_request_to_file"
+        log_path = "certbot_apache.configurator.logger.warning"
+        with mock.patch(ocsp_path) as mock_ocsp:
+            mock_ocsp.return_value = False
+            with mock.patch(log_path) as mock_log:
+                self.call_mocked_py2(self.config.enable_ocsp_prefetch,
+                                self.lineage,
+                                ["ocspvhost.com"])
+                self.assertTrue(mock_log.called)
+                self.assertTrue(
+                    "trying to prefetch OCSP" in mock_log.call_args[0][0])
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator._ocsp_refresh_if_needed")
+    def test_ocsp_prefetch_update_noop(self, mock_refresh):
+        self.config.update_ocsp_prefetch(None)
+        self.assertFalse(mock_refresh.called)
+
+    def test_ocsp_prefetch_preflight_check_noerror(self):
+        self.call_mocked_py2(self.config._ensure_ocsp_prefetch_compatibility)
+        self.call_mocked_py3(self.config._ensure_ocsp_prefetch_compatibility)
+        mockdbm_path = "certbot_apache.tests.ocsp_prefetch_test.Mockdbm_impl"
+        with mock.patch(mockdbm_path) as mock_dbm:
+            mock_dbm.library = 'Not Berkeley DB'
+            self.assertRaises(errors.NotSupportedError,
+                              self.call_mocked_py3,
+                              self.config._ensure_ocsp_prefetch_compatibility)
+
+    def test_ocsp_prefetch_open_dbm_no_file(self):
+        open(self.db_fullpath, 'a').close()
+        db_not_exists = self.db_path+"nonsense"
+        self.call_mocked_py2(self.config._ocsp_dbm_open, self.db_path)
+        self.assertRaises(errors.PluginError,
+                          self.call_mocked_py2, self.config._ocsp_dbm_open, db_not_exists)
+
+    def test_ocsp_prefetch_py2_open_file_error(self):
+        open(self.db_fullpath, 'a').close()
+        mock_db = mock.MagicMock()
+        mock_db.hashopen.side_effect = Exception("error")
+        sys.modules["bsddb"] = mock_db
+        self.assertRaises(errors.PluginError,
+                            self.config._ocsp_dbm_open,
+                            self.db_path)
+
+    def test_ocsp_prefetch_py3_open_file_error(self):
+        open(self.db_fullpath, 'a').close()
+        mock_db = mock.MagicMock()
+        mock_db.ndbm.open.side_effect = Exception("error")
+        sys.modules["dbm"] = mock_db
+        sys.modules["bsddb"] = None
+        self.assertRaises(errors.PluginError,
+                            self.config._ocsp_dbm_open,
+                            self.db_path)
+
+    def test_ocsp_prefetch_open_close_py2_noerror(self):
+        expected_val = b'whatever_value'
+        open(self.db_fullpath, 'a').close()
+        db = self.call_mocked_py2(
+            self.config._ocsp_dbm_open, self.db_path)
+        db[b'key'] = expected_val
+        self.call_mocked_py2(self.config._ocsp_dbm_close, db)
+        db2 = self.call_mocked_py2(self.config._ocsp_dbm_open, self.db_path)
+        self.assertEquals(db2[b'key'], expected_val)
+
+    def test_ocsp_prefetch_open_close_py3_noerror(self):
+        expected_val = b'whatever_value'
+        open(self.db_fullpath, 'a').close()
+        db = self.call_mocked_py3(
+            self.config._ocsp_dbm_open, self.db_path)
+        db[b'key'] = expected_val
+        self.call_mocked_py3(self.config._ocsp_dbm_close, db)
+        db2 = self.call_mocked_py3(self.config._ocsp_dbm_open, self.db_path)
+        self.assertEquals(db2[b'key'], expected_val)
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/parser_test.py

@@ -282,11 +282,11 @@ class BasicParserTest(util.ParserTest):
         self.assertRaises(
             errors.PluginError, self.parser.update_runtime_variables)
 
-    @mock.patch("certbot_apache.configurator.ApacheConfigurator.constant")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.option")
     @mock.patch("certbot_apache.parser.subprocess.Popen")
-    def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_const):
+    def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_opt):
         mock_popen.side_effect = OSError
-        mock_const.return_value = "nonexistent"
+        mock_opt.return_value = "nonexistent"
         self.assertRaises(
             errors.MisconfigurationError,
             self.parser.update_runtime_variables)
@@ -299,6 +299,13 @@ class BasicParserTest(util.ParserTest):
             errors.MisconfigurationError,
             self.parser.update_runtime_variables)
 
+    def test_add_comment(self):
+        from certbot_apache.parser import get_aug_path
+        self.parser.add_comment(get_aug_path(self.parser.loc["name"]), "123456")
+        comm = self.parser.find_comments("123456")
+        self.assertEquals(len(comm), 1)
+        self.assertTrue(self.parser.loc["name"] in comm[0])
+
 
 class ParserInitTest(util.ApacheTest):
     def setUp(self):  # pylint: disable=arguments-differ

certbot-apache/certbot_apache/tests/util.py

@@ -87,7 +87,6 @@ class ParserTest(ApacheTest):
 def get_apache_configurator(  # pylint: disable=too-many-arguments, too-many-locals
         config_path, vhost_path,
         config_dir, work_dir, version=(2, 4, 7),
-        conf=None,
         os_info="generic",
         conf_vhost_path=None):
     """Create an Apache Configurator with the specified options.
@@ -98,9 +97,10 @@ def get_apache_configurator(  # pylint: disable=too-many-arguments, too-many-loc
     backups = os.path.join(work_dir, "backups")
     mock_le_config = mock.MagicMock(
         apache_server_root=config_path,
-        apache_vhost_root=conf_vhost_path,
+        apache_vhost_root=None,
         apache_le_vhost_ext="-le-ssl.conf",
         apache_challenge_location=config_path,
+        apache_enmod=None,
         backup_dir=backups,
         config_dir=config_dir,
         http01_port=80,
@@ -108,37 +108,25 @@ def get_apache_configurator(  # pylint: disable=too-many-arguments, too-many-loc
         in_progress_dir=os.path.join(backups, "IN_PROGRESS"),
         work_dir=work_dir)
 
-    orig_os_constant = configurator.ApacheConfigurator(mock_le_config,
-                                                       name="apache",
-                                                       version=version).constant
-
-    def mock_os_constant(key, vhost_path=vhost_path):
-        """Mock default vhost path"""
-        if key == "vhost_root":
-            return vhost_path
-        else:
-            return orig_os_constant(key)
-
-    with mock.patch("certbot_apache.configurator.ApacheConfigurator.constant") as mock_cons:
-        mock_cons.side_effect = mock_os_constant
-        with mock.patch("certbot_apache.configurator.util.run_script"):
-            with mock.patch("certbot_apache.configurator.util."
-                            "exe_exists") as mock_exe_exists:
-                mock_exe_exists.return_value = True
-                with mock.patch("certbot_apache.parser.ApacheParser."
-                                "update_runtime_variables"):
-                    try:
-                        config_class = entrypoint.OVERRIDE_CLASSES[os_info]
-                    except KeyError:
-                        config_class = configurator.ApacheConfigurator
-                    config = config_class(config=mock_le_config, name="apache",
-                        version=version)
-                    # This allows testing scripts to set it a bit more
-                    # quickly
-                    if conf is not None:
-                        config.conf = conf  # pragma: no cover
-
-                    config.prepare()
+    with mock.patch("certbot_apache.configurator.util.run_script"):
+        with mock.patch("certbot_apache.configurator.util."
+                        "exe_exists") as mock_exe_exists:
+            mock_exe_exists.return_value = True
+            with mock.patch("certbot_apache.parser.ApacheParser."
+                            "update_runtime_variables"):
+                try:
+                    config_class = entrypoint.OVERRIDE_CLASSES[os_info]
+                except KeyError:
+                    config_class = configurator.ApacheConfigurator
+                config = config_class(config=mock_le_config, name="apache",
+                    version=version)
+                if not conf_vhost_path:
+                    config_class.OS_DEFAULTS["vhost_root"] = vhost_path
+                else:
+                    # Custom virtualhost path was requested
+                    config.config.apache_vhost_root = conf_vhost_path
+                config.config.apache_ctl = config_class.OS_DEFAULTS["ctl"]
+                config.prepare()
     return config
 
 

certbot-apache/certbot_apache/tls_sni_01.py

@@ -3,6 +3,7 @@
 import os
 import logging
 
+from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
 from certbot.plugins import common
 from certbot.errors import PluginError, MissingCommandlineFlag
 
@@ -93,7 +94,7 @@ class ApacheTlsSni01(common.TLSSNI01):
         :rtype: set
 
         """
-        addrs = set()
+        addrs = set()  # type: Set[obj.Addr]
         config_text = "<IfModule mod_ssl.c>\n"
 
         for achall in self.achalls:
@@ -123,7 +124,8 @@ class ApacheTlsSni01(common.TLSSNI01):
             self.configurator.config.tls_sni_01_port)))
 
         try:
-            vhost = self.configurator.choose_vhost(achall.domain, temp=True)
+            vhost = self.configurator.choose_vhost(achall.domain,
+                                                   create_if_no_ssl=False)
         except (PluginError, MissingCommandlineFlag):
             # We couldn't find the virtualhost for this domain, possibly
             # because it's a new vhost that's not configured yet

certbot-apache/local-oldest-requirements.txt

@@ -1,2 +1,2 @@
-acme[dev]==0.21.1
-certbot[dev]==0.21.1
+acme[dev]==0.25.0
+-e .[dev]

certbot-apache/setup.py

@@ -1,16 +1,14 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.23.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    'acme>=0.21.1',
-    'certbot>=0.21.1',
+    'acme>=0.25.0',
+    'certbot>=0.26.0',
     'mock',
     'python-augeas',
     'setuptools',
@@ -33,7 +31,7 @@ setup(
     license='Apache License 2.0',
     python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
-        'Development Status :: 3 - Alpha',
+        'Development Status :: 5 - Production/Stable',
         'Environment :: Plugins',
         'Intended Audience :: System Administrators',
         'License :: OSI Approved :: Apache Software License',
@@ -45,6 +43,7 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.22.2"
+LE_AUTO_VERSION="0.27.1"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -1055,41 +1055,32 @@ cffi==1.10.0 \
     --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \
     --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5
 ConfigArgParse==0.12.0 \
-    --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339
+    --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \
+    --no-binary ConfigArgParse
 configobj==5.0.6 \
-    --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902
-cryptography==2.0.2 \
-    --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \
-    --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \
-    --hash=sha256:da9291502cbc87dc0284a20c56876e4d2e68deac61cc43df4aec934e44ca97b1 \
-    --hash=sha256:0954f8813095f581669330e0a2d5e726c33ac7f450c1458fac58bab54595e516 \
-    --hash=sha256:d68b0cc40a8432ed3fc84876c519de704d6001800ec22b136e75ae841910c45b \
-    --hash=sha256:2f8ad9580ab4da645cfea52a91d2da99a49a1e76616d8be68441a986fad652b0 \
-    --hash=sha256:cc00b4511294f5f6b65c4e77a1a9c62f52490a63d2c120f3872176b40a82351e \
-    --hash=sha256:cf896020f6a9f095a547b3d672c8db1ef2ed71fca11250731fa1d4a4cb8b1590 \
-    --hash=sha256:e0fdb8322206fa02aa38f71519ff75dce2eb481b7e1110e2936795cb376bb6ee \
-    --hash=sha256:277538466657ca5d6637f80be100242f9831d75138b788d718edd3aab34621f8 \
-    --hash=sha256:2c77eb0560f54ce654ab82d6b2a64327a71ee969b29022bf9746ca311c9f5069 \
-    --hash=sha256:755a7853b679e79d0a799351c092a9b0271f95ff54c8dd8823d8b527a2926a86 \
-    --hash=sha256:77197a2d525e761cdd4c771180b4bd0d80703654c6385e4311cbbbe2beb56fa1 \
-    --hash=sha256:eb8bb79d0ab00c931c8333b745f06fec481a51c52d70acd4ee95d6093ba5c386 \
-    --hash=sha256:131f61de82ef28f3e20beb4bfc24f9692d28cecfd704e20e6c7f070f7793013a \
-    --hash=sha256:ac35435974b2e27cd4520f29c191d7da36f4189aa3264e52c4c6c6d089ab6142 \
-    --hash=sha256:04b6ea99daa2a8460728794213d76d45ad58ea247dc7e7ff148d7dd726e87863 \
-    --hash=sha256:2b9442f8b4c3d575f6cc3db0e856034e0f5a9d55ecd636f52d8c496795b26952 \
-    --hash=sha256:b3d3b3ecba1fe1bdb6f180770a137f877c8f07571f7b2934bb269475bcf0e5e8 \
-    --hash=sha256:670a58c0d75cb0e78e73dd003bd96d4440bbb1f2bc041dcf7b81767ca4fb0ce9 \
-    --hash=sha256:5af84d23bdb86b5e90aca263df1424b43f1748480bfcde3ac2a3cbe622612468 \
-    --hash=sha256:ba22e8eefabdd7aca37d0c0c00d2274000d2cebb5cce9e5a710cb55bf8797b31 \
-    --hash=sha256:b798b22fa7e92b439547323b8b719d217f1e1b7677585cfeeedf3b55c70bb7fb \
-    --hash=sha256:59cff28af8cce96cb7e94a459726e1d88f6f5fa75097f9dcbebd99118d64ea4c \
-    --hash=sha256:fe859e445abc9ba9e97950ddafb904e23234c4ecb76b0fae6c86e80592ce464a \
-    --hash=sha256:655f3c474067f1e277430f23cc0549f0b1dc99b82aec6e53f80b9b2db7f76f11 \
-    --hash=sha256:0ebc2be053c9a03a2f3e20a466e87bf12a51586b3c79bd2a22171b073a805346 \
-    --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \
-    --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \
-    --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab
-enum34==1.1.2 \
+    --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \
+    --no-binary configobj
+cryptography==2.2.2 \
+    --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \
+    --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \
+    --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \
+    --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \
+    --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \
+    --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \
+    --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \
+    --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \
+    --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \
+    --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \
+    --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \
+    --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \
+    --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \
+    --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \
+    --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \
+    --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \
+    --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \
+    --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \
+    --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887
+enum34==1.1.2 ; python_version < '3.4' \
     --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \
     --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501
 funcsigs==1.0.2 \
@@ -1101,9 +1092,9 @@ idna==2.5 \
 ipaddress==1.0.16 \
     --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \
     --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0
-josepy==1.0.1 \
-    --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \
-    --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc
+josepy==1.1.0 \
+    --hash=sha256:1309a25aac3caeff5239729c58ff9b583f7d022ffdb1553406ddfc8e5b52b76e \
+    --hash=sha256:fb5c62c77d26e04df29cb5ecd01b9ce69b6fcc9e521eb1ca193b7faa2afa7086
 linecache2==1.0.0 \
     --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \
     --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c
@@ -1112,7 +1103,8 @@ mock==1.3.0 \
     --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \
     --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6
 ordereddict==1.1 \
-    --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f
+    --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \
+    --no-binary ordereddict
 packaging==16.8 \
     --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \
     --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e
@@ -1138,7 +1130,8 @@ pyRFC3339==1.0 \
     --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \
     --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535
 python-augeas==0.5.0 \
-    --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2
+    --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \
+    --no-binary python-augeas
 pytz==2015.7 \
     --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \
     --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \
@@ -1166,9 +1159,11 @@ unittest2==1.1.0 \
     --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \
     --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579
 zope.component==4.2.2 \
-    --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a
+    --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \
+    --no-binary zope.component
 zope.event==4.1.0 \
-    --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786
+    --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \
+    --no-binary zope.event
 zope.interface==4.1.3 \
     --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \
     --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \
@@ -1187,6 +1182,9 @@ zope.interface==4.1.3 \
     --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \
     --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \
     --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392
+requests-toolbelt==0.8.0 \
+    --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \
+    --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5
 
 # Contains the requirements for the letsencrypt package.
 #
@@ -1199,18 +1197,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.22.2 \
-    --hash=sha256:c8c63bdf0fed6258bdbc892454314ec37bcd1c35a7f62524a083d93ccdfc420d \
-    --hash=sha256:e6e3639293e78397f31f7d99e3c63aff82d91e2b0d50d146ee3c77f830464bef
-acme==0.22.2 \
-    --hash=sha256:59a55244612ee305d2caa6bb4cddd400fb60ec841bf011ed29a2899832a682c2 \
-    --hash=sha256:0ecd0ea369f53d5bc744d6e72717f9af2e1ceb558d109dbd433148851027adb4
-certbot-apache==0.22.2 \
-    --hash=sha256:b5340d4b9190358fde8eb6a5be0def37e32014b5142ee79ef5d2319ccbbde754 \
-    --hash=sha256:3cd26912bb5732d917ddf7aad2fe870090d4ece9a408b2c2de8e9723ec99c759
-certbot-nginx==0.22.2 \
-    --hash=sha256:91feef0d879496835d355e82841f92e5ecb5abbf6f23ea0ee5bbb8f5a92b278a \
-    --hash=sha256:b10bf04c1a20cf878d5e0d1877deb0e0780bc31b0ffda08ce7199bbc39d0753b
+certbot==0.27.1 \
+    --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \
+    --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a
+acme==0.27.1 \
+    --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \
+    --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90
+certbot-apache==0.27.1 \
+    --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \
+    --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854
+certbot-nginx==0.27.1 \
+    --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \
+    --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------

certbot-compatibility-test/Dockerfile

@@ -14,7 +14,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only
 # the above is not likely to change, so by putting it further up the
 # Dockerfile we make sure we cache as much as possible
 
-COPY setup.py README.rst CHANGES.rst MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/
+COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/
 
 # all above files are necessary for setup.py, however, package source
 # code directory has to be copied separately to a subdirectory...

certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py

@@ -59,9 +59,6 @@ class Proxy(configurators_common.Proxy):
             setattr(self.le_config, "apache_" + k,
                     entrypoint.ENTRYPOINT.OS_DEFAULTS[k])
 
-        # An alias
-        self.le_config.apache_handle_modules = self.le_config.apache_handle_mods
-
         self._configurator = entrypoint.ENTRYPOINT(
             config=configuration.NamespaceConfig(self.le_config),
             name="apache")

certbot-compatibility-test/certbot_compatibility_test/test_driver.py

@@ -15,6 +15,7 @@ from six.moves import xrange  # pylint: disable=import-error,redefined-builtin
 from acme import challenges
 from acme import crypto_util
 from acme import messages
+from acme.magic_typing import List, Tuple  # pylint: disable=unused-import, no-name-in-module
 from certbot import achallenges
 from certbot import errors as le_errors
 from certbot.tests import acme_util
@@ -52,9 +53,8 @@ def test_authenticator(plugin, config, temp_dir):
 
     try:
         responses = plugin.perform(achalls)
-    except le_errors.Error as error:
-        logger.error("Performing challenges on %s caused an error:", config)
-        logger.exception(error)
+    except le_errors.Error:
+        logger.error("Performing challenges on %s caused an error:", config, exc_info=True)
         return False
 
     success = True
@@ -82,9 +82,8 @@ def test_authenticator(plugin, config, temp_dir):
     if success:
         try:
             plugin.cleanup(achalls)
-        except le_errors.Error as error:
-            logger.error("Challenge cleanup for %s caused an error:", config)
-            logger.exception(error)
+        except le_errors.Error:
+            logger.error("Challenge cleanup for %s caused an error:", config, exc_info=True)
             success = False
 
         if _dirs_are_unequal(config, backup):
@@ -147,9 +146,8 @@ def test_deploy_cert(plugin, temp_dir, domains):
         try:
             plugin.deploy_cert(domain, cert_path, util.KEY_PATH, cert_path, cert_path)
             plugin.save()  # Needed by the Apache plugin
-        except le_errors.Error as error:
-            logger.error("**** Plugin failed to deploy certificate for %s:", domain)
-            logger.exception(error)
+        except le_errors.Error:
+            logger.error("**** Plugin failed to deploy certificate for %s:", domain, exc_info=True)
             return False
 
     if not _save_and_restart(plugin, "deployed"):
@@ -179,7 +177,7 @@ def test_enhancements(plugin, domains):
                      "enhancements")
         return False
 
-    domains_and_info = [(domain, []) for domain in domains]
+    domains_and_info = [(domain, []) for domain in domains]  # type: List[Tuple[str, List[bool]]]
 
     for domain, info in domains_and_info:
         try:
@@ -192,10 +190,9 @@ def test_enhancements(plugin, domains):
             # Don't immediately fail because a redirect may already be enabled
             logger.warning("*** Plugin failed to enable redirect for %s:", domain)
             logger.warning("%s", error)
-        except le_errors.Error as error:
+        except le_errors.Error:
             logger.error("*** An error occurred while enabling redirect for %s:",
-                         domain)
-            logger.exception(error)
+                         domain, exc_info=True)
 
     if not _save_and_restart(plugin, "enhanced"):
         return False
@@ -222,9 +219,8 @@ def _save_and_restart(plugin, title=None):
         plugin.save(title)
         plugin.restart()
         return True
-    except le_errors.Error as error:
-        logger.error("*** Plugin failed to save and restart server:")
-        logger.exception(error)
+    except le_errors.Error:
+        logger.error("*** Plugin failed to save and restart server:", exc_info=True)
         return False
 
 
@@ -232,9 +228,8 @@ def test_rollback(plugin, config, backup):
     """Tests the rollback checkpoints function"""
     try:
         plugin.rollback_checkpoints(1337)
-    except le_errors.Error as error:
-        logger.error("*** Plugin raised an exception during rollback:")
-        logger.exception(error)
+    except le_errors.Error:
+        logger.error("*** Plugin raised an exception during rollback:", exc_info=True)
         return False
 
     if _dirs_are_unequal(config, backup):
@@ -263,21 +258,21 @@ def _dirs_are_unequal(dir1, dir2):
             logger.error("The following files and directories are only "
                          "present in one directory")
             if dircmp.left_only:
-                logger.error(dircmp.left_only)
+                logger.error(str(dircmp.left_only))
             else:
-                logger.error(dircmp.right_only)
+                logger.error(str(dircmp.right_only))
             return True
         elif dircmp.common_funny or dircmp.funny_files:
             logger.error("The following files and directories could not be "
                          "compared:")
             if dircmp.common_funny:
-                logger.error(dircmp.common_funny)
+                logger.error(str(dircmp.common_funny))
             else:
-                logger.error(dircmp.funny_files)
+                logger.error(str(dircmp.funny_files))
             return True
         elif dircmp.diff_files:
             logger.error("The following files differ:")
-            logger.error(dircmp.diff_files)
+            logger.error(str(dircmp.diff_files))
             return True
 
         for subdir in dircmp.subdirs.itervalues():
@@ -354,9 +349,8 @@ def main():
                     success = test_authenticator(plugin, config, temp_dir)
                 if success and args.install:
                     success = test_installer(args, plugin, config, temp_dir)
-            except errors.Error as error:
-                logger.error("Tests on %s raised:", config)
-                logger.exception(error)
+            except errors.Error:
+                logger.error("Tests on %s raised:", config, exc_info=True)
                 success = False
 
             if success:

certbot-compatibility-test/certbot_compatibility_test/util.py

@@ -26,12 +26,12 @@ def create_le_config(parent_dir):
     config = copy.deepcopy(constants.CLI_DEFAULTS)
 
     le_dir = os.path.join(parent_dir, "certbot")
-    config["config_dir"] = os.path.join(le_dir, "config")
-    config["work_dir"] = os.path.join(le_dir, "work")
-    config["logs_dir"] = os.path.join(le_dir, "logs_dir")
-    os.makedirs(config["config_dir"])
-    os.mkdir(config["work_dir"])
-    os.mkdir(config["logs_dir"])
+    os.mkdir(le_dir)
+    for dir_name in ("config", "logs", "work"):
+        full_path = os.path.join(le_dir, dir_name)
+        os.mkdir(full_path)
+        full_name = dir_name + "_dir"
+        config[full_name] = full_path
 
     config["domains"] = None
 

certbot-compatibility-test/certbot_compatibility_test/validator.py

@@ -33,7 +33,7 @@ class Validator(object):
         try:
             presented_cert = crypto_util.probe_sni(name, host, port)
         except acme_errors.Error as error:
-            logger.exception(error)
+            logger.exception(str(error))
             return False
 
         return presented_cert.digest("sha256") == cert.digest("sha256")
@@ -86,8 +86,7 @@ class Validator(object):
             return False
 
         try:
-            _, max_age_value = max_age[0]
-            max_age_value = int(max_age_value)
+            max_age_value = int(max_age[0][1])
         except ValueError:
             logger.error("Server responded with invalid HSTS header field")
             return False

certbot-compatibility-test/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.23.0.dev0'
+version = '0.28.0.dev0'
 
 install_requires = [
     'certbot',
@@ -46,6 +46,7 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
     ],

certbot-dns-cloudflare/setup.py

@@ -1,10 +1,8 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.23.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -44,6 +42,7 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-cloudxns/setup.py

@@ -1,10 +1,8 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.23.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -44,6 +42,7 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py

@@ -50,7 +50,8 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic
 
 
 class DigitalOceanClientTest(unittest.TestCase):
-    id = 1
+
+    id_num = 1
     record_prefix = "_acme-challenge"
     record_name = record_prefix + "." + DOMAIN
     record_content = "bar"
@@ -70,7 +71,7 @@ class DigitalOceanClientTest(unittest.TestCase):
 
         domain_mock = mock.MagicMock()
         domain_mock.name = DOMAIN
-        domain_mock.create_new_domain_record.return_value = {'domain_record': {'id': self.id}}
+        domain_mock.create_new_domain_record.return_value = {'domain_record': {'id': self.id_num}}
 
         self.manager.get_all_domains.return_value = [wrong_domain_mock, domain_mock]
 

certbot-dns-digitalocean/setup.py

@@ -1,10 +1,8 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.23.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -45,6 +43,7 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-dnsimple/setup.py

@@ -1,10 +1,8 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.23.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -44,6 +42,7 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-dnsmadeeasy/setup.py

@@ -1,10 +1,8 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.23.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -44,6 +42,7 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-gehirn/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-gehirn
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-gehirn

certbot-dns-gehirn/LICENSE.txt

@@ -0,0 +1,190 @@
+   Copyright 2018 Electronic Frontier Foundation and others
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS

certbot-dns-gehirn/MANIFEST.in

@@ -0,0 +1,3 @@
+include LICENSE.txt
+include README.rst
+recursive-include docs *

certbot-dns-gehirn/README.rst

@@ -0,0 +1 @@
+Gehirn Infrastracture Service DNS Authenticator plugin for Certbot

certbot-dns-gehirn/certbot_dns_gehirn/__init__.py

@@ -0,0 +1,88 @@
+"""
+The `~certbot_dns_gehirn.dns_gehirn` plugin automates the process of completing
+a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently
+removing, TXT records using the Gehirn Infrastracture Service DNS API.
+
+
+Named Arguments
+---------------
+
+========================================  =====================================
+``--dns-gehirn-credentials``              Gehirn Infrastracture Service
+                                          credentials_ INI file.
+                                          (Required)
+``--dns-gehirn-propagation-seconds``      The number of seconds to wait for DNS
+                                          to propagate before asking the ACME
+                                          server to verify the DNS record.
+                                          (Default: 30)
+========================================  =====================================
+
+
+Credentials
+-----------
+
+Use of this plugin requires a configuration file containing
+Gehirn Infrastracture Service DNS API credentials,
+obtained from your Gehirn Infrastracture Service
+`dashboard <https://gis.gehirn.jp/>`_.
+
+.. code-block:: ini
+   :name: credentials.ini
+   :caption: Example credentials file:
+
+   # Gehirn Infrastracture Service API credentials used by Certbot
+   dns_gehirn_api_token  = 00000000-0000-0000-0000-000000000000
+   dns_gehirn_api_secret = MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw
+
+The path to this file can be provided interactively or using the
+``--dns-gehirn-credentials`` command-line argument. Certbot records the path
+to this file for use during renewal, but does not store the file's contents.
+
+.. caution::
+   You should protect these API credentials as you would the password to your
+   Gehirn Infrastracture Service account. Users who can read this file can use
+   these credentials to issue arbitrary API calls on your behalf. Users who can
+   cause Certbot to run using these credentials can complete a ``dns-01``
+   challenge to acquire new certificates or revoke existing certificates for
+   associated domains, even if those domains aren't being managed by this server.
+
+Certbot will emit a warning if it detects that the credentials file can be
+accessed by other users on your system. The warning reads "Unsafe permissions
+on credentials configuration file", followed by the path to the credentials
+file. This warning will be emitted each time Certbot uses the credentials file,
+including for renewal, and cannot be silenced except by addressing the issue
+(e.g., by using a command like ``chmod 600`` to restrict access to the file).
+
+
+Examples
+--------
+
+.. code-block:: bash
+   :caption: To acquire a certificate for ``example.com``
+
+   certbot certonly \\
+     --dns-gehirn \\
+     --dns-gehirn-credentials ~/.secrets/certbot/gehirn.ini \\
+     -d example.com
+
+.. code-block:: bash
+   :caption: To acquire a single certificate for both ``example.com`` and
+             ``www.example.com``
+
+   certbot certonly \\
+     --dns-gehirn \\
+     --dns-gehirn-credentials ~/.secrets/certbot/gehirn.ini \\
+     -d example.com \\
+     -d www.example.com
+
+.. code-block:: bash
+   :caption: To acquire a certificate for ``example.com``, waiting 60 seconds
+             for DNS propagation
+
+   certbot certonly \\
+     --dns-gehirn \\
+     --dns-gehirn-credentials ~/.secrets/certbot/gehirn.ini \\
+     --dns-gehirn-propagation-seconds 60 \\
+     -d example.com
+
+"""

certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py

@@ -0,0 +1,84 @@
+"""DNS Authenticator for Gehirn Infrastracture Service DNS."""
+import logging
+
+import zope.interface
+from lexicon.providers import gehirn
+
+from certbot import interfaces
+from certbot.plugins import dns_common
+from certbot.plugins import dns_common_lexicon
+
+logger = logging.getLogger(__name__)
+
+DASHBOARD_URL = "https://gis.gehirn.jp/"
+
+@zope.interface.implementer(interfaces.IAuthenticator)
+@zope.interface.provider(interfaces.IPluginFactory)
+class Authenticator(dns_common.DNSAuthenticator):
+    """DNS Authenticator for Gehirn Infrastracture Service DNS
+
+    This Authenticator uses the Gehirn Infrastracture Service API to fulfill
+    a dns-01 challenge.
+    """
+
+    description = 'Obtain certificates using a DNS TXT record ' + \
+                  '(if you are using Gehirn Infrastracture Service for DNS).'
+    ttl = 60
+
+    def __init__(self, *args, **kwargs):
+        super(Authenticator, self).__init__(*args, **kwargs)
+        self.credentials = None
+
+    @classmethod
+    def add_parser_arguments(cls, add):  # pylint: disable=arguments-differ
+        super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30)
+        add('credentials', help='Gehirn Infrastracture Service credentials file.')
+
+    def more_info(self):  # pylint: disable=missing-docstring,no-self-use
+        return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \
+               'the Gehirn Infrastracture Service API.'
+
+    def _setup_credentials(self):
+        self.credentials = self._configure_credentials(
+            'credentials',
+            'Gehirn Infrastracture Service credentials file',
+            {
+                'api-token': 'API token for Gehirn Infrastracture Service ' + \
+                             'API obtained from {0}'.format(DASHBOARD_URL),
+                'api-secret': 'API secret for Gehirn Infrastracture Service ' + \
+                              'API obtained from {0}'.format(DASHBOARD_URL),
+            }
+        )
+
+    def _perform(self, domain, validation_name, validation):
+        self._get_gehirn_client().add_txt_record(domain, validation_name, validation)
+
+    def _cleanup(self, domain, validation_name, validation):
+        self._get_gehirn_client().del_txt_record(domain, validation_name, validation)
+
+    def _get_gehirn_client(self):
+        return _GehirnLexiconClient(
+            self.credentials.conf('api-token'),
+            self.credentials.conf('api-secret'),
+            self.ttl
+        )
+
+
+class _GehirnLexiconClient(dns_common_lexicon.LexiconClient):
+    """
+    Encapsulates all communication with the Gehirn Infrastracture Service via Lexicon.
+    """
+
+    def __init__(self, api_token, api_secret, ttl):
+        super(_GehirnLexiconClient, self).__init__()
+
+        self.provider = gehirn.Provider({
+            'auth_token': api_token,
+            'auth_secret': api_secret,
+            'ttl': ttl,
+        })
+
+    def _handle_http_error(self, e, domain_name):
+        if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')):
+            return  # Expected errors when zone name guess is wrong
+        return super(_GehirnLexiconClient, self)._handle_http_error(e, domain_name)

certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn_test.py

@@ -0,0 +1,55 @@
+"""Tests for certbot_dns_gehirn.dns_gehirn."""
+
+import os
+import unittest
+
+import mock
+from requests.exceptions import HTTPError
+
+from certbot.plugins import dns_test_common
+from certbot.plugins import dns_test_common_lexicon
+from certbot.plugins.dns_test_common import DOMAIN
+from certbot.tests import util as test_util
+
+API_TOKEN = '00000000-0000-0000-0000-000000000000'
+API_SECRET = 'MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw'
+
+class AuthenticatorTest(test_util.TempDirTestCase,
+                        dns_test_common_lexicon.BaseLexiconAuthenticatorTest):
+
+    def setUp(self):
+        super(AuthenticatorTest, self).setUp()
+
+        from certbot_dns_gehirn.dns_gehirn import Authenticator
+
+        path = os.path.join(self.tempdir, 'file.ini')
+        dns_test_common.write(
+            {"gehirn_api_token": API_TOKEN, "gehirn_api_secret": API_SECRET},
+            path
+        )
+
+        self.config = mock.MagicMock(gehirn_credentials=path,
+                                     gehirn_propagation_seconds=0)  # don't wait during tests
+
+        self.auth = Authenticator(self.config, "gehirn")
+
+        self.mock_client = mock.MagicMock()
+        # _get_gehirn_client | pylint: disable=protected-access
+        self.auth._get_gehirn_client = mock.MagicMock(return_value=self.mock_client)
+
+
+class GehirnLexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconClientTest):
+    DOMAIN_NOT_FOUND = HTTPError('404 Client Error: Not Found for url: {0}.'.format(DOMAIN))
+    LOGIN_ERROR = HTTPError('401 Client Error: Unauthorized for url: {0}.'.format(DOMAIN))
+
+    def setUp(self):
+        from certbot_dns_gehirn.dns_gehirn import _GehirnLexiconClient
+
+        self.client = _GehirnLexiconClient(API_TOKEN, API_SECRET, 0)
+
+        self.provider_mock = mock.MagicMock()
+        self.client.provider = self.provider_mock
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot-dns-gehirn/docs/.gitignore

@@ -0,0 +1 @@
+/_build/

certbot-dns-gehirn/docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = certbot-dns-gehirn
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

certbot-dns-gehirn/docs/api.rst

@@ -0,0 +1,8 @@
+=================
+API Documentation
+=================
+
+.. toctree::
+   :glob:
+
+   api/**

certbot-dns-gehirn/docs/api/dns_gehirn.rst

@@ -0,0 +1,5 @@
+:mod:`certbot_dns_gehirn.dns_gehirn`
+------------------------------------
+
+.. automodule:: certbot_dns_gehirn.dns_gehirn
+   :members:

certbot-dns-gehirn/docs/conf.py

@@ -0,0 +1,180 @@
+# -*- coding: utf-8 -*-
+#
+# certbot-dns-gehirn documentation build configuration file, created by
+# sphinx-quickstart on Wed May 10 18:30:40 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.autodoc',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'sphinx.ext.coverage',
+    'sphinx.ext.viewcode']
+
+autodoc_member_order = 'bysource'
+autodoc_default_flags = ['show-inheritance', 'private-members']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'certbot-dns-gehirn'
+copyright = u'2018, Certbot Project'
+author = u'Certbot Project'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = u'0'
+# The full version, including alpha/beta/rc tags.
+release = u'0'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = 'en'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+default_role = 'py:obj'
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+
+# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs
+# on_rtd is whether we are on readthedocs.org
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+if not on_rtd:  # only import and set the theme if we're building docs locally
+    import sphinx_rtd_theme
+    html_theme = 'sphinx_rtd_theme'
+    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+# otherwise, readthedocs.org uses their theme by default, so no need to specify it
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'certbot-dns-gehirndoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'certbot-dns-gehirn.tex', u'certbot-dns-gehirn Documentation',
+     u'Certbot Project', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'certbot-dns-gehirn', u'certbot-dns-gehirn Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'certbot-dns-gehirn', u'certbot-dns-gehirn Documentation',
+     author, 'certbot-dns-gehirn', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/', None),
+    'acme': ('https://acme-python.readthedocs.org/en/latest/', None),
+    'certbot': ('https://certbot.eff.org/docs/', None),
+}

certbot-dns-gehirn/docs/index.rst

@@ -0,0 +1,28 @@
+.. certbot-dns-gehirn documentation master file, created by
+   sphinx-quickstart on Wed May 10 18:30:40 2017.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to certbot-dns-gehirn's documentation!
+==============================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+.. toctree::
+   :maxdepth: 1
+
+   api
+
+.. automodule:: certbot_dns_gehirn
+   :members:
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

certbot-dns-gehirn/docs/make.bat

@@ -0,0 +1,36 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+set SPHINXPROJ=certbot-dns-gehirn
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

certbot-dns-gehirn/readthedocs.org.requirements.txt

@@ -0,0 +1,12 @@
+# readthedocs.org gives no way to change the install command to "pip
+# install -e .[docs]" (that would in turn install documentation
+# dependencies), but it allows to specify a requirements.txt file at
+# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259)
+
+# Although ReadTheDocs certainly doesn't need to install the project
+# in --editable mode (-e), just "pip install .[docs]" does not work as
+# expected and "pip install -e .[docs]" must be used instead
+
+-e acme
+-e .
+-e certbot-dns-gehirn[docs]

certbot-dns-gehirn/setup.cfg

@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal = 1

certbot-dns-gehirn/setup.py

@@ -0,0 +1,66 @@
+import sys
+
+from setuptools import setup
+from setuptools import find_packages
+
+
+version = '0.28.0.dev0'
+
+# Please update tox.ini when modifying dependency version requirements
+install_requires = [
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
+    'dns-lexicon>=2.1.22',
+    'mock',
+    'setuptools',
+    'zope.interface',
+]
+
+docs_extras = [
+    'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
+    'sphinx_rtd_theme',
+]
+
+setup(
+    name='certbot-dns-gehirn',
+    version=version,
+    description="Gehirn Infrastracture Service DNS Authenticator plugin for Certbot",
+    url='https://github.com/certbot/certbot',
+    author="Certbot Project",
+    author_email='client-dev@letsencrypt.org',
+    license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Environment :: Plugins',
+        'Intended Audience :: System Administrators',
+        'License :: OSI Approved :: Apache Software License',
+        'Operating System :: POSIX :: Linux',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Topic :: Internet :: WWW/HTTP',
+        'Topic :: Security',
+        'Topic :: System :: Installation/Setup',
+        'Topic :: System :: Networking',
+        'Topic :: System :: Systems Administration',
+        'Topic :: Utilities',
+    ],
+
+    packages=find_packages(),
+    include_package_data=True,
+    install_requires=install_requires,
+    extras_require={
+        'docs': docs_extras,
+    },
+    entry_points={
+        'certbot.plugins': [
+            'dns-gehirn = certbot_dns_gehirn.dns_gehirn:Authenticator',
+        ],
+    },
+    test_suite='certbot_dns_gehirn',
+)

certbot-dns-google/MANIFEST.in

@@ -1,3 +1,4 @@
 include LICENSE.txt
 include README.rst
 recursive-include docs *
+recursive-include certbot_dns_google/testdata *

certbot-dns-google/setup.py

@@ -1,10 +1,8 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.23.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
@@ -49,6 +47,7 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-linode/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-linode
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-linode

certbot-dns-linode/LICENSE.txt

@@ -0,0 +1,190 @@
+   Copyright 2015 Electronic Frontier Foundation and others
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS

certbot-dns-linode/MANIFEST.in

@@ -0,0 +1,3 @@
+include LICENSE.txt
+include README.rst
+recursive-include docs *

certbot-dns-linode/README.rst

@@ -0,0 +1 @@
+Linode DNS Authenticator plugin for Certbot

certbot-dns-linode/certbot_dns_linode/__init__.py

@@ -0,0 +1,92 @@
+"""
+The `~certbot_dns_linode.dns_linode` plugin automates the process of
+completing a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and
+subsequently removing, TXT records using the Linode API.
+
+
+Named Arguments
+---------------
+
+==========================================  ===================================
+``--dns-linode-credentials``                Linode credentials_ INI file.
+                                            (Required)
+``--dns-linode-propagation-seconds``        The number of seconds to wait for
+                                            DNS to propagate before asking the
+                                            ACME server to verify the DNS
+                                            record.
+                                            (Default: 1200 because Linode
+                                            updates its first DNS every 15
+                                            minutes and we allow 5 more minutes
+                                            for the update to reach the other 5
+                                            servers)
+==========================================  ===================================
+
+
+Credentials
+-----------
+
+Use of this plugin requires a configuration file containing Linode API
+credentials, obtained from your Linode account's `Applications & API
+Tokens page <https://manager.linode.com/profile/api>`_.
+
+.. code-block:: ini
+   :name: credentials.ini
+   :caption: Example credentials file:
+
+   # Linode API credentials used by Certbot
+   dns_linode_key = 0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ64
+
+The path to this file can be provided interactively or using the
+``--dns-linode-credentials`` command-line argument. Certbot records the path
+to this file for use during renewal, but does not store the file's contents.
+
+.. caution::
+   You should protect these API credentials as you would the password to your
+   Linode account. Users who can read this file can use these credentials
+   to issue arbitrary API calls on your behalf. Users who can cause Certbot to
+   run using these credentials can complete a ``dns-01`` challenge to acquire
+   new certificates or revoke existing certificates for associated domains,
+   even if those domains aren't being managed by this server.
+
+Certbot will emit a warning if it detects that the credentials file can be
+accessed by other users on your system. The warning reads "Unsafe permissions
+on credentials configuration file", followed by the path to the credentials
+file. This warning will be emitted each time Certbot uses the credentials file,
+including for renewal, and cannot be silenced except by addressing the issue
+(e.g., by using a command like ``chmod 600`` to restrict access to the file).
+
+
+Examples
+--------
+
+.. code-block:: bash
+   :caption: To acquire a certificate for ``example.com``
+
+   certbot certonly \\
+     --dns-linode \\
+     --dns-linode-credentials ~/.secrets/certbot/linode.ini \\
+     -d example.com
+
+.. code-block:: bash
+   :caption: To acquire a single certificate for both ``example.com`` and
+             ``www.example.com``
+
+   certbot certonly \\
+     --dns-linode \\
+     --dns-linode-credentials ~/.secrets/certbot/linode.ini \\
+     -d example.com \\
+     -d www.example.com
+
+.. code-block:: bash
+   :caption: To acquire a certificate for ``example.com``, waiting 1000 seconds
+             for DNS propagation (Linode updates its first DNS every 15 minutes
+             and we allow some extra time for the update to reach the other 5
+             servers)
+
+   certbot certonly \\
+     --dns-linode \\
+     --dns-linode-credentials ~/.secrets/certbot/linode.ini \\
+     --dns-linode-propagation-seconds 1000 \\
+     -d example.com
+
+"""

certbot-dns-linode/certbot_dns_linode/dns_linode.py

@@ -0,0 +1,72 @@
+"""DNS Authenticator for Linode."""
+import logging
+
+import zope.interface
+from lexicon.providers import linode
+
+from certbot import errors
+from certbot import interfaces
+from certbot.plugins import dns_common
+from certbot.plugins import dns_common_lexicon
+
+logger = logging.getLogger(__name__)
+
+API_KEY_URL = 'https://manager.linode.com/profile/api'
+
+@zope.interface.implementer(interfaces.IAuthenticator)
+@zope.interface.provider(interfaces.IPluginFactory)
+class Authenticator(dns_common.DNSAuthenticator):
+    """DNS Authenticator for Linode
+
+    This Authenticator uses the Linode API to fulfill a dns-01 challenge.
+    """
+
+    description = 'Obtain certs using a DNS TXT record (if you are using Linode for DNS).'
+
+    def __init__(self, *args, **kwargs):
+        super(Authenticator, self).__init__(*args, **kwargs)
+        self.credentials = None
+
+    @classmethod
+    def add_parser_arguments(cls, add):  # pylint: disable=arguments-differ
+        super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=1200)
+        add('credentials', help='Linode credentials INI file.')
+
+    def more_info(self):  # pylint: disable=missing-docstring,no-self-use
+        return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \
+               'the Linode API.'
+
+    def _setup_credentials(self):
+        self.credentials = self._configure_credentials(
+            'credentials',
+            'Linode credentials INI file',
+            {
+                'key': 'API key for Linode account, obtained from {0}'.format(API_KEY_URL)
+            }
+        )
+
+    def _perform(self, domain, validation_name, validation):
+        self._get_linode_client().add_txt_record(domain, validation_name, validation)
+
+    def _cleanup(self, domain, validation_name, validation):
+        self._get_linode_client().del_txt_record(domain, validation_name, validation)
+
+    def _get_linode_client(self):
+        return _LinodeLexiconClient(self.credentials.conf('key'))
+
+class _LinodeLexiconClient(dns_common_lexicon.LexiconClient):
+    """
+    Encapsulates all communication with the Linode API.
+    """
+
+    def __init__(self, api_key):
+        super(_LinodeLexiconClient, self).__init__()
+        self.provider = linode.Provider({
+            'auth_token': api_key
+        })
+
+    def _handle_general_error(self, e, domain_name):
+        if not str(e).startswith('Domain not found'):
+            return errors.PluginError('Unexpected error determining zone identifier for {0}: {1}'
+                                      .format(domain_name, e))
+

certbot-dns-linode/certbot_dns_linode/dns_linode_test.py

@@ -0,0 +1,47 @@
+"""Tests for certbot_dns_linode.dns_linode."""
+
+import os
+import unittest
+
+import mock
+
+from certbot.plugins import dns_test_common
+from certbot.plugins import dns_test_common_lexicon
+from certbot.tests import util as test_util
+
+TOKEN = 'a-token'
+
+class AuthenticatorTest(test_util.TempDirTestCase,
+                        dns_test_common_lexicon.BaseLexiconAuthenticatorTest):
+
+    def setUp(self):
+        super(AuthenticatorTest, self).setUp()
+
+        from certbot_dns_linode.dns_linode import Authenticator
+
+        path = os.path.join(self.tempdir, 'file.ini')
+        dns_test_common.write({"linode_key": TOKEN}, path)
+
+        self.config = mock.MagicMock(linode_credentials=path,
+                                     linode_propagation_seconds=0)  # don't wait during tests
+
+        self.auth = Authenticator(self.config, "linode")
+
+        self.mock_client = mock.MagicMock()
+        # _get_linode_client | pylint: disable=protected-access
+        self.auth._get_linode_client = mock.MagicMock(return_value=self.mock_client)
+
+class LinodeLexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconClientTest):
+
+    DOMAIN_NOT_FOUND = Exception('Domain not found')
+
+    def setUp(self):
+        from certbot_dns_linode.dns_linode import _LinodeLexiconClient
+
+        self.client = _LinodeLexiconClient(TOKEN)
+
+        self.provider_mock = mock.MagicMock()
+        self.client.provider = self.provider_mock
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot-dns-linode/docs/.gitignore

@@ -0,0 +1 @@
+/_build/

certbot-dns-linode/docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = certbot-dns-linode
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

certbot-dns-linode/docs/api.rst

@@ -0,0 +1,8 @@
+=================
+API Documentation
+=================
+
+.. toctree::
+   :glob:
+
+   api/**

certbot-dns-linode/docs/api/dns_linode.rst

@@ -0,0 +1,5 @@
+:mod:`certbot_dns_linode.dns_linode`
+------------------------------------------------
+
+.. automodule:: certbot_dns_linode.dns_linode
+   :members: