Merge branch 'master-uptsream' into apache-integration-tests

# Conflicts:
#	.travis.yml
#	tox.ini

.travis.yml

@@ -6,6 +6,16 @@ cache:
         - $HOME/.cache/pip
 
 before_script:
+  # Install required apt packages
+  - |
+    if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then
+      ./certbot-auto --non-interactive --os-packages-only
+      sudo -E apt-get -yq --no-install-suggests --no-install-recommends install nginx-light
+      sudo -E /etc/init.d/nginx stop
+      sudo -E apt-get -yq --no-install-suggests --no-install-recommends install apache2
+      sudo -E /etc/init.d/apache2 stop
+      sudo -E chmod 777 -R /var/lib/apache2/module
+    fi
   - 'if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ulimit -n 1024 ; fi'
   # On Travis, the fastest parallelization for integration tests has proved to be 4.
   - 'if [[ "$TOXENV" == *"integration"* ]]; then export PYTEST_ADDOPTS="--numprocesses 4"; fi'
@@ -71,15 +81,9 @@ matrix:
     - python: "3.8-dev"
       env: TOXENV=py38
       <<: *not-on-master
-    - sudo: required
-      env: TOXENV=apache_compat
-      services: docker
-      before_install:
-      addons:
+    - env: TOXENV=apache_compat
       <<: *not-on-master
-    - sudo: required
-      env: TOXENV=le_auto_xenial
-      services: docker
+    - env: TOXENV=le_auto_xenial
       <<: *not-on-master
     - python: "2.7"
       env: TOXENV=apacheconftest-with-pebble
@@ -89,11 +93,7 @@ matrix:
       <<: *not-on-master
 
     # Extended test suite on cron jobs and pushes to tested branches other than master
-    - sudo: required
-      env: TOXENV=nginx_compat
-      services: docker
-      before_install:
-      addons:
+    - env: TOXENV=nginx_compat
       <<: *extended-test-suite
     - python: "2.7"
       env:
@@ -122,13 +122,9 @@ matrix:
       <<: *extended-test-suite
     - python: "2.7"
       env: ACME_SERVER=boulder-v1 TOXENV=integration
-      sudo: required
-      services: docker
       <<: *extended-test-suite
     - python: "2.7"
       env: ACME_SERVER=boulder-v2 TOXENV=integration
-      sudo: required
-      services: docker
       <<: *extended-test-suite
     - python: "2.7"
       env: ACME_SERVER=boulder-v1 TOXENV=integration-certbot-oldest
@@ -136,8 +132,6 @@ matrix:
       # cryptography we support cannot be compiled against the version of
       # OpenSSL in Xenial or newer.
       dist: trusty
-      sudo: required
-      services: docker
       <<: *extended-test-suite
     - python: "2.7"
       env: ACME_SERVER=boulder-v2 TOXENV=integration-certbot-oldest
@@ -145,8 +139,6 @@ matrix:
       # cryptography we support cannot be compiled against the version of
       # OpenSSL in Xenial or newer.
       dist: trusty
-      sudo: required
-      services: docker
       <<: *extended-test-suite
     - python: "2.7"
       env: ACME_SERVER=boulder-v1 TOXENV=integration-nginx-oldest
@@ -154,8 +146,6 @@ matrix:
       # cryptography we support cannot be compiled against the version of
       # OpenSSL in Xenial or newer.
       dist: trusty
-      sudo: required
-      services: docker
       <<: *extended-test-suite
     - python: "2.7"
       env: ACME_SERVER=boulder-v2 TOXENV=integration-nginx-oldest
@@ -163,8 +153,6 @@ matrix:
       # cryptography we support cannot be compiled against the version of
       # OpenSSL in Xenial or newer.
       dist: trusty
-      sudo: required
-      services: docker
       <<: *extended-test-suite
     - python: "3.4"
       env: TOXENV=py34
@@ -183,43 +171,27 @@ matrix:
       <<: *extended-test-suite
     - python: "3.4"
       env: ACME_SERVER=boulder-v1 TOXENV=integration
-      sudo: required
-      services: docker
       <<: *extended-test-suite
     - python: "3.4"
       env: ACME_SERVER=boulder-v2 TOXENV=integration
-      sudo: required
-      services: docker
       <<: *extended-test-suite
     - python: "3.5"
       env: ACME_SERVER=boulder-v1 TOXENV=integration
-      sudo: required
-      services: docker
       <<: *extended-test-suite
     - python: "3.5"
       env: ACME_SERVER=boulder-v2 TOXENV=integration
-      sudo: required
-      services: docker
       <<: *extended-test-suite
     - python: "3.6"
       env: ACME_SERVER=boulder-v1 TOXENV=integration
-      sudo: required
-      services: docker
       <<: *extended-test-suite
     - python: "3.6"
       env: ACME_SERVER=boulder-v2 TOXENV=integration
-      sudo: required
-      services: docker
       <<: *extended-test-suite
     - python: "3.7"
       env: ACME_SERVER=boulder-v1 TOXENV=integration
-      sudo: required
-      services: docker
       <<: *extended-test-suite
     - python: "3.7"
       env: ACME_SERVER=boulder-v2 TOXENV=integration
-      sudo: required
-      services: docker
       <<: *extended-test-suite
     - python: "3.8-dev"
       env: ACME_SERVER=boulder-v1 TOXENV=integration
@@ -227,21 +199,11 @@ matrix:
     - python: "3.8-dev"
       env: ACME_SERVER=boulder-v2 TOXENV=integration
       <<: *extended-test-suite
-    - sudo: required
-      env: TOXENV=le_auto_jessie
-      services: docker
+    - env: TOXENV=le_auto_jessie
       <<: *extended-test-suite
-    - sudo: required
-      env: TOXENV=le_auto_centos6
-      services: docker
+    - env: TOXENV=le_auto_centos6
       <<: *extended-test-suite
-    - sudo: required
-      env: TOXENV=docker_dev
-      services: docker
-      addons:
-        apt:
-          packages:  # don't install nginx and apache
-            - libaugeas0
+    - env: TOXENV=docker_dev
       <<: *extended-test-suite
     - language: generic
       env: TOXENV=py27
@@ -268,22 +230,6 @@ matrix:
             - python3
       <<: *extended-test-suite
 
-# container-based infrastructure
-sudo: false
-
-addons:
-  apt:
-    packages:  # Keep in sync with letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh and Boulder.
-    - python-dev
-    - gcc
-    - libaugeas0
-    - libssl-dev
-    - libffi-dev
-    - ca-certificates
-    # For certbot-nginx integration testing
-    - nginx-light
-    - openssl
-
 # tools/pip_install.py is used to pin packages to a known working version
 # except in tests where the environment variable CERTBOT_NO_PIN is set.
 # virtualenv is listed here explicitly to make sure it is upgraded when

certbot-ci/certbot_integration_tests/apache_tests/apache_config.py

@@ -0,0 +1,171 @@
+import shutil
+import subprocess
+import os
+
+import pkg_resources
+import getpass
+
+
+def construct_apache_config_dir(apache_root, http_port, https_port, key_path=None,
+                                cert_path=None, wtf_prefix='le'):
+    config_path = os.path.join(apache_root, 'config')
+    shutil.copytree('/etc/apache2', config_path, symlinks=True)
+
+    webroot_path = os.path.join(apache_root, 'www')
+    os.mkdir(webroot_path)
+    with open(os.path.join(webroot_path, 'index.html'), 'w') as file_h:
+        file_h.write('Hello World!')
+
+    main_config_path = os.path.join(config_path, 'apache2.conf')
+    with open(main_config_path, 'w') as file_h:
+        file_h.write('''\
+ServerRoot "{config}"
+DefaultRuntimeDir ${{APACHE_RUN_DIR}}
+PidFile ${{APACHE_PID_FILE}}
+Timeout 300
+KeepAlive On
+MaxKeepAliveRequests 100
+KeepAliveTimeout 5
+User ${{APACHE_RUN_USER}}
+Group ${{APACHE_RUN_GROUP}}
+HostnameLookups Off
+ErrorLog ${{APACHE_LOG_DIR}}/error.log
+LogLevel warn
+
+IncludeOptional mods-enabled/*.load
+IncludeOptional mods-enabled/*.conf
+
+Include ports.conf
+
+<Directory />
+    Options FollowSymLinks
+    AllowOverride None
+    Require all denied
+</Directory>
+
+<Directory /usr/share>
+    AllowOverride None
+    Require all granted
+</Directory>
+
+<Directory {webroot}/>
+    Options Indexes FollowSymLinks
+    AllowOverride None
+    Require all granted
+</Directory>
+
+AccessFileName .htaccess
+
+<FilesMatch "^\.ht">
+    Require all denied
+</FilesMatch>
+
+LogFormat "%v:%p %h %l %u %t \\"%r\\" %>s %O \\"%{{Referer}}i\\" \\"%{{User-Agent}}i\\"" vhost_combined
+LogFormat "%h %l %u %t \\"%r\\" %>s %O \\"%{{Referer}}i\\" \\"%{{User-Agent}}i\\"" combined
+LogFormat "%h %l %u %t \\"%r\\" %>s %O" common
+LogFormat "%{{Referer}}i -> %U" referer
+LogFormat "%{{User-agent}}i" agent
+
+IncludeOptional conf-enabled/*.conf
+IncludeOptional sites-enabled/*.conf
+'''.format(config=config_path, webroot=webroot_path))
+
+    with open(os.path.join(config_path, 'ports.conf'), 'w') as file_h:
+        file_h.write('''\
+Listen {http}
+<IfModule ssl_module>
+    Listen {https}
+</IfModule>
+<IfModule mod_gnutls.c>
+    Listen {https}
+</IfModule>
+'''.format(http=http_port, https=https_port))
+
+    new_environ = os.environ.copy()
+    new_environ['APACHE_CONFDIR'] = config_path
+
+    run_path = os.path.join(apache_root, 'run')
+    lock_path = os.path.join(apache_root, 'lock')
+    logs_path = os.path.join(apache_root, 'logs')
+    os.mkdir(run_path)
+    os.mkdir(lock_path)
+    os.mkdir(logs_path)
+
+    user = getpass.getuser()
+    user = user if user != 'root' else 'www-data'
+    group = user
+
+    pid_file = os.path.join(run_path, 'apache.pid')
+
+    with open(os.path.join(config_path, 'envvars'), 'w') as file_h:
+        file_h.write('''\
+unset HOME
+export APACHE_RUN_USER={user}
+export APACHE_RUN_GROUP={group}
+export APACHE_PID_FILE={pid_file}
+export APACHE_RUN_DIR={run_path}
+export APACHE_LOCK_DIR={lock_path}
+export APACHE_LOG_DIR={logs_path}
+export LANG=C
+'''.format(user=user, group=group, pid_file=pid_file,
+           run_path=run_path, lock_path=lock_path, logs_path=logs_path))
+
+    new_environ['APACHE_RUN_USER'] = user
+    new_environ['APACHE_RUN_GROUP'] = group
+    new_environ['APACHE_PID_FILE'] = pid_file
+    new_environ['APACHE_RUN_DIR'] = run_path
+    new_environ['APACHE_LOCK_DIR'] = lock_path
+    new_environ['APACHE_LOG_DIR'] = logs_path
+
+    le_host = 'apache.{0}.wtf'.format(wtf_prefix)
+
+    with open(os.path.join(config_path, 'sites-available', '000-default.conf'), 'w') as file_h:
+        file_h.write('''\
+<VirtualHost *:{http}>
+    ServerAdmin webmaster@localhost
+    ServerName {le_host}
+    DocumentRoot {webroot}
+    
+    ErrorLog ${{APACHE_LOG_DIR}}/error.log
+    CustomLog ${{APACHE_LOG_DIR}}/access.log combined
+</VirtualHost>
+'''.format(http=http_port, le_host=le_host, webroot=webroot_path))
+
+    key_path = key_path if key_path \
+        else pkg_resources.resource_filename('certbot_integration_tests', 'assets/key.pem')
+    cert_path = cert_path if cert_path \
+        else pkg_resources.resource_filename('certbot_integration_tests', 'assets/cert.pem')
+
+    with open(os.path.join(config_path, 'sites-available', 'default-ssl.conf'), 'w') as file_h:
+        file_h.write('''\
+<IfModule mod_ssl.c>
+    <VirtualHost _default_:{https}>
+        ServerAdmin webmaster@localhost
+        ServerName {le_host}
+        DocumentRoot {webroot}
+        
+        ErrorLog ${{APACHE_LOG_DIR}}/error.log
+        CustomLog ${{APACHE_LOG_DIR}}/access.log combined
+        
+        SSLEngine on
+        SSLCertificateFile {cert_path}
+        SSLCertificateKeyFile {key_path}
+        
+        <FilesMatch "\.(cgi|shtml|phtml|php)$">
+            SSLOptions +StdEnvVars
+        </FilesMatch>
+        
+        <Directory /usr/lib/cgi-bin>
+            SSLOptions +StdEnvVars
+        </Directory>
+    </VirtualHost>
+</IfModule>
+'''.format(https=https_port, le_host=le_host, webroot=webroot_path,
+           cert_path=cert_path, key_path=key_path))
+
+    return new_environ, config_path, pid_file
+
+
+def test():
+    env = construct_apache_config_dir('/tmp/test1', 5001, 5002)
+    subprocess.call(['apache2ctl', '-DFOREGROUND'], env=env)

certbot-ci/certbot_integration_tests/apache_tests/context.py

@@ -0,0 +1,51 @@
+import errno
+import os
+import signal
+import subprocess
+
+from certbot_integration_tests.certbot_tests import context as certbot_context
+from certbot_integration_tests.apache_tests import apache_config
+from certbot_integration_tests.utils import certbot_call
+
+
+class IntegrationTestsContext(certbot_context.IntegrationTestsContext):
+    def __init__(self, request):
+        super(IntegrationTestsContext, self).__init__(request)
+
+        subprocess.check_output(['chmod', '+x', self.workspace])
+
+        self.apache_root = os.path.join(self.workspace, 'apache')
+        os.mkdir(self.apache_root)
+
+        self.env, self.config_dir, self.pid_file = apache_config.construct_apache_config_dir(
+            self.apache_root, self.http_01_port, self.tls_alpn_01_port,
+            wtf_prefix=self.worker_id)
+
+    def cleanup(self):
+        self._stop_apache()
+        super(IntegrationTestsContext, self).cleanup()
+
+    def certbot_test_apache(self, args):
+        command = ['--authenticator', 'apache', '--installer', 'apache',
+                   '--apache-server-root', self.config_dir,
+                   '--apache-challenge-location', self.apache_root]
+        command.extend(args)
+
+        return certbot_call.certbot_test(
+            command, self.directory_url, self.http_01_port, self.tls_alpn_01_port,
+            self.config_dir, self.workspace, env=self.env, force_renew=True)
+
+    def _stop_apache(self):
+        try:
+            with open(self.pid_file) as file_h:
+                pid = int(file_h.read().strip())
+        except BaseException:
+            pid = None
+
+        if pid:
+            try:
+                os.kill(pid, signal.SIGTERM)
+            except OSError as err:
+                # Ignore "No such process" error, Apache may already be stopped.
+                if err.errno != errno.ESRCH:
+                    raise

certbot-ci/certbot_integration_tests/apache_tests/test_main.py

@@ -0,0 +1,18 @@
+import pytest
+
+from certbot_integration_tests.apache_tests import context as apache_context
+
+
+@pytest.fixture()
+def context(request):
+    # Fixture request is a built-in pytest fixture describing current test request.
+    integration_test_context = apache_context.IntegrationTestsContext(request)
+    try:
+        yield integration_test_context
+    finally:
+        integration_test_context.cleanup()
+
+
+def test_it(context):
+    command = ['-d', 'apache.{0}.wtf'.format(context.worker_id)]
+    context.certbot_test_apache(command)

certbot-ci/certbot_integration_tests/utils/certbot_call.py

@@ -11,7 +11,7 @@ from certbot_integration_tests.utils.constants import *
 
 
 def certbot_test(certbot_args, directory_url, http_01_port, tls_alpn_01_port,
-                 config_dir, workspace, force_renew=True):
+                 config_dir, workspace, env=None, force_renew=True):
     """
     Invoke the certbot executable available in PATH in a test context for the given args.
     The test context consists in running certbot in debug mode, with various flags suitable
@@ -23,18 +23,20 @@ def certbot_test(certbot_args, directory_url, http_01_port, tls_alpn_01_port,
     :param int tls_alpn_01_port: port for the TLS-ALPN-01 challenges
     :param str config_dir: certbot configuration directory to use
     :param str workspace: certbot current directory to use
+    :param obj env: the environment variables to use (default: None, then current env will be used)
     :param bool force_renew: set False to not force renew existing certificates (default: True)
     :return: stdout as string
     :rtype: str
     """
+    new_environ = env if env else os.environ.copy()
     command, env = _prepare_args_env(certbot_args, directory_url, http_01_port, tls_alpn_01_port,
-                                     config_dir, workspace, force_renew)
+                                     config_dir, workspace, force_renew, new_environ)
 
     return subprocess.check_output(command, universal_newlines=True, cwd=workspace, env=env)
 
 
-def _prepare_environ(workspace):
-    new_environ = os.environ.copy()
+def _prepare_environ(workspace, new_environ):
+    new_environ = new_environ.copy()
     new_environ['TMPDIR'] = workspace
 
     # So, pytest is nice, and a little too nice for our usage.
@@ -80,9 +82,9 @@ def _compute_additional_args(workspace, environ, force_renew):
 
 
 def _prepare_args_env(certbot_args, directory_url, http_01_port, tls_alpn_01_port,
-                      config_dir, workspace, force_renew):
+                      config_dir, workspace, force_renew, new_environ):
 
-    new_environ = _prepare_environ(workspace)
+    new_environ = _prepare_environ(workspace, new_environ)
     additional_args = _compute_additional_args(workspace, new_environ, force_renew)
 
     command = [

tox.ini

@@ -223,13 +223,14 @@ passenv = DOCKER_*
 
 [testenv:integration]
 commands =
-    {[base]pip_install} acme . certbot-nginx certbot-ci
+    {[base]pip_install} acme . certbot-nginx certbot-apache certbot-ci
     pytest certbot-ci/certbot_integration_tests \
         --acme-server={env:ACME_SERVER:pebble} \
-        --cov=acme --cov=certbot --cov=certbot_nginx --cov-report= \
-        --cov-config=certbot-ci/certbot_integration_tests/.coveragerc
+        --cov=acme --cov=certbot --cov=certbot_nginx --cov=certbot_apache \
+        --cov-report= --cov-config=certbot-ci/certbot_integration_tests/.coveragerc
     coverage report --include 'certbot/*' --show-missing --fail-under=65
     coverage report --include 'certbot-nginx/*' --show-missing --fail-under=74
+    coverage report --include 'certbot-apache/*' --show-missing --fail-under=55
 passenv = DOCKER_*
 
 [testenv:integration-certbot]