Merge branch 'test-update-fedora-ami' into update-fedora-and-ubuntu-amis

Fedora 28 is likely to reach its EOL soon.

.travis.yml

@@ -30,70 +30,6 @@ extended-test-suite: &extended-test-suite
 
 matrix:
   include:
-    # Main test suite
-    - python: "2.7"
-      env: ACME_SERVER=pebble TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *not-on-master
-
-    # This job is always executed, including on master
-    - python: "2.7"
-      env: TOXENV=py27-cover FYI="py27 tests + code coverage"
-
-    - sudo: required
-      env: TOXENV=nginx_compat
-      services: docker
-      before_install:
-      addons:
-      <<: *not-on-master
-    - python: "2.7"
-      env: TOXENV=lint
-      <<: *not-on-master
-    - python: "3.4"
-      env: TOXENV=mypy
-      <<: *not-on-master
-    - python: "3.5"
-      env: TOXENV=mypy
-      <<: *not-on-master
-    - python: "2.7"
-      env: TOXENV='py27-{acme,apache,certbot,dns,nginx,postfix}-oldest'
-      sudo: required
-      services: docker
-      <<: *not-on-master
-    - python: "3.4"
-      env: TOXENV=py34
-      sudo: required
-      services: docker
-      <<: *not-on-master
-    - python: "3.7"
-      dist: xenial
-      env: TOXENV=py37
-      sudo: required
-      services: docker
-      <<: *not-on-master
-    - sudo: required
-      env: TOXENV=apache_compat
-      services: docker
-      before_install:
-      addons:
-      <<: *not-on-master
-    - sudo: required
-      env: TOXENV=le_auto_trusty
-      services: docker
-      before_install:
-      addons:
-      <<: *not-on-master
-    - python: "2.7"
-      env: TOXENV=apacheconftest-with-pebble
-      sudo: required
-      services: docker
-      <<: *not-on-master
-    - python: "2.7"
-      env: TOXENV=nginxroundtrip
-      <<: *not-on-master
-
-    # Extended test suite on cron jobs and pushes to tested branches other than master
     - python: "2.7"
       env:
         - TOXENV=travis-test-farm-apache2
@@ -116,139 +52,6 @@ matrix:
         - TOXENV=travis-test-farm-sdists
         - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
       <<: *extended-test-suite
-    - python: "3.7"
-      dist: xenial
-      env: TOXENV=py37 CERTBOT_NO_PIN=1
-      <<: *extended-test-suite
-    - python: "2.7"
-      env: ACME_SERVER=boulder-v1 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "2.7"
-      env: ACME_SERVER=boulder-v2 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "2.7"
-      env: TOXENV=py27-certbot-oldest
-      <<: *extended-test-suite
-    - python: "2.7"
-      env: TOXENV=py27-nginx-oldest
-      <<: *extended-test-suite
-    - python: "2.7"
-      env: ACME_SERVER=boulder-v1 TOXENV=integration-certbot-oldest
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "2.7"
-      env: ACME_SERVER=boulder-v2 TOXENV=integration-certbot-oldest
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "2.7"
-      env: ACME_SERVER=boulder-v1 TOXENV=integration-nginx-oldest
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "2.7"
-      env: ACME_SERVER=boulder-v2 TOXENV=integration-nginx-oldest
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "3.4"
-      env: TOXENV=py34
-      <<: *extended-test-suite
-    - python: "3.5"
-      env: TOXENV=py35
-      <<: *extended-test-suite
-    - python: "3.6"
-      env: TOXENV=py36
-      <<: *extended-test-suite
-    - python: "3.7"
-      dist: xenial
-      env: TOXENV=py37
-      <<: *extended-test-suite
-    - python: "3.4"
-      env: ACME_SERVER=boulder-v1 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "3.4"
-      env: ACME_SERVER=boulder-v2 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "3.5"
-      env: ACME_SERVER=boulder-v1 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "3.5"
-      env: ACME_SERVER=boulder-v2 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "3.6"
-      env: ACME_SERVER=boulder-v1 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "3.6"
-      env: ACME_SERVER=boulder-v2 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "3.7"
-      dist: xenial
-      env: ACME_SERVER=boulder-v1 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - python: "3.7"
-      dist: xenial
-      env: ACME_SERVER=boulder-v2 TOXENV=integration
-      sudo: required
-      services: docker
-      <<: *extended-test-suite
-    - sudo: required
-      env: TOXENV=le_auto_xenial
-      services: docker
-      <<: *extended-test-suite
-    - sudo: required
-      env: TOXENV=le_auto_jessie
-      services: docker
-      <<: *extended-test-suite
-    - sudo: required
-      env: TOXENV=le_auto_centos6
-      services: docker
-      <<: *extended-test-suite
-    - sudo: required
-      env: TOXENV=docker_dev
-      services: docker
-      addons:
-        apt:
-          packages:  # don't install nginx and apache
-            - libaugeas0
-      <<: *extended-test-suite
-    - language: generic
-      env: TOXENV=py27
-      os: osx
-      addons:
-        homebrew:
-          packages:
-            - augeas
-            - python2
-      <<: *extended-test-suite
-    - language: generic
-      env: TOXENV=py3
-      os: osx
-      addons:
-        homebrew:
-          packages:
-            - augeas
-            - python3
-      <<: *extended-test-suite
 
 # container-based infrastructure
 sudo: false
@@ -274,10 +77,3 @@ after_success: '[ "$TOXENV" == "py27-cover" ] && codecov -F linux'
 
 notifications:
   email: false
-  irc:
-    channels:
-      - secure: "SGWZl3ownKx9xKVV2VnGt7DqkTmutJ89oJV9tjKhSs84kLijU6EYdPnllqISpfHMTxXflNZuxtGo0wTDYHXBuZL47w1O32W6nzuXdra5zC+i4sYQwYULUsyfOv9gJX8zWAULiK0Z3r0oho45U+FR5ZN6TPCidi8/eGU+EEPwaAw="
-    on_cancel: never
-    on_success: never
-    on_failure: always
-    use_notice: true

tests/letstest/apache2_targets.yaml

@@ -1,8 +1,8 @@
 targets:
   #-----------------------------------------------------------------------------
   #Ubuntu
-  - ami: ami-064bd2d44a1d6c097
-    name: ubuntu18.10
+  - ami: ami-08ab45c4343f5f5c6
+    name: ubuntu19.04
     type: ubuntu
     virt: hvm
     user: ubuntu
@@ -40,8 +40,8 @@ targets:
     user: admin
   #-----------------------------------------------------------------------------
   # Fedora
-  - ami: ami-5c69df23
-    name: fedora28
+  - ami: ami-00bbc6858140f19ed
+    name: fedora30
     type: centos
     virt: hvm
     user: fedora

tests/letstest/scripts/set_python_envvars.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+# This is a simple script that can be sourced to set Python environment
+# variables for use in Certbot's letstest test farm tests.
+
+# Some distros like Fedora may only have an executable named python3 installed.
+if command -v python; then
+    PYTHON_NAME="python"
+    VENV_SCRIPT="tools/venv.py"
+    VENV_PATH="venv"
+else
+    # We could check for "python2" here, however, the addition of "python3"
+    # only systems is what necessitated this change so checking for "python2"
+    # isn't necessary.
+    PYTHON_NAME="python3"
+    VENV_PATH="venv3"
+    VENV_SCRIPT="tools/venv3.py"
+fi

tests/letstest/scripts/test_apache2.sh

@@ -45,8 +45,10 @@ if [ $? -ne 0 ] ; then
     exit 1
 fi
 
-python tools/venv.py -e acme[dev] -e .[dev,docs] -e certbot-apache
-sudo venv/bin/certbot -v --debug --text --agree-dev-preview --agree-tos \
+. tests/letstest/scripts/set_python_envvars.sh
+
+"$VENV_SCRIPT" -e acme[dev] -e .[dev,docs] -e certbot-apache
+sudo "$VENV_PATH/bin/certbot" -v --debug --text --agree-dev-preview --agree-tos \
                    --renew-by-default --redirect --register-unsafely-without-email \
                    --domain $PUBLIC_HOSTNAME --server $BOULDER_URL
 if [ $? -ne 0 ] ; then
@@ -55,7 +57,7 @@ fi
 
 if [ "$OS_TYPE" = "ubuntu" ] ; then
     export SERVER="$BOULDER_URL"
-    venv/bin/tox -e apacheconftest
+    "$VENV_PATH/bin/tox" -e apacheconftest
 else
     echo Not running hackish apache tests on $OS_TYPE
 fi

tests/letstest/scripts/test_leauto_upgrades.sh

@@ -21,7 +21,7 @@ if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | se
     # 0.20.0 is the latest version of letsencrypt-auto that doesn't install
     # Python 3 on RHEL 6.
     INITIAL_VERSION="0.20.0"
-    RUN_PYTHON3_TESTS=1
+    RUN_RHEL6_TESTS=1
 else
     # 0.33.x is the oldest version of letsencrypt-auto that works on Fedora 29+.
     INITIAL_VERSION="0.33.1"
@@ -32,14 +32,16 @@ if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | tail -n1 |
     exit 1
 fi
 
+. tests/letstest/scripts/set_python_envvars.sh
+
 # Now that python and openssl have been installed, we can set up a fake server
 # to provide a new version of letsencrypt-auto. First, we start the server and
 # directory to be served.
 MY_TEMP_DIR=$(mktemp -d)
 PORT_FILE="$MY_TEMP_DIR/port"
-SERVER_PATH=$(tools/readlink.py tools/simple_http_server.py)
+SERVER_PATH=$("$PYTHON_NAME" tools/readlink.py tools/simple_http_server.py)
 cd "$MY_TEMP_DIR"
-"$SERVER_PATH" 0 > $PORT_FILE &
+"$PYTHON_NAME" "$SERVER_PATH" 0 > $PORT_FILE &
 SERVER_PID=$!
 trap 'kill "$SERVER_PID" && rm -rf "$MY_TEMP_DIR"' EXIT
 cd ~-
@@ -72,7 +74,7 @@ iQIDAQAB
 -----END PUBLIC KEY-----
 "
 
-if [ "$RUN_PYTHON3_TESTS" = 1 ]; then
+if [ "$RUN_RHEL6_TESTS" = 1 ]; then
     if command -v python3; then
         echo "Didn't expect Python 3 to be installed!"
         exit 1
@@ -110,7 +112,7 @@ if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then
     exit 1
 fi
 
-if [ "$RUN_PYTHON3_TESTS" = 1 ]; then
+if [ "$RUN_RHEL6_TESTS" = 1 ]; then
     if ! command -v python3; then
         echo "Python3 wasn't properly installed"
         exit 1
@@ -120,7 +122,7 @@ if [ "$RUN_PYTHON3_TESTS" = 1 ]; then
         exit 1
     fi
 
-    if [ "$(tools/readlink.py $OLD_VENV_PATH)" != "/opt/eff.org/certbot/venv" ]; then
+    if [ "$("$PYTHON_NAME" tools/readlink.py $OLD_VENV_PATH)" != "/opt/eff.org/certbot/venv" ]; then
         echo symlink from old venv path not properly created!
         exit 1
     fi

tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh

@@ -18,6 +18,8 @@ export PATH="$LE_AUTO_DIR:$PATH"
 
 letsencrypt-auto --os-packages-only --debug --version
 
+. tests/letstest/scripts/set_python_envvars.sh
+
 # Create a venv-like layout at the old virtual environment path to test that a
 # symlink is properly created when letsencrypt-auto runs.
 HOME=${HOME:-~root}
@@ -32,7 +34,8 @@ letsencrypt-auto certonly --no-self-upgrade -v --standalone --debug \
                    --register-unsafely-without-email \
                    --domain $PUBLIC_HOSTNAME --server $BOULDER_URL
 
-if [ "$(tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt)" != "/opt/eff.org/certbot/venv" ]; then
+LINK_PATH=$("$PYTHON_NAME" tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt)
+if [ "$LINK_PATH" != "/opt/eff.org/certbot/venv" ]; then
     echo symlink from old venv path not properly created!
     exit 1
 fi

tests/letstest/scripts/test_sdists.sh

@@ -6,16 +6,20 @@ cd letsencrypt
 PLUGINS="certbot-apache certbot-nginx"
 PYTHON_MAJOR_VERSION=$(/opt/eff.org/certbot/venv/bin/python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1)
 TEMP_DIR=$(mktemp -d)
-VERSION=$(letsencrypt-auto-source/version.py)
 
 if [ "$PYTHON_MAJOR_VERSION" = "3" ]; then
+    # Some distros like Fedora may only have an executable named python3 installed.
+    PYTHON_NAME="python3"
     VENV_PATH="venv3"
     VENV_SCRIPT="tools/venv3.py"
 else
+    PYTHON_NAME="python"
     VENV_SCRIPT="tools/venv.py"
     VENV_PATH="venv"
 fi
 
+VERSION=$("$PYTHON_NAME" letsencrypt-auto-source/version.py)
+
 # setup venv
 "$VENV_SCRIPT" --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt
 . "$VENV_PATH/bin/activate"

tests/letstest/targets.yaml

@@ -1,8 +1,8 @@
 targets:
   #-----------------------------------------------------------------------------
   #Ubuntu
-  - ami: ami-064bd2d44a1d6c097
-    name: ubuntu18.10
+  - ami: ami-08ab45c4343f5f5c6
+    name: ubuntu19.04
     type: ubuntu
     virt: hvm
     user: ubuntu
@@ -49,8 +49,8 @@ targets:
     type: centos
     virt: hvm
     user: ec2-user
-  - ami: ami-5c69df23
-    name: fedora28
+  - ami: ami-00bbc6858140f19ed
+    name: fedora30
     type: centos
     virt: hvm
     user: fedora

tools/simple_http_server.py

@@ -1,8 +1,11 @@
 #!/usr/bin/env python
-"""A version of Python 2.x's SimpleHTTPServer that flushes its output."""
-from BaseHTTPServer import HTTPServer
-from SimpleHTTPServer import SimpleHTTPRequestHandler
+"""A version of Python's SimpleHTTPServer that flushes its output."""
 import sys
+try:
+    from http.server import HTTPServer, SimpleHTTPRequestHandler
+except ImportError:
+    from BaseHTTPServer import HTTPServer
+    from SimpleHTTPServer import SimpleHTTPRequestHandler
 
 def serve_forever(port=0):
     """Spins up an HTTP server on all interfaces and the given port.