make tests fail

.travis.yml

@@ -169,7 +169,7 @@ notifications:
   email: false
   irc:
     channels:
-      - "chat.freenode.net#letsencrypt-dev"
+      - secure: "HeXUVxLjB5dLl7JdTvY3EicLkZM5ZNtEa75x4Eud8a656y/6cpuhJli+devRwaAhq7fx7WkkZWWtcm9bJFx/vKwUYr7OCN2OJBNKviQfLxtBENt3gcMHALA+MNi5pQPw5ARe4NvEHABfyGTn6oti+5iFXcgAMGDq11RgDpHR+Bk="
     on_success: never
     on_failure: always
     use_notice: true

tox.ini

@@ -60,8 +60,7 @@ source_paths =
 
 [testenv:py26]
 commands =
-    {[base]install_and_test} {[base]py26_packages}
-    python tests/lock_test.py
+    {toxinidir}/tools/fail.sh
 
 [testenv]
 commands =
@@ -114,6 +113,7 @@ commands =
 [testenv:cover]
 basepython = python2.7
 commands =
+    {toxinidir}/tools/fail.sh
     {[base]install_packages}
     ./tox.cover.sh
 
@@ -124,12 +124,14 @@ basepython = python2.7
 # duplicate code checking; if one of the commands fails, others will
 # continue, but tox return code will reflect previous error
 commands =
+    {toxinidir}/tools/fail.sh
     {[base]install_packages}
     pylint --reports=n --rcfile=.pylintrc {[base]source_paths}
 
 [testenv:mypy]
 basepython = python3.4
 commands =
+    {toxinidir}/tools/fail.sh
     {[base]pip_install} mypy
     {[base]install_packages}
     mypy --py2 --ignore-missing-imports {[base]source_paths}
@@ -137,11 +139,13 @@ commands =
 [testenv:apacheconftest]
 #basepython = python2.7
 commands =
+    {toxinidir}/tools/fail.sh
     {[base]pip_install} acme . certbot-apache certbot-compatibility-test
     {toxinidir}/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test --debian-modules
 
 [testenv:nginxroundtrip]
 commands =
+    {toxinidir}/tools/fail.sh
     {[base]pip_install} acme . certbot-apache certbot-nginx
     python certbot-compatibility-test/nginx/roundtrip.py certbot-compatibility-test/nginx/nginx-roundtrip-testdata
 
@@ -149,10 +153,12 @@ commands =
 # allow users to run the modification check by running `tox`
 [testenv:modification]
 commands =
+    {toxinidir}/tools/fail.sh
     {toxinidir}/tests/modification-check.sh
 
 [testenv:apache_compat]
 commands =
+    {toxinidir}/tools/fail.sh
     docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile .
     docker build -t apache-compat -f certbot-compatibility-test/Dockerfile-apache .
     docker run --rm -it apache-compat -c apache.tar.gz -vvvv
@@ -162,6 +168,7 @@ passenv = DOCKER_*
 
 [testenv:nginx_compat]
 commands =
+    {toxinidir}/tools/fail.sh
     docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile .
     docker build -t nginx-compat -f certbot-compatibility-test/Dockerfile-nginx .
     docker run --rm -it nginx-compat -c nginx.tar.gz -vv -aie
@@ -173,6 +180,7 @@ passenv = DOCKER_*
 # At the moment, this tests under Python 2.7 only, as only that version is
 # readily available on the Precise Docker image.
 commands =
+    {toxinidir}/tools/fail.sh
     docker build -f letsencrypt-auto-source/Dockerfile.precise -t lea letsencrypt-auto-source
     docker run --rm -t -i lea
 whitelist_externals =
@@ -183,6 +191,7 @@ passenv = DOCKER_*
 # At the moment, this tests under Python 2.7 only, as only that version is
 # readily available on the Trusty Docker image.
 commands =
+    {toxinidir}/tools/fail.sh
     {toxinidir}/tests/modification-check.sh
     docker build -f letsencrypt-auto-source/Dockerfile.trusty -t lea letsencrypt-auto-source
     docker run --rm -t -i lea
@@ -196,6 +205,7 @@ passenv =
 # At the moment, this tests under Python 2.7 only, as only that version is
 # readily available on the Wheezy Docker image.
 commands =
+    {toxinidir}/tools/fail.sh
     docker build -f letsencrypt-auto-source/Dockerfile.wheezy -t lea letsencrypt-auto-source
     docker run --rm -t -i lea
 whitelist_externals =
@@ -206,6 +216,7 @@ passenv = DOCKER_*
 # At the moment, this tests under Python 2.6 only, as only that version is
 # readily available on the CentOS 6 Docker image.
 commands =
+    {toxinidir}/tools/fail.sh
     docker build -f letsencrypt-auto-source/Dockerfile.centos6 -t lea letsencrypt-auto-source
     docker run --rm -t -i lea
 whitelist_externals =
@@ -216,6 +227,7 @@ passenv = DOCKER_*
 # tests the Dockerfile-dev file to ensure development with it works
 # as expected
 commands =
+    {toxinidir}/tools/fail.sh
     docker-compose run --rm --service-ports development bash -c 'tox -e lint'
 whitelist_externals =
     docker