rewrite coverage tests (#9669)
In addition to the speed improvements in CI, the speed improvements locally with both this https://github.com/certbot/certbot/pull/9666 which this builds on is even more significant. After it's been run once so it's had a chance to set up the different virtual environments, `tox` locally now takes 39 seconds on my laptop when it used to take 137 seconds.
This commit is contained in:
Brad Warren
19
.coveragerc
19
.coveragerc
@@ -1,5 +1,24 @@
|
||||
[run]
|
||||
omit = */setup.py
|
||||
source =
|
||||
acme
|
||||
certbot
|
||||
certbot-apache
|
||||
certbot-dns-cloudflare
|
||||
certbot-dns-digitalocean
|
||||
certbot-dns-dnsimple
|
||||
certbot-dns-dnsmadeeasy
|
||||
certbot-dns-gehirn
|
||||
certbot-dns-google
|
||||
certbot-dns-linode
|
||||
certbot-dns-luadns
|
||||
certbot-dns-nsone
|
||||
certbot-dns-ovh
|
||||
certbot-dns-rfc2136
|
||||
certbot-dns-route53
|
||||
certbot-dns-sakuracloud
|
||||
certbot-nginx
|
||||
|
||||
[report]
|
||||
omit = */setup.py
|
||||
show_missing = True
|
||||
|
||||
87
tox.cover.py
87
tox.cover.py
@@ -1,87 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
DEFAULT_PACKAGES = [
|
||||
'certbot', 'acme', 'certbot_apache', 'certbot_dns_cloudflare',
|
||||
'certbot_dns_digitalocean', 'certbot_dns_dnsimple', 'certbot_dns_dnsmadeeasy',
|
||||
'certbot_dns_gehirn', 'certbot_dns_google', 'certbot_dns_linode', 'certbot_dns_luadns',
|
||||
'certbot_dns_nsone', 'certbot_dns_ovh', 'certbot_dns_rfc2136', 'certbot_dns_route53',
|
||||
'certbot_dns_sakuracloud', 'certbot_nginx']
|
||||
|
||||
COVER_THRESHOLDS = {
|
||||
'certbot': {'linux': 94, 'windows': 96},
|
||||
'acme': {'linux': 100, 'windows': 99},
|
||||
'certbot_apache': {'linux': 100, 'windows': 100},
|
||||
'certbot_dns_cloudflare': {'linux': 98, 'windows': 98},
|
||||
'certbot_dns_digitalocean': {'linux': 98, 'windows': 98},
|
||||
'certbot_dns_dnsimple': {'linux': 98, 'windows': 98},
|
||||
'certbot_dns_dnsmadeeasy': {'linux': 99, 'windows': 99},
|
||||
'certbot_dns_gehirn': {'linux': 97, 'windows': 97},
|
||||
'certbot_dns_google': {'linux': 99, 'windows': 99},
|
||||
'certbot_dns_linode': {'linux': 98, 'windows': 98},
|
||||
'certbot_dns_luadns': {'linux': 98, 'windows': 98},
|
||||
'certbot_dns_nsone': {'linux': 99, 'windows': 99},
|
||||
'certbot_dns_ovh': {'linux': 97, 'windows': 97},
|
||||
'certbot_dns_rfc2136': {'linux': 99, 'windows': 99},
|
||||
'certbot_dns_route53': {'linux': 92, 'windows': 92},
|
||||
'certbot_dns_sakuracloud': {'linux': 97, 'windows': 97},
|
||||
'certbot_nginx': {'linux': 97, 'windows': 97},
|
||||
}
|
||||
|
||||
SKIP_PROJECTS_ON_WINDOWS = ['certbot-apache']
|
||||
|
||||
|
||||
def cover(package):
|
||||
threshold = COVER_THRESHOLDS.get(package)['windows' if os.name == 'nt' else 'linux']
|
||||
|
||||
pkg_dir = package.replace('_', '-')
|
||||
|
||||
if os.name == 'nt' and pkg_dir in SKIP_PROJECTS_ON_WINDOWS:
|
||||
print((
|
||||
'Info: currently {0} is not supported on Windows and will not be tested/covered.'
|
||||
.format(pkg_dir)))
|
||||
return
|
||||
|
||||
subprocess.check_call([sys.executable, '-m', 'pytest',
|
||||
'--cov', pkg_dir, '--cov-append', '--cov-report=', pkg_dir])
|
||||
try:
|
||||
subprocess.check_call([
|
||||
sys.executable, '-m', 'coverage', 'report', '--fail-under',
|
||||
str(threshold), '--include', '{0}/*'.format(pkg_dir),
|
||||
'--show-missing'])
|
||||
except subprocess.CalledProcessError as err:
|
||||
print(err)
|
||||
print('Test coverage on', pkg_dir,
|
||||
'did not meet threshold of {0}%.'.format(threshold))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
description = """
|
||||
This script is used by tox.ini (and thus by Travis CI and Azure Pipelines) in
|
||||
order to generate separate stats for each package. It should be removed once
|
||||
those packages are moved to a separate repo."""
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
parser.add_argument('--packages', nargs='+')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
packages = args.packages or DEFAULT_PACKAGES
|
||||
|
||||
# --cov-append is on, make sure stats are correct
|
||||
try:
|
||||
os.remove('.coverage')
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
for package in packages:
|
||||
cover(package)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
33
tox.ini
33
tox.ini
@@ -73,8 +73,37 @@ basepython = python3.7
|
||||
setenv = CERTBOT_OLDEST=1
|
||||
commands = {[testenv:py]commands}
|
||||
|
||||
[testenv:cover{,-win,-posix}]
|
||||
commands = python tox.cover.py
|
||||
[testenv:cover{,-posix}]
|
||||
coverage_report = python -m coverage report
|
||||
# These coverage report commands are used on both posix and windows
|
||||
common_coverage_report_commands =
|
||||
{[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-cloudflare/*
|
||||
{[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-digitalocean/*
|
||||
{[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-dnsimple/*
|
||||
{[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-dnsmadeeasy/*
|
||||
{[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-gehirn/*
|
||||
{[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-google/*
|
||||
{[testenv:cover]coverage_report} --fail-under 100 --include certbot-dns-linode/*
|
||||
{[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-luadns/*
|
||||
{[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-nsone/*
|
||||
{[testenv:cover]coverage_report} --fail-under 98 --include certbot-dns-ovh/*
|
||||
{[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-rfc2136/*
|
||||
{[testenv:cover]coverage_report} --fail-under 94 --include certbot-dns-route53/*
|
||||
{[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-sakuracloud/*
|
||||
{[testenv:cover]coverage_report} --fail-under 98 --include certbot-nginx/*
|
||||
commands =
|
||||
{[testenv:py]commands} --cov --cov-report=
|
||||
{[testenv:cover]coverage_report} --fail-under 100 --include acme/*
|
||||
{[testenv:cover]coverage_report} --fail-under 95 --include certbot/*
|
||||
{[testenv:cover]coverage_report} --fail-under 100 --include certbot-apache/*
|
||||
{[testenv:cover]common_coverage_report_commands}
|
||||
|
||||
[testenv:cover-win]
|
||||
commands =
|
||||
{[testenv:py-win]commands} --cov --cov-report=
|
||||
{[testenv:cover]coverage_report} --fail-under 99 --include acme/*
|
||||
{[testenv:cover]coverage_report} --fail-under 96 --include certbot/*
|
||||
{[testenv:cover]common_coverage_report_commands}
|
||||
|
||||
[testenv:lint{,-win,-posix}]
|
||||
commands = python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths}
|
||||
|
||||
Reference in New Issue
Block a user