undo some stuff

acme/setup.py

@@ -14,7 +14,7 @@ install_requires = [
     'PyOpenSSL>=17.3.0',
     'pyrfc3339',
     'pytz',
-    'requests>=2.6.0',
+    'requests>=2.14.2',
     'requests-toolbelt>=0.3.0',
     'setuptools>=39.0.1',
 ]

certbot-dns-cloudxns/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup
 version = '1.17.0.dev0'
 
 install_requires = [
-    'dns-lexicon>=3.1.0',  # Changed `rtype` parameter name
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot-dns-dnsimple/setup.py

@@ -7,6 +7,9 @@ from setuptools import setup
 version = '1.17.0.dev0'
 
 install_requires = [
+    # This version of lexicon is required to address the problem described in
+    # https://github.com/AnalogJ/lexicon/issues/387.
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]
@@ -25,18 +28,6 @@ elif 'bdist_wheel' in sys.argv[1:]:
 if os.environ.get('SNAP_BUILD'):
     install_requires.append('packaging')
 
-# This package normally depends on dns-lexicon>=3.2.1 to address the
-# problem described in https://github.com/AnalogJ/lexicon/issues/387,
-# however, the fix there has been backported to older versions of
-# lexicon found in various Linux distros. This conditional helps us test
-# that we've maintained compatibility with these versions of lexicon
-# which allows us to potentially upgrade our packages in these distros
-# as necessary.
-if os.environ.get('CERTBOT_OLDEST') == '1':
-    install_requires.append('dns-lexicon>=3.1.0')  # Changed parameter name
-else:
-    install_requires.append('dns-lexicon>=3.2.1')
-
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',

certbot-dns-dnsmadeeasy/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup
 version = '1.17.0.dev0'
 
 install_requires = [
-    'dns-lexicon>=3.1.0',  # Changed `rtype` parameter name
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot-dns-gehirn/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup
 version = '1.17.0.dev0'
 
 install_requires = [
-    'dns-lexicon>=3.1.0',  # Changed `rtype` parameter name
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot-dns-linode/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup
 version = '1.17.0.dev0'
 
 install_requires = [
-    'dns-lexicon>=3.1.0',  # Changed `rtype` parameter name
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot-dns-luadns/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup
 version = '1.17.0.dev0'
 
 install_requires = [
-    'dns-lexicon>=3.1.0',  # Changed `rtype` parameter name
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot-dns-nsone/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup
 version = '1.17.0.dev0'
 
 install_requires = [
-    'dns-lexicon>=3.1.0',  # Changed `rtype` parameter name
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot-dns-ovh/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup
 version = '1.17.0.dev0'
 
 install_requires = [
-    'dns-lexicon>=3.1.0',  # Changed `rtype` parameter name
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot-dns-sakuracloud/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup
 version = '1.17.0.dev0'
 
 install_requires = [
-    'dns-lexicon>=3.1.0',  # Changed `rtype` parameter name
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot/setup.py

@@ -70,13 +70,10 @@ dev_extras = [
     'azure-devops',
     'ipdb',
     'PyGithub',
-    'pip',
     # poetry 1.2.0+ is required for it to pin pip, setuptools, and wheel. See
     # https://github.com/python-poetry/poetry/issues/1584.
     'poetry>=1.2.0a1',
-    'tox',
     'twine',
-    'wheel',
 ]
 
 docs_extras = [
@@ -87,16 +84,22 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+# Tools like pip, wheel, and tox are listed here to ensure they are properly
+# pinned and installed during automated testing.
 test_extras = [
     'coverage',
     'mypy',
+    'pip',
     'pylint',
     'pytest',
     'pytest-cov',
     'pytest-xdist',
+    'setuptools',
+    'tox',
     # typing-extensions is required to import typing.Protocol and make the mypy checks
     # pass (along with pylint about non-existent objects) on Python 3.6 & 3.7
     'typing-extensions',
+    'wheel',
 ]
 
 

pytest.ini

@@ -13,6 +13,10 @@
 # The current warnings being ignored are:
 # 1) The warning raised when importing certbot.tests.util and the external mock
 #    library is installed.
+# 2) An ImportWarning is raised with older versions of setuptools and
+#    zope.interface. See
+#    https://github.com/zopefoundation/zope.interface/issues/68 for more info.
 filterwarnings =
     error
     ignore:The external mock module:PendingDeprecationWarning
+    ignore:.*zope. missing __init__:ImportWarning

tools/pinning/common/export-pinned-dependencies.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# This script accepts a directory containing a pyproject.toml file configured
+# for use with poetry and generates and prints the pinned dependencies of that
+# file. Any dependencies on acme or those referencing certbot will be removed
+# from the output. The exported requirements are printed to stdout.
+set -euo pipefail
+
+if [ -z ${1+x} ]; then
+    echo "Usage:" >&2
+    echo "$0 PYPROJECT_TOML_DIRECTORY" >&2
+    exit 1
+fi
+
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+WORK_DIR="$1"
+
+if ! command -v poetry >/dev/null || [ $(poetry --version | grep -oE '[0-9]+\.[0-9]+' | sed 's/\.//') -lt 12 ]; then
+    echo "Please install poetry 1.2+." >&2
+    echo "You may need to recreate Certbot's virtual environment and activate it." >&2
+    exit 1
+fi
+
+# Old eggs can cause outdated dependency information to be used by poetry so we
+# delete them before generating the lock file. See
+# https://github.com/python-poetry/poetry/issues/4103 for more info.
+rm -rf ${REPO_ROOT}/*.egg-info
+
+cd "${WORK_DIR}"
+
+if [ -f poetry.lock ]; then
+    rm poetry.lock
+fi
+
+poetry lock >&2
+trap 'rm poetry.lock' EXIT
+
+# We need to remove local packages from the output.
+poetry export --without-hashes | sed '/^acme @/d; /certbot/d;'

tools/pinning/normal/pyproject.toml

@@ -12,28 +12,28 @@ python = "^3.6"
 # Any local packages that have dependencies on other local packages must be
 # listed below before the package it depends on. For instance, certbot depends
 # on acme so certbot must be listed before acme.
-certbot-ci = {path = "../../certbot-ci"}
-certbot-compatibility-test = {path = "../../certbot-compatibility-test"}
-certbot-dns-cloudflare = {path = "../../certbot-dns-cloudflare", extras = ["docs"]}
-certbot-dns-cloudxns = {path = "../../certbot-dns-cloudxns", extras = ["docs"]}
-certbot-dns-digitalocean = {path = "../../certbot-dns-digitalocean", extras = ["docs"]}
-certbot-dns-dnsimple = {path = "../../certbot-dns-dnsimple", extras = ["docs"]}
-certbot-dns-dnsmadeeasy = {path = "../../certbot-dns-dnsmadeeasy", extras = ["docs"]}
-certbot-dns-gehirn = {path = "../../certbot-dns-gehirn", extras = ["docs"]}
-certbot-dns-google = {path = "../../certbot-dns-google", extras = ["docs"]}
-certbot-dns-linode = {path = "../../certbot-dns-linode", extras = ["docs"]}
-certbot-dns-luadns = {path = "../../certbot-dns-luadns", extras = ["docs"]}
-certbot-dns-nsone = {path = "../../certbot-dns-nsone", extras = ["docs"]}
-certbot-dns-ovh = {path = "../../certbot-dns-ovh", extras = ["docs"]}
-certbot-dns-rfc2136 = {path = "../../certbot-dns-rfc2136", extras = ["docs"]}
-certbot-dns-route53 = {path = "../../certbot-dns-route53", extras = ["docs"]}
-certbot-dns-sakuracloud = {path = "../../certbot-dns-sakuracloud", extras = ["docs"]}
-certbot-nginx = {path = "../../certbot-nginx"}
-certbot-apache = {path = "../../certbot-apache", extras = ["dev"]}
-certbot = {path = "../../certbot", extras = ["all"]}
-acme = {path = "../../acme", extras = ["docs", "test"]}
-letstest = {path = "../../letstest"}
-windows-installer = {path = "../../windows-installer"}
+certbot-ci = {path = "../../../certbot-ci"}
+certbot-compatibility-test = {path = "../../../certbot-compatibility-test"}
+certbot-dns-cloudflare = {path = "../../../certbot-dns-cloudflare", extras = ["docs"]}
+certbot-dns-cloudxns = {path = "../../../certbot-dns-cloudxns", extras = ["docs"]}
+certbot-dns-digitalocean = {path = "../../../certbot-dns-digitalocean", extras = ["docs"]}
+certbot-dns-dnsimple = {path = "../../../certbot-dns-dnsimple", extras = ["docs"]}
+certbot-dns-dnsmadeeasy = {path = "../../../certbot-dns-dnsmadeeasy", extras = ["docs"]}
+certbot-dns-gehirn = {path = "../../../certbot-dns-gehirn", extras = ["docs"]}
+certbot-dns-google = {path = "../../../certbot-dns-google", extras = ["docs"]}
+certbot-dns-linode = {path = "../../../certbot-dns-linode", extras = ["docs"]}
+certbot-dns-luadns = {path = "../../../certbot-dns-luadns", extras = ["docs"]}
+certbot-dns-nsone = {path = "../../../certbot-dns-nsone", extras = ["docs"]}
+certbot-dns-ovh = {path = "../../../certbot-dns-ovh", extras = ["docs"]}
+certbot-dns-rfc2136 = {path = "../../../certbot-dns-rfc2136", extras = ["docs"]}
+certbot-dns-route53 = {path = "../../../certbot-dns-route53", extras = ["docs"]}
+certbot-dns-sakuracloud = {path = "../../../certbot-dns-sakuracloud", extras = ["docs"]}
+certbot-nginx = {path = "../../../certbot-nginx"}
+certbot-apache = {path = "../../../certbot-apache", extras = ["dev"]}
+certbot = {path = "../../../certbot", extras = ["all"]}
+acme = {path = "../../../acme", extras = ["docs", "test"]}
+letstest = {path = "../../../letstest"}
+windows-installer = {path = "../../../windows-installer"}
 
 # Extra dependencies
 # awscli is just listed here as a performance optimization. As of writing this,

tools/pinning/normal/repin.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# This script accepts no arguments and automates the process of updating
+# Certbot's dependencies. Dependencies can be pinned to older versions by
+# modifying pyproject.toml in the same directory as this file.
+set -euo pipefail
+
+WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+COMMON_DIR="$(dirname "${WORK_DIR}")/common"
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+RELATIVE_SCRIPT_PATH="$(realpath --relative-to "$REPO_ROOT" "$WORK_DIR")/$(basename "${BASH_SOURCE[0]}")"
+REQUIREMENTS_FILE="$REPO_ROOT/tools/requirements.txt"
+
+PINNINGS=$("${COMMON_DIR}/export-pinned-dependencies.sh" "${WORK_DIR}")
+cat << EOF > "$REQUIREMENTS_FILE"
+# This file was generated by $RELATIVE_SCRIPT_PATH and can be updated using
+# that script.
+#
+# It is normally used as constraints to pip, however, it has the name
+# requirements.txt so that is scanned by GitHub. See
+# https://docs.github.com/en/github/visualizing-repository-data-with-graphs/about-the-dependency-graph#supported-package-ecosystems
+# for more info.
+EOF
+echo "${PINNINGS}" >> "${REQUIREMENTS_FILE}"

tools/pinning/oldest/pyproject.toml

@@ -0,0 +1,142 @@
+[tool.poetry]
+name = "certbot-pinner"
+version = "0.1.0"
+description = "A simple project for pinning Certbot's dependencies using Poetry."
+authors = ["Certbot Project"]
+license = "Apache License 2.0"
+
+[tool.poetry.dependencies]
+# The Python version here should be kept in sync with the one used in our
+# oldest tests in tox.ini.
+python = "3.6"
+
+# Local dependencies
+# Any local packages that have dependencies on other local packages must be
+# listed below before the package it depends on. For instance, certbot depends
+# on acme so certbot must be listed before acme.
+certbot-ci = {path = "../../../certbot-ci"}
+certbot-dns-cloudflare = {path = "../../../certbot-dns-cloudflare"}
+certbot-dns-cloudxns = {path = "../../../certbot-dns-cloudxns"}
+certbot-dns-digitalocean = {path = "../../../certbot-dns-digitalocean"}
+certbot-dns-dnsimple = {path = "../../../certbot-dns-dnsimple"}
+certbot-dns-dnsmadeeasy = {path = "../../../certbot-dns-dnsmadeeasy"}
+certbot-dns-gehirn = {path = "../../../certbot-dns-gehirn"}
+certbot-dns-google = {path = "../../../certbot-dns-google"}
+certbot-dns-linode = {path = "../../../certbot-dns-linode"}
+certbot-dns-luadns = {path = "../../../certbot-dns-luadns"}
+certbot-dns-nsone = {path = "../../../certbot-dns-nsone"}
+certbot-dns-ovh = {path = "../../../certbot-dns-ovh"}
+certbot-dns-rfc2136 = {path = "../../../certbot-dns-rfc2136"}
+certbot-dns-route53 = {path = "../../../certbot-dns-route53"}
+certbot-dns-sakuracloud = {path = "../../../certbot-dns-sakuracloud"}
+certbot-nginx = {path = "../../../certbot-nginx"}
+certbot-apache = {path = "../../../certbot-apache", extras = ["dev"]}
+certbot = {path = "../../../certbot", extras = ["test"]}
+acme = {path = "../../../acme", extras = ["test"]}
+
+# Oldest dependencies
+# We specify the oldest versions our dependencies that we're trying to keep
+# support for below. Usually these version numbers are taken from the packages
+# of our dependencies available in popular LTS Linux distros. Keeping
+# compatibility with those versions makes it much easier for OS maintainers to
+# update their Certbot packages.
+#
+# When updating these dependencies, we should ideally try to only update them
+# to the oldest version of the dependency that is found in a non-EOL'd version
+# of CentOS, Debian, or Ubuntu that has Certbot packages in their OS
+# repositories using a version of Python we support. If the distro is EOL'd or
+# using a version of Python we don't support, it can be ignored.
+
+# CentOS/RHEL 7 EPEL dependencies
+# Some of these dependencies may be stricter than necessary because they
+# initially referred to the Python 2 packages in CentOS/RHEL 7 with EPEL.
+cffi = "1.9.1"
+chardet = "2.2.1"
+ipaddress = "1.0.16"
+mock = "1.0.1"
+ndg-httpsclient = "0.3.2"
+ply = "3.4"
+pyOpenSSL = "17.3.0"
+pyasn1 = "0.1.9"
+pycparser = "2.14"
+pyRFC3339 = "1.0"
+python-augeas = "0.5.0"
+oauth2client = "4.0.0"
+requests = "2.14.2"
+urllib3 = "1.10.2"
+# Package names containing "." need to be quoted.
+"zope.component" = "4.1.0"
+"zope.event" = "4.0.3"
+"zope.interface" = "4.0.5"
+
+# Debian Jessie Backports dependencies
+# Debian Jessie has reached end of life so these dependencies can probably be
+# updated as needed or desired.
+pbr = "1.8.0"
+pytz = "2012rc0"
+
+# Debian Buster dependencies
+google-api-python-client = "1.5.5"
+pyparsing = "2.2.0"
+
+# Our setup.py dependencies
+apacheconfig = "0.3.2"
+cloudflare = "1.5.1"
+python-digitalocean = "1.11"
+
+# Ubuntu Xenial dependencies
+# Ubuntu Xenial only has versions of Python which we do not support available
+# so these dependencies can probably be updated as needed or desired.
+ConfigArgParse = "0.10.0"
+funcsigs = "0.4"
+# Package names containing "." need to be quoted.
+"zope.hookable" = "4.0.4"
+
+# Ubuntu Bionic dependencies.
+cryptography = "2.1.4"
+distro = "1.0.1"
+httplib2 = "0.9.2"
+idna = "2.6"
+setuptools = "39.0.1"
+six = "1.11.0"
+
+# Ubuntu Focal dependencies
+asn1crypto = "0.24.0"
+configobj = "5.0.6"
+parsedatetime = "2.4"
+
+# Plugin dependencies
+# These aren't necessarily the oldest versions we need to support
+# Tracking at https://github.com/certbot/certbot/issues/6473
+boto3 = "1.4.7"
+botocore = "1.7.41"
+dns-lexicon = "3.2.1"
+
+# Build dependencies
+# Since there doesn't appear to
+# doesn't appear to be a good way to automatically track down and pin build
+# dependencies in Python (see
+# https://discuss.python.org/t/how-to-pin-build-dependencies/8238), we list any
+# build dependencies here to ensure they're pinned for extra stability.
+
+# cython is a build dependency of pyyaml
+cython = "*"
+
+# Other dependencies
+# We add any dependencies that must be specified in this file for any another
+# reason below.
+
+# pip's new dependency resolver fails on local packages that depend on each
+# other when those packages are requested with extras such as 'certbot[dev]' so
+# let's pin it back for now. See https://github.com/pypa/pip/issues/9204.
+pip = "20.2.4"
+
+# wheel 0.34.1+ does not support the version of setuptools pinned above (and
+# wheel 0.34.0 is buggy).
+wheel = "<0.34.0"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

tools/pinning/oldest/repin.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# This script accepts no arguments and automates the process of updating
+# Certbot's dependencies used for our "oldest" tests. Dependencies can be
+# pinned to older versions by modifying pyproject.toml in the same directory as
+# this file.
+set -euo pipefail
+
+WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+COMMON_DIR="$(dirname "${WORK_DIR}")/common"
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+RELATIVE_SCRIPT_PATH="$(realpath --relative-to "$REPO_ROOT" "$WORK_DIR")/$(basename "${BASH_SOURCE[0]}")"
+CONSTRAINTS_FILE="$REPO_ROOT/tools/oldest_constraints.txt"
+
+PINNINGS=$("${COMMON_DIR}/export-pinned-dependencies.sh" "${WORK_DIR}")
+cat << EOF > "$CONSTRAINTS_FILE"
+# This file was generated by $RELATIVE_SCRIPT_PATH and can be updated using
+# that script.
+EOF
+echo "${PINNINGS}" >> "${CONSTRAINTS_FILE}"

tools/pip_install.py

@@ -1,54 +1,23 @@
 #!/usr/bin/env python
 # pip installs packages using pinned package versions. If CERTBOT_OLDEST is set
-# to 1, a combination of tools/oldest_constraints.txt and
-# tools/dev_constraints.txt is used, otherwise, tools/requirements.txt is used.
+# to 1, tools/oldest_constraints.txt is used, otherwise, tools/requirements.txt
+# is used.
 
 from __future__ import absolute_import
 from __future__ import print_function
 
-import contextlib
 import os
-import re
-import shutil
 import subprocess
 import sys
 import tempfile
 
-import merge_requirements as merge_module
 import readlink
 
 
-# Once this code doesn't need to support Python 2, we can simply use
-# tempfile.TemporaryDirectory.
-@contextlib.contextmanager
-def temporary_directory():
-    dirpath = tempfile.mkdtemp()
-    try:
-        yield dirpath
-    finally:
-        shutil.rmtree(dirpath)
-
-
 def find_tools_path():
     return os.path.dirname(readlink.main(__file__))
 
 
-def certbot_oldest_processing(tools_path, constraints_path):
-    # The order of the files in this list matters as files specified later can
-    # override the pinnings found in earlier files.
-    pinning_files = [os.path.join(tools_path, 'dev_constraints.txt'),
-                     os.path.join(tools_path, 'oldest_constraints.txt')]
-    with open(constraints_path, 'w') as fd:
-        fd.write(merge_module.main(*pinning_files))
-
-
-def certbot_normal_processing(tools_path, constraints_path):
-    repo_path = os.path.dirname(tools_path)
-    requirements = os.path.normpath(os.path.join(
-        repo_path, 'tools/requirements.txt'))
-    shutil.copy(requirements, constraints_path)
-
-
 def call_with_print(command, env=None):
     if not env:
         env = os.environ
@@ -66,17 +35,20 @@ def pip_install_with_print(args_str, env=None):
 def main(args):
     tools_path = find_tools_path()
 
-    with temporary_directory() as working_dir:
+    with tempfile.TemporaryDirectory() as working_dir:
         if os.environ.get('CERTBOT_NO_PIN') == '1':
             # With unpinned dependencies, there is no constraint
             pip_install_with_print(' '.join(args))
         else:
-            # Otherwise, we merge requirements to build the constraints and pin dependencies
-            constraints_path = os.path.join(working_dir, 'constraints.txt')
+            # Otherwise, we pick the constraints file based on the environment
+            # variable CERTBOT_OLDEST.
+            repo_path = os.path.dirname(tools_path)
             if os.environ.get('CERTBOT_OLDEST') == '1':
-                certbot_oldest_processing(tools_path, constraints_path)
+                constraints_path = os.path.normpath(os.path.join(
+                    repo_path, 'tools', 'oldest_constraints.txt'))
             else:
-                certbot_normal_processing(tools_path, constraints_path)
+                constraints_path = os.path.normpath(os.path.join(
+                    repo_path, 'tools', 'requirements.txt'))
 
             env = os.environ.copy()
             env["PIP_CONSTRAINT"] = constraints_path

tox.ini

@@ -43,6 +43,9 @@ setenv =
 # Setting basepython allows the tests to fail fast if that version of Python
 # isn't available instead of potentially trying to use a newer version of
 # Python which is unlikely to work.
+#
+# This version should be kept in sync with the one declared in
+# tools/pinning/oldest/pyproject.toml.
 basepython = python3.6
 commands =
     {[testenv]commands}