Focus on snap tests

This reverts commit 0b7c361e03.

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -1,95 +1,95 @@
 jobs:
-  - job: docker_build
+#  - job: docker_build
-    pool:
+#    pool:
-      vmImage: ubuntu-18.04
+#      vmImage: ubuntu-18.04
-    strategy:
+#    strategy:
-      matrix:
+#      matrix:
-        amd64:
+#        amd64:
-          DOCKER_ARCH: amd64
+#          DOCKER_ARCH: amd64
-        # Do not run the heavy non-amd64 builds for test branches
+#        # Do not run the heavy non-amd64 builds for test branches
-        ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
+#        ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
-          arm32v6:
+#          arm32v6:
-            DOCKER_ARCH: arm32v6
+#            DOCKER_ARCH: arm32v6
-          arm64v8:
+#          arm64v8:
-            DOCKER_ARCH: arm64v8
+#            DOCKER_ARCH: arm64v8
-    steps:
+#    steps:
-      - bash: tools/docker/build.sh $(dockerTag) $DOCKER_ARCH
+#      - bash: tools/docker/build.sh $(dockerTag) $DOCKER_ARCH
-        displayName: Build the Docker images
+#        displayName: Build the Docker images
-      # We don't filter for the Docker Hub organization to continue to allow
+#      # We don't filter for the Docker Hub organization to continue to allow
-      # easy testing of these scripts on forks.
+#      # easy testing of these scripts on forks.
-      - bash: |
+#      - bash: |
-          DOCKER_IMAGES=$(docker images --filter reference='*/certbot' --filter reference='*/dns-*' --format '{{.Repository}}')
+#          DOCKER_IMAGES=$(docker images --filter reference='*/certbot' --filter reference='*/dns-*' --format '{{.Repository}}')
-          docker save --output images.tar $DOCKER_IMAGES
+#          docker save --output images.tar $DOCKER_IMAGES
-        displayName: Save the Docker images
+#        displayName: Save the Docker images
-      # If the name of the tar file or artifact changes, the deploy stage will
+#      # If the name of the tar file or artifact changes, the deploy stage will
-      # also need to be updated.
+#      # also need to be updated.
-      - bash: mv images.tar $(Build.ArtifactStagingDirectory)
+#      - bash: mv images.tar $(Build.ArtifactStagingDirectory)
-        displayName: Prepare Docker artifact
+#        displayName: Prepare Docker artifact
-      - task: PublishPipelineArtifact@1
+#      - task: PublishPipelineArtifact@1
-        inputs:
+#        inputs:
-          path: $(Build.ArtifactStagingDirectory)
+#          path: $(Build.ArtifactStagingDirectory)
-          artifact: docker_$(DOCKER_ARCH)
+#          artifact: docker_$(DOCKER_ARCH)
-        displayName: Store Docker artifact
+#        displayName: Store Docker artifact
-  - job: installer_build
+#  - job: installer_build
-    pool:
+#    pool:
-      vmImage: vs2017-win2016
+#      vmImage: vs2017-win2016
-    steps:
+#    steps:
-      - task: UsePythonVersion@0
+#      - task: UsePythonVersion@0
-        inputs:
+#        inputs:
-          versionSpec: 3.7
+#          versionSpec: 3.7
-          architecture: x86
+#          architecture: x86
-          addToPath: true
+#          addToPath: true
-      - script: python windows-installer/construct.py
+#      - script: python windows-installer/construct.py
-        displayName: Build Certbot installer
+#        displayName: Build Certbot installer
-      - task: CopyFiles@2
+#      - task: CopyFiles@2
-        inputs:
+#        inputs:
-          sourceFolder: $(System.DefaultWorkingDirectory)/windows-installer/build/nsis
+#          sourceFolder: $(System.DefaultWorkingDirectory)/windows-installer/build/nsis
-          contents: '*.exe'
+#          contents: '*.exe'
-          targetFolder: $(Build.ArtifactStagingDirectory)
+#          targetFolder: $(Build.ArtifactStagingDirectory)
-      - task: PublishPipelineArtifact@1
+#      - task: PublishPipelineArtifact@1
-        inputs:
+#        inputs:
-          path: $(Build.ArtifactStagingDirectory)
+#          path: $(Build.ArtifactStagingDirectory)
-          # If we change the artifact's name, it should also be changed in tools/create_github_release.py
+#          # If we change the artifact's name, it should also be changed in tools/create_github_release.py
-          artifact: windows-installer
+#          artifact: windows-installer
-        displayName: Publish Windows installer
+#        displayName: Publish Windows installer
-  - job: installer_run
+#  - job: installer_run
-    dependsOn: installer_build
+#    dependsOn: installer_build
-    strategy:
+#    strategy:
-      matrix:
+#      matrix:
-        win2019:
+#        win2019:
-          imageName: windows-2019
+#          imageName: windows-2019
-        win2016:
+#        win2016:
-          imageName: vs2017-win2016
+#          imageName: vs2017-win2016
-    pool:
+#    pool:
-      vmImage: $(imageName)
+#      vmImage: $(imageName)
-    steps:
+#    steps:
-      - powershell: |
+#      - powershell: |
-          if ($PSVersionTable.PSVersion.Major -ne 5) {
+#          if ($PSVersionTable.PSVersion.Major -ne 5) {
-              throw "Powershell version is not 5.x"
+#              throw "Powershell version is not 5.x"
-          }
+#          }
-        condition: eq(variables['imageName'], 'vs2017-win2016')
+#        condition: eq(variables['imageName'], 'vs2017-win2016')
-        displayName: Check Powershell 5.x is used in vs2017-win2016
+#        displayName: Check Powershell 5.x is used in vs2017-win2016
-      - task: UsePythonVersion@0
+#      - task: UsePythonVersion@0
-        inputs:
+#        inputs:
-          versionSpec: 3.8
+#          versionSpec: 3.8
-          addToPath: true
+#          addToPath: true
-      - task: DownloadPipelineArtifact@2
+#      - task: DownloadPipelineArtifact@2
-        inputs:
+#        inputs:
-          artifact: windows-installer
+#          artifact: windows-installer
-          path: $(Build.SourcesDirectory)/bin
+#          path: $(Build.SourcesDirectory)/bin
-        displayName: Retrieve Windows installer
+#        displayName: Retrieve Windows installer
-      - script: |
+#      - script: |
-          py -3 -m venv venv
+#          py -3 -m venv venv
-          venv\Scripts\python tools\pip_install.py -e certbot-ci
+#          venv\Scripts\python tools\pip_install.py -e certbot-ci
-        displayName: Prepare Certbot-CI
+#        displayName: Prepare Certbot-CI
-      - script: |
+#      - script: |
-          set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
+#          set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
-          venv\Scripts\python -m pytest certbot-ci\windows_installer_integration_tests --allow-persistent-changes --installer-path $(Build.SourcesDirectory)\bin\certbot-beta-installer-win32.exe
+#          venv\Scripts\python -m pytest certbot-ci\windows_installer_integration_tests --allow-persistent-changes --installer-path $(Build.SourcesDirectory)\bin\certbot-beta-installer-win32.exe
-        displayName: Run windows installer integration tests
+#        displayName: Run windows installer integration tests
-      - script: |
+#      - script: |
-          set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
+#          set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
-          venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
+#          venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
-        displayName: Run certbot integration tests
+#        displayName: Run certbot integration tests
   - job: snaps_build
     pool:
       vmImage: ubuntu-18.04

.azure-pipelines/templates/stages/test-and-package-stage.yml

@@ -1,6 +1,6 @@
 stages:
   - stage: TestAndPackage
     jobs:
-      - template: ../jobs/standard-tests-jobs.yml
+#      - template: ../jobs/standard-tests-jobs.yml
-      - template: ../jobs/extended-tests-jobs.yml
+#      - template: ../jobs/extended-tests-jobs.yml
       - template: ../jobs/packaging-jobs.yml

certbot.wrapper

@@ -1,38 +0,0 @@
-#!/bin/sh
-#
-# TODO: We may want to consider rewriting this script in Python. See
-# https://github.com/certbot/certbot/issues/8251 for more info.
-set -e
-
-# This code is based on snapcraft's own patch to work around this problem at
-# https://github.com/snapcore/snapcraft/blob/a97fb5c7ea553a1bd20f4887a7c3393e75761890/patches/ctypes_init.diff.
-# We may not build the Certbot snap for all of these architectures (and as of
-# writing this we do not), but we keep the code for them to avoid having to
-# solve this problem again in the future if we add support for new
-# architectures.
-case "${SNAP_ARCH}" in
-    'arm64')
-        ARCH_TRIPLET='aarch64-linux-gnu';;
-    'armhf')
-        ARCH_TRIPLET='arm-linux-gnueabihf';;
-    'i386')
-        ARCH_TRIPLET='i386-linux-gnu';;
-    'ppc64el')
-        ARCH_TRIPLET='powerpc64le-linux-gnu';;
-    'powerpc')
-        ARCH_TRIPLET='powerpc-linux-gnu';;
-    'amd64')
-        ARCH_TRIPLET='x86_64-linux-gnu';;
-    's390x')
-        ARCH_TRIPLET='s390x-linux-gnu';;
-    *)
-        echo "Unrecongized value of SNAP_ARCH: ${SNAP_ARCH}" >&2
-        exit 1
-esac
-
-export CERTBOT_AUGEAS_PATH="${SNAP}/usr/lib/${ARCH_TRIPLET}/libaugeas.so.0"
-
-CERTBOT_PLUGIN_PATH="$(curl -s --unix-socket /run/snapd.socket "http://localhost/v2/connections?snap=certbot&interface=content" | jq -r  '.result.established | map(select(.plug.plug == "plugin" and ."plug-attrs".content == "certbot-1") | "/snap/"+.slot.snap+"/current/lib/python3.8/site-packages/" ) | join(":")')"
-export CERTBOT_PLUGIN_PATH
-
-exec certbot "$@" --preconfigured-renewal

certbot/certbot/_internal/main.py

@@ -28,6 +28,7 @@ from certbot._internal import hooks
 from certbot._internal import log
 from certbot._internal import renewal
 from certbot._internal import reporter
+from certbot._internal import snap_config
 from certbot._internal import storage
 from certbot._internal import updater
 from certbot._internal.plugins import disco as plugins_disco
@@ -1325,6 +1326,9 @@ def main(cli_args=None):
 
     log.pre_arg_parse_setup()
 
+    if os.environ.get('CERTBOT_SNAPPED') == 'True':
+        cli_args = snap_config.prepare_env(cli_args)
+
     plugins = plugins_disco.PluginsRegistry.find_all()
     logger.debug("certbot version: %s", certbot.__version__)
     # do not log `config`, as it contains sensitive data (e.g. revoke --key)!

certbot/certbot/_internal/snap_config.py

@@ -0,0 +1,102 @@
+"""Module configuring Certbot in a snap environment"""
+import logging
+import socket
+
+from requests import Session
+from requests.adapters import HTTPAdapter
+from requests.exceptions import HTTPError
+from requests.exceptions import RequestException
+
+from acme.magic_typing import List
+from certbot.compat import os
+from certbot.errors import Error
+
+try:
+    from urllib3.connection import HTTPConnection
+    from urllib3.connectionpool import HTTPConnectionPool
+except ImportError:
+    # Stub imports for oldest requirements, that will never be used in snaps.
+    HTTPConnection = object
+    HTTPConnectionPool = object
+
+
+_ARCH_TRIPLET_MAP = {
+    'arm64': 'aarch64-linux-gnu',
+    'armhf': 'arm-linux-gnueabihf',
+    'i386': 'i386-linux-gnu',
+    'ppc64el': 'powerpc64le-linux-gnu',
+    'powerpc': 'powerpc-linux-gnu',
+    'amd64': 'x86_64-linux-gnu',
+    's390x': 's390x-linux-gnu',
+}
+
+LOGGER = logging.getLogger(__name__)
+
+
+def prepare_env(cli_args):
+    # type: (List[str]) -> List[str]
+    """
+    Prepare runtime environment for a certbot execution in snap.
+    :param list cli_args: List of command line arguments
+    :return: Update list of command line arguments
+    :rtype: list
+    """
+    snap_arch = os.environ.get('SNAP_ARCH')
+
+    if snap_arch not in _ARCH_TRIPLET_MAP:
+        raise Error('Unrecognized value of SNAP_ARCH: {0}'.format(snap_arch))
+
+    os.environ['CERTBOT_AUGEAS_PATH'] = '{0}/usr/lib/{1}/libaugeas.so.0'.format(
+        os.environ.get('SNAP'), _ARCH_TRIPLET_MAP[snap_arch])
+
+    session = Session()
+    session.mount('http://snapd/', _SnapdAdapter())
+
+    try:
+        response = session.get('http://snapd/v2/connections?snap=certbot&interface=content')
+        response.raise_for_status()
+    except RequestException as e:
+        if isinstance(e, HTTPError) and e.response.status_code == 404:
+            LOGGER.error('An error occurred while fetching Certbot snap plugins: '
+                         'your version of snapd is outdated.')
+            LOGGER.error('Please run "sudo snap install core; sudo snap refresh" '
+                         'in your terminal and try again.')
+        else:
+            LOGGER.error('An error occurred while fetching Certbot snap plugins: '
+                         'make sure the snapd service is running.')
+        raise e
+
+    data = response.json()
+    connections = ['/snap/{0}/current/lib/python3.8/site-packages/'.format(item['slot']['snap'])
+                   for item in data.get('result', {}).get('established', [])
+                   if item.get('plug', {}).get('plug') == 'plugin'
+                   and item.get('plug-attrs', {}).get('content') == 'certbot-1']
+
+    os.environ['CERTBOT_PLUGIN_PATH'] = ':'.join(connections)
+
+    cli_args.append('--preconfigured-renewal')
+
+    return cli_args
+
+
+class _SnapdConnection(HTTPConnection):
+    def __init__(self):
+        super(_SnapdConnection, self).__init__("localhost")
+        self.sock = None
+
+    def connect(self):
+        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        self.sock.connect("/run/snapd.socket")
+
+
+class _SnapdConnectionPool(HTTPConnectionPool):
+    def __init__(self):
+        super(_SnapdConnectionPool, self).__init__("localhost")
+
+    def _new_conn(self):
+        return _SnapdConnection()
+
+
+class _SnapdAdapter(HTTPAdapter):
+    def get_connection(self, url, proxies=None):
+        return _SnapdConnectionPool()

snap/snapcraft.yaml

@@ -20,13 +20,13 @@ adopt-info: certbot
 
 apps:
   certbot:
-    command: certbot.wrapper
+    command: bin/python3 $SNAP/bin/certbot
     environment:
       PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
       AUGEAS_LENS_LIB: "$SNAP/usr/share/augeas/lenses/dist"
       CERTBOT_SNAPPED: "True"
   renew:
-    command: certbot.wrapper -q renew
+    command: bin/python3 $SNAP/bin/certbot -q renew
     daemon: oneshot
     environment:
       PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
@@ -71,9 +71,6 @@ parts:
       - python3-distutils
       - python3-pkg-resources
       - python3.8-minimal
-      # added for certbot.wrapper script:
-      - curl
-      - jq
     # To build cryptography and cffi if needed
     build-packages: [gcc, libffi-dev, libssl-dev, git, libaugeas-dev, python3-dev]
     build-environment:
@@ -84,10 +81,6 @@ parts:
         cd $SNAPCRAFT_PART_SRC
         python3 tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt | grep -v python-augeas > snap-constraints.txt
         snapcraftctl set-version `git describe|sed s/^v//`
-  wrappers:
-    plugin: dump
-    source: .
-    stage: [certbot.wrapper]
   shared-metadata:
     plugin: dump
     source: .