diff --git a/.gitignore b/.gitignore index e744a82a2..54545e883 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,8 @@ dist*/ /venv*/ /kgs/ /.tox/ -/releases/ +/releases*/ +/log* letsencrypt.log certbot.log letsencrypt-auto-source/letsencrypt-auto.sig.lzma.base64 @@ -39,6 +40,7 @@ tests/letstest/venv/ # pytest cache .cache .mypy_cache/ +.pytest_cache/ # docker files .docker diff --git a/.travis.yml b/.travis.yml index b671c0e8d..acdf365bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,15 +13,15 @@ before_script: matrix: include: - python: "2.7" - env: TOXENV=py27_install BOULDER_INTEGRATION=v1 + env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=all TOXENV=py27_install sudo: required services: docker - python: "2.7" - env: TOXENV=py27_install BOULDER_INTEGRATION=v2 + env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=all TOXENV=py27_install sudo: required services: docker - python: "2.7" - env: TOXENV=cover NUMPROCESSES=2 FYI="this also tests py27" + env: TOXENV=cover FYI="this also tests py27" - sudo: required env: TOXENV=nginx_compat services: docker @@ -90,12 +90,12 @@ addons: - nginx-light - openssl -install: "travis_retry $(command -v pip || command -v pip3) install tox coveralls" +install: "travis_retry $(command -v pip || command -v pip3) install codecov tox" script: - travis_retry tox - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)' -after_success: '[ "$TOXENV" == "cover" ] && coveralls' +after_success: '[ "$TOXENV" == "cover" ] && codecov' notifications: email: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d384ad30..5dd51ef16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,83 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.28.0 - master + +### Added + +* `revoke` accepts `--cert-name`, and doesn't accept both `--cert-name` and `--cert-path`. + +### Changed + +* `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges. + +### Fixed + +* Match Nginx parser update in allowing variable names to start with `${`. +* Correct OVH integration tests on machines without internet access. +* Stop caching the results of ipv6_info in http01.py + +## 0.27.1 - 2018-09-06 + +### Fixed + +* Fixed parameter name in OpenSUSE overrides for default parameters in the + Apache plugin. Certbot on OpenSUSE works again. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* certbot-apache + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/60?closed=1 + +## 0.27.0 - 2018-09-05 + +### Added + +* The Apache plugin now accepts the parameter --apache-ctl which can be + used to configure the path to the Apache control script. + +### Changed + +* When using `acme.client.ClientV2` (or + `acme.client.BackwardsCompatibleClientV2` with an ACME server that supports a + newer version of the ACME protocol), an `acme.errors.ConflictError` will be + raised if you try to create an ACME account with a key that has already been + used. Previously, a JSON parsing error was raised in this scenario when using + the library with Let's Encrypt's ACMEv2 endpoint. + +### Fixed + +* When Apache is not installed, Certbot's Apache plugin no longer prints + messages about being unable to find apachectl to the terminal when the plugin + is not selected. +* If you're using the Apache plugin with the --apache-vhost-root flag set to a + directory containing a disabled virtual host for the domain you're requesting + a certificate for, the virtual host will now be temporarily enabled if + necessary to pass the HTTP challenge. +* The documentation for the Certbot package can now be built using Sphinx 1.6+. +* You can now call `query_registration` without having to first call + `new_account` on `acme.client.ClientV2` objects. +* The requirement of `setuptools>=1.0` has been removed from `certbot-dns-ovh`. +* Names in certbot-dns-sakuracloud's tests have been updated to refer to Sakura + Cloud rather than NS1 whose plugin certbot-dns-sakuracloud was based on. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* acme +* certbot +* certbot-apache +* certbot-dns-ovh +* certbot-dns-sakuracloud + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/57?closed=1 + ## 0.26.1 - 2018-07-17 ### Fixed diff --git a/CHANGES.rst b/CHANGES.rst deleted file mode 100644 index 3b92c125f..000000000 --- a/CHANGES.rst +++ /dev/null @@ -1,8 +0,0 @@ -ChangeLog -========= - -To see the changes in a given release, view the issues closed in a given -release's GitHub milestone: - - - `Past releases `_ - - `Upcoming releases `_ diff --git a/Dockerfile b/Dockerfile index 28cd6b323..d1296b30f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ EXPOSE 80 443 VOLUME /etc/letsencrypt /var/lib/letsencrypt WORKDIR /opt/certbot -COPY CHANGES.rst README.rst setup.py src/ +COPY CHANGELOG.md README.rst setup.py src/ COPY acme src/acme COPY certbot src/certbot diff --git a/Dockerfile-old b/Dockerfile-old index 7bce82e0c..da48c7fc8 100644 --- a/Dockerfile-old +++ b/Dockerfile-old @@ -34,7 +34,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only # Dockerfile we make sure we cache as much as possible -COPY setup.py README.rst CHANGES.rst MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/ +COPY setup.py README.rst CHANGELOG.md MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/ # all above files are necessary for setup.py and venv setup, however, # package source code directory has to be copied separately to a diff --git a/MANIFEST.in b/MANIFEST.in index 434a156b7..7f529c7a7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include README.rst -include CHANGES.rst +include CHANGELOG.md include CONTRIBUTING.md include LICENSE.txt include linter_plugin.py diff --git a/README.rst b/README.rst index a18028aee..d75b44a65 100644 --- a/README.rst +++ b/README.rst @@ -6,7 +6,7 @@ Anyone who has gone through the trouble of setting up a secure website knows wha How you use Certbot depends on the configuration of your web server. The best way to get started is to use our `interactive guide `_. It generates instructions based on your configuration settings. In most cases, you’ll need `root or administrator access `_ to your web server to run Certbot. -If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issued by Let’s Encrypt. +Certbot is meant to be run directly on your web server, not on your personal computer. If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issued by Let’s Encrypt. Certbot is a fully-featured, extensible client for the Let's Encrypt CA (or any other CA that speaks the `ACME @@ -107,8 +107,8 @@ ACME working area in github: https://github.com/ietf-wg-acme/acme :target: https://travis-ci.org/certbot/certbot :alt: Travis CI status -.. |coverage| image:: https://coveralls.io/repos/certbot/certbot/badge.svg?branch=master - :target: https://coveralls.io/r/certbot/certbot +.. |coverage| image:: https://codecov.io/gh/certbot/certbot/branch/master/graph/badge.svg + :target: https://codecov.io/gh/certbot/certbot :alt: Coverage status .. |docs| image:: https://readthedocs.org/projects/letsencrypt/badge/ diff --git a/acme/setup.py b/acme/setup.py index 88592013c..85492d9a3 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000..67ad67c16 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,12 @@ +# AppVeyor CI pipeline, executed on Windows Server 2016/2012 R2 +branches: + only: + - master + - /^\d+\.\d+\.x$/ # version branches like X.X.X + - /^test-.*$/ + +build: off + +test_script: + - ps: Write-Host "Hello, world!" + \ No newline at end of file diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index bb77e2e41..da632dc81 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -1,5 +1,6 @@ """Apache Configuration based off of Augeas Configurator.""" # pylint: disable=too-many-lines +import copy import fnmatch import logging import os @@ -97,48 +98,72 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): vhost_root="/etc/apache2/sites-available", vhost_files="*", logs_root="/var/log/apache2", + ctl="apache2ctl", version_cmd=['apache2ctl', '-v'], - apache_cmd="apache2ctl", restart_cmd=['apache2ctl', 'graceful'], conftest_cmd=['apache2ctl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/apache2", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( "certbot_apache", "options-ssl-apache.conf") ) - def constant(self, key): - """Get constant for OS_DEFAULTS""" - return self.OS_DEFAULTS.get(key) + def option(self, key): + """Get a value from options""" + return self.options.get(key) + + def _prepare_options(self): + """ + Set the values possibly changed by command line parameters to + OS_DEFAULTS constant dictionary + """ + opts = ["enmod", "dismod", "le_vhost_ext", "server_root", "vhost_root", + "logs_root", "challenge_location", "handle_modules", "handle_sites", + "ctl"] + for o in opts: + # Config options use dashes instead of underscores + if self.conf(o.replace("_", "-")) is not None: + self.options[o] = self.conf(o.replace("_", "-")) + else: + self.options[o] = self.OS_DEFAULTS[o] + + # Special cases + self.options["version_cmd"][0] = self.option("ctl") + self.options["restart_cmd"][0] = self.option("ctl") + self.options["conftest_cmd"][0] = self.option("ctl") @classmethod def add_parser_arguments(cls, add): + # When adding, modifying or deleting command line arguments, be sure to + # include the changes in the list used in method _prepare_options() to + # ensure consistent behavior. add("enmod", default=cls.OS_DEFAULTS["enmod"], - help="Path to the Apache 'a2enmod' binary.") + help="Path to the Apache 'a2enmod' binary") add("dismod", default=cls.OS_DEFAULTS["dismod"], - help="Path to the Apache 'a2dismod' binary.") + help="Path to the Apache 'a2dismod' binary") add("le-vhost-ext", default=cls.OS_DEFAULTS["le_vhost_ext"], - help="SSL vhost configuration extension.") + help="SSL vhost configuration extension") add("server-root", default=cls.OS_DEFAULTS["server_root"], - help="Apache server root directory.") + help="Apache server root directory") add("vhost-root", default=None, help="Apache server VirtualHost configuration root") add("logs-root", default=cls.OS_DEFAULTS["logs_root"], help="Apache server logs directory") add("challenge-location", default=cls.OS_DEFAULTS["challenge_location"], - help="Directory path for challenge configuration.") - add("handle-modules", default=cls.OS_DEFAULTS["handle_mods"], - help="Let installer handle enabling required modules for you. " + + help="Directory path for challenge configuration") + add("handle-modules", default=cls.OS_DEFAULTS["handle_modules"], + help="Let installer handle enabling required modules for you " + "(Only Ubuntu/Debian currently)") add("handle-sites", default=cls.OS_DEFAULTS["handle_sites"], - help="Let installer handle enabling sites for you. " + + help="Let installer handle enabling sites for you " + "(Only Ubuntu/Debian currently)") - util.add_deprecated_argument(add, argument_name="ctl", nargs=1) + add("ctl", default=cls.OS_DEFAULTS["ctl"], + help="Full path to Apache control script") util.add_deprecated_argument( add, argument_name="init-script", nargs=1) @@ -169,7 +194,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.parser = None self.version = version self.vhosts = None - self.vhostroot = None + self.options = copy.deepcopy(self.OS_DEFAULTS) self._enhance_func = {"redirect": self._enable_redirect, "ensure-http-header": self._set_http_header, "staple-ocsp": self._enable_ocsp_stapling} @@ -201,12 +226,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): except ImportError: raise errors.NoInstallationError("Problem in Augeas installation") + self._prepare_options() + # Verify Apache is installed - restart_cmd = self.constant("restart_cmd")[0] - if not util.exe_exists(restart_cmd): - if not path_surgery(restart_cmd): - raise errors.NoInstallationError( - 'Cannot find Apache control command {0}'.format(restart_cmd)) + self._verify_exe_availability(self.option("ctl")) # Make sure configuration is valid self.config_test() @@ -226,12 +249,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "version 1.2.0 or higher, please make sure you have you have " "those installed.") - # Parse vhost-root if defined on cli - if not self.conf("vhost-root"): - self.vhostroot = self.constant("vhost_root") - else: - self.vhostroot = os.path.abspath(self.conf("vhost-root")) - self.parser = self.get_parser() # Check for errors in parsing files with Augeas @@ -245,13 +262,20 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Prevent two Apache plugins from modifying a config at once try: - util.lock_dir_until_exit(self.conf("server-root")) + util.lock_dir_until_exit(self.option("server_root")) except (OSError, errors.LockError): logger.debug("Encountered error:", exc_info=True) raise errors.PluginError( - "Unable to lock %s", self.conf("server-root")) + "Unable to lock %s", self.option("server_root")) self._prepared = True + def _verify_exe_availability(self, exe): + """Checks availability of Apache executable""" + if not util.exe_exists(exe): + if not path_surgery(exe): + raise errors.NoInstallationError( + 'Cannot find Apache executable {0}'.format(exe)) + def _check_aug_version(self): """ Checks that we have recent enough version of libaugeas. If augeas version is recent enough, it will support case insensitive @@ -269,8 +293,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def get_parser(self): """Initializes the ApacheParser""" + # If user provided vhost_root value in command line, use it return parser.ApacheParser( - self.aug, self.conf("server-root"), self.conf("vhost-root"), + self.aug, self.option("server_root"), self.conf("vhost-root"), self.version, configurator=self) def _wildcard_domain(self, domain): @@ -1037,7 +1062,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param boolean temp: If the change is temporary """ - if self.conf("handle-modules"): + if self.option("handle_modules"): if self.version >= (2, 4) and ("socache_shmcb_module" not in self.parser.modules): self.enable_mod("socache_shmcb", temp=temp) @@ -1066,7 +1091,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): Duplicates vhost and adds default ssl options New vhost will reside as (nonssl_vhost.path) + - ``self.constant("le_vhost_ext")`` + ``self.option("le_vhost_ext")`` .. note:: This function saves the configuration @@ -1165,18 +1190,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ if self.conf("vhost-root") and os.path.exists(self.conf("vhost-root")): - # Defined by user on CLI - - fp = os.path.join(os.path.realpath(self.vhostroot), + fp = os.path.join(os.path.realpath(self.option("vhost_root")), os.path.basename(non_ssl_vh_fp)) else: # Use non-ssl filepath fp = os.path.realpath(non_ssl_vh_fp) if fp.endswith(".conf"): - return fp[:-(len(".conf"))] + self.conf("le_vhost_ext") + return fp[:-(len(".conf"))] + self.option("le_vhost_ext") else: - return fp + self.conf("le_vhost_ext") + return fp + self.option("le_vhost_ext") def _sift_rewrite_rule(self, line): """Decides whether a line should be copied to a SSL vhost. @@ -2025,7 +2048,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): addr in self._get_proposed_addrs(ssl_vhost)), servername, serveralias, " ".join(rewrite_rule_args), - self.conf("logs-root"))) + self.option("logs_root"))) def _write_out_redirect(self, ssl_vhost, text): # This is the default name @@ -2037,7 +2060,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if len(ssl_vhost.name) < (255 - (len(redirect_filename) + 1)): redirect_filename = "le-redirect-%s.conf" % ssl_vhost.name - redirect_filepath = os.path.join(self.vhostroot, + redirect_filepath = os.path.join(self.option("vhost_root"), redirect_filename) # Register the new file that will be created @@ -2158,18 +2181,18 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ error = "" try: - util.run_script(self.constant("restart_cmd")) + util.run_script(self.option("restart_cmd")) except errors.SubprocessError as err: logger.info("Unable to restart apache using %s", - self.constant("restart_cmd")) - alt_restart = self.constant("restart_cmd_alt") + self.option("restart_cmd")) + alt_restart = self.option("restart_cmd_alt") if alt_restart: logger.debug("Trying alternative restart command: %s", alt_restart) # There is an alternative restart command available # This usually is "restart" verb while original is "graceful" try: - util.run_script(self.constant( + util.run_script(self.option( "restart_cmd_alt")) return except errors.SubprocessError as secerr: @@ -2185,7 +2208,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - util.run_script(self.constant("conftest_cmd")) + util.run_script(self.option("conftest_cmd")) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -2201,11 +2224,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - stdout, _ = util.run_script(self.constant("version_cmd")) + stdout, _ = util.run_script(self.option("version_cmd")) except errors.SubprocessError: raise errors.PluginError( "Unable to run %s -v" % - self.constant("version_cmd")) + self.option("version_cmd")) regex = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE) matches = regex.findall(stdout) @@ -2295,7 +2318,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # certbot for unprivileged users via setuid), this function will need # to be modified. return common.install_version_controlled_file(options_ssl, options_ssl_digest, - self.constant("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES) + self.option("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES) def enable_autohsts(self, _unused_lineage, domains): """ diff --git a/certbot-apache/certbot_apache/display_ops.py b/certbot-apache/certbot_apache/display_ops.py index 097b84b96..db1c3cca4 100644 --- a/certbot-apache/certbot_apache/display_ops.py +++ b/certbot-apache/certbot_apache/display_ops.py @@ -113,8 +113,7 @@ def _vhost_menu(domain, vhosts): code, tag = zope.component.getUtility(interfaces.IDisplay).menu( "We were unable to find a vhost with a ServerName " "or Address of {0}.{1}Which virtual host would you " - "like to choose?\n(note: conf files with multiple " - "vhosts are not yet supported)".format(domain, os.linesep), + "like to choose?".format(domain, os.linesep), choices, force_interactive=True) except errors.MissingCommandlineFlag: msg = ( diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index 37545e9cc..22598baca 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -6,6 +6,7 @@ from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-m from certbot import errors from certbot.plugins import common from certbot_apache.obj import VirtualHost # pylint: disable=unused-import +from certbot_apache.parser import get_aug_path logger = logging.getLogger(__name__) @@ -172,4 +173,9 @@ class ApacheHttp01(common.TLSSNI01): self.configurator.parser.add_dir( vhost.path, "Include", self.challenge_conf_post) + if not vhost.enabled: + self.configurator.parser.add_dir( + get_aug_path(self.configurator.parser.loc["default"]), + "Include", vhost.filep) + self.moded_vhosts.add(vhost) diff --git a/certbot-apache/certbot_apache/override_arch.py b/certbot-apache/certbot_apache/override_arch.py index ea5155a3c..c5620e9f9 100644 --- a/certbot-apache/certbot_apache/override_arch.py +++ b/certbot-apache/certbot_apache/override_arch.py @@ -16,14 +16,14 @@ class ArchConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/httpd/conf", vhost_files="*.conf", logs_root="/var/log/httpd", + ctl="apachectl", version_cmd=['apachectl', '-v'], - apache_cmd="apachectl", restart_cmd=['apachectl', 'graceful'], conftest_cmd=['apachectl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/httpd/conf", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( diff --git a/certbot-apache/certbot_apache/override_centos.py b/certbot-apache/certbot_apache/override_centos.py index 0b6b12b96..a4f1b84ec 100644 --- a/certbot-apache/certbot_apache/override_centos.py +++ b/certbot-apache/certbot_apache/override_centos.py @@ -18,25 +18,33 @@ class CentOSConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/httpd/conf.d", vhost_files="*.conf", logs_root="/var/log/httpd", + ctl="apachectl", version_cmd=['apachectl', '-v'], - apache_cmd="apachectl", restart_cmd=['apachectl', 'graceful'], restart_cmd_alt=['apachectl', 'restart'], conftest_cmd=['apachectl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/httpd/conf.d", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( "certbot_apache", "centos-options-ssl-apache.conf") ) + def _prepare_options(self): + """ + Override the options dictionary initialization in order to support + alternative restart cmd used in CentOS. + """ + super(CentOSConfigurator, self)._prepare_options() + self.options["restart_cmd_alt"][0] = self.option("ctl") + def get_parser(self): """Initializes the ApacheParser""" return CentOSParser( - self.aug, self.conf("server-root"), self.conf("vhost-root"), + self.aug, self.option("server_root"), self.option("vhost_root"), self.version, configurator=self) diff --git a/certbot-apache/certbot_apache/override_darwin.py b/certbot-apache/certbot_apache/override_darwin.py index 53741d504..4e2a6acac 100644 --- a/certbot-apache/certbot_apache/override_darwin.py +++ b/certbot-apache/certbot_apache/override_darwin.py @@ -16,14 +16,14 @@ class DarwinConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/apache2/other", vhost_files="*.conf", logs_root="/var/log/apache2", - version_cmd=['/usr/sbin/httpd', '-v'], - apache_cmd="/usr/sbin/httpd", + ctl="apachectl", + version_cmd=['apachectl', '-v'], restart_cmd=['apachectl', 'graceful'], conftest_cmd=['apachectl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/other", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( diff --git a/certbot-apache/certbot_apache/override_debian.py b/certbot-apache/certbot_apache/override_debian.py index 02dffc3f7..0caa619d2 100644 --- a/certbot-apache/certbot_apache/override_debian.py +++ b/certbot-apache/certbot_apache/override_debian.py @@ -23,14 +23,14 @@ class DebianConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/apache2/sites-available", vhost_files="*", logs_root="/var/log/apache2", + ctl="apache2ctl", version_cmd=['apache2ctl', '-v'], - apache_cmd="apache2ctl", restart_cmd=['apache2ctl', 'graceful'], conftest_cmd=['apache2ctl', 'configtest'], enmod="a2enmod", dismod="a2dismod", le_vhost_ext="-le-ssl.conf", - handle_mods=True, + handle_modules=True, handle_sites=True, challenge_location="/etc/apache2", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( @@ -134,11 +134,11 @@ class DebianConfigurator(configurator.ApacheConfigurator): # Generate reversal command. # Try to be safe here... check that we can probably reverse before # applying enmod command - if not util.exe_exists(self.conf("dismod")): + if not util.exe_exists(self.option("dismod")): raise errors.MisconfigurationError( "Unable to find a2dismod, please make sure a2enmod and " "a2dismod are configured correctly for certbot.") self.reverter.register_undo_command( - temp, [self.conf("dismod"), "-f", mod_name]) - util.run_script([self.conf("enmod"), mod_name]) + temp, [self.option("dismod"), "-f", mod_name]) + util.run_script([self.option("enmod"), mod_name]) diff --git a/certbot-apache/certbot_apache/override_gentoo.py b/certbot-apache/certbot_apache/override_gentoo.py index 165e44c96..556e3225e 100644 --- a/certbot-apache/certbot_apache/override_gentoo.py +++ b/certbot-apache/certbot_apache/override_gentoo.py @@ -18,25 +18,33 @@ class GentooConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/apache2/vhosts.d", vhost_files="*.conf", logs_root="/var/log/apache2", - version_cmd=['/usr/sbin/apache2', '-v'], - apache_cmd="apache2ctl", + ctl="apache2ctl", + version_cmd=['apache2ctl', '-v'], restart_cmd=['apache2ctl', 'graceful'], restart_cmd_alt=['apache2ctl', 'restart'], conftest_cmd=['apache2ctl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/vhosts.d", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( "certbot_apache", "options-ssl-apache.conf") ) + def _prepare_options(self): + """ + Override the options dictionary initialization in order to support + alternative restart cmd used in Gentoo. + """ + super(GentooConfigurator, self)._prepare_options() + self.options["restart_cmd_alt"][0] = self.option("ctl") + def get_parser(self): """Initializes the ApacheParser""" return GentooParser( - self.aug, self.conf("server-root"), self.conf("vhost-root"), + self.aug, self.option("server_root"), self.option("vhost_root"), self.version, configurator=self) @@ -61,7 +69,7 @@ class GentooParser(parser.ApacheParser): def update_modules(self): """Get loaded modules from httpd process, and add them to DOM""" - mod_cmd = [self.configurator.constant("apache_cmd"), "modules"] + mod_cmd = [self.configurator.option("ctl"), "modules"] matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module") for mod in matches: self.add_mod(mod.strip()) diff --git a/certbot-apache/certbot_apache/override_suse.py b/certbot-apache/certbot_apache/override_suse.py index a67054b5b..3d0043afe 100644 --- a/certbot-apache/certbot_apache/override_suse.py +++ b/certbot-apache/certbot_apache/override_suse.py @@ -16,14 +16,14 @@ class OpenSUSEConfigurator(configurator.ApacheConfigurator): vhost_root="/etc/apache2/vhosts.d", vhost_files="*.conf", logs_root="/var/log/apache2", + ctl="apache2ctl", version_cmd=['apache2ctl', '-v'], - apache_cmd="apache2ctl", restart_cmd=['apache2ctl', 'graceful'], conftest_cmd=['apache2ctl', 'configtest'], enmod="a2enmod", dismod="a2dismod", le_vhost_ext="-le-ssl.conf", - handle_mods=False, + handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/vhosts.d", MOD_SSL_CONF_SRC=pkg_resources.resource_filename( diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py index 02337f8d4..148f052d0 100644 --- a/certbot-apache/certbot_apache/parser.py +++ b/certbot-apache/certbot_apache/parser.py @@ -69,7 +69,7 @@ class ApacheParser(object): # Must also attempt to parse additional virtual host root if vhostroot: self.parse_file(os.path.abspath(vhostroot) + "/" + - self.configurator.constant("vhost_files")) + self.configurator.option("vhost_files")) # check to see if there were unparsed define statements if version < (2, 4): @@ -152,7 +152,7 @@ class ApacheParser(object): """Get Defines from httpd process""" variables = dict() - define_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D", + define_cmd = [self.configurator.option("ctl"), "-t", "-D", "DUMP_RUN_CFG"] matches = self.parse_from_subprocess(define_cmd, r"Define: ([^ \n]*)") try: @@ -179,7 +179,7 @@ class ApacheParser(object): # configuration files _ = self.find_dir("Include") - inc_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D", + inc_cmd = [self.configurator.option("ctl"), "-t", "-D", "DUMP_INCLUDES"] matches = self.parse_from_subprocess(inc_cmd, r"\(.*\) (.*)") if matches: @@ -190,7 +190,7 @@ class ApacheParser(object): def update_modules(self): """Get loaded modules from httpd process, and add them to DOM""" - mod_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D", + mod_cmd = [self.configurator.option("ctl"), "-t", "-D", "DUMP_MODULES"] matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module") for mod in matches: diff --git a/certbot-apache/certbot_apache/tests/autohsts_test.py b/certbot-apache/certbot_apache/tests/autohsts_test.py index 73da33f15..c5d720dd3 100644 --- a/certbot-apache/certbot_apache/tests/autohsts_test.py +++ b/certbot-apache/certbot_apache/tests/autohsts_test.py @@ -119,6 +119,9 @@ class AutoHSTSTest(util.ApacheTest): cur_val = maxage.format(constants.AUTOHSTS_STEPS[i+1]) self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), cur_val) + # Ensure that the value is raised to max + self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), + maxage.format(constants.AUTOHSTS_STEPS[-1])) # Make permanent self.config.deploy_autohsts(mock_lineage) self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path), diff --git a/certbot-apache/certbot_apache/tests/centos_test.py b/certbot-apache/certbot_apache/tests/centos_test.py index 4ee8b5dcf..46b857c3e 100644 --- a/certbot-apache/certbot_apache/tests/centos_test.py +++ b/certbot-apache/certbot_apache/tests/centos_test.py @@ -135,5 +135,7 @@ class MultipleVhostsTestCentOS(util.ApacheTest): errors.SubprocessError, errors.SubprocessError] self.assertRaises(errors.MisconfigurationError, self.config.restart) + + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 350262634..0fb89c95a 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -115,9 +115,22 @@ class MultipleVhostsTest(util.ApacheTest): # Weak test.. ApacheConfigurator.add_parser_arguments(mock.MagicMock()) + def test_add_parser_arguments_all_configurators(self): # pylint: disable=no-self-use + from certbot_apache.entrypoint import OVERRIDE_CLASSES + for cls in OVERRIDE_CLASSES.values(): + cls.add_parser_arguments(mock.MagicMock()) + + def test_all_configurators_defaults_defined(self): + from certbot_apache.entrypoint import OVERRIDE_CLASSES + from certbot_apache.configurator import ApacheConfigurator + parameters = set(ApacheConfigurator.OS_DEFAULTS.keys()) + for cls in OVERRIDE_CLASSES.values(): + self.assertTrue(parameters.issubset(set(cls.OS_DEFAULTS.keys()))) + def test_constant(self): - self.assertEqual(self.config.constant("server_root"), "/etc/apache2") - self.assertEqual(self.config.constant("nonexistent"), None) + self.assertTrue("debian_apache_2_4/multiple_vhosts/apache" in + self.config.option("server_root")) + self.assertEqual(self.config.option("nonexistent"), None) @certbot_util.patch_get_utility() def test_get_all_names(self, mock_getutility): @@ -651,22 +664,10 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(ssl_vhost_slink.name, "nonsym.link") def test_make_vhost_ssl_nonexistent_vhost_path(self): - def conf_side_effect(arg): - """ Mock function for ApacheConfigurator.conf """ - confvars = { - "vhost-root": "/tmp/nonexistent", - "le_vhost_ext": "-le-ssl.conf", - "handle-sites": True} - return confvars[arg] - - with mock.patch( - "certbot_apache.configurator.ApacheConfigurator.conf" - ) as mock_conf: - mock_conf.side_effect = conf_side_effect - ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[1]) - self.assertEqual(os.path.dirname(ssl_vhost.filep), - os.path.dirname(os.path.realpath( - self.vh_truth[1].filep))) + ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[1]) + self.assertEqual(os.path.dirname(ssl_vhost.filep), + os.path.dirname(os.path.realpath( + self.vh_truth[1].filep))) def test_make_vhost_ssl(self): ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) @@ -1583,7 +1584,7 @@ class AugeasVhostsTest(util.ApacheTest): broken_vhost) class MultiVhostsTest(util.ApacheTest): - """Test vhosts with illegal names dependent on augeas version.""" + """Test configuration with multiple virtualhosts in a single file.""" # pylint: disable=protected-access def setUp(self): # pylint: disable=arguments-differ @@ -1703,7 +1704,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): self.config.updated_mod_ssl_conf_digest) def _current_ssl_options_hash(self): - return crypto_util.sha256sum(self.config.constant("MOD_SSL_CONF_SRC")) + return crypto_util.sha256sum(self.config.option("MOD_SSL_CONF_SRC")) def _assert_current_file(self): self.assertTrue(os.path.isfile(self.config.mod_ssl_conf)) @@ -1739,7 +1740,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): self.assertFalse(mock_logger.warning.called) self.assertTrue(os.path.isfile(self.config.mod_ssl_conf)) self.assertEqual(crypto_util.sha256sum( - self.config.constant("MOD_SSL_CONF_SRC")), + self.config.option("MOD_SSL_CONF_SRC")), self._current_ssl_options_hash()) self.assertNotEqual(crypto_util.sha256sum(self.config.mod_ssl_conf), self._current_ssl_options_hash()) @@ -1755,7 +1756,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): "%s has been manually modified; updated file " "saved to %s. We recommend updating %s for security purposes.") self.assertEqual(crypto_util.sha256sum( - self.config.constant("MOD_SSL_CONF_SRC")), + self.config.option("MOD_SSL_CONF_SRC")), self._current_ssl_options_hash()) # only print warning once with mock.patch("certbot.plugins.common.logger") as mock_logger: diff --git a/certbot-apache/certbot_apache/tests/debian_test.py b/certbot-apache/certbot_apache/tests/debian_test.py index fde8d4c35..bb1d64278 100644 --- a/certbot-apache/certbot_apache/tests/debian_test.py +++ b/certbot-apache/certbot_apache/tests/debian_test.py @@ -20,7 +20,7 @@ class MultipleVhostsTestDebian(util.ApacheTest): def setUp(self): # pylint: disable=arguments-differ super(MultipleVhostsTestDebian, self).setUp() self.config = util.get_apache_configurator( - self.config_path, None, self.config_dir, self.work_dir, + self.config_path, self.vhost_path, self.config_dir, self.work_dir, os_info="debian") self.config = self.mock_deploy_cert(self.config) self.vh_truth = util.get_vh_truth(self.temp_dir, diff --git a/certbot-apache/certbot_apache/tests/gentoo_test.py b/certbot-apache/certbot_apache/tests/gentoo_test.py index d32551267..0681e30b5 100644 --- a/certbot-apache/certbot_apache/tests/gentoo_test.py +++ b/certbot-apache/certbot_apache/tests/gentoo_test.py @@ -117,7 +117,7 @@ class MultipleVhostsTestGentoo(util.ApacheTest): self.config.parser.modules = set() with mock.patch("certbot.util.get_os_info") as mock_osi: - # Make sure we have the have the CentOS httpd constants + # Make sure we have the have the Gentoo httpd constants mock_osi.return_value = ("gentoo", "123") self.config.parser.update_runtime_variables() diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 98bf412ae..9c729b08c 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -10,6 +10,7 @@ from certbot import achallenges from certbot import errors from certbot.tests import acme_util +from certbot_apache.parser import get_aug_path from certbot_apache.tests import util @@ -134,6 +135,21 @@ class ApacheHttp01Test(util.ApacheTest): def test_perform_3_achall_apache_2_4(self): self.combinations_perform_test(num_achalls=3, minor_version=4) + def test_activate_disabled_vhost(self): + vhosts = [v for v in self.config.vhosts if v.name == "certbot.demo"] + achalls = [ + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=((b'a' * 16))), + "pending"), + domain="certbot.demo", account_key=self.account_key)] + vhosts[0].enabled = False + self.common_perform_test(achalls, vhosts) + matches = self.config.parser.find_dir( + "Include", vhosts[0].filep, + get_aug_path(self.config.parser.loc["default"])) + self.assertEqual(len(matches), 1) + def combinations_perform_test(self, num_achalls, minor_version): """Test perform with the given achall count and Apache version.""" achalls = self.achalls[:num_achalls] diff --git a/certbot-apache/certbot_apache/tests/parser_test.py b/certbot-apache/certbot_apache/tests/parser_test.py index f95f1b346..d62fd54e8 100644 --- a/certbot-apache/certbot_apache/tests/parser_test.py +++ b/certbot-apache/certbot_apache/tests/parser_test.py @@ -282,11 +282,11 @@ class BasicParserTest(util.ParserTest): self.assertRaises( errors.PluginError, self.parser.update_runtime_variables) - @mock.patch("certbot_apache.configurator.ApacheConfigurator.constant") + @mock.patch("certbot_apache.configurator.ApacheConfigurator.option") @mock.patch("certbot_apache.parser.subprocess.Popen") - def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_const): + def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_opt): mock_popen.side_effect = OSError - mock_const.return_value = "nonexistent" + mock_opt.return_value = "nonexistent" self.assertRaises( errors.MisconfigurationError, self.parser.update_runtime_variables) diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 6d3cfa109..9329ccb20 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -97,9 +97,10 @@ def get_apache_configurator( # pylint: disable=too-many-arguments, too-many-loc backups = os.path.join(work_dir, "backups") mock_le_config = mock.MagicMock( apache_server_root=config_path, - apache_vhost_root=conf_vhost_path, + apache_vhost_root=None, apache_le_vhost_ext="-le-ssl.conf", apache_challenge_location=config_path, + apache_enmod=None, backup_dir=backups, config_dir=config_dir, http01_port=80, @@ -107,33 +108,25 @@ def get_apache_configurator( # pylint: disable=too-many-arguments, too-many-loc in_progress_dir=os.path.join(backups, "IN_PROGRESS"), work_dir=work_dir) - orig_os_constant = configurator.ApacheConfigurator(mock_le_config, - name="apache", - version=version).constant - - def mock_os_constant(key, vhost_path=vhost_path): - """Mock default vhost path""" - if key == "vhost_root": - return vhost_path - else: - return orig_os_constant(key) - - with mock.patch("certbot_apache.configurator.ApacheConfigurator.constant") as mock_cons: - mock_cons.side_effect = mock_os_constant - with mock.patch("certbot_apache.configurator.util.run_script"): - with mock.patch("certbot_apache.configurator.util." - "exe_exists") as mock_exe_exists: - mock_exe_exists.return_value = True - with mock.patch("certbot_apache.parser.ApacheParser." - "update_runtime_variables"): - try: - config_class = entrypoint.OVERRIDE_CLASSES[os_info] - except KeyError: - config_class = configurator.ApacheConfigurator - config = config_class(config=mock_le_config, name="apache", - version=version) - - config.prepare() + with mock.patch("certbot_apache.configurator.util.run_script"): + with mock.patch("certbot_apache.configurator.util." + "exe_exists") as mock_exe_exists: + mock_exe_exists.return_value = True + with mock.patch("certbot_apache.parser.ApacheParser." + "update_runtime_variables"): + try: + config_class = entrypoint.OVERRIDE_CLASSES[os_info] + except KeyError: + config_class = configurator.ApacheConfigurator + config = config_class(config=mock_le_config, name="apache", + version=version) + if not conf_vhost_path: + config_class.OS_DEFAULTS["vhost_root"] = vhost_path + else: + # Custom virtualhost path was requested + config.config.apache_vhost_root = conf_vhost_path + config.config.apache_ctl = config_class.OS_DEFAULTS["ctl"] + config.prepare() return config diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index f435bb1a9..aa5908f9e 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index e097719db..076c45e39 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.1" +LE_AUTO_VERSION="0.27.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.26.1 \ - --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ - --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 -acme==0.26.1 \ - --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ - --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be -certbot-apache==0.26.1 \ - --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ - --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 -certbot-nginx==0.26.1 \ - --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ - --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 +certbot==0.27.1 \ + --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ + --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a +acme==0.27.1 \ + --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ + --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 +certbot-apache==0.27.1 \ + --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ + --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 +certbot-nginx==0.27.1 \ + --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ + --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/Dockerfile b/certbot-compatibility-test/Dockerfile index fe55a68a6..803b4a1b9 100644 --- a/certbot-compatibility-test/Dockerfile +++ b/certbot-compatibility-test/Dockerfile @@ -14,7 +14,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only # the above is not likely to change, so by putting it further up the # Dockerfile we make sure we cache as much as possible -COPY setup.py README.rst CHANGES.rst MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/ +COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/ # all above files are necessary for setup.py, however, package source # code directory has to be copied separately to a subdirectory... diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py index 1d2cfdeca..82195264b 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py @@ -59,9 +59,6 @@ class Proxy(configurators_common.Proxy): setattr(self.le_config, "apache_" + k, entrypoint.ENTRYPOINT.OS_DEFAULTS[k]) - # An alias - self.le_config.apache_handle_modules = self.le_config.apache_handle_mods - self._configurator = entrypoint.ENTRYPOINT( config=configuration.NamespaceConfig(self.le_config), name="apache") diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index e4ccc719a..8dac3e047 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 05649d4d0..b823cf98f 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 911f9e052..3daf933cb 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 9dd318296..feb2a0d58 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 09b11def0..b6efcf360 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 2ca3213bf..c268eaa8f 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index e9ead6546..fc147f85c 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 3c7402f25..86d36bcb3 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/certbot_dns_linode/__init__.py b/certbot-dns-linode/certbot_dns_linode/__init__.py index 0c445f45d..0a6ccec61 100644 --- a/certbot-dns-linode/certbot_dns_linode/__init__.py +++ b/certbot-dns-linode/certbot_dns_linode/__init__.py @@ -14,7 +14,11 @@ Named Arguments DNS to propagate before asking the ACME server to verify the DNS record. - (Default: 960) + (Default: 1200 because Linode + updates its first DNS every 15 + minutes and we allow 5 more minutes + for the update to reach the other 5 + servers) ========================================== =================================== @@ -74,13 +78,15 @@ Examples -d www.example.com .. code-block:: bash - :caption: To acquire a certificate for ``example.com``, waiting 60 seconds - for DNS propagation + :caption: To acquire a certificate for ``example.com``, waiting 1000 seconds + for DNS propagation (Linode updates its first DNS every 15 minutes + and we allow some extra time for the update to reach the other 5 + servers) certbot certonly \\ --dns-linode \\ --dns-linode-credentials ~/.secrets/certbot/linode.ini \\ - --dns-linode-propagation-seconds 60 \\ + --dns-linode-propagation-seconds 1000 \\ -d example.com """ diff --git a/certbot-dns-linode/certbot_dns_linode/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/dns_linode.py index 323c0810a..cc29ce842 100644 --- a/certbot-dns-linode/certbot_dns_linode/dns_linode.py +++ b/certbot-dns-linode/certbot_dns_linode/dns_linode.py @@ -29,7 +29,7 @@ class Authenticator(dns_common.DNSAuthenticator): @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=960) + super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=1200) add('credentials', help='Linode credentials INI file.') def more_info(self): # pylint: disable=missing-docstring,no-self-use diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 327224c9c..1d196403f 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 1f92f7dce..a5c06d90e 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 0b4241afb..474093a5b 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index e0ce785a1..1f3acbf62 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,14 +4,14 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'acme>=0.21.1', 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'dns-lexicon>=2.7.3', # Correct OVH integration tests 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index bd54ec4c5..c009ef032 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 5f0b26f6e..2bae0c3d0 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index b7cfc15b5..9f8bfbbdb 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index d31a54c17..d526381a2 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -136,7 +136,9 @@ class NginxConfigurator(common.Installer): """ # Verify Nginx is installed if not util.exe_exists(self.conf('ctl')): - raise errors.NoInstallationError + raise errors.NoInstallationError( + "Could not find a usable 'nginx' binary. Ensure nginx exists, " + "the binary is executable, and your PATH is set correctly.") # Make sure configuration is valid self.config_test() diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index 677ce0737..e46d7b9b9 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -40,8 +40,6 @@ class NginxHttp01(common.ChallengePerformer): super(NginxHttp01, self).__init__(configurator) self.challenge_conf = os.path.join( configurator.config.config_dir, "le_http_01_cert_challenge.conf") - self._ipv6 = None - self._ipv6only = None def perform(self): """Perform a challenge on Nginx. @@ -102,6 +100,7 @@ class NginxHttp01(common.ChallengePerformer): config = [self._make_or_mod_server_block(achall) for achall in self.achalls] config = [x for x in config if x is not None] config = nginxparser.UnspacedList(config) + logger.debug("Generated server block:\n%s", str(config)) self.configurator.reverter.register_file_creation( True, self.challenge_conf) @@ -120,9 +119,7 @@ class NginxHttp01(common.ChallengePerformer): self.configurator.config.http01_port) port = self.configurator.config.http01_port - if self._ipv6 is None or self._ipv6only is None: - self._ipv6, self._ipv6only = self.configurator.ipv6_info(port) - ipv6, ipv6only = self._ipv6, self._ipv6only + ipv6, ipv6only = self.configurator.ipv6_info(port) if ipv6: # If IPv6 is active in Nginx configuration diff --git a/certbot-nginx/certbot_nginx/nginxparser.py b/certbot-nginx/certbot_nginx/nginxparser.py index 8818bc040..bfb75adcc 100644 --- a/certbot-nginx/certbot_nginx/nginxparser.py +++ b/certbot-nginx/certbot_nginx/nginxparser.py @@ -26,7 +26,7 @@ class RawNginxParser(object): dquoted = QuotedString('"', multiline=True, unquoteResults=False, escChar='\\') squoted = QuotedString("'", multiline=True, unquoteResults=False, escChar='\\') quoted = dquoted | squoted - head_tokenchars = Regex(r"[^{};\s'\"]") # if (last_space) + head_tokenchars = Regex(r"(\$\{)|[^{};\s'\"]") # if (last_space) tail_tokenchars = Regex(r"(\$\{)|[^{;\s]") # else tokenchars = Combine(head_tokenchars + ZeroOrMore(tail_tokenchars)) paren_quote_extend = Combine(quoted + Literal(')') + ZeroOrMore(tail_tokenchars)) diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 7d1da2e73..a5cf2892e 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -222,7 +222,7 @@ class NginxParser(object): return os.path.join(self.root, name) raise errors.NoInstallationError( - "Could not find configuration root") + "Could not find Nginx root configuration file (nginx.conf)") def filedump(self, ext='tmp', lazy=True): """Dumps parsed configurations into files. @@ -395,12 +395,17 @@ class NginxParser(object): addr.ipv6only = False for directive in enclosing_block[new_vhost.path[-1]][1]: if len(directive) > 0 and directive[0] == 'listen': - if 'default_server' in directive: - del directive[directive.index('default_server')] - if 'default' in directive: - del directive[directive.index('default')] - if 'ipv6only=on' in directive: - del directive[directive.index('ipv6only=on')] + # Exclude one-time use parameters which will cause an error if repeated. + # https://nginx.org/en/docs/http/ngx_http_core_module.html#listen + exclude = set(('default_server', 'default', 'setfib', 'fastopen', 'backlog', + 'rcvbuf', 'sndbuf', 'accept_filter', 'deferred', 'bind', + 'ipv6only', 'reuseport', 'so_keepalive')) + + for param in exclude: + # See: github.com/certbot/certbot/pull/6223#pullrequestreview-143019225 + keys = [x.split('=')[0] for x in directive] + if param in keys: + del directive[keys.index(param)] return new_vhost diff --git a/certbot-nginx/certbot_nginx/tests/http_01_test.py b/certbot-nginx/certbot_nginx/tests/http_01_test.py index 0f764e92e..ed3c257ee 100644 --- a/certbot-nginx/certbot_nginx/tests/http_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/http_01_test.py @@ -12,6 +12,7 @@ from certbot import achallenges from certbot.plugins import common_test from certbot.tests import acme_util +from certbot_nginx.obj import Addr from certbot_nginx.tests import util @@ -108,6 +109,41 @@ class HttpPerformTest(util.NginxTest): # self.assertEqual(vhost.addrs, set(v_addr2_print)) # self.assertEqual(vhost.names, set([response.z_domain.decode('ascii')])) + @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info") + def test_default_listen_addresses_no_memoization(self, ipv6_info): + # pylint: disable=protected-access + ipv6_info.return_value = (True, True) + self.http01._default_listen_addresses() + self.assertEqual(ipv6_info.call_count, 1) + ipv6_info.return_value = (False, False) + self.http01._default_listen_addresses() + self.assertEqual(ipv6_info.call_count, 2) + + @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info") + def test_default_listen_addresses_t_t(self, ipv6_info): + # pylint: disable=protected-access + ipv6_info.return_value = (True, True) + addrs = self.http01._default_listen_addresses() + http_addr = Addr.fromstring("80") + http_ipv6_addr = Addr.fromstring("[::]:80") + self.assertEqual(addrs, [http_addr, http_ipv6_addr]) + + @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info") + def test_default_listen_addresses_t_f(self, ipv6_info): + # pylint: disable=protected-access + ipv6_info.return_value = (True, False) + addrs = self.http01._default_listen_addresses() + http_addr = Addr.fromstring("80") + http_ipv6_addr = Addr.fromstring("[::]:80 ipv6only=on") + self.assertEqual(addrs, [http_addr, http_ipv6_addr]) + + @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info") + def test_default_listen_addresses_f_f(self, ipv6_info): + # pylint: disable=protected-access + ipv6_info.return_value = (False, False) + addrs = self.http01._default_listen_addresses() + http_addr = Addr.fromstring("80") + self.assertEqual(addrs, [http_addr]) if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py index dd31ebac8..7fc4576c3 100644 --- a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py +++ b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py @@ -271,6 +271,8 @@ class TestRawNginxParser(unittest.TestCase): location ~ ^/users/(.+\.(?:gif|jpe?g|png))$ { alias /data/w3/images/$1; } + + proxy_set_header X-Origin-URI ${scheme}://${http_host}/$request_uri; """ parsed = loads(test) self.assertEqual(parsed, [[['if', '($http_user_agent', '~', 'MSIE)'], @@ -281,7 +283,8 @@ class TestRawNginxParser(unittest.TestCase): [['return', '403']]], [['if', '($args', '~', 'post=140)'], [['rewrite', '^', 'http://example.com/']]], [['location', '~', '^/users/(.+\\.(?:gif|jpe?g|png))$'], - [['alias', '/data/w3/images/$1']]]] + [['alias', '/data/w3/images/$1']]], + ['proxy_set_header', 'X-Origin-URI', '${scheme}://${http_host}/$request_uri']] ) def test_edge_cases(self): @@ -289,10 +292,6 @@ class TestRawNginxParser(unittest.TestCase): parsed = loads(r'"hello\""; # blah "heh heh"') self.assertEqual(parsed, [['"hello\\""'], ['#', ' blah "heh heh"']]) - # empty var as block - parsed = loads(r"${}") - self.assertEqual(parsed, [[['$'], []]]) - # if with comment parsed = loads("""if ($http_cookie ~* "id=([^;]+)(?:;|$)") { # blah ) }""") @@ -342,10 +341,9 @@ class TestRawNginxParser(unittest.TestCase): ]) # variable weirdness - parsed = loads("directive $var;") - self.assertEqual(parsed, [['directive', '$var']]) + parsed = loads("directive $var ${var} $ ${};") + self.assertEqual(parsed, [['directive', '$var', '${var}', '$', '${}']]) self.assertRaises(ParseException, loads, "server {server_name test.com};") - self.assertRaises(ParseException, loads, "directive ${var};") self.assertEqual(loads("blag${dfgdfg};"), [['blag${dfgdfg}']]) self.assertRaises(ParseException, loads, "blag${dfgdf{g};") diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index 0fd37e0cb..60ec1ed1a 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -51,9 +51,6 @@ class NginxTlsSni01(common.TLSSNI01): default_addr = "{0} ssl".format( self.configurator.config.tls_sni_01_port) - ipv6, ipv6only = self.configurator.ipv6_info( - self.configurator.config.tls_sni_01_port) - for achall in self.achalls: vhosts = self.configurator.choose_vhosts(achall.domain, create_if_no_match=True) @@ -61,6 +58,9 @@ class NginxTlsSni01(common.TLSSNI01): if vhosts and vhosts[0].addrs: addresses.append(list(vhosts[0].addrs)) else: + # choose_vhosts might have modified vhosts, so put this after + ipv6, ipv6only = self.configurator.ipv6_info( + self.configurator.config.tls_sni_01_port) if ipv6: # If IPv6 is active in Nginx configuration ipv6_addr = "[::]:{0} ssl".format( @@ -141,6 +141,8 @@ class NginxTlsSni01(common.TLSSNI01): with open(self.challenge_conf, "w") as new_conf: nginxparser.dump(config, new_conf) + logger.debug("Generated server block:\n%s", str(config)) + def _make_server_block(self, achall, addrs): """Creates a server block for a challenge. diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index 38ed5debe..bcd02d197 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,2 +1,2 @@ acme[dev]==0.26.0 --e .[dev] +certbot[dev]==0.22.0 diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 02aaa6581..3c8a66ee5 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.27.0.dev0' +version = '0.28.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-nginx/tests/boulder-integration.sh b/certbot-nginx/tests/boulder-integration.sh index 980b5d45a..194413f1d 100755 --- a/certbot-nginx/tests/boulder-integration.sh +++ b/certbot-nginx/tests/boulder-integration.sh @@ -35,6 +35,7 @@ test_deployment_and_rollback() { } export default_server="default_server" +nginx -v reload_nginx certbot_test_nginx --domains nginx.wtf run test_deployment_and_rollback nginx.wtf diff --git a/certbot/__init__.py b/certbot/__init__.py index 3b0b77f6c..ab23926c9 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.27.0.dev0' +__version__ = '0.28.0.dev0' diff --git a/certbot/account.py b/certbot/account.py index 59ceb42e0..76135c3aa 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -17,6 +17,7 @@ import zope.component from acme import fields as acme_fields from acme import messages +from certbot import compat from certbot import constants from certbot import errors from certbot import interfaces @@ -140,7 +141,7 @@ class AccountFileStorage(interfaces.AccountStorage): """ def __init__(self, config): self.config = config - util.make_or_verify_dir(config.accounts_dir, 0o700, os.geteuid(), + util.make_or_verify_dir(config.accounts_dir, 0o700, compat.os_geteuid(), self.config.strict_permissions) def _account_dir_path(self, account_id): @@ -323,7 +324,7 @@ class AccountFileStorage(interfaces.AccountStorage): def _save(self, account, acme, regr_only): account_dir_path = self._account_dir_path(account.id) - util.make_or_verify_dir(account_dir_path, 0o700, os.geteuid(), + util.make_or_verify_dir(account_dir_path, 0o700, compat.os_geteuid(), self.config.strict_permissions) try: with open(self._regr_path(account_dir_path), "w") as regr_file: diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index d1205835a..2a67f8765 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -8,6 +8,7 @@ import traceback import zope.component from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module +from certbot import compat from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -104,7 +105,7 @@ def lineage_for_certname(cli_config, certname): """Find a lineage object with name certname.""" configs_dir = cli_config.renewal_configs_dir # Verify the directory is there - util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid()) + util.make_or_verify_dir(configs_dir, mode=0o755, uid=compat.os_geteuid()) try: renewal_file = storage.renewal_file_for_certname(cli_config, certname) except errors.CertStorageError: @@ -353,7 +354,7 @@ def _describe_certs(config, parsed_certs, parse_failures): notify("Found the following {0}certs:".format(match)) notify(_report_human_readable(config, parsed_certs)) if parse_failures: - notify("\nThe following renewal configuration files " + notify("\nThe following renewal configurations " "were invalid:") notify(_report_lines(parse_failures)) @@ -374,7 +375,7 @@ def _search_lineages(cli_config, func, initial_rv, *args): """ configs_dir = cli_config.renewal_configs_dir # Verify the directory is there - util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid()) + util.make_or_verify_dir(configs_dir, mode=0o755, uid=compat.os_geteuid()) rv = initial_rv for renewal_file in storage.renewal_conf_files(cli_config): diff --git a/certbot/cli.py b/certbot/cli.py index 2c4aa6530..99bf33180 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -96,7 +96,7 @@ obtain, install, and renew certificates: manage certificates: certificates Display information about certificates you have from Certbot - revoke Revoke a certificate (supply --cert-path) + revoke Revoke a certificate (supply --cert-path or --cert-name) delete Delete a certificate manage your account with Let's Encrypt: @@ -387,9 +387,10 @@ VERB_HELP = [ "usage": "\n\n certbot delete --cert-name CERTNAME\n\n" }), ("revoke", { - "short": "Revoke a certificate specified with --cert-path", + "short": "Revoke a certificate specified with --cert-path or --cert-name", "opts": "Options for revocation of certificates", - "usage": "\n\n certbot revoke --cert-path /path/to/fullchain.pem [options]\n\n" + "usage": "\n\n certbot revoke [--cert-path /path/to/fullchain.pem | " + "--cert-name example.com] [options]\n\n" }), ("register", { "short": "Register for account with Let's Encrypt / other ACME server", @@ -1333,7 +1334,7 @@ def _paths_parser(helpful): add(sections, "--cert-path", type=os.path.abspath, default=flag_default("auth_cert_path"), help=cph) elif verb == "revoke": - add(sections, "--cert-path", type=read_file, required=True, help=cph) + add(sections, "--cert-path", type=read_file, required=False, help=cph) else: add(sections, "--cert-path", type=os.path.abspath, help=cph) diff --git a/certbot/client.py b/certbot/client.py index 4d4915a27..e634b6bd9 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -24,6 +24,7 @@ import certbot from certbot import account from certbot import auth_handler from certbot import cli +from certbot import compat from certbot import constants from certbot import crypto_util from certbot import eff @@ -447,7 +448,7 @@ class Client(object): """ for path in cert_path, chain_path, fullchain_path: util.make_or_verify_dir( - os.path.dirname(path), 0o755, os.geteuid(), + os.path.dirname(path), 0o755, compat.os_geteuid(), self.config.strict_permissions) diff --git a/certbot/compat.py b/certbot/compat.py new file mode 100644 index 000000000..1dc89dfd8 --- /dev/null +++ b/certbot/compat.py @@ -0,0 +1,140 @@ +""" +Compatibility layer to run certbot both on Linux and Windows. + +The approach used here is similar to Modernizr for Web browsers. +We do not check the plateform type to determine if a particular logic is supported. +Instead, we apply a logic, and then fallback to another logic if first logic +is not supported at runtime. + +Then logic chains are abstracted into single functions to be exposed to certbot. +""" +import os +import select +import sys +import errno +import ctypes + +from certbot import errors + +try: + # Linux specific + import fcntl # pylint: disable=import-error +except ImportError: + # Windows specific + import msvcrt # pylint: disable=import-error + +UNPRIVILEGED_SUBCOMMANDS_ALLOWED = [ + 'certificates', 'enhance', 'revoke', 'delete', + 'register', 'unregister', 'config_changes', 'plugins'] +def raise_for_non_administrative_windows_rights(subcommand): + """ + On Windows, raise if current shell does not have the administrative rights. + Do nothing on Linux. + + :param str subcommand: The subcommand (like 'certonly') passed to the certbot client. + + :raises .errors.Error: If the provided subcommand must be run on a shell with + administrative rights, and current shell does not have these rights. + + """ + # Why not simply try ctypes.windll.shell32.IsUserAnAdmin() and catch AttributeError ? + # Because windll exists only on a Windows runtime, and static code analysis engines + # do not like at all non existent objects when run from Linux (even if we handle properly + # all the cases in the code). + # So we access windll only by reflection to trick theses engines. + if hasattr(ctypes, 'windll') and subcommand not in UNPRIVILEGED_SUBCOMMANDS_ALLOWED: + windll = getattr(ctypes, 'windll') + if windll.shell32.IsUserAnAdmin() == 0: + raise errors.Error( + 'Error, "{0}" subcommand must be run on a shell with administrative rights.' + .format(subcommand)) + +def os_geteuid(): + """ + Get current user uid + + :returns: The current user uid. + :rtype: int + + """ + try: + # Linux specific + return os.geteuid() + except AttributeError: + # Windows specific + return 0 + +def readline_with_timeout(timeout, prompt): + """ + Read user input to return the first line entered, or raise after specified timeout. + + :param float timeout: The timeout in seconds given to the user. + :param str prompt: The prompt message to display to the user. + + :returns: The first line entered by the user. + :rtype: str + + """ + try: + # Linux specific + # + # Call to select can only be done like this on UNIX + rlist, _, _ = select.select([sys.stdin], [], [], timeout) + if not rlist: + raise errors.Error( + "Timed out waiting for answer to prompt '{0}'".format(prompt)) + return rlist[0].readline() + except OSError: + # Windows specific + # + # No way with select to make a timeout to the user input on Windows, + # as select only supports socket in this case. + # So no timeout on Windows for now. + return sys.stdin.readline() + +def lock_file(fd): + """ + Lock the file linked to the specified file descriptor. + + :param int fd: The file descriptor of the file to lock. + + """ + if 'fcntl' in sys.modules: + # Linux specific + fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + else: + # Windows specific + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + +def release_locked_file(fd, path): + """ + Remove, close, and release a lock file specified by its file descriptor and its path. + + :param int fd: The file descriptor of the lock file. + :param str path: The path of the lock file. + + """ + # Linux specific + # + # It is important the lock file is removed before it's released, + # otherwise: + # + # process A: open lock file + # process B: release lock file + # process A: lock file + # process A: check device and inode + # process B: delete file + # process C: open and lock a different file at the same path + try: + os.remove(path) + except OSError as err: + if err.errno == errno.EACCES: + # Windows specific + # We will not be able to remove a file before closing it. + # To avoid race conditions described for Linux, we will not delete the lockfile, + # just close it to be reused on the next Certbot call. + pass + else: + raise + finally: + os.close(fd) diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 943e2c87f..6193a8fbf 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -25,6 +25,7 @@ from OpenSSL import SSL # type: ignore from acme import crypto_util as acme_crypto_util from acme.magic_typing import IO # pylint: disable=unused-import, no-name-in-module +from certbot import compat from certbot import errors from certbot import interfaces from certbot import util @@ -60,7 +61,7 @@ def init_save_key(key_size, key_dir, keyname="key-certbot.pem"): config = zope.component.getUtility(interfaces.IConfig) # Save file - util.make_or_verify_dir(key_dir, 0o700, os.geteuid(), + util.make_or_verify_dir(key_dir, 0o700, compat.os_geteuid(), config.strict_permissions) key_f, key_path = util.unique_file( os.path.join(key_dir, keyname), 0o600, "wb") @@ -91,7 +92,7 @@ def init_save_csr(privkey, names, path): privkey.pem, names, must_staple=config.must_staple) # Save CSR - util.make_or_verify_dir(path, 0o755, os.geteuid(), + util.make_or_verify_dir(path, 0o755, compat.os_geteuid(), config.strict_permissions) csr_f, csr_filename = util.unique_file( os.path.join(path, "csr-certbot.pem"), 0o644, "wb") diff --git a/certbot/display/util.py b/certbot/display/util.py index e157a1123..9a813d4b7 100644 --- a/certbot/display/util.py +++ b/certbot/display/util.py @@ -1,12 +1,12 @@ """Certbot display.""" import logging import os -import select import sys import textwrap import zope.interface +from certbot import compat from certbot import constants from certbot import interfaces from certbot import errors @@ -79,13 +79,8 @@ def input_with_timeout(prompt=None, timeout=36000.0): sys.stdout.write(prompt) sys.stdout.flush() - # select can only be used like this on UNIX - rlist, _, _ = select.select([sys.stdin], [], [], timeout) - if not rlist: - raise errors.Error( - "Timed out waiting for answer to prompt '{0}'".format(prompt)) + line = compat.readline_with_timeout(timeout, prompt) - line = rlist[0].readline() if not line: raise EOFError return line.rstrip('\n') diff --git a/certbot/lock.py b/certbot/lock.py index 5f59cc090..3ff46518d 100644 --- a/certbot/lock.py +++ b/certbot/lock.py @@ -1,9 +1,9 @@ """Implements file locks for locking files and directories in UNIX.""" import errno -import fcntl import logging import os +from certbot import compat from certbot import errors logger = logging.getLogger(__name__) @@ -74,7 +74,7 @@ class LockFile(object): """ try: - fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + compat.lock_file(fd) except IOError as err: if err.errno in (errno.EACCES, errno.EAGAIN): logger.debug( @@ -118,22 +118,7 @@ class LockFile(object): def release(self): """Remove, close, and release the lock file.""" - # It is important the lock file is removed before it's released, - # otherwise: - # - # process A: open lock file - # process B: release lock file - # process A: lock file - # process A: check device and inode - # process B: delete file - # process C: open and lock a different file at the same path - # - # Calling os.remove on a file that's in use doesn't work on - # Windows, but neither does locking with fcntl. try: - os.remove(self._path) + compat.release_locked_file(self._fd, self._path) finally: - try: - os.close(self._fd) - finally: - self._fd = None + self._fd = None diff --git a/certbot/log.py b/certbot/log.py index 89626af99..b883936f3 100644 --- a/certbot/log.py +++ b/certbot/log.py @@ -23,6 +23,7 @@ import traceback from acme import messages +from certbot import compat from certbot import constants from certbot import errors from certbot import util @@ -133,7 +134,7 @@ def setup_log_file_handler(config, logfile, fmt): # TODO: logs might contain sensitive data such as contents of the # private key! #525 util.set_up_core_dir( - config.logs_dir, 0o700, os.geteuid(), config.strict_permissions) + config.logs_dir, 0o700, compat.os_geteuid(), config.strict_permissions) log_file_path = os.path.join(config.logs_dir, logfile) try: handler = logging.handlers.RotatingFileHandler( diff --git a/certbot/main.py b/certbot/main.py index 2cba8745b..6cd2bbfac 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -19,6 +19,7 @@ from certbot import account from certbot import cert_manager from certbot import cli from certbot import client +from certbot import compat from certbot import configuration from certbot import constants from certbot import crypto_util @@ -531,8 +532,7 @@ def _determine_account(config): def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-branches """Does the user want to delete their now-revoked certs? If run in non-interactive mode, - deleting happens automatically, unless if both `--cert-name` and `--cert-path` were - specified with conflicting values. + deleting happens automatically. :param config: parsed command line arguments :type config: interfaces.IConfig @@ -556,50 +556,13 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b reporter_util.add_message("Not deleting revoked certs.", reporter_util.LOW_PRIORITY) return - if not (config.certname or config.cert_path): - raise errors.Error('At least one of --cert-path or --cert-name must be specified.') + # config.cert_path must have been set + # config.certname may have been set + assert config.cert_path - if config.certname and config.cert_path: - # first, check if certname and cert_path imply the same certs - implied_cert_name = cert_manager.cert_path_to_lineage(config) - - if implied_cert_name != config.certname: - cert_path_implied_cert_name = cert_manager.cert_path_to_lineage(config) - cert_path_implied_conf = storage.renewal_file_for_certname(config, - cert_path_implied_cert_name) - cert_path_cert = storage.RenewableCert(cert_path_implied_conf, config) - cert_path_info = cert_manager.human_readable_cert_info(config, cert_path_cert, - skip_filter_checks=True) - - cert_name_implied_conf = storage.renewal_file_for_certname(config, config.certname) - cert_name_cert = storage.RenewableCert(cert_name_implied_conf, config) - cert_name_info = cert_manager.human_readable_cert_info(config, cert_name_cert) - - msg = ("You specified conflicting values for --cert-path and --cert-name. " - "Which did you mean to select?") - choices = [cert_path_info, cert_name_info] - try: - code, index = display.menu(msg, - choices, ok_label="Select", force_interactive=True) - except errors.MissingCommandlineFlag: - error_msg = ('To run in non-interactive mode, you must either specify only one of ' - '--cert-path or --cert-name, or both must point to the same certificate lineages.') - raise errors.Error(error_msg) - - if code != display_util.OK or not index in range(0, len(choices)): - raise errors.Error("User ended interaction.") - - if index == 0: - config.certname = cert_path_implied_cert_name - else: - config.cert_path = storage.cert_path_for_cert_name(config, config.certname) - - elif config.cert_path: + if not config.certname: config.certname = cert_manager.cert_path_to_lineage(config) - else: # if only config.certname was specified - config.cert_path = storage.cert_path_for_cert_name(config, config.certname) - # don't delete if the archive_dir is used by some other lineage archive_dir = storage.full_archive_path( configobj.ConfigObj(storage.renewal_file_for_certname(config, config.certname)), @@ -1065,6 +1028,14 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config """ # For user-agent construction config.installer = config.authenticator = None + + if config.cert_path is None and config.certname: + config.cert_path = storage.cert_path_for_cert_name(config, config.certname) + elif not config.cert_path or (config.cert_path and config.certname): + # intentionally not supporting --cert-path & --cert-name together, + # to avoid dealing with mismatched values + raise errors.Error("Error! Exactly one of --cert-path or --cert-name must be specified!") + if config.key_path is not None: # revocation by cert key logger.debug("Revoking %s using cert key %s", config.cert_path[0], config.key_path[0]) @@ -1077,7 +1048,6 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config acme = client.acme_from_config_key(config, acc.key, acc.regr) cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0] logger.debug("Reason code for revocation: %s", config.reason) - try: acme.revoke(jose.ComparableX509(cert), config.reason) _delete_if_appropriate(config) @@ -1289,16 +1259,16 @@ def make_or_verify_needed_dirs(config): """ util.set_up_core_dir(config.config_dir, constants.CONFIG_DIRS_MODE, - os.geteuid(), config.strict_permissions) + compat.os_geteuid(), config.strict_permissions) util.set_up_core_dir(config.work_dir, constants.CONFIG_DIRS_MODE, - os.geteuid(), config.strict_permissions) + compat.os_geteuid(), config.strict_permissions) hook_dirs = (config.renewal_pre_hooks_dir, config.renewal_deploy_hooks_dir, config.renewal_post_hooks_dir,) for hook_dir in hook_dirs: util.make_or_verify_dir(hook_dir, - uid=os.geteuid(), + uid=compat.os_geteuid(), strict=config.strict_permissions) @@ -1333,6 +1303,7 @@ def main(cli_args=sys.argv[1:]): :raises errors.Error: error if plugin command is not supported """ + log.pre_arg_parse_setup() plugins = plugins_disco.PluginsRegistry.find_all() @@ -1346,6 +1317,10 @@ def main(cli_args=sys.argv[1:]): config = configuration.NamespaceConfig(args) zope.component.provideUtility(config) + # On windows, shell without administrative right cannot create symlinks required by certbot. + # So we check the rights before continuing. + compat.raise_for_non_administrative_windows_rights(config.verb) + try: log.post_arg_parse_setup(config) make_or_verify_needed_dirs(config) diff --git a/certbot/plugins/manual.py b/certbot/plugins/manual.py index 53533d35a..8723a1c62 100644 --- a/certbot/plugins/manual.py +++ b/certbot/plugins/manual.py @@ -94,6 +94,16 @@ using the secret key {key} when it receives a TLS ClientHello with the SNI extension set to {sni_domain} +""" + _SUBSEQUENT_CHALLENGE_INSTRUCTIONS = """ +(This must be set up in addition to the previous challenges; do not remove, +replace, or undo the previous challenge tasks yet.) +""" + _SUBSEQUENT_DNS_CHALLENGE_INSTRUCTIONS = """ +(This must be set up in addition to the previous challenges; do not remove, +replace, or undo the previous challenge tasks yet. Note that you might be +asked to create multiple distinct TXT records with the same name. This is +permitted by DNS standards.) """ def __init__(self, *args, **kwargs): @@ -103,6 +113,8 @@ when it receives a TLS ClientHello with the SNI extension set to self.env = dict() \ # type: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] self.tls_sni_01 = None + self.subsequent_dns_challenge = False + self.subsequent_any_challenge = False @classmethod def add_parser_arguments(cls, add): @@ -212,8 +224,17 @@ when it receives a TLS ClientHello with the SNI extension set to key=self.tls_sni_01.get_key_path(achall), port=self.config.tls_sni_01_port, sni_domain=self.tls_sni_01.get_z_domain(achall)) + if isinstance(achall.chall, challenges.DNS01): + if self.subsequent_dns_challenge: + # 2nd or later dns-01 challenge + msg += self._SUBSEQUENT_DNS_CHALLENGE_INSTRUCTIONS + self.subsequent_dns_challenge = True + elif self.subsequent_any_challenge: + # 2nd or later challenge of another type + msg += self._SUBSEQUENT_CHALLENGE_INSTRUCTIONS display = zope.component.getUtility(interfaces.IDisplay) display.notification(msg, wrap=False, force_interactive=True) + self.subsequent_any_challenge = True def cleanup(self, achalls): # pylint: disable=missing-docstring if self.conf('cleanup-hook'): diff --git a/certbot/plugins/manual_test.py b/certbot/plugins/manual_test.py index e5c22b377..ba4eb6889 100644 --- a/certbot/plugins/manual_test.py +++ b/certbot/plugins/manual_test.py @@ -20,8 +20,9 @@ class AuthenticatorTest(test_util.TempDirTestCase): super(AuthenticatorTest, self).setUp() self.http_achall = acme_util.HTTP01_A self.dns_achall = acme_util.DNS01_A + self.dns_achall_2 = acme_util.DNS01_A_2 self.tls_sni_achall = acme_util.TLSSNI01_A - self.achalls = [self.http_achall, self.dns_achall, self.tls_sni_achall] + self.achalls = [self.http_achall, self.dns_achall, self.tls_sni_achall, self.dns_achall_2] for d in ["config_dir", "work_dir", "in_progress"]: os.mkdir(os.path.join(self.tempdir, d)) # "backup_dir" and "temp_checkpoint_dir" get created in diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index ad2257e1d..5c682c3ff 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -9,18 +9,19 @@ logger = logging.getLogger(__name__) def get_prefixes(path): """Retrieves all possible path prefixes of a path, in descending order of length. For instance, - /a/b/c/ => ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/'] + (linux) /a/b/c returns ['/a/b/c', '/a/b', '/a', '/'] + (windows) C:\\a\\b\\c returns ['C:\\a\\b\\c', 'C:\\a\\b', 'C:\\a', 'C:'] :param str path: the path to break into prefixes :returns: all possible path prefixes of given path in descending order :rtype: `list` of `str` """ - prefix = path + prefix = os.path.normpath(path) prefixes = [] while len(prefix) > 0: prefixes.append(prefix) prefix, _ = os.path.split(prefix) - # break once we hit '/' + # break once we hit the root path if prefix == prefixes[-1]: break return prefixes @@ -51,6 +52,6 @@ def path_surgery(cmd): return True else: expanded = " expanded" if any(added) else "" - logger.warning("Failed to find executable %s in%s PATH: %s", cmd, - expanded, path) + logger.debug("Failed to find executable %s in%s PATH: %s", cmd, + expanded, path) return False diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index 2c0e476ae..b2f9c79ea 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -9,16 +9,15 @@ class GetPrefixTest(unittest.TestCase): """Tests for certbot.plugins.get_prefixes.""" def test_get_prefix(self): from certbot.plugins.util import get_prefixes - self.assertEqual(get_prefixes("/a/b/c/"), ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/']) - self.assertEqual(get_prefixes("/"), ["/"]) - self.assertEqual(get_prefixes("a"), ["a"]) + self.assertEqual(get_prefixes('/a/b/c'), ['/a/b/c', '/a/b', '/a', '/']) + self.assertEqual(get_prefixes('/'), ['/']) + self.assertEqual(get_prefixes('a'), ['a']) class PathSurgeryTest(unittest.TestCase): """Tests for certbot.plugins.path_surgery.""" - @mock.patch("certbot.plugins.util.logger.warning") @mock.patch("certbot.plugins.util.logger.debug") - def test_path_surgery(self, mock_debug, mock_warn): + def test_path_surgery(self, mock_debug): from certbot.plugins.util import path_surgery all_path = {"PATH": "/usr/local/bin:/bin/:/usr/sbin/:/usr/local/sbin/"} with mock.patch.dict('os.environ', all_path): @@ -26,14 +25,12 @@ class PathSurgeryTest(unittest.TestCase): mock_exists.return_value = True self.assertEqual(path_surgery("eg"), True) self.assertEqual(mock_debug.call_count, 0) - self.assertEqual(mock_warn.call_count, 0) self.assertEqual(os.environ["PATH"], all_path["PATH"]) no_path = {"PATH": "/tmp/"} with mock.patch.dict('os.environ', no_path): path_surgery("thingy") - self.assertEqual(mock_debug.call_count, 1) - self.assertEqual(mock_warn.call_count, 1) - self.assertTrue("Failed to find" in mock_warn.call_args[0][0]) + self.assertEqual(mock_debug.call_count, 2) + self.assertTrue("Failed to find" in mock_debug.call_args[0][0]) self.assertTrue("/usr/local/bin" in os.environ["PATH"]) self.assertTrue("/tmp" in os.environ["PATH"]) diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 5d0d7d586..529094705 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -170,7 +170,9 @@ to serve all files under specified web root ({0}).""" old_umask = os.umask(0o022) try: stat_path = os.stat(path) - for prefix in sorted(util.get_prefixes(self.full_roots[name]), key=len): + # We ignore the last prefix in the next iteration, + # as it does not correspond to a folder path ('/' or 'C:') + for prefix in sorted(util.get_prefixes(self.full_roots[name])[:-1], key=len): try: # This is coupled with the "umask" call above because # os.mkdir's "mode" parameter may not always work: @@ -180,7 +182,7 @@ to serve all files under specified web root ({0}).""" # Set owner as parent directory if possible try: os.chown(prefix, stat_path.st_uid, stat_path.st_gid) - except OSError as exception: + except (OSError, AttributeError) as exception: logger.info("Unable to change owner and uid of webroot directory") logger.debug("Error was: %s", exception) except OSError as exception: diff --git a/certbot/renewal.py b/certbot/renewal.py index f50131028..ecc8b1f2f 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -359,7 +359,7 @@ def _renew_describe_results(config, renew_successes, renew_failures, notify_error(report(renew_failures, "failure")) if parse_failures: - notify("\nAdditionally, the following renewal configuration files " + notify("\nAdditionally, the following renewal configurations " "were invalid: ") notify(report(parse_failures, "parsefail")) diff --git a/certbot/reverter.py b/certbot/reverter.py index 683c0cc32..5d56615fd 100644 --- a/certbot/reverter.py +++ b/certbot/reverter.py @@ -10,6 +10,7 @@ import traceback import six import zope.component +from certbot import compat from certbot import constants from certbot import errors from certbot import interfaces @@ -65,7 +66,7 @@ class Reverter(object): self.config = config util.make_or_verify_dir( - config.backup_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), + config.backup_dir, constants.CONFIG_DIRS_MODE, compat.os_geteuid(), self.config.strict_permissions) def revert_temporary_config(self): @@ -219,7 +220,7 @@ class Reverter(object): """ util.make_or_verify_dir( - cp_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), + cp_dir, constants.CONFIG_DIRS_MODE, compat.os_geteuid(), self.config.strict_permissions) op_fd, existing_filepaths = self._read_and_append( @@ -433,7 +434,7 @@ class Reverter(object): cp_dir = self.config.in_progress_dir util.make_or_verify_dir( - cp_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), + cp_dir, constants.CONFIG_DIRS_MODE, compat.os_geteuid(), self.config.strict_permissions) return cp_dir diff --git a/certbot/tests/acme_util.py b/certbot/tests/acme_util.py index 53a2f214a..2f9445694 100644 --- a/certbot/tests/acme_util.py +++ b/certbot/tests/acme_util.py @@ -21,6 +21,7 @@ HTTP01 = challenges.HTTP01( TLSSNI01 = challenges.TLSSNI01( token=jose.b64decode(b"evaGxfADs6pSRb2LAv9IZf17Dt3juxGJyPCt92wrDoA")) DNS01 = challenges.DNS01(token=b"17817c66b60ce2e4012dfad92657527a") +DNS01_2 = challenges.DNS01(token=b"cafecafecafecafecafecafe0feedbac") CHALLENGES = [HTTP01, TLSSNI01, DNS01] @@ -49,6 +50,7 @@ def chall_to_challb(chall, status): # pylint: disable=redefined-outer-name TLSSNI01_P = chall_to_challb(TLSSNI01, messages.STATUS_PENDING) HTTP01_P = chall_to_challb(HTTP01, messages.STATUS_PENDING) DNS01_P = chall_to_challb(DNS01, messages.STATUS_PENDING) +DNS01_P_2 = chall_to_challb(DNS01_2, messages.STATUS_PENDING) CHALLENGES_P = [HTTP01_P, TLSSNI01_P, DNS01_P] @@ -57,6 +59,7 @@ CHALLENGES_P = [HTTP01_P, TLSSNI01_P, DNS01_P] HTTP01_A = auth_handler.challb_to_achall(HTTP01_P, JWK, "example.com") TLSSNI01_A = auth_handler.challb_to_achall(TLSSNI01_P, JWK, "example.net") DNS01_A = auth_handler.challb_to_achall(DNS01_P, JWK, "example.org") +DNS01_A_2 = auth_handler.challb_to_achall(DNS01_P_2, JWK, "esimerkki.example.org") ACHALLENGES = [HTTP01_A, TLSSNI01_A, DNS01_A] diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index 59fb2cea9..eb8bcff89 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -48,18 +48,23 @@ class NamespaceConfigTest(test_util.ConfigTestCase): mock_constants.TEMP_CHECKPOINT_DIR = 't' self.assertEqual( - self.config.accounts_dir, os.path.join( - self.config.config_dir, 'acc/acme-server.org:443/new')) + os.path.normpath(self.config.accounts_dir), + os.path.normpath(os.path.join(self.config.config_dir, 'acc/acme-server.org:443/new'))) self.assertEqual( - self.config.backup_dir, os.path.join(self.config.work_dir, 'backups')) + os.path.normpath(self.config.backup_dir), + os.path.normpath(os.path.join(self.config.work_dir, 'backups'))) self.assertEqual( - self.config.csr_dir, os.path.join(self.config.config_dir, 'csr')) + os.path.normpath(self.config.csr_dir), + os.path.normpath(os.path.join(self.config.config_dir, 'csr'))) self.assertEqual( - self.config.in_progress_dir, os.path.join(self.config.work_dir, '../p')) + os.path.normpath(self.config.in_progress_dir), + os.path.normpath(os.path.join(self.config.work_dir, '../p'))) self.assertEqual( - self.config.key_dir, os.path.join(self.config.config_dir, 'keys')) + os.path.normpath(self.config.key_dir), + os.path.normpath(os.path.join(self.config.config_dir, 'keys'))) self.assertEqual( - self.config.temp_checkpoint_dir, os.path.join(self.config.work_dir, 't')) + os.path.normpath(self.config.temp_checkpoint_dir), + os.path.normpath(os.path.join(self.config.work_dir, 't'))) def test_absolute_paths(self): from certbot.configuration import NamespaceConfig diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 1dfc21c30..f5cf29047 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -34,7 +34,7 @@ class InputWithTimeoutTest(unittest.TestCase): def test_input(self, prompt=None): expected = "foo bar" stdin = six.StringIO(expected + "\n") - with mock.patch("certbot.display.util.select.select") as mock_select: + with mock.patch("certbot.compat.select.select") as mock_select: mock_select.return_value = ([stdin], [], [],) self.assertEqual(self._call(prompt), expected) @@ -321,11 +321,7 @@ class FileOutputDisplayTest(unittest.TestCase): class NoninteractiveDisplayTest(unittest.TestCase): - """Test non-interactive display. - - These tests are pretty easy! - - """ + """Test non-interactive display. These tests are pretty easy!""" def setUp(self): super(NoninteractiveDisplayTest, self).setUp() self.mock_stdout = mock.MagicMock() diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index e1a4f8c8a..51469e8c1 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -89,7 +89,7 @@ class LockFileTest(test_util.TempDirTestCase): lock_file.release() self.assertFalse(os.path.exists(self.lock_path)) - @mock.patch('certbot.lock.fcntl.lockf') + @mock.patch('certbot.compat.fcntl.lockf') def test_unexpected_lockf_err(self, mock_lockf): msg = 'hi there' mock_lockf.side_effect = IOError(msg) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index cc4e6c293..8334068c9 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -19,6 +19,7 @@ from six.moves import reload_module # pylint: disable=import-error from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import account from certbot import cli +from certbot import compat from certbot import constants from certbot import configuration from certbot import crypto_util @@ -239,6 +240,8 @@ class RevokeTest(test_util.TempDirTestCase): shutil.copy(CERT_PATH, self.tempdir) self.tmp_cert_path = os.path.abspath(os.path.join(self.tempdir, 'cert_512.pem')) + with open(self.tmp_cert_path, 'r') as f: + self.tmp_cert = (self.tmp_cert_path, f.read()) self.patches = [ mock.patch('acme.client.BackwardsCompatibleClientV2'), @@ -268,9 +271,10 @@ class RevokeTest(test_util.TempDirTestCase): for patch in self.patches: patch.stop() - def _call(self, extra_args=""): - args = 'revoke --cert-path={0} ' + extra_args - args = args.format(self.tmp_cert_path).split() + def _call(self, args=None): + if not args: + args = 'revoke --cert-path={0} ' + args = args.format(self.tmp_cert_path).split() plugins = disco.PluginsRegistry.find_all() config = configuration.NamespaceConfig( cli.prepare_and_parse_args(plugins, args)) @@ -286,12 +290,25 @@ class RevokeTest(test_util.TempDirTestCase): mock_revoke = mock_acme_client.BackwardsCompatibleClientV2().revoke expected = [] for reason, code in constants.REVOCATION_REASONS.items(): - self._call("--reason " + reason) + args = 'revoke --cert-path={0} --reason {1}'.format(self.tmp_cert_path, reason).split() + self._call(args) expected.append(mock.call(mock.ANY, code)) - self._call("--reason " + reason.upper()) + args = 'revoke --cert-path={0} --reason {1}'.format(self.tmp_cert_path, + reason.upper()).split() + self._call(args) expected.append(mock.call(mock.ANY, code)) self.assertEqual(expected, mock_revoke.call_args_list) + @mock.patch('certbot.main._delete_if_appropriate') + @mock.patch('certbot.storage.cert_path_for_cert_name') + def test_revoke_by_certname(self, mock_cert_path_for_cert_name, + mock_delete_if_appropriate): + args = 'revoke --cert-name=example.com'.split() + mock_cert_path_for_cert_name.return_value = self.tmp_cert + mock_delete_if_appropriate.return_value = False + self._call(args) + self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) + @mock.patch('certbot.main._delete_if_appropriate') def test_revocation_success(self, mock_delete_if_appropriate): self._call() @@ -358,25 +375,6 @@ class DeleteIfAppropriateTest(test_util.ConfigTestCase): self._call(config) mock_delete.assert_not_called() - # pylint: disable=too-many-arguments - @mock.patch('certbot.storage.renewal_file_for_certname') - @mock.patch('certbot.cert_manager.match_and_check_overlaps') - @mock.patch('certbot.storage.full_archive_path') - @mock.patch('certbot.cert_manager.delete') - @mock.patch('certbot.storage.cert_path_for_cert_name') - @test_util.patch_get_utility() - def test_cert_name_only(self, mock_get_utility, - mock_cert_path_for_cert_name, mock_delete, mock_archive, - mock_overlapping_archive_dirs, mock_renewal_file_for_certname): - # pylint: disable = unused-argument - config = self.config - config.certname = "example.com" - config.cert_path = "" - mock_cert_path_for_cert_name.return_value = "/some/reasonable/path" - mock_overlapping_archive_dirs.return_value = False - self._call(config) - self.assertEqual(mock_delete.call_count, 1) - # pylint: disable=too-many-arguments @mock.patch('certbot.storage.renewal_file_for_certname') @mock.patch('certbot.cert_manager.match_and_check_overlaps') @@ -439,89 +437,6 @@ class DeleteIfAppropriateTest(test_util.ConfigTestCase): self.assertEqual(mock_delete.call_count, 1) self.assertFalse(mock_get_utility().yesno.called) - # pylint: disable=too-many-arguments - @mock.patch('certbot.storage.renewal_file_for_certname') - @mock.patch('certbot.cert_manager.match_and_check_overlaps') - @mock.patch('certbot.storage.full_archive_path') - @mock.patch('certbot.cert_manager.delete') - @mock.patch('certbot.cert_manager.cert_path_to_lineage') - @test_util.patch_get_utility() - def test_certname_and_cert_path_match(self, mock_get_utility, - mock_cert_path_to_lineage, mock_delete, mock_archive, - mock_overlapping_archive_dirs, mock_renewal_file_for_certname): - # pylint: disable = unused-argument - config = self.config - config.certname = "example.com" - config.cert_path = "/some/reasonable/path" - mock_cert_path_to_lineage.return_value = config.certname - mock_overlapping_archive_dirs.return_value = False - self._call(config) - self.assertEqual(mock_delete.call_count, 1) - - # pylint: disable=too-many-arguments - @mock.patch('certbot.cert_manager.match_and_check_overlaps') - @mock.patch('certbot.storage.full_archive_path') - @mock.patch('certbot.cert_manager.delete') - @mock.patch('certbot.cert_manager.human_readable_cert_info') - @mock.patch('certbot.storage.RenewableCert') - @mock.patch('certbot.storage.renewal_file_for_certname') - @mock.patch('certbot.cert_manager.cert_path_to_lineage') - @test_util.patch_get_utility() - def test_certname_and_cert_path_mismatch(self, mock_get_utility, - mock_cert_path_to_lineage, mock_renewal_file_for_certname, - mock_RenewableCert, mock_human_readable_cert_info, - mock_delete, mock_archive, mock_overlapping_archive_dirs): - # pylint: disable=unused-argument - config = self.config - config.certname = "example.com" - config.cert_path = "/some/reasonable/path" - mock_cert_path_to_lineage = "something else" - mock_RenewableCert.return_value = mock.Mock() - mock_human_readable_cert_info.return_value = "" - mock_overlapping_archive_dirs.return_value = False - from certbot.display import util as display_util - util_mock = mock_get_utility() - util_mock.menu.return_value = (display_util.OK, 0) - self._call(config) - self.assertEqual(mock_delete.call_count, 1) - - # pylint: disable=too-many-arguments - @mock.patch('certbot.cert_manager.match_and_check_overlaps') - @mock.patch('certbot.storage.full_archive_path') - @mock.patch('certbot.cert_manager.delete') - @mock.patch('certbot.cert_manager.human_readable_cert_info') - @mock.patch('certbot.storage.RenewableCert') - @mock.patch('certbot.storage.renewal_file_for_certname') - @mock.patch('certbot.cert_manager.cert_path_to_lineage') - @test_util.patch_get_utility() - def test_noninteractive_certname_cert_path_mismatch(self, mock_get_utility, - mock_cert_path_to_lineage, mock_renewal_file_for_certname, - mock_RenewableCert, mock_human_readable_cert_info, - mock_delete, mock_archive, mock_overlapping_archive_dirs): - # pylint: disable=unused-argument - config = self.config - config.certname = "example.com" - config.cert_path = "/some/reasonable/path" - mock_cert_path_to_lineage.return_value = "some-reasonable-path.com" - mock_RenewableCert.return_value = mock.Mock() - mock_human_readable_cert_info.return_value = "" - mock_overlapping_archive_dirs.return_value = False - # Test for non-interactive mode - util_mock = mock_get_utility() - util_mock.menu.side_effect = errors.MissingCommandlineFlag("Oh no.") - self.assertRaises(errors.Error, self._call, config) - mock_delete.assert_not_called() - - @mock.patch('certbot.cert_manager.delete') - @test_util.patch_get_utility() - def test_no_certname_or_cert_path(self, mock_get_utility, mock_delete): - # pylint: disable=unused-argument - config = self.config - config.certname = None - config.cert_path = None - self.assertRaises(errors.Error, self._call, config) - mock_delete.assert_not_called() - class DetermineAccountTest(test_util.ConfigTestCase): """Tests for certbot.main._determine_account.""" @@ -1577,7 +1492,7 @@ class MakeOrVerifyNeededDirs(test_util.ConfigTestCase): for core_dir in (self.config.config_dir, self.config.work_dir,): mock_util.set_up_core_dir.assert_any_call( core_dir, constants.CONFIG_DIRS_MODE, - os.geteuid(), self.config.strict_permissions + compat.os_geteuid(), self.config.strict_permissions ) hook_dirs = (self.config.renewal_pre_hooks_dir, @@ -1586,7 +1501,7 @@ class MakeOrVerifyNeededDirs(test_util.ConfigTestCase): for hook_dir in hook_dirs: # default mode of 755 is used mock_util.make_or_verify_dir.assert_any_call( - hook_dir, uid=os.geteuid(), + hook_dir, uid=compat.os_geteuid(), strict=self.config.strict_permissions) diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 8434d11de..d505ea76c 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -362,7 +362,6 @@ def lock_and_call(func, lock_path): child.join() assert child.exitcode == 0 - def hold_lock(cv, lock_path): # pragma: no cover """Acquire a file lock at lock_path and wait to release it. diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 0e280f3ab..689b4108d 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -10,6 +10,7 @@ import mock import six from six.moves import reload_module # pylint: disable=import-error +from certbot import compat from certbot import errors import certbot.tests.util as test_util @@ -116,7 +117,7 @@ class SetUpCoreDirTest(test_util.TempDirTestCase): @mock.patch('certbot.util.lock_dir_until_exit') def test_success(self, mock_lock): new_dir = os.path.join(self.tempdir, 'new') - self._call(new_dir, 0o700, os.geteuid(), False) + self._call(new_dir, 0o700, compat.os_geteuid(), False) self.assertTrue(os.path.exists(new_dir)) self.assertEqual(mock_lock.call_count, 1) @@ -124,7 +125,7 @@ class SetUpCoreDirTest(test_util.TempDirTestCase): def test_failure(self, mock_make_or_verify): mock_make_or_verify.side_effect = OSError self.assertRaises(errors.Error, self._call, - self.tempdir, 0o700, os.geteuid(), False) + self.tempdir, 0o700, compat.os_geteuid(), False) class MakeOrVerifyDirTest(test_util.TempDirTestCase): diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 931ea4c62..4ed9f0731 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -108,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.26.1 + "". (default: CertbotACMEClient/0.27.1 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -475,16 +475,15 @@ apache: Apache Web Server plugin - Beta --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary. (default: - a2enmod) + Path to the Apache 'a2enmod' binary (default: a2enmod) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary. (default: + Path to the Apache 'a2dismod' binary (default: a2dismod) --apache-le-vhost-ext APACHE_LE_VHOST_EXT - SSL vhost configuration extension. (default: -le- + SSL vhost configuration extension (default: -le- ssl.conf) --apache-server-root APACHE_SERVER_ROOT - Apache server root directory. (default: /etc/apache2) + Apache server root directory (default: /etc/apache2) --apache-vhost-root APACHE_VHOST_ROOT Apache server VirtualHost configuration root (default: None) @@ -492,14 +491,17 @@ apache: Apache server logs directory (default: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION - Directory path for challenge configuration. (default: + Directory path for challenge configuration (default: /etc/apache2) --apache-handle-modules APACHE_HANDLE_MODULES - Let installer handle enabling required modules for - you. (Only Ubuntu/Debian currently) (default: True) + Let installer handle enabling required modules for you + (Only Ubuntu/Debian currently) (default: True) --apache-handle-sites APACHE_HANDLE_SITES - Let installer handle enabling sites for you. (Only + Let installer handle enabling sites for you (Only Ubuntu/Debian currently) (default: True) + --apache-ctl APACHE_CTL + Full path to Apache control script (default: + apache2ctl) certbot-route53:auth: Obtain certificates using a DNS TXT record (if you are using AWS Route53 diff --git a/docs/conf.py b/docs/conf.py index 09bb44285..2e6c5a9b7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,6 +40,7 @@ needs_sphinx = '1.0' # ones. extensions = [ 'sphinx.ext.autodoc', + 'sphinx.ext.imgconverter', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', diff --git a/docs/install.rst b/docs/install.rst index f7504baa5..fc6abad7a 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -9,6 +9,8 @@ Get Certbot About Certbot ============= +*Certbot is meant to be run directly on a web server*, normally by a system administrator. In most cases, running Certbot on your personal computer is not a useful option. The instructions below relate to installing and running Certbot on a server. + Certbot is packaged for many common operating systems and web servers. Check whether ``certbot`` (or ``letsencrypt``) is packaged for your web server's OS by visiting certbot.eff.org_, where you will also find the correct installation instructions for diff --git a/docs/using.rst b/docs/using.rst index 946c12bc6..2f45feca9 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -190,10 +190,11 @@ If you'd like to obtain a wildcard certificate from Let's Encrypt or run ``certbot`` on a machine other than your target webserver, you can use one of Certbot's DNS plugins. -These plugins are still in the process of being packaged -by many distributions and cannot currently be installed with ``certbot-auto``. -If, however, you are comfortable installing the certificates yourself, -you can run these plugins with :ref:`Docker `. +These plugins are not included in a default Certbot installation and must be +installed separately. While the DNS plugins cannot currently be used with +``certbot-auto``, they are available in many OS package managers and as Docker +images. Visit https://certbot.eff.org to learn the best way to use the DNS +plugins on your system. Once installed, you can find documentation on how to use each plugin at: @@ -904,7 +905,7 @@ Lock Files When processing a validation Certbot writes a number of lock files on your system to prevent multiple instances from overwriting each other's changes. This means -that be default two instances of Certbot will not be able to run in parallel. +that by default two instances of Certbot will not be able to run in parallel. Since the directories used by Certbot are configurable, Certbot will write a lock file for all of the directories it uses. This include Certbot's diff --git a/letsencrypt-auto b/letsencrypt-auto index e097719db..076c45e39 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.26.1" +LE_AUTO_VERSION="0.27.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.26.1 \ - --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ - --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 -acme==0.26.1 \ - --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ - --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be -certbot-apache==0.26.1 \ - --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ - --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 -certbot-nginx==0.26.1 \ - --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ - --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 +certbot==0.27.1 \ + --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ + --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a +acme==0.27.1 \ + --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ + --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 +certbot-apache==0.27.1 \ + --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ + --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 +certbot-nginx==0.27.1 \ + --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ + --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 9f6706931..747d98e2d 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -Version: GnuPG v2 -iQEcBAABCAAGBQJbTSv8AAoJEE0XyZXNl3Xy12sH/1FgV3SDVG0T1jgKQOYEUwrq -cmpjdav8YPgFOSQDOcyFZG0DNcRfTskZt45IMkBLLnXq2PuPvkppc1+akP81vMoK -NXHHS+PXDMjnBW4NFkexoM06KRF1SyHnvqsOg13w7UW2CjsAgtazGF5BucNCnjPH -XJTwUf4uhKxeUb0Xkva1OPH++oTWz8+SYgWr/iMggkBrK8y04QUUJ6lyCO6MZgcE -3JcECG7CwMK+hW0gCUkCSNZ0NzOBALCd9wCxNGszgkeJXrrW73oUpZmGC5BxIwYY -o6lcF0qo7Jb92t4B3+7JhulMC5JoVoG4lpiXpKQFFCT0P4pZKotIomKNMATmnB4= -=hzUL +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAluRtuUACgkQTRfJlc2X +dfIvhgf7BrKDo9wjHU8Yb2h1O63OJmoYSQMqM4Q44OVkTTjHQZgDYrOflbegq9g+ +nxxOcMakiPTxvefZOecczKGTZZ/S+A/w5kH/9vJbxW0277iNnYsj1G59m1UPNzgn +ECFL5AUKhl/RF3NWSpe2XhGA7ybls8LAidwxeS3b3nXNeuXIspKd84AIAqaWlpOa +I16NhJsU8VOq6I5RCgkx4WgmmUhCmzjLbYDH7rjj1dehCZa0Y63mlMdTKKs4BJSk +AtSVVV6nTupZdHPJtpQ1RxcT6iTy8Nr13cVuKnluui7KZ/uktOdB0H1o5kuWchvm +8/oqLVSfoqjhU6Fn/11Af+iCnpICUw== +=QRnC -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 765072c3f..740cb9c73 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.0.dev0" +LE_AUTO_VERSION="0.28.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -195,7 +195,7 @@ if [ "$1" = "--cb-auto-has-root" ]; then else SetRootAuthMechanism if [ -n "$SUDO" ]; then - echo "Requesting to rerun $0 with root privileges..." + say "Requesting to rerun $0 with root privileges..." $SUDO "$0" --cb-auto-has-root "$@" exit 0 fi @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.26.1 \ - --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ - --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 -acme==0.26.1 \ - --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ - --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be -certbot-apache==0.26.1 \ - --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ - --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 -certbot-nginx==0.26.1 \ - --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ - --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 +certbot==0.27.1 \ + --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ + --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a +acme==0.27.1 \ + --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ + --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 +certbot-apache==0.27.1 \ + --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ + --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 +certbot-nginx==0.27.1 \ + --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ + --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index d1306d03d..b717e359b 100644 Binary files a/letsencrypt-auto-source/letsencrypt-auto.sig and b/letsencrypt-auto-source/letsencrypt-auto.sig differ diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 9d0f27009..6d2977832 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -195,7 +195,7 @@ if [ "$1" = "--cb-auto-has-root" ]; then else SetRootAuthMechanism if [ -n "$SUDO" ]; then - echo "Requesting to rerun $0 with root privileges..." + say "Requesting to rerun $0 with root privileges..." $SUDO "$0" --cb-auto-has-root "$@" exit 0 fi diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index feb3f1c3a..b9cd42694 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.26.1 \ - --hash=sha256:4e2ffdeebb7f5097600bcb1ca19131441fa021f952b443ca7454a279337af609 \ - --hash=sha256:4983513d63f7f36e24a07873ca2d6ea1c0101aa6cb1cd825cda02ed520f6ca66 -acme==0.26.1 \ - --hash=sha256:d47841e66adc1336ecca2f0d41a247c1b62307c981be6d07996bbf3f95af1dc5 \ - --hash=sha256:86e7b5f4654cb19215f16c0e6225750db7421f68ef6a0a040a61796f24e690be -certbot-apache==0.26.1 \ - --hash=sha256:c16acb49bd4f84fff25bcbb7eaf74412145efe9b68ce46e1803be538894f2ce3 \ - --hash=sha256:b7fa327e987b892d64163e7519bdeaf9723d78275ef6c438272848894ace6d87 -certbot-nginx==0.26.1 \ - --hash=sha256:c0048dc83672dc90805a8ddf513be3e48c841d6e91607e91e8657c1785d65660 \ - --hash=sha256:d0c95a32625e0f1612d7fcf9021e6e050ba3d879823489d1edd2478a78ae6624 +certbot==0.27.1 \ + --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ + --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a +acme==0.27.1 \ + --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ + --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 +certbot-apache==0.27.1 \ + --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ + --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 +certbot-nginx==0.27.1 \ + --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ + --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f diff --git a/pull_request_template.md b/pull_request_template.md new file mode 100644 index 000000000..c071d4135 --- /dev/null +++ b/pull_request_template.md @@ -0,0 +1 @@ +Be sure to edit the `master` section of `CHANGELOG.md` with a line describing this PR before it gets merged. diff --git a/setup.py b/setup.py index 1827c4d42..f8f5feadc 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,6 @@ init_fn = os.path.join(here, 'certbot', '__init__.py') meta = dict(re.findall(r"""__([a-z]+)__ = '([^']+)""", read_file(init_fn))) readme = read_file(os.path.join(here, 'README.rst')) -changes = read_file(os.path.join(here, 'CHANGES.rst')) version = meta['version'] # This package relies on PyOpenSSL, requests, and six, however, it isn't @@ -70,8 +69,8 @@ dev3_extras = [ docs_extras = [ 'repoze.sphinx.autointerface', - # autodoc_member_order = 'bysource', autodoc_default_flags, and #4686 - 'Sphinx >=1.0,<=1.5.6', + # sphinx.ext.imgconverter + 'Sphinx >=1.6', 'sphinx_rtd_theme', ] @@ -79,7 +78,7 @@ setup( name='certbot', version=version, description="ACME client", - long_description=readme, # later: + '\n\n' + changes + long_description=readme, url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", author_email='client-dev@letsencrypt.org', diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 2f9130489..3e16fcbbc 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -1,496 +1,16 @@ #!/bin/bash -# Simple integration test. Make sure to activate virtualenv beforehand -# (source venv/bin/activate) and that you are running Boulder test -# instance (see ./boulder-fetch.sh). -# -# Environment variables: -# SERVER: Passed as "certbot --server" argument. -# -# Note: this script is called by Boulder integration test suite! -set -eux +set -e -. ./tests/integration/_common.sh -export PATH="$PATH:/usr/sbin" # /usr/sbin/nginx - -cleanup_and_exit() { - EXIT_STATUS=$? - if SERVER_STILL_RUNNING=`ps -p $python_server_pid -o pid=` - then - echo Kill server subprocess, left running by abnormal exit - kill $SERVER_STILL_RUNNING - fi - if [ -f "$HOOK_DIRS_TEST" ]; then - rm -f "$HOOK_DIRS_TEST" - fi - exit $EXIT_STATUS -} - -trap cleanup_and_exit EXIT - -export HOOK_DIRS_TEST="$(mktemp)" -renewal_hooks_root="$config_dir/renewal-hooks" -renewal_hooks_dirs=$(echo "$renewal_hooks_root/"{pre,deploy,post}) -renewal_dir_pre_hook="$(echo $renewal_hooks_dirs | cut -f 1 -d " ")/hook.sh" -renewal_dir_deploy_hook="$(echo $renewal_hooks_dirs | cut -f 2 -d " ")/hook.sh" -renewal_dir_post_hook="$(echo $renewal_hooks_dirs | cut -f 3 -d " ")/hook.sh" - -# Creates hooks in Certbot's renewal hook directory that write to a file -CreateDirHooks() { - for hook_dir in $renewal_hooks_dirs; do - mkdir -p $hook_dir - hook_path="$hook_dir/hook.sh" - cat << EOF > "$hook_path" -#!/bin/bash -xe -if [ "\$0" = "$renewal_dir_deploy_hook" ]; then - if [ -z "\$RENEWED_DOMAINS" -o -z "\$RENEWED_LINEAGE" ]; then - echo "Environment variables not properly set!" >&2 - exit 1 +if [ "$INTEGRATION_TEST" = "certbot" ]; then + tests/certbot-boulder-integration.sh +elif [ "$INTEGRATION_TEST" = "nginx" ]; then + certbot-nginx/tests/boulder-integration.sh +else + tests/certbot-boulder-integration.sh + # Most CI systems set this variable to true. + # If the tests are running as part of CI, Nginx should be available. + if ${CI:-false} || type nginx; then + certbot-nginx/tests/boulder-integration.sh fi fi -echo \$(basename \$(dirname "\$0")) >> "\$HOOK_DIRS_TEST" -EOF - chmod +x "$hook_path" - done -} - -# Asserts that the hooks created by CreateDirHooks have been run once and -# resets the file. -# -# Arguments: -# The number of times the deploy hook should have been run. (It should run -# once for each certificate that was issued in that run of Certbot.) -CheckDirHooks() { - expected="pre\n" - for ((i=0; i<$1; i++)); do - expected=$expected"deploy\n" - done - expected=$expected"post" - - if ! diff "$HOOK_DIRS_TEST" <(echo -e "$expected"); then - echo "Unexpected directory hook output!" >&2 - echo "Expected:" >&2 - echo -e "$expected" >&2 - echo "Got:" >&2 - cat "$HOOK_DIRS_TEST" >&2 - exit 1 - fi - - rm -f "$HOOK_DIRS_TEST" - export HOOK_DIRS_TEST="$(mktemp)" -} - -common_no_force_renew() { - certbot_test_no_force_renew \ - --authenticator standalone \ - --installer null \ - "$@" -} - -common() { - common_no_force_renew \ - --renew-by-default \ - "$@" -} - -export HOOK_TEST="/tmp/hook$$" -CheckHooks() { - if [ $(head -n1 "$HOOK_TEST") = "wtf.pre" ]; then - expected="wtf.pre\ndeploy\n" - if [ $(sed '3q;d' "$HOOK_TEST") = "deploy" ]; then - expected=$expected"deploy\nwtf2.pre\n" - else - expected=$expected"wtf2.pre\ndeploy\n" - fi - expected=$expected"deploy\ndeploy\nwtf.post\nwtf2.post" - else - expected="wtf2.pre\ndeploy\n" - if [ $(sed '3q;d' "$HOOK_TEST") = "deploy" ]; then - expected=$expected"deploy\nwtf.pre\n" - else - expected=$expected"wtf.pre\ndeploy\n" - fi - expected=$expected"deploy\ndeploy\nwtf2.post\nwtf.post" - fi - - if ! cmp --quiet <(echo -e "$expected") "$HOOK_TEST" ; then - echo Hooks did not run as expected\; got >&2 - cat "$HOOK_TEST" >&2 - echo -e "Expected\n$expected" >&2 - rm "$HOOK_TEST" - exit 1 - fi - rm "$HOOK_TEST" -} - -# Checks if deploy is in the hook output and deletes the file -DeployInHookOutput() { - CONTENTS=$(cat "$HOOK_TEST") - rm "$HOOK_TEST" - grep deploy <(echo "$CONTENTS") -} - -# Asserts that there is a saved renew_hook for a lineage. -# -# Arguments: -# Name of lineage to check -CheckSavedRenewHook() { - if ! grep renew_hook "$config_dir/renewal/$1.conf"; then - echo "Hook wasn't saved as renew_hook" >&2 - exit 1 - fi -} - -# Asserts the deploy hook was properly run and saved and deletes the hook file -# -# Arguments: -# Lineage name of the issued cert -CheckDeployHook() { - if ! DeployInHookOutput; then - echo "The deploy hook wasn't run" >&2 - exit 1 - fi - CheckSavedRenewHook $1 -} - -# Asserts the renew hook wasn't run but was saved and deletes the hook file -# -# Arguments: -# Lineage name of the issued cert -# Asserts the deploy hook wasn't run and deletes the hook file -CheckRenewHook() { - if DeployInHookOutput; then - echo "The renew hook was incorrectly run" >&2 - exit 1 - fi - CheckSavedRenewHook $1 -} - -# Return success only if input contains exactly $1 lines of text, of -# which $2 different values occur in the first field. -TotalAndDistinctLines() { - total=$1 - distinct=$2 - awk '{a[$1] = 1}; END {exit(NR !='$total' || length(a) !='$distinct')}' -} - -# Cleanup coverage data -coverage erase - -# test for regressions of #4719 -get_num_tmp_files() { - ls -1 /tmp | wc -l -} -num_tmp_files=$(get_num_tmp_files) -common --csr / && echo expected error && exit 1 || true -common --help -common --help all -common --version -if [ $(get_num_tmp_files) -ne $num_tmp_files ]; then - echo "New files or directories created in /tmp!" - exit 1 -fi -CreateDirHooks - -common register -for dir in $renewal_hooks_dirs; do - if [ ! -d "$dir" ]; then - echo "Hook directory not created by Certbot!" >&2 - exit 1 - fi -done - -common unregister - -common register --email ex1@domain.org,ex2@domain.org - -common register --update-registration --email ex1@domain.org - -common register --update-registration --email ex1@domain.org,ex2@domain.org - -common plugins --init --prepare | grep webroot - -# We start a server listening on the port for the -# unrequested challenge to prevent regressions in #3601. -python ./tests/run_http_server.py $http_01_port & -python_server_pid=$! - -certname="le1.wtf" -common --domains le1.wtf --preferred-challenges tls-sni-01 auth \ - --cert-name $certname \ - --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ - --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ - --deploy-hook 'echo deploy >> "$HOOK_TEST"' -kill $python_server_pid -CheckDeployHook $certname - -python ./tests/run_http_server.py $tls_sni_01_port & -python_server_pid=$! -certname="le2.wtf" -common --domains le2.wtf --preferred-challenges http-01 run \ - --cert-name $certname \ - --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ - --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ - --deploy-hook 'echo deploy >> "$HOOK_TEST"' -kill $python_server_pid -CheckDeployHook $certname - -certname="le.wtf" -common certonly -a manual -d le.wtf --rsa-key-size 4096 --cert-name $certname \ - --manual-auth-hook ./tests/manual-http-auth.sh \ - --manual-cleanup-hook ./tests/manual-http-cleanup.sh \ - --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \ - --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \ - --renew-hook 'echo deploy >> "$HOOK_TEST"' -CheckRenewHook $certname - -certname="dns.le.wtf" -common -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni run \ - --cert-name $certname \ - --manual-auth-hook ./tests/manual-dns-auth.sh \ - --manual-cleanup-hook ./tests/manual-dns-cleanup.sh \ - --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \ - --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \ - --renew-hook 'echo deploy >> "$HOOK_TEST"' -CheckRenewHook $certname - -common certonly --cert-name newname -d newname.le.wtf - -export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ - OPENSSL_CNF=examples/openssl.cnf -./examples/generate-csr.sh le3.wtf -common auth --csr "$CSR_PATH" \ - --cert-path "${root}/csr/cert.pem" \ - --chain-path "${root}/csr/chain.pem" -openssl x509 -in "${root}/csr/cert.pem" -text -openssl x509 -in "${root}/csr/chain.pem" -text - -common --domains le3.wtf install \ - --cert-path "${root}/csr/cert.pem" \ - --key-path "${root}/key.pem" - -CheckCertCount() { - CERTCOUNT=`ls "${root}/conf/archive/$1/cert"* | wc -l` - if [ "$CERTCOUNT" -ne "$2" ] ; then - echo Wrong cert count, not "$2" `ls "${root}/conf/archive/$1/"*` - exit 1 - fi -} - -CheckCertCount "le.wtf" 1 -# This won't renew (because it's not time yet) -common_no_force_renew renew -CheckCertCount "le.wtf" 1 -if [ -s "$HOOK_DIRS_TEST" ]; then - echo "Directory hooks were executed for non-renewal!" >&2; - exit 1 -fi - -rm -rf "$renewal_hooks_root" -# renew using HTTP manual auth hooks -common renew --cert-name le.wtf --authenticator manual -CheckCertCount "le.wtf" 2 - -# test renewal with no executables in hook directories -for hook_dir in $renewal_hooks_dirs; do - touch "$hook_dir/file" - mkdir "$hook_dir/dir" -done -# renew using DNS manual auth hooks -common renew --cert-name dns.le.wtf --authenticator manual -CheckCertCount "dns.le.wtf" 2 - -# test with disabled directory hooks -rm -rf "$renewal_hooks_root" -CreateDirHooks -# This will renew because the expiry is less than 10 years from now -sed -i "4arenew_before_expiry = 4 years" "$root/conf/renewal/le.wtf.conf" -common_no_force_renew renew --rsa-key-size 2048 --no-directory-hooks -CheckCertCount "le.wtf" 3 -if [ -s "$HOOK_DIRS_TEST" ]; then - echo "Directory hooks were executed with --no-directory-hooks!" >&2 - exit 1 -fi - -# The 4096 bit setting should persist to the first renewal, but be overridden in the second - -size1=`wc -c ${root}/conf/archive/le.wtf/privkey1.pem | cut -d" " -f1` -size2=`wc -c ${root}/conf/archive/le.wtf/privkey2.pem | cut -d" " -f1` -size3=`wc -c ${root}/conf/archive/le.wtf/privkey3.pem | cut -d" " -f1` -# 4096 bit PEM keys are about ~3270 bytes, 2048 ones are about 1700 bytes -if [ "$size1" -lt 3000 ] || [ "$size2" -lt 3000 ] || [ "$size3" -gt 1800 ] ; then - echo key sizes violate assumptions: - ls -l "${root}/conf/archive/le.wtf/privkey"* - exit 1 -fi - -# --renew-by-default is used, so renewal should occur -[ -f "$HOOK_TEST" ] && rm -f "$HOOK_TEST" -common renew -CheckCertCount "le.wtf" 4 -CheckHooks -CheckDirHooks 5 - -# test with overlapping directory hooks on the command line -common renew --cert-name le2.wtf \ - --pre-hook "$renewal_dir_pre_hook" \ - --deploy-hook "$renewal_dir_deploy_hook" \ - --post-hook "$renewal_dir_post_hook" -CheckDirHooks 1 - -# test with overlapping directory hooks in the renewal conf files -common renew --cert-name le2.wtf -CheckDirHooks 1 - -# manual-dns-auth.sh will skip completing the challenge for domains that begin -# with fail. -common -a manual -d dns1.le.wtf,fail.dns1.le.wtf \ - --allow-subset-of-names \ - --preferred-challenges dns,tls-sni \ - --manual-auth-hook ./tests/manual-dns-auth.sh \ - --manual-cleanup-hook ./tests/manual-dns-cleanup.sh - -if common certificates | grep "fail\.dns1\.le\.wtf"; then - echo "certificate should not have been issued for domain!" >&2 - exit 1 -fi - -# reuse-key -common --domains reusekey.le.wtf --reuse-key -common renew --cert-name reusekey.le.wtf -CheckCertCount "reusekey.le.wtf" 2 -ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* -# The final awk command here exits successfully if its input consists of -# exactly two lines with identical first fields, and unsuccessfully otherwise. -sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 2 1 - -# don't reuse key (just by forcing reissuance without --reuse-key) -common --cert-name reusekey.le.wtf --domains reusekey.le.wtf --force-renewal -CheckCertCount "reusekey.le.wtf" 3 -ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* -# Exactly three lines, of which exactly two identical first fields. -sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 3 2 - -# Nonetheless, all three certificates are different even though two of them -# share the same subject key. -sha256sum "${root}/conf/archive/reusekey.le.wtf/cert"* | TotalAndDistinctLines 3 3 - -# ECDSA -openssl ecparam -genkey -name secp384r1 -out "${root}/privkey-p384.pem" -SAN="DNS:ecdsa.le.wtf" openssl req -new -sha256 \ - -config "${OPENSSL_CNF:-openssl.cnf}" \ - -key "${root}/privkey-p384.pem" \ - -subj "/" \ - -reqexts san \ - -outform der \ - -out "${root}/csr-p384.der" -common auth --csr "${root}/csr-p384.der" \ - --cert-path "${root}/csr/cert-p384.pem" \ - --chain-path "${root}/csr/chain-p384.pem" -openssl x509 -in "${root}/csr/cert-p384.pem" -text | grep 'ASN1 OID: secp384r1' - -# OCSP Must Staple -common auth --must-staple --domains "must-staple.le.wtf" -openssl x509 -in "${root}/conf/live/must-staple.le.wtf/cert.pem" -text | grep '1.3.6.1.5.5.7.1.24' - -# revoke by account key -common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" --delete-after-revoke -# revoke renewed -common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" --no-delete-after-revoke -if [ ! -d "$root/conf/live/le1.wtf" ]; then - echo "cert deleted when --no-delete-after-revoke was used!" - exit 1 -fi -common delete --cert-name le1.wtf -# revoke by cert key -common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ - --key-path "$root/conf/live/le2.wtf/privkey.pem" - -# Get new certs to test revoke with a reason, by account and by cert key -common --domains le1.wtf -common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" \ - --reason cessationOfOperation -common --domains le2.wtf -common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ - --key-path "$root/conf/live/le2.wtf/privkey.pem" \ - --reason keyCompromise - -common unregister - -out=$(common certificates) -subdomains="le dns.le newname.le must-staple.le" -for subdomain in $subdomains; do - domain="$subdomain.wtf" - if ! echo $out | grep "$domain"; then - echo "$domain not in certificates output!" - exit 1; - fi -done - -# Testing that revocation also deletes by default -subdomains="le1 le2" -for subdomain in $subdomains; do - domain="$subdomain.wtf" - if echo $out | grep "$domain"; then - echo "Revoked $domain in certificates output! Should not be!" - exit 1; - fi -done - -# Test that revocation raises correct error if --cert-name and --cert-path don't match -common --domains le1.wtf -common --domains le2.wtf -out=$(common revoke --cert-path "$root/conf/live/le1.wtf/fullchain.pem" --cert-name "le2.wtf" 2>&1) || true -if ! echo $out | grep "or both must point to the same certificate lineages."; then - echo "Non-interactive revoking with mismatched --cert-name and --cert-path " - echo "did not raise the correct error!" - exit 1 -fi - -# Revoking by matching --cert-name and --cert-path deletes -common --domains le1.wtf -common revoke --cert-path "$root/conf/live/le1.wtf/fullchain.pem" --cert-name "le1.wtf" -out=$(common certificates) -if echo $out | grep "le1.wtf"; then - echo "Cert le1.wtf should've been deleted! Was revoked via matching --cert-path & --cert-name" - exit 1 -fi - -# Test that revocation doesn't delete if multiple lineages share an archive dir -common --domains le1.wtf -common --domains le2.wtf -sed -i "s|^archive_dir = .*$|archive_dir = $root/conf/archive/le1.wtf|" "$root/conf/renewal/le2.wtf.conf" -#common update_symlinks # not needed, but a bit more context for what this test is about -out=$(common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem") -if ! echo $out | grep "Not deleting revoked certs due to overlapping archive dirs"; then - echo "Deleted a cert that had an overlapping archive dir with another lineage!" - exit 1 -fi - -cert_name="must-staple.le.wtf" -common delete --cert-name $cert_name -archive="$root/conf/archive/$cert_name" -conf="$root/conf/renewal/$cert_name.conf" -live="$root/conf/live/$cert_name" -for path in $archive $conf $live; do - if [ -e $path ]; then - echo "Lineage not properly deleted!" - exit 1 - fi -done - -# Test ACMEv2-only features -if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then - common -a manual -d '*.le4.wtf,le4.wtf' --preferred-challenges dns \ - --manual-auth-hook ./tests/manual-dns-auth.sh \ - --manual-cleanup-hook ./tests/manual-dns-cleanup.sh -fi - -coverage report --fail-under 65 --include 'certbot/*' --show-missing - -# Most CI systems set this variable to true. -# If the tests are running as part of CI, Nginx should be available. -if ${CI:-false} || type nginx; -then - . ./certbot-nginx/tests/boulder-integration.sh -fi diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh new file mode 100755 index 000000000..e250e591b --- /dev/null +++ b/tests/certbot-boulder-integration.sh @@ -0,0 +1,482 @@ +#!/bin/bash +# Simple integration test. Make sure to activate virtualenv beforehand +# (source venv/bin/activate) and that you are running Boulder test +# instance (see ./boulder-fetch.sh). +# +# Environment variables: +# SERVER: Passed as "certbot --server" argument. +# +# Note: this script is called by Boulder integration test suite! + +set -eux + +# Check that python executable is available in the PATH. Fail immediatly if not. +command -v python > /dev/null || (echo "Error, python executable is not in the PATH" && exit 1) + +. ./tests/integration/_common.sh +export PATH="$PATH:/usr/sbin" # /usr/sbin/nginx + +cleanup_and_exit() { + EXIT_STATUS=$? + if SERVER_STILL_RUNNING=`ps -p $python_server_pid -o pid=` + then + echo Kill server subprocess, left running by abnormal exit + kill $SERVER_STILL_RUNNING + fi + if [ -f "$HOOK_DIRS_TEST" ]; then + rm -f "$HOOK_DIRS_TEST" + fi + exit $EXIT_STATUS +} + +trap cleanup_and_exit EXIT + +export HOOK_DIRS_TEST="$(mktemp)" +renewal_hooks_root="$config_dir/renewal-hooks" +renewal_hooks_dirs=$(echo "$renewal_hooks_root/"{pre,deploy,post}) +renewal_dir_pre_hook="$(echo $renewal_hooks_dirs | cut -f 1 -d " ")/hook.sh" +renewal_dir_deploy_hook="$(echo $renewal_hooks_dirs | cut -f 2 -d " ")/hook.sh" +renewal_dir_post_hook="$(echo $renewal_hooks_dirs | cut -f 3 -d " ")/hook.sh" + +# Creates hooks in Certbot's renewal hook directory that write to a file +CreateDirHooks() { + for hook_dir in $renewal_hooks_dirs; do + mkdir -p $hook_dir + hook_path="$hook_dir/hook.sh" + cat << EOF > "$hook_path" +#!/bin/bash -xe +if [ "\$0" = "$renewal_dir_deploy_hook" ]; then + if [ -z "\$RENEWED_DOMAINS" -o -z "\$RENEWED_LINEAGE" ]; then + echo "Environment variables not properly set!" >&2 + exit 1 + fi +fi +echo \$(basename \$(dirname "\$0")) >> "\$HOOK_DIRS_TEST" +EOF + chmod +x "$hook_path" + done +} + +# Asserts that the hooks created by CreateDirHooks have been run once and +# resets the file. +# +# Arguments: +# The number of times the deploy hook should have been run. (It should run +# once for each certificate that was issued in that run of Certbot.) +CheckDirHooks() { + expected="pre\n" + for ((i=0; i<$1; i++)); do + expected=$expected"deploy\n" + done + expected=$expected"post" + + if ! diff "$HOOK_DIRS_TEST" <(echo -e "$expected"); then + echo "Unexpected directory hook output!" >&2 + echo "Expected:" >&2 + echo -e "$expected" >&2 + echo "Got:" >&2 + cat "$HOOK_DIRS_TEST" >&2 + exit 1 + fi + + rm -f "$HOOK_DIRS_TEST" + export HOOK_DIRS_TEST="$(mktemp)" +} + +common_no_force_renew() { + certbot_test_no_force_renew \ + --authenticator standalone \ + --installer null \ + "$@" +} + +common() { + common_no_force_renew \ + --renew-by-default \ + "$@" +} + +export HOOK_TEST="/tmp/hook$$" +CheckHooks() { + if [ $(head -n1 "$HOOK_TEST") = "wtf.pre" ]; then + expected="wtf.pre\ndeploy\n" + if [ $(sed '3q;d' "$HOOK_TEST") = "deploy" ]; then + expected=$expected"deploy\nwtf2.pre\n" + else + expected=$expected"wtf2.pre\ndeploy\n" + fi + expected=$expected"deploy\ndeploy\nwtf.post\nwtf2.post" + else + expected="wtf2.pre\ndeploy\n" + if [ $(sed '3q;d' "$HOOK_TEST") = "deploy" ]; then + expected=$expected"deploy\nwtf.pre\n" + else + expected=$expected"wtf.pre\ndeploy\n" + fi + expected=$expected"deploy\ndeploy\nwtf2.post\nwtf.post" + fi + + if ! cmp --quiet <(echo -e "$expected") "$HOOK_TEST" ; then + echo Hooks did not run as expected\; got >&2 + cat "$HOOK_TEST" >&2 + echo -e "Expected\n$expected" >&2 + rm "$HOOK_TEST" + exit 1 + fi + rm "$HOOK_TEST" +} + +# Checks if deploy is in the hook output and deletes the file +DeployInHookOutput() { + CONTENTS=$(cat "$HOOK_TEST") + rm "$HOOK_TEST" + grep deploy <(echo "$CONTENTS") +} + +# Asserts that there is a saved renew_hook for a lineage. +# +# Arguments: +# Name of lineage to check +CheckSavedRenewHook() { + if ! grep renew_hook "$config_dir/renewal/$1.conf"; then + echo "Hook wasn't saved as renew_hook" >&2 + exit 1 + fi +} + +# Asserts the deploy hook was properly run and saved and deletes the hook file +# +# Arguments: +# Lineage name of the issued cert +CheckDeployHook() { + if ! DeployInHookOutput; then + echo "The deploy hook wasn't run" >&2 + exit 1 + fi + CheckSavedRenewHook $1 +} + +# Asserts the renew hook wasn't run but was saved and deletes the hook file +# +# Arguments: +# Lineage name of the issued cert +# Asserts the deploy hook wasn't run and deletes the hook file +CheckRenewHook() { + if DeployInHookOutput; then + echo "The renew hook was incorrectly run" >&2 + exit 1 + fi + CheckSavedRenewHook $1 +} + +# Return success only if input contains exactly $1 lines of text, of +# which $2 different values occur in the first field. +TotalAndDistinctLines() { + total=$1 + distinct=$2 + awk '{a[$1] = 1}; END {exit(NR !='$total' || length(a) !='$distinct')}' +} + +# Cleanup coverage data +coverage erase + +# test for regressions of #4719 +get_num_tmp_files() { + ls -1 /tmp | wc -l +} +num_tmp_files=$(get_num_tmp_files) +common --csr / > /dev/null && echo expected error && exit 1 || true +common --help > /dev/null +common --help all > /dev/null +common --version > /dev/null +if [ $(get_num_tmp_files) -ne $num_tmp_files ]; then + echo "New files or directories created in /tmp!" + exit 1 +fi +CreateDirHooks + +common register +for dir in $renewal_hooks_dirs; do + if [ ! -d "$dir" ]; then + echo "Hook directory not created by Certbot!" >&2 + exit 1 + fi +done + +common unregister + +common register --email ex1@domain.org,ex2@domain.org + +common register --update-registration --email ex1@domain.org + +common register --update-registration --email ex1@domain.org,ex2@domain.org + +common plugins --init --prepare | grep webroot + +# We start a server listening on the port for the +# unrequested challenge to prevent regressions in #3601. +python ./tests/run_http_server.py $http_01_port & +python_server_pid=$! + +certname="le1.wtf" +common --domains le1.wtf --preferred-challenges tls-sni-01 auth \ + --cert-name $certname \ + --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ + --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ + --deploy-hook 'echo deploy >> "$HOOK_TEST"' +kill $python_server_pid +CheckDeployHook $certname + +python ./tests/run_http_server.py $tls_sni_01_port & +python_server_pid=$! +certname="le2.wtf" +common --domains le2.wtf --preferred-challenges http-01 run \ + --cert-name $certname \ + --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ + --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ + --deploy-hook 'echo deploy >> "$HOOK_TEST"' +kill $python_server_pid +CheckDeployHook $certname + +certname="le.wtf" +common certonly -a manual -d le.wtf --rsa-key-size 4096 --cert-name $certname \ + --manual-auth-hook ./tests/manual-http-auth.sh \ + --manual-cleanup-hook ./tests/manual-http-cleanup.sh \ + --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \ + --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \ + --renew-hook 'echo deploy >> "$HOOK_TEST"' +CheckRenewHook $certname + +certname="dns.le.wtf" +common -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni run \ + --cert-name $certname \ + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh \ + --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \ + --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \ + --renew-hook 'echo deploy >> "$HOOK_TEST"' +CheckRenewHook $certname + +common certonly --cert-name newname -d newname.le.wtf + +export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ + OPENSSL_CNF=examples/openssl.cnf +./examples/generate-csr.sh le3.wtf +common auth --csr "$CSR_PATH" \ + --cert-path "${root}/csr/cert.pem" \ + --chain-path "${root}/csr/chain.pem" +openssl x509 -in "${root}/csr/cert.pem" -text +openssl x509 -in "${root}/csr/chain.pem" -text + +common --domains le3.wtf install \ + --cert-path "${root}/csr/cert.pem" \ + --key-path "${root}/key.pem" + +CheckCertCount() { + CERTCOUNT=`ls "${root}/conf/archive/$1/cert"* | wc -l` + if [ "$CERTCOUNT" -ne "$2" ] ; then + echo Wrong cert count, not "$2" `ls "${root}/conf/archive/$1/"*` + exit 1 + fi +} + +CheckCertCount "le.wtf" 1 +# This won't renew (because it's not time yet) +common_no_force_renew renew +CheckCertCount "le.wtf" 1 +if [ -s "$HOOK_DIRS_TEST" ]; then + echo "Directory hooks were executed for non-renewal!" >&2; + exit 1 +fi + +rm -rf "$renewal_hooks_root" +# renew using HTTP manual auth hooks +common renew --cert-name le.wtf --authenticator manual +CheckCertCount "le.wtf" 2 + +# test renewal with no executables in hook directories +for hook_dir in $renewal_hooks_dirs; do + touch "$hook_dir/file" + mkdir "$hook_dir/dir" +done +# renew using DNS manual auth hooks +common renew --cert-name dns.le.wtf --authenticator manual +CheckCertCount "dns.le.wtf" 2 + +# test with disabled directory hooks +rm -rf "$renewal_hooks_root" +CreateDirHooks +# This will renew because the expiry is less than 10 years from now +sed -i "4arenew_before_expiry = 4 years" "$root/conf/renewal/le.wtf.conf" +common_no_force_renew renew --rsa-key-size 2048 --no-directory-hooks +CheckCertCount "le.wtf" 3 +if [ -s "$HOOK_DIRS_TEST" ]; then + echo "Directory hooks were executed with --no-directory-hooks!" >&2 + exit 1 +fi + +# The 4096 bit setting should persist to the first renewal, but be overridden in the second + +size1=`wc -c ${root}/conf/archive/le.wtf/privkey1.pem | cut -d" " -f1` +size2=`wc -c ${root}/conf/archive/le.wtf/privkey2.pem | cut -d" " -f1` +size3=`wc -c ${root}/conf/archive/le.wtf/privkey3.pem | cut -d" " -f1` +# 4096 bit PEM keys are about ~3270 bytes, 2048 ones are about 1700 bytes +if [ "$size1" -lt 3000 ] || [ "$size2" -lt 3000 ] || [ "$size3" -gt 1800 ] ; then + echo key sizes violate assumptions: + ls -l "${root}/conf/archive/le.wtf/privkey"* + exit 1 +fi + +# --renew-by-default is used, so renewal should occur +[ -f "$HOOK_TEST" ] && rm -f "$HOOK_TEST" +common renew +CheckCertCount "le.wtf" 4 +CheckHooks +CheckDirHooks 5 + +# test with overlapping directory hooks on the command line +common renew --cert-name le2.wtf \ + --pre-hook "$renewal_dir_pre_hook" \ + --deploy-hook "$renewal_dir_deploy_hook" \ + --post-hook "$renewal_dir_post_hook" +CheckDirHooks 1 + +# test with overlapping directory hooks in the renewal conf files +common renew --cert-name le2.wtf +CheckDirHooks 1 + +# manual-dns-auth.sh will skip completing the challenge for domains that begin +# with fail. +common -a manual -d dns1.le.wtf,fail.dns1.le.wtf \ + --allow-subset-of-names \ + --preferred-challenges dns,tls-sni \ + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh + +if common certificates | grep "fail\.dns1\.le\.wtf"; then + echo "certificate should not have been issued for domain!" >&2 + exit 1 +fi + +# reuse-key +common --domains reusekey.le.wtf --reuse-key +common renew --cert-name reusekey.le.wtf +CheckCertCount "reusekey.le.wtf" 2 +ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* +# The final awk command here exits successfully if its input consists of +# exactly two lines with identical first fields, and unsuccessfully otherwise. +sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 2 1 + +# don't reuse key (just by forcing reissuance without --reuse-key) +common --cert-name reusekey.le.wtf --domains reusekey.le.wtf --force-renewal +CheckCertCount "reusekey.le.wtf" 3 +ls -l "${root}/conf/archive/reusekey.le.wtf/privkey"* +# Exactly three lines, of which exactly two identical first fields. +sha256sum "${root}/conf/archive/reusekey.le.wtf/privkey"* | TotalAndDistinctLines 3 2 + +# Nonetheless, all three certificates are different even though two of them +# share the same subject key. +sha256sum "${root}/conf/archive/reusekey.le.wtf/cert"* | TotalAndDistinctLines 3 3 + +# ECDSA +openssl ecparam -genkey -name secp384r1 -out "${root}/privkey-p384.pem" +SAN="DNS:ecdsa.le.wtf" openssl req -new -sha256 \ + -config "${OPENSSL_CNF:-openssl.cnf}" \ + -key "${root}/privkey-p384.pem" \ + -subj "/" \ + -reqexts san \ + -outform der \ + -out "${root}/csr-p384.der" +common auth --csr "${root}/csr-p384.der" \ + --cert-path "${root}/csr/cert-p384.pem" \ + --chain-path "${root}/csr/chain-p384.pem" +openssl x509 -in "${root}/csr/cert-p384.pem" -text | grep 'ASN1 OID: secp384r1' + +# OCSP Must Staple +common auth --must-staple --domains "must-staple.le.wtf" +openssl x509 -in "${root}/conf/live/must-staple.le.wtf/cert.pem" -text | grep '1.3.6.1.5.5.7.1.24' + +# revoke by account key +common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" --delete-after-revoke +# revoke renewed +common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" --no-delete-after-revoke +if [ ! -d "$root/conf/live/le1.wtf" ]; then + echo "cert deleted when --no-delete-after-revoke was used!" + exit 1 +fi +common delete --cert-name le1.wtf +# revoke by cert key +common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ + --key-path "$root/conf/live/le2.wtf/privkey.pem" + +# Get new certs to test revoke with a reason, by account and by cert key +common --domains le1.wtf +common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" \ + --reason cessationOfOperation +common --domains le2.wtf +common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ + --key-path "$root/conf/live/le2.wtf/privkey.pem" \ + --reason keyCompromise + +common unregister + +out=$(common certificates) +subdomains="le dns.le newname.le must-staple.le" +for subdomain in $subdomains; do + domain="$subdomain.wtf" + if ! echo $out | grep "$domain"; then + echo "$domain not in certificates output!" + exit 1; + fi +done + +# Testing that revocation also deletes by default +subdomains="le1 le2" +for subdomain in $subdomains; do + domain="$subdomain.wtf" + if echo $out | grep "$domain"; then + echo "Revoked $domain in certificates output! Should not be!" + exit 1; + fi +done + +# Test that revocation raises correct error when both --cert-name and --cert-path specified +common --domains le1.wtf +out=$(common revoke --cert-path "$root/conf/live/le1.wtf/fullchain.pem" --cert-name "le1.wtf" 2>&1) || true +if ! echo $out | grep "Exactly one of --cert-path or --cert-name must be specified"; then + echo "Non-interactive revoking with both --cert-name and --cert-path " + echo "did not raise the correct error!" + exit 1 +fi + +# Test that revocation doesn't delete if multiple lineages share an archive dir +common --domains le1.wtf +common --domains le2.wtf +sed -i "s|^archive_dir = .*$|archive_dir = $root/conf/archive/le1.wtf|" "$root/conf/renewal/le2.wtf.conf" +#common update_symlinks # not needed, but a bit more context for what this test is about +out=$(common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem") +if ! echo $out | grep "Not deleting revoked certs due to overlapping archive dirs"; then + echo "Deleted a cert that had an overlapping archive dir with another lineage!" + exit 1 +fi + +cert_name="must-staple.le.wtf" +common delete --cert-name $cert_name +archive="$root/conf/archive/$cert_name" +conf="$root/conf/renewal/$cert_name.conf" +live="$root/conf/live/$cert_name" +for path in $archive $conf $live; do + if [ -e $path ]; then + echo "Lineage not properly deleted!" + exit 1 + fi +done + +# Test ACMEv2-only features +if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then + common -a manual -d '*.le4.wtf,le4.wtf' --preferred-challenges dns \ + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh +fi + +coverage report --fail-under 64 --include 'certbot/*' --show-missing diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 0ae9636d4..320328d9e 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -103,13 +103,32 @@ LOGDIR = "" #points to logging / working directory # boto3/AWS api globals AWS_SESSION = None EC2 = None +SECURITY_GROUP_NAME = 'certbot-security-group' +SUBNET_NAME = 'certbot-subnet' # Boto3/AWS automation functions #------------------------------------------------------------------------------- -def make_security_group(): +def should_use_subnet(subnet): + """Should we use the given subnet for these tests? + + We should if it is the default subnet for the availability zone or the + subnet is named "certbot-subnet". + + """ + if not subnet.map_public_ip_on_launch: + return False + if subnet.default_for_az: + return True + for tag in subnet.tags: + if tag['Key'] == 'Name' and tag['Value'] == SUBNET_NAME: + return True + return False + +def make_security_group(vpc): + """Creates a security group in the given VPC.""" # will fail if security group of GroupName already exists # cannot have duplicate SGs of the same name - mysg = EC2.create_security_group(GroupName="letsencrypt_test", + mysg = vpc.create_security_group(GroupName=SECURITY_GROUP_NAME, Description='security group for automated testing') mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=22, ToPort=22) mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=80, ToPort=80) @@ -123,14 +142,16 @@ def make_security_group(): def make_instance(instance_name, ami_id, keyname, + security_group_id, + subnet_id, machine_type='t2.micro', - security_groups=['letsencrypt_test'], userdata=""): #userdata contains bash or cloud-init script new_instance = EC2.create_instances( BlockDeviceMappings=_get_block_device_mappings(ami_id), ImageId=ami_id, - SecurityGroups=security_groups, + SecurityGroupIds=[security_group_id], + SubnetId=subnet_id, KeyName=keyname, MinCount=1, MaxCount=1, @@ -294,7 +315,7 @@ def grab_certbot_log(): sudo('if [ -f ./certbot.log ]; then \ cat ./certbot.log; else echo "[nolocallog]"; fi') -def create_client_instances(targetlist): +def create_client_instances(targetlist, security_group_id, subnet_id): "Create a fleet of client instances" instances = [] print("Creating instances: ", end="") @@ -314,6 +335,8 @@ def create_client_instances(targetlist): target['ami'], KEYNAME, machine_type=machine_type, + security_group_id=security_group_id, + subnet_id=subnet_id, userdata=userdata)) print() return instances @@ -418,14 +441,28 @@ print("Connecting to EC2 using\n profile %s\n keyname %s\n keyfile %s"%(PROFILE, AWS_SESSION = boto3.session.Session(profile_name=PROFILE) EC2 = AWS_SESSION.resource('ec2') +print("Determining Subnet") +for subnet in EC2.subnets.all(): + if should_use_subnet(subnet): + subnet_id = subnet.id + vpc_id = subnet.vpc.id + break +else: + print("No usable subnet exists!") + print("Please create a VPC with a subnet named {0}".format(SUBNET_NAME)) + print("that maps public IPv4 addresses to instances launched in the subnet.") + sys.exit(1) + print("Making Security Group") +vpc = EC2.Vpc(vpc_id) sg_exists = False -for sg in EC2.security_groups.all(): - if sg.group_name == 'letsencrypt_test': +for sg in vpc.security_groups.all(): + if sg.group_name == SECURITY_GROUP_NAME: + security_group_id = sg.id sg_exists = True - print(" %s already exists"%'letsencrypt_test') + print(" %s already exists"%SECURITY_GROUP_NAME) if not sg_exists: - make_security_group() + security_group_id = make_security_group(vpc).id time.sleep(30) boulder_preexists = False @@ -446,11 +483,12 @@ else: KEYNAME, machine_type='t2.micro', #machine_type='t2.medium', - security_groups=['letsencrypt_test']) + security_group_id=security_group_id, + subnet_id=subnet_id) try: if not cl_args.boulderonly: - instances = create_client_instances(targetlist) + instances = create_client_instances(targetlist, security_group_id, subnet_id) # Configure and launch boulder server #------------------------------------------------------------------------------- diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 766b4ea09..57ce4811a 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -48,13 +48,13 @@ targets: # CentOS # These Marketplace AMIs must, irritatingly, have their terms manually # agreed to on the AWS marketplace site for any new AWS account using them... - - ami: ami-61bbf104 + - ami: ami-9887c6e7 name: centos7 type: centos virt: hvm user: centos # centos6 requires EPEL repo added - - ami: ami-57cd8732 + - ami: ami-1585c46a name: centos6 type: centos virt: hvm diff --git a/tools/_release.sh b/tools/_release.sh new file mode 100755 index 000000000..dab4eec3a --- /dev/null +++ b/tools/_release.sh @@ -0,0 +1,249 @@ +#!/bin/bash -xe +# Release packages to PyPI + +if [ "$RELEASE_DIR" = "" ]; then + echo Please run this script through the tools/release.sh wrapper script or set the environment + echo variable RELEASE_DIR to the directory where the release should be built. + exit 1 +fi + +version="$1" +echo Releasing production version "$version"... +nextversion="$2" +RELEASE_BRANCH="candidate-$version" + +if [ "$RELEASE_OPENSSL_PUBKEY" = "" ] ; then + RELEASE_OPENSSL_PUBKEY="`realpath \`dirname $0\``/eff-pubkey.pem" +fi +RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-A2CFB51FA275A7286234E7B24D17C995CD9775F2} +# Needed to fix problems with git signatures and pinentry +export GPG_TTY=$(tty) + +# port for a local Python Package Index (used in testing) +PORT=${PORT:-1234} + +# subpackages to be released (the way developers think about them) +SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" +SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" + +# subpackages to be released (the way the script thinks about them) +SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" +SUBPKGS_NO_CERTBOT="$SUBPKGS_IN_AUTO_NO_CERTBOT $SUBPKGS_NOT_IN_AUTO" +SUBPKGS="$SUBPKGS_IN_AUTO $SUBPKGS_NOT_IN_AUTO" +subpkgs_modules="$(echo $SUBPKGS | sed s/-/_/g)" +# certbot_compatibility_test is not packaged because: +# - it is not meant to be used by anyone else than Certbot devs +# - it causes problems when running pytest - the latter tries to +# run everything that matches test*, while there are no unittests +# there + +tag="v$version" +mv "dist.$version" "dist.$version.$(date +%s).bak" || true +git tag --delete "$tag" || true + +tmpvenv=$(mktemp -d) +virtualenv --no-site-packages -p python2 $tmpvenv +. $tmpvenv/bin/activate +# update setuptools/pip just like in other places in the repo +pip install -U setuptools +pip install -U pip # latest pip => no --pre for dev releases +pip install -U wheel # setup.py bdist_wheel + +# newer versions of virtualenv inherit setuptools/pip/wheel versions +# from current env when creating a child env +pip install -U virtualenv + +root_without_le="$version.$$" +root="$RELEASE_DIR/le.$root_without_le" + +echo "Cloning into fresh copy at $root" # clean repo = no artifacts +git clone . $root +git rev-parse HEAD +cd $root +if [ "$RELEASE_BRANCH" != "candidate-$version" ] ; then + git branch -f "$RELEASE_BRANCH" +fi +git checkout "$RELEASE_BRANCH" + +for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test . +do + sed -i 's/\.dev0//' "$pkg_dir/setup.py" + git add "$pkg_dir/setup.py" +done + +SetVersion() { + ver="$1" + # bumping Certbot's version number is done differently + for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test + do + setup_file="$pkg_dir/setup.py" + if [ $(grep -c '^version' "$setup_file") != 1 ]; then + echo "Unexpected count of version variables in $setup_file" + exit 1 + fi + sed -i "s/^version.*/version = '$ver'/" $pkg_dir/setup.py + done + init_file="certbot/__init__.py" + if [ $(grep -c '^__version' "$init_file") != 1 ]; then + echo "Unexpected count of __version variables in $init_file" + exit 1 + fi + sed -i "s/^__version.*/__version__ = '$ver'/" "$init_file" + + git add $SUBPKGS certbot-compatibility-test +} + +SetVersion "$version" + +echo "Preparing sdists and wheels" +for pkg_dir in . $SUBPKGS_NO_CERTBOT +do + cd $pkg_dir + + python setup.py clean + rm -rf build dist + python setup.py sdist + python setup.py bdist_wheel + + echo "Signing ($pkg_dir)" + for x in dist/*.tar.gz dist/*.whl + do + gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 $x + done + + cd - +done + + +mkdir "dist.$version" +mv dist "dist.$version/certbot" +for pkg_dir in $SUBPKGS_NO_CERTBOT +do + mv $pkg_dir/dist "dist.$version/$pkg_dir/" +done + +echo "Testing packages" +cd "dist.$version" +# start local PyPI +python -m SimpleHTTPServer $PORT & +# cd .. is NOT done on purpose: we make sure that all subpackages are +# installed from local PyPI rather than current directory (repo root) +virtualenv --no-site-packages ../venv +. ../venv/bin/activate +pip install -U setuptools +pip install -U pip +# Now, use our local PyPI. Disable cache so we get the correct KGS even if we +# (or our dependencies) have conditional dependencies implemented with if +# statements in setup.py and we have cached wheels lying around that would +# cause those ifs to not be evaluated. +pip install \ + --no-cache-dir \ + --extra-index-url http://localhost:$PORT \ + $SUBPKGS +# stop local PyPI +kill $! +cd ~- + +# get a snapshot of the CLI help for the docs +# We set CERTBOT_DOCS to use dummy values in example user-agent string. +CERTBOT_DOCS=1 certbot --help all > docs/cli-help.txt +jws --help > acme/docs/jws-help.txt + +cd .. +# freeze before installing anything else, so that we know end-user KGS +# make sure "twine upload" doesn't catch "kgs" +if [ -d kgs ] ; then + echo Deleting old kgs... + rm -rf kgs +fi +mkdir kgs +kgs="kgs/$version" +pip freeze | tee $kgs +pip install pytest +for module in $subpkgs_modules ; do + echo testing $module + pytest --pyargs $module +done +cd ~- + +# pin pip hashes of the things we just built +for pkg in $SUBPKGS_IN_AUTO ; do + echo $pkg==$version \\ + pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python2 -c 'from sys import stdin; input = stdin.read(); print " ", input.replace("\n--hash", " \\\n --hash"),' +done > letsencrypt-auto-source/pieces/certbot-requirements.txt +deactivate + +# there should be one requirement specifier and two hashes for each subpackage +expected_count=$(expr $(echo $SUBPKGS_IN_AUTO | wc -w) \* 3) +if ! wc -l letsencrypt-auto-source/pieces/certbot-requirements.txt | grep -qE "^\s*$expected_count " ; then + echo Unexpected pip hash output + exit 1 +fi + +# ensure we have the latest built version of leauto +letsencrypt-auto-source/build.py + +# and that it's signed correctly +tools/offline-sigrequest.sh || true +while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ + letsencrypt-auto-source/letsencrypt-auto.sig \ + letsencrypt-auto-source/letsencrypt-auto ; do + echo "The signature on letsencrypt-auto is not correct." + read -p "Would you like this script to try and sign it again [Y/n]?" response + case $response in + [yY][eE][sS]|[yY]|"") + tools/offline-sigrequest.sh || true;; + *) + ;; + esac +done + +# This signature is not quite as strong, but easier for people to verify out of band +while ! gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 letsencrypt-auto-source/letsencrypt-auto; do + echo "Unable to sign letsencrypt-auto using $RELEASE_KEY." + echo "Make sure your OpenPGP card is in your computer if you are using one." + echo "You may need to take the card out and put it back in again." + read -p "Press enter to try signing again." +done +# We can't rename the openssl letsencrypt-auto.sig for compatibility reasons, +# but we can use the right name for certbot-auto.asc from day one +mv letsencrypt-auto-source/letsencrypt-auto.asc letsencrypt-auto-source/certbot-auto.asc + +# copy leauto to the root, overwriting the previous release version +cp -p letsencrypt-auto-source/letsencrypt-auto certbot-auto +cp -p letsencrypt-auto-source/letsencrypt-auto letsencrypt-auto + +git add certbot-auto letsencrypt-auto letsencrypt-auto-source docs/cli-help.txt +git diff --cached +while ! git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version"; do + echo "Unable to sign the release commit using git." + echo "You may have to configure git to use gpg2 by running:" + echo 'git config --global gpg.program $(command -v gpg2)' + read -p "Press enter to try signing again." +done +git tag --local-user "$RELEASE_GPG_KEY" --sign --message "Release $version" "$tag" + +cd .. +echo Now in $PWD +name=${root_without_le%.*} +ext="${root_without_le##*.}" +rev="$(git rev-parse --short HEAD)" +echo tar cJvf $name.$rev.tar.xz $name.$rev +echo gpg2 -U $RELEASE_GPG_KEY --detach-sign --armor $name.$rev.tar.xz +cd ~- + +echo "New root: $root" +echo "Test commands (in the letstest repo):" +echo 'python multitester.py targets.yaml $AWS_KEY $USERNAME scripts/test_leauto_upgrades.sh --alt_pip $YOUR_PIP_REPO --branch public-beta' +echo 'python multitester.py targets.yaml $AWK_KEY $USERNAME scripts/test_letsencrypt_auto_certonly_standalone.sh --branch candidate-0.1.1' +echo 'python multitester.py --saveinstances targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh' +echo "In order to upload packages run the following command:" +echo twine upload "$root/dist.$version/*/*" + +if [ "$RELEASE_BRANCH" = candidate-"$version" ] ; then + SetVersion "$nextversion".dev0 + letsencrypt-auto-source/build.py + git add letsencrypt-auto-source/letsencrypt-auto + git diff + git commit -m "Bump version to $nextversion" +fi diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 5a16b8cba..00ecee03e 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -12,7 +12,7 @@ botocore==1.7.41 cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 -dns-lexicon==2.2.1 +dns-lexicon==2.7.3 dnspython==1.15.0 docutils==0.14 execnet==1.5.0 @@ -48,7 +48,7 @@ pylint==1.4.2 pytest==3.2.5 pytest-cov==2.5.1 pytest-forked==0.2 -pytest-xdist==1.20.1 +pytest-xdist==1.22.5 python-dateutil==2.6.1 python-digitalocean==1.11 PyYAML==3.13 @@ -60,8 +60,9 @@ s3transfer==0.1.11 scandir==1.6 simplegeneric==0.8.1 snowballstemmer==1.2.1 -Sphinx==1.5.6 +Sphinx==1.7.5 sphinx-rtd-theme==0.2.4 +sphinxcontrib-websupport==1.0.1 tldextract==2.2.0 tox==2.9.1 tqdm==4.19.4 diff --git a/tools/release.sh b/tools/release.sh index 880563b4b..ae3e78dc1 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -1,11 +1,5 @@ -#!/bin/bash -xe -# Release dev packages to PyPI - -Usage() { - echo Usage: - echo "$0 [ --production ]" - exit 1 -} +#!/bin/bash -e +# Release packages to PyPI if [ "`dirname $0`" != "tools" ] ; then echo Please run this script from the repo root @@ -13,235 +7,38 @@ if [ "`dirname $0`" != "tools" ] ; then fi CheckVersion() { - # Args: - if ! echo "$2" | grep -q -e '[0-9]\+.[0-9]\+.[0-9]\+' ; then + # Args: + if ! echo "$1" | grep -q -e '[0-9]\+.[0-9]\+.[0-9]\+' ; then echo "$1 doesn't look like 1.2.3" + echo "Usage:" + echo "$0 RELEASE_VERSION NEXT_VERSION" exit 1 fi } -if [ "$1" = "--production" ] ; then - version="$2" - CheckVersion Version "$version" - echo Releasing production version "$version"... - nextversion="$3" - CheckVersion "Next version" "$nextversion" - RELEASE_BRANCH="candidate-$version" -else - version=`grep "__version__" certbot/__init__.py | cut -d\' -f2 | sed s/\.dev0//` - version="$version.dev$(date +%Y%m%d)1" - RELEASE_BRANCH="dev-release" - echo Releasing developer version "$version"... -fi +CheckVersion "$1" +CheckVersion "$2" -if [ "$RELEASE_OPENSSL_PUBKEY" = "" ] ; then - RELEASE_OPENSSL_PUBKEY="`realpath \`dirname $0\``/eff-pubkey.pem" -fi -RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-A2CFB51FA275A7286234E7B24D17C995CD9775F2} -# Needed to fix problems with git signatures and pinentry -export GPG_TTY=$(tty) - -# port for a local Python Package Index (used in testing) -PORT=${PORT:-1234} - -# subpackages to be released (the way developers think about them) -SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" -SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" - -# subpackages to be released (the way the script thinks about them) -SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" -SUBPKGS_NO_CERTBOT="$SUBPKGS_IN_AUTO_NO_CERTBOT $SUBPKGS_NOT_IN_AUTO" -SUBPKGS="$SUBPKGS_IN_AUTO $SUBPKGS_NOT_IN_AUTO" -subpkgs_modules="$(echo $SUBPKGS | sed s/-/_/g)" -# certbot_compatibility_test is not packaged because: -# - it is not meant to be used by anyone else than Certbot devs -# - it causes problems when running pytest - the latter tries to -# run everything that matches test*, while there are no unittests -# there - -tag="v$version" -mv "dist.$version" "dist.$version.$(date +%s).bak" || true -git tag --delete "$tag" || true - -tmpvenv=$(mktemp -d) -virtualenv --no-site-packages -p python2 $tmpvenv -. $tmpvenv/bin/activate -# update setuptools/pip just like in other places in the repo -pip install -U setuptools -pip install -U pip # latest pip => no --pre for dev releases -pip install -U wheel # setup.py bdist_wheel - -# newer versions of virtualenv inherit setuptools/pip/wheel versions -# from current env when creating a child env -pip install -U virtualenv - -root_without_le="$version.$$" -root="./releases/le.$root_without_le" - -echo "Cloning into fresh copy at $root" # clean repo = no artifacts -git clone . $root -git rev-parse HEAD -cd $root -if [ "$RELEASE_BRANCH" != "candidate-$version" ] ; then - git branch -f "$RELEASE_BRANCH" -fi -git checkout "$RELEASE_BRANCH" - -for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test . -do - sed -i 's/\.dev0//' "$pkg_dir/setup.py" -done -# We only add Certbot's setup.py here because the other files are added in the -# call to SetVersion below. -git add -p setup.py - -SetVersion() { - ver="$1" - # bumping Certbot's version number is done differently - for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test - do - sed -i "s/^version.*/version = '$ver'/" $pkg_dir/setup.py - done - sed -i "s/^__version.*/__version__ = '$ver'/" certbot/__init__.py - - # interactive user input - git add -p $SUBPKGS certbot-compatibility-test - -} - -SetVersion "$version" - -echo "Preparing sdists and wheels" -for pkg_dir in . $SUBPKGS_NO_CERTBOT -do - cd $pkg_dir - - python setup.py clean - rm -rf build dist - python setup.py sdist - python setup.py bdist_wheel - - echo "Signing ($pkg_dir)" - for x in dist/*.tar.gz dist/*.whl - do - gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 $x - done - - cd - -done - - -mkdir "dist.$version" -mv dist "dist.$version/certbot" -for pkg_dir in $SUBPKGS_NO_CERTBOT -do - mv $pkg_dir/dist "dist.$version/$pkg_dir/" -done - -echo "Testing packages" -cd "dist.$version" -# start local PyPI -python -m SimpleHTTPServer $PORT & -# cd .. is NOT done on purpose: we make sure that all subpackages are -# installed from local PyPI rather than current directory (repo root) -virtualenv --no-site-packages ../venv -. ../venv/bin/activate -pip install -U setuptools -pip install -U pip -# Now, use our local PyPI. Disable cache so we get the correct KGS even if we -# (or our dependencies) have conditional dependencies implemented with if -# statements in setup.py and we have cached wheels lying around that would -# cause those ifs to not be evaluated. -pip install \ - --no-cache-dir \ - --extra-index-url http://localhost:$PORT \ - $SUBPKGS -# stop local PyPI -kill $! -cd ~- - -# get a snapshot of the CLI help for the docs -# We set CERTBOT_DOCS to use dummy values in example user-agent string. -CERTBOT_DOCS=1 certbot --help all > docs/cli-help.txt -jws --help > acme/docs/jws-help.txt - -cd .. -# freeze before installing anything else, so that we know end-user KGS -# make sure "twine upload" doesn't catch "kgs" -if [ -d kgs ] ; then - echo Deleting old kgs... - rm -rf kgs -fi -mkdir kgs -kgs="kgs/$version" -pip freeze | tee $kgs -pip install pytest -for module in $subpkgs_modules ; do - echo testing $module - pytest --pyargs $module -done -cd ~- - -# pin pip hashes of the things we just built -for pkg in $SUBPKGS_IN_AUTO ; do - echo $pkg==$version \\ - pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python2 -c 'from sys import stdin; input = stdin.read(); print " ", input.replace("\n--hash", " \\\n --hash"),' -done > letsencrypt-auto-source/pieces/certbot-requirements.txt -deactivate - -# there should be one requirement specifier and two hashes for each subpackage -expected_count=$(expr $(echo $SUBPKGS_IN_AUTO | wc -w) \* 3) -if ! wc -l letsencrypt-auto-source/pieces/certbot-requirements.txt | grep -qE "^\s*$expected_count " ; then - echo Unexpected pip hash output +if [ "$RELEASE_GPG_KEY" = "" ] && ! gpg2 --card-status >/dev/null 2>&1; then + echo OpenPGP card not found! + echo Please insert your PGP card and run this script again. exit 1 fi -# ensure we have the latest built version of leauto -letsencrypt-auto-source/build.py - -# and that it's signed correctly -while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ - letsencrypt-auto-source/letsencrypt-auto.sig \ - letsencrypt-auto-source/letsencrypt-auto ; do - read -p "Please correctly sign letsencrypt-auto with offline-signrequest.sh" -done - -# This signature is not quite as strong, but easier for people to verify out of band -gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 letsencrypt-auto-source/letsencrypt-auto -# We can't rename the openssl letsencrypt-auto.sig for compatibility reasons, -# but we can use the right name for certbot-auto.asc from day one -mv letsencrypt-auto-source/letsencrypt-auto.asc letsencrypt-auto-source/certbot-auto.asc - -# copy leauto to the root, overwriting the previous release version -cp -p letsencrypt-auto-source/letsencrypt-auto certbot-auto -cp -p letsencrypt-auto-source/letsencrypt-auto letsencrypt-auto - -git add certbot-auto letsencrypt-auto letsencrypt-auto-source docs/cli-help.txt -git diff --cached -git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" -git tag --local-user "$RELEASE_GPG_KEY" --sign --message "Release $version" "$tag" - -cd .. -echo Now in $PWD -name=${root_without_le%.*} -ext="${root_without_le##*.}" -rev="$(git rev-parse --short HEAD)" -echo tar cJvf $name.$rev.tar.xz $name.$rev -echo gpg2 -U $RELEASE_GPG_KEY --detach-sign --armor $name.$rev.tar.xz -cd ~- - -echo "New root: $root" -echo "Test commands (in the letstest repo):" -echo 'python multitester.py targets.yaml $AWS_KEY $USERNAME scripts/test_leauto_upgrades.sh --alt_pip $YOUR_PIP_REPO --branch public-beta' -echo 'python multitester.py targets.yaml $AWK_KEY $USERNAME scripts/test_letsencrypt_auto_certonly_standalone.sh --branch candidate-0.1.1' -echo 'python multitester.py --saveinstances targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh' -echo "In order to upload packages run the following command:" -echo twine upload "$root/dist.$version/*/*" - -if [ "$RELEASE_BRANCH" = candidate-"$version" ] ; then - SetVersion "$nextversion".dev0 - letsencrypt-auto-source/build.py - git add letsencrypt-auto-source/letsencrypt-auto - git diff - git commit -m "Bump version to $nextversion" +if ! command -v script >/dev/null 2>&1; then + echo The command script was not found. + echo Please install it. + exit 1 +fi + +export RELEASE_DIR="./releases" +mv "$RELEASE_DIR" "$RELEASE_DIR.$(date +%s).bak" || true +LOG_PATH="log" +mv "$LOG_PATH" "$LOG_PATH.$(date +%s).bak" || true + +# Work with both Linux and macOS versions of script +if script --help | grep -q -- '--command'; then + script --command "tools/_release.sh $1 $2" "$LOG_PATH" +else + script "$LOG_PATH" tools/_release.sh "$1" "$2" fi diff --git a/tox.cover.sh b/tox.cover.sh index 6440d1b48..c68e757de 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -6,9 +6,7 @@ # generate separate stats for each package. It should be removed once # those packages are moved to separate repo. # -# -e makes sure we fail fast and don't submit coveralls submit - -NUMPROCESSES=${NUMPROCESSES:=auto} +# -e makes sure we fail fast and don't submit to codecov if [ "xxx$1" = "xxx" ]; then pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_ovh certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" @@ -63,7 +61,7 @@ cover () { fi pkg_dir=$(echo "$1" | tr _ -) - pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses "$NUMPROCESSES" --pyargs "$1" + pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses "auto" --pyargs "$1" coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing } diff --git a/tox.ini b/tox.ini index 0676a0da4..9db06f78c 100644 --- a/tox.ini +++ b/tox.ini @@ -64,6 +64,7 @@ source_paths = tests/lock_test.py [testenv] +passenv = TRAVIS commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py @@ -121,7 +122,6 @@ commands = [testenv:cover] basepython = python2.7 -passenv = NUMPROCESSES commands = {[base]install_packages} ./tox.cover.sh @@ -166,7 +166,9 @@ commands = docker run --rm -it apache-compat -c apache.tar.gz -vvvv whitelist_externals = docker -passenv = DOCKER_* +passenv = + DOCKER_* + TRAVIS [testenv:nginx_compat] commands = @@ -175,7 +177,9 @@ commands = docker run --rm -it nginx-compat -c nginx.tar.gz -vv -aie whitelist_externals = docker -passenv = DOCKER_* +passenv = + DOCKER_* + TRAVIS [testenv:le_auto_precise] # At the moment, this tests under Python 2.7 only, as only that version is @@ -185,7 +189,9 @@ commands = docker run --rm -t -i lea whitelist_externals = docker -passenv = DOCKER_* +passenv = + DOCKER_* + TRAVIS [testenv:le_auto_trusty] # At the moment, this tests under Python 2.7 only, as only that version is @@ -198,6 +204,7 @@ whitelist_externals = docker passenv = DOCKER_* + TRAVIS TRAVIS_BRANCH [testenv:le_auto_wheezy]