From 04340ae8b1d6ae89d90f804ca19c685746d446f9 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 15 Dec 2016 03:02:06 +0100 Subject: [PATCH 1/9] pytest: only rewrite the testsuite, fixes #1938 do not rewrite the borg application code, just the test code, so the bytecode tested is identical / very close to the bytecode used in practice. --- conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index d622e6df1..197af6fd1 100644 --- a/conftest.py +++ b/conftest.py @@ -5,7 +5,7 @@ import sys import pytest # needed to get pretty assertion failures in unit tests: -pytest.register_assert_rewrite('borg') +pytest.register_assert_rewrite('borg.testsuite') # This is a hack to fix path problems because "borg" (the package) is in the source root. # When importing the conftest an "import borg" can incorrectly import the borg from the From 6b6ddecd9301cbd3a657bb6d78e58554f2aa7882 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 17 Dec 2016 00:25:49 +0100 Subject: [PATCH 2/9] document windows 10 linux subsystem install also add note about remote repos being broken on cygwin. --- README.rst | 1 + docs/installation.rst | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 07e8b0d39..6655e77a6 100644 --- a/README.rst +++ b/README.rst @@ -75,6 +75,7 @@ Main features * FreeBSD * OpenBSD and NetBSD (no xattrs/ACLs support or binaries yet) * Cygwin (not supported, no binaries yet) + * Linux Subsystem of Windows 10 (not supported) **Free and Open Source Software** * security and functionality can be audited independently diff --git a/docs/installation.rst b/docs/installation.rst index d3bc73007..bcd5d4796 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -257,12 +257,21 @@ and commands to make fuse work for using the mount command. sysctl vfs.usermount=1 +Windows 10's Linux Subsystem +++++++++++++++++++++++++++++ + +.. note:: + Running under Windows 10's Linux Subsystem is experimental and has not been tested much yet. + +Just follow the Ubuntu Linux installation steps. You can omit the FUSE stuff, it won't work anyway. + + Cygwin ++++++ .. note:: Running under Cygwin is experimental and has only been tested with Cygwin - (x86-64) v2.5.2. + (x86-64) v2.5.2. Remote repositories are known broken, local repositories should work. Use the Cygwin installer to install the dependencies:: From 61370082d63af8abd2ad40ecf962df17b2476f66 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 17 Dec 2016 00:37:00 +0100 Subject: [PATCH 3/9] catch errno.ENOSYS for mknod (win 10 lxsys) mknod raises this when running as non-root under Windows 10's Linux Subsystem. --- borg/testsuite/archiver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 67b5e581b..815c8943a 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -311,7 +311,8 @@ class ArchiverTestCase(ArchiverTestCaseBase): except PermissionError: have_root = False except OSError as e: - if e.errno != errno.EINVAL: + # Note: ENOSYS "Function not implemented" happens as non-root on Win 10 Linux Subsystem. + if e.errno not in (errno.EINVAL, errno.ENOSYS): raise have_root = False return have_root From 420c984f05ff8bc1b16c3b4b661bdaebe285a421 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 30 Oct 2016 18:18:05 +0100 Subject: [PATCH 4/9] fix wrong duration if clock jumps during create --- borg/archive.py | 9 ++++++--- borg/archiver.py | 6 +++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index ebfa091f9..f93e0a8c9 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -1,5 +1,5 @@ from contextlib import contextmanager -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta from getpass import getuser from itertools import groupby import errno @@ -186,7 +186,7 @@ class Archive: def __init__(self, repository, key, manifest, name, cache=None, create=False, checkpoint_interval=300, numeric_owner=False, noatime=False, noctime=False, progress=False, - chunker_params=CHUNKER_PARAMS, start=None, end=None): + chunker_params=CHUNKER_PARAMS, start=None, start_monotonic=None, end=None): self.cwd = os.getcwd() self.key = key self.repository = repository @@ -200,9 +200,12 @@ class Archive: self.numeric_owner = numeric_owner self.noatime = noatime self.noctime = noctime + assert (start is None) == (start_monotonic is None), 'Logic error: if start is given, start_monotonic must be given as well and vice versa.' if start is None: start = datetime.utcnow() + start_monotonic = time.monotonic() self.start = start + self.start_monotonic = start_monotonic if end is None: end = datetime.utcnow() self.end = end @@ -302,7 +305,7 @@ Number of files: {0.stats.nfiles}'''.format( raise self.AlreadyExists(name) self.items_buffer.flush(flush=True) if timestamp is None: - self.end = datetime.utcnow() + self.end = self.start + timedelta(seconds=time.monotonic() - self.start_monotonic) start = self.start end = self.end else: diff --git a/borg/archiver.py b/borg/archiver.py index cb02d4d8a..5ee612972 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -5,7 +5,6 @@ from operator import attrgetter import argparse import functools import inspect -import io import os import re import shlex @@ -13,6 +12,7 @@ import signal import stat import sys import textwrap +import time import traceback import collections @@ -263,7 +263,6 @@ class Archiver: if args.progress: archive.stats.show_progress(final=True) if args.stats: - archive.end = datetime.utcnow() log_multi(DASHES, str(archive), DASHES, @@ -276,6 +275,7 @@ class Archiver: self.ignore_inode = args.ignore_inode dry_run = args.dry_run t0 = datetime.utcnow() + t0_monotonic = time.monotonic() if not dry_run: key.compressor = Compressor(**args.compression) with Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) as cache: @@ -283,7 +283,7 @@ class Archiver: create=True, checkpoint_interval=args.checkpoint_interval, numeric_owner=args.numeric_owner, noatime=args.noatime, noctime=args.noctime, progress=args.progress, - chunker_params=args.chunker_params, start=t0) + chunker_params=args.chunker_params, start=t0, start_monotonic=t0_monotonic) create_inner(archive, cache) else: create_inner(None, None) From 2dc558a02e24461f0cda9869cb49bcd73038a815 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 30 Oct 2016 18:18:23 +0100 Subject: [PATCH 5/9] fix create progress not updating if clock jumps --- borg/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/helpers.py b/borg/helpers.py index ff7d16075..a8106aeef 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -219,7 +219,7 @@ class Statistics: return format_file_size(self.csize) def show_progress(self, item=None, final=False, stream=None, dt=None): - now = time.time() + now = time.monotonic() if dt is None or now - self.last_progress > dt: self.last_progress = now columns, lines = get_terminal_size() From a8d921a54cbb9a5e7b86668bd04a050c23110ad4 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sat, 17 Dec 2016 11:37:56 +0100 Subject: [PATCH 6/9] base archive timestamps on end time The assumption is that if the clock jumps during the Borg run that it was jump-corrected and is now correct, while the start timestamp would be wrong. --- borg/archive.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borg/archive.py b/borg/archive.py index f93e0a8c9..5001d3108 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -305,7 +305,8 @@ Number of files: {0.stats.nfiles}'''.format( raise self.AlreadyExists(name) self.items_buffer.flush(flush=True) if timestamp is None: - self.end = self.start + timedelta(seconds=time.monotonic() - self.start_monotonic) + self.end = datetime.utcnow() + self.start = self.end - timedelta(seconds=time.monotonic() - self.start_monotonic) start = self.start end = self.end else: From f5d6093ccc0b41ea7465dfded9091b0d95d01403 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sat, 17 Dec 2016 11:46:20 +0100 Subject: [PATCH 7/9] fix checkpoints when clock jumps --- borg/archive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 5001d3108..64be0f4d8 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -215,7 +215,7 @@ class Archive: self.chunker = Chunker(self.key.chunk_seed, *chunker_params) if name in manifest.archives: raise self.AlreadyExists(name) - self.last_checkpoint = time.time() + self.last_checkpoint = time.monotonic() i = 0 while True: self.checkpoint_name = '%s.checkpoint%s' % (name, i and ('.%d' % i) or '') @@ -290,9 +290,9 @@ Number of files: {0.stats.nfiles}'''.format( if self.show_progress: self.stats.show_progress(item=item, dt=0.2) self.items_buffer.add(item) - if time.time() - self.last_checkpoint > self.checkpoint_interval: + if time.monotonic() - self.last_checkpoint > self.checkpoint_interval: self.write_checkpoint() - self.last_checkpoint = time.time() + self.last_checkpoint = time.monotonic() def write_checkpoint(self): self.save(self.checkpoint_name) From 445365b3ff19830aefe2c7b1332cba05703fcb97 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sat, 17 Dec 2016 12:00:25 +0100 Subject: [PATCH 8/9] update changes --- docs/changes.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index 4ff900e7d..71677bcb3 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -82,6 +82,9 @@ Bug fixes: - skip corrupted chunks during manifest rebuild - fix TypeError in integrity error handler, #1903, #1894 - fix location parser for archives with @ char (regression introduced in 1.0.8), #1930 +- fix wrong duration/timestamps if system clock jumped during a create +- fix progress display not updating if system clock jumps backwards +- fix checkpoint interval being incorrect if system clock jumps Other changes: From baa8baafdbd342bdaec0e39aa59fc160bea09e2c Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sat, 17 Dec 2016 12:55:16 +0100 Subject: [PATCH 9/9] create: fix duration if --timestamp is given --- borg/archive.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 64be0f4d8..641fd08dd 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -304,15 +304,17 @@ Number of files: {0.stats.nfiles}'''.format( if name in self.manifest.archives: raise self.AlreadyExists(name) self.items_buffer.flush(flush=True) + duration = timedelta(seconds=time.monotonic() - self.start_monotonic) if timestamp is None: self.end = datetime.utcnow() - self.start = self.end - timedelta(seconds=time.monotonic() - self.start_monotonic) + self.start = self.end - duration start = self.start end = self.end else: self.end = timestamp - start = timestamp - end = timestamp # we only have 1 value + self.start = timestamp - duration + end = timestamp + start = self.start metadata = StableDict({ 'version': 1, 'name': name,