Merge pull request #3023 from ThomasWaldmann/fix-2994-1.0

Fix #2994 (1.0-maint)

borg/archive.py

@@ -17,7 +17,7 @@ import time
 from io import BytesIO
 from . import xattr
 from .helpers import Error, uid2user, user2uid, gid2group, group2gid, bin_to_hex, \
-    parse_timestamp, to_localtime, format_time, format_timedelta, remove_surrogates, \
+    parse_timestamp, to_localtime, ISO_FORMAT, format_time, format_timedelta, remove_surrogates, \
     Manifest, Statistics, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, \
     ProgressIndicatorPercent, IntegrityError, set_ec, EXIT_WARNING, safe_ns
 from .platform import acl_get, acl_set
@@ -321,8 +321,8 @@ Number of files: {0.stats.nfiles}'''.format(
             'cmdline': sys.argv,
             'hostname': socket.gethostname(),
             'username': getuser(),
-            'time': start.isoformat(),
-            'time_end': end.isoformat(),
+            'time': start.strftime(ISO_FORMAT),
+            'time_end': end.strftime(ISO_FORMAT),
         })
         data = self.key.pack_and_authenticate_metadata(metadata, context=b'archive')
         self.id = self.key.id_hash(data)

borg/helpers.py

@@ -39,6 +39,12 @@ import msgpack.fallback
 
 import socket
 
+# never use datetime.isoformat(), it is evil. always use one of these:
+# datetime.strftime(ISO_FORMAT)  # output always includes .microseconds
+# datetime.strftime(ISO_FORMAT_NO_USECS)  # output never includes microseconds
+ISO_FORMAT_NO_USECS = '%Y-%m-%dT%H:%M:%S'
+ISO_FORMAT = ISO_FORMAT_NO_USECS + '.%f'
+
 # 20 MiB minus 41 bytes for a Repository header (because the "size" field in the Repository includes
 # the header, and the total size was set to 20 MiB).
 MAX_DATA_SIZE = 20971479
@@ -306,11 +312,11 @@ class Manifest:
             self.config[b'tam_required'] = True
         # self.timestamp needs to be strictly monotonically increasing. Clocks often are not set correctly
         if self.timestamp is None:
-            self.timestamp = datetime.utcnow().isoformat()
+            self.timestamp = datetime.utcnow().strftime(ISO_FORMAT)
         else:
-            prev_ts = datetime.strptime(self.timestamp, "%Y-%m-%dT%H:%M:%S.%f")
-            incremented = (prev_ts + timedelta(microseconds=1)).isoformat()
-            self.timestamp = max(incremented, datetime.utcnow().isoformat())
+            prev_ts = parse_timestamp(self.timestamp, tzinfo=None)
+            incremented = (prev_ts + timedelta(microseconds=1)).strftime(ISO_FORMAT)
+            self.timestamp = max(incremented, datetime.utcnow().strftime(ISO_FORMAT))
         # include checks for limits as enforced by limited unpacker (used by load())
         assert len(self.archives) <= MAX_ARCHIVES
         assert all(len(name) <= 255 for name in self.archives)
@@ -485,12 +491,13 @@ def to_localtime(ts):
     return datetime(*time.localtime((ts - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds())[:6])
 
 
-def parse_timestamp(timestamp):
+def parse_timestamp(timestamp, tzinfo=timezone.utc):
     """Parse a ISO 8601 timestamp string"""
-    if '.' in timestamp:  # microseconds might not be present
-        return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%f').replace(tzinfo=timezone.utc)
-    else:
-        return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc)
+    fmt = ISO_FORMAT if '.' in timestamp else ISO_FORMAT_NO_USECS
+    dt = datetime.strptime(timestamp, fmt)
+    if tzinfo is not None:
+        dt = dt.replace(tzinfo=tzinfo)
+    return dt
 
 
 def load_excludes(fh):

borg/repository.py

@@ -14,7 +14,7 @@ from .logger import create_logger
 logger = create_logger()
 
 from .helpers import Error, ErrorWithTraceback, IntegrityError, Location, ProgressIndicatorPercent, bin_to_hex
-from .helpers import LIST_SCAN_LIMIT, MAX_OBJECT_SIZE, MAX_DATA_SIZE
+from .helpers import LIST_SCAN_LIMIT, MAX_OBJECT_SIZE, MAX_DATA_SIZE, ISO_FORMAT
 from .hashindex import NSIndex
 from .locking import Lock, LockError, LockErrorT
 from .lrucache import LRUCache
@@ -273,7 +273,8 @@ class Repository:
                   os.path.join(self.path, 'index.%d' % transaction_id))
         if self.append_only:
             with open(os.path.join(self.path, 'transactions'), 'a') as log:
-                print('transaction %d, UTC time %s' % (transaction_id, datetime.utcnow().isoformat()), file=log)
+                print('transaction %d, UTC time %s' % (
+                      transaction_id, datetime.utcnow().strftime(ISO_FORMAT)), file=log)
         # Remove old indices
         current = '.%d' % transaction_id
         for name in os.listdir(self.path):

borg/testsuite/archiver.py

@@ -25,7 +25,7 @@ from ..archiver import Archiver
 from ..cache import Cache
 from ..crypto import bytes_to_long, num_aes_blocks
 from ..helpers import Manifest, PatternMatcher, parse_pattern, EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, bin_to_hex, \
-    get_security_dir, MAX_S, MandatoryFeatureUnsupported, Location
+    get_security_dir, MAX_S, MandatoryFeatureUnsupported, Location, ISO_FORMAT
 from ..key import RepoKey, KeyfileKey, Passphrase, TAMRequiredError
 from ..keymanager import RepoIdMismatch, NotABorgKeyFile
 from ..remote import RemoteRepository, PathNotAllowed
@@ -1766,7 +1766,7 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
                 'version': 1,
                 'archives': {},
                 'config': {},
-                'timestamp': (datetime.utcnow() + timedelta(days=1)).isoformat(),
+                'timestamp': (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT),
             })))
             repository.commit()
 
@@ -1778,7 +1778,7 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
             repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb({
                 'version': 1,
                 'archives': {},
-                'timestamp': (datetime.utcnow() + timedelta(days=1)).isoformat(),
+                'timestamp': (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT),
             })))
             repository.commit()