Merge pull request #2464 from ThomasWaldmann/limit-y2038-1.0

embrace y2038 issue to support 32bit platforms (1.0-maint)

borg/helpers.py

@@ -744,10 +744,30 @@ def replace_placeholders(text):
 
 
 # Not too rarely, we get crappy timestamps from the fs, that overflow some computations.
-# As they are crap anyway, nothing is lost if we just clamp them to the max valid value.
-# msgpack can only pack uint64. datetime is limited to year 9999.
-MAX_NS = 18446744073000000000  # less than 2**64 - 1 ns. also less than y9999.
-MAX_S = MAX_NS // 1000000000
+# As they are crap anyway (valid filesystem timestamps always refer to the past up to
+# the present, but never to the future), nothing is lost if we just clamp them to the
+# maximum value we can support.
+# As long as people are using borg on 32bit platforms to access borg archives, we must
+# keep this value True. But we can expect that we can stop supporting 32bit platforms
+# well before coming close to the year 2038, so this will never be a practical problem.
+SUPPORT_32BIT_PLATFORMS = True  # set this to False before y2038.
+
+if SUPPORT_32BIT_PLATFORMS:
+    # second timestamps will fit into a signed int32 (platform time_t limit).
+    # nanosecond timestamps thus will naturally fit into a signed int64.
+    # subtract last 48h to avoid any issues that could be caused by tz calculations.
+    # this is in the year 2038, so it is also less than y9999 (which is a datetime internal limit).
+    # msgpack can pack up to uint64.
+    MAX_S = 2**31-1 - 48*3600
+    MAX_NS = MAX_S * 1000000000
+else:
+    # nanosecond timestamps will fit into a signed int64.
+    # subtract last 48h to avoid any issues that could be caused by tz calculations.
+    # this is in the year 2262, so it is also less than y9999 (which is a datetime internal limit).
+    # round down to 1e9 multiple, so MAX_NS corresponds precisely to a integer MAX_S.
+    # msgpack can pack up to uint64.
+    MAX_NS = (2**63-1 - 48*3600*1000000000) // 1000000000 * 1000000000
+    MAX_S = MAX_NS // 1000000000
 
 
 def safe_s(ts):

borg/testsuite/archiver.py

@@ -25,7 +25,7 @@ from ..archiver import Archiver
 from ..cache import Cache
 from ..crypto import bytes_to_long, num_aes_blocks
 from ..helpers import Manifest, PatternMatcher, parse_pattern, EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, bin_to_hex, \
-    get_security_dir
+    get_security_dir, MAX_S
 from ..key import RepoKey, KeyfileKey, Passphrase, TAMRequiredError
 from ..keymanager import RepoIdMismatch, NotABorgKeyFile
 from ..remote import RemoteRepository, PathNotAllowed
@@ -272,12 +272,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         """
         # File
         self.create_regular_file('empty', size=0)
-        # next code line raises OverflowError on 32bit cpu (raspberry pi 2):
-        # 2600-01-01 > 2**64 ns
-        # os.utime('input/empty', (19880895600, 19880895600))
-        # thus, we better test with something not that far in future:
-        # 2038-01-19 (1970 + 2^31 - 1 seconds) is the 32bit "deadline":
-        os.utime('input/empty', (2**31 - 1, 2**31 - 1))
+        os.utime('input/empty', (MAX_S, MAX_S))
         self.create_regular_file('file1', size=1024 * 80)
         self.create_regular_file('flagfile', size=1024)
         # Directory

borg/testsuite/helpers.py

@@ -17,7 +17,7 @@ from ..helpers import Location, format_file_size, format_timedelta, format_line,
     StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \
     ProgressIndicatorPercent, ProgressIndicatorEndless, parse_pattern, load_exclude_file, load_pattern_file, \
     PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern, \
-    Buffer, safe_ns, safe_s
+    Buffer, safe_ns, safe_s, SUPPORT_32BIT_PLATFORMS
 
 from . import BaseTestCase, FakeInputs
 
@@ -1145,15 +1145,29 @@ def test_format_line_erroneous():
 
 
 def test_safe_timestamps():
-    # ns fit into uint64
-    assert safe_ns(2 ** 64) < 2 ** 64
-    assert safe_ns(-1) == 0
-    # s are so that their ns conversion fits into uint64
-    assert safe_s(2 ** 64) * 1000000000 < 2 ** 64
-    assert safe_s(-1) == 0
-    # datetime won't fall over its y10k problem
-    beyond_y10k = 2 ** 100
-    with pytest.raises(OverflowError):
-        datetime.utcfromtimestamp(beyond_y10k)
-    assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2500, 12, 31)
-    assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2500, 12, 31)
+    if SUPPORT_32BIT_PLATFORMS:
+        # ns fit into int64
+        assert safe_ns(2 ** 64) <= 2 ** 63 - 1
+        assert safe_ns(-1) == 0
+        # s fit into int32
+        assert safe_s(2 ** 64) <= 2 ** 31 - 1
+        assert safe_s(-1) == 0
+        # datetime won't fall over its y10k problem
+        beyond_y10k = 2 ** 100
+        with pytest.raises(OverflowError):
+            datetime.utcfromtimestamp(beyond_y10k)
+        assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2038, 1, 1)
+        assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2038, 1, 1)
+    else:
+        # ns fit into int64
+        assert safe_ns(2 ** 64) <= 2 ** 63 - 1
+        assert safe_ns(-1) == 0
+        # s are so that their ns conversion fits into int64
+        assert safe_s(2 ** 64) * 1000000000 <= 2 ** 63 - 1
+        assert safe_s(-1) == 0
+        # datetime won't fall over its y10k problem
+        beyond_y10k = 2 ** 100
+        with pytest.raises(OverflowError):
+            datetime.utcfromtimestamp(beyond_y10k)
+        assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2262, 1, 1)
+        assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2262, 1, 1)