From 6a929c689ef4b4e80b5e8c25e38b7c4fa2bca2c3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 8 Apr 2017 15:46:24 +0200 Subject: [PATCH] embrace y2038 issue to support 32bit platforms (cherry picked from commit de76a6b8218bf0f20aa31e92ca92edab37483f34) --- borg/helpers.py | 28 ++++++++++++++++++++++---- borg/testsuite/archiver.py | 9 ++------- borg/testsuite/helpers.py | 40 +++++++++++++++++++++++++------------- 3 files changed, 53 insertions(+), 24 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index cf69cd7dc..ed9d1fae5 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -744,10 +744,30 @@ def replace_placeholders(text): # Not too rarely, we get crappy timestamps from the fs, that overflow some computations. -# As they are crap anyway, nothing is lost if we just clamp them to the max valid value. -# msgpack can only pack uint64. datetime is limited to year 9999. -MAX_NS = 18446744073000000000 # less than 2**64 - 1 ns. also less than y9999. -MAX_S = MAX_NS // 1000000000 +# As they are crap anyway (valid filesystem timestamps always refer to the past up to +# the present, but never to the future), nothing is lost if we just clamp them to the +# maximum value we can support. +# As long as people are using borg on 32bit platforms to access borg archives, we must +# keep this value True. But we can expect that we can stop supporting 32bit platforms +# well before coming close to the year 2038, so this will never be a practical problem. +SUPPORT_32BIT_PLATFORMS = True # set this to False before y2038. + +if SUPPORT_32BIT_PLATFORMS: + # second timestamps will fit into a signed int32 (platform time_t limit). + # nanosecond timestamps thus will naturally fit into a signed int64. + # subtract last 48h to avoid any issues that could be caused by tz calculations. + # this is in the year 2038, so it is also less than y9999 (which is a datetime internal limit). + # msgpack can pack up to uint64. + MAX_S = 2**31-1 - 48*3600 + MAX_NS = MAX_S * 1000000000 +else: + # nanosecond timestamps will fit into a signed int64. + # subtract last 48h to avoid any issues that could be caused by tz calculations. + # this is in the year 2262, so it is also less than y9999 (which is a datetime internal limit). + # round down to 1e9 multiple, so MAX_NS corresponds precisely to a integer MAX_S. + # msgpack can pack up to uint64. + MAX_NS = (2**63-1 - 48*3600*1000000000) // 1000000000 * 1000000000 + MAX_S = MAX_NS // 1000000000 def safe_s(ts): diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 904f20cf2..9e92c0716 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -25,7 +25,7 @@ from ..archiver import Archiver from ..cache import Cache from ..crypto import bytes_to_long, num_aes_blocks from ..helpers import Manifest, PatternMatcher, parse_pattern, EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, bin_to_hex, \ - get_security_dir + get_security_dir, MAX_S from ..key import RepoKey, KeyfileKey, Passphrase, TAMRequiredError from ..keymanager import RepoIdMismatch, NotABorgKeyFile from ..remote import RemoteRepository, PathNotAllowed @@ -272,12 +272,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): """ # File self.create_regular_file('empty', size=0) - # next code line raises OverflowError on 32bit cpu (raspberry pi 2): - # 2600-01-01 > 2**64 ns - # os.utime('input/empty', (19880895600, 19880895600)) - # thus, we better test with something not that far in future: - # 2038-01-19 (1970 + 2^31 - 1 seconds) is the 32bit "deadline": - os.utime('input/empty', (2**31 - 1, 2**31 - 1)) + os.utime('input/empty', (MAX_S, MAX_S)) self.create_regular_file('file1', size=1024 * 80) self.create_regular_file('flagfile', size=1024) # Directory diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 1d22e4eae..a1571b70a 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -17,7 +17,7 @@ from ..helpers import Location, format_file_size, format_timedelta, format_line, StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \ ProgressIndicatorPercent, ProgressIndicatorEndless, parse_pattern, load_exclude_file, load_pattern_file, \ PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern, \ - Buffer, safe_ns, safe_s + Buffer, safe_ns, safe_s, SUPPORT_32BIT_PLATFORMS from . import BaseTestCase, FakeInputs @@ -1145,15 +1145,29 @@ def test_format_line_erroneous(): def test_safe_timestamps(): - # ns fit into uint64 - assert safe_ns(2 ** 64) < 2 ** 64 - assert safe_ns(-1) == 0 - # s are so that their ns conversion fits into uint64 - assert safe_s(2 ** 64) * 1000000000 < 2 ** 64 - assert safe_s(-1) == 0 - # datetime won't fall over its y10k problem - beyond_y10k = 2 ** 100 - with pytest.raises(OverflowError): - datetime.utcfromtimestamp(beyond_y10k) - assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2500, 12, 31) - assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2500, 12, 31) + if SUPPORT_32BIT_PLATFORMS: + # ns fit into int64 + assert safe_ns(2 ** 64) <= 2 ** 63 - 1 + assert safe_ns(-1) == 0 + # s fit into int32 + assert safe_s(2 ** 64) <= 2 ** 31 - 1 + assert safe_s(-1) == 0 + # datetime won't fall over its y10k problem + beyond_y10k = 2 ** 100 + with pytest.raises(OverflowError): + datetime.utcfromtimestamp(beyond_y10k) + assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2038, 1, 1) + assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2038, 1, 1) + else: + # ns fit into int64 + assert safe_ns(2 ** 64) <= 2 ** 63 - 1 + assert safe_ns(-1) == 0 + # s are so that their ns conversion fits into int64 + assert safe_s(2 ** 64) * 1000000000 <= 2 ** 63 - 1 + assert safe_s(-1) == 0 + # datetime won't fall over its y10k problem + beyond_y10k = 2 ** 100 + with pytest.raises(OverflowError): + datetime.utcfromtimestamp(beyond_y10k) + assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2262, 1, 1) + assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2262, 1, 1)