From 199f708b4f802485d4da4fe71bdda93e27188b17 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 7 Nov 2025 17:17:00 +0100 Subject: [PATCH] add granularity_sleep, fixes #9150 --- src/borg/testsuite/__init__.py | 43 ++++++++++++++++++- src/borg/testsuite/archiver/__init__.py | 4 +- .../testsuite/archiver/create_cmd_test.py | 18 ++++---- src/borg/testsuite/archiver/diff_cmd_test.py | 17 +++----- .../testsuite/archiver/extract_cmd_test.py | 5 +-- 5 files changed, 61 insertions(+), 26 deletions(-) diff --git a/src/borg/testsuite/__init__.py b/src/borg/testsuite/__init__.py index b117b32b0..008a5d8ef 100644 --- a/src/borg/testsuite/__init__.py +++ b/src/borg/testsuite/__init__.py @@ -11,6 +11,7 @@ import stat import sys import sysconfig import tempfile +import time import unittest # Note: this is used by borg.selftest, do not *require* pytest functionality here. @@ -21,7 +22,7 @@ except: # noqa from ..fuse_impl import llfuse, has_llfuse, has_pyfuse3 # NOQA from .. import platform -from ..platformflags import is_win32 +from ..platformflags import is_win32, is_darwin # Does this version of llfuse support ns precision? have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, "st_mtime_ns") if llfuse else False @@ -54,6 +55,46 @@ def same_ts_ns(ts_ns1, ts_ns2): return diff_ts <= diff_max +def granularity_sleep(*, ctime_quirk=False): + """Sleep long enough to overcome filesystem timestamp granularity and related platform quirks. + + Purpose + - Ensure that successive file operations land on different timestamp "ticks" across filesystems + and operating systems, so tests that compare mtime/ctime are reliable. + + Default rationale (ctime_quirk=False) + - macOS: Some volumes may still be HFS+ (1 s timestamp granularity). To be safe across APFS and HFS+, + sleep 1.0 s on Darwin. + - Windows/NTFS: Although NTFS stores timestamps with 100 ns units, actual updates can be delayed by + scheduling/metadata behavior. Sleep a short but noticeable amount (0.2 s). + - Linux/BSD and others: Modern filesystems (ext4, XFS, Btrfs, ZFS, UFS2, etc.) typically have + sub-second granularity; a small delay (0.02 s) is sufficient in practice. + + Windows ctime quirk (ctime_quirk=True) + - On Windows, ``stat().st_ctime`` is the file creation time, not "metadata change time" as on Unix. + - NTFS implements a feature called "file system tunneling" that preserves certain metadata — including + creation time — for short intervals when a file is deleted and a new file with the same name is + created in the same directory. The default tunneling window is about 15 seconds. + - Consequence: If a test deletes a file and quickly recreates it with the same name, the creation time + (st_ctime) may remain unchanged for up to ~15 s, causing flakiness when tests expect a changed ctime. + - When ``ctime_quirk=True`` this helper sleeps long enough on Windows (15.0 s) to exceed the tunneling + window so the new file receives a fresh creation time. On non-Windows platforms this flag has no + special effect beyond the normal, short sleep. + + Parameters + - ctime_quirk: bool (default False) + If True, apply the Windows NTFS tunneling workaround (15 s sleep on Windows). Ignored elsewhere. + """ + if is_darwin: + duration = 1.0 + elif is_win32: + duration = 0.2 if not ctime_quirk else 15.0 + else: + # Default for Linux/BSD and others with fine-grained timestamps + duration = 0.02 + time.sleep(duration) + + rejected_dotdot_paths = ( "..", "../", diff --git a/src/borg/testsuite/archiver/__init__.py b/src/borg/testsuite/archiver/__init__.py index 130c1344c..e140d829d 100644 --- a/src/borg/testsuite/archiver/__init__.py +++ b/src/borg/testsuite/archiver/__init__.py @@ -28,7 +28,7 @@ from ...remote import RemoteRepository from ...repository import Repository from .. import has_lchflags, has_mknod, is_utime_fully_supported, have_fuse_mtime_ns, st_mtime_ns_round, no_selinux from .. import changedir -from .. import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported +from .. import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, granularity_sleep from ..platform.platform_test import is_win32 from ...xattr import get_all @@ -249,7 +249,7 @@ def create_test_files(input_path, create_hardlinks=True): if e.errno not in (errno.EINVAL, errno.ENOSYS): raise have_root = False - time.sleep(1) # "empty" must have newer timestamp than other files + granularity_sleep() # "empty" must have newer timestamp than other files create_regular_file(input_path, "empty", size=0) return have_root diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py index 8e084a8ac..d4b0d1caf 100644 --- a/src/borg/testsuite/archiver/create_cmd_test.py +++ b/src/borg/testsuite/archiver/create_cmd_test.py @@ -6,7 +6,6 @@ import shutil import socket import stat import subprocess -import time import pytest @@ -14,7 +13,7 @@ from ... import platform from ...constants import * # NOQA from ...constants import zeros from ...manifest import Manifest -from ...platform import is_win32, is_darwin +from ...platform import is_win32 from ...repository import Repository from ...helpers import CommandError, BackupPermissionError from .. import has_lchflags, has_mknod @@ -27,6 +26,7 @@ from .. import ( is_birthtime_fully_supported, same_ts_ns, is_root, + granularity_sleep, ) from . import ( cmd, @@ -650,7 +650,7 @@ def test_file_status(archivers, request): clearly incomplete: only tests for the weird "unchanged" status for now""" archiver = request.getfixturevalue(archivers) create_regular_file(archiver.input_path, "file1", size=1024 * 80) - time.sleep(1) # file2 must have newer timestamps than file1 + granularity_sleep() # file2 must have newer timestamps than file1 create_regular_file(archiver.input_path, "file2", size=1024 * 80) cmd(archiver, "repo-create", RK_ENCRYPTION) output = cmd(archiver, "create", "--list", "test", "input") @@ -671,7 +671,7 @@ def test_file_status_cs_cache_mode(archivers, request): archiver = request.getfixturevalue(archivers) """test that a changed file with faked "previous" mtime still gets backed up in ctime,size cache_mode""" create_regular_file(archiver.input_path, "file1", contents=b"123") - time.sleep(1) # file2 must have newer timestamps than file1 + granularity_sleep() # file2 must have newer timestamps than file1 create_regular_file(archiver.input_path, "file2", size=10) cmd(archiver, "repo-create", RK_ENCRYPTION) cmd(archiver, "create", "test", "input", "--list", "--files-cache=ctime,size") @@ -688,7 +688,7 @@ def test_file_status_ms_cache_mode(archivers, request): """test that a chmod'ed file with no content changes does not get chunked again in mtime,size cache_mode""" archiver = request.getfixturevalue(archivers) create_regular_file(archiver.input_path, "file1", size=10) - time.sleep(1) # file2 must have newer timestamps than file1 + granularity_sleep() # file2 must have newer timestamps than file1 create_regular_file(archiver.input_path, "file2", size=10) cmd(archiver, "repo-create", RK_ENCRYPTION) cmd(archiver, "create", "--list", "--files-cache=mtime,size", "test", "input") @@ -704,7 +704,7 @@ def test_file_status_rc_cache_mode(archivers, request): """test that files get rechunked unconditionally in rechunk,ctime cache mode""" archiver = request.getfixturevalue(archivers) create_regular_file(archiver.input_path, "file1", size=10) - time.sleep(1) # file2 must have newer timestamps than file1 + granularity_sleep() # file2 must have newer timestamps than file1 create_regular_file(archiver.input_path, "file2", size=10) cmd(archiver, "repo-create", RK_ENCRYPTION) cmd(archiver, "create", "--list", "--files-cache=rechunk,ctime", "test", "input") @@ -717,7 +717,7 @@ def test_file_status_excluded(archivers, request): """test that excluded paths are listed""" archiver = request.getfixturevalue(archivers) create_regular_file(archiver.input_path, "file1", size=1024 * 80) - time.sleep(1) # file2 must have newer timestamps than file1 + granularity_sleep() # file2 must have newer timestamps than file1 create_regular_file(archiver.input_path, "file2", size=1024 * 80) if has_lchflags: create_regular_file(archiver.input_path, "file3", size=1024 * 80) @@ -760,7 +760,7 @@ def test_file_status_counters(archivers, request): assert result["Modified files"] == 0 # Archive a dir with two added files create_regular_file(archiver.input_path, "testfile1", contents=b"test1") - time.sleep(1.0 if is_darwin else 0.01) # testfile2 must have newer timestamps than testfile1 + granularity_sleep() # testfile2 must have newer timestamps than testfile1 create_regular_file(archiver.input_path, "testfile2", contents=b"test2") result = cmd(archiver, "create", "--stats", "test_archive", archiver.input_path) result = to_dict(result) @@ -800,7 +800,7 @@ def test_create_json(archivers, request): def test_create_topical(archivers, request): archiver = request.getfixturevalue(archivers) create_regular_file(archiver.input_path, "file1", size=1024 * 80) - time.sleep(1) # file2 must have newer timestamps than file1 + granularity_sleep() # file2 must have newer timestamps than file1 create_regular_file(archiver.input_path, "file2", size=1024 * 80) cmd(archiver, "repo-create", RK_ENCRYPTION) # no listing by default diff --git a/src/borg/testsuite/archiver/diff_cmd_test.py b/src/borg/testsuite/archiver/diff_cmd_test.py index 92f1af0fa..155d654b7 100644 --- a/src/borg/testsuite/archiver/diff_cmd_test.py +++ b/src/borg/testsuite/archiver/diff_cmd_test.py @@ -6,8 +6,8 @@ import time import pytest from ...constants import * # NOQA -from .. import are_symlinks_supported, are_hardlinks_supported -from ...platformflags import is_win32, is_darwin +from .. import are_symlinks_supported, are_hardlinks_supported, granularity_sleep +from ...platformflags import is_win32 from . import ( cmd, create_regular_file, @@ -54,7 +54,7 @@ def test_basic_functionality(archivers, request): create_regular_file(archiver.input_path, "file_replaced", contents=b"0" * 4096) os.unlink("input/file_removed") os.unlink("input/file_removed2") - time.sleep(1) # macOS HFS+ has a 1s timestamp granularity + granularity_sleep() Path("input/file_touched").touch() os.rmdir("input/dir_replaced_with_file") create_regular_file(archiver.input_path, "dir_replaced_with_file", size=8192) @@ -269,19 +269,14 @@ def test_time_diffs(archivers, request): cmd(archiver, "create", "archive1", "input") time.sleep(0.1) os.unlink("input/test_file") - if is_win32: - # Sleeping for 15s because Windows doesn't refresh ctime if file is deleted and recreated within 15 seconds. - time.sleep(15) - elif is_darwin: - time.sleep(1) # HFS has a 1s timestamp granularity + granularity_sleep(ctime_quirk=True) create_regular_file(archiver.input_path, "test_file", size=15) cmd(archiver, "create", "archive2", "input") output = cmd(archiver, "diff", "archive1", "archive2", "--format", "'{mtime}{ctime} {path}{NL}'") assert "mtime" in output assert "ctime" in output # Should show up on Windows as well since it is a new file. - if is_darwin: - time.sleep(1) # HFS has a 1s timestamp granularity + granularity_sleep() os.chmod("input/test_file", 0o777) cmd(archiver, "create", "archive3", "input") output = cmd(archiver, "diff", "archive2", "archive3", "--format", "'{mtime}{ctime} {path}{NL}'") @@ -395,7 +390,7 @@ def test_sort_by_all_keys_with_directions(archivers, request, sort_key): cmd(archiver, "create", "s0", "input") # Ensure that subsequent modifications happen on a later timestamp tick than s0 - time.sleep(1.0 if is_darwin else 0.1) # HFS+ has ~1s timestamp granularity on macOS + granularity_sleep() # Create differences for second archive os.unlink("input/a_removed") diff --git a/src/borg/testsuite/archiver/extract_cmd_test.py b/src/borg/testsuite/archiver/extract_cmd_test.py index 8726b087d..d9e2e8a00 100644 --- a/src/borg/testsuite/archiver/extract_cmd_test.py +++ b/src/borg/testsuite/archiver/extract_cmd_test.py @@ -1,7 +1,6 @@ import errno import os import shutil -import time import stat from unittest.mock import patch @@ -13,7 +12,7 @@ from ...chunkers import has_seek_hole from ...constants import * # NOQA from ...helpers import EXIT_WARNING, BackupPermissionError, bin_to_hex from ...helpers import flags_noatime, flags_normal -from .. import changedir, same_ts_ns +from .. import changedir, same_ts_ns, granularity_sleep from .. import are_symlinks_supported, are_hardlinks_supported, is_utime_fully_supported, is_birthtime_fully_supported from ...platform import get_birthtime_ns from ...platformflags import is_darwin, is_freebsd, is_win32 @@ -728,7 +727,7 @@ def test_extract_continue(archivers, request): # make a hard link, so it does not free the inode when unlinking input/file3 os.link("input/file3", "hardlink-to-keep-inode-f3") os.remove("input/file3") - time.sleep(1) # needed due to timestamp granularity of apple hfs+ + granularity_sleep() with changedir("output"): # now try to continue extracting, using the same archive, same output dir: