Merge pull request #56120 from nextcloud/feat/snowflake-file-sequence

lib/composer/composer/autoload_classmap.php

@@ -2110,8 +2110,11 @@ return array(
     'OC\\Share\\Constants' => $baseDir . '/lib/private/Share/Constants.php',
     'OC\\Share\\Helper' => $baseDir . '/lib/private/Share/Helper.php',
     'OC\\Share\\Share' => $baseDir . '/lib/private/Share/Share.php',
+    'OC\\Snowflake\\APCuSequence' => $baseDir . '/lib/private/Snowflake/APCuSequence.php',
     'OC\\Snowflake\\Decoder' => $baseDir . '/lib/private/Snowflake/Decoder.php',
+    'OC\\Snowflake\\FileSequence' => $baseDir . '/lib/private/Snowflake/FileSequence.php',
     'OC\\Snowflake\\Generator' => $baseDir . '/lib/private/Snowflake/Generator.php',
+    'OC\\Snowflake\\ISequence' => $baseDir . '/lib/private/Snowflake/ISequence.php',
     'OC\\SpeechToText\\SpeechToTextManager' => $baseDir . '/lib/private/SpeechToText/SpeechToTextManager.php',
     'OC\\SpeechToText\\TranscriptionJob' => $baseDir . '/lib/private/SpeechToText/TranscriptionJob.php',
     'OC\\StreamImage' => $baseDir . '/lib/private/StreamImage.php',

lib/composer/composer/autoload_static.php

@@ -21,10 +21,6 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         array (
             'NCU\\' => 4,
         ),
-        'B' => 
-        array (
-            'Bamarni\\Composer\\Bin\\' => 21,
-        ),
     );
 
     public static $prefixDirsPsr4 = array (
@@ -44,10 +40,6 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         array (
             0 => __DIR__ . '/../../..' . '/lib/unstable',
         ),
-        'Bamarni\\Composer\\Bin\\' => 
-        array (
-            0 => __DIR__ . '/..' . '/bamarni/composer-bin-plugin/src',
-        ),
     );
 
     public static $fallbackDirsPsr4 = array (
@@ -2159,8 +2151,11 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\Share\\Constants' => __DIR__ . '/../../..' . '/lib/private/Share/Constants.php',
         'OC\\Share\\Helper' => __DIR__ . '/../../..' . '/lib/private/Share/Helper.php',
         'OC\\Share\\Share' => __DIR__ . '/../../..' . '/lib/private/Share/Share.php',
+        'OC\\Snowflake\\APCuSequence' => __DIR__ . '/../../..' . '/lib/private/Snowflake/APCuSequence.php',
         'OC\\Snowflake\\Decoder' => __DIR__ . '/../../..' . '/lib/private/Snowflake/Decoder.php',
+        'OC\\Snowflake\\FileSequence' => __DIR__ . '/../../..' . '/lib/private/Snowflake/FileSequence.php',
         'OC\\Snowflake\\Generator' => __DIR__ . '/../../..' . '/lib/private/Snowflake/Generator.php',
+        'OC\\Snowflake\\ISequence' => __DIR__ . '/../../..' . '/lib/private/Snowflake/ISequence.php',
         'OC\\SpeechToText\\SpeechToTextManager' => __DIR__ . '/../../..' . '/lib/private/SpeechToText/SpeechToTextManager.php',
         'OC\\SpeechToText\\TranscriptionJob' => __DIR__ . '/../../..' . '/lib/private/SpeechToText/TranscriptionJob.php',
         'OC\\StreamImage' => __DIR__ . '/../../..' . '/lib/private/StreamImage.php',

lib/private/Server.php

@@ -115,8 +115,11 @@ use OC\Settings\DeclarativeManager;
 use OC\SetupCheck\SetupCheckManager;
 use OC\Share20\ProviderFactory;
 use OC\Share20\ShareHelper;
+use OC\Snowflake\APCuSequence;
 use OC\Snowflake\Decoder;
+use OC\Snowflake\FileSequence;
 use OC\Snowflake\Generator;
+use OC\Snowflake\ISequence;
 use OC\SpeechToText\SpeechToTextManager;
 use OC\SystemTag\ManagerFactory as SystemTagManagerFactory;
 use OC\Talk\Broker;
@@ -1262,6 +1265,16 @@ class Server extends ServerContainer implements IServerContainer {
 		$this->registerAlias(ISignatureManager::class, SignatureManager::class);
 
 		$this->registerAlias(IGenerator::class, Generator::class);
+		$this->registerService(ISequence::class, function (ContainerInterface $c): ISequence {
+			if (PHP_SAPI !== 'cli') {
+				$sequence = $c->get(APCuSequence::class);
+				if ($sequence->isAvailable()) {
+					return $sequence;
+				}
+			}
+
+			return $c->get(FileSequence::class);
+		}, false);
 		$this->registerAlias(IDecoder::class, Decoder::class);
 
 		$this->connectDispatcher();

lib/private/Snowflake/APCuSequence.php

@@ -0,0 +1,36 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+namespace OC\Snowflake;
+
+use Override;
+
+class APCuSequence implements ISequence {
+	#[Override]
+	public function isAvailable(): bool {
+		return PHP_SAPI !== 'cli' && function_exists('apcu_enabled') && apcu_enabled();
+	}
+
+	#[Override]
+	public function nextId(int $serverId, int $seconds, int $milliseconds): int|false {
+		if ((int)apcu_cache_info(true)['creation_time'] === $seconds) {
+			// APCu cache was just started
+			// It means a sequence was maybe deleted
+			return false;
+		}
+
+		$key = 'seq:' . $seconds . ':' . $milliseconds;
+		$sequenceId = apcu_inc($key, success: $success, ttl: 1);
+		if ($success === true) {
+			return $sequenceId;
+		}
+
+		throw new \Exception('Failed to generate SnowflakeId with APCu');
+	}
+}

lib/private/Snowflake/FileSequence.php

@@ -0,0 +1,89 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+namespace OC\Snowflake;
+
+use OCP\ITempManager;
+use Override;
+
+class FileSequence implements ISequence {
+	/** Number of files to use */
+	private const NB_FILES = 20;
+	/** Lock filename format **/
+	private const LOCK_FILE_FORMAT = 'seq-%03d.lock';
+	/** Delete sequences after SEQUENCE_TTL seconds **/
+	private const SEQUENCE_TTL = 30;
+
+	private string $workDir;
+
+	public function __construct(
+		ITempManager $tempManager,
+	) {
+		$this->workDir = $tempManager->getTemporaryFolder('.snowflakes');
+	}
+
+	#[Override]
+	public function isAvailable(): bool {
+		return true;
+	}
+
+	#[Override]
+	public function nextId(int $serverId, int $seconds, int $milliseconds): int {
+		// Open lock file
+		$filePath = $this->getFilePath($milliseconds % self::NB_FILES);
+		$fp = fopen($filePath, 'c+');
+		if ($fp === false) {
+			throw new \Exception('Unable to open sequence ID file: ' . $filePath);
+		}
+		if (!flock($fp, LOCK_EX)) {
+			throw new \Exception('Unable to acquire lock on sequence ID file: ' . $filePath);
+		}
+
+		// Read content
+		$content = (string)fgets($fp);
+		$locks = $content === ''
+			? []
+			: json_decode($content, true, 3, JSON_THROW_ON_ERROR);
+
+		// Generate new ID
+		if (isset($locks[$seconds])) {
+			if (isset($locks[$seconds][$milliseconds])) {
+				++$locks[$seconds][$milliseconds];
+			} else {
+				$locks[$seconds][$milliseconds] = 0;
+			}
+		} else {
+			$locks[$seconds] = [
+				$milliseconds => 0
+			];
+		}
+
+		// Clean old sequence IDs
+		$cleanBefore = $seconds - self::SEQUENCE_TTL;
+		$locks = array_filter($locks, static function ($key) use ($cleanBefore) {
+			return $key >= $cleanBefore;
+		}, ARRAY_FILTER_USE_KEY);
+
+		// Write data
+		ftruncate($fp, 0);
+		$content = json_encode($locks, JSON_THROW_ON_ERROR);
+		rewind($fp);
+		fwrite($fp, $content);
+		fsync($fp);
+
+		// Release lock
+		fclose($fp);
+
+		return $locks[$seconds][$milliseconds];
+	}
+
+	private function getFilePath(int $fileId): string {
+		return $this->workDir . sprintf(self::LOCK_FILE_FORMAT, $fileId);
+	}
+}

lib/private/Snowflake/Generator.php

@@ -23,17 +23,18 @@ use Override;
 final class Generator implements IGenerator {
 	public function __construct(
 		private readonly ITimeFactory $timeFactory,
+		private readonly ISequence $sequenceGenerator,
 	) {
 	}
 
 	#[Override]
 	public function nextId(): string {
-		// Time related
+		// Relative time
 		[$seconds, $milliseconds] = $this->getCurrentTime();
 
 		$serverId = $this->getServerId() & 0x1FF; // Keep 9 bits
 		$isCli = (int)$this->isCli(); // 1 bit
-		$sequenceId = $this->getSequenceId($seconds, $milliseconds, $serverId); //  12 bits
+		$sequenceId = $this->sequenceGenerator->nextId($seconds, $milliseconds, $serverId); //  12 bits
 		if ($sequenceId > 0xFFF || $sequenceId === false) {
 			// Throttle a bit, wait for next millisecond
 			usleep(1000);
@@ -106,33 +107,4 @@ final class Generator implements IGenerator {
 	private function isCli(): bool {
 		return PHP_SAPI === 'cli';
 	}
-
-	/**
-	 * Generates sequence ID from APCu (general case) or random if APCu disabled or CLI
-	 *
-	 * @return int|false Sequence ID or false if APCu not ready
-	 * @throws \Exception if there is an error with APCu
-	 */
-	private function getSequenceId(int $seconds, int $milliseconds, int $serverId): int|false {
-		$key = 'seq:' . $seconds . ':' . $milliseconds;
-
-		// Use APCu as fastest local cache, but not shared between processes in CLI
-		if (!$this->isCli() && function_exists('apcu_enabled') && apcu_enabled()) {
-			if ((int)apcu_cache_info(true)['creation_time'] === $seconds) {
-				// APCu cache was just started
-				// It means a sequence was maybe deleted
-				return false;
-			}
-
-			$sequenceId = apcu_inc($key, success: $success, ttl: 1);
-			if ($success === true) {
-				return $sequenceId;
-			}
-
-			throw new \Exception('Failed to generate SnowflakeId with APCu');
-		}
-
-		// Otherwise, just return a random number
-		return random_int(0, 0xFFF - 1);
-	}
 }

lib/private/Snowflake/ISequence.php

@@ -0,0 +1,25 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+namespace OC\Snowflake;
+
+/**
+ * Generates sequence IDs
+ */
+interface ISequence {
+	/**
+	 * Check if generator is available
+	 */
+	public function isAvailable(): bool;
+
+	/**
+	 * Returns next sequence ID for current time and server
+	 */
+	public function nextId(int $serverId, int $seconds, int $milliseconds): int|false;
+}

tests/lib/Snowflake/APCuTest.php

@@ -0,0 +1,23 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace Test\Snowflake;
+
+use OC\Snowflake\APCuSequence;
+
+/**
+ * @package Test
+ */
+class APCuTest extends ISequenceBase {
+	private string $path;
+
+	public function setUp():void {
+		$this->sequence = new APCuSequence();
+	}
+}

tests/lib/Snowflake/FileSequenceTest.php

@@ -0,0 +1,35 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace Test\Snowflake;
+
+use OC\Snowflake\FileSequence;
+use OCP\ITempManager;
+
+/**
+ * @package Test
+ */
+class FileSequenceTest extends ISequenceBase {
+	private string $path;
+
+	public function setUp():void {
+		$tempManager = $this->createMock(ITempManager::class);
+		$this->path = uniqid(sys_get_temp_dir() . '/php_test_seq_', true);
+		mkdir($this->path);
+		$tempManager->method('getTemporaryFolder')->willReturn($this->path);
+		$this->sequence = new FileSequence($tempManager);
+	}
+
+	public function tearDown():void {
+		foreach (glob($this->path . '/*') as $file) {
+			unlink($file);
+		}
+		rmdir($this->path);
+	}
+}

tests/lib/Snowflake/GeneratorTest.php

@@ -12,9 +12,11 @@ namespace Test\Snowflake;
 use OC\AppFramework\Utility\TimeFactory;
 use OC\Snowflake\Decoder;
 use OC\Snowflake\Generator;
+use OC\Snowflake\ISequence;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\Snowflake\IGenerator;
 use PHPUnit\Framework\Attributes\DataProvider;
+use PHPUnit\Framework\MockObject\MockObject;
 use Test\TestCase;
 
 /**
@@ -22,12 +24,17 @@ use Test\TestCase;
  */
 class GeneratorTest extends TestCase {
 	private Decoder $decoder;
+	private ISequence&MockObject $sequence;
 
 	public function setUp():void {
 		$this->decoder = new Decoder();
+		$this->sequence = $this->createMock(ISequence::class);
+		$this->sequence->method('isAvailable')->willReturn(true);
+		$this->sequence->method('nextId')->willReturn(421);
 	}
+
 	public function testGenerator(): void {
-		$generator = new Generator(new TimeFactory());
+		$generator = new Generator(new TimeFactory(), $this->sequence);
 		$snowflakeId = $generator->nextId();
 		$data = $this->decoder->decode($generator->nextId());
 
@@ -53,7 +60,7 @@ class GeneratorTest extends TestCase {
 		$timeFactory = $this->createMock(ITimeFactory::class);
 		$timeFactory->method('now')->willReturn($dt);
 
-		$generator = new Generator($timeFactory);
+		$generator = new Generator($timeFactory, $this->sequence);
 		$data = $this->decoder->decode($generator->nextId());
 
 		$this->assertEquals($expectedSeconds, ($data['createdAt']->format('U') - IGenerator::TS_OFFSET));

tests/lib/Snowflake/ISequenceBase.php

@@ -0,0 +1,45 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace Test\Snowflake;
+
+use OC\Snowflake\ISequence;
+use Test\TestCase;
+
+/**
+ * @package Test
+ */
+abstract class ISequenceBase extends TestCase {
+	protected ISequence $sequence;
+
+	public function testGenerator(): void {
+		if (!$this->sequence->isAvailable()) {
+			$this->markTestSkipped('Sequence ID generator ' . get_class($this->sequence) . 'is’nt available. Skip');
+		}
+
+		$nb = 1000;
+		$ids = [];
+		$server = 42;
+		for ($i = 0; $i < $nb; ++$i) {
+			$time = explode('.', (string)microtime(true));
+			$seconds = (int)$time[0];
+			$milliseconds = str_pad(substr($time[1] ?? '0', 0, 3), 3, '0');
+			$id = $this->sequence->nextId($server, $seconds, (int)$milliseconds);
+			$ids[] = sprintf('%d_%s_%d', $seconds, $milliseconds, $id);
+			usleep(100);
+		}
+
+		// Is it unique?
+		$this->assertCount($nb, array_unique($ids));
+		// Is it sequential?
+		$sortedIds = $ids;
+		natsort($sortedIds);
+		$this->assertSame($sortedIds, $ids);
+	}
+}