diff --git a/lib/private/Files/Cache/FileAccess.php b/lib/private/Files/Cache/FileAccess.php index 11a95b5d897..c3f3614f3ca 100644 --- a/lib/private/Files/Cache/FileAccess.php +++ b/lib/private/Files/Cache/FileAccess.php @@ -10,6 +10,7 @@ namespace OC\Files\Cache; use OC\FilesMetadata\FilesMetadataManager; use OC\SystemConfig; +use OCP\DB\Exception; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Cache\IFileAccess; use OCP\Files\IMimeTypeLoader; @@ -94,4 +95,128 @@ class FileAccess implements IFileAccess { $rows = $query->executeQuery()->fetchAll(); return $this->rowsToEntries($rows); } + + public function getByAncestorInStorage(int $storageId, int $folderId, int $fileIdCursor = 0, int $maxResults = 100, array $mimeTypeIds = [], bool $endToEndEncrypted = true, bool $serverSideEncrypted = true): \Generator { + $qb = $this->getQuery(); + $qb->select('path') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($folderId, IQueryBuilder::PARAM_INT))); + $result = $qb->executeQuery(); + /** @var array{path:string}|false $root */ + $root = $result->fetch(); + $result->closeCursor(); + + if ($root === false) { + throw new Exception('Could not fetch storage root'); + } + + $qb = $this->getQuery(); + + $path = $root['path'] === '' ? '' : $root['path'] . '/'; + + $qb->selectDistinct('*') + ->from('filecache', 'f') + ->where($qb->expr()->like('f.path', $qb->createNamedParameter($this->connection->escapeLikeParameter($path) . '%'))) + ->andWhere($qb->expr()->eq('f.storage', $qb->createNamedParameter($storageId))) + ->andWhere($qb->expr()->gt('f.fileid', $qb->createNamedParameter($fileIdCursor, IQueryBuilder::PARAM_INT))); + + if (!$endToEndEncrypted) { + // End to end encrypted files are descendants of a folder with encrypted=1 + // Use a subquery to check the `encrypted` status of the parent folder + $subQuery = $this->getQuery()->select('p.encrypted') + ->from('filecache', 'p') + ->andWhere($qb->expr()->eq('p.fileid', 'f.parent')) + ->getSQL(); + + $qb->andWhere( + $qb->expr()->eq($qb->createFunction(sprintf('(%s)', $subQuery)), $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)) + ); + } + + if (!$serverSideEncrypted) { + // Server side encrypted files have encrypted=1 directly + $qb->andWhere($qb->expr()->eq('f.encrypted', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))); + } + + if (count($mimeTypeIds) > 0) { + $qb->andWhere($qb->expr()->in('f.mimetype', $qb->createNamedParameter($mimeTypeIds, IQueryBuilder::PARAM_INT_ARRAY))); + } + + if ($maxResults !== 0) { + $qb->setMaxResults($maxResults); + } + $qb->orderBy('f.fileid', 'ASC'); + $files = $qb->executeQuery(); + + while ( + /** @var array */ + $row = $files->fetch() + ) { + yield Cache::cacheEntryFromData($row, $this->mimeTypeLoader); + } + + $files->closeCursor(); + } + + public function getDistinctMounts(array $mountProviders = [], bool $onlyUserFilesMounts = true): \Generator { + $qb = $this->connection->getQueryBuilder(); + $qb->selectDistinct(['root_id', 'storage_id', 'mount_provider_class']) + ->from('mounts'); + if ($onlyUserFilesMounts) { + $qb->andWhere( + $qb->expr()->orX( + $qb->expr()->like('mount_point', $qb->createNamedParameter('/%/files/%')), + $qb->expr()->in('mount_provider_class', $qb->createNamedParameter([ + \OC\Files\Mount\LocalHomeMountProvider::class, + \OC\Files\Mount\ObjectHomeMountProvider::class, + ], IQueryBuilder::PARAM_STR_ARRAY)) + ) + ); + } + if (count($mountProviders) > 0) { + $qb->andWhere($qb->expr()->in('mount_provider_class', $qb->createNamedParameter($mountProviders, IQueryBuilder::PARAM_STR_ARRAY))); + } + $qb->orderBy('root_id', 'ASC'); + $result = $qb->executeQuery(); + + while ( + /** @var array{storage_id:int, root_id:int,mount_provider_class:string} $row */ + $row = $result->fetch() + ) { + $storageId = (int)$row['storage_id']; + $rootId = (int)$row['root_id']; + $overrideRoot = $rootId; + // LocalHomeMountProvider is the default provider for user home directories + // ObjectHomeMountProvider is the home directory provider for when S3 primary storage is used + if ($onlyUserFilesMounts && in_array($row['mount_provider_class'], [ + \OC\Files\Mount\LocalHomeMountProvider::class, + \OC\Files\Mount\ObjectHomeMountProvider::class, + ], true)) { + // Only crawl files, not cache or trashbin + $qb = $this->getQuery(); + try { + $qb->select('fileid') + ->from('filecache') + ->where($qb->expr()->eq('storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('parent', $qb->createNamedParameter($rootId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('path', $qb->createNamedParameter('files'))); + /** @var array|false $root */ + $root = $qb->executeQuery()->fetch(); + if ($root !== false) { + $overrideRoot = (int)$root['fileid']; + } + } catch (Exception $e) { + $this->logger->error('Could not fetch home storage files root for storage ' . $storageId, ['exception' => $e]); + continue; + } + } + // Reference to root_id is still necessary even if we have the overridden_root_id, because storage_id and root_id uniquely identify a mount + yield [ + 'storage_id' => $storageId, + 'root_id' => $rootId, + 'overridden_root' => $overrideRoot, + ]; + } + $result->closeCursor(); + } } diff --git a/lib/public/Files/Cache/IFileAccess.php b/lib/public/Files/Cache/IFileAccess.php index 51945b55a25..7a993d81e7a 100644 --- a/lib/public/Files/Cache/IFileAccess.php +++ b/lib/public/Files/Cache/IFileAccess.php @@ -79,4 +79,36 @@ interface IFileAccess { * @since 29.0.0 */ public function getByFileIdsInStorage(array $fileIds, int $storageId): array; + + /** + * Retrieves files stored in a specific storage that have a specified ancestor in the file hierarchy. + * Allows filtering by mime types, encryption status, and limits the number of results. + * + * @param int $storageId The ID of the storage to search within. + * @param int $folderId The file ID of the ancestor to base the search on. + * @param int $fileIdCursor The last processed file ID. Only files with a higher ID will be included. Defaults to 0. + * @param int $maxResults The maximum number of results to retrieve. If set to 0, all matching files will be retrieved. + * @param list $mimeTypeIds An array of mime types to filter the results. If empty, no mime type filtering will be applied. + * @param bool $endToEndEncrypted Whether to include EndToEndEncrypted files + * @param bool $serverSideEncrypted Whether to include ServerSideEncrypted files + * @return \Generator A generator yielding matching files as cache entries. + * @throws \OCP\DB\Exception + * + * @since 32.0.0 + */ + public function getByAncestorInStorage(int $storageId, int $folderId, int $fileIdCursor = 0, int $maxResults = 100, array $mimeTypeIds = [], bool $endToEndEncrypted = true, bool $serverSideEncrypted = true): \Generator; + + /** + * Retrieves a list of all distinct mounts. + * Allows filtering by specific mount providers. + * Optionally rewrites home directory root paths to avoid cache and trashbin. + * + * @param list $mountProviders An array of mount provider class names to filter. If empty, all providers will be included. + * @param bool $onlyUserFilesMounts Whether to rewrite the root IDs for home directories to only include user files and to only consider mounts with mount points in the user files. + * @return \Generator A generator yielding mount configurations as an array containing 'storage_id', 'root_id', and 'override_root'. + * @throws \OCP\DB\Exception + * + * @since 32.0.0 + */ + public function getDistinctMounts(array $mountProviders = [], bool $onlyUserFilesMounts = true): \Generator; } diff --git a/tests/lib/Files/Cache/FileAccessTest.php b/tests/lib/Files/Cache/FileAccessTest.php new file mode 100644 index 00000000000..59fa2494ea8 --- /dev/null +++ b/tests/lib/Files/Cache/FileAccessTest.php @@ -0,0 +1,438 @@ +dbConnection = \OCP\Server::get(IDBConnection::class); + + // Ensure FileAccess is instantiated with the real connection + $this->fileAccess = new FileAccess( + $this->dbConnection, + \OCP\Server::get(\OC\SystemConfig::class), + \OCP\Server::get(LoggerInterface::class), + \OCP\Server::get(\OC\FilesMetadata\FilesMetadataManager::class), + \OCP\Server::get(\OCP\Files\IMimeTypeLoader::class) + ); + + // Clear and prepare `filecache` table for tests + $queryBuilder = $this->dbConnection->getQueryBuilder()->runAcrossAllShards(); + $queryBuilder->delete('filecache')->executeStatement(); + + // Clean up potential leftovers from other tests + $queryBuilder = $this->dbConnection->getQueryBuilder(); + $queryBuilder->delete('mounts')->executeStatement(); + + + $this->setUpTestDatabaseForGetDistinctMounts(); + $this->setUpTestDatabaseForGetByAncestorInStorage(); + } + + private function setUpTestDatabaseForGetDistinctMounts(): void { + $queryBuilder = $this->dbConnection->getQueryBuilder(); + + // Insert test data + $queryBuilder->insert('mounts') + ->values([ + 'storage_id' => $queryBuilder->createNamedParameter(1, IQueryBuilder::PARAM_INT), + 'root_id' => $queryBuilder->createNamedParameter(10, IQueryBuilder::PARAM_INT), + 'mount_provider_class' => $queryBuilder->createNamedParameter('TestProviderClass1'), + 'mount_point' => $queryBuilder->createNamedParameter('/files'), + 'user_id' => $queryBuilder->createNamedParameter('test'), + ]) + ->executeStatement(); + + $queryBuilder->insert('mounts') + ->values([ + 'storage_id' => $queryBuilder->createNamedParameter(3, IQueryBuilder::PARAM_INT), + 'root_id' => $queryBuilder->createNamedParameter(30, IQueryBuilder::PARAM_INT), + 'mount_provider_class' => $queryBuilder->createNamedParameter('TestProviderClass1'), + 'mount_point' => $queryBuilder->createNamedParameter('/documents'), + 'user_id' => $queryBuilder->createNamedParameter('test'), + ]) + ->executeStatement(); + + $queryBuilder->insert('mounts') + ->values([ + 'storage_id' => $queryBuilder->createNamedParameter(4, IQueryBuilder::PARAM_INT), + 'root_id' => $queryBuilder->createNamedParameter(31, IQueryBuilder::PARAM_INT), + 'mount_provider_class' => $queryBuilder->createNamedParameter('TestProviderClass2'), + 'mount_point' => $queryBuilder->createNamedParameter('/foobar'), + 'user_id' => $queryBuilder->createNamedParameter('test'), + ]) + ->executeStatement(); + } + + /** + * Test that getDistinctMounts returns all mounts without filters + */ + public function testGetDistinctMountsWithoutFilters(): void { + $result = iterator_to_array($this->fileAccess->getDistinctMounts([], false)); + + $this->assertCount(3, $result); + + $this->assertEquals([ + 'storage_id' => 1, + 'root_id' => 10, + 'overridden_root' => 10, + ], $result[0]); + + $this->assertEquals([ + 'storage_id' => 3, + 'root_id' => 30, + 'overridden_root' => 30, + ], $result[1]); + + $this->assertEquals([ + 'storage_id' => 4, + 'root_id' => 31, + 'overridden_root' => 31, + ], $result[2]); + } + + /** + * Test that getDistinctMounts applies filtering by mount providers + */ + public function testGetDistinctMountsWithMountProviderFilter(): void { + $result = iterator_to_array($this->fileAccess->getDistinctMounts(['TestProviderClass1'], false)); + + $this->assertCount(2, $result); + + $this->assertEquals([ + 'storage_id' => 1, + 'root_id' => 10, + 'overridden_root' => 10, + ], $result[0]); + + $this->assertEquals([ + 'storage_id' => 3, + 'root_id' => 30, + 'overridden_root' => 30, + ], $result[1]); + } + + /** + * Test that getDistinctMounts rewrites home directory paths + */ + public function testGetDistinctMountsWithRewriteHomeDirectories(): void { + // Add additional test data for a home directory mount + $queryBuilder = $this->dbConnection->getQueryBuilder(); + $queryBuilder->insert('mounts') + ->values([ + 'storage_id' => $queryBuilder->createNamedParameter(4, IQueryBuilder::PARAM_INT), + 'root_id' => $queryBuilder->createNamedParameter(40, IQueryBuilder::PARAM_INT), + 'mount_provider_class' => $queryBuilder->createNamedParameter(\OC\Files\Mount\LocalHomeMountProvider::class), + 'mount_point' => $queryBuilder->createNamedParameter('/home/user'), + 'user_id' => $queryBuilder->createNamedParameter('test'), + ]) + ->executeStatement(); + + // Add a mount that is mounted in the home directory + $queryBuilder = $this->dbConnection->getQueryBuilder(); + $queryBuilder->insert('mounts') + ->values([ + 'storage_id' => $queryBuilder->createNamedParameter(5, IQueryBuilder::PARAM_INT), + 'root_id' => $queryBuilder->createNamedParameter(41, IQueryBuilder::PARAM_INT), + 'mount_provider_class' => $queryBuilder->createNamedParameter('TestMountProvider3'), + 'mount_point' => $queryBuilder->createNamedParameter('/test/files/foobar'), + 'user_id' => $queryBuilder->createNamedParameter('test'), + ]) + ->executeStatement(); + + // Simulate adding a "files" directory to the filecache table + $queryBuilder = $this->dbConnection->getQueryBuilder()->runAcrossAllShards(); + $queryBuilder->delete('filecache')->executeStatement(); + $queryBuilder = $this->dbConnection->getQueryBuilder(); + $queryBuilder->insert('filecache') + ->values([ + 'fileid' => $queryBuilder->createNamedParameter(99, IQueryBuilder::PARAM_INT), + 'storage' => $queryBuilder->createNamedParameter(4, IQueryBuilder::PARAM_INT), + 'parent' => $queryBuilder->createNamedParameter(40), + 'name' => $queryBuilder->createNamedParameter('files'), + 'path' => $queryBuilder->createNamedParameter('files'), + 'path_hash' => $queryBuilder->createNamedParameter(md5('files')), + ]) + ->executeStatement(); + + $result = iterator_to_array($this->fileAccess->getDistinctMounts()); + + $this->assertCount(2, $result); + + $this->assertEquals([ + 'storage_id' => 4, + 'root_id' => 40, + 'overridden_root' => 99, + ], $result[0]); + + $this->assertEquals([ + 'storage_id' => 5, + 'root_id' => 41, + 'overridden_root' => 41, + ], $result[1]); + } + + private function setUpTestDatabaseForGetByAncestorInStorage(): void { + // prepare `filecache` table for tests + $queryBuilder = $this->dbConnection->getQueryBuilder(); + + $queryBuilder->insert('filecache') + ->values([ + 'fileid' => 1, + 'parent' => 0, + 'path' => $queryBuilder->createNamedParameter('files'), + 'path_hash' => $queryBuilder->createNamedParameter(md5('files')), + 'storage' => $queryBuilder->createNamedParameter(1), + 'name' => $queryBuilder->createNamedParameter('files'), + 'mimetype' => 1, + 'encrypted' => 0, + ]) + ->executeStatement(); + + $queryBuilder->insert('filecache') + ->values([ + 'fileid' => 2, + 'parent' => 1, + 'path' => $queryBuilder->createNamedParameter('files/documents'), + 'path_hash' => $queryBuilder->createNamedParameter(md5('files/documents')), + 'storage' => $queryBuilder->createNamedParameter(1), + 'name' => $queryBuilder->createNamedParameter('documents'), + 'mimetype' => 2, + 'encrypted' => 1, + ]) + ->executeStatement(); + + $queryBuilder->insert('filecache') + ->values([ + 'fileid' => 3, + 'parent' => 1, + 'path' => $queryBuilder->createNamedParameter('files/photos'), + 'path_hash' => $queryBuilder->createNamedParameter(md5('files/photos')), + 'storage' => $queryBuilder->createNamedParameter(1), + 'name' => $queryBuilder->createNamedParameter('photos'), + 'mimetype' => 3, + 'encrypted' => 1, + ]) + ->executeStatement(); + + $queryBuilder->insert('filecache') + ->values([ + 'fileid' => 4, + 'parent' => 3, + 'path' => $queryBuilder->createNamedParameter('files/photos/endtoendencrypted'), + 'path_hash' => $queryBuilder->createNamedParameter(md5('files/photos/endtoendencrypted')), + 'storage' => $queryBuilder->createNamedParameter(1), + 'name' => $queryBuilder->createNamedParameter('endtoendencrypted'), + 'mimetype' => 4, + 'encrypted' => 0, + ]) + ->executeStatement(); + + $queryBuilder->insert('filecache') + ->values([ + 'fileid' => 5, + 'parent' => 1, + 'path' => $queryBuilder->createNamedParameter('files/serversideencrypted'), + 'path_hash' => $queryBuilder->createNamedParameter(md5('files/serversideencrypted')), + 'storage' => $queryBuilder->createNamedParameter(1), + 'name' => $queryBuilder->createNamedParameter('serversideencrypted'), + 'mimetype' => 4, + 'encrypted' => 1, + ]) + ->executeStatement(); + + $queryBuilder->insert('filecache') + ->values([ + 'fileid' => 6, + 'parent' => 0, + 'path' => $queryBuilder->createNamedParameter('files/storage2'), + 'path_hash' => $queryBuilder->createNamedParameter(md5('files/storage2')), + 'storage' => $queryBuilder->createNamedParameter(2), + 'name' => $queryBuilder->createNamedParameter('storage2'), + 'mimetype' => 5, + 'encrypted' => 0, + ]) + ->executeStatement(); + + $queryBuilder->insert('filecache') + ->values([ + 'fileid' => 7, + 'parent' => 6, + 'path' => $queryBuilder->createNamedParameter('files/storage2/file'), + 'path_hash' => $queryBuilder->createNamedParameter(md5('files/storage2/file')), + 'storage' => $queryBuilder->createNamedParameter(2), + 'name' => $queryBuilder->createNamedParameter('file'), + 'mimetype' => 6, + 'encrypted' => 0, + ]) + ->executeStatement(); + } + + /** + * Test fetching files by ancestor in storage. + */ + public function testGetByAncestorInStorage(): void { + $generator = $this->fileAccess->getByAncestorInStorage( + 1, // storageId + 1, // rootId + 0, // lastFileId + 10, // maxResults + [], // mimeTypes + true, // include end-to-end encrypted files + true, // include server-side encrypted files + ); + + $result = iterator_to_array($generator); + + $this->assertCount(4, $result); + + $paths = array_map(fn (CacheEntry $entry) => $entry->getPath(), $result); + $this->assertEquals([ + 'files/documents', + 'files/photos', + 'files/photos/endtoendencrypted', + 'files/serversideencrypted', + ], $paths); + } + + /** + * Test filtering by mime types. + */ + public function testGetByAncestorInStorageWithMimeTypes(): void { + $generator = $this->fileAccess->getByAncestorInStorage( + 1, + 1, + 0, + 10, + [2], // Only include documents (mimetype=2) + true, + true, + ); + + $result = iterator_to_array($generator); + + $this->assertCount(1, $result); + $this->assertEquals('files/documents', $result[0]->getPath()); + } + + /** + * Test excluding end-to-end encrypted files. + */ + public function testGetByAncestorInStorageWithoutEndToEndEncrypted(): void { + $generator = $this->fileAccess->getByAncestorInStorage( + 1, + 1, + 0, + 10, + [], + false, // exclude end-to-end encrypted files + true, + ); + + $result = iterator_to_array($generator); + + $this->assertCount(3, $result); + $paths = array_map(fn (CacheEntry $entry) => $entry->getPath(), $result); + $this->assertEquals(['files/documents', 'files/photos', 'files/serversideencrypted'], $paths); + } + + /** + * Test excluding server-side encrypted files. + */ + public function testGetByAncestorInStorageWithoutServerSideEncrypted(): void { + $generator = $this->fileAccess->getByAncestorInStorage( + 1, + 1, + 0, + 10, + [], + true, + false, // exclude server-side encrypted files + ); + + $result = iterator_to_array($generator); + + $this->assertCount(1, $result); + $this->assertEquals('files/photos/endtoendencrypted', $result[0]->getPath()); + } + + /** + * Test max result limits. + */ + public function testGetByAncestorInStorageWithMaxResults(): void { + $generator = $this->fileAccess->getByAncestorInStorage( + 1, + 1, + 0, + 1, // Limit to 1 result + [], + true, + true, + ); + + $result = iterator_to_array($generator); + + $this->assertCount(1, $result); + $this->assertEquals('files/documents', $result[0]->getPath()); + } + + /** + * Test rootId filter + */ + public function testGetByAncestorInStorageWithRootIdFilter(): void { + $generator = $this->fileAccess->getByAncestorInStorage( + 1, + 3, // Filter by rootId + 0, + 10, + [], + true, + true, + ); + + $result = iterator_to_array($generator); + + $this->assertCount(1, $result); + $this->assertEquals('files/photos/endtoendencrypted', $result[0]->getPath()); + } + + /** + * Test rootId filter + */ + public function testGetByAncestorInStorageWithStorageFilter(): void { + $generator = $this->fileAccess->getByAncestorInStorage( + 2, // Filter by storage + 6, // and by rootId + 0, + 10, + [], + true, + true, + ); + + $result = iterator_to_array($generator); + + $this->assertCount(1, $result); + $this->assertEquals('files/storage2/file', $result[0]->getPath()); + } +}