Merge pull request #51810 from nextcloud/feat/getByAncestorInStorage

feat: Add new methods to list distinct mounts and retrieve all files in a mount
This commit is contained in:
Andy Scherzinger 2025-07-15 12:14:45 +02:00 committed by GitHub
commit af5acc35cd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 595 additions and 0 deletions

View file

@ -10,6 +10,7 @@ namespace OC\Files\Cache;
use OC\FilesMetadata\FilesMetadataManager;
use OC\SystemConfig;
use OCP\DB\Exception;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\Cache\IFileAccess;
use OCP\Files\IMimeTypeLoader;
@ -94,4 +95,128 @@ class FileAccess implements IFileAccess {
$rows = $query->executeQuery()->fetchAll();
return $this->rowsToEntries($rows);
}
public function getByAncestorInStorage(int $storageId, int $folderId, int $fileIdCursor = 0, int $maxResults = 100, array $mimeTypeIds = [], bool $endToEndEncrypted = true, bool $serverSideEncrypted = true): \Generator {
$qb = $this->getQuery();
$qb->select('path')
->from('filecache')
->where($qb->expr()->eq('fileid', $qb->createNamedParameter($folderId, IQueryBuilder::PARAM_INT)));
$result = $qb->executeQuery();
/** @var array{path:string}|false $root */
$root = $result->fetch();
$result->closeCursor();
if ($root === false) {
throw new Exception('Could not fetch storage root');
}
$qb = $this->getQuery();
$path = $root['path'] === '' ? '' : $root['path'] . '/';
$qb->selectDistinct('*')
->from('filecache', 'f')
->where($qb->expr()->like('f.path', $qb->createNamedParameter($this->connection->escapeLikeParameter($path) . '%')))
->andWhere($qb->expr()->eq('f.storage', $qb->createNamedParameter($storageId)))
->andWhere($qb->expr()->gt('f.fileid', $qb->createNamedParameter($fileIdCursor, IQueryBuilder::PARAM_INT)));
if (!$endToEndEncrypted) {
// End to end encrypted files are descendants of a folder with encrypted=1
// Use a subquery to check the `encrypted` status of the parent folder
$subQuery = $this->getQuery()->select('p.encrypted')
->from('filecache', 'p')
->andWhere($qb->expr()->eq('p.fileid', 'f.parent'))
->getSQL();
$qb->andWhere(
$qb->expr()->eq($qb->createFunction(sprintf('(%s)', $subQuery)), $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))
);
}
if (!$serverSideEncrypted) {
// Server side encrypted files have encrypted=1 directly
$qb->andWhere($qb->expr()->eq('f.encrypted', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
}
if (count($mimeTypeIds) > 0) {
$qb->andWhere($qb->expr()->in('f.mimetype', $qb->createNamedParameter($mimeTypeIds, IQueryBuilder::PARAM_INT_ARRAY)));
}
if ($maxResults !== 0) {
$qb->setMaxResults($maxResults);
}
$qb->orderBy('f.fileid', 'ASC');
$files = $qb->executeQuery();
while (
/** @var array */
$row = $files->fetch()
) {
yield Cache::cacheEntryFromData($row, $this->mimeTypeLoader);
}
$files->closeCursor();
}
public function getDistinctMounts(array $mountProviders = [], bool $onlyUserFilesMounts = true): \Generator {
$qb = $this->connection->getQueryBuilder();
$qb->selectDistinct(['root_id', 'storage_id', 'mount_provider_class'])
->from('mounts');
if ($onlyUserFilesMounts) {
$qb->andWhere(
$qb->expr()->orX(
$qb->expr()->like('mount_point', $qb->createNamedParameter('/%/files/%')),
$qb->expr()->in('mount_provider_class', $qb->createNamedParameter([
\OC\Files\Mount\LocalHomeMountProvider::class,
\OC\Files\Mount\ObjectHomeMountProvider::class,
], IQueryBuilder::PARAM_STR_ARRAY))
)
);
}
if (count($mountProviders) > 0) {
$qb->andWhere($qb->expr()->in('mount_provider_class', $qb->createNamedParameter($mountProviders, IQueryBuilder::PARAM_STR_ARRAY)));
}
$qb->orderBy('root_id', 'ASC');
$result = $qb->executeQuery();
while (
/** @var array{storage_id:int, root_id:int,mount_provider_class:string} $row */
$row = $result->fetch()
) {
$storageId = (int)$row['storage_id'];
$rootId = (int)$row['root_id'];
$overrideRoot = $rootId;
// LocalHomeMountProvider is the default provider for user home directories
// ObjectHomeMountProvider is the home directory provider for when S3 primary storage is used
if ($onlyUserFilesMounts && in_array($row['mount_provider_class'], [
\OC\Files\Mount\LocalHomeMountProvider::class,
\OC\Files\Mount\ObjectHomeMountProvider::class,
], true)) {
// Only crawl files, not cache or trashbin
$qb = $this->getQuery();
try {
$qb->select('fileid')
->from('filecache')
->where($qb->expr()->eq('storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($qb->expr()->eq('parent', $qb->createNamedParameter($rootId, IQueryBuilder::PARAM_INT)))
->andWhere($qb->expr()->eq('path', $qb->createNamedParameter('files')));
/** @var array|false $root */
$root = $qb->executeQuery()->fetch();
if ($root !== false) {
$overrideRoot = (int)$root['fileid'];
}
} catch (Exception $e) {
$this->logger->error('Could not fetch home storage files root for storage ' . $storageId, ['exception' => $e]);
continue;
}
}
// Reference to root_id is still necessary even if we have the overridden_root_id, because storage_id and root_id uniquely identify a mount
yield [
'storage_id' => $storageId,
'root_id' => $rootId,
'overridden_root' => $overrideRoot,
];
}
$result->closeCursor();
}
}

View file

@ -79,4 +79,36 @@ interface IFileAccess {
* @since 29.0.0
*/
public function getByFileIdsInStorage(array $fileIds, int $storageId): array;
/**
* Retrieves files stored in a specific storage that have a specified ancestor in the file hierarchy.
* Allows filtering by mime types, encryption status, and limits the number of results.
*
* @param int $storageId The ID of the storage to search within.
* @param int $folderId The file ID of the ancestor to base the search on.
* @param int $fileIdCursor The last processed file ID. Only files with a higher ID will be included. Defaults to 0.
* @param int $maxResults The maximum number of results to retrieve. If set to 0, all matching files will be retrieved.
* @param list<int> $mimeTypeIds An array of mime types to filter the results. If empty, no mime type filtering will be applied.
* @param bool $endToEndEncrypted Whether to include EndToEndEncrypted files
* @param bool $serverSideEncrypted Whether to include ServerSideEncrypted files
* @return \Generator<ICacheEntry> A generator yielding matching files as cache entries.
* @throws \OCP\DB\Exception
*
* @since 32.0.0
*/
public function getByAncestorInStorage(int $storageId, int $folderId, int $fileIdCursor = 0, int $maxResults = 100, array $mimeTypeIds = [], bool $endToEndEncrypted = true, bool $serverSideEncrypted = true): \Generator;
/**
* Retrieves a list of all distinct mounts.
* Allows filtering by specific mount providers.
* Optionally rewrites home directory root paths to avoid cache and trashbin.
*
* @param list<string> $mountProviders An array of mount provider class names to filter. If empty, all providers will be included.
* @param bool $onlyUserFilesMounts Whether to rewrite the root IDs for home directories to only include user files and to only consider mounts with mount points in the user files.
* @return \Generator<array{storage_id: int, root_id: int, overridden_root: int}> A generator yielding mount configurations as an array containing 'storage_id', 'root_id', and 'override_root'.
* @throws \OCP\DB\Exception
*
* @since 32.0.0
*/
public function getDistinctMounts(array $mountProviders = [], bool $onlyUserFilesMounts = true): \Generator;
}

View file

@ -0,0 +1,438 @@
<?php
/**
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace Test\Files\Cache;
use OC\Files\Cache\CacheEntry;
use OC\Files\Cache\FileAccess;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\IDBConnection;
use Psr\Log\LoggerInterface;
use Test\TestCase;
/**
* @group DB
*/
class FileAccessTest extends TestCase {
private IDBConnection $dbConnection;
private FileAccess $fileAccess;
protected function setUp(): void {
parent::setUp();
// Setup the actual database connection (assume the database is configured properly in PHPUnit setup)
$this->dbConnection = \OCP\Server::get(IDBConnection::class);
// Ensure FileAccess is instantiated with the real connection
$this->fileAccess = new FileAccess(
$this->dbConnection,
\OCP\Server::get(\OC\SystemConfig::class),
\OCP\Server::get(LoggerInterface::class),
\OCP\Server::get(\OC\FilesMetadata\FilesMetadataManager::class),
\OCP\Server::get(\OCP\Files\IMimeTypeLoader::class)
);
// Clear and prepare `filecache` table for tests
$queryBuilder = $this->dbConnection->getQueryBuilder()->runAcrossAllShards();
$queryBuilder->delete('filecache')->executeStatement();
// Clean up potential leftovers from other tests
$queryBuilder = $this->dbConnection->getQueryBuilder();
$queryBuilder->delete('mounts')->executeStatement();
$this->setUpTestDatabaseForGetDistinctMounts();
$this->setUpTestDatabaseForGetByAncestorInStorage();
}
private function setUpTestDatabaseForGetDistinctMounts(): void {
$queryBuilder = $this->dbConnection->getQueryBuilder();
// Insert test data
$queryBuilder->insert('mounts')
->values([
'storage_id' => $queryBuilder->createNamedParameter(1, IQueryBuilder::PARAM_INT),
'root_id' => $queryBuilder->createNamedParameter(10, IQueryBuilder::PARAM_INT),
'mount_provider_class' => $queryBuilder->createNamedParameter('TestProviderClass1'),
'mount_point' => $queryBuilder->createNamedParameter('/files'),
'user_id' => $queryBuilder->createNamedParameter('test'),
])
->executeStatement();
$queryBuilder->insert('mounts')
->values([
'storage_id' => $queryBuilder->createNamedParameter(3, IQueryBuilder::PARAM_INT),
'root_id' => $queryBuilder->createNamedParameter(30, IQueryBuilder::PARAM_INT),
'mount_provider_class' => $queryBuilder->createNamedParameter('TestProviderClass1'),
'mount_point' => $queryBuilder->createNamedParameter('/documents'),
'user_id' => $queryBuilder->createNamedParameter('test'),
])
->executeStatement();
$queryBuilder->insert('mounts')
->values([
'storage_id' => $queryBuilder->createNamedParameter(4, IQueryBuilder::PARAM_INT),
'root_id' => $queryBuilder->createNamedParameter(31, IQueryBuilder::PARAM_INT),
'mount_provider_class' => $queryBuilder->createNamedParameter('TestProviderClass2'),
'mount_point' => $queryBuilder->createNamedParameter('/foobar'),
'user_id' => $queryBuilder->createNamedParameter('test'),
])
->executeStatement();
}
/**
* Test that getDistinctMounts returns all mounts without filters
*/
public function testGetDistinctMountsWithoutFilters(): void {
$result = iterator_to_array($this->fileAccess->getDistinctMounts([], false));
$this->assertCount(3, $result);
$this->assertEquals([
'storage_id' => 1,
'root_id' => 10,
'overridden_root' => 10,
], $result[0]);
$this->assertEquals([
'storage_id' => 3,
'root_id' => 30,
'overridden_root' => 30,
], $result[1]);
$this->assertEquals([
'storage_id' => 4,
'root_id' => 31,
'overridden_root' => 31,
], $result[2]);
}
/**
* Test that getDistinctMounts applies filtering by mount providers
*/
public function testGetDistinctMountsWithMountProviderFilter(): void {
$result = iterator_to_array($this->fileAccess->getDistinctMounts(['TestProviderClass1'], false));
$this->assertCount(2, $result);
$this->assertEquals([
'storage_id' => 1,
'root_id' => 10,
'overridden_root' => 10,
], $result[0]);
$this->assertEquals([
'storage_id' => 3,
'root_id' => 30,
'overridden_root' => 30,
], $result[1]);
}
/**
* Test that getDistinctMounts rewrites home directory paths
*/
public function testGetDistinctMountsWithRewriteHomeDirectories(): void {
// Add additional test data for a home directory mount
$queryBuilder = $this->dbConnection->getQueryBuilder();
$queryBuilder->insert('mounts')
->values([
'storage_id' => $queryBuilder->createNamedParameter(4, IQueryBuilder::PARAM_INT),
'root_id' => $queryBuilder->createNamedParameter(40, IQueryBuilder::PARAM_INT),
'mount_provider_class' => $queryBuilder->createNamedParameter(\OC\Files\Mount\LocalHomeMountProvider::class),
'mount_point' => $queryBuilder->createNamedParameter('/home/user'),
'user_id' => $queryBuilder->createNamedParameter('test'),
])
->executeStatement();
// Add a mount that is mounted in the home directory
$queryBuilder = $this->dbConnection->getQueryBuilder();
$queryBuilder->insert('mounts')
->values([
'storage_id' => $queryBuilder->createNamedParameter(5, IQueryBuilder::PARAM_INT),
'root_id' => $queryBuilder->createNamedParameter(41, IQueryBuilder::PARAM_INT),
'mount_provider_class' => $queryBuilder->createNamedParameter('TestMountProvider3'),
'mount_point' => $queryBuilder->createNamedParameter('/test/files/foobar'),
'user_id' => $queryBuilder->createNamedParameter('test'),
])
->executeStatement();
// Simulate adding a "files" directory to the filecache table
$queryBuilder = $this->dbConnection->getQueryBuilder()->runAcrossAllShards();
$queryBuilder->delete('filecache')->executeStatement();
$queryBuilder = $this->dbConnection->getQueryBuilder();
$queryBuilder->insert('filecache')
->values([
'fileid' => $queryBuilder->createNamedParameter(99, IQueryBuilder::PARAM_INT),
'storage' => $queryBuilder->createNamedParameter(4, IQueryBuilder::PARAM_INT),
'parent' => $queryBuilder->createNamedParameter(40),
'name' => $queryBuilder->createNamedParameter('files'),
'path' => $queryBuilder->createNamedParameter('files'),
'path_hash' => $queryBuilder->createNamedParameter(md5('files')),
])
->executeStatement();
$result = iterator_to_array($this->fileAccess->getDistinctMounts());
$this->assertCount(2, $result);
$this->assertEquals([
'storage_id' => 4,
'root_id' => 40,
'overridden_root' => 99,
], $result[0]);
$this->assertEquals([
'storage_id' => 5,
'root_id' => 41,
'overridden_root' => 41,
], $result[1]);
}
private function setUpTestDatabaseForGetByAncestorInStorage(): void {
// prepare `filecache` table for tests
$queryBuilder = $this->dbConnection->getQueryBuilder();
$queryBuilder->insert('filecache')
->values([
'fileid' => 1,
'parent' => 0,
'path' => $queryBuilder->createNamedParameter('files'),
'path_hash' => $queryBuilder->createNamedParameter(md5('files')),
'storage' => $queryBuilder->createNamedParameter(1),
'name' => $queryBuilder->createNamedParameter('files'),
'mimetype' => 1,
'encrypted' => 0,
])
->executeStatement();
$queryBuilder->insert('filecache')
->values([
'fileid' => 2,
'parent' => 1,
'path' => $queryBuilder->createNamedParameter('files/documents'),
'path_hash' => $queryBuilder->createNamedParameter(md5('files/documents')),
'storage' => $queryBuilder->createNamedParameter(1),
'name' => $queryBuilder->createNamedParameter('documents'),
'mimetype' => 2,
'encrypted' => 1,
])
->executeStatement();
$queryBuilder->insert('filecache')
->values([
'fileid' => 3,
'parent' => 1,
'path' => $queryBuilder->createNamedParameter('files/photos'),
'path_hash' => $queryBuilder->createNamedParameter(md5('files/photos')),
'storage' => $queryBuilder->createNamedParameter(1),
'name' => $queryBuilder->createNamedParameter('photos'),
'mimetype' => 3,
'encrypted' => 1,
])
->executeStatement();
$queryBuilder->insert('filecache')
->values([
'fileid' => 4,
'parent' => 3,
'path' => $queryBuilder->createNamedParameter('files/photos/endtoendencrypted'),
'path_hash' => $queryBuilder->createNamedParameter(md5('files/photos/endtoendencrypted')),
'storage' => $queryBuilder->createNamedParameter(1),
'name' => $queryBuilder->createNamedParameter('endtoendencrypted'),
'mimetype' => 4,
'encrypted' => 0,
])
->executeStatement();
$queryBuilder->insert('filecache')
->values([
'fileid' => 5,
'parent' => 1,
'path' => $queryBuilder->createNamedParameter('files/serversideencrypted'),
'path_hash' => $queryBuilder->createNamedParameter(md5('files/serversideencrypted')),
'storage' => $queryBuilder->createNamedParameter(1),
'name' => $queryBuilder->createNamedParameter('serversideencrypted'),
'mimetype' => 4,
'encrypted' => 1,
])
->executeStatement();
$queryBuilder->insert('filecache')
->values([
'fileid' => 6,
'parent' => 0,
'path' => $queryBuilder->createNamedParameter('files/storage2'),
'path_hash' => $queryBuilder->createNamedParameter(md5('files/storage2')),
'storage' => $queryBuilder->createNamedParameter(2),
'name' => $queryBuilder->createNamedParameter('storage2'),
'mimetype' => 5,
'encrypted' => 0,
])
->executeStatement();
$queryBuilder->insert('filecache')
->values([
'fileid' => 7,
'parent' => 6,
'path' => $queryBuilder->createNamedParameter('files/storage2/file'),
'path_hash' => $queryBuilder->createNamedParameter(md5('files/storage2/file')),
'storage' => $queryBuilder->createNamedParameter(2),
'name' => $queryBuilder->createNamedParameter('file'),
'mimetype' => 6,
'encrypted' => 0,
])
->executeStatement();
}
/**
* Test fetching files by ancestor in storage.
*/
public function testGetByAncestorInStorage(): void {
$generator = $this->fileAccess->getByAncestorInStorage(
1, // storageId
1, // rootId
0, // lastFileId
10, // maxResults
[], // mimeTypes
true, // include end-to-end encrypted files
true, // include server-side encrypted files
);
$result = iterator_to_array($generator);
$this->assertCount(4, $result);
$paths = array_map(fn (CacheEntry $entry) => $entry->getPath(), $result);
$this->assertEquals([
'files/documents',
'files/photos',
'files/photos/endtoendencrypted',
'files/serversideencrypted',
], $paths);
}
/**
* Test filtering by mime types.
*/
public function testGetByAncestorInStorageWithMimeTypes(): void {
$generator = $this->fileAccess->getByAncestorInStorage(
1,
1,
0,
10,
[2], // Only include documents (mimetype=2)
true,
true,
);
$result = iterator_to_array($generator);
$this->assertCount(1, $result);
$this->assertEquals('files/documents', $result[0]->getPath());
}
/**
* Test excluding end-to-end encrypted files.
*/
public function testGetByAncestorInStorageWithoutEndToEndEncrypted(): void {
$generator = $this->fileAccess->getByAncestorInStorage(
1,
1,
0,
10,
[],
false, // exclude end-to-end encrypted files
true,
);
$result = iterator_to_array($generator);
$this->assertCount(3, $result);
$paths = array_map(fn (CacheEntry $entry) => $entry->getPath(), $result);
$this->assertEquals(['files/documents', 'files/photos', 'files/serversideencrypted'], $paths);
}
/**
* Test excluding server-side encrypted files.
*/
public function testGetByAncestorInStorageWithoutServerSideEncrypted(): void {
$generator = $this->fileAccess->getByAncestorInStorage(
1,
1,
0,
10,
[],
true,
false, // exclude server-side encrypted files
);
$result = iterator_to_array($generator);
$this->assertCount(1, $result);
$this->assertEquals('files/photos/endtoendencrypted', $result[0]->getPath());
}
/**
* Test max result limits.
*/
public function testGetByAncestorInStorageWithMaxResults(): void {
$generator = $this->fileAccess->getByAncestorInStorage(
1,
1,
0,
1, // Limit to 1 result
[],
true,
true,
);
$result = iterator_to_array($generator);
$this->assertCount(1, $result);
$this->assertEquals('files/documents', $result[0]->getPath());
}
/**
* Test rootId filter
*/
public function testGetByAncestorInStorageWithRootIdFilter(): void {
$generator = $this->fileAccess->getByAncestorInStorage(
1,
3, // Filter by rootId
0,
10,
[],
true,
true,
);
$result = iterator_to_array($generator);
$this->assertCount(1, $result);
$this->assertEquals('files/photos/endtoendencrypted', $result[0]->getPath());
}
/**
* Test rootId filter
*/
public function testGetByAncestorInStorageWithStorageFilter(): void {
$generator = $this->fileAccess->getByAncestorInStorage(
2, // Filter by storage
6, // and by rootId
0,
10,
[],
true,
true,
);
$result = iterator_to_array($generator);
$this->assertCount(1, $result);
$this->assertEquals('files/storage2/file', $result[0]->getPath());
}
}