upstream/nextcloud
1
0
Fork 0
mirror of https://github.com/nextcloud/server.git synced 2026-04-15 22:11:17 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 35

Merge pull request #41914 from nextcloud/s3-copy-size-limit

Browse source 
only do a multipart s3 copy when above the regular copy limit
This commit is contained in:
Julius Härtl 2023-12-29 11:28:54 +01:00 committed by GitHub
commit 77cec80feb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 94 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
apps/files_external/tests/Storage/Amazons3MultiPartTest.php Normal file
View file

@ -0,0 +1,68 @@
<?php
/**
* @copyright Copyright (c) 2023 Robin Appelman <robin@icewind.nl>
*
* @author Maxence Lange <maxence@artificial-owl.com>
* @author Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\files_external\tests\Storage;
use OCA\Files_External\Lib\Storage\AmazonS3;
/**
* Class Amazons3Test
*
* @group DB
*
* @package OCA\Files_External\Tests\Storage
*/
class Amazons3MultiPartTest extends \Test\Files\Storage\Storage {
private $config;
/** @var AmazonS3 */
protected $instance;
protected function setUp(): void {
parent::setUp();
$this->config = include('files_external/tests/config.amazons3.php');
if (! is_array($this->config) or ! $this->config['run']) {
$this->markTestSkipped('AmazonS3 backend not configured');
}
$this->instance = new AmazonS3($this->config + [
'putSizeLimit' => 1,
'copySizeLimit' => 1,
]);
}
protected function tearDown(): void {
if ($this->instance) {
$this->instance->rmdir('');
}
parent::tearDown();
}
public function testStat() {
$this->markTestSkipped('S3 doesn\'t update the parents folder mtime');
}
public function testHashInFileName() {
$this->markTestSkipped('Localstack has a bug with hashes in filename');
}
}

7
lib/private/Files/ObjectStore/S3ConnectionTrait.php
View file

@ -71,6 +71,11 @@ trait S3ConnectionTrait {
/** @var int */
private $putSizeLimit;
/** @var int */
private $copySizeLimit;
private bool $useMultipartCopy = true;
protected $test;
protected function parseParams($params) {
@ -87,6 +92,8 @@ trait S3ConnectionTrait {
$this->storageClass = !empty($params['storageClass']) ? $params['storageClass'] : 'STANDARD';
$this->uploadPartSize = $params['uploadPartSize'] ?? 524288000;
$this->putSizeLimit = $params['putSizeLimit'] ?? 104857600;
$this->copySizeLimit = $params['copySizeLimit'] ?? 5242880000;
$this->useMultipartCopy = (bool)($params['useMultipartCopy'] ?? true);
$params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region'];
$params['hostname'] = empty($params['hostname']) ? 's3.' . $params['region'] . '.amazonaws.com' : $params['hostname'];
if (!isset($params['port']) || $params['port'] === '') {

30
lib/private/Files/ObjectStore/S3ObjectTrait.php
View file

@ -196,16 +196,24 @@ trait S3ObjectTrait {
'Key' => $from,
] + $this->getSSECParameters());
$copy = new MultipartCopy($this->getConnection(), [
"source_bucket" => $this->getBucket(),
"source_key" => $from
], array_merge([
"bucket" => $this->getBucket(),
"key" => $to,
"acl" => "private",
"params" => $this->getSSECParameters() + $this->getSSECParameters(true),
"source_metadata" => $sourceMetadata
], $options));
$copy->copy();
$size = (int)($sourceMetadata->get('Size') ?? $sourceMetadata->get('ContentLength'));
if ($this->useMultipartCopy && $size > $this->copySizeLimit) {
$copy = new MultipartCopy($this->getConnection(), [
"source_bucket" => $this->getBucket(),
"source_key" => $from
], array_merge([
"bucket" => $this->getBucket(),
"key" => $to,
"acl" => "private",
"params" => $this->getSSECParameters() + $this->getSSECParameters(true),
"source_metadata" => $sourceMetadata
], $options));
$copy->copy();
} else {
$this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', array_merge([
'params' => $this->getSSECParameters() + $this->getSSECParameters(true)
], $options));
}
}
}