Merge pull request #40487 from nextcloud/backport/40183/stable27

[stable27] SFTP improvements

.github/workflows/sftp.yml

@@ -0,0 +1,75 @@
+name: SFTP unit tests
+on:
+  push:
+    branches:
+      - master
+      - stable*
+    paths:
+      - 'apps/files_external/**'
+  pull_request:
+    paths:
+      - 'apps/files_external/**'
+
+env:
+  APP_NAME: files_external
+
+jobs:
+  sftp-tests:
+    runs-on: ubuntu-latest
+
+    if: ${{ github.repository_owner != 'nextcloud-gmbh' }}
+
+    strategy:
+      # do not stop on another job's failure
+      fail-fast: false
+      matrix:
+        php-versions: ['8.0']
+        sftpd: ['openssh']
+
+    name: php${{ matrix.php-versions }}-${{ matrix.sftpd }}
+
+    steps:
+      - name: Checkout server
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: Set up sftpd
+        run: |
+          sudo mkdir /tmp/sftp
+          sudo chown -R 0777 /tmp/sftp
+          if [[ "${{ matrix.sftpd }}" == 'openssh' ]]; then docker run -p 2222:22 --name sftp -d -v /tmp/sftp:/home/test atmoz/sftp "test:test:::data"; fi
+      - name: Set up php ${{ matrix.php-versions }}
+        uses: shivammathur/setup-php@c5fc0d8281aba02c7fda07d3a70cc5371548067d #v2.25.2
+        with:
+          php-version: ${{ matrix.php-versions }}
+          tools: phpunit:9
+          extensions: mbstring, fileinfo, intl, sqlite, pdo_sqlite, zip, gd
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Nextcloud
+        run: |
+          mkdir data
+          ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
+          ./occ app:enable --force ${{ env.APP_NAME }}
+          php -S localhost:8080 &
+      - name: PHPUnit
+        run: |
+          echo "<?php return ['run' => true, 'host' => 'localhost:2222','user' => 'test','password' => 'test', 'root' => 'data'];" > apps/${{ env.APP_NAME }}/tests/config.sftp.php
+          phpunit --configuration tests/phpunit-autotest-external.xml apps/files_external/tests/Storage/SftpTest.php
+      - name: sftpd logs
+        if: always()
+        run: |
+          ls -l /tmp/sftp
+          docker logs sftp
+
+  sftp-summary:
+    runs-on: ubuntu-latest
+    needs: sftp-tests
+
+    if: always()
+
+    steps:
+      - name: Summary status
+        run: if ${{ needs.sftp-tests.result != 'success' }}; then exit 1; fi

apps/files_external/lib/Lib/Storage/AmazonS3.php

@@ -497,7 +497,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 
 				try {
 					return $this->readObject($path);
-				} catch (S3Exception $e) {
+				} catch (\Exception $e) {
 					$this->logger->error($e->getMessage(), [
 						'app' => 'files_external',
 						'exception' => $e,

apps/files_external/lib/Lib/Storage/SFTP.php

@@ -36,15 +36,21 @@
  */
 namespace OCA\Files_External\Lib\Storage;
 
+use Icewind\Streams\CountWrapper;
 use Icewind\Streams\IteratorDirectory;
 use Icewind\Streams\RetryWrapper;
+use OC\Files\Filesystem;
+use OC\Files\Storage\Common;
+use OCP\Constants;
+use OCP\Files\FileInfo;
+use OCP\Files\IMimeTypeDetector;
 use phpseclib\Net\SFTP\Stream;
 
 /**
  * Uses phpseclib's Net\SFTP class and the Net\SFTP\Stream stream wrapper to
  * provide access to SFTP servers.
  */
-class SFTP extends \OC\Files\Storage\Common {
+class SFTP extends Common {
 	private $host;
 	private $user;
 	private $root;
@@ -56,6 +62,9 @@ class SFTP extends \OC\Files\Storage\Common {
 	 * @var \phpseclib\Net\SFTP
 	 */
 	protected $client;
+	private IMimeTypeDetector $mimeTypeDetector;
+
+	const COPY_CHUNK_SIZE = 8 * 1024 * 1024;
 
 	/**
 	 * @param string $host protocol://server:port
@@ -111,6 +120,7 @@ class SFTP extends \OC\Files\Storage\Common {
 
 		$this->root = '/' . ltrim($this->root, '/');
 		$this->root = rtrim($this->root, '/') . '/';
+		$this->mimeTypeDetector = \OC::$server->get(IMimeTypeDetector::class);
 	}
 
 	/**
@@ -370,20 +380,24 @@ class SFTP extends \OC\Files\Storage\Common {
 	public function fopen($path, $mode) {
 		try {
 			$absPath = $this->absPath($path);
+			$connection = $this->getConnection();
 			switch ($mode) {
 				case 'r':
 				case 'rb':
-					if (!$this->file_exists($path)) {
+					$stat = $this->stat($path);
+					if (!$stat) {
 						return false;
 					}
 					SFTPReadStream::register();
-					$context = stream_context_create(['sftp' => ['session' => $this->getConnection()]]);
+					$context = stream_context_create(['sftp' => ['session' => $connection, 'size' => $stat['size']]]);
 					$handle = fopen('sftpread://' . trim($absPath, '/'), 'r', false, $context);
 					return RetryWrapper::wrap($handle);
 				case 'w':
 				case 'wb':
 					SFTPWriteStream::register();
-					$context = stream_context_create(['sftp' => ['session' => $this->getConnection()]]);
+					// the SFTPWriteStream doesn't go through the "normal" methods so it doesn't clear the stat cache.
+					$connection->_remove_from_stat_cache($absPath);
+					$context = stream_context_create(['sftp' => ['session' => $connection]]);
 					return fopen('sftpwrite://' . trim($absPath, '/'), 'w', false, $context);
 				case 'a':
 				case 'ab':
@@ -395,7 +409,7 @@ class SFTP extends \OC\Files\Storage\Common {
 				case 'x+':
 				case 'c':
 				case 'c+':
-					$context = stream_context_create(['sftp' => ['session' => $this->getConnection()]]);
+					$context = stream_context_create(['sftp' => ['session' => $connection]]);
 					$handle = fopen($this->constructUrl($path), $mode, false, $context);
 					return RetryWrapper::wrap($handle);
 			}
@@ -450,14 +464,14 @@ class SFTP extends \OC\Files\Storage\Common {
 	}
 
 	/**
-	 * {@inheritdoc}
+	 * @return array{mtime: int, size: int, ctime: int}|false
 	 */
 	public function stat($path) {
 		try {
 			$stat = $this->getConnection()->stat($this->absPath($path));
 
-			$mtime = $stat ? $stat['mtime'] : -1;
-			$size = $stat ? $stat['size'] : 0;
+			$mtime = $stat ? (int)$stat['mtime'] : -1;
+			$size = $stat ? (int)$stat['size'] : 0;
 
 			return ['mtime' => $mtime, 'size' => $size, 'ctime' => -1];
 		} catch (\Exception $e) {
@@ -476,4 +490,99 @@ class SFTP extends \OC\Files\Storage\Common {
 		$url = 'sftp://' . urlencode($this->user) . '@' . $this->host . ':' . $this->port . $this->root . $path;
 		return $url;
 	}
+
+	public function file_put_contents($path, $data) {
+		/** @psalm-suppress InternalMethod */
+		$result = $this->getConnection()->put($this->absPath($path), $data);
+		if ($result) {
+			return strlen($data);
+		} else {
+			return false;
+		}
+	}
+
+	public function writeStream(string $path, $stream, int $size = null): int {
+		if ($size === null) {
+			$stream = CountWrapper::wrap($stream, function (int $writtenSize) use (&$size) {
+				$size = $writtenSize;
+			});
+			if (!$stream) {
+				throw new \Exception("Failed to wrap stream");
+			}
+		}
+		/** @psalm-suppress InternalMethod */
+		$result = $this->getConnection()->put($this->absPath($path), $stream);
+		fclose($stream);
+		if ($result) {
+			return $size;
+		} else {
+			throw new \Exception("Failed to write steam to sftp storage");
+		}
+	}
+
+	public function copy($source, $target) {
+		if ($this->is_dir($source) || $this->is_dir($target)) {
+			return parent::copy($source, $target);
+		} else {
+			$absSource = $this->absPath($source);
+			$absTarget = $this->absPath($target);
+
+			$connection = $this->getConnection();
+			$size = $connection->size($absSource);
+			if ($size === false) {
+				return false;
+			}
+			for ($i = 0; $i < $size; $i += self::COPY_CHUNK_SIZE) {
+				/** @psalm-suppress InvalidArgument */
+				$chunk = $connection->get($absSource, false, $i, self::COPY_CHUNK_SIZE);
+				if ($chunk === false) {
+					return false;
+				}
+				/** @psalm-suppress InternalMethod */
+				if (!$connection->put($absTarget, $chunk, \phpseclib\Net\SFTP::SOURCE_STRING, $i)) {
+					return false;
+				}
+			}
+			return true;
+		}
+	}
+
+	public function getPermissions($path) {
+		$stat = $this->getConnection()->stat($this->absPath($path));
+		if (!$stat) {
+			return 0;
+		}
+		if ($stat['type'] === NET_SFTP_TYPE_DIRECTORY) {
+			return Constants::PERMISSION_ALL;
+		} else {
+			return Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
+		}
+	}
+
+	public function getMetaData($path) {
+		$stat = $this->getConnection()->stat($this->absPath($path));
+		if (!$stat) {
+			return null;
+		}
+
+		if ($stat['type'] === NET_SFTP_TYPE_DIRECTORY) {
+			$stat['permissions'] = Constants::PERMISSION_ALL;
+		} else {
+			$stat['permissions'] = Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
+		}
+
+		if ($stat['type'] === NET_SFTP_TYPE_DIRECTORY) {
+			$stat['size'] = -1;
+			$stat['mimetype'] = FileInfo::MIMETYPE_FOLDER;
+		} else {
+			$stat['mimetype'] = $this->mimeTypeDetector->detectPath($path);
+		}
+
+		$stat['etag'] = $this->getETag($path);
+		$stat['storage_mtime'] = $stat['mtime'];
+		$stat['name'] = basename($path);
+
+		$keys = ['size', 'mtime', 'mimetype', 'etag', 'storage_mtime', 'permissions', 'name'];
+		return array_intersect_key($stat, array_flip($keys));
+	}
 }

apps/files_external/lib/Lib/Storage/SFTPReadStream.php

@@ -49,6 +49,8 @@ class SFTPReadStream implements File {
 	private $eof = false;
 
 	private $buffer = '';
+	private bool $pendingRead = false;
+	private int $size = 0;
 
 	public static function register($protocol = 'sftpread') {
 		if (in_array($protocol, stream_get_wrappers(), true)) {
@@ -75,6 +77,9 @@ class SFTPReadStream implements File {
 		} else {
 			throw new \BadMethodCallException('Invalid context, session not set');
 		}
+		if (isset($context['size'])) {
+			$this->size = $context['size'];
+		}
 		return $context;
 	}
 
@@ -118,7 +123,25 @@ class SFTPReadStream implements File {
 	}
 
 	public function stream_seek($offset, $whence = SEEK_SET) {
-		return false;
+		switch ($whence) {
+			case SEEK_SET:
+				$this->seekTo($offset);
+				break;
+			case SEEK_CUR:
+				$this->seekTo($this->readPosition + $offset);
+				break;
+			case SEEK_END:
+				$this->seekTo($this->size + $offset);
+				break;
+		}
+		return true;
+	}
+
+	private function seekTo(int $offset): void {
+		$this->internalPosition = $offset;
+		$this->readPosition = $offset;
+		$this->buffer = '';
+		$this->request_chunk(256 * 1024);
 	}
 
 	public function stream_tell() {
@@ -142,11 +165,17 @@ class SFTPReadStream implements File {
 	}
 
 	private function request_chunk($size) {
+		if ($this->pendingRead) {
+			$this->sftp->_get_sftp_packet();
+		}
+
 		$packet = pack('Na*N3', strlen($this->handle), $this->handle, $this->internalPosition / 4294967296, $this->internalPosition, $size);
+		$this->pendingRead = true;
 		return $this->sftp->_send_sftp_packet(NET_SFTP_READ, $packet);
 	}
 
 	private function read_chunk() {
+		$this->pendingRead = false;
 		$response = $this->sftp->_get_sftp_packet();
 
 		switch ($this->sftp->packet_type) {
@@ -195,6 +224,10 @@ class SFTPReadStream implements File {
 	}
 
 	public function stream_close() {
+		// we still have a read request incoming that needs to be handled before we can close
+		if ($this->pendingRead) {
+			$this->sftp->_get_sftp_packet();
+		}
 		if (!$this->sftp->_close_handle($this->handle)) {
 			return false;
 		}