From 8e0b78c746404c8e385908cad1be6dcff6e8ebd1 Mon Sep 17 00:00:00 2001 From: Vincent Petry Date: Wed, 16 Mar 2016 17:57:54 +0100 Subject: [PATCH 1/2] Chunk upload for GDrive Instead of storing the WHOLE file in memory in a PHP variable, use the library's chunk upload support. --- apps/files_external/lib/google.php | 45 ++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/apps/files_external/lib/google.php b/apps/files_external/lib/google.php index 5e5716cf438..112f7e9c57f 100644 --- a/apps/files_external/lib/google.php +++ b/apps/files_external/lib/google.php @@ -490,18 +490,15 @@ class Google extends \OC\Files\Storage\Common { $path = self::$tempFiles[$tmpFile]; $parentFolder = $this->getDriveFile(dirname($path)); if ($parentFolder) { - // TODO Research resumable upload $mimetype = \OC::$server->getMimeTypeDetector()->detect($tmpFile); - $data = file_get_contents($tmpFile); $params = array( - 'data' => $data, 'mimeType' => $mimetype, - 'uploadType' => 'media' ); $result = false; if ($this->file_exists($path)) { $file = $this->getDriveFile($path); - $result = $this->service->files->update($file->getId(), $file, $params); + $this->client->setDefer(true); + $request = $this->service->files->update($file->getId(), $file, $params); } else { $file = new \Google_Service_Drive_DriveFile(); $file->setTitle(basename($path)); @@ -509,8 +506,44 @@ class Google extends \OC\Files\Storage\Common { $parent = new \Google_Service_Drive_ParentReference(); $parent->setId($parentFolder->getId()); $file->setParents(array($parent)); - $result = $this->service->files->insert($file, $params); + $this->client->setDefer(true); + $request = $this->service->files->insert($file, $params); } + + $chunkSizeBytes = 10 * 1024 * 1024; + + // Create a media file upload to represent our upload process. + $media = new \Google_Http_MediaFileUpload( + $this->client, + $request, + 'text/plain', + null, + true, + $chunkSizeBytes + ); + $media->setFileSize(filesize($tmpFile)); + + // Upload the various chunks. $status will be false until the process is + // complete. + $status = false; + $handle = fopen($tmpFile, 'rb'); + while (!$status && !feof($handle)) { + $chunk = fread($handle, $chunkSizeBytes); + $status = $media->nextChunk($chunk); + } + + // The final value of $status will be the data from the API for the object + // that has been uploaded. + $result = false; + if ($status !== false) { + $result = $status; + } + + fclose($handle); + + // Reset to the client to execute requests immediately in the future. + $this->client->setDefer(false); + if ($result) { $this->setDriveFile($path, $result); } From 1e1c0885c63c3568e74966b1f8ea33fc3cd3ccb6 Mon Sep 17 00:00:00 2001 From: Vincent Petry Date: Thu, 17 Mar 2016 12:36:47 +0100 Subject: [PATCH 2/2] Only use GDrive chunks when needed --- apps/files_external/lib/google.php | 72 ++++++++++++++++++------------ 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/apps/files_external/lib/google.php b/apps/files_external/lib/google.php index 112f7e9c57f..b79f42d1e00 100644 --- a/apps/files_external/lib/google.php +++ b/apps/files_external/lib/google.php @@ -493,11 +493,23 @@ class Google extends \OC\Files\Storage\Common { $mimetype = \OC::$server->getMimeTypeDetector()->detect($tmpFile); $params = array( 'mimeType' => $mimetype, + 'uploadType' => 'media' ); $result = false; + + $chunkSizeBytes = 10 * 1024 * 1024; + + $useChunking = false; + $size = filesize($tmpFile); + if ($size > $chunkSizeBytes) { + $useChunking = true; + } else { + $params['data'] = file_get_contents($tmpFile); + } + if ($this->file_exists($path)) { $file = $this->getDriveFile($path); - $this->client->setDefer(true); + $this->client->setDefer($useChunking); $request = $this->service->files->update($file->getId(), $file, $params); } else { $file = new \Google_Service_Drive_DriveFile(); @@ -506,41 +518,43 @@ class Google extends \OC\Files\Storage\Common { $parent = new \Google_Service_Drive_ParentReference(); $parent->setId($parentFolder->getId()); $file->setParents(array($parent)); - $this->client->setDefer(true); + $this->client->setDefer($useChunking); $request = $this->service->files->insert($file, $params); } - $chunkSizeBytes = 10 * 1024 * 1024; + if ($useChunking) { + // Create a media file upload to represent our upload process. + $media = new \Google_Http_MediaFileUpload( + $this->client, + $request, + 'text/plain', + null, + true, + $chunkSizeBytes + ); + $media->setFileSize($size); - // Create a media file upload to represent our upload process. - $media = new \Google_Http_MediaFileUpload( - $this->client, - $request, - 'text/plain', - null, - true, - $chunkSizeBytes - ); - $media->setFileSize(filesize($tmpFile)); + // Upload the various chunks. $status will be false until the process is + // complete. + $status = false; + $handle = fopen($tmpFile, 'rb'); + while (!$status && !feof($handle)) { + $chunk = fread($handle, $chunkSizeBytes); + $status = $media->nextChunk($chunk); + } - // Upload the various chunks. $status will be false until the process is - // complete. - $status = false; - $handle = fopen($tmpFile, 'rb'); - while (!$status && !feof($handle)) { - $chunk = fread($handle, $chunkSizeBytes); - $status = $media->nextChunk($chunk); + // The final value of $status will be the data from the API for the object + // that has been uploaded. + $result = false; + if ($status !== false) { + $result = $status; + } + + fclose($handle); + } else { + $result = $request; } - // The final value of $status will be the data from the API for the object - // that has been uploaded. - $result = false; - if ($status !== false) { - $result = $status; - } - - fclose($handle); - // Reset to the client to execute requests immediately in the future. $this->client->setDefer(false);