feat: support cross-bucket copy and move operations (#18)

Author: RomainLanz

.github/workflows/checks.yml

@@ -45,6 +45,7 @@ jobs:
         env:
           GCS_KEY: ${{ secrets.GCS_KEY }}
           GCS_BUCKET: drive-gcs
+          GCS_OTHER_BUCKET: flydrive-other-bucket
           GCS_FINE_GRAINED_ACL_BUCKET: drive-gcs-no-uniform-acl
   tests-gcs:
     runs-on: ${{ matrix.os }}
@@ -69,6 +70,7 @@ jobs:
         env:
           GCS_KEY: ${{ secrets.GCS_KEY }}
           GCS_BUCKET: drive-gcs
+          GCS_OTHER_BUCKET: flydrive-other-bucket
           GCS_FINE_GRAINED_ACL_BUCKET: drive-gcs-no-uniform-acl
   tests-s3:
     runs-on: ${{ matrix.os }}
@@ -93,6 +95,7 @@ jobs:
         env:
           S3_SERVICE: do
           S3_BUCKET: testing-flydrive
+          S3_OTHER_BUCKET: drive-other-bucket
           S3_ACCESS_KEY: ${{ secrets.DO_ACCESS_KEY }}
           S3_ACCESS_SECRET: ${{ secrets.DO_ACCESS_SECRET }}
           S3_ENDPOINT: https://sgp1.digitaloceanspaces.com
@@ -121,6 +124,7 @@ jobs:
         env:
           S3_SERVICE: r2
           S3_BUCKET: testing-flydrive
+          S3_OTHER_BUCKET: flydrive-other-bucket
           S3_ACCESS_KEY: ${{ secrets.R2_ACCESS_KEY }}
           S3_ACCESS_SECRET: ${{ secrets.R2_ACCESS_SECRET }}
           S3_ENDPOINT: https://b7d56a259a224b185a70dd6e6f77d9c3.r2.cloudflarestorage.com

drivers/gcs/driver.ts

@@ -23,6 +23,7 @@ import { DriveFile } from '../../src/driver_file.js'
 import { DriveDirectory } from '../../src/drive_directory.js'
 import type {
   WriteOptions,
+  CopyMoveOptions,
   ObjectMetaData,
   DriverContract,
   SignedURLOptions,
@@ -381,58 +382,69 @@ export class GCSDriver implements DriverContract {
   /**
    * Copies the source file to the destination. Both paths must
    * be within the root location.
+   *
+   * Use the "destinationBucket" option to copy the file to a different bucket.
    */
-  async copy(source: string, destination: string, options?: WriteOptions): Promise<void> {
+  async copy(source: string, destination: string, options?: CopyMoveOptions): Promise<void> {
+    const { destinationBucket, ...writeOptions } = options || {}
+    const targetBucket = destinationBucket || this.options.bucket
+
     debug(
       'copying file from %s:%s to %s:%s',
       this.options.bucket,
       source,
-      this.options.bucket,
+      targetBucket,
       destination
     )
-    const bucket = this.#storage.bucket(this.options.bucket)
-    options = options || {}
+
+    const sourceBucket = this.#storage.bucket(this.options.bucket)
 
     /**
      * Copy visibility from the source file to the
      * desintation when no inline visibility is
      * defined and not using usingUniformAcl
      */
-    if (!options.visibility && !this.#usingUniformAcl) {
-      const [isFilePublic] = await bucket.file(source).isPublic()
-      options.visibility = isFilePublic ? 'public' : 'private'
+    if (!writeOptions.visibility && !this.#usingUniformAcl) {
+      const [isFilePublic] = await sourceBucket.file(source).isPublic()
+      writeOptions.visibility = isFilePublic ? 'public' : 'private'
     }
 
-    await bucket.file(source).copy(destination, this.#getSaveOptions(options))
+    const target = destinationBucket
+      ? this.#storage.bucket(destinationBucket).file(destination)
+      : destination
+
+    await sourceBucket.file(source).copy(target, this.#getSaveOptions(writeOptions))
   }
 
   /**
    * Moves the source file to the destination. Both paths must
    * be within the root location.
+   *
+   * Use the "destinationBucket" option to move the file to a different bucket.
    */
-  async move(source: string, destination: string, options?: WriteOptions): Promise<void> {
-    debug(
-      'moving file from %s:%s to %s:%s',
-      this.options.bucket,
-      source,
-      this.options.bucket,
-      destination
-    )
+  async move(source: string, destination: string, options?: CopyMoveOptions): Promise<void> {
+    const { destinationBucket, ...writeOptions } = options || {}
+    const targetBucket = destinationBucket || this.options.bucket
 
-    const bucket = this.#storage.bucket(this.options.bucket)
-    options = options || {}
+    debug('moving file from %s:%s to %s:%s', this.options.bucket, source, targetBucket, destination)
+
+    const sourceBucket = this.#storage.bucket(this.options.bucket)
 
     /**
      * Copy visibility from the source file to the
      * desintation when no inline visibility is
      * defined and not using usingUniformAcl
      */
-    if (!options.visibility && !this.#usingUniformAcl) {
-      const [isFilePublic] = await bucket.file(source).isPublic()
-      options.visibility = isFilePublic ? 'public' : 'private'
+    if (!writeOptions.visibility && !this.#usingUniformAcl) {
+      const [isFilePublic] = await sourceBucket.file(source).isPublic()
+      writeOptions.visibility = isFilePublic ? 'public' : 'private'
     }
 
-    await bucket.file(source).move(destination, this.#getSaveOptions(options))
+    const target = destinationBucket
+      ? this.#storage.bucket(destinationBucket).file(destination)
+      : destination
+
+    await sourceBucket.file(source).move(target, this.#getSaveOptions(writeOptions))
   }
 
   /**

drivers/s3/driver.ts

@@ -39,6 +39,7 @@ import { DriveFile } from '../../src/driver_file.js'
 import { DriveDirectory } from '../../src/drive_directory.js'
 import type {
   WriteOptions,
+  CopyMoveOptions,
   DriverContract,
   ObjectMetaData,
   ObjectVisibility,
@@ -583,49 +584,50 @@ export class S3Driver implements DriverContract {
   /**
    * Copies the source file to the destination. Both paths must
    * be within the root location.
+   *
+   * Use the "destinationBucket" option to copy the file to a different bucket.
    */
-  async copy(source: string, destination: string, options?: WriteOptions): Promise<void> {
+  async copy(source: string, destination: string, options?: CopyMoveOptions): Promise<void> {
+    const { destinationBucket, ...writeOptions } = options || {}
+    const targetBucket = destinationBucket || this.options.bucket
+
     debug(
       'copying file from %s:%s to %s:%s',
       this.options.bucket,
       source,
-      this.options.bucket,
+      targetBucket,
       destination
     )
 
-    options = options || {}
-
     /**
      * Copy visibility from the source file to the
      * destination when no inline visibility is
      * defined
      */
-    if (!options.visibility && this.#supportsACL) {
-      options.visibility = await this.getVisibility(source)
+    if (!writeOptions.visibility && this.#supportsACL) {
+      writeOptions.visibility = await this.getVisibility(source)
     }
 
     await this.#client.send(
       this.createCopyObjectCommand(this.#client, {
-        ...this.#getSaveOptions(destination, options),
+        ...this.#getSaveOptions(destination, writeOptions),
         Key: destination,
         CopySource: `/${this.options.bucket}/${source}`,
-        Bucket: this.options.bucket,
+        Bucket: targetBucket,
       })
     )
   }
 
   /**
    * Moves the source file to the destination. Both paths must
    * be within the root location.
+   *
+   * Use the "destinationBucket" option to move the file to a different bucket.
    */
-  async move(source: string, destination: string, options?: WriteOptions): Promise<void> {
-    debug(
-      'moving file from %s:%s to %s:%s',
-      this.options.bucket,
-      source,
-      this.options.bucket,
-      destination
-    )
+  async move(source: string, destination: string, options?: CopyMoveOptions): Promise<void> {
+    const targetBucket = options?.destinationBucket || this.options.bucket
+
+    debug('moving file from %s:%s to %s:%s', this.options.bucket, source, targetBucket, destination)
 
     await this.copy(source, destination, options)
     await this.delete(source)

src/disk.ts

@@ -22,6 +22,7 @@ import type {
   DriverContract,
   ObjectVisibility,
   SignedURLOptions,
+  CopyMoveOptions,
 } from './types.js'
 
 /**
@@ -169,13 +170,13 @@ export class Disk {
   }
 
   /**
-   * Copies file from the "source" to the "destination" within the
-   * same bucket or the root location of local filesystem.
+   * Copies file from the "source" to the "destination". Use the "destinationBucket"
+   * option to copy the file to a different bucket.
    *
    * Use "copyFromFs" method to copy files from local filesystem to
    * a cloud provider
    */
-  async copy(source: string, destination: string, options?: WriteOptions): Promise<void> {
+  async copy(source: string, destination: string, options?: CopyMoveOptions): Promise<void> {
     source = this.#normalizer.normalize(source)
     destination = this.#normalizer.normalize(destination)
     try {
@@ -193,13 +194,13 @@ export class Disk {
   }
 
   /**
-   * Moves file from the "source" to the "destination" within the
-   * same bucket or the root location of local filesystem.
+   * Moves file from the "source" to the "destination". Use the "destinationBucket"
+   * option to move the file to a different bucket.
    *
    * Use "moveFromFs" method to move files from local filesystem to
    * a cloud provider
    */
-  async move(source: string, destination: string, options?: WriteOptions): Promise<void> {
+  async move(source: string, destination: string, options?: CopyMoveOptions): Promise<void> {
     source = this.#normalizer.normalize(source)
     destination = this.#normalizer.normalize(destination)
     try {

src/types.ts

@@ -42,6 +42,13 @@ export type WriteOptions = {
   [key: string]: any
 }
 
+/**
+ * Options accepted by the copy and move operations.
+ */
+export type CopyMoveOptions = WriteOptions & {
+  destinationBucket?: string
+}
+
 /**
  * Options accepted during the creation of a signed URL.
  */
@@ -155,15 +162,19 @@ export interface DriverContract {
    * Copy the file from within the disk root location. Both
    * the "source" and "destination" will be the key names
    * and not absolute paths.
+   *
+   * Use the "destinationBucket" option to copy the file to a different bucket.
    */
-  copy(source: string, destination: string, options?: WriteOptions): Promise<void>
+  copy(source: string, destination: string, options?: CopyMoveOptions): Promise<void>
 
   /**
    * Move the file from within the disk root location. Both
    * the "source" and "destination" will be the key names
    * and not absolute paths.
+   *
+   * Use the "destinationBucket" option to move the file to a different bucket.
    */
-  move(source: string, destination: string, options?: WriteOptions): Promise<void>
+  move(source: string, destination: string, options?: CopyMoveOptions): Promise<void>
 
   /**
    * Delete the file for the given key. Should not throw

tests/drivers/gcs/copy.spec.ts

@@ -11,7 +11,7 @@ import { test } from '@japa/runner'
 import string from '@poppinss/utils/string'
 import { Storage } from '@google-cloud/storage'
 import { GCSDriver } from '../../../drivers/gcs/driver.js'
-import { GCS_BUCKET, GCS_FINE_GRAINED_ACL_BUCKET, GCS_KEY } from './env.js'
+import { GCS_BUCKET, GCS_FINE_GRAINED_ACL_BUCKET, GCS_KEY, GCS_OTHER_BUCKET } from './env.js'
 
 /**
  * Direct access to Google cloud storage bucket
@@ -20,6 +20,9 @@ import { GCS_BUCKET, GCS_FINE_GRAINED_ACL_BUCKET, GCS_KEY } from './env.js'
 const bucket = new Storage({
   credentials: GCS_KEY,
 }).bucket(GCS_BUCKET)
+const otherBucket = new Storage({
+  credentials: GCS_KEY,
+}).bucket(GCS_OTHER_BUCKET)
 const noUniformedAclBucket = new Storage({
   credentials: GCS_KEY,
 }).bucket(GCS_FINE_GRAINED_ACL_BUCKET)
@@ -28,6 +31,7 @@ test.group('GCS Driver | copy', (group) => {
   group.each.setup(() => {
     return async () => {
       await bucket.deleteFiles()
+      await otherBucket.deleteFiles()
       await noUniformedAclBucket.deleteFiles()
     }
   })
@@ -109,4 +113,53 @@ test.group('GCS Driver | copy', (group) => {
     const existsResponse = await noUniformedAclBucket.file(source).exists()
     assert.isTrue(existsResponse[0])
   })
+
+  test('copy file with explicit bucket option', async ({ assert }) => {
+    const source = `${string.random(6)}.txt`
+    const destination = `${string.random(6)}.txt`
+    const contents = 'Hello world'
+
+    const fdgcs = new GCSDriver({
+      visibility: 'public',
+      bucket: GCS_BUCKET,
+      credentials: GCS_KEY,
+      usingUniformAcl: true,
+    })
+    await fdgcs.put(source, contents)
+    await fdgcs.copy(source, destination, { destinationBucket: GCS_BUCKET })
+
+    assert.equal(await fdgcs.get(destination), contents)
+    const [exists] = await bucket.file(source).exists()
+    assert.isTrue(exists)
+  })
+
+  test('copy file to another bucket with explicit bucket option', async ({ assert }) => {
+    const source = `${string.random(6)}.txt`
+    const destination = `${string.random(6)}.txt`
+    const contents = 'Hello world'
+
+    const sourceDriver = new GCSDriver({
+      visibility: 'public',
+      bucket: GCS_BUCKET,
+      credentials: GCS_KEY,
+      usingUniformAcl: true,
+    })
+    const destinationDriver = new GCSDriver({
+      visibility: 'public',
+      bucket: GCS_OTHER_BUCKET,
+      credentials: GCS_KEY,
+      usingUniformAcl: true,
+    })
+
+    await sourceDriver.put(source, contents)
+    await sourceDriver.copy(source, destination, { destinationBucket: GCS_OTHER_BUCKET })
+
+    assert.equal(await destinationDriver.get(destination), contents)
+
+    const [sourceExists] = await bucket.file(source).exists()
+    const [destinationExists] = await otherBucket.file(destination).exists()
+
+    assert.isTrue(sourceExists)
+    assert.isTrue(destinationExists)
+  })
 })

tests/drivers/gcs/env.ts

@@ -12,9 +12,11 @@ import { Env } from '@adonisjs/env'
 const env = await Env.create(new URL('../../../', import.meta.url), {
   GCS_KEY: Env.schema.string(),
   GCS_BUCKET: Env.schema.string(),
+  GCS_OTHER_BUCKET: Env.schema.string(),
   GCS_FINE_GRAINED_ACL_BUCKET: Env.schema.string(),
 })
 
 export const GCS_BUCKET = env.get('GCS_BUCKET')
+export const GCS_OTHER_BUCKET = env.get('GCS_OTHER_BUCKET')
 export const GCS_KEY = JSON.parse(env.get('GCS_KEY'))
 export const GCS_FINE_GRAINED_ACL_BUCKET = env.get('GCS_FINE_GRAINED_ACL_BUCKET')