Configurable s3 adapter (#2358)

composer.json

@@ -64,6 +64,7 @@
         "twig/twig": "^2.14 || ^3.0"
     },
     "require-dev": {
+        "async-aws/simple-s3": "^1.0",
         "aws/aws-sdk-php": "^3.0",
         "dama/doctrine-test-bundle": "^6.7",
         "doctrine/doctrine-bundle": "^2.3.2",
@@ -89,13 +90,13 @@
         "sonata-project/classification-bundle": "^4.0",
         "sonata-project/doctrine-orm-admin-bundle": "^4.0",
         "symfony/browser-kit": "^4.4 || ^5.4 || ^6.0",
-        "symfony/filesystem": "^4.4 || ^5.4 || ^6.0",
         "symfony/messenger": "^4.4 || ^5.4 || ^6.0",
         "symfony/phpunit-bridge": "^6.1",
         "symfony/security-csrf": "^4.4 || ^5.4 || ^6.0",
         "vimeo/psalm": "^4.7.2"
     },
     "conflict": {
+        "async-aws/simple-s3": "<1.0",
         "aws/aws-sdk-php": "<3.0",
         "doctrine/mongodb-odm": "<2.2",
         "doctrine/orm": "<2.9",
@@ -107,6 +108,8 @@
         "sonata-project/doctrine-orm-admin-bundle": "<4.0"
     },
     "suggest": {
+        "async-aws/simple-s3": "If you want to use async Amazon S3 adapter",
+        "aws/aws-sdk-php": "If you want to use Amazon S3",
         "liip/imagine-bundle": "If you want on-the-fly thumbnail generation or image filtering (scale, crop, watermark...).",
         "sonata-project/classification-bundle": "If you want to categorize your media items.",
         "sonata-project/doctrine-orm-admin-bundle": "If you want to persist entities.",

docs/reference/advanced_configuration.rst

@@ -111,6 +111,7 @@ Full configuration options:
                 cache_control: max-age=86400 # or any other
                 meta:
                     key1: value1 #any amount of metas(sent as x-amz-meta-key1 = value1)
+                async: false
 
             replicate:
                 primary: sonata.media.adapter.filesystem.s3

docs/reference/amazon_s3.rst

@@ -34,6 +34,23 @@ This is a sample configuration to enable amazon S3 as a filesystem and provider:
                 version: '%s3_version%' # defaults to "latest" (cf. https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_configuration.html#cfg-version)
                 endpoint: '%s3_endpoint%' # defaults to null (cf. https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_configuration.html#endpoint)
 
+Async adapter
+-------------
+In order to use async S3 adapter, you will need to require the following package:
+
+.. code-block:: bash
+
+    composer require async-aws/simple-s3
+
+.. code-block:: yaml
+
+    # config/packages/sonata_media.yaml
+
+    sonata_media:
+        filesystem:
+            s3:
+                async: true
+
 .. note::
 
     This bundle is currently using KNP Gaufrette as S3 adapter.

src/DependencyInjection/Configuration.php

@@ -254,6 +254,9 @@ private function addFilesystemSection(ArrayNodeDefinition $node): void
                                     ->prototype('scalar')
                                     ->end()
                                 ->end()
+                                ->booleanNode('async')
+                                    ->defaultFalse()
+                                ->end()
                             ->end()
                         ->end()
 

src/DependencyInjection/SonataMediaExtension.php

@@ -13,6 +13,10 @@
 
 namespace Sonata\MediaBundle\DependencyInjection;
 
+use AsyncAws\SimpleS3\SimpleS3Client;
+use Aws\S3\S3Client;
+use Gaufrette\Adapter\AsyncAwsS3;
+use Gaufrette\Adapter\AwsS3;
 use Sonata\Doctrine\Mapper\Builder\OptionsBuilder;
 use Sonata\Doctrine\Mapper\DoctrineCollector;
 use Symfony\Component\Config\Definition\Processor;
@@ -260,8 +264,21 @@ public function configureFilesystemAdapter(ContainerBuilder $container, array $c
 
         // add the default configuration for the S3 filesystem
         if ($container->hasDefinition('sonata.media.adapter.filesystem.s3') && isset($config['filesystem']['s3'])) {
+            $async = true === $config['filesystem']['s3']['async'];
+            if ($async && !class_exists(SimpleS3Client::class)) {
+                throw new \RuntimeException('You must install "async-aws/simple-s3" to use async S3 adapter');
+            } elseif (!$async && !class_exists(S3Client::class)) {
+                throw new \RuntimeException('You must install "aws/aws-sdk-php" to use Amazon S3 filesystem');
+            }
+
+            $adapterClass = $async ? AsyncAwsS3::class : AwsS3::class;
+            $clientReference = new Reference(
+                $async ? 'sonata.media.adapter.service.s3.async' : 'sonata.media.adapter.service.s3'
+            );
+
             $container->getDefinition('sonata.media.adapter.filesystem.s3')
-                ->replaceArgument(0, new Reference('sonata.media.adapter.service.s3'))
+                ->setClass($adapterClass)
+                ->replaceArgument(0, $clientReference)
                 ->replaceArgument(1, $config['filesystem']['s3']['bucket'])
                 ->replaceArgument(2, ['create' => $config['filesystem']['s3']['create'], 'region' => $config['filesystem']['s3']['region'], 'directory' => $config['filesystem']['s3']['directory'], 'ACL' => $config['filesystem']['s3']['acl']]);
 
@@ -292,6 +309,18 @@ public function configureFilesystemAdapter(ContainerBuilder $container, array $c
 
             $container->getDefinition('sonata.media.adapter.service.s3')
                 ->replaceArgument(0, $arguments);
+
+            if ($async) {
+                if (isset($arguments['credentials']['key'], $arguments['credentials']['secret'])) {
+                    $arguments['accessKeyId'] = $arguments['credentials']['key'];
+                    $arguments['accessKeySecret'] = $arguments['credentials']['secret'];
+                    unset($arguments['credentials']);
+                }
+
+                unset($arguments['version']);
+                $container->getDefinition('sonata.media.adapter.service.s3.async')
+                    ->replaceArgument(0, $arguments);
+            }
         } else {
             $container->removeDefinition('sonata.media.adapter.filesystem.s3');
             $container->removeDefinition('sonata.media.filesystem.s3');

src/Resources/config/gaufrette.php

@@ -11,6 +11,7 @@
  * file that was distributed with this source code.
  */
 
+use AsyncAws\SimpleS3\SimpleS3Client;
 use Aws\S3\S3Client;
 use Gaufrette\Adapter\AwsS3;
 use Gaufrette\Adapter\Ftp;
@@ -31,9 +32,6 @@
         ->set('sonata.media.adapter.filesystem.local', Local::class)
         ->set('sonata.media.adapter.filesystem.ftp', Ftp::class)
 
-        ->set('sonata.media.adapter.service.s3', S3Client::class)
-            ->args([[]])
-
         ->set('sonata.media.adapter.filesystem.s3', AwsS3::class)
             ->args(['', '', ''])
 
@@ -75,4 +73,16 @@
             ->args([[]])
 
         ->set('sonata.media.metadata.noop', NoopMetadataBuilder::class);
+
+    if (class_exists(S3Client::class)) {
+        $containerConfigurator->services()
+            ->set('sonata.media.adapter.service.s3', S3Client::class)
+            ->args([[]]);
+    }
+
+    if (class_exists(SimpleS3Client::class)) {
+        $containerConfigurator->services()
+            ->set('sonata.media.adapter.service.s3.async', SimpleS3Client::class)
+            ->args([[]]);
+    }
 };

tests/Controller/MediaAdminControllerTest.php

@@ -128,6 +128,10 @@ public function testListAction(): void
         $this->configureSetFormTheme($formView, ['filterTheme']);
         $this->configureSetCsrfToken('sonata.batch');
         $this->configureRender('templateList', 'renderResponse');
+
+        /**
+         * @psalm-suppress DeprecatedMethod
+         */
         $datagrid->expects(static::exactly(3))->method('setValue')->withConsecutive(
             ['context', null, 'another_context'],
             ['category', null, 1]

tests/DependencyInjection/SonataMediaExtensionTest.php

@@ -13,8 +13,12 @@
 
 namespace Sonata\MediaBundle\Tests\DependencyInjection;
 
+use AsyncAws\SimpleS3\SimpleS3Client;
 use Aws\CloudFront\CloudFrontClient;
+use Aws\S3\S3Client;
 use Aws\Sdk;
+use Gaufrette\Adapter\AsyncAwsS3;
+use Gaufrette\Adapter\AwsS3;
 use Imagine\Gd\Imagine as GdImagine;
 use Imagine\Gmagick\Imagine as GmagicImagine;
 use Imagine\Imagick\Imagine as ImagicImagine;
@@ -237,18 +241,30 @@ public function testLoadWithSonataAdminCustomConfiguration(): void
      * @param array<string, mixed> $expected
      * @param array<string, mixed> $configs
      */
-    public function testLoadWithFilesystemConfigurationV3(array $expected, array $configs): void
-    {
+    public function testLoadWithFilesystemConfigurationV3(
+        array $expected,
+        array $configs
+    ): void {
         if (!class_exists(Sdk::class)) {
             static::markTestSkipped('This test requires aws/aws-sdk-php 3.x.');
         }
 
         $this->load($configs);
 
+        static::assertSame(
+            S3Client::class,
+            $this->container->getDefinition('sonata.media.adapter.service.s3')->getClass()
+        );
+
         static::assertSame(
             $expected,
             $this->container->getDefinition('sonata.media.adapter.service.s3')->getArgument(0)
         );
+
+        static::assertSame(
+            AwsS3::class,
+            $this->container->getDefinition('sonata.media.adapter.filesystem.s3')->getClass()
+        );
     }
 
     /**
@@ -344,6 +360,51 @@ public function dataFilesystemConfigurationAwsV3(): iterable
         ];
     }
 
+    public function testLoadWithFilesystemConfigurationV3ASync(): void
+    {
+        if (!class_exists(SimpleS3Client::class)) {
+            static::markTestSkipped('This test requires async-aws/simple-s3.');
+        }
+
+        $expected = [
+            'region' => 'region',
+            'endpoint' => 'endpoint',
+            'accessKeyId' => 'access',
+            'accessKeySecret' => 'secret',
+        ];
+
+        $configs = [
+            'filesystem' => [
+                's3' => [
+                    'async' => true,
+                    'bucket' => 'bucket_name',
+                    'region' => 'region',
+                    'version' => 'version',
+                    'endpoint' => 'endpoint',
+                    'secretKey' => 'secret',
+                    'accessKey' => 'access',
+                ],
+            ],
+        ];
+
+        $this->load($configs);
+
+        static::assertSame(
+            SimpleS3Client::class,
+            $this->container->getDefinition('sonata.media.adapter.service.s3.async')->getClass()
+        );
+
+        static::assertSame(
+            $expected,
+            $this->container->getDefinition('sonata.media.adapter.service.s3.async')->getArgument(0)
+        );
+
+        static::assertSame(
+            AsyncAwsS3::class,
+            $this->container->getDefinition('sonata.media.adapter.filesystem.s3')->getClass()
+        );
+    }
+
     public function testMediaPool(): void
     {
         $this->load();