From 933736cfddc547732ead4fabcadd89412d5d9297 Mon Sep 17 00:00:00 2001 From: ak Date: Tue, 11 Aug 2026 18:35:17 +0800 Subject: [PATCH] =?UTF-8?q?M6:=20S3=20=E9=99=84=E4=BB=B6=E6=94=AF=E6=8C=81?= =?UTF-8?q?=EF=BC=88R2/COS/OSS=EF=BC=89+=20attachments:sync-s3=20=E5=91=BD?= =?UTF-8?q?=E4=BB=A4=EF=BC=88Multipart/=E5=B9=82=E7=AD=89=EF=BC=89?= =?UTF-8?q?=EF=BC=8CMinIO=20=E5=AE=9E=E6=B5=8B=E9=80=9A=E8=BF=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/Console/Commands/AttachmentsSyncS3.php | 170 ++++++++++ composer.json | 1 + composer.lock | 344 ++++++++++++++++++++- config/filesystems.php | 2 +- 4 files changed, 515 insertions(+), 2 deletions(-) create mode 100644 app/Console/Commands/AttachmentsSyncS3.php diff --git a/app/Console/Commands/AttachmentsSyncS3.php b/app/Console/Commands/AttachmentsSyncS3.php new file mode 100644 index 0000000..74b8140 --- /dev/null +++ b/app/Console/Commands/AttachmentsSyncS3.php @@ -0,0 +1,170 @@ +option('disk') ?: MediaDisk::name(); + $s3 = Storage::disk($disk); + + $this->info("目标磁盘：{$disk}"); + + $query = Media::query()->where('collection_name', 'attachments'); + + if (! $this->option('force')) { + $query->where(function ($q) use ($disk) { + $q->whereRaw('JSON_EXTRACT(custom_properties, "$.pending_sync") = true') + ->orWhere('disk', '!=', $disk); + }); + } + + $mediaItems = $query->get(); + + if ($mediaItems->isEmpty()) { + $this->info('没有需要同步的附件'); + + return self::SUCCESS; + } + + $bar = $this->output->createProgressBar($mediaItems->count()); + $bar->start(); + + $synced = 0; + $skipped = 0; + $failed = 0; + + foreach ($mediaItems as $media) { + $result = $this->syncOne($media, $s3, $disk); + + match ($result) { + 'synced' => $synced++, + 'skipped' => $skipped++, + 'failed' => $failed++, + }; + + $bar->advance(); + } + + $bar->finish(); + $this->newLine(); + $this->line("已同步：{$synced}，跳过：{$skipped}，失败：{$failed}"); + + return $failed === 0 ? self::SUCCESS : self::FAILURE; + } + + private function syncOne(Media $media, \Illuminate\Contracts\Filesystem\Filesystem $s3, string $disk): string + { + // 1. 找源文件：优先已存在磁盘上的文件，其次老 attachments 目录 + $source = $this->locateSource($media); + + if (! $source) { + return 'skipped'; + } + + $targetPath = $this->targetPath($media, $disk); + + try { + if ($s3->exists($targetPath) && ! $this->option('force')) { + $this->markSynced($media, $disk); + + return 'synced'; + } + + $this->upload($s3, $targetPath, $source); + + $this->markSynced($media, $disk); + + return 'synced'; + } catch (\Throwable $e) { + $this->error(" [{$media->id}] {$media->file_name}: {$e->getMessage()}"); + + return 'failed'; + } + } + + private function locateSource(Media $media): ?string + { + // 老磁盘（public/local）上的文件 + if ($media->disk !== 'uploads') { + $local = Storage::disk($media->disk); + $path = $media->getPath(); + + if ($local->exists($path)) { + return $local->path($path); + } + } + + // 老 sablog attachments 目录 + $legacyDir = $this->option('legacy-dir'); + $legacyPath = $media->getCustomProperty('legacy_filepath'); + + if ($legacyDir && $legacyPath) { + $candidate = rtrim($legacyDir, '/').'/'.ltrim($legacyPath, '/'); + + if (is_file($candidate)) { + return $candidate; + } + } + + return null; + } + + private function targetPath(Media $media, string $disk): string + { + // medialibrary 默认路径约定：{media_id}/{filename} + return $media->getKey().'/'.$media->file_name; + } + + private function upload(\Illuminate\Contracts\Filesystem\Filesystem $s3, string $targetPath, string $source): void + { + $size = filesize($source); + $threshold = (int) config('media.multipart_threshold', 50 * 1024 * 1024); + + if ($size > $threshold && method_exists($s3, 'getClient')) { + // 大文件：AWS SDK MultipartUpload + $client = $s3->getClient(); + $bucket = $s3->getConfig('bucket'); + + $uploader = new \Aws\S3\MultipartUploader($client, $source, [ + 'bucket' => $bucket, + 'key' => $targetPath, + 'before_initiate' => function (\Aws\Command $command) use ($source) { + $command['ContentType'] = File::mimeType($source) ?: 'application/octet-stream'; + }, + ]); + + $uploader->upload(); + + return; + } + + $s3->putFileAs('', $source, $targetPath, [ + 'visibility' => config('media.visibility', 'public'), + 'ContentType' => File::mimeType($source) ?: 'application/octet-stream', + ]); + } + + private function markSynced(Media $media, string $disk): void + { + $media->update([ + 'disk' => $disk, + 'conversions_disk' => $disk, + 'custom_properties' => array_merge($media->custom_properties ?? [], ['pending_sync' => false]), + ]); + } +} diff --git a/composer.json b/composer.json index 5aaad77..f09ed74 100644 --- a/composer.json +++ b/composer.json @@ -12,6 +12,7 @@ "laravel/framework": "^12.0", "laravel/tinker": "^2.10.1", "league/commonmark": "^2.9", + "league/flysystem-aws-s3-v3": "^3.35", "league/html-to-markdown": "^5.1", "spatie/laravel-backup": "^9.0", "spatie/laravel-medialibrary": "^11.0", diff --git a/composer.lock b/composer.lock index 0415452..d780129 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b3635d8ae6b629801f16a9df99622e56", + "content-hash": "3b5974911686ef4cf9590ee3210822a5", "packages": [ { "name": "anourvalar/eloquent-serialize", @@ -71,6 +71,157 @@ }, "time": "2026-08-07T06:14:19+00:00" }, + { + "name": "aws/aws-crt-php", + "version": "v1.2.7", + "source": { + "type": "git", + "url": "https://github.com/awslabs/aws-crt-php.git", + "reference": "d71d9906c7bb63a28295447ba12e74723bd3730e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/awslabs/aws-crt-php/zipball/d71d9906c7bb63a28295447ba12e74723bd3730e", + "reference": "d71d9906c7bb63a28295447ba12e74723bd3730e", + "shasum": "" + }, + "require": { + "php": ">=5.5" + }, + "require-dev": { + "phpunit/phpunit": "^4.8.35||^5.6.3||^9.5", + "yoast/phpunit-polyfills": "^1.0" + }, + "suggest": { + "ext-awscrt": "Make sure you install awscrt native extension to use any of the functionality." + }, + "type": "library", + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "AWS SDK Common Runtime Team", + "email": "aws-sdk-common-runtime@amazon.com" + } + ], + "description": "AWS Common Runtime for PHP", + "homepage": "https://github.com/awslabs/aws-crt-php", + "keywords": [ + "amazon", + "aws", + "crt", + "sdk" + ], + "support": { + "issues": "https://github.com/awslabs/aws-crt-php/issues", + "source": "https://github.com/awslabs/aws-crt-php/tree/v1.2.7" + }, + "time": "2024-10-18T22:15:13+00:00" + }, + { + "name": "aws/aws-sdk-php", + "version": "3.391.2", + "source": { + "type": "git", + "url": "https://github.com/aws/aws-sdk-php.git", + "reference": "50e1ed57a0b75921efcbcabe2f8702e7508c501c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/50e1ed57a0b75921efcbcabe2f8702e7508c501c", + "reference": "50e1ed57a0b75921efcbcabe2f8702e7508c501c", + "shasum": "" + }, + "require": { + "aws/aws-crt-php": "^1.2.3", + "ext-json": "*", + "ext-pcre": "*", + "ext-simplexml": "*", + "guzzlehttp/guzzle": "^7.8.2 || ^8.0", + "guzzlehttp/promises": "^2.0.3 || ^3.0", + "guzzlehttp/psr7": "^2.6.3 || ^3.0", + "mtdowling/jmespath.php": "^2.9.1", + "php": ">=8.1", + "psr/http-message": "^1.0 || ^2.0", + "symfony/filesystem": "^v5.4.45 || ^v6.4.3 || ^v7.1.0 || ^v8.0.0" + }, + "require-dev": { + "andrewsville/php-token-reflection": "^1.4", + "aws/aws-php-sns-message-validator": "~1.0", + "behat/behat": "~3.0", + "composer/composer": "^2.7.8", + "dms/phpunit-arraysubset-asserts": "^v0.5.0", + "doctrine/cache": "~1.4", + "ext-dom": "*", + "ext-openssl": "*", + "ext-sockets": "*", + "phpunit/phpunit": "^10.0", + "psr/cache": "^2.0 || ^3.0", + "psr/simple-cache": "^2.0 || ^3.0", + "sebastian/comparator": "^1.2.3 || ^4.0 || ^5.0", + "yoast/phpunit-polyfills": "^2.0" + }, + "suggest": { + "aws/aws-php-sns-message-validator": "To validate incoming SNS notifications", + "doctrine/cache": "To use the DoctrineCacheAdapter", + "ext-curl": "To send requests using cURL", + "ext-openssl": "Allows working with CloudFront private distributions and verifying received SNS messages", + "ext-pcntl": "To use client-side monitoring", + "ext-sockets": "To use client-side monitoring" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.0-dev" + } + }, + "autoload": { + "files": [ + "src/functions.php" + ], + "psr-4": { + "Aws\\": "src/" + }, + "exclude-from-classmap": [ + "src/data/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Amazon Web Services", + "homepage": "https://aws.amazon.com" + } + ], + "description": "AWS SDK for PHP - Use Amazon Web Services in your PHP project", + "homepage": "https://aws.amazon.com/sdk-for-php", + "keywords": [ + "amazon", + "aws", + "cloud", + "dynamodb", + "ec2", + "glacier", + "s3", + "sdk" + ], + "support": { + "forum": "https://github.com/aws/aws-sdk-php/discussions", + "issues": "https://github.com/aws/aws-sdk-php/issues", + "source": "https://github.com/aws/aws-sdk-php/tree/3.391.2" + }, + "time": "2026-08-10T18:12:09+00:00" + }, { "name": "blade-ui-kit/blade-heroicons", "version": "2.7.0", @@ -2975,6 +3126,61 @@ }, "time": "2026-07-06T14:42:07+00:00" }, + { + "name": "league/flysystem-aws-s3-v3", + "version": "3.35.2", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/flysystem-aws-s3-v3.git", + "reference": "8475ef9adfc6498b85469e2abec6fe3118cd08c4" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/flysystem-aws-s3-v3/zipball/8475ef9adfc6498b85469e2abec6fe3118cd08c4", + "reference": "8475ef9adfc6498b85469e2abec6fe3118cd08c4", + "shasum": "" + }, + "require": { + "aws/aws-sdk-php": "^3.371.5", + "league/flysystem": "^3.10.0", + "league/mime-type-detection": "^1.0.0", + "php": "^8.0.2" + }, + "conflict": { + "guzzlehttp/guzzle": "<7.0", + "guzzlehttp/ringphp": "<1.1.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "League\\Flysystem\\AwsS3V3\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Frank de Jonge", + "email": "info@frankdejonge.nl" + } + ], + "description": "AWS S3 filesystem adapter for Flysystem.", + "keywords": [ + "Flysystem", + "aws", + "file", + "files", + "filesystem", + "s3", + "storage" + ], + "support": { + "source": "https://github.com/thephpleague/flysystem-aws-s3-v3/tree/3.35.2" + }, + "time": "2026-07-01T23:25:49+00:00" + }, { "name": "league/flysystem-local", "version": "3.31.0", @@ -3759,6 +3965,72 @@ ], "time": "2026-01-02T08:56:05+00:00" }, + { + "name": "mtdowling/jmespath.php", + "version": "2.9.2", + "source": { + "type": "git", + "url": "https://github.com/jmespath/jmespath.php.git", + "reference": "2157c5e50e813ec6a96c1eed3be7f64a20fb32a8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/jmespath/jmespath.php/zipball/2157c5e50e813ec6a96c1eed3be7f64a20fb32a8", + "reference": "2157c5e50e813ec6a96c1eed3be7f64a20fb32a8", + "shasum": "" + }, + "require": { + "php": "^7.2.5 || ^8.0", + "symfony/polyfill-mbstring": "^1.17" + }, + "require-dev": { + "composer/xdebug-handler": "^3.0.3", + "phpunit/phpunit": "^8.5.52" + }, + "bin": [ + "bin/jp.php" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.9-dev" + } + }, + "autoload": { + "files": [ + "src/JmesPath.php" + ], + "psr-4": { + "JmesPath\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Graham Campbell", + "email": "hello@gjcampbell.co.uk", + "homepage": "https://github.com/GrahamCampbell" + }, + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + } + ], + "description": "Declaratively specify how to extract elements from a JSON document", + "keywords": [ + "json", + "jsonpath" + ], + "support": { + "issues": "https://github.com/jmespath/jmespath.php/issues", + "source": "https://github.com/jmespath/jmespath.php/tree/2.9.2" + }, + "time": "2026-07-06T18:56:19+00:00" + }, { "name": "nesbot/carbon", "version": "3.13.2", @@ -7495,6 +7767,76 @@ ], "time": "2026-06-05T06:23:12+00:00" }, + { + "name": "symfony/filesystem", + "version": "v7.4.15", + "source": { + "type": "git", + "url": "https://github.com/symfony/filesystem.git", + "reference": "ff16a16bf87fdf264638b8f6995b3515975e3c79" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/ff16a16bf87fdf264638b8f6995b3515975e3c79", + "reference": "ff16a16bf87fdf264638b8f6995b3515975e3c79", + "shasum": "" + }, + "require": { + "php": ">=8.2", + "symfony/polyfill-ctype": "~1.8", + "symfony/polyfill-mbstring": "~1.8" + }, + "require-dev": { + "symfony/process": "^6.4|^7.0|^8.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\Filesystem\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Provides basic utilities for the filesystem", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/filesystem/tree/v7.4.15" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-07-22T07:36:05+00:00" + }, { "name": "symfony/finder", "version": "v7.4.14", diff --git a/config/filesystems.php b/config/filesystems.php index 6b1506d..7be64f7 100644 --- a/config/filesystems.php +++ b/config/filesystems.php @@ -72,7 +72,7 @@ return [ 'secret' => env('S3_SECRET_KEY'), 'region' => env('S3_REGION', 'auto'), 'bucket' => env('S3_BUCKET'), - 'url' => env('S3_URL'), + 'url' => env('S3_URL') ?: (env('S3_ENDPOINT') ? rtrim(env('S3_ENDPOINT'), '/').'/'.env('S3_BUCKET') : null), 'endpoint' => env('S3_ENDPOINT'), 'use_path_style_endpoint' => env('S3_USE_PATH_STYLE_ENDPOINT', false), 'visibility' => env('S3_VISIBILITY', 'public'),