feat: CMS-1202 send image to queue to process instead of handling it sync

.env.example

@@ -35,9 +35,12 @@ SESSION_DOMAIN=null
 
 BROADCAST_CONNECTION=log
 FILESYSTEM_DISK=local
-QUEUE_CONNECTION=database
+QUEUE_CONNECTION=redis
 
-CACHE_STORE=database
+# redis (not database): the async transform's ShouldBeUnique dispatch lock and the
+# cache flag / failure sentinel / frame memo all use the cache store. A fresh app
+# ships no migrations, so `database` has no cache/cache_locks tables to write to.
+CACHE_STORE=redis
 # CACHE_PREFIX=
 
 MEMCACHED_HOST=127.0.0.1
@@ -85,4 +88,14 @@ IMAGE_TRANSFORM_MAX_ANIMATED_FRAMES=30
 # cache disk with a bucket/S3 lifecycle rule instead.
 IMAGE_TRANSFORM_CACHE_DISK=local
 
+# Async transform (see "Async processing" in README). On a cache miss the
+# request 302s to the original (short TTL) and queues the transform instead of
+# blocking on it. Requires a running `php artisan queue:work redis`.
+IMAGE_TRANSFORM_ASYNC_ENABLED=true
+# IMAGE_TRANSFORM_QUEUE=default
+# IMAGE_TRANSFORM_QUEUE_CONNECTION=
+# IMAGE_TRANSFORM_PENDING_MAX_AGE=60
+# IMAGE_TRANSFORM_JOB_UNIQUE_FOR=300
+# IMAGE_TRANSFORM_FAILED_LIFETIME=3600
+
 VITE_APP_NAME="${APP_NAME}"

README.md

@@ -21,5 +21,47 @@ Transformed images are cached on the disk set by `IMAGE_TRANSFORM_CACHE_DISK` (d
 
   Do **not** append `*`. S3 and R2 lifecycle filters match a *literal* key prefix; they do not support wildcards. A prefix of `_cache/image-transform-url/*` is taken literally, matches no cache object, and the rule silently expires nothing — leaving the bucket unbounded. To target the whole bucket instead, use an empty prefix, not `*`.
 
+# Allowed sizes + formats
+
+Without a whitelist, `width=`/`height=` accept any positive integer, so a client can
+enumerate `width=101`, `width=102`, `width=103`... — each one a distinct cache entry
+(and, on the async path, a distinct queued job). `config/image-transform-url.php` →
+`sizes` bounds this: a requested width/height is rounded to the **nearest** value in
+the list (not rejected), so e.g. `width=203` and `width=210` both resolve to the same
+200px cache/dedup entry. Default: `[200, 400, 600, 800, 1000, 1280, 1440, 1920]`. Set
+to `[]` to disable.
+
+`qualities` applies the same nearest-value clamp to `quality=` (default:
+`[60, 75, 90, 100]`) — otherwise `quality=1..100` is a 100-value enumeration space
+per size/format combo that the size whitelist alone doesn't bound.
+
+`allowed_formats` restricts the `format=` option — an explicit format not in the list
+404s (default: `['webp']` only). Omitting `format=` entirely is unaffected by this
+whitelist; the vendor still defaults to the source's own mime type in that case.
+
+# Async processing
+
+On a cache **miss** the server does not transform inline. It **302s to the original**
+(short-lived, `Cache-Control: public, max-age=60`) so the client gets an image
+immediately, and queues the transform in the background. The next request for the
+same URL hits the now-populated cache and gets the optimized image (200 `X-Cache: HIT`).
+
+- **Required:** a running worker — `php artisan queue:work redis --queue=default`
+  (use Supervisor/Horizon in production). Without a worker, misses keep redirecting
+  to the original and never optimize.
+- `QUEUE_CONNECTION=redis` (phpredis extension required). Ensure a `failed_jobs` table
+  exists (`php artisan queue:failed-table && php artisan migrate`), **or** set
+  `QUEUE_FAILED_DRIVER=null` — the app's own failure sentinel does not need it.
+- Recommended: `CACHE_STORE=redis` so the dispatch dedup lock (`ShouldBeUnique`) is
+  atomic and fast.
+- On a job failure (undecodable source, encoder error), a permanent sentinel is
+  cached and the request path serves the long-cache (30-day) redirect instead of
+  re-dispatching — matching the synchronous decode-failure behavior.
+- Tuning knobs (see `config/image-transform-url.php` → `async`):
+  `IMAGE_TRANSFORM_ASYNC_ENABLED` (kill-switch → synchronous transform),
+  `IMAGE_TRANSFORM_PENDING_MAX_AGE` (temporary redirect TTL),
+  `IMAGE_TRANSFORM_JOB_UNIQUE_FOR` (dispatch dedup window),
+  `IMAGE_TRANSFORM_FAILED_LIFETIME` (failure sentinel TTL).
+
 # More documentation
 Please read [here](https://image-transform-url.julian.center/installation)

app/Actions/TransformImageAction.php

@@ -7,11 +7,13 @@
 use AceOfAces\LaravelImageTransformUrl\Actions\TransformImageAction as BaseTransformImageAction;
 use AceOfAces\LaravelImageTransformUrl\ValueObjects\ImageResult;
 use AceOfAces\LaravelImageTransformUrl\ValueObjects\ImageSource;
+use App\Jobs\ProcessImageTransform;
 use Illuminate\Http\Exceptions\HttpResponseException;
 use Illuminate\Support\Facades\App;
 use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\File;
 use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\RateLimiter;
 use Illuminate\Support\Facades\Storage;
 use Intervention\Image\Exceptions\DecoderException;
 use Intervention\Image\Exceptions\EncoderException;
@@ -56,11 +58,40 @@ public function handle(?string $ip, ?string $pathPrefix, string $options, ?strin
             // $pathPrefix/$path by reference — so the cache key below matches the
             // vendor's writer, including on the default (no-prefix) route.
             $source = $this->handlePath($pathPrefix, $path);
+            $options = $this->normalizeOptions($options);
 
             if ($cached = $this->servedFromCache($pathPrefix, $path, $options)) {
                 return $cached; // disk-native cache read — works for S3 (the vendor's File:: read does not)
             }
 
+            // When image is not optimized, dispatch to the queue and return a temporary redirect
+            if (config()->boolean('image-transform-url.async.enabled')) {
+                // Prior permanent failure — serve the long-cache redirect, do not re-dispatch.
+                if (Cache::has($this->failureSentinelKey($pathPrefix, $path, $options))) {
+                    $this->redirectToOriginal($source);
+                }
+
+                $this->guardAnimatedFrames($ip, $source, (string) $path); // over-cap → permanent redirect
+
+                // Throttle only the DISPATCH, never the redirect. width/height/quality
+                // are clamped to a fixed whitelist above (normalizeOptions()), so a
+                // param-enumeration attack no longer forks the dedup key — but the
+                // whitelist is config-driven (empty `sizes`/`qualities` disables it),
+                // so keep an IP+path throttle as a queue-flood guard. Crucially it
+                // gates ONLY the enqueue: an over-limit miss still gets the temporary
+                // redirect below, so a cache miss ALWAYS serves an image (a hard 429
+                // here would hand the client a broken image during the processing gap).
+                if ($this->dispatchRateLimitPassed($ip, (string) $path)) {
+                    ProcessImageTransform::dispatch($pathPrefix, $path, $options); // deduped by ShouldBeUnique
+                }
+
+                // TEMPORARY redirect — short TTL so the CDN re-checks and picks up the HIT.
+                $this->redirectToOriginal($source, [
+                    'Cache-Control' => 'public, max-age='.config()->integer('image-transform-url.async.pending_redirect_max_age'),
+                ]);
+            }
+
+            // If image is optimized, show the image
             $this->guardAnimatedFrames($ip, $source, (string) $path); // may throttle + redirect
 
             return parent::handle($ip, $pathPrefix, $options, $path);
@@ -69,16 +100,87 @@ public function handle(?string $ip, ?string $pathPrefix, string $options, ?strin
         } catch (HttpExceptionInterface $e) {
             throw $e; // preserve HTTP control-flow (404 not-found, 429 rate-limit, etc.)
         } catch (DecoderException|EncoderException|NotSupportedException|\ImagickException $e) {
-            // Decode/encode failure — serve the original instead of a 500. Intervention
-            // wraps decode faults in DecoderException, but its Imagick encoders throw
-            // ImagickException bare (e.g. CacheResourcesExhausted under a policy.xml /
-            // memory limit), so catch that too. ImagickException only originates from
-            // Imagick ops, so genuine faults (S3, config, TypeError) still surface as 5xx.
+            // Decode/encode failure — serve the original instead of a 500.
             report($e);
             $this->redirectToOriginal($source); // throws HttpResponseException
         }
     }
 
+    /**
+     * App-level abuse guard: canonicalize width/height to the nearest allowed
+     * size and reject a non-whitelisted explicit format, before anything else
+     * reads/writes the cache or dispatches a job. Re-serializes with sorted
+     * keys so option order can't fork the cache/dedup key either (e.g.
+     * `format=webp,width=200` and `width=200,format=webp` collapse to one).
+     */
+    protected function normalizeOptions(string $rawOptions): string
+    {
+        $options = static::parseOptions($rawOptions);
+
+        if (array_key_exists('width', $options)) {
+            $options['width'] = $this->nearestAllowedValue((int) $options['width'], config()->array('image-transform-url.sizes'));
+        }
+
+        if (array_key_exists('height', $options)) {
+            $options['height'] = $this->nearestAllowedValue((int) $options['height'], config()->array('image-transform-url.sizes'));
+        }
+
+        if (array_key_exists('quality', $options)) {
+            $options['quality'] = $this->nearestAllowedValue((int) $options['quality'], config()->array('image-transform-url.qualities'));
+        }
+
+        if (array_key_exists('format', $options)) {
+            abort_unless(
+                in_array($options['format'], config()->array('image-transform-url.allowed_formats'), true),
+                404,
+            );
+        }
+
+        ksort($options);
+
+        return collect($options)->map(fn ($value, $key) => "{$key}={$value}")->implode(',');
+    }
+
+    /**
+     * Round a requested width/height/quality to the nearest value in the given
+     * whitelist, bounding cache/queue cardinality per source image (an
+     * enumerated width=101, width=102, ... all collapse onto one entry). An
+     * empty whitelist disables the guard — the raw value passes through.
+     */
+    protected function nearestAllowedValue(int $value, array $whitelist): int
+    {
+        if ($whitelist === []) {
+            return $value;
+        }
+
+        return collect($whitelist)->sortBy(fn (int $allowed) => abs($allowed - $value))->first();
+    }
+
+    /**
+     * Non-aborting dispatch throttle for the async miss path: returns whether the
+     * source may be enqueued on this request. Mirrors the vendor rate-limit KEY
+     * (image-transform-url:$ip:$path) so both share one bucket, but returns a bool
+     * instead of aborting 429 — the caller still serves the temporary redirect when
+     * this is false, so a miss never hands the client a broken image. Returns true
+     * when rate limiting is disabled or the current environment is exempt.
+     */
+    protected function dispatchRateLimitPassed(?string $ip, string $path): bool
+    {
+        if (
+            ! config()->boolean('image-transform-url.rate_limit.enabled') ||
+            in_array(App::environment(), config()->array('image-transform-url.rate_limit.disabled_for_environments'), true)
+        ) {
+            return true;
+        }
+
+        return (bool) RateLimiter::attempt(
+            key: 'image-transform-url:'.$ip.':'.$path,
+            maxAttempts: config()->integer('image-transform-url.rate_limit.max_attempts'),
+            callback: fn () => true,
+            decaySeconds: config()->integer('image-transform-url.rate_limit.decay_seconds'),
+        );
+    }
+
     /**
      * Redirect animations with more than the configured frame count straight to
      * the original, before Imagick decodes (and coalesces) them into GBs of RAM.
@@ -218,6 +320,80 @@ protected function storeCachedImage(?string $pathPrefix, ?string $path, array $o
         }
     }
 
+    /**
+     * Background half of the async miss path (called from ProcessImageTransform).
+     * Reuses the vendor pipeline (DRY — no re-implementing Intervention), then
+     * stores the result explicitly and exactly once from the returned bytes.
+     */
+    public function processAndCache(?string $pathPrefix, ?string $path, string $rawOptions): void
+    {
+        // Trusted internal call — the rate limiter is a public-abuse guard, and the
+        // parent's own cache read/deferred-store must not run (we store explicitly
+        // below). A persistent `queue:work` process reuses the same booted app
+        // across many jobs (config is NOT reset between them), so these overrides
+        // MUST be restored — otherwise the first async job permanently disables
+        // caching/rate-limiting for every request that process (or, under the
+        // `sync` queue driver, the current request/test) handles afterwards.
+        $originalRateLimit = config()->boolean('image-transform-url.rate_limit.enabled');
+        $originalCacheEnabled = config()->boolean('image-transform-url.cache.enabled');
+
+        config()->set('image-transform-url.rate_limit.enabled', false);
+        config()->set('image-transform-url.cache.enabled', false);
+
+        try {
+            $result = parent::handle(null, $pathPrefix, $rawOptions, $path);
+
+            $this->storeResultBytes($pathPrefix, $path, $rawOptions, $result);
+        } finally {
+            config()->set('image-transform-url.rate_limit.enabled', $originalRateLimit);
+            config()->set('image-transform-url.cache.enabled', $originalCacheEnabled);
+        }
+    }
+
+    /**
+     * Store the transform from an ImageResult's raw bytes
+     */
+    protected function storeResultBytes(?string $pathPrefix, ?string $path, string $rawOptions, ImageResult $result): void
+    {
+        $options = static::parseOptions($rawOptions);
+        $diskName = config()->string('image-transform-url.cache.disk');
+
+        Storage::disk($diskName)->put($this->getCacheEndPath($pathPrefix, $path, $options), $result->content);
+
+        Cache::put(
+            key: 'image-transform-url:'.$this->getCachePath($pathPrefix, $path, $options),
+            value: true,
+            ttl: config()->integer('image-transform-url.cache.lifetime'),
+        );
+
+        if (config("filesystems.disks.{$diskName}.driver") !== 's3') {
+            $this->manageCacheSize();
+        }
+    }
+
+    /**
+     * Cache key for the permanent-failure sentinel written by the job's failed().
+     */
+    protected function failureSentinelKey(?string $pathPrefix, ?string $path, string $rawOptions): string
+    {
+        $options = static::parseOptions($rawOptions);
+
+        return 'image-transform-url:failed:'.$this->getCacheEndPath($pathPrefix, $path, $options);
+    }
+
+    /**
+     * Mark a transform as permanently failed so the request path stops
+     * re-dispatching and serves the long-cache permanent redirect instead.
+     */
+    public function markTransformFailed(?string $pathPrefix, ?string $path, string $rawOptions): void
+    {
+        Cache::put(
+            $this->failureSentinelKey($pathPrefix, $path, $rawOptions),
+            true,
+            config()->integer('image-transform-url.async.failed_lifetime'),
+        );
+    }
+
     /**
      * Read the raw source bytes for a disk- or local-based source.
      */
@@ -233,19 +409,24 @@ protected function readSourceBytes(ImageSource $source): string
      * an image (and the app never streams a broken/huge file through PHP). Takes
      * the already-resolved source (no re-resolve / extra metadata calls). The
      * bucket is public (see README/config), so a plain url() is correct and stays
-     * CDN-cacheable; the configured cache headers let the CDN absorb it. Only disk
-     * sources have a public URL — a local source has none, so it 404s honestly
-     * rather than emitting a malformed /storage//abs/path redirect. Throws, so the
-     * caller never falls through.
+     * CDN-cacheable. Only disk sources have a public URL — a local source has
+     * none, so it 404s honestly rather than emitting a malformed /storage//abs/path
+     * redirect. Throws, so the caller never falls through.
+     *
+     * $headers defaults to the permanent (30-day) config headers — used for the
+     * frame-guard/failure/sentinel redirects, where the source will never
+     * transform. The async-miss path passes a short-lived header set instead, so
+     * the CDN re-checks origin once the queued transform finishes.
      */
-    protected function redirectToOriginal(?ImageSource $source): never
+    protected function redirectToOriginal(?ImageSource $source, ?array $headers = null): never
     {
         abort_unless($source?->type === 'disk', 404);
+        $headers ??= config()->array('image-transform-url.headers');
 
         $url = Storage::disk((string) $source->disk)->url($source->path);
 
         throw new HttpResponseException(
-            redirect()->away($url)->withHeaders(config()->array('image-transform-url.headers'))
+            redirect()->away($url)->withHeaders($headers)
         );
     }
 }