Optimize academy

2026-06-09 13:16:01 +02:00
parent f89ee937c0
commit 5af95f6533
109 changed files with 6862 additions and 719 deletions
--- a/app/Jobs/AutoTagArtworkJob.php
+++ b/app/Jobs/AutoTagArtworkJob.php
@@ -54,6 +54,10 @@ final class AutoTagArtworkJob implements ShouldQueue

    public function handle(TagService $tagService, TagNormalizer $normalizer, ?VisionService $vision = null): void
    {
+        if (! (bool) config('vision.auto_tagging.enabled', false)) {
+            return;
+        }
+
        $vision ??= app(VisionService::class);

        if (! $vision->isEnabled()) {
--- a/app/Jobs/GenerateDerivativesJob.php
+++ b/app/Jobs/GenerateDerivativesJob.php
@@ -54,10 +54,18 @@ final class GenerateDerivativesJob implements ShouldQueue
        }

        // Auto-tagging is async and must never block publish.
-        AutoTagArtworkJob::dispatch($this->artworkId, $this->hash)->afterCommit();
-        DetectArtworkMaturityJob::dispatch($this->artworkId, $this->hash)->afterCommit();
-        GenerateArtworkEmbeddingJob::dispatch($this->artworkId, $this->hash)->afterCommit();
-        AnalyzeArtworkAiAssistJob::dispatch($this->artworkId)->afterCommit();
+        if ((bool) config('vision.auto_tagging.enabled', false)) {
+            AutoTagArtworkJob::dispatch($this->artworkId, $this->hash)->afterCommit();
+        }
+        if ((bool) config('vision.upload.maturity.enabled', false)) {
+            DetectArtworkMaturityJob::dispatch($this->artworkId, $this->hash)->afterCommit();
+        }
+        if ((bool) config('vision.upload.embeddings.enabled', true)) {
+            GenerateArtworkEmbeddingJob::dispatch($this->artworkId, $this->hash)->afterCommit();
+        }
+        if ((bool) config('vision.upload.ai_assist.enabled', false)) {
+            AnalyzeArtworkAiAssistJob::dispatch($this->artworkId)->afterCommit();
+        }
    }

    public function failed(\Throwable $exception): void
--- a/app/Jobs/RecComputeSimilarByTagsJob.php
+++ b/app/Jobs/RecComputeSimilarByTagsJob.php
@@ -9,9 +9,11 @@ use App\Models\RecArtworkRec;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;

 /**
 * Compute tag-based (+ category boost) similarity for artworks.
@@ -30,6 +32,7 @@ final class RecComputeSimilarByTagsJob implements ShouldQueue
    public function __construct(
        private readonly ?int $artworkId = null,
        private readonly int  $batchSize = 200,
+        private readonly ?int $afterArtworkId = null,
    ) {
        $queue = (string) config('recommendations.queue', 'default');
        if ($queue !== '') {
@@ -37,6 +40,22 @@ final class RecComputeSimilarByTagsJob implements ShouldQueue
        }
    }

+    /**
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        if ($this->artworkId === null) {
+            return [];
+        }
+
+        return [
+            (new WithoutOverlapping('rec-similar-tags:'.$this->artworkId))
+                ->expireAfter($this->timeout + 60)
+                ->dontRelease(),
+        ];
+    }
+
    public function handle(): void
    {
        $modelVersion = (string) config('recommendations.similarity.model_version', 'sim_v1');
@@ -51,19 +70,68 @@ final class RecComputeSimilarByTagsJob implements ShouldQueue
            ->pluck('cnt', 'tag_id')
            ->all();

-        $query = Artwork::query()->public()->published()->select('id', 'user_id');
-
        if ($this->artworkId !== null) {
-            $query->where('id', $this->artworkId);
+            $artwork = Artwork::query()->public()->published()->select('id', 'user_id')->find($this->artworkId);
+
+            if (! $artwork instanceof Artwork) {
+                return;
+            }
+
+            $this->processArtworkSafely($artwork, $tagFreqs, $modelVersion, $candidatePool, $maxPerAuthor, $resultLimit);
+
+            return;
        }

-        $query->chunkById($this->batchSize, function ($artworks) use (
-            $tagFreqs, $modelVersion, $candidatePool, $maxPerAuthor, $resultLimit
-        ) {
-            foreach ($artworks as $artwork) {
-                $this->processArtwork($artwork, $tagFreqs, $modelVersion, $candidatePool, $maxPerAuthor, $resultLimit);
-            }
-        });
+        $artworks = Artwork::query()
+            ->public()
+            ->published()
+            ->select('id', 'user_id')
+            ->when($this->afterArtworkId !== null, fn ($query) => $query->where('id', '>', $this->afterArtworkId))
+            ->orderBy('id')
+            ->limit($this->batchSize)
+            ->get();
+
+        if ($artworks->isEmpty()) {
+            return;
+        }
+
+        foreach ($artworks as $artwork) {
+            $this->processArtworkSafely($artwork, $tagFreqs, $modelVersion, $candidatePool, $maxPerAuthor, $resultLimit);
+        }
+
+        if ($artworks->count() === $this->batchSize) {
+            static::dispatch(null, $this->batchSize, (int) $artworks->last()->id);
+        }
+    }
+
+    public function failed(\Throwable $exception): void
+    {
+        Log::error('[RecComputeSimilarByTags] Job failed permanently.', [
+            'artwork_id' => $this->artworkId,
+            'batch_size' => $this->batchSize,
+            'after_artwork_id' => $this->afterArtworkId,
+            'attempts' => $this->attempts(),
+            'exception_class' => $exception::class,
+            'exception_message' => $exception->getMessage(),
+        ]);
+    }
+
+    private function processArtworkSafely(
+        Artwork $artwork,
+        array $tagFreqs,
+        string $modelVersion,
+        int $candidatePool,
+        int $maxPerAuthor,
+        int $resultLimit,
+    ): void {
+        try {
+            $this->processArtwork($artwork, $tagFreqs, $modelVersion, $candidatePool, $maxPerAuthor, $resultLimit);
+        } catch (\Throwable $exception) {
+            Log::warning("[RecComputeSimilarByTags] Failed for artwork {$artwork->id}: {$exception->getMessage()}", [
+                'artwork_id' => $artwork->id,
+                'exception_class' => $exception::class,
+            ]);
+        }
    }

    private function processArtwork(
--- a/app/Jobs/RecComputeSimilarHybridJob.php
+++ b/app/Jobs/RecComputeSimilarHybridJob.php
@@ -9,6 +9,7 @@ use App\Models\RecArtworkRec;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\DB;
@@ -25,7 +26,10 @@ final class RecComputeSimilarHybridJob implements ShouldQueue
 {
    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

-    public int $tries   = 2;
+    // This recompute is idempotent and already guards per-artwork execution.
+    // Keep retries to a minimum so transient failures do not turn into
+    // Horizon's max-attempt exception noise.
+    public int $tries   = 1;
    public int $timeout = 900;

    public function __construct(
@@ -38,6 +42,24 @@ final class RecComputeSimilarHybridJob implements ShouldQueue
        }
    }

+    /**
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        if ($this->artworkId === null) {
+            return [];
+        }
+
+        return [
+            // Many artwork lifecycle events can queue this same recompute burstily.
+            // Keep only one in flight per artwork and drop overlapping duplicates.
+            (new WithoutOverlapping('rec-similar-hybrid:'.$this->artworkId))
+                ->expireAfter($this->timeout + 60)
+                ->dontRelease(),
+        ];
+    }
+
    public function handle(): void
    {
        $modelVersion   = (string) config('recommendations.similarity.model_version', 'sim_v1');
@@ -50,26 +72,90 @@ final class RecComputeSimilarHybridJob implements ShouldQueue
            ? (array) config('recommendations.similarity.weights_with_vector')
            : (array) config('recommendations.similarity.weights_without_vector');

-        $query = Artwork::query()->public()->published()->select('id', 'user_id');
-
        if ($this->artworkId !== null) {
-            $query->where('id', $this->artworkId);
+            $artwork = Artwork::query()
+                ->public()
+                ->published()
+                ->select('id', 'user_id')
+                ->find($this->artworkId);
+
+            if (! $artwork instanceof Artwork) {
+                return;
+            }
+
+            $this->processArtworkSafely(
+                collect([$artwork]),
+                $modelVersion,
+                $vectorEnabled,
+                $resultLimit,
+                $maxPerAuthor,
+                $minCatsTop12,
+                $weights,
+            );
+
+            return;
        }

-        $query->chunkById($this->batchSize, function ($artworks) use (
-            $modelVersion, $vectorEnabled, $resultLimit, $maxPerAuthor, $minCatsTop12, $weights
-        ) {
-            foreach ($artworks as $artwork) {
-                try {
-                    $this->processArtwork(
-                        $artwork, $modelVersion, $vectorEnabled, $resultLimit,
-                        $maxPerAuthor, $minCatsTop12, $weights
-                    );
-                } catch (\Throwable $e) {
-                    Log::warning("[RecComputeSimilarHybrid] Failed for artwork {$artwork->id}: {$e->getMessage()}");
-                }
+        Artwork::query()
+            ->public()
+            ->published()
+            ->select('id', 'user_id')
+            ->chunkById($this->batchSize, function ($artworks) use (
+                $modelVersion, $vectorEnabled, $resultLimit, $maxPerAuthor, $minCatsTop12, $weights
+            ) {
+                $this->processArtworkSafely(
+                    $artworks,
+                    $modelVersion,
+                    $vectorEnabled,
+                    $resultLimit,
+                    $maxPerAuthor,
+                    $minCatsTop12,
+                    $weights,
+                );
+            });
+    }
+
+    public function failed(\Throwable $exception): void
+    {
+        Log::error('[RecComputeSimilarHybrid] Job failed permanently.', [
+            'artwork_id' => $this->artworkId,
+            'batch_size' => $this->batchSize,
+            'attempts' => $this->attempts(),
+            'exception_class' => $exception::class,
+            'exception_message' => $exception->getMessage(),
+        ]);
+    }
+
+    /**
+     * @param  iterable<Artwork>  $artworks
+     */
+    private function processArtworkSafely(
+        iterable $artworks,
+        string $modelVersion,
+        bool $vectorEnabled,
+        int $resultLimit,
+        int $maxPerAuthor,
+        int $minCatsTop12,
+        array $weights,
+    ): void {
+        foreach ($artworks as $artwork) {
+            try {
+                $this->processArtwork(
+                    $artwork,
+                    $modelVersion,
+                    $vectorEnabled,
+                    $resultLimit,
+                    $maxPerAuthor,
+                    $minCatsTop12,
+                    $weights,
+                );
+            } catch (\Throwable $e) {
+                Log::warning("[RecComputeSimilarHybrid] Failed for artwork {$artwork->id}: {$e->getMessage()}", [
+                    'artwork_id' => $artwork->id,
+                    'exception_class' => $e::class,
+                ]);
            }
-        });
+        }
    }

    private function processArtwork(