[6.x] Improve term/entry stache warming performance

src/Stache/Indexes/Index.php

@@ -58,6 +58,17 @@ public function push($value)
         $this->items[] = $value;
     }
 
+    /**
+     * Bulk-sets the index items without going through getItems(). Used by
+     * Store::warmValueIndexes() to write pre-accumulated values in one shot.
+     */
+    public function setItems(array $items): static
+    {
+        $this->items = $items;
+
+        return $this;
+    }
+
     public function load()
     {
         if ($this->loaded) {

src/Stache/Indexes/Terms/Associations.php

@@ -2,32 +2,40 @@
 
 namespace Statamic\Stache\Indexes\Terms;
 
+use Statamic\Facades\Stache;
 use Statamic\Facades\Taxonomy;
 use Statamic\Stache\Indexes\Index;
+use Statamic\Support\Arr;
 use Statamic\Support\Str;
 
 class Associations extends Index
 {
+    /**
+     * Builds the term→entry association map for a taxonomy from the entries
+     * stores' value indexes rather than by loading every Entry from disk.
+     *
+     * During stache:warm those indexes are already cached (Pass 1 runs before
+     * this index is built in Pass 2). Outside of warming they'll be built on
+     * demand, which costs the same as the old query-based approach did.
+     */
     public function getItems()
     {
         return Taxonomy::findByHandle($handle = $this->store->childKey())
             ->collections()
             ->flatMap(function ($collection) use ($handle) {
-                return $collection->queryEntries()
-                    ->where($handle, '<>', null)
-                    ->get()
-                    ->flatMap(function ($entry) use ($handle) {
-                        return collect($entry->value($handle))
-                            ->map(function ($value) use ($entry) {
-                                return [
-                                    'value' => $value,
-                                    'slug' => Str::slug($value),
-                                    'entry' => $entry->id(),
-                                    'collection' => $entry->collectionHandle(),
-                                    'site' => $entry->locale(),
-                                ];
-                            });
-                    })->all();
+                $entries = Stache::store('entries')->store($collection->handle());
+                $ids = $entries->index('id');
+                $sites = $entries->index('site');
+
+                return $entries->index($handle)->items()
+                    ->reject(fn ($terms) => is_null($terms))
+                    ->flatMap(fn ($terms, $key) => collect(Arr::wrap($terms))->map(fn ($term) => [
+                        'value' => $term,
+                        'slug' => Str::slug($term),
+                        'entry' => $ids->get($key),
+                        'collection' => $collection->handle(),
+                        'site' => $sites->get($key),
+                    ]));
             })->all();
     }
 

src/Stache/Stache.php

@@ -137,7 +137,7 @@ public function warm()
         if ($this->shouldUseParallelWarming($stores)) {
             $this->warmInParallel($stores);
         } else {
-            $stores->each->warm();
+            $this->warmSequentially($stores);
         }
 
         $this->stopTimer();
@@ -280,6 +280,25 @@ protected function shouldUseParallelWarming($stores): bool
         return true;
     }
 
+    /**
+     * Two-pass warm: Pass 1 caches all per-item value indexes (including entries'
+     * taxonomy indexes) across every store before Pass 2 runs. This lets
+     * Terms\Associations read from the cache in Pass 2 instead of loading all
+     * Entry objects from disk, which was the main source of slow warm times.
+     *
+     * Stores that define their own warm() are warmed through it instead, so
+     * third-party overrides aren't silently bypassed.
+     */
+    protected function warmSequentially($stores)
+    {
+        [$overriding, $twoPass] = $stores->partition->overridesWarm();
+
+        $overriding->each->warm();
+
+        $twoPass->each->warmValueIndexes();
+        $twoPass->each->warmOtherIndexes();
+    }
+
     protected function warmInParallel($stores)
     {
         try {
@@ -297,7 +316,9 @@ protected function warmInParallel($stores)
 
             $closures = $chunks->map(function ($chunk) {
                 return function () use ($chunk) {
-                    return $chunk->each->warm()->keys()->all();
+                    $this->warmSequentially($chunk);
+
+                    return $chunk->keys()->all();
                 };
             })->all();
 
@@ -310,7 +331,7 @@ protected function warmInParallel($stores)
             Concurrency::driver($driver)->run($closures);
         } catch (\Exception $e) {
             Log::warning('Parallel warming failed, falling back to sequential: '.$e->getMessage());
-            $stores->each->warm();
+            $this->warmSequentially($stores);
         }
     }
 

src/Stache/Stores/AggregateStore.php

@@ -87,13 +87,35 @@ public function warm()
         $this->discoverStores()->each->warm();
     }
 
+    /**
+     * AggregateStore's own warm() just fans out to child stores, which is exactly what
+     * warmValueIndexes()/warmOtherIndexes() do too, so it can still take the two-pass
+     * path. Subclasses that define their own warm() are reported as overriding it.
+     */
+    public function overridesWarm(): bool
+    {
+        return (new \ReflectionMethod($this, 'warm'))->getDeclaringClass()->getName() !== self::class;
+    }
+
     public function resetMemoizedState()
     {
         parent::resetMemoizedState();
 
         $this->stores->each->resetMemoizedState();
     }
 
+    /** @see Store::warmValueIndexes() */
+    public function warmValueIndexes()
+    {
+        $this->discoverStores()->each->warmValueIndexes();
+    }
+
+    /** @see Store::warmOtherIndexes() */
+    public function warmOtherIndexes()
+    {
+        $this->discoverStores()->each->warmOtherIndexes();
+    }
+
     public function paths()
     {
         return $this->discoverStores()->flatMap(function ($store) {

src/Stache/Stores/Store.php

@@ -409,10 +409,64 @@ public function clear()
     }
 
     public function warm()
+    {
+        $this->warmValueIndexes();
+        $this->warmOtherIndexes();
+    }
+
+    /**
+     * Whether this store provides its own warm() implementation. Such stores are warmed
+     * by calling warm() directly rather than through the two-pass split, so third-party
+     * overrides registered via Stache::registerStore() keep being honoured.
+     */
+    public function overridesWarm(): bool
+    {
+        return (new \ReflectionMethod($this, 'warm'))->getDeclaringClass()->getName() !== self::class;
+    }
+
+    /**
+     * Pass 1 of the 2-pass warm. Loads every file once and accumulates values for
+     * all per-item value indexes in a single loop, then writes each index to the
+     * cache in one shot. This ensures entries' taxonomy indexes (e.g. `categories`)
+     * are cached before Pass 2 runs, so Terms\Associations can build from them.
+     */
+    public function warmValueIndexes()
+    {
+        $valueIndexes = $this->resolveIndexes()->filter(fn ($index) => $this->isPerItemValueIndex($index));
+
+        if ($valueIndexes->isEmpty()) {
+            return;
+        }
+
+        $accumulated = $valueIndexes->map(fn () => [])->all();
+
+        foreach ($this->paths()->keys() as $key) {
+            if (! $item = $this->getItem($key)) {
+                continue;
+            }
+
+            foreach ($valueIndexes as $name => $index) {
+                $accumulated[$name][$key] = $index->getItemValue($item);
+            }
+        }
+
+        $valueIndexes->each(function ($index, $name) use ($accumulated) {
+            $index->setItems($accumulated[$name])->cache();
+        });
+    }
+
+    /**
+     * Pass 2 of the 2-pass warm. Runs after all stores have completed Pass 1, so
+     * indexes that depend on other indexes (e.g. Terms\Associations) can read
+     * them from the cache instead of loading Entry objects from disk.
+     */
+    public function warmOtherIndexes()
     {
         $this->shouldCacheFileItems = true;
 
-        $this->resolveIndexes()->each->update();
+        $this->resolveIndexes()
+            ->reject(fn ($index) => $this->isPerItemValueIndex($index))
+            ->each->update();
 
         $this->shouldCacheFileItems = false;
         $this->fileItems = null;
@@ -426,4 +480,15 @@ public function keys()
 
         return $this->keys = (new Keys($this))->load();
     }
+
+    /**
+     * Whether an index can be built purely by mapping getItemValue() over each item.
+     * Subclasses of Value that override getItems() (e.g. Terms\Value, which merges in
+     * on-the-fly terms) need their own build logic and belong in Pass 2.
+     */
+    private function isPerItemValueIndex(Index $index): bool
+    {
+        return $index instanceof Indexes\Value
+            && (new \ReflectionMethod($index, 'getItems'))->getDeclaringClass()->getName() === Indexes\Value::class;
+    }
 }

tests/Stache/StacheTest.php

@@ -2,21 +2,30 @@
 
 namespace Tests\Stache;
 
+use Facades\Tests\Factories\EntryFactory;
 use Illuminate\Filesystem\Filesystem;
 use Illuminate\Support\Collection;
 use PHPUnit\Framework\Attributes\DataProvider;
 use PHPUnit\Framework\Attributes\Test;
 use Statamic\Facades\Blink;
+use Statamic\Facades\Blueprint;
+use Statamic\Facades\Collection as CollectionFacade;
+use Statamic\Facades\Stache as StacheFacade;
+use Statamic\Facades\Taxonomy;
+use Statamic\Facades\Term;
 use Statamic\Stache\NullLockStore;
 use Statamic\Stache\Stache;
 use Statamic\Stache\Stores\ChildStore;
 use Statamic\Stache\Stores\CollectionsStore;
 use Statamic\Stache\Stores\EntriesStore;
 use Symfony\Component\Lock\LockFactory;
+use Tests\PreventSavingStacheItemsToDisk;
 use Tests\TestCase;
 
 class StacheTest extends TestCase
 {
+    use PreventSavingStacheItemsToDisk;
+
     protected $stache;
 
     public function setUp(): void
@@ -195,4 +204,79 @@ public static function watcherProvider()
             ['production', 'config' => null, 'expected' => false],
         ];
     }
+
+    #[Test]
+    public function warming_builds_term_associations_for_single_item_taxonomy_fields()
+    {
+        Taxonomy::make('tags')->save();
+        CollectionFacade::make('blog')->taxonomies(['tags'])->save();
+        Blueprint::make('blog')->setNamespace('collections.blog')->setContents(['fields' => [
+            ['handle' => 'tags', 'field' => ['type' => 'terms', 'taxonomies' => ['tags'], 'max_items' => 1]],
+        ]])->save();
+        Term::make('alfa')->taxonomy('tags')->data(['title' => 'Alfa'])->save();
+        EntryFactory::collection('blog')->id('1')->slug('one')->data(['tags' => 'alfa'])->create();
+        EntryFactory::collection('blog')->id('2')->slug('two')->data(['tags' => 'bravo'])->create();
+        EntryFactory::collection('blog')->id('3')->slug('three')->data([])->create();
+
+        StacheFacade::clear();
+        StacheFacade::warm();
+
+        $this->assertEquals([
+            ['value' => 'alfa', 'slug' => 'alfa', 'entry' => '1', 'collection' => 'blog', 'site' => 'en'],
+            ['value' => 'bravo', 'slug' => 'bravo', 'entry' => '2', 'collection' => 'blog', 'site' => 'en'],
+        ], StacheFacade::store('terms')->store('tags')->index('associations')->items()->all());
+    }
+
+    #[Test]
+    public function warming_includes_on_the_fly_terms_in_term_value_indexes()
+    {
+        Taxonomy::make('tags')->save();
+        CollectionFacade::make('blog')->taxonomies(['tags'])->save();
+        Term::make('alfa')->taxonomy('tags')->data(['title' => 'Alfa'])->save();
+        EntryFactory::collection('blog')->id('1')->slug('one')->data(['tags' => ['alfa', 'bravo']])->create();
+
+        StacheFacade::clear();
+        StacheFacade::warm();
+
+        $store = StacheFacade::store('terms')->store('tags');
+
+        $this->assertEquals([
+            ['value' => 'alfa', 'slug' => 'alfa', 'entry' => '1', 'collection' => 'blog', 'site' => 'en'],
+            ['value' => 'bravo', 'slug' => 'bravo', 'entry' => '1', 'collection' => 'blog', 'site' => 'en'],
+        ], $store->index('associations')->items()->all());
+
+        $this->assertEquals(['en::alfa' => 'alfa', 'en::bravo' => 'bravo'], $store->index('slug')->items()->all());
+        $this->assertEquals(['en::alfa' => 'en', 'en::bravo' => 'en'], $store->index('site')->items()->all());
+    }
+
+    #[Test]
+    public function warming_calls_warm_on_stores_that_override_it()
+    {
+        $store = new class($this->stache, \Mockery::mock(Filesystem::class)) extends CollectionsStore
+        {
+            public $warmed = false;
+
+            public function warm()
+            {
+                $this->warmed = true;
+            }
+
+            public function warmValueIndexes()
+            {
+                throw new \Exception('Should not be called on a store that overrides warm().');
+            }
+
+            public function warmOtherIndexes()
+            {
+                throw new \Exception('Should not be called on a store that overrides warm().');
+            }
+        };
+
+        $this->stache->registerStore($store);
+        $this->stache->setLockFactory(new LockFactory(new NullLockStore));
+
+        $this->stache->warm();
+
+        $this->assertTrue($store->warmed);
+    }
 }