FOUR-33794 Fix Ellucian clients are reporting slow performance

ProcessMaker/Cache/AbstractCacheFactory.php

@@ -40,7 +40,7 @@ public static function create(CacheManager $cacheManager, CacheMetricsInterface
         $cache = static::createInstance($cacheManager);
 
         // Wrap with metrics decorator
-        if (static::$storeMetrics) {
+        if (static::$storeMetrics && config('metrics.enabled', false)) {
             return new CacheMetricsDecorator($cache, $metrics);
         }
 

ProcessMaker/Console/Kernel.php

@@ -56,46 +56,59 @@ protected function schedule(Schedule $schedule)
             }
         }
 
-        $clearInterval = config('metrics.clear_interval', 10);
-        switch ((int) $clearInterval) {
+        $this->scheduleMetricsClear($schedule);
+
+        // evaluate cases retention policy
+        $schedule->command('cases:retention:evaluate')
+            ->daily()
+            ->onOneServer()
+            ->withoutOverlapping()
+            ->runInBackground();
+
+        // 5 minutes is recommended in https://laravel.com/docs/12.x/horizon#metrics
+        $schedule->command('horizon:snapshot')->everyFiveMinutes();
+    }
+
+    protected function scheduleMetricsClear(Schedule $schedule): void
+    {
+        if (!config('metrics.enabled', false)) {
+            return;
+        }
+
+        $clearInterval = (int) config('metrics.clear_interval', 1440);
+
+        $event = $schedule->command('metrics:clear --stagger');
+        switch ($clearInterval) {
             case 1:
-                $schedule->command('metrics:clear')->everyMinute();
+                $event->everyMinute();
                 break;
             case 2:
-                $schedule->command('metrics:clear')->everyTwoMinutes();
+                $event->everyTwoMinutes();
                 break;
             case 5:
-                $schedule->command('metrics:clear')->everyFiveMinutes();
+                $event->everyFiveMinutes();
                 break;
             case 10:
-                $schedule->command('metrics:clear')->everyTenMinutes();
+                $event->everyTenMinutes();
                 break;
             case 15:
-                $schedule->command('metrics:clear')->everyFifteenMinutes();
+                $event->everyFifteenMinutes();
                 break;
             case 30:
-                $schedule->command('metrics:clear')->everyThirtyMinutes();
+                $event->everyThirtyMinutes();
                 break;
             case 60:
-                $schedule->command('metrics:clear')->hourly();
+                $event->hourly();
                 break;
             case 1440:
-                $schedule->command('metrics:clear')->daily();
+                $event->daily();
                 break;
             default:
-                $schedule->command('metrics:clear')->cron("*/{$clearInterval} * * * *");
+                $event->cron("*/{$clearInterval} * * * *");
                 break;
         }
 
-        // evaluate cases retention policy
-        $schedule->command('cases:retention:evaluate')
-            ->daily()
-            ->onOneServer()
-            ->withoutOverlapping()
-            ->runInBackground();
-
-        // 5 minutes is recommended in https://laravel.com/docs/12.x/horizon#metrics
-        $schedule->command('horizon:snapshot')->everyFiveMinutes();
+        $event->runInBackground()->withoutOverlapping($clearInterval);
     }
 
     /**

ProcessMaker/Exception/MetricsDisabledException.php

@@ -0,0 +1,15 @@
+<?php
+
+namespace ProcessMaker\Exception;
+
+use RuntimeException;
+
+class MetricsDisabledException extends RuntimeException
+{
+    public function __construct()
+    {
+        parent::__construct(
+            'Prometheus metrics are disabled. Set METRICS_ENABLED=true to enable metric collection.'
+        );
+    }
+}

ProcessMaker/Facades/Metrics.php

@@ -15,6 +15,8 @@
  * @method static void counterInc(string $name, string $help = null, array $labels = [])
  * @method static void histogramObserve(string $name, string $help = null, array $labels = [], array $buckets = [0.1, 1, 5, 10], float $executionTime)
  * @method static void clearMetrics()
+ * @method static void collectQueueMetrics()
+ * @method static bool isEnabled()
  */
 class Metrics extends Facade
 {

ProcessMaker/Services/MetricsService.php

@@ -6,6 +6,7 @@
 use Laravel\Horizon\Contracts\JobRepository;
 use Laravel\Horizon\Contracts\MetricsRepository;
 use Laravel\Horizon\Contracts\WorkloadRepository;
+use ProcessMaker\Exception\MetricsDisabledException;
 use ProcessMaker\Facades\Metrics;
 use ProcessMaker\Multitenancy\Tenant;
 use Prometheus\CollectorRegistry;
@@ -22,9 +23,9 @@ class MetricsService
     /**
      * The CollectorRegistry instance used by the MetricsService.
      *
-     * @var CollectorRegistry
+     * @var CollectorRegistry|null
      */
-    private $collectionRegistry;
+    private ?CollectorRegistry $collectionRegistry = null;
 
     /**
      * The namespace used by the MetricsService.
@@ -41,8 +42,21 @@ class MetricsService
     public function __construct(private $adapter = null)
     {
         $this->namespace = config('app.prometheus_namespace', 'app');
+        if (!$this->isEnabled() && $adapter === null) {
+            return;
+        }
+
+        $this->initializeRegistry($adapter);
+    }
+
+    public function isEnabled(): bool
+    {
+        return filter_var(config('metrics.enabled', false), FILTER_VALIDATE_BOOLEAN);
+    }
+
+    private function initializeRegistry($adapter = null): void
+    {
         try {
-            // Set up Redis as the adapter if none is provided
             if ($adapter === null) {
                 $redis = app('redis')->client();
                 $adapter = PrometheusRedis::fromExistingConnection($redis);
@@ -57,14 +71,27 @@ public function __construct(private $adapter = null)
         }
     }
 
+    private function registry(): CollectorRegistry
+    {
+        if (!$this->isEnabled()) {
+            throw new MetricsDisabledException();
+        }
+
+        if (!isset($this->collectionRegistry)) {
+            $this->initializeRegistry($this->adapter);
+        }
+
+        return $this->collectionRegistry;
+    }
+
     /**
      * Get the collection registry.
      *
      * @return CollectorRegistry The collection registry instance.
      */
     public function getCollectionRegistry(): CollectorRegistry
     {
-        return $this->collectionRegistry;
+        return $this->registry();
     }
 
     /**
@@ -79,7 +106,7 @@ public function counter(string $name, string|null $help = null, array $labels =
     {
         $help = $help ?? $name;
 
-        return $this->collectionRegistry->getOrRegisterCounter(
+        return $this->registry()->getOrRegisterCounter(
             $this->namespace,
             $name,
             $help,
@@ -99,7 +126,7 @@ public function gauge(string $name, string|null $help = null, array $labels = []
     {
         $help = $help ?? $name;
 
-        return $this->collectionRegistry->getOrRegisterGauge(
+        return $this->registry()->getOrRegisterGauge(
             $this->namespace,
             $name,
             $help,
@@ -120,7 +147,7 @@ public function histogram(string $name, string $help = null, array $labels = [],
     {
         $help = $help ?? $name;
 
-        return $this->collectionRegistry->getOrRegisterHistogram(
+        return $this->registry()->getOrRegisterHistogram(
             $this->namespace,
             $name,
             $help,
@@ -138,7 +165,11 @@ public function histogram(string $name, string $help = null, array $labels = [],
      */
     public function setGauge(string $name, float $value, array $labelValues = []): void
     {
-        $gauge = $this->collectionRegistry->getGauge($this->namespace, $name);
+        if (!$this->isEnabled()) {
+            return;
+        }
+
+        $gauge = $this->registry()->getGauge($this->namespace, $name);
         $gauge->set($value, $labelValues);
     }
 
@@ -149,6 +180,10 @@ public function setGauge(string $name, float $value, array $labelValues = []): v
      */
     public function renderMetrics(): string
     {
+        if (!$this->isEnabled() || !isset($this->collectionRegistry)) {
+            return '';
+        }
+
         $renderer = new RenderTextFormat();
         $metrics = $this->collectionRegistry->getMetricFamilySamples();
 
@@ -166,6 +201,10 @@ public function renderMetrics(): string
      */
     public function counterInc(string $name, string $help = null, array $labels = []): void
     {
+        if (!$this->isEnabled()) {
+            return;
+        }
+
         // Add system labels
         $labels = $this->addSystemLabels($labels);
         $labelKeys = array_keys($labels);
@@ -185,6 +224,10 @@ public function counterInc(string $name, string $help = null, array $labels = []
      */
     public function histogramObserve(string $name, string $help = null, array $labels = [], array $buckets = [0.1, 1, 5, 10], float $executionTime = 0): void
     {
+        if (!$this->isEnabled()) {
+            return;
+        }
+
         // Add system labels
         $labels = $this->addSystemLabels($labels);
         $labelKeys = array_keys($labels);
@@ -220,6 +263,10 @@ public function addSystemLabels(array $labels)
 
     public function clearMetrics(): void
     {
+        if (!$this->isEnabled() || !isset($this->collectionRegistry)) {
+            return;
+        }
+
         $this->collectionRegistry->wipeStorage();
     }
 
@@ -243,6 +290,10 @@ private function getApplicationVersion()
      */
     public function collectQueueMetrics(): void
     {
+        if (!$this->isEnabled()) {
+            return;
+        }
+
         $metricsRepository = app(MetricsRepository::class);
         $jobsRepository = app(JobRepository::class);
         $workloadRepository = app(WorkloadRepository::class);

config/metrics.php

@@ -1,6 +1,12 @@
 <?php
 
 return [
-    // Interval in minutes for clearing metrics. Can be set via METRICS_CLEAR_INTERVAL env variable.
-    'clear_interval' => env('METRICS_CLEAR_INTERVAL', 10),
+    // Enable Prometheus metric collection (counters, gauges, histograms). Default off.
+    'enabled' => env('METRICS_ENABLED', false),
+
+    // Interval in minutes for clearing metrics. Default 24 hours (1440 minutes).
+    'clear_interval' => env('METRICS_CLEAR_INTERVAL', 1440),
+
+    // Max random delay (seconds) before scheduled clear (--stagger). Capped at clear_interval × 60 at runtime.
+    'clear_stagger_max_seconds' => env('METRICS_CLEAR_STAGGER_MAX_SECONDS', 60),
 ];

routes/console.php

@@ -41,7 +41,16 @@
     }
 })->purpose('Display an inspiring quote');
 
-Artisan::command('metrics:clear', function () {
+Artisan::command('metrics:clear {--stagger : Random delay before clear (scheduler only)}', function () {
+    if ($this->option('stagger')) {
+        $intervalSeconds = max(0, (int) config('metrics.clear_interval', 1440) * 60);
+        $staggerMax = max(0, (int) config('metrics.clear_stagger_max_seconds', 60));
+        $maxSeconds = min($staggerMax, $intervalSeconds);
+        if ($maxSeconds > 0) {
+            sleep(random_int(0, $maxSeconds));
+        }
+    }
+
     ProcessMaker\Facades\Metrics::clearMetrics();
     echo "Metrics cleared.\n";
 })->purpose('Clear all metrics using Metrics::clearMetrics');

tests/Feature/Metrics/TaskMetricsTest.php

@@ -2,6 +2,7 @@
 
 namespace Tests\Feature\Metrics;
 
+use Illuminate\Support\Facades\Config;
 use ProcessMaker\Facades\Metrics;
 use ProcessMaker\Facades\WorkflowManager;
 use ProcessMaker\Models\User;
@@ -10,6 +11,12 @@
 
 class TaskMetricsTest extends TestCase
 {
+    protected function setUp(): void
+    {
+        parent::setUp();
+        Config::set('metrics.enabled', true);
+    }
+
     public function test_task_metric_was_stored()
     {
         $user = User::factory()->create([

tests/Feature/MetricsFacadeTest.php

@@ -3,6 +3,7 @@
 namespace Tests\Feature;
 
 use Illuminate\Support\Facades\App;
+use Illuminate\Support\Facades\Config;
 use ProcessMaker\Facades\Metrics;
 use ProcessMaker\Services\MetricsService;
 use Prometheus\Storage\InMemory;
@@ -19,6 +20,8 @@ protected function setUp(): void
     {
         parent::setUp();
 
+        Config::set('metrics.enabled', true);
+
         // Simulate the MetricsService with an InMemory adapter
         $adapter = new InMemory();
         $mockService = new MetricsService($adapter);

tests/unit/MetricsServiceTest.php

@@ -5,6 +5,7 @@
 use Illuminate\Support\Facades\App;
 use Illuminate\Support\Facades\Config;
 use Mockery;
+use ProcessMaker\Exception\MetricsDisabledException;
 use ProcessMaker\Facades\Metrics;
 use ProcessMaker\Services\MetricsService;
 use Prometheus\CollectorRegistry;
@@ -30,6 +31,8 @@ protected function setUp(): void
     {
         parent::setUp();
 
+        Config::set('metrics.enabled', true);
+
         // Use InMemory storage for testing
         $adapter = new InMemory();
         $this->metricsService = new MetricsService($adapter);
@@ -265,4 +268,18 @@ public function testClearMetrics()
         // Call clearMetrics which should call wipeStorage on the registry.
         $service->clearMetrics();
     }
+
+    public function testMetricsDisabledSkipsRegistration(): void
+    {
+        Config::set('metrics.enabled', false);
+
+        $adapter = new InMemory();
+        $service = new MetricsService($adapter);
+
+        $service->counterInc('disabled_counter', 'Should not register', ['foo' => 'bar']);
+        $this->assertSame('', $service->renderMetrics());
+
+        $this->expectException(MetricsDisabledException::class);
+        $service->counter('disabled_counter_direct');
+    }
 }