From fea8c03af1d713e4b00a0bee993f621ce2680e81 Mon Sep 17 00:00:00 2001 From: Viacheslav Huliaiev Date: Wed, 21 Oct 2026 22:12:15 +0200 Subject: [PATCH 1/2] fix(chat): pass the chat user instructions to the Context Agent The "Chat User Instructions for Chat Completions" admin setting was only used for classic chat. With the Context Agent the instructions were never sent, so agentic chat ignored them. Pass the session's system message as the optional `user_instructions` input to the classic, multimodal and audio agency tasks, if the provider declares it (Context Agent appends it to its own system prompt). Older Context Agent versions keep working as before. Fixes #338 Signed-off-by: Viacheslav Huliaiev --- lib/Service/ChatService.php | 49 +++++-- .../ContextAgentAudioInteractionProvider.php | 9 ++ src/components/AdminSettings.vue | 4 + tests/unit/Service/ChatServiceAgencyTest.php | 131 ++++++++++++++++++ 4 files changed, 184 insertions(+), 9 deletions(-) create mode 100644 tests/unit/Service/ChatServiceAgencyTest.php diff --git a/lib/Service/ChatService.php b/lib/Service/ChatService.php index 1f27de531..538466669 100644 --- a/lib/Service/ChatService.php +++ b/lib/Service/ChatService.php @@ -456,15 +456,7 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $ } } else { // classic chat - $systemPrompt = ''; - try { - $firstMessage = $this->messageMapper->getFirstNMessages($sessionId, 1, Message::ROLE_SYSTEM); - $systemPrompt = $firstMessage->getContent(); - } catch (DoesNotExistException $e) { - $this->logger->info('No system message found in the session', ['exception' => $e, 'sessionId' => $sessionId]); - } catch (MultipleObjectsReturnedException|Exception $e) { - throw new InternalException(previous: $e); - } + $systemPrompt = $this->getSessionSystemPrompt($sessionId); try { $history = $this->getRawLastMessages($sessionId); } catch (Exception|AppConfigTypeConflictException $e) { @@ -655,6 +647,39 @@ public function isMultimodalContextAgentAvailable(): bool { return in_array(\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypeIds()); } + /** + * Get the user instructions stored as the system message of the session (empty string if there is none) + * + * @throws InternalException + */ + private function getSessionSystemPrompt(int $sessionId): string { + try { + $firstMessage = $this->messageMapper->getFirstNMessages($sessionId, 1, Message::ROLE_SYSTEM); + return $firstMessage->getContent(); + } catch (DoesNotExistException $e) { + $this->logger->info('No system message found in the session', ['exception' => $e, 'sessionId' => $sessionId]); + } catch (MultipleObjectsReturnedException|Exception $e) { + throw new InternalException(previous: $e); + } + return ''; + } + + /** + * Pass the session's user instructions to an agency task if its provider supports them. + * The agent appends them to its own system prompt, they do not replace it. + * + * @throws InternalException + */ + private function addAgencyUserInstructions(array &$taskInput, string $taskTypeId, int $sessionId): void { + if (!isset($this->taskProcessingManager->getAvailableTaskTypes()[$taskTypeId]['optionalInputShape']['user_instructions'])) { + return; + } + $userInstructions = $this->getSessionSystemPrompt($sessionId); + if ($userInstructions !== '') { + $taskInput['user_instructions'] = $userInstructions; + } + } + private function getAudioHistory(array $history): array { // history is a list of JSON strings // The content is the remote audio ID (or the transcription as fallback) @@ -838,6 +863,8 @@ private function scheduleAgencyTask( $taskInput['memories'] = $this->sessionSummaryService->getMemories($userId); } /** @psalm-suppress UndefinedClass */ + $this->addAgencyUserInstructions($taskInput, \OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID, $sessionId); + /** @psalm-suppress UndefinedClass */ $task = new Task( \OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID, $taskInput, @@ -887,6 +914,8 @@ private function scheduleAgencyMultimodalTask( ]; $taskInput['memories'] = $this->sessionSummaryService->getMemories($userId); /** @psalm-suppress UndefinedClass */ + $this->addAgencyUserInstructions($taskInput, \OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, $sessionId); + /** @psalm-suppress UndefinedClass */ $task = new Task( \OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, $taskInput, @@ -983,6 +1012,8 @@ private function scheduleAgencyAudioTask( $taskInput['memories'] = $this->sessionSummaryService->getMemories($userId); } /** @psalm-suppress UndefinedClass */ + $this->addAgencyUserInstructions($taskInput, \OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID, $sessionId); + /** @psalm-suppress UndefinedClass */ $task = new Task( \OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID, $taskInput, diff --git a/lib/TaskProcessing/ContextAgentAudioInteractionProvider.php b/lib/TaskProcessing/ContextAgentAudioInteractionProvider.php index a523d72f1..820d4a216 100644 --- a/lib/TaskProcessing/ContextAgentAudioInteractionProvider.php +++ b/lib/TaskProcessing/ContextAgentAudioInteractionProvider.php @@ -68,6 +68,11 @@ public function getOptionalInputShape(): array { $this->l->t('The memories to be injected into the chat session.'), EShapeType::ListOfTexts ), + 'user_instructions' => new ShapeDescriptor( + $this->l->t('User instructions'), + $this->l->t('Additional instructions for the agent, appended to its system prompt.'), + EShapeType::Text + ), ]; } @@ -134,6 +139,10 @@ public function process(?string $userId, array $input, callable $reportProgress) if (isset($input['memories'], $this->taskProcessingManager->getAvailableTaskTypes()[ContextAgentAudioInteraction::ID]['optionalInputShape']['memories'])) { $contextAgentTaskInput['memories'] = $input['memories']; } + /** @psalm-suppress UndefinedClass */ + if (isset($input['user_instructions'], $this->taskProcessingManager->getAvailableTaskTypes()[ContextAgentInteraction::ID]['optionalInputShape']['user_instructions'])) { + $contextAgentTaskInput['user_instructions'] = $input['user_instructions']; + } /** @psalm-suppress UndefinedClass,InvalidArgument */ $task = new Task( ContextAgentInteraction::ID, diff --git a/src/components/AdminSettings.vue b/src/components/AdminSettings.vue index dea8caa66..f8df88667 100644 --- a/src/components/AdminSettings.vue +++ b/src/components/AdminSettings.vue @@ -144,6 +144,10 @@ {{ t('assistant', 'It is passed on to the LLM for it to better understand the context.') }}
{{ t('assistant', '"{user}" is a placeholder for the user\'s display name.') }} + createMock(SessionMapper::class); + $sessionMapper->method('exists')->willReturn(true); + $session = new Session(); + $session->setUserId(self::USER_ID); + $sessionMapper->method('getUserSession')->willReturn($session); + + $humanMessage = new Message(); + $humanMessage->setSessionId(self::SESSION_ID); + $humanMessage->setRole(Message::ROLE_HUMAN); + $humanMessage->setContent('What is your name?'); + $humanMessage->setAttachments('[]'); + $this->messageMapper = $this->createMock(MessageMapper::class); + $this->messageMapper->method('getLastHumanMessage')->willReturn($humanMessage); + + // only the classic Context Agent is available, no multimodal or audio agency + $this->taskProcessingManager = $this->createMock(IManager::class); + $this->taskProcessingManager->method('getAvailableTaskTypeIds')->willReturn([ContextAgentInteraction::ID]); + $this->taskProcessingManager->method('getUserTasksByApp')->willReturn([]); + $this->taskProcessingManager->method('scheduleTask')->willReturnCallback(function (Task $task): void { + $this->scheduledTask = $task; + }); + + $this->service = new ChatService( + $this->createMock(IUserManager::class), + $this->createMock(IAppConfig::class), + $this->createMock(IL10N::class), + $sessionMapper, + $this->messageMapper, + $this->createMock(IDBConnection::class), + $this->createMock(SessionSummaryService::class), + $this->taskProcessingManager, + $this->createMock(LoggerInterface::class), + $this->createMock(ITimeFactory::class), + ); + } + + private function setAgentOptionalInputs(array $optionalInputShape): void { + $this->taskProcessingManager->method('getAvailableTaskTypes')->willReturn([ + ContextAgentInteraction::ID => ['optionalInputShape' => $optionalInputShape], + ]); + } + + private function systemMessage(string $content): Message { + $message = new Message(); + $message->setSessionId(self::SESSION_ID); + $message->setRole(Message::ROLE_SYSTEM); + $message->setContent($content); + return $message; + } + + public function testUserInstructionsArePassedToTheAgent(): void { + $this->setAgentOptionalInputs(['user_instructions' => 'shape']); + $this->messageMapper->method('getFirstNMessages') + ->with(self::SESSION_ID, 1, Message::ROLE_SYSTEM) + ->willReturn($this->systemMessage(self::USER_INSTRUCTIONS)); + + $this->service->scheduleMessageGeneration(self::USER_ID, self::SESSION_ID); + + $this->assertNotNull($this->scheduledTask); + $this->assertSame(ContextAgentInteraction::ID, $this->scheduledTask->getTaskTypeId()); + $this->assertSame(self::USER_INSTRUCTIONS, $this->scheduledTask->getInput()['user_instructions'] ?? null); + } + + public function testUserInstructionsAreNotSentWhenTheAgentDoesNotSupportThem(): void { + // e.g. an older Context Agent version that does not declare the optional input + $this->setAgentOptionalInputs([]); + $this->messageMapper->expects($this->never())->method('getFirstNMessages'); + + $this->service->scheduleMessageGeneration(self::USER_ID, self::SESSION_ID); + + $this->assertNotNull($this->scheduledTask); + $this->assertArrayNotHasKey('user_instructions', $this->scheduledTask->getInput()); + } + + public function testUserInstructionsAreNotSentWhenTheSessionHasNoSystemMessage(): void { + $this->setAgentOptionalInputs(['user_instructions' => 'shape']); + $this->messageMapper->method('getFirstNMessages')->willThrowException(new DoesNotExistException('no system message')); + + $this->service->scheduleMessageGeneration(self::USER_ID, self::SESSION_ID); + + $this->assertNotNull($this->scheduledTask); + $this->assertArrayNotHasKey('user_instructions', $this->scheduledTask->getInput()); + } +} From 3f8434789340e42bae679113e92c4223535a2371 Mon Sep 17 00:00:00 2001 From: Viacheslav Huliaiev Date: Thu, 22 Oct 2026 13:13:52 +0200 Subject: [PATCH 2/2] fix(chat): address review feedback on the agent user instructions - Do not pass the chat user instructions to assignment runs, they run unattended - Ask the Context Agent for plain text in audio interactions (output_mode=speech), the reply goes to text-to-speech - Run the agent without instructions if they cannot be read - Fetch the available task types only once - Mention the required Context Agent version in the admin hint Signed-off-by: Viacheslav Huliaiev --- lib/Service/ChatService.php | 36 ++++++++++++------- .../ContextAgentAudioInteractionProvider.php | 10 ++++-- src/components/AdminSettings.vue | 2 +- tests/unit/Service/ChatServiceAgencyTest.php | 23 ++++++++++++ 4 files changed, 55 insertions(+), 16 deletions(-) diff --git a/lib/Service/ChatService.php b/lib/Service/ChatService.php index 538466669..b2be960a3 100644 --- a/lib/Service/ChatService.php +++ b/lib/Service/ChatService.php @@ -567,8 +567,10 @@ public function scheduleAssignmentMessageGeneration(?string $userId, int $sessio $lastConversationToken = self::EMPTY_CONVERSATION_TOKEN; // classic agency + // The chat user instructions are meant for interactive chats (e.g. asking for clarification), + // an assignment runs unattended with its own prompt $prompt = $lastUserMessage->getContent(); - $taskId = $this->scheduleAgencyTask($userId, $prompt, 0, $lastConversationToken, $sessionId); + $taskId = $this->scheduleAgencyTask($userId, $prompt, 0, $lastConversationToken, $sessionId, withUserInstructions: false); return $taskId; } @@ -667,14 +669,18 @@ private function getSessionSystemPrompt(int $sessionId): string { /** * Pass the session's user instructions to an agency task if its provider supports them. * The agent appends them to its own system prompt, they do not replace it. - * - * @throws InternalException + * If they cannot be read, the task is scheduled without them. */ - private function addAgencyUserInstructions(array &$taskInput, string $taskTypeId, int $sessionId): void { - if (!isset($this->taskProcessingManager->getAvailableTaskTypes()[$taskTypeId]['optionalInputShape']['user_instructions'])) { + private function addAgencyUserInstructions(array &$taskInput, array $optionalInputShape, int $sessionId): void { + if (!isset($optionalInputShape['user_instructions'])) { + return; + } + try { + $userInstructions = $this->getSessionSystemPrompt($sessionId); + } catch (InternalException $e) { + $this->logger->warning('Could not get the user instructions of the session, the agent will run without them', ['exception' => $e, 'sessionId' => $sessionId]); return; } - $userInstructions = $this->getSessionSystemPrompt($sessionId); if ($userInstructions !== '') { $taskInput['user_instructions'] = $userInstructions; } @@ -850,6 +856,7 @@ private function scheduleAgencyTask( int $confirmation, string $conversationToken, int $sessionId, + bool $withUserInstructions = true, ): int { $customId = 'chatty-llm:' . $sessionId; $this->checkIfSessionIsThinking($userId, $customId); @@ -859,11 +866,13 @@ private function scheduleAgencyTask( 'conversation_token' => $conversationToken, ]; /** @psalm-suppress UndefinedClass */ - if (isset($this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID]['optionalInputShape']['memories'])) { + $optionalInputShape = $this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID]['optionalInputShape'] ?? []; + if (isset($optionalInputShape['memories'])) { $taskInput['memories'] = $this->sessionSummaryService->getMemories($userId); } - /** @psalm-suppress UndefinedClass */ - $this->addAgencyUserInstructions($taskInput, \OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID, $sessionId); + if ($withUserInstructions) { + $this->addAgencyUserInstructions($taskInput, $optionalInputShape, $sessionId); + } /** @psalm-suppress UndefinedClass */ $task = new Task( \OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID, @@ -914,7 +923,8 @@ private function scheduleAgencyMultimodalTask( ]; $taskInput['memories'] = $this->sessionSummaryService->getMemories($userId); /** @psalm-suppress UndefinedClass */ - $this->addAgencyUserInstructions($taskInput, \OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, $sessionId); + $optionalInputShape = $this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID]['optionalInputShape'] ?? []; + $this->addAgencyUserInstructions($taskInput, $optionalInputShape, $sessionId); /** @psalm-suppress UndefinedClass */ $task = new Task( \OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, @@ -1008,11 +1018,11 @@ private function scheduleAgencyAudioTask( 'conversation_token' => $conversationToken, ]; /** @psalm-suppress UndefinedClass */ - if (isset($this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID]['optionalInputShape']['memories'])) { + $optionalInputShape = $this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID]['optionalInputShape'] ?? []; + if (isset($optionalInputShape['memories'])) { $taskInput['memories'] = $this->sessionSummaryService->getMemories($userId); } - /** @psalm-suppress UndefinedClass */ - $this->addAgencyUserInstructions($taskInput, \OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID, $sessionId); + $this->addAgencyUserInstructions($taskInput, $optionalInputShape, $sessionId); /** @psalm-suppress UndefinedClass */ $task = new Task( \OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID, diff --git a/lib/TaskProcessing/ContextAgentAudioInteractionProvider.php b/lib/TaskProcessing/ContextAgentAudioInteractionProvider.php index 820d4a216..d5de92831 100644 --- a/lib/TaskProcessing/ContextAgentAudioInteractionProvider.php +++ b/lib/TaskProcessing/ContextAgentAudioInteractionProvider.php @@ -135,14 +135,20 @@ public function process(?string $userId, array $input, callable $reportProgress) 'confirmation' => $confirmation, 'conversation_token' => $conversationToken, ]; + $availableTaskTypes = $this->taskProcessingManager->getAvailableTaskTypes(); /** @psalm-suppress UndefinedClass */ - if (isset($input['memories'], $this->taskProcessingManager->getAvailableTaskTypes()[ContextAgentAudioInteraction::ID]['optionalInputShape']['memories'])) { + if (isset($input['memories'], $availableTaskTypes[ContextAgentAudioInteraction::ID]['optionalInputShape']['memories'])) { $contextAgentTaskInput['memories'] = $input['memories']; } /** @psalm-suppress UndefinedClass */ - if (isset($input['user_instructions'], $this->taskProcessingManager->getAvailableTaskTypes()[ContextAgentInteraction::ID]['optionalInputShape']['user_instructions'])) { + if (isset($input['user_instructions'], $availableTaskTypes[ContextAgentInteraction::ID]['optionalInputShape']['user_instructions'])) { $contextAgentTaskInput['user_instructions'] = $input['user_instructions']; } + // the reply is read out loud, so the agent must not format it (e.g. with Markdown) + /** @psalm-suppress UndefinedClass */ + if (isset($availableTaskTypes[ContextAgentInteraction::ID]['optionalInputShape']['output_mode'])) { + $contextAgentTaskInput['output_mode'] = 'speech'; + } /** @psalm-suppress UndefinedClass,InvalidArgument */ $task = new Task( ContextAgentInteraction::ID, diff --git a/src/components/AdminSettings.vue b/src/components/AdminSettings.vue index f8df88667..6606deb3c 100644 --- a/src/components/AdminSettings.vue +++ b/src/components/AdminSettings.vue @@ -146,7 +146,7 @@ {{ t('assistant', '"{user}" is a placeholder for the user\'s display name.') }} assertNotNull($this->scheduledTask); $this->assertArrayNotHasKey('user_instructions', $this->scheduledTask->getInput()); } + + public function testTheAgentRunsWithoutUserInstructionsWhenTheyCannotBeRead(): void { + $this->setAgentOptionalInputs(['user_instructions' => 'shape']); + $this->messageMapper->method('getFirstNMessages')->willThrowException(new Exception('database error')); + + $this->service->scheduleMessageGeneration(self::USER_ID, self::SESSION_ID); + + $this->assertNotNull($this->scheduledTask); + $this->assertArrayNotHasKey('user_instructions', $this->scheduledTask->getInput()); + } + + public function testUserInstructionsAreNotSentForAssignmentRuns(): void { + // assignments run unattended, instructions like "ask for clarification" do not apply + $this->setAgentOptionalInputs(['user_instructions' => 'shape']); + $this->messageMapper->expects($this->never())->method('getFirstNMessages'); + + $this->service->scheduleAssignmentMessageGeneration(self::USER_ID, self::SESSION_ID); + + $this->assertNotNull($this->scheduledTask); + $this->assertSame(ContextAgentInteraction::ID, $this->scheduledTask->getTaskTypeId()); + $this->assertArrayNotHasKey('user_instructions', $this->scheduledTask->getInput()); + } }