diff --git a/lib/Service/ChatService.php b/lib/Service/ChatService.php
index 1f27de531..b2be960a3 100644
--- a/lib/Service/ChatService.php
+++ b/lib/Service/ChatService.php
@@ -456,15 +456,7 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $
}
} else {
// classic chat
- $systemPrompt = '';
- try {
- $firstMessage = $this->messageMapper->getFirstNMessages($sessionId, 1, Message::ROLE_SYSTEM);
- $systemPrompt = $firstMessage->getContent();
- } catch (DoesNotExistException $e) {
- $this->logger->info('No system message found in the session', ['exception' => $e, 'sessionId' => $sessionId]);
- } catch (MultipleObjectsReturnedException|Exception $e) {
- throw new InternalException(previous: $e);
- }
+ $systemPrompt = $this->getSessionSystemPrompt($sessionId);
try {
$history = $this->getRawLastMessages($sessionId);
} catch (Exception|AppConfigTypeConflictException $e) {
@@ -575,8 +567,10 @@ public function scheduleAssignmentMessageGeneration(?string $userId, int $sessio
$lastConversationToken = self::EMPTY_CONVERSATION_TOKEN;
// classic agency
+ // The chat user instructions are meant for interactive chats (e.g. asking for clarification),
+ // an assignment runs unattended with its own prompt
$prompt = $lastUserMessage->getContent();
- $taskId = $this->scheduleAgencyTask($userId, $prompt, 0, $lastConversationToken, $sessionId);
+ $taskId = $this->scheduleAgencyTask($userId, $prompt, 0, $lastConversationToken, $sessionId, withUserInstructions: false);
return $taskId;
}
@@ -655,6 +649,43 @@ public function isMultimodalContextAgentAvailable(): bool {
return in_array(\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypeIds());
}
+ /**
+ * Get the user instructions stored as the system message of the session (empty string if there is none)
+ *
+ * @throws InternalException
+ */
+ private function getSessionSystemPrompt(int $sessionId): string {
+ try {
+ $firstMessage = $this->messageMapper->getFirstNMessages($sessionId, 1, Message::ROLE_SYSTEM);
+ return $firstMessage->getContent();
+ } catch (DoesNotExistException $e) {
+ $this->logger->info('No system message found in the session', ['exception' => $e, 'sessionId' => $sessionId]);
+ } catch (MultipleObjectsReturnedException|Exception $e) {
+ throw new InternalException(previous: $e);
+ }
+ return '';
+ }
+
+ /**
+ * Pass the session's user instructions to an agency task if its provider supports them.
+ * The agent appends them to its own system prompt, they do not replace it.
+ * If they cannot be read, the task is scheduled without them.
+ */
+ private function addAgencyUserInstructions(array &$taskInput, array $optionalInputShape, int $sessionId): void {
+ if (!isset($optionalInputShape['user_instructions'])) {
+ return;
+ }
+ try {
+ $userInstructions = $this->getSessionSystemPrompt($sessionId);
+ } catch (InternalException $e) {
+ $this->logger->warning('Could not get the user instructions of the session, the agent will run without them', ['exception' => $e, 'sessionId' => $sessionId]);
+ return;
+ }
+ if ($userInstructions !== '') {
+ $taskInput['user_instructions'] = $userInstructions;
+ }
+ }
+
private function getAudioHistory(array $history): array {
// history is a list of JSON strings
// The content is the remote audio ID (or the transcription as fallback)
@@ -825,6 +856,7 @@ private function scheduleAgencyTask(
int $confirmation,
string $conversationToken,
int $sessionId,
+ bool $withUserInstructions = true,
): int {
$customId = 'chatty-llm:' . $sessionId;
$this->checkIfSessionIsThinking($userId, $customId);
@@ -834,9 +866,13 @@ private function scheduleAgencyTask(
'conversation_token' => $conversationToken,
];
/** @psalm-suppress UndefinedClass */
- if (isset($this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID]['optionalInputShape']['memories'])) {
+ $optionalInputShape = $this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID]['optionalInputShape'] ?? [];
+ if (isset($optionalInputShape['memories'])) {
$taskInput['memories'] = $this->sessionSummaryService->getMemories($userId);
}
+ if ($withUserInstructions) {
+ $this->addAgencyUserInstructions($taskInput, $optionalInputShape, $sessionId);
+ }
/** @psalm-suppress UndefinedClass */
$task = new Task(
\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID,
@@ -887,6 +923,9 @@ private function scheduleAgencyMultimodalTask(
];
$taskInput['memories'] = $this->sessionSummaryService->getMemories($userId);
/** @psalm-suppress UndefinedClass */
+ $optionalInputShape = $this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID]['optionalInputShape'] ?? [];
+ $this->addAgencyUserInstructions($taskInput, $optionalInputShape, $sessionId);
+ /** @psalm-suppress UndefinedClass */
$task = new Task(
\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID,
$taskInput,
@@ -979,9 +1018,11 @@ private function scheduleAgencyAudioTask(
'conversation_token' => $conversationToken,
];
/** @psalm-suppress UndefinedClass */
- if (isset($this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID]['optionalInputShape']['memories'])) {
+ $optionalInputShape = $this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID]['optionalInputShape'] ?? [];
+ if (isset($optionalInputShape['memories'])) {
$taskInput['memories'] = $this->sessionSummaryService->getMemories($userId);
}
+ $this->addAgencyUserInstructions($taskInput, $optionalInputShape, $sessionId);
/** @psalm-suppress UndefinedClass */
$task = new Task(
\OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID,
diff --git a/lib/TaskProcessing/ContextAgentAudioInteractionProvider.php b/lib/TaskProcessing/ContextAgentAudioInteractionProvider.php
index a523d72f1..d5de92831 100644
--- a/lib/TaskProcessing/ContextAgentAudioInteractionProvider.php
+++ b/lib/TaskProcessing/ContextAgentAudioInteractionProvider.php
@@ -68,6 +68,11 @@ public function getOptionalInputShape(): array {
$this->l->t('The memories to be injected into the chat session.'),
EShapeType::ListOfTexts
),
+ 'user_instructions' => new ShapeDescriptor(
+ $this->l->t('User instructions'),
+ $this->l->t('Additional instructions for the agent, appended to its system prompt.'),
+ EShapeType::Text
+ ),
];
}
@@ -130,10 +135,20 @@ public function process(?string $userId, array $input, callable $reportProgress)
'confirmation' => $confirmation,
'conversation_token' => $conversationToken,
];
+ $availableTaskTypes = $this->taskProcessingManager->getAvailableTaskTypes();
/** @psalm-suppress UndefinedClass */
- if (isset($input['memories'], $this->taskProcessingManager->getAvailableTaskTypes()[ContextAgentAudioInteraction::ID]['optionalInputShape']['memories'])) {
+ if (isset($input['memories'], $availableTaskTypes[ContextAgentAudioInteraction::ID]['optionalInputShape']['memories'])) {
$contextAgentTaskInput['memories'] = $input['memories'];
}
+ /** @psalm-suppress UndefinedClass */
+ if (isset($input['user_instructions'], $availableTaskTypes[ContextAgentInteraction::ID]['optionalInputShape']['user_instructions'])) {
+ $contextAgentTaskInput['user_instructions'] = $input['user_instructions'];
+ }
+ // the reply is read out loud, so the agent must not format it (e.g. with Markdown)
+ /** @psalm-suppress UndefinedClass */
+ if (isset($availableTaskTypes[ContextAgentInteraction::ID]['optionalInputShape']['output_mode'])) {
+ $contextAgentTaskInput['output_mode'] = 'speech';
+ }
/** @psalm-suppress UndefinedClass,InvalidArgument */
$task = new Task(
ContextAgentInteraction::ID,
diff --git a/src/components/AdminSettings.vue b/src/components/AdminSettings.vue
index dea8caa66..6606deb3c 100644
--- a/src/components/AdminSettings.vue
+++ b/src/components/AdminSettings.vue
@@ -144,6 +144,10 @@
{{ t('assistant', 'It is passed on to the LLM for it to better understand the context.') }}
{{ t('assistant', '"{user}" is a placeholder for the user\'s display name.') }}
+
+
+ {{ t('assistant', 'When chatting through the Context Agent, these instructions are added to the agent\'s own instructions (requires Context Agent 2.10.0 or later).') }}
+
createMock(SessionMapper::class);
+ $sessionMapper->method('exists')->willReturn(true);
+ $session = new Session();
+ $session->setUserId(self::USER_ID);
+ $sessionMapper->method('getUserSession')->willReturn($session);
+
+ $humanMessage = new Message();
+ $humanMessage->setSessionId(self::SESSION_ID);
+ $humanMessage->setRole(Message::ROLE_HUMAN);
+ $humanMessage->setContent('What is your name?');
+ $humanMessage->setAttachments('[]');
+ $this->messageMapper = $this->createMock(MessageMapper::class);
+ $this->messageMapper->method('getLastHumanMessage')->willReturn($humanMessage);
+
+ // only the classic Context Agent is available, no multimodal or audio agency
+ $this->taskProcessingManager = $this->createMock(IManager::class);
+ $this->taskProcessingManager->method('getAvailableTaskTypeIds')->willReturn([ContextAgentInteraction::ID]);
+ $this->taskProcessingManager->method('getUserTasksByApp')->willReturn([]);
+ $this->taskProcessingManager->method('scheduleTask')->willReturnCallback(function (Task $task): void {
+ $this->scheduledTask = $task;
+ });
+
+ $this->service = new ChatService(
+ $this->createMock(IUserManager::class),
+ $this->createMock(IAppConfig::class),
+ $this->createMock(IL10N::class),
+ $sessionMapper,
+ $this->messageMapper,
+ $this->createMock(IDBConnection::class),
+ $this->createMock(SessionSummaryService::class),
+ $this->taskProcessingManager,
+ $this->createMock(LoggerInterface::class),
+ $this->createMock(ITimeFactory::class),
+ );
+ }
+
+ private function setAgentOptionalInputs(array $optionalInputShape): void {
+ $this->taskProcessingManager->method('getAvailableTaskTypes')->willReturn([
+ ContextAgentInteraction::ID => ['optionalInputShape' => $optionalInputShape],
+ ]);
+ }
+
+ private function systemMessage(string $content): Message {
+ $message = new Message();
+ $message->setSessionId(self::SESSION_ID);
+ $message->setRole(Message::ROLE_SYSTEM);
+ $message->setContent($content);
+ return $message;
+ }
+
+ public function testUserInstructionsArePassedToTheAgent(): void {
+ $this->setAgentOptionalInputs(['user_instructions' => 'shape']);
+ $this->messageMapper->method('getFirstNMessages')
+ ->with(self::SESSION_ID, 1, Message::ROLE_SYSTEM)
+ ->willReturn($this->systemMessage(self::USER_INSTRUCTIONS));
+
+ $this->service->scheduleMessageGeneration(self::USER_ID, self::SESSION_ID);
+
+ $this->assertNotNull($this->scheduledTask);
+ $this->assertSame(ContextAgentInteraction::ID, $this->scheduledTask->getTaskTypeId());
+ $this->assertSame(self::USER_INSTRUCTIONS, $this->scheduledTask->getInput()['user_instructions'] ?? null);
+ }
+
+ public function testUserInstructionsAreNotSentWhenTheAgentDoesNotSupportThem(): void {
+ // e.g. an older Context Agent version that does not declare the optional input
+ $this->setAgentOptionalInputs([]);
+ $this->messageMapper->expects($this->never())->method('getFirstNMessages');
+
+ $this->service->scheduleMessageGeneration(self::USER_ID, self::SESSION_ID);
+
+ $this->assertNotNull($this->scheduledTask);
+ $this->assertArrayNotHasKey('user_instructions', $this->scheduledTask->getInput());
+ }
+
+ public function testUserInstructionsAreNotSentWhenTheSessionHasNoSystemMessage(): void {
+ $this->setAgentOptionalInputs(['user_instructions' => 'shape']);
+ $this->messageMapper->method('getFirstNMessages')->willThrowException(new DoesNotExistException('no system message'));
+
+ $this->service->scheduleMessageGeneration(self::USER_ID, self::SESSION_ID);
+
+ $this->assertNotNull($this->scheduledTask);
+ $this->assertArrayNotHasKey('user_instructions', $this->scheduledTask->getInput());
+ }
+
+ public function testTheAgentRunsWithoutUserInstructionsWhenTheyCannotBeRead(): void {
+ $this->setAgentOptionalInputs(['user_instructions' => 'shape']);
+ $this->messageMapper->method('getFirstNMessages')->willThrowException(new Exception('database error'));
+
+ $this->service->scheduleMessageGeneration(self::USER_ID, self::SESSION_ID);
+
+ $this->assertNotNull($this->scheduledTask);
+ $this->assertArrayNotHasKey('user_instructions', $this->scheduledTask->getInput());
+ }
+
+ public function testUserInstructionsAreNotSentForAssignmentRuns(): void {
+ // assignments run unattended, instructions like "ask for clarification" do not apply
+ $this->setAgentOptionalInputs(['user_instructions' => 'shape']);
+ $this->messageMapper->expects($this->never())->method('getFirstNMessages');
+
+ $this->service->scheduleAssignmentMessageGeneration(self::USER_ID, self::SESSION_ID);
+
+ $this->assertNotNull($this->scheduledTask);
+ $this->assertSame(ContextAgentInteraction::ID, $this->scheduledTask->getTaskTypeId());
+ $this->assertArrayNotHasKey('user_instructions', $this->scheduledTask->getInput());
+ }
+}