diff --git a/lib/Service/ChatService.php b/lib/Service/ChatService.php index 1f27de531..b2be960a3 100644 --- a/lib/Service/ChatService.php +++ b/lib/Service/ChatService.php @@ -456,15 +456,7 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $ } } else { // classic chat - $systemPrompt = ''; - try { - $firstMessage = $this->messageMapper->getFirstNMessages($sessionId, 1, Message::ROLE_SYSTEM); - $systemPrompt = $firstMessage->getContent(); - } catch (DoesNotExistException $e) { - $this->logger->info('No system message found in the session', ['exception' => $e, 'sessionId' => $sessionId]); - } catch (MultipleObjectsReturnedException|Exception $e) { - throw new InternalException(previous: $e); - } + $systemPrompt = $this->getSessionSystemPrompt($sessionId); try { $history = $this->getRawLastMessages($sessionId); } catch (Exception|AppConfigTypeConflictException $e) { @@ -575,8 +567,10 @@ public function scheduleAssignmentMessageGeneration(?string $userId, int $sessio $lastConversationToken = self::EMPTY_CONVERSATION_TOKEN; // classic agency + // The chat user instructions are meant for interactive chats (e.g. asking for clarification), + // an assignment runs unattended with its own prompt $prompt = $lastUserMessage->getContent(); - $taskId = $this->scheduleAgencyTask($userId, $prompt, 0, $lastConversationToken, $sessionId); + $taskId = $this->scheduleAgencyTask($userId, $prompt, 0, $lastConversationToken, $sessionId, withUserInstructions: false); return $taskId; } @@ -655,6 +649,43 @@ public function isMultimodalContextAgentAvailable(): bool { return in_array(\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypeIds()); } + /** + * Get the user instructions stored as the system message of the session (empty string if there is none) + * + * @throws InternalException + */ + private function getSessionSystemPrompt(int $sessionId): string { + try { + $firstMessage = $this->messageMapper->getFirstNMessages($sessionId, 1, Message::ROLE_SYSTEM); + return $firstMessage->getContent(); + } catch (DoesNotExistException $e) { + $this->logger->info('No system message found in the session', ['exception' => $e, 'sessionId' => $sessionId]); + } catch (MultipleObjectsReturnedException|Exception $e) { + throw new InternalException(previous: $e); + } + return ''; + } + + /** + * Pass the session's user instructions to an agency task if its provider supports them. + * The agent appends them to its own system prompt, they do not replace it. + * If they cannot be read, the task is scheduled without them. + */ + private function addAgencyUserInstructions(array &$taskInput, array $optionalInputShape, int $sessionId): void { + if (!isset($optionalInputShape['user_instructions'])) { + return; + } + try { + $userInstructions = $this->getSessionSystemPrompt($sessionId); + } catch (InternalException $e) { + $this->logger->warning('Could not get the user instructions of the session, the agent will run without them', ['exception' => $e, 'sessionId' => $sessionId]); + return; + } + if ($userInstructions !== '') { + $taskInput['user_instructions'] = $userInstructions; + } + } + private function getAudioHistory(array $history): array { // history is a list of JSON strings // The content is the remote audio ID (or the transcription as fallback) @@ -825,6 +856,7 @@ private function scheduleAgencyTask( int $confirmation, string $conversationToken, int $sessionId, + bool $withUserInstructions = true, ): int { $customId = 'chatty-llm:' . $sessionId; $this->checkIfSessionIsThinking($userId, $customId); @@ -834,9 +866,13 @@ private function scheduleAgencyTask( 'conversation_token' => $conversationToken, ]; /** @psalm-suppress UndefinedClass */ - if (isset($this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID]['optionalInputShape']['memories'])) { + $optionalInputShape = $this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID]['optionalInputShape'] ?? []; + if (isset($optionalInputShape['memories'])) { $taskInput['memories'] = $this->sessionSummaryService->getMemories($userId); } + if ($withUserInstructions) { + $this->addAgencyUserInstructions($taskInput, $optionalInputShape, $sessionId); + } /** @psalm-suppress UndefinedClass */ $task = new Task( \OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID, @@ -887,6 +923,9 @@ private function scheduleAgencyMultimodalTask( ]; $taskInput['memories'] = $this->sessionSummaryService->getMemories($userId); /** @psalm-suppress UndefinedClass */ + $optionalInputShape = $this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID]['optionalInputShape'] ?? []; + $this->addAgencyUserInstructions($taskInput, $optionalInputShape, $sessionId); + /** @psalm-suppress UndefinedClass */ $task = new Task( \OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, $taskInput, @@ -979,9 +1018,11 @@ private function scheduleAgencyAudioTask( 'conversation_token' => $conversationToken, ]; /** @psalm-suppress UndefinedClass */ - if (isset($this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID]['optionalInputShape']['memories'])) { + $optionalInputShape = $this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID]['optionalInputShape'] ?? []; + if (isset($optionalInputShape['memories'])) { $taskInput['memories'] = $this->sessionSummaryService->getMemories($userId); } + $this->addAgencyUserInstructions($taskInput, $optionalInputShape, $sessionId); /** @psalm-suppress UndefinedClass */ $task = new Task( \OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID, diff --git a/lib/TaskProcessing/ContextAgentAudioInteractionProvider.php b/lib/TaskProcessing/ContextAgentAudioInteractionProvider.php index a523d72f1..d5de92831 100644 --- a/lib/TaskProcessing/ContextAgentAudioInteractionProvider.php +++ b/lib/TaskProcessing/ContextAgentAudioInteractionProvider.php @@ -68,6 +68,11 @@ public function getOptionalInputShape(): array { $this->l->t('The memories to be injected into the chat session.'), EShapeType::ListOfTexts ), + 'user_instructions' => new ShapeDescriptor( + $this->l->t('User instructions'), + $this->l->t('Additional instructions for the agent, appended to its system prompt.'), + EShapeType::Text + ), ]; } @@ -130,10 +135,20 @@ public function process(?string $userId, array $input, callable $reportProgress) 'confirmation' => $confirmation, 'conversation_token' => $conversationToken, ]; + $availableTaskTypes = $this->taskProcessingManager->getAvailableTaskTypes(); /** @psalm-suppress UndefinedClass */ - if (isset($input['memories'], $this->taskProcessingManager->getAvailableTaskTypes()[ContextAgentAudioInteraction::ID]['optionalInputShape']['memories'])) { + if (isset($input['memories'], $availableTaskTypes[ContextAgentAudioInteraction::ID]['optionalInputShape']['memories'])) { $contextAgentTaskInput['memories'] = $input['memories']; } + /** @psalm-suppress UndefinedClass */ + if (isset($input['user_instructions'], $availableTaskTypes[ContextAgentInteraction::ID]['optionalInputShape']['user_instructions'])) { + $contextAgentTaskInput['user_instructions'] = $input['user_instructions']; + } + // the reply is read out loud, so the agent must not format it (e.g. with Markdown) + /** @psalm-suppress UndefinedClass */ + if (isset($availableTaskTypes[ContextAgentInteraction::ID]['optionalInputShape']['output_mode'])) { + $contextAgentTaskInput['output_mode'] = 'speech'; + } /** @psalm-suppress UndefinedClass,InvalidArgument */ $task = new Task( ContextAgentInteraction::ID, diff --git a/src/components/AdminSettings.vue b/src/components/AdminSettings.vue index dea8caa66..6606deb3c 100644 --- a/src/components/AdminSettings.vue +++ b/src/components/AdminSettings.vue @@ -144,6 +144,10 @@ {{ t('assistant', 'It is passed on to the LLM for it to better understand the context.') }}
{{ t('assistant', '"{user}" is a placeholder for the user\'s display name.') }} + createMock(SessionMapper::class); + $sessionMapper->method('exists')->willReturn(true); + $session = new Session(); + $session->setUserId(self::USER_ID); + $sessionMapper->method('getUserSession')->willReturn($session); + + $humanMessage = new Message(); + $humanMessage->setSessionId(self::SESSION_ID); + $humanMessage->setRole(Message::ROLE_HUMAN); + $humanMessage->setContent('What is your name?'); + $humanMessage->setAttachments('[]'); + $this->messageMapper = $this->createMock(MessageMapper::class); + $this->messageMapper->method('getLastHumanMessage')->willReturn($humanMessage); + + // only the classic Context Agent is available, no multimodal or audio agency + $this->taskProcessingManager = $this->createMock(IManager::class); + $this->taskProcessingManager->method('getAvailableTaskTypeIds')->willReturn([ContextAgentInteraction::ID]); + $this->taskProcessingManager->method('getUserTasksByApp')->willReturn([]); + $this->taskProcessingManager->method('scheduleTask')->willReturnCallback(function (Task $task): void { + $this->scheduledTask = $task; + }); + + $this->service = new ChatService( + $this->createMock(IUserManager::class), + $this->createMock(IAppConfig::class), + $this->createMock(IL10N::class), + $sessionMapper, + $this->messageMapper, + $this->createMock(IDBConnection::class), + $this->createMock(SessionSummaryService::class), + $this->taskProcessingManager, + $this->createMock(LoggerInterface::class), + $this->createMock(ITimeFactory::class), + ); + } + + private function setAgentOptionalInputs(array $optionalInputShape): void { + $this->taskProcessingManager->method('getAvailableTaskTypes')->willReturn([ + ContextAgentInteraction::ID => ['optionalInputShape' => $optionalInputShape], + ]); + } + + private function systemMessage(string $content): Message { + $message = new Message(); + $message->setSessionId(self::SESSION_ID); + $message->setRole(Message::ROLE_SYSTEM); + $message->setContent($content); + return $message; + } + + public function testUserInstructionsArePassedToTheAgent(): void { + $this->setAgentOptionalInputs(['user_instructions' => 'shape']); + $this->messageMapper->method('getFirstNMessages') + ->with(self::SESSION_ID, 1, Message::ROLE_SYSTEM) + ->willReturn($this->systemMessage(self::USER_INSTRUCTIONS)); + + $this->service->scheduleMessageGeneration(self::USER_ID, self::SESSION_ID); + + $this->assertNotNull($this->scheduledTask); + $this->assertSame(ContextAgentInteraction::ID, $this->scheduledTask->getTaskTypeId()); + $this->assertSame(self::USER_INSTRUCTIONS, $this->scheduledTask->getInput()['user_instructions'] ?? null); + } + + public function testUserInstructionsAreNotSentWhenTheAgentDoesNotSupportThem(): void { + // e.g. an older Context Agent version that does not declare the optional input + $this->setAgentOptionalInputs([]); + $this->messageMapper->expects($this->never())->method('getFirstNMessages'); + + $this->service->scheduleMessageGeneration(self::USER_ID, self::SESSION_ID); + + $this->assertNotNull($this->scheduledTask); + $this->assertArrayNotHasKey('user_instructions', $this->scheduledTask->getInput()); + } + + public function testUserInstructionsAreNotSentWhenTheSessionHasNoSystemMessage(): void { + $this->setAgentOptionalInputs(['user_instructions' => 'shape']); + $this->messageMapper->method('getFirstNMessages')->willThrowException(new DoesNotExistException('no system message')); + + $this->service->scheduleMessageGeneration(self::USER_ID, self::SESSION_ID); + + $this->assertNotNull($this->scheduledTask); + $this->assertArrayNotHasKey('user_instructions', $this->scheduledTask->getInput()); + } + + public function testTheAgentRunsWithoutUserInstructionsWhenTheyCannotBeRead(): void { + $this->setAgentOptionalInputs(['user_instructions' => 'shape']); + $this->messageMapper->method('getFirstNMessages')->willThrowException(new Exception('database error')); + + $this->service->scheduleMessageGeneration(self::USER_ID, self::SESSION_ID); + + $this->assertNotNull($this->scheduledTask); + $this->assertArrayNotHasKey('user_instructions', $this->scheduledTask->getInput()); + } + + public function testUserInstructionsAreNotSentForAssignmentRuns(): void { + // assignments run unattended, instructions like "ask for clarification" do not apply + $this->setAgentOptionalInputs(['user_instructions' => 'shape']); + $this->messageMapper->expects($this->never())->method('getFirstNMessages'); + + $this->service->scheduleAssignmentMessageGeneration(self::USER_ID, self::SESSION_ID); + + $this->assertNotNull($this->scheduledTask); + $this->assertSame(ContextAgentInteraction::ID, $this->scheduledTask->getTaskTypeId()); + $this->assertArrayNotHasKey('user_instructions', $this->scheduledTask->getInput()); + } +}