From db2a3cf8c89767eaaa1dbfa49e5a7c9b9a6d7ba3 Mon Sep 17 00:00:00 2001 From: Andrey Borysenko Date: Tue, 2 Jan 2024 00:20:17 +0300 Subject: [PATCH] Text-Processing APIs implementation (#191) Signed-off-by: Andrey Borysenko Co-authored-by: Alexander Piskun --- appinfo/routes.php | 5 + docs/tech_details/api/index.rst | 1 + docs/tech_details/api/speechtotext.rst | 4 + docs/tech_details/api/textprocessing.rst | 72 ++++++ lib/AppInfo/Application.php | 5 + lib/Capabilities.php | 4 + lib/Controller/TextProcessingController.php | 93 +++++++ .../TextProcessing/TextProcessingProvider.php | 69 +++++ .../TextProcessingProviderMapper.php | 72 ++++++ lib/Migration/Version1006Date202401011308.php | 60 +++++ lib/Service/AppAPIService.php | 2 + lib/Service/SpeechToTextService.php | 3 +- lib/Service/TextProcessingService.php | 237 ++++++++++++++++++ 13 files changed, 626 insertions(+), 1 deletion(-) create mode 100644 docs/tech_details/api/textprocessing.rst create mode 100644 lib/Controller/TextProcessingController.php create mode 100644 lib/Db/TextProcessing/TextProcessingProvider.php create mode 100644 lib/Db/TextProcessing/TextProcessingProviderMapper.php create mode 100644 lib/Migration/Version1006Date202401011308.php create mode 100644 lib/Service/TextProcessingService.php diff --git a/appinfo/routes.php b/appinfo/routes.php index 9bedeaea..96afecf8 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -105,5 +105,10 @@ ['name' => 'speechToText#registerProvider', 'url' => '/api/v1/ai_provider/speech_to_text', 'verb' => 'POST'], ['name' => 'speechToText#unregisterProvider', 'url' => '/api/v1/ai_provider/speech_to_text', 'verb' => 'DELETE'], ['name' => 'speechToText#getProvider', 'url' => '/api/v1/ai_provider/speech_to_text', 'verb' => 'GET'], + + // Text-Processing + ['name' => 'textProcessing#registerProvider', 'url' => '/api/v1/ai_provider/text_processing', 'verb' => 'POST'], + ['name' => 'textProcessing#unregisterProvider', 'url' => '/api/v1/ai_provider/text_processing', 'verb' => 'DELETE'], + ['name' => 'textProcessing#getProvider', 'url' => '/api/v1/ai_provider/text_processing', 'verb' => 'GET'], ], ]; diff --git a/docs/tech_details/api/index.rst b/docs/tech_details/api/index.rst index 8374f026..6140f541 100644 --- a/docs/tech_details/api/index.rst +++ b/docs/tech_details/api/index.rst @@ -19,4 +19,5 @@ AppAPI Nextcloud APIs notifications talkbots speechtotext + textprocessing other_ocs diff --git a/docs/tech_details/api/speechtotext.rst b/docs/tech_details/api/speechtotext.rst index 49117fe8..0c2669bc 100644 --- a/docs/tech_details/api/speechtotext.rst +++ b/docs/tech_details/api/speechtotext.rst @@ -4,6 +4,10 @@ Speech-To-Text AppAPI provides a Speech-To-Text (STT) provider registration API for the ExApps. +.. note:: + + Available since Nextcloud 29. + Registering ExApp STT provider (OCS) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/tech_details/api/textprocessing.rst b/docs/tech_details/api/textprocessing.rst new file mode 100644 index 00000000..a4a80802 --- /dev/null +++ b/docs/tech_details/api/textprocessing.rst @@ -0,0 +1,72 @@ +=============== +Text-Processing +=============== + +AppAPI provides a Text-Processing providers registration mechanism for ExApps. + +.. note:: + + Available since Nextcloud 29. + +Registering text-processing provider (OCS) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +OCS endpoint: ``POST /apps/app_api/api/v1/ai_provider/text_processing`` + +Request data +************ + +.. code-block:: json + + { + "name": "unique_provider_name", + "display_name": "Provider Display Name", + "action_handler": "/handler_route_on_ex_app", + "task_type": "supported_task_type", + } + +.. note:: + + ``action_type`` is a class name of the Text-Processing task type that can be found in the list of supported task types. + +Response +******** + +On successful registration response with status code 200 is returned. + +Unregistering text-processing provider (OCS) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +OCS endpoint: ``DELETE /apps/app_api/api/v1/ai_provider/text_processing`` + +Request data +************ + +.. code-block:: json + + { + "name": "unique_provider_name", + } + +Response +******** + +On successful unregister response with status code 200 is returned. + + +Get list of supported Text-Processing task types (capabilities) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are limited number of Task Types that can be used for Text-Processing. +You can get list of supported Text-Processing task types from OCS capabilities. + +Response +******** + +.. code-block:: json + + { + "text_processing": { + "task_types": ["free_prompt", "headline", "summary", "topics"] + } + } diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index f75956fe..e0b1009b 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -16,6 +16,7 @@ use OCA\AppAPI\Profiler\AppAPIDataCollector; use OCA\AppAPI\PublicCapabilities; use OCA\AppAPI\Service\SpeechToTextService; +use OCA\AppAPI\Service\TextProcessingService; use OCA\AppAPI\Service\UI\TopMenuService; use OCA\DAV\Events\SabrePluginAuthInitEvent; use OCA\Files\Event\LoadAdditionalScriptsEvent; @@ -66,6 +67,10 @@ public function register(IRegistrationContext $context): void { /** @var SpeechToTextService $speechToTextService */ $speechToTextService = $container->get(SpeechToTextService::class); $speechToTextService->registerExAppSpeechToTextProviders($context, $container->getServer()); + + /** @var TextProcessingService $textProcessingService */ + $textProcessingService = $container->get(TextProcessingService::class); + $textProcessingService->registerExAppTextProcessingProviders($context, $container->getServer()); } catch (NotFoundExceptionInterface|ContainerExceptionInterface) { } } diff --git a/lib/Capabilities.php b/lib/Capabilities.php index 85bd8dc2..212b31f6 100644 --- a/lib/Capabilities.php +++ b/lib/Capabilities.php @@ -9,6 +9,7 @@ use OCA\AppAPI\Service\AppAPIService; use OCA\AppAPI\Service\ExAppScopesService; +use OCA\AppAPI\Service\TextProcessingService; use OCP\App\IAppManager; use OCP\Capabilities\ICapability; use OCP\IConfig; @@ -29,6 +30,9 @@ public function getCapabilities(): array { $capabilities = [ 'loglevel' => intval($this->config->getSystemValue('loglevel', 2)), 'version' => $this->appManager->getAppVersion(Application::APP_ID), + 'text_processing' => [ + 'task_types' => array_keys(TextProcessingService::TASK_TYPES), + ] ]; $this->attachExAppScopes($capabilities); return [ diff --git a/lib/Controller/TextProcessingController.php b/lib/Controller/TextProcessingController.php new file mode 100644 index 00000000..78a30f5f --- /dev/null +++ b/lib/Controller/TextProcessingController.php @@ -0,0 +1,93 @@ +request = $request; + } + + #[NoCSRFRequired] + #[PublicPage] + #[AppAPIAuth] + public function registerProvider( + string $name, + string $displayName, + string $actionHandler, + string $taskType + ): DataResponse { + $ncVersion = $this->config->getSystemValueString('version', '0.0.0'); + if (version_compare($ncVersion, '29.0', '<')) { + return new DataResponse([], Http::STATUS_NOT_IMPLEMENTED); + } + + $provider = $this->textProcessingService->registerTextProcessingProvider( + $this->request->getHeader('EX-APP-ID'), $name, $displayName, $actionHandler, $taskType, + ); + + if ($provider === null) { + return new DataResponse([], Http::STATUS_BAD_REQUEST); + } + + return new DataResponse(); + } + + #[NoCSRFRequired] + #[PublicPage] + #[AppAPIAuth] + public function unregisterProvider(string $name): Response { + $ncVersion = $this->config->getSystemValueString('version', '0.0.0'); + if (version_compare($ncVersion, '29.0', '<')) { + return new DataResponse([], Http::STATUS_NOT_IMPLEMENTED); + } + + $unregistered = $this->textProcessingService->unregisterTextProcessingProvider( + $this->request->getHeader('EX-APP-ID'), $name + ); + + if ($unregistered === null) { + return new DataResponse([], Http::STATUS_NOT_FOUND); + } + + return new DataResponse(); + } + + #[NoCSRFRequired] + #[PublicPage] + #[AppAPIAuth] + public function getProvider(string $name): DataResponse { + $ncVersion = $this->config->getSystemValueString('version', '0.0.0'); + if (version_compare($ncVersion, '29.0', '<')) { + return new DataResponse([], Http::STATUS_NOT_IMPLEMENTED); + } + $result = $this->textProcessingService->getExAppTextProcessingProvider( + $this->request->getHeader('EX-APP-ID'), $name + ); + if (!$result) { + return new DataResponse([], Http::STATUS_NOT_FOUND); + } + return new DataResponse($result, Http::STATUS_OK); + } +} diff --git a/lib/Db/TextProcessing/TextProcessingProvider.php b/lib/Db/TextProcessing/TextProcessingProvider.php new file mode 100644 index 00000000..1be668d4 --- /dev/null +++ b/lib/Db/TextProcessing/TextProcessingProvider.php @@ -0,0 +1,69 @@ +addType('appid', 'string'); + $this->addType('name', 'string'); + $this->addType('displayName', 'string'); + $this->addType('actionHandler', 'string'); + $this->addType('taskType', 'string'); + + if (isset($params['id'])) { + $this->setId($params['id']); + } + if (isset($params['appid'])) { + $this->setAppid($params['appid']); + } + if (isset($params['name'])) { + $this->setName($params['name']); + } + if (isset($params['display_name'])) { + $this->setDisplayName($params['display_name']); + } + if (isset($params['action_handler'])) { + $this->setActionHandler($params['action_handler']); + } + if (isset($params['task_type'])) { + $this->setTaskType($params['task_type']); + } + } + + public function jsonSerialize(): array { + return [ + 'id' => $this->getId(), + 'appid' => $this->getAppid(), + 'name' => $this->getName(), + 'display_name' => $this->getDisplayName(), + 'action_handler' => $this->getActionHandler(), + 'task_type' => $this->getTaskType(), + ]; + } +} diff --git a/lib/Db/TextProcessing/TextProcessingProviderMapper.php b/lib/Db/TextProcessing/TextProcessingProviderMapper.php new file mode 100644 index 00000000..525a2649 --- /dev/null +++ b/lib/Db/TextProcessing/TextProcessingProviderMapper.php @@ -0,0 +1,72 @@ + + */ +class TextProcessingProviderMapper extends QBMapper { + public function __construct(IDBConnection $db) { + parent::__construct($db, 'ex_text_processing'); + } + + /** + * @throws Exception + */ + public function findAllEnabled(): array { + $qb = $this->db->getQueryBuilder(); + $result = $qb->select( + 'ex_text_processing.appid', + 'ex_text_processing.name', + 'ex_text_processing.display_name', + 'ex_text_processing.action_handler', + 'ex_text_processing.task_type', + ) + ->from($this->tableName, 'ex_text_processing') + ->innerJoin('ex_text_processing', 'ex_apps', 'exa', 'exa.appid = ex_text_processing.appid') + ->where( + $qb->expr()->eq('exa.enabled', $qb->createNamedParameter(1, IQueryBuilder::PARAM_INT)) + )->executeQuery(); + return $result->fetchAll(); + } + + /** + * @param string $appId + * @param string $name + * + * @throws DoesNotExistException + * @throws Exception + * @throws MultipleObjectsReturnedException + * + * @return TextProcessingProvider + */ + public function findByAppidName(string $appId, string $name): TextProcessingProvider { + $qb = $this->db->getQueryBuilder(); + return $this->findEntity($qb->select('*') + ->from($this->tableName) + ->where($qb->expr()->eq('appid', $qb->createNamedParameter($appId), IQueryBuilder::PARAM_STR)) + ->andWhere($qb->expr()->eq('name', $qb->createNamedParameter($name), IQueryBuilder::PARAM_STR)) + ); + } + + /** + * @throws Exception + */ + public function removeAllByAppId(string $appId): int { + $qb = $this->db->getQueryBuilder(); + $qb->delete($this->tableName) + ->where( + $qb->expr()->eq('appid', $qb->createNamedParameter($appId, IQueryBuilder::PARAM_STR)) + ); + return $qb->executeStatement(); + } +} diff --git a/lib/Migration/Version1006Date202401011308.php b/lib/Migration/Version1006Date202401011308.php new file mode 100644 index 00000000..7473daa3 --- /dev/null +++ b/lib/Migration/Version1006Date202401011308.php @@ -0,0 +1,60 @@ +hasTable('ex_text_processing')) { + $table = $schema->createTable('ex_text_processing'); + + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + ]); + $table->addColumn('appid', Types::STRING, [ + 'notnull' => true, + 'length' => 32, + ]); + $table->addColumn('name', Types::STRING, [ + 'notnull' => true, + 'length' => 64, + ]); + $table->addColumn('display_name', Types::STRING, [ + 'notnull' => true, + 'length' => 64, + ]); + // ExApp route to forward the action + $table->addColumn('action_handler', Types::STRING, [ + 'notnull' => true, + 'length' => 410, + ]); + $table->addColumn('task_type', Types::STRING, [ + 'notnull' => true, + 'length' => 64, + ]); + + $table->setPrimaryKey(['id']); + $table->addUniqueIndex(['appid', 'name'], 'text_processing__idx'); + } + + return $schema; + } +} diff --git a/lib/Service/AppAPIService.php b/lib/Service/AppAPIService.php index a88e85a1..8e533daf 100644 --- a/lib/Service/AppAPIService.php +++ b/lib/Service/AppAPIService.php @@ -61,6 +61,7 @@ public function __construct( private readonly StylesService $stylesService, private readonly FilesActionsMenuService $filesActionsMenuService, private readonly SpeechToTextService $speechToTextService, + private readonly TextProcessingService $textProcessingService, private readonly ISecureRandom $random, private readonly IUserSession $userSession, private readonly ISession $session, @@ -150,6 +151,7 @@ public function unregisterExApp(string $appId): ?ExApp { $this->scriptsService->deleteExAppScripts($appId); $this->stylesService->deleteExAppStyles($appId); $this->speechToTextService->unregisterExAppSpeechToTextProviders($appId); + $this->textProcessingService->unregisterExAppTextProcessingProviders($appId); $this->cache->remove('/exApp_' . $appId); return $exApp; } catch (Exception $e) { diff --git a/lib/Service/SpeechToTextService.php b/lib/Service/SpeechToTextService.php index a93a7228..f090f355 100644 --- a/lib/Service/SpeechToTextService.php +++ b/lib/Service/SpeechToTextService.php @@ -165,8 +165,9 @@ public function getName(): string { } public function transcribeFile(File $file, float $maxExecutionTime = 0): string { - $route = $this->sttProvider->getActionHandler(); + /** @var AppAPIService $service */ $service = $this->serverContainer->get(AppAPIService::class); + $route = $this->sttProvider->getActionHandler(); try { $fileHandle = $file->fopen('r'); diff --git a/lib/Service/TextProcessingService.php b/lib/Service/TextProcessingService.php new file mode 100644 index 00000000..806673d6 --- /dev/null +++ b/lib/Service/TextProcessingService.php @@ -0,0 +1,237 @@ + 'OCP\TextProcessing\FreePromptTaskType', + 'headline' => 'OCP\TextProcessing\HeadlineTaskType', + 'summary' => 'OCP\TextProcessing\SummaryTaskType', + 'topics' => 'OCP\TextProcessing\TopicsTaskType', + ]; + + private ICache $cache; + + public function __construct( + ICacheFactory $cacheFactory, + private readonly TextProcessingProviderMapper $mapper, + private readonly ?string $userId, + private readonly LoggerInterface $logger, + ) { + $this->cache = $cacheFactory->createDistributed(Application::APP_ID . '/ex__text_processing_providers'); + } + + public function getRegisteredTextProcessingProviders(): array { + try { + $cacheKey = '/ex_text_processing_providers'; + $providers = $this->cache->get($cacheKey); + if ($providers === null) { + $providers = $this->mapper->findAllEnabled(); + $this->cache->set($cacheKey, $providers); + } + + return array_map(function ($provider) { + return $provider instanceof TextProcessingProvider ? $provider : new TextProcessingProvider($provider); + }, $providers); + } catch (Exception) { + return []; + } + } + + public function getExAppTextProcessingProvider(string $appId, string $name): ?TextProcessingProvider { + $cacheKey = '/ex_text_processing_providers_' . $appId . '_' . $name; + $cached = $this->cache->get($cacheKey); + if ($cached !== null) { + return $cached instanceof TextProcessingProvider ? $cached : new TextProcessingProvider($cached); + } + + try { + $textProcessingProvider = $this->mapper->findByAppidName($appId, $name); + } catch (DoesNotExistException|MultipleObjectsReturnedException|Exception) { + return null; + } + $this->cache->set($cacheKey, $textProcessingProvider); + return $textProcessingProvider; + } + + public function registerTextProcessingProvider( + string $appId, + string $name, + string $displayName, + string $actionHandler, + string $taskType + ): ?TextProcessingProvider { + try { + $textProcessingProvider = $this->mapper->findByAppidName($appId, $name); + } catch (DoesNotExistException|MultipleObjectsReturnedException|Exception) { + $textProcessingProvider = null; + } + try { + if (!$this->isTaskTypeValid($taskType)) { + return null; + } + + $newTextProcessingProvider = new TextProcessingProvider([ + 'appid' => $appId, + 'name' => $name, + 'display_name' => $displayName, + 'action_handler' => ltrim($actionHandler, '/'), + 'task_type' => $taskType, + ]); + + if ($textProcessingProvider !== null) { + $newTextProcessingProvider->setId($textProcessingProvider->getId()); + } + + $textProcessingProvider = $this->mapper->insertOrUpdate($newTextProcessingProvider); + $this->cache->set('/ex_text_processing_providers_' . $appId . '_' . $name, $textProcessingProvider); + $this->resetCacheEnabled(); + } catch (Exception $e) { + $this->logger->error( + sprintf('Failed to register ExApp %s TextProcessingProvider %s. Error: %s', $appId, $name, $e->getMessage()), ['exception' => $e] + ); + return null; + } + return $textProcessingProvider; + } + + public function unregisterTextProcessingProvider(string $appId, string $name): ?TextProcessingProvider { + try { + $textProcessingProvider = $this->getExAppTextProcessingProvider($appId, $name); + if ($textProcessingProvider === null) { + return null; + } + $this->mapper->delete($textProcessingProvider); + $this->cache->remove('/ex_text_processing_providers_' . $appId . '_' . $name); + $this->resetCacheEnabled(); + return $textProcessingProvider; + } catch (Exception $e) { + $this->logger->error(sprintf('Failed to unregister ExApp %s TextProcessingProvider %s. Error: %s', $appId, $name, $e->getMessage()), ['exception' => $e]); + return null; + } + } + + /** + * Register dynamically ExApps TextProcessing providers with ID using anonymous classes. + * + * @param IRegistrationContext $context + * @param IServerContainer $serverContainer + * + * @return void + */ + public function registerExAppTextProcessingProviders(IRegistrationContext &$context, IServerContainer $serverContainer): void { + $exAppsProviders = $this->getRegisteredTextProcessingProviders(); + /** @var TextProcessingProvider $exAppProvider */ + foreach ($exAppsProviders as $exAppProvider) { + if (!$this->isTaskTypeValid($exAppProvider->getTaskType())) { + continue; + } + + $className = '\\OCA\\AppAPI\\' . $exAppProvider->getAppid() . '\\' . $exAppProvider->getName(); + $provider = $this->getAnonymousExAppProvider($exAppProvider, $className, $serverContainer); + $context->registerService($className, function () use ($provider) { + return $provider; + }); + $context->registerTextProcessingProvider($className); + } + } + + /** + * @psalm-suppress UndefinedClass, MissingDependency, InvalidReturnStatement, InvalidReturnType + */ + private function getAnonymousExAppProvider( + TextProcessingProvider $provider, + string $className, + IServerContainer $serverContainer + ): IProviderWithId { + return new class($provider, $serverContainer, $className, $this->userId) implements IProviderWithId, IProviderWithUserId { + public function __construct( + private readonly TextProcessingProvider $provider, + private readonly IServerContainer $serverContainer, + private readonly string $className, + private ?string $userId, + ) { + } + + public function getId(): string { + return $this->className; + } + + public function getName(): string { + return $this->provider->getDisplayName(); + } + + public function process(string $prompt, float $maxExecutionTime = 0): string { + /** @var AppAPIService $service */ + $service = $this->serverContainer->get(AppAPIService::class); + $route = $this->provider->getActionHandler(); + + $response = $service->requestToExAppById($this->provider->getAppid(), + $route, + $this->userId, + 'POST', + params: [ + 'prompt' => $prompt, + 'max_execution_time' => $maxExecutionTime, + ], + options: [ + 'timeout' => $maxExecutionTime, + ], + ); + if (is_array($response)) { + throw new \Exception(sprintf('Failed process text task: %s:%s:%s. Error: %s', + $this->provider->getAppid(), + $this->provider->getName(), + $this->provider->getTaskType(), + $response['error'] + )); + } + return $response->getBody(); + } + + public function getTaskType(): string { + return TextProcessingService::TASK_TYPES[$this->provider->getTaskType()]; + } + + public function setUserId(?string $userId): void { + $this->userId = $userId; + } + }; + } + + private function isTaskTypeValid(string $getActionType): bool { + return in_array($getActionType, array_keys(self::TASK_TYPES)); + } + + private function resetCacheEnabled(): void { + $this->cache->remove('/ex_text_processing_providers'); + } + + public function unregisterExAppTextProcessingProviders(string $appId): int { + try { + $result = $this->mapper->removeAllByAppId($appId); + } catch (Exception) { + $result = -1; + } + $this->cache->clear('/ex_text_processing_providers_' . $appId); + $this->resetCacheEnabled(); + return $result; + } +}