Skip to content

Commit

Permalink
Rework of App:Register (#128)
Browse files Browse the repository at this point in the history
Fixes: #122


1. Made sending the "init" request in a separate process
2. If "init" request fails with STATUS_NOT_IMPLEMENTED or
STATUS_NOT_FOUND sets the progress to 100
3. Added `wait-finish` optional parameter to `app_api:app:register` occ
command.

What is missing:

- [x] Global option: how long the "/init" request can be proceed.
- [x] Docs update for this
- [x] Update nc_py_api CI for this
- [x] Added test for registering ExApp that does not have "/init"
endpoint.


This allows to implement ExApp without "/init" endpoint and made it
optional.

---------

Signed-off-by: Alexander Piskun <bigcat88@icloud.com>
Signed-off-by: Andrey Borysenko <andrey18106x@gmail.com>
Co-authored-by: Andrey Borysenko <andrey18106x@gmail.com>
  • Loading branch information
bigcat88 and andrey18106 authored Nov 27, 2023
1 parent 9ce6fc3 commit ec0324b
Show file tree
Hide file tree
Showing 15 changed files with 390 additions and 127 deletions.
111 changes: 109 additions & 2 deletions .github/workflows/tests-special.yml
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ jobs:
php occ app_api:daemon:register manual_install "Manual Install" manual-install 0 0 0
php occ app_api:app:register nc_py_api manual_install --json-info \
"{\"appid\":\"$APP_ID\",\"name\":\"$APP_ID\",\"daemon_config_name\":\"manual_install\",\"version\":\"$APP_VERSION\",\"secret\":\"$APP_SECRET\",\"host\":\"localhost\",\"port\":$APP_PORT,\"scopes\":{\"required\":[\"SYSTEM\", \"FILES\", \"FILES_SHARING\"],\"optional\":[\"USER_INFO\", \"USER_STATUS\", \"NOTIFICATIONS\", \"WEATHER_STATUS\", \"TALK\"]},\"protocol\":\"http\",\"system_app\":1}" \
--force-scopes
--force-scopes --wait-finish
kill -15 $(cat /tmp/_install.pid)
timeout 3m tail --pid=$(cat /tmp/_install.pid) -f /dev/null
Expand All @@ -134,11 +134,118 @@ jobs:
path: data/nextcloud.log
if-no-files-found: warn

no-init-endpoint:
runs-on: ubuntu-22.04
name: Without Init Endpoint

services:
postgres:
image: ghcr.io/nextcloud/continuous-integration-postgres-14:latest
ports:
- 4444:5432/tcp
env:
POSTGRES_USER: root
POSTGRES_PASSWORD: rootpassword
POSTGRES_DB: nextcloud
options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5

steps:
- uses: actions/setup-python@v4
with:
python-version: '3.11'

- name: Set app env
run: echo "APP_NAME=${GITHUB_REPOSITORY##*/}" >> $GITHUB_ENV

- name: Checkout server
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
with:
submodules: true
repository: nextcloud/server
ref: 'stable27'

- name: Checkout Notifications
uses: actions/checkout@v3
with:
repository: nextcloud/notifications
ref: ${{ matrix.server-version }}
path: apps/notifications

- name: Checkout AppAPI
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
with:
path: apps/${{ env.APP_NAME }}

- name: Set up php
uses: shivammathur/setup-php@4bd44f22a98a19e0950cbad5f31095157cc9621b # v2
with:
php-version: '8.1'
extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, session, simplexml, xmlreader, xmlwriter, zip, zlib, pgsql, pdo_pgsql
coverage: none
ini-file: development
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- name: Check composer file existence
id: check_composer
uses: andstor/file-existence-action@20b4d2e596410855db8f9ca21e96fbe18e12930b # v2
with:
files: apps/${{ env.APP_NAME }}/composer.json

- name: Set up dependencies
if: steps.check_composer.outputs.files_exists == 'true'
working-directory: apps/${{ env.APP_NAME }}
run: composer i

- name: Set up Nextcloud
env:
DB_PORT: 4444
run: |
mkdir data
./occ maintenance:install --verbose --database=pgsql --database-name=nextcloud --database-host=127.0.0.1 \
--database-port=$DB_PORT --database-user=root --database-pass=rootpassword \
--admin-user admin --admin-pass admin
./occ app:enable notifications
./occ app:enable --force ${{ env.APP_NAME }}
- name: Run Nextcloud
run: PHP_CLI_SERVER_WORKERS=2 php -S 127.0.0.1:8080 &

- name: Checkout NcPyApi
uses: actions/checkout@v3
with:
path: nc_py_api
repository: cloud-py-api/nc_py_api

- name: Install NcPyApi
working-directory: nc_py_api
run: python3 -m pip -v install ".[dev]"

- name: Register NcPyApi
run: |
python3 apps/${{ env.APP_NAME }}/tests/install_no_init.py &
echo $! > /tmp/_install.pid
sleep 5s
php occ app_api:daemon:register manual_install "Manual Install" manual-install 0 0 0
php occ app_api:app:register nc_py_api manual_install --json-info \
"{\"appid\":\"$APP_ID\",\"name\":\"$APP_ID\",\"daemon_config_name\":\"manual_install\",\"version\":\"$APP_VERSION\",\"secret\":\"$APP_SECRET\",\"host\":\"localhost\",\"port\":$APP_PORT,\"scopes\":{\"required\":[\"SYSTEM\", \"FILES\", \"FILES_SHARING\"],\"optional\":[\"USER_INFO\", \"USER_STATUS\", \"NOTIFICATIONS\", \"WEATHER_STATUS\", \"TALK\"]},\"protocol\":\"http\",\"system_app\":1}" \
--force-scopes --wait-finish
kill -15 $(cat /tmp/_install.pid)
timeout 3m tail --pid=$(cat /tmp/_install.pid) -f /dev/null
- name: Upload NC logs
if: always()
uses: actions/upload-artifact@v3
with:
name: install_no_init.log
path: data/nextcloud.log
if-no-files-found: warn

tests-special-success:
permissions:
contents: none
runs-on: ubuntu-22.04
needs: [app-version-higher]
needs: [app-version-higher, no-init-endpoint]
name: TestsSpecial-OK
steps:
- run: echo "Tests special passed successfully"
3 changes: 2 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/).

## [Unreleased]

## [1.3.0 - 2023-12-0x]
## [1.3.0 - 2023-11-28]

### Changed

- Reworked: algorithm of `app:register` and occ cli command, "/init" endpoint now is optional. #128
- Reworked: `app_api:app:unregister` occ cli command, make it much robust. #127

## [1.2.2 - 2023-11-13]
Expand Down
4 changes: 4 additions & 0 deletions appinfo/info.xml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ to join us in shaping a more versatile, stable, and secure app landscape.
<php min-version="8.1"/>
<nextcloud min-version="27" max-version="29"/>
</dependencies>
<background-jobs>
<job>OCA\AppAPI\BackgroundJob\ExAppInitStatusCheckJob</job>
</background-jobs>
<repair-steps>
<install>
<step>OCA\AppAPI\Migration\DataInitializationStep</step>
Expand All @@ -70,6 +73,7 @@ to join us in shaping a more versatile, stable, and secure app landscape.
<command>OCA\AppAPI\Command\ExApp\Deploy</command>
<command>OCA\AppAPI\Command\ExApp\Register</command>
<command>OCA\AppAPI\Command\ExApp\Unregister</command>
<command>OCA\AppAPI\Command\ExApp\DispatchInit</command>
<command>OCA\AppAPI\Command\ExApp\Update</command>
<command>OCA\AppAPI\Command\ExApp\Enable</command>
<command>OCA\AppAPI\Command\ExApp\Disable</command>
Expand Down
6 changes: 5 additions & 1 deletion docs/tech_details/Deployment.rst
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,11 @@ Application installation scheme
1. AppAPI deploys the application and launches it.
2. AppAPI for `N` seconds (default ``90``) checks the ``/heartbeat`` endpoint with ``GET`` request.
3. AppAPI sends a ``POST`` to the ``/init`` endpoint.
4. ExApp sends an integer from ``0`` to ``100` to the OCS endpoint ``apps/app_api/apps/status`` indicating the initialization progress. After sending ``100``, the application is considered initialized.

.. note:: if ExApp do not implements ``/init`` endpoint and
AppAPI receives 501 or 404 status error, AppAPI enables the application by going to point 5.

4. **ExApp** sends an integer from ``0`` to ``100`` to the OCS endpoint ``apps/app_api/apps/status`` indicating the initialization progress. After sending ``100``, the application is considered initialized.
5. AppAPI sends a PUT to the ``/enabled`` endpoint.

ExApp info.xml schema
Expand Down
16 changes: 7 additions & 9 deletions docs/tech_details/InstallationFlow.rst
Original file line number Diff line number Diff line change
Expand Up @@ -21,20 +21,18 @@ The application, in response to the request "/heartbeat", should return json: ``
Init
----

.. note:: Starting from this point, all requests made by AppAPI contains :ref:`auth-headers`.

After application is ready, which is determined by previous step,
AppAPI sends ``POST`` request to the ``/init`` application endpoint.

*Application should response with empty JSON, if initialization takes long time it should be done in background and not in this request handler.*

.. note:: Starting from this point, all requests made by AppAPI contains :ref:`auth-headers`.

If the application does not need to carry out long initialization, it can immediately execute an ``OCS request`` to
``/ocs/v1.php/apps/app_api/apps/status/$APP_ID`` with such a payload in json format::
*If the application does not need to carry out long initialization, it has an option to not implement "/init" endpoint, so
AppAPI will get 404 or 501 error on it's request, and consider that initialization is done and this section can be skipped.*

{"progress": 100}
In case you want to implement "/init" endpoint, your application should:

If the application initialization takes a long time, the application should periodically send an ``OCS request`` to
``/ocs/v1.php/apps/app_api/apps/status/$APP_ID`` with the progress value.
1. In "/init" handler: Response with empty JSON on AppAPI call.
2. In background job: Send an ``OCS request`` to ``/ocs/v1.php/apps/app_api/apps/status/$APP_ID`` with the progress value.

Possible values for **progress** are integers from 1 to 100;
after receiving the value 100, the **application is considered initialized and ready to work**.
Expand Down
49 changes: 49 additions & 0 deletions lib/BackgroundJob/ExAppInitStatusCheckJob.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
<?php

declare(strict_types=1);

namespace OCA\AppAPI\BackgroundJob;

use OCA\AppAPI\AppInfo\Application;
use OCA\AppAPI\Db\ExAppMapper;
use OCA\AppAPI\Service\AppAPIService;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\TimedJob;
use OCP\DB\Exception;
use OCP\IConfig;

class ExAppInitStatusCheckJob extends TimedJob {
private const everyMinuteInterval = 60;

public function __construct(
ITimeFactory $time,
private ExAppMapper $mapper,
private AppAPIService $service,
private IConfig $config,
) {
parent::__construct($time);

$this->setInterval(self::everyMinuteInterval);
}

protected function run($argument): void {
// Iterate over all ExApp and check for status.init_start_time if it is older than ex_app_init_timeout minutes
// set status.progress=0 and status.error message with timeout error
try {
$exApps = $this->mapper->findAll();
$initTimeoutMinutes = intval($this->config->getAppValue(Application::APP_ID, 'ex_app_init_timeout', '40'));
foreach ($exApps as $exApp) {
$status = json_decode($exApp->getStatus(), true);
if (!isset($status['init_start_time'])) {
continue;
}
if (($status['init_start_time'] + $initTimeoutMinutes * 60) > time()) {
$this->service->setAppInitProgress(
$exApp->getAppId(), 0, sprintf('ExApp %s initialization timed out (%sm)', $exApp->getAppid(), $initTimeoutMinutes * 60)
);
}
}
} catch (Exception) {
}
}
}
23 changes: 0 additions & 23 deletions lib/BackgroundJob/ExAppStatusUpdateJob.php

This file was deleted.

5 changes: 2 additions & 3 deletions lib/Command/ExApp/Deploy.php
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int
return 2;
}

$defaultDaemonConfigName = $this->config->getAppValue(Application::APP_ID, 'default_daemon_config');
$daemonConfigName = $input->getArgument('daemon-config-name');
if (!isset($daemonConfigName) && $defaultDaemonConfigName !== '') {
$daemonConfigName = $defaultDaemonConfigName;
if (!isset($daemonConfigName)) {
$daemonConfigName = $this->config->getAppValue(Application::APP_ID, 'default_daemon_config');
}
$daemonConfig = $this->daemonConfigService->getDaemonConfigByName($daemonConfigName);
if ($daemonConfig === null) {
Expand Down
39 changes: 39 additions & 0 deletions lib/Command/ExApp/DispatchInit.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
<?php

declare(strict_types=1);

namespace OCA\AppAPI\Command\ExApp;

use OCA\AppAPI\Service\AppAPIService;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;

class DispatchInit extends Command {

public function __construct(
private AppAPIService $service,
) {
parent::__construct();
}

protected function configure(): void {
$this->setHidden(true);
$this->setName('app_api:app:dispatch_init');
$this->setDescription('Internal command to dispatch init command');

$this->addArgument('appid', InputArgument::REQUIRED);
}

protected function execute(InputInterface $input, OutputInterface $output): int {
$appId = $input->getArgument('appid');
$exApp = $this->service->getExApp($appId);
if ($exApp === null) {
$output->writeln(sprintf('ExApp %s not found. Failed to dispatch init.', $appId));
return 1;
}
$this->service->dispatchExAppInitInternal($exApp);
return 0;
}
}
Loading

0 comments on commit ec0324b

Please sign in to comment.