chore: 0.31.0 release with crc service

CHANGELOG.rst

@@ -1,5 +1,57 @@
 .. _changelog:
 
+0.31.0
+------
+
+Renku ``0.31.0`` introduces the compute resource control (CRC) service, enabling Renku administrators to manage access to 
+specific computing resources. The service exposes an HTTP API for the administrators to interact with. In this way, 
+Renku administrators can create, update or delete resource pools, and can add to or remove users from resource pools. 
+Please note that a user interface for the CRC has not been added yet but it will be added in a future release. Currently,
+the only way for administrators to interact with the CRC service is through the swagger page which can be found at the path
+``/swagger/?urls.primaryName=crc%20service`` appended to the base URL of a Renku deployment. A Renku administrator is any user who
+has the `renku-admin` realm role. Assigning users to this role can be performed by the Keycloak administrator via the Keycloak UI
+or API.
+
+The CRC service also brings changes to the user interface for launching sessions, specifically when it comes to selecting 
+compute resources for a specific session. With this version we have grouped different pre-set configurations of memory, CPU, 
+RAM and GPU in resource classes. Resource classes are further grouped in resource pools and users are asked to select the resource
+pool and class they wish to use when they launch a session rather than separately specify memory, CPU, RAM and GPU requirements. 
+The selection for the amount of disk storage required is also changed but now has more freedom than before. Users can now select disk
+storage with a slider that only has a maximum limit and no pre-set steps. For projects where the users have specified resource requests
+in the project settings the UI will provide hints as to which resource classes are suitable based on the settings. When the quick launch
+button is used to start a session the closest equal or greater resource class based on the project settings will be automatically selected.
+
+Apart from the changes needed to support compute resource access features, support has also been added for common R file extensions.
+
+User-Facing Changes
+~~~~~~~~~~~~~~~~~~~
+
+**🌟 New Features**
+
+- 🧑‍💻 **UI**: Update session start options and project settings to use compute resource pools
+  (`#2484 <https://github.com/SwissDataScienceCenter/renku-ui/issues/2484>`_).
+
+**🐞 Bug Fixes**
+
+- 〽️ **UI**: Support common R file extensions
+  (`#2638 <https://github.com/SwissDataScienceCenter/renku-ui/issues/2638>`_).
+
+Internal Changes
+~~~~~~~~~~~~~~~~
+
+**🌟 New Features**
+
+- **Infrastructure**: Add the compute resource control service.
+- **Renku Notebooks**: Use the compute resource control (CRC) service.
+
+Individual components
+~~~~~~~~~~~~~~~~~~~~~~
+
+- `renku-data-services 0.0.1 <https://github.com/SwissDataScienceCenter/renku-data-services/releases/tag/v0.0.1>`_
+- `renku-notebooks 1.16.0 <https://github.com/SwissDataScienceCenter/renku-notebooks/releases/tag/1.16.0>`_
+- `renku-ui 3.9.0 <https://github.com/SwissDataScienceCenter/renku-ui/releases/tag/3.9.0>`_
+
+
 0.30.1
 ------
 
@@ -18,6 +70,7 @@ Individual components
 
 - `renku-gateway 0.21.1 <https://github.com/SwissDataScienceCenter/renku-gateway/releases/tag/0.21.1>`_
 
+
 0.30.0
 ------
 

cypress-tests/cypress/e2e/publicProject.cy.ts

@@ -194,18 +194,21 @@ describe("Basic public project functionality", () => {
   it("Can view and modify sessions settings", () => {
     cy.dataCy("project-navbar").contains("Settings").should("exist").click();
     cy.intercept("/ui-server/api/renku/*/config.set").as("configSet");
-    // ? The settings page refreshes when stale. We should wait for that only when it's invoked.
-    let configInvocations = 0;
-    cy.intercept("/ui-server/api/renku/*/config.show?git_url=*", req => { configInvocations++; }).as("getConfig");
+    cy.intercept("/ui-server/api/renku/*/config.show?git_url=*").as(
+      "getConfig"
+    );
 
-    const navigateToSettingsSessions = () => {
-      const invoked = configInvocations;
+    const navigateToSettingsSessions = ({
+      waitForApis,
+    }: { waitForApis?: boolean } = {}) => {
       robustNavigateToProjectPage("/settings");
       cy.get(".form-rk-green form").contains("Project Tags").should("exist");
       robustNavigateToProjectPage("/settings/sessions");
       cy.get("h3").contains("Session settings").should("exist");
-      if (invoked > configInvocations)
-        cy.wait("@configShow", { timeout: TIMEOUTS.long });
+      cy.intercept("/ui-server/api/data/resource_pools").as("getResourcePools");
+      if (waitForApis) {
+        cy.wait("@getConfig", { timeout: TIMEOUTS.long });
+      }
     };
 
     // Make sure the renku.ini is in a pristine state
@@ -215,20 +218,31 @@ describe("Basic public project functionality", () => {
     cy.get("pre.hljs").should("be.visible");
     cy.get("pre.hljs").contains("cpu_request").should("not.exist");
 
-    navigateToSettingsSessions();
-    cy.get("div.form-rk-green div.row").contains("button", "0.5").should("exist").click();
+    // Add a compute requirement for sessions
+    navigateToSettingsSessions({ waitForApis: true });
+    cy.contains("label", "Number of CPUs")
+      .parent()
+      .find("input.form-control")
+      .should("exist")
+      .click()
+      .type("1.5")
+      .blur();
+    cy.contains(".badge", "Saving");
     cy.wait("@configSet");
-    cy.get("div.form-rk-green div.success-feedback").contains("Updated.").should("be.visible");
+    cy.contains(".badge", "Saved");
 
     robustNavigateToProjectPage("/files");
     cy.get("div#tree-content").contains("renku.ini").should("exist").click();
     cy.get(".hljs.language-ini").contains("[interactive]").should("be.visible");
-    cy.get("pre.hljs").contains("cpu_request = 0.5").should("exist");
+    cy.get("pre.hljs").contains("cpu_request = 1.5").should("exist");
 
     navigateToSettingsSessions();
-    cy.get("#cpu_request_reset").should("be.visible").click();
+    cy.get("#project-settings-sessions-interactive-cpu-request-reset")
+      .should("be.visible")
+      .click();
+    cy.contains(".badge", "Saving");
     cy.wait("@configSet");
-    cy.get("div.form-rk-green div.success-feedback").contains("Updated.").should("exist");
+    cy.contains(".badge", "Saved");
 
     robustNavigateToProjectPage("/files");
     cy.get("div#tree-content").contains("renku.ini").should("exist").click();

helm-chart/renku/requirements.yaml

@@ -9,11 +9,11 @@ dependencies:
 - name: renku-ui
   alias: ui
   repository: "https://swissdatasciencecenter.github.io/helm-charts/"
-  version: 3.8.1
+  version: 3.9.0
 - name: renku-notebooks
   alias: notebooks
   repository: "https://swissdatasciencecenter.github.io/helm-charts/"
-  version: 1.15.3
+  version: 1.16.0
 - name: renku-gateway
   alias: gateway
   repository: "https://swissdatasciencecenter.github.io/helm-charts/"

helm-chart/renku/templates/configmap.yaml

@@ -10,9 +10,10 @@ metadata:
     heritage: {{ .Release.Service }}
 data:
   {{- if and .Values.gitlab.enabled }}
+  # DO NOT USE set -x you will leak the password in the logs
   init-gitlab.sh: |-
     #!/usr/bin/env bash
-    set -ex
+    set -e
     env
 
     GITLAB_SERVICE_URL="http://{{ template "gitlab.fullname" . }}{{ .Values.global.gitlab.urlPrefix }}"
@@ -35,9 +36,10 @@ data:
   # Init scripts that populate /docker-entrypoint-initdb.d
 
   # Unashamedly copied from: https://github.com/docker-library/postgres/blob/master/9.6/docker-entrypoint.sh
+  # DO NOT USE set -x you will leak the password in the logs
   init-postgres.sh: |-
     #!/bin/bash
-    set -ex
+    set -e
     env
 
     until sleep 1; pg_isready; do
@@ -67,9 +69,9 @@ data:
     done
 
   {{- if .Values.keycloakx.enabled }}
+  # DO NOT USE set -x you will leak the password in the logs
   init-keycloak-db.sh: |-
     #!/bin/bash
-    set -x
 
     KEYCLOAK_POSTGRES_PASSWORD=$(cat /keycloak-postgres/KC_DB_PASSWORD)
 
@@ -86,9 +88,9 @@ data:
   {{- end }}
 
   {{- if .Values.gitlab.enabled }}
+  # DO NOT USE set -x you will leak the password in the logs
   init-gitlab-db.sh: |-
       #!/bin/bash
-      set -x
 
       GITLAB_POSTGRES_PASSWORD=$(cat /gitlab-postgres/gitlab-postgres-password)
 
@@ -108,9 +110,9 @@ data:
 
   {{- if .Values.graph.enabled }}
 
+  # DO NOT USE set -x you will leak the password in the logs
   init-dbEventLog-db.sh: |-
       #!/bin/bash
-      set -x
 
       DB_EVENT_LOG_POSTGRES_PASSWORD=$(cat /graph-db-postgres/graph-dbEventLog-postgresPassword)
       DB_EVENT_LOG_DB_NAME=event_log
@@ -127,9 +129,9 @@ data:
         grant all privileges on schema "public" to "{{ .Values.global.graph.dbEventLog.postgresUser }}";
       EOSQL
 
+  # DO NOT USE set -x you will leak the password in the logs
   init-triplesGenerator-db.sh: |-
       #!/bin/bash
-      set -x
 
       DB_TG_LOG_POSTGRES_PASSWORD=$(cat /graph-tg-postgres/graph-triplesGenerator-postgresPassword)
       DB_TG_LOG_DB_NAME=triples_generator
@@ -149,7 +151,6 @@ data:
 
   init-tokenRepository-db.sh: |-
       #!/bin/bash
-      set -x
 
       TOKEN_REPOSITORY_POSTGRES_PASSWORD=$(cat /graph-token-postgres/graph-tokenRepository-postgresPassword)
       TOKEN_REPOSITORY_DB_NAME=projects_tokens
@@ -166,3 +167,23 @@ data:
         grant all privileges on schema "public" to "{{ .Values.global.graph.tokenRepository.postgresUser }}";
       EOSQL
   {{- end }}
+
+  # DO NOT USE set -x you will leak the password in the logs
+  init-renku-db.sh: |-
+    #!/bin/bash
+
+    PASSWORD=$(cat /db-common-password/password)
+    DB_NAME=renku
+
+    psql -v ON_ERROR_STOP=1 <<-EOSQL
+      create user "{{ .Values.global.db.common.username }}" password '$PASSWORD';
+      create database "$DB_NAME" WITH OWNER "{{ .Values.global.db.common.username }}";
+    EOSQL
+
+    psql postgres -v ON_ERROR_STOP=1 --dbname "$DB_NAME" <<-EOSQL
+      create extension if not exists "pg_trgm";
+      revoke all on schema "public" from "public";
+      grant all privileges on database "$DB_NAME" to "{{ .Values.global.db.common.username }}";
+      grant all privileges on database "$DB_NAME" to "postgres";
+      grant all privileges on schema "public" to "{{ .Values.global.db.common.username }}";
+    EOSQL

helm-chart/renku/templates/crc/configmap.yaml

@@ -0,0 +1,16 @@
+{{- if and .Values.notebooks.serverOptions .Values.notebooks.serverDefaults -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "renku.fullname" . }}-server-options
+  labels:
+    app: {{ template "renku.name" . }}
+    chart: {{ template "renku.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  server_options.json: |
+    {{ .Values.notebooks.serverOptions | default dict | toJson }}
+  server_defaults.json: |
+    {{ toJson .Values.notebooks.serverDefaults }}
+{{- end -}}

helm-chart/renku/templates/crc/deployment.yaml

@@ -0,0 +1,108 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "renku.fullname" . }}-crc
+  labels:
+    app: renku-crc
+    chart: {{ template "renku.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  {{- if not .Values.crc.autoscaling.enabled }}
+  replicas: {{ .Values.crc.replicaCount }}
+  {{- end }}
+  strategy: 
+    {{- toYaml .Values.crc.updateStrategy | nindent 4 }}
+  selector:
+    matchLabels:
+      app: renku-crc
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: renku-crc
+        release: {{ .Release.Name }}
+      {{- with .Values.crc.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      automountServiceAccountToken: {{ .Values.global.debug }}
+      initContainers:
+        {{- include "certificates.initContainer" . | nindent 8 }}
+      containers:
+        - name: crc
+          image: "{{ .Values.crc.image.repository }}:{{ .Values.crc.image.tag }}"
+          imagePullPolicy: {{ .Values.crc.image.pullPolicy }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          ports:
+            - name: http
+              containerPort: 8000
+              protocol: TCP
+          env:
+            - name: DB_HOST
+              value: {{ template "postgresql.fullname" . }}
+            - name: DB_USER
+              value: {{ .Values.global.db.common.username }}
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.global.db.common.passwordSecretName }}
+                  key: password
+            - name: KEYCLOAK_URL
+              value: {{ (printf "%s://%s/auth/" (include "gateway.protocol" .) .Values.global.renku.domain) | quote}}
+            - name: KEYCLOAK_TOKEN_SIGNATURE_ALGS
+              value: "RS256"
+            - name: SERVER_DEFAULTS
+              value: /etc/renku-crc/server_options/server_defaults.json
+            - name: SERVER_OPTIONS
+              value: /etc/renku-crc/server_options/server_options.json
+            - name: K8S_NAMESPACE
+              value: {{ .Release.Namespace | quote }}
+          volumeMounts:
+            - name: server-options
+              mountPath: /etc/renku-crc/server_options
+            {{- include "certificates.volumeMounts.system" . | nindent 12 }}
+          livenessProbe:
+            httpGet:
+              path: /api/data/version
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            failureThreshold: 6
+          readinessProbe:
+            httpGet:
+              path: /api/data/version
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 2
+            failureThreshold: 2
+          startupProbe:
+            httpGet:
+              path: /api/data/version
+              port: http
+            periodSeconds: 5
+            failureThreshold: 60
+          resources:
+            {{ toYaml .Values.crc.resources | nindent 12 }}
+    {{- with .Values.crc.nodeSelector }}
+      nodeSelector:
+        {{ toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.crc.affinity }}
+      affinity:
+        {{ toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.crc.tolerations }}
+      tolerations:
+        {{ toYaml . | nindent 8 }}
+    {{- end }}
+      volumes:
+        - name: server-options
+          configMap:
+            name: {{ template "renku.fullname" . }}-server-options
+        {{- include "certificates.volumes" . | nindent 8 }}
+      serviceAccountName: {{ template "renku.fullname" . }}-crc