Merge pull request #4053 from wireapp/release_2024-05-21_12_44

Release 2024-05-21 - (expected chart version 5.3.0)

.hlint.yaml

@@ -13,6 +13,7 @@
 - ignore: { name: Use section }
 - ignore: { name: "Use :" }
 - ignore: { name: Use underscore }
+- ignore: { name: Use error }
 
 # custom rules:
 - hint: { lhs: (() <$), rhs: void }

CHANGELOG.md

@@ -1,3 +1,47 @@
+# [2024-05-21] (Chart Release 5.3.0)
+
+## API changes
+
+
+* /mls/keys use JWK instead of bare keys as MLS removal keys (#3548)
+
+* The `cipher_suite` field is not present anymore in objects corresponding to newly created conversations (#4009)
+
+
+## Features
+
+
+* Upgrade `rusty-jwt-tools` to support `ecdsa_secp256r1_sha256` (#4035)
+
+* gundeck: Delete all APNS_VOIP and APNS_VOIP_SANDBOX push tokens (#4044)
+
+
+## Bug fixes and other updates
+
+
+* gundeck: Fix parsing errors for SNS ARN for VOIP Tokens (#4040)
+
+* Fix hardcoded ciphersuite when switching to mixed (#4048)
+
+
+## Internal changes
+
+
+* Add tool to determine number of phone-only users (#4024)
+
+* Log federator request ID on exceptions (#4037)
+
+* Update mls-test-cli to version 0.12 (#4039)
+
+* Remove inbucket helm chart. (#4032)
+
+* Finish servantifying galley and remove wai-routing dependency (#4018)
+
+* New subsystem for user management. (#3977)
+
+* Clean up syntax of test cases that occur in BSI audit. (#4041)
+
+
 # [2024-04-26] (Chart Release 4.43.0)
 
 ## Bug fixes and other updates

Makefile

@@ -18,7 +18,7 @@ fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice		\
 calling-test demo-smtp elasticsearch-curator elasticsearch-external				\
 elasticsearch-ephemeral minio-external cassandra-external						\
 nginx-ingress-controller ingress-nginx-controller nginx-ingress-services reaper restund coturn		\
-inbucket k8ssandra-test-cluster postgresql ldap-scim-bridge smallstep-accomp
+k8ssandra-test-cluster postgresql ldap-scim-bridge smallstep-accomp
 KIND_CLUSTER_NAME     := wire-server
 HELM_PARALLELISM      ?= 1 # 1 for sequential tests; 6 for all-parallel tests
 
@@ -129,8 +129,9 @@ devtest:
 .PHONY: sanitize-pr
 sanitize-pr:
 	./hack/bin/generate-local-nix-packages.sh
+	make formatf
 	make hlint-inplace-pr
-	make format
+	make hlint-check-pr  # sometimes inplace has been observed not to do its job very well.
 	make git-add-cassandra-schema
 	@git diff-files --quiet -- || ( echo "There are unstaged changes, please take a look, consider committing them, and try again."; exit 1 )
 	@git diff-index --quiet --cached HEAD -- || ( echo "There are staged changes, please take a look, consider committing them, and try again."; exit 1 )
@@ -154,16 +155,28 @@ ghcid:
 
 # Used by CI
 .PHONY: lint-all
-lint-all: treefmt-check check-local-nix-derivations
+lint-all: formatc hlint-check-all check-local-nix-derivations treefmt-check
+
+.PHONY: hlint-check-all
+hlint-check-all:
+	./tools/hlint.sh -f all -m check
 
 .PHONY: hlint-inplace-all
 hlint-inplace-all:
 	./tools/hlint.sh -f all -m inplace
 
+.PHONY: hlint-check-pr
+hlint-check-pr:
+	./tools/hlint.sh -f pr -m check
+
 .PHONY: hlint-inplace-pr
 hlint-inplace-pr:
 	./tools/hlint.sh -f pr -m inplace
 
+.PHONY: hlint-check
+hlint-check:
+	./tools/hlint.sh -f changeset -m check
+
 .PHONY: hlint-inplace
 hlint-inplace:
 	./tools/hlint.sh -f changeset -m inplace
@@ -179,27 +192,35 @@ check-local-nix-derivations: regen-local-nix-derivations
 services: init install
 	$(MAKE) -C services/nginz
 
-# formats everything according to treefmt rules
-# this may take a while (5 minutes) on first run but should be instant on 
-# any subsequent run except after you have changed files.
+# formats all Haskell files (which don't contain CPP)
 .PHONY: format
 format:
-	treefmt
+	./tools/ormolu.sh
+
+# formats all Haskell files changed in this PR, even if local changes are not committed to git
+.PHONY: formatf
+formatf:
+	./tools/ormolu.sh -f pr
 
-# checks the format
+# formats all Haskell files even if local changes are not committed to git
+.PHONY: formatf-all
+formatf-all:
+	./tools/ormolu.sh -f all
+
+# checks that all Haskell files are formatted; fail if a `make format` run is needed.
 .PHONY: formatc
-formatc: 
-	treefmt-check
+formatc:
+	./tools/ormolu.sh -c
 
 # For any Haskell or Rust file, update or add a license header if necessary.
 # Headers should be added according to Ormolu's formatting rules, but please check just in case.
 .PHONY: add-license
 add-license:
-	# Check headroom is installed.
+	# Check headroom is installed. If not, please run 'stack install headroom'
 	command -v headroom
 	headroom run
 	@echo ""
-	@echo "you might want to run 'make format' now to make sure ormolu is happy"
+	@echo "you might want to run 'make formatf' now to make sure ormolu is happy"
 
 .PHONY: treefmt
 treefmt:
@@ -276,13 +297,16 @@ db-reset: c
 	./dist/brig-index reset \
 		--elasticsearch-index-prefix directory \
 		--elasticsearch-server https://localhost:9200 \
+	  --elasticsearch-ca-cert ./services/brig/test/resources/elasticsearch-ca.pem \
 		--elasticsearch-credentials ./services/brig/test/resources/elasticsearch-credentials.yaml > /dev/null
 	./dist/brig-index reset \
 		--elasticsearch-index-prefix directory2 \
 		--elasticsearch-server https://localhost:9200 \
+	  --elasticsearch-ca-cert ./services/brig/test/resources/elasticsearch-ca.pem \
 		--elasticsearch-credentials ./services/brig/test/resources/elasticsearch-credentials.yaml > /dev/null
 	./integration/scripts/integration-dynamic-backends-brig-index.sh \
 		--elasticsearch-server https://localhost:9200 \
+	  --elasticsearch-ca-cert ./services/brig/test/resources/elasticsearch-ca.pem \
 		--elasticsearch-credentials ./services/brig/test/resources/elasticsearch-credentials.yaml > /dev/null
 
 
@@ -544,8 +568,8 @@ helm-template-%: clean-charts charts-integration
 	./hack/bin/helm-template.sh $(*)
 
 # Ask the security team for the `DEPENDENCY_TRACK_API_KEY` (if you need it)
-# changing the directory is necessary because of some quirkiness of how 
-# runhaskell / ghci behaves (it doesn't find modules that aren't in the same 
+# changing the directory is necessary because of some quirkiness of how
+# runhaskell / ghci behaves (it doesn't find modules that aren't in the same
 # directory as the script that is being executed)
 .PHONY: upload-bombon
 upload-bombon:

cabal.project

@@ -48,6 +48,7 @@ packages:
   , tools/db/inconsistencies/
   , tools/db/migrate-sso-feature-flag/
   , tools/db/move-team/
+  , tools/db/phone-users/
   , tools/db/repair-handles/
   , tools/db/repair-brig-clients-table/
   , tools/db/service-backfill/
@@ -127,6 +128,8 @@ package proxy
     ghc-options: -Werror
 package mlsstats
     ghc-options: -Werror
+package phone-users
+    ghc-options: -Werror
 package rabbitmq-consumer
     ghc-options: -Werror
 package repair-handles
@@ -179,6 +182,6 @@ package fedcalls
 -- - these packages have bounds that are justified with their current
 --   dependency set, however, we have updated their dependencies, such
 --   that they work with newer base and ghc (api) versions
-allow-newer: 
+allow-newer:
   , proto-lens-protoc:base
   , proto-lens-protoc:ghc

cassandra-schema.cql

@@ -1720,19 +1720,17 @@ CREATE TABLE galley_test.mls_proposal_refs (
     AND speculative_retry = '99PERCENTILE';
 CREATE KEYSPACE gundeck_test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}  AND durable_writes = true;
 
-CREATE TABLE gundeck_test.push (
-    ptoken text,
-    app text,
-    transport int,
-    client text,
-    connection blob,
-    usr uuid,
-    PRIMARY KEY (ptoken, app, transport)
-) WITH CLUSTERING ORDER BY (app ASC, transport ASC)
-    AND bloom_filter_fp_chance = 0.1
+CREATE TABLE gundeck_test.data_migration (
+    id int,
+    version int,
+    date timestamp,
+    descr text,
+    PRIMARY KEY (id, version)
+) WITH CLUSTERING ORDER BY (version ASC)
+    AND bloom_filter_fp_chance = 0.01
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
     AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
     AND crc_check_chance = 1.0
     AND dclocal_read_repair_chance = 0.1
@@ -1790,10 +1788,16 @@ CREATE TABLE gundeck_test.meta (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE gundeck_test.notification_payload (
-    id uuid PRIMARY KEY,
-    payload blob
-) WITH bloom_filter_fp_chance = 0.1
+CREATE TABLE gundeck_test.push (
+    ptoken text,
+    app text,
+    transport int,
+    client text,
+    connection blob,
+    usr uuid,
+    PRIMARY KEY (ptoken, app, transport)
+) WITH CLUSTERING ORDER BY (app ASC, transport ASC)
+    AND bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
@@ -1832,6 +1836,24 @@ CREATE TABLE gundeck_test.user_push (
     AND min_index_interval = 128
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
+
+CREATE TABLE gundeck_test.notification_payload (
+    id uuid PRIMARY KEY,
+    payload blob
+) WITH bloom_filter_fp_chance = 0.1
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0.1
+    AND default_time_to_live = 0
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE';
 CREATE KEYSPACE spar_test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}  AND durable_writes = true;
 
 CREATE TABLE spar_test.bind_cookie (

charts/cassandra-migrations/templates/galley-migrate-data.yaml

@@ -20,7 +20,6 @@ spec:
     metadata:
       name: "{{.Release.Name}}"
       labels:
-        app: galley-migrate-data
         app: galley-migrate-data
         heritage: {{.Release.Service | quote }}
         release: {{.Release.Name | quote }}

charts/cassandra-migrations/templates/gundeck-migrate-data.yaml

@@ -0,0 +1,59 @@
+# This jobs runs migrations on the gundeck DB using the gundeck-migrate-data tool.
+# The source for the tool can be found at services/gundeck in the wire-server
+# repository.
+{{- if .Values.enableGundeckMigrations }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: gundeck-migrate-data
+  labels:
+    app: "cassandra-migrations"
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "10"
+    "helm.sh/hook-delete-policy": "before-hook-creation"
+spec:
+  template:
+    metadata:
+      name: "{{.Release.Name}}"
+      labels:
+        app: gundeck-migrate-data
+        heritage: {{.Release.Service | quote }}
+        release: {{.Release.Name | quote }}
+        chart: "{{.Chart.Name}}-{{.Chart.Version}}"
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: gundeck-migrate-data
+          image: "{{ .Values.images.gundeckMigrateData }}:{{ .Values.images.tag }}"
+          imagePullPolicy: {{ default "" .Values.imagePullPolicy | quote }}
+        {{- if eq (include "includeSecurityContext" .) "true" }}
+          securityContext:
+            {{- toYaml .Values.podSecurityContext | nindent 12 }}
+        {{- end }}
+          args:
+           - --cassandra-host
+           - "{{ template "cassandraGundeckHost" . }}"
+           - --cassandra-port
+           - "9042"
+           - --cassandra-keyspace
+           - gundeck
+           {{- if eq (include "useTlsGundeck" .) "true" }}
+           - --tls-ca-certificate-file
+           - /certs/gundeck/{{- (include "tlsSecretRefGundeck" . | fromYaml).key }}
+           {{- end }}
+          {{- if eq (include "useTlsGundeck" .) "true" }}
+          volumeMounts:
+            - name: gundeck-cassandra-cert
+              mountPath: "/certs/gundeck"
+          {{- end }}
+      {{- if eq (include "useTlsGundeck" .) "true" }}
+      volumes:
+        - name: gundeck-cassandra-cert
+          secret:
+            secretName: {{ (include "tlsSecretRefGundeck" . | fromYaml).name }}
+      {{- end }}
+{{- end }}

charts/cassandra-migrations/values.yaml

@@ -5,6 +5,7 @@ images:
   galley: quay.io/wire/galley-schema
   spar: quay.io/wire/spar-schema
   galleyMigrateData: quay.io/wire/galley-migrate-data
+  gundeckMigrateData: quay.io/wire/gundeck-migrate-data
   sparMigrateData: quay.io/wire/spar-migrate-data
 
 # Setting cassandra host name and replication is mandatory to specify.