Merge pull request #93 from PermanentOrg/per-9561_add_account_space_u…

…pdate_lambda

Add account_space_update lambda handler

.github/workflows/test.yml

@@ -30,8 +30,10 @@ jobs:
                   ref: main
                   path: ./devenv
             - run: (cd stela; npm install --production=false)
+            - run : (cd stela; npm run build -ws)
             - run: touch stela/.env
             - run: (cd devenv; docker compose up database_setup -d; docker logs devenv-database_setup-1)
             - run: (cd stela/packages/api; npm run start-containers)
             - run: (cd stela/packages/api; docker compose run stela npm run test-ci)
+            - run: (cd stela; npm run test -w @stela/account_space_updater)
             - uses: codecov/codecov-action@v2

Dockerfile.account_space_updater

@@ -0,0 +1,29 @@
+FROM public.ecr.aws/lambda/nodejs:18 as builder
+WORKDIR /usr/local/apps/stela/
+
+COPY package.json ./
+COPY tsconfig.build.json ./
+COPY tsconfig.json ./
+COPY jest.config.js ./
+COPY packages ./packages
+
+RUN npm install -g npm@8.19.3
+RUN npm install
+RUN npm install -ws
+RUN npm run build -ws
+
+
+FROM public.ecr.aws/lambda/nodejs:18 as final
+WORKDIR ${LAMBDA_TASK_ROOT}
+
+COPY --from=builder /usr/local/apps/stela/packages/account_space_updater/dist ./packages/account_space_updater/dist
+COPY --from=builder /usr/local/apps/stela/packages/account_space_updater/package.json ./packages/account_space_updater/package.json
+COPY --from=builder /usr/local/apps/stela/packages/logger/dist ./packages/logger/dist
+COPY --from=builder /usr/local/apps/stela/packages/logger/package.json ./packages/logger/package.json
+COPY --from=builder /usr/local/apps/stela/package.json ./package.json
+COPY --from=builder /usr/local/apps/stela/package-lock.json ./package-lock.json
+
+RUN npm install -g npm@8.19.3
+RUN npm install --workspace @stela/account_space_updater
+
+CMD ["packages/account_space_updater/dist/index.handler"]

package.json

@@ -2,7 +2,12 @@
   "name": "stela",
   "version": "0.0.0",
   "description": "A monolithic backend for Permanent.org",
-  "workspaces": ["./packages/logger", "./packages/api", "./packages/archivematica_cleanup"],
+  "workspaces": [
+    "./packages/logger",
+    "./packages/api",
+    "./packages/archivematica_cleanup",
+    "./packages/account_space_updater"
+  ],
   "repository": {
     "type": "git",
     "url": "git+https://github.com/PermanentOrg/stela.git"

packages/account_space_updater/.sqlfluff

@@ -0,0 +1,10 @@
+[sqlfluff]
+dialect = postgres
+ignore = templating
+templater = placeholder
+
+[sqlfluff:indentation]
+tab_space_size = 2
+
+[sqlfluff:templater:placeholder]
+param_style = colon

packages/account_space_updater/jest.config.js

@@ -0,0 +1,7 @@
+sharedConfig = require("../../jest.config");
+module.exports = {
+  ...sharedConfig,
+  moduleNameMapper: {
+    "^@stela/(.*)$": "<rootDir>/../$1/src",
+  },
+};

packages/account_space_updater/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@stela/account_space_updater",
+  "version": "1.0.0",
+  "description": "A lambda to update a Permanent account's space usage",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/PermanentOrg/stela.git"
+  },
+  "author": "Permanent.org",
+  "license": "AGPL-3.0",
+  "bugs": {
+    "url": "https://github.com/PermanentOrg/stela/issues"
+  },
+  "homepage": "https://permanent.org",
+  "engines": {
+    "node": ">=18.0"
+  },
+  "main": "dist/index.js",
+  "scripts": {
+    "check-types": "tsc --noEmit",
+    "check-format": "prettier --check src",
+    "eslint": "eslint --max-warnings 0 ./src --ext .ts",
+    "lint-sql": "docker run -i --rm -v $PWD:/sql sqlfluff/sqlfluff:2.3.5 lint --dialect postgres --nocolor src/queries/",
+    "lint": "npm run check-format && npm run check-types && npm run eslint && npm run lint-sql",
+    "build": "rm -rf dist/* && tsc -p tsconfig.build.json && tscp -p tsconfig.tscp.json",
+    "start": "node dist/index.js",
+    "clear-database": "psql postgresql://postgres:permanent@localhost:5432 -c 'DROP DATABASE IF EXISTS test_permanent;'",
+    "create-database": "psql postgresql://postgres:permanent@localhost:5432 -c 'CREATE DATABASE test_permanent;'",
+    "set-up-database": "docker exec --env='PGPASSWORD=permanent' devenv-database-1 pg_dump -U postgres permanent --schema-only | psql postgresql://postgres:permanent@localhost:5432/test_permanent",
+    "test": "npm run clear-database && npm run create-database && npm run set-up-database && node --experimental-vm-modules ../../node_modules/jest/bin/jest.js -i --silent=false"
+  },
+  "dependencies": {
+    "@sentry/node": "^7.57.0",
+    "@stela/logger": "^1.0.0",
+    "ajv": "^8.16.0",
+    "dotenv": "^8.2.0",
+    "require-env-variable": "^3.1.2",
+    "tinypg": "^7.0.0"
+  },
+  "devDependencies": {
+    "@types/aws-lambda": "^8.10.119"
+  }
+}

packages/account_space_updater/src/__mocks__/database.ts

@@ -0,0 +1,11 @@
+import * as path from "path";
+import { TinyPg } from "tinypg";
+
+const db = new TinyPg({
+  connection_string:
+    "postgres://postgres:permanent@localhost:5432/test_permanent",
+  root_dir: [path.resolve(__dirname, "..")],
+  capture_stack_trace: true,
+});
+
+export { db };

packages/account_space_updater/src/database.ts

@@ -0,0 +1,7 @@
+import * as path from "path";
+import { TinyPg } from "tinypg";
+
+export const db = new TinyPg({
+  connection_string: process.env["DATABASE_URL"] ?? "",
+  root_dir: [path.resolve(__dirname, ".")],
+});

packages/account_space_updater/src/env.ts

@@ -0,0 +1,8 @@
+import { config } from "dotenv";
+import { requireEnv } from "require-env-variable";
+
+config({ path: "../../.env" });
+
+requireEnv("DATABASE_URL");
+requireEnv("SENTRY_DSN");
+requireEnv("ENV");

packages/account_space_updater/src/fixtures/create_test_account_space.sql

@@ -0,0 +1,24 @@
+INSERT INTO
+account_space (
+  accountid,
+  spaceleft,
+  spacetotal,
+  fileleft,
+  filetotal,
+  status,
+  type,
+  createddt,
+  updateddt
+)
+VALUES
+(
+  2,
+  2147483648,
+  2147483648,
+  100000,
+  100000,
+  'status.generic.ok',
+  'type.generic.placeholder',
+  CURRENT_TIMESTAMP,
+  CURRENT_TIMESTAMP
+);

packages/account_space_updater/src/fixtures/create_test_accounts.sql

@@ -0,0 +1,20 @@
+INSERT INTO
+account (
+  accountid,
+  primaryemail,
+  status,
+  notificationpreferences,
+  type,
+  fullname,
+  subject
+)
+VALUES
+(
+  2,
+  'test@permanent.org',
+  'status.auth.ok',
+  '{}',
+  'type.account.standard',
+  'Jack Rando',
+  null
+);

packages/account_space_updater/src/fixtures/create_test_archives.sql

@@ -0,0 +1,4 @@
+INSERT INTO
+archive (archiveid, archivenbr, payeraccountid, public, type, thumburl200)
+VALUES
+(1, '0001-0001', 2, false, 'type.archive.person', null);

packages/account_space_updater/src/fixtures/create_test_files.sql

@@ -0,0 +1,16 @@
+INSERT INTO
+file (
+  fileid,
+  archiveid,
+  status,
+  createddt,
+  updateddt
+)
+VALUES
+(
+  1,
+  1,
+  'status.generic.ok',
+  CURRENT_TIMESTAMP,
+  CURRENT_TIMESTAMP
+);

packages/account_space_updater/src/fixtures/create_test_record_files.sql

@@ -0,0 +1,18 @@
+INSERT INTO
+record_file (
+  recordid,
+  fileid,
+  status,
+  type,
+  createddt,
+  updateddt
+)
+VALUES
+(
+  1,
+  1,
+  'status.generic.ok',
+  'type.generic.placeholder',
+  CURRENT_TIMESTAMP,
+  CURRENT_TIMESTAMP
+);

packages/account_space_updater/src/fixtures/create_test_records.sql

@@ -0,0 +1,26 @@
+INSERT INTO
+record (
+  recordid,
+  archiveid,
+  publicdt,
+  displayname,
+  uploadaccountid,
+  size,
+  uploadpayeraccountid,
+  uploadfilename,
+  status,
+  type
+)
+VALUES
+(
+  1,
+  1,
+  '2023-06-21',
+  'Public File',
+  2,
+  1024,
+  2,
+  'public_file.jpg',
+  'status.generic.ok',
+  'type.record.image'
+);