feat: s3 external storage, biome linting

.gitignore

@@ -42,5 +42,4 @@ dist
 
 packages/web/.next
 packages/api/data
-packages/*/.turbo
-packages/*/.eslintcache
+packages/*/.turbo

.vscode/settings.json

@@ -2,11 +2,6 @@
   "typescript.tsdk": "node_modules/typescript/lib",
   "typescript.preferences.importModuleSpecifier": "relative",
   "npm.scriptExplorerExclude": ["^((?!watch|generate:watch).)*$"],
-  "eslint.workingDirectories": [
-    {
-      "pattern": "./packages/*"
-    }
-  ],
   "files.associations": {
     ".microrc": "jsonc",
     ".microrc.example": "jsonc",
@@ -25,5 +20,13 @@
     // const class = 'value'
     // const selectedClass = 'value'
     ["const [a-zA-Z]+s = ['\"`]([^\"`'`]*)"]
-  ]
+  ],
+
+  /// BIOME
+  "editor.defaultFormatter": "biomejs.biome",
+  "editor.formatOnSave": true,
+  "editor.codeActionsOnSave": {
+    "quickfix.biome": "explicit",
+    "source.organizeImports.biome": "never"
+  }
 }

biome.json

@@ -0,0 +1,6 @@
+{
+    "$schema": "https://biomejs.dev/schemas/1.6.1/schema.json",
+    "extends": [
+        "@atlasbot/configs/biome"
+    ]
+}

compose.yml

@@ -13,3 +13,15 @@ services:
       - POSTGRES_USER=micro
       - POSTGRES_PASSWORD=youshallnotpass
       - POSTGRES_DB=micro
+
+  minio:
+    image: minio/minio
+    container_name: micro_minio
+    restart: unless-stopped
+    command: server /data --console-address ":9001"
+    ports:
+      - 127.0.0.1:9000:9000 
+      - 127.0.0.1:9001:9001
+    environment:
+      MINIO_ROOT_USER: test
+      MINIO_ROOT_PASSWORD: testtest

package.json

@@ -7,7 +7,7 @@
   "private": true,
   "engines": {
     "node": ">=16",
-    "pnpm": ">=7"
+    "pnpm": ">=9"
   },
   "scripts": {
     "build": "turbo run build",
@@ -17,9 +17,10 @@
     "test": "turbo run test"
   },
   "devDependencies": {
-    "syncpack": "^12.3.0",
-    "turbo": "1.12.3",
-    "typescript": "^5.3.3"
-  },
-  "packageManager": "pnpm@7.0.0"
-}
+    "@atlasbot/configs": "^10.6.1",
+    "@biomejs/biome": "^1.7.3",
+    "syncpack": "^12.3.2",
+    "turbo": "1.13.3",
+    "typescript": "^5.4.5"
+  }
+}

packages/api/package.json

@@ -11,87 +11,83 @@
   },
   "scripts": {
     "build": "tsc --noEmit && tsup",
-    "lint": "eslint src --fix --cache",
     "start": "node dist/main.js",
     "test": "vitest run",
-    "watch": "tsup --watch --onSuccess \"node dist/main.js\""
+    "watch": "tsup --watch --onSuccess \"node dist/main.js\"",
+    "mikro-orm": "tsup --no-dts --silent && MIKRO_ORM_MIGRATIONS_PATH=src/migrations mikro-orm"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.577.0",
     "@fastify/cookie": "^9.2.0",
     "@fastify/helmet": "^11.1.1",
-    "@fastify/multipart": "^8.1.0",
+    "@fastify/multipart": "^8.2.0",
     "@jenyus-org/graphql-utils": "^1.5.0",
-    "@mercuriusjs/gateway": "^2.2.0",
-    "@mikro-orm/core": "^5.9.7",
-    "@mikro-orm/migrations": "^5.9.7",
+    "@mercuriusjs/gateway": "^3.0.1",
+    "@mikro-orm/core": "^6.2.5",
+    "@mikro-orm/migrations": "^6.2.5",
     "@mikro-orm/nestjs": "^5.2.3",
-    "@mikro-orm/postgresql": "^5.9.7",
-    "@nestjs/common": "^10.3.2",
-    "@nestjs/core": "^10.3.2",
+    "@mikro-orm/postgresql": "^6.2.5",
+    "@nestjs/common": "^10.3.8",
+    "@nestjs/core": "^10.3.8",
     "@nestjs/graphql": "^12.1.1",
     "@nestjs/jwt": "^10.2.0",
     "@nestjs/mercurius": "^12.1.1",
     "@nestjs/passport": "^10.0.3",
-    "@nestjs/platform-fastify": "^10.3.2",
-    "@nestjs/schedule": "^4.0.1",
+    "@nestjs/platform-fastify": "^10.3.8",
+    "@nestjs/schedule": "^4.0.2",
     "@ryanke/venera": "^1.0.5",
     "bcryptjs": "^2.4.3",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.0",
-    "fastify": "^4.26.0",
+    "fastify": "^4.27.0",
     "fluent-ffmpeg": "^2.1.2",
     "graphql": "^16.8.1",
-    "mercurius": "^13.3.3",
+    "mercurius": "^14.1.0",
     "mime-types": "^2.1.35",
-    "nodemailer": "^6.9.9",
+    "nodemailer": "^6.9.13",
     "otplib": "^12.0.1",
     "passport": "^0.7.0",
     "passport-jwt": "^4.0.1",
     "rxjs": "^7.8.1",
-    "sharp": "^0.33.1",
-    "stream-size": "^0.0.6",
+    "sharp": "^0.33.3",
     "utf-8-validate": "^6.0.3"
   },
   "devDependencies": {
-    "@atlasbot/configs": "^10.5.15",
-    "@mikro-orm/cli": "^5.9.7",
-    "@swc/core": "^1.4.0",
+    "@atlasbot/configs": "^10.6.1",
+    "@mikro-orm/cli": "^6.2.5",
+    "@swc/core": "^1.5.3",
     "@types/bcryptjs": "^2.4.6",
     "@types/bytes": "^3.1.4",
     "@types/dedent": "^0.7.2",
     "@types/fluent-ffmpeg": "^2.1.24",
     "@types/luxon": "^3.4.0",
     "@types/mime-types": "^2.1.4",
     "@types/ms": "^0.7.34",
-    "@types/node": "^20.11.17",
-    "@types/nodemailer": "^6.4.14",
+    "@types/node": "^20.12.10",
+    "@types/nodemailer": "^6.4.15",
     "@types/passport-jwt": "^4.0.0",
     "@types/utf-8-validate": "^5.0.2",
     "bytes": "^3.1.2",
     "chalk": "^5.3.0",
     "content-range": "^2.0.2",
-    "dedent": "^1.5.1",
+    "dedent": "^1.5.3",
     "escape-string-regexp": "^5.0.0",
     "file-type": "^19.0.0",
     "handlebars": "^4.7.8",
     "istextorbinary": "^9.5.0",
     "luxon": "^3.4.4",
     "ms": "^3.0.0-canary.1",
-    "nanoid": "^5.0.5",
-    "normalize-url": "^8.0.0",
+    "nanoid": "^5.0.7",
+    "normalize-url": "^8.0.1",
     "pretty-bytes": "^6.1.1",
-    "reflect-metadata": "^0.2.1",
-    "ts-node": "^10.9.2",
+    "reflect-metadata": "^0.2.2",
     "tsup": "^8.0.2",
-    "typescript": "^5.3.3",
-    "vitest": "^1.2.2",
-    "zod": "^3.22.4",
-    "zod-validation-error": "^3.0.0"
+    "typescript": "^5.4.5",
+    "vitest": "^1.6.0",
+    "zod": "^3.23.7",
+    "zod-validation-error": "^3.2.0"
   },
   "mikro-orm": {
-    "useTsNode": true,
-    "configPaths": [
-      "./src/orm.config.ts"
-    ]
+    "configPaths": ["./dist/orm.config.js"]
   }
 }

packages/api/src/config.ts

@@ -1,19 +1,19 @@
-import { loadConfig } from '@ryanke/venera';
-import c from 'chalk';
-import { randomBytes } from 'crypto';
-import dedent from 'dedent';
-import escapeStringRegexp from 'escape-string-regexp';
-import ms from 'ms';
-import z, { any, array, boolean, number, record, strictObject, string, union } from 'zod';
-import { fromZodError } from 'zod-validation-error';
-import { expandMime } from './helpers/expand-mime.js';
-import { HostService } from './modules/host/host.service.js';
-import { parseBytes } from './helpers/parse-bytes.js';
+import { loadConfig } from "@ryanke/venera";
+import c from "chalk";
+import { randomBytes } from "crypto";
+import dedent from "dedent";
+import escapeStringRegexp from "escape-string-regexp";
+import ms from "ms";
+import z, { any, array, boolean, literal, number, record, strictObject, string, union } from "zod";
+import { fromZodError } from "zod-validation-error";
+import { expandMime } from "./helpers/expand-mime.js";
+import { HostService } from "./modules/host/host.service.js";
+import { parseBytes } from "./helpers/parse-bytes.js";
 
 export type MicroHost = ReturnType<typeof enhanceHost>;
 
 const schema = strictObject({
-  databaseUrl: string().startsWith('postgresql://'),
+  databaseUrl: string().startsWith("postgresql://"),
   secret: string().min(6),
   inquiries: string().email(),
   uploadLimit: string().transform(parseBytes),
@@ -23,6 +23,23 @@ const schema = strictObject({
     .optional()
     .transform((value) => (value ? new Set(expandMime(value)) : null)),
   storagePath: string(),
+  externalStorage: strictObject({
+    type: literal("s3"),
+
+    bucket: string(),
+    region: string(),
+    endpoint: string().url().optional(),
+    forcePathStyle: boolean().optional(),
+    credentials: strictObject({
+      accessKeyId: string(),
+      secretAccessKey: string(),
+    }),
+    filter: strictObject({
+      decayDuration: string().transform(ms), // how long before local files are uploaded
+      maxSize: string().transform(parseBytes).optional(),
+      minSize: string().transform(parseBytes).optional(),
+    }),
+  }).optional(),
   restrictFilesToHost: boolean().default(true),
   purge: strictObject({
     overLimit: string().transform(parseBytes),
@@ -44,14 +61,14 @@ const schema = strictObject({
       url: z
         .string()
         .url()
-        .transform((value) => value.replace(/\/$/, '')),
+        .transform((value) => value.replace(/\/$/, "")),
       tags: array(string()).optional(),
       redirect: string().url().optional(),
     }),
   ),
 });
 
-const data = loadConfig('micro');
+const data = loadConfig("micro");
 const result = schema.safeParse(data);
 if (!result.success) {
   console.dir({ data, error: result.error }, { depth: null });
@@ -62,12 +79,12 @@ if (!result.success) {
 const getWildcardPattern = (url: string) => {
   const normalised = HostService.normaliseHostUrl(url);
   const escaped = escapeStringRegexp(normalised);
-  const pattern = escaped.replace('\\{\\{username\\}\\}', '(?<username>[a-z0-9-{}]+?)');
-  return new RegExp(`^(https?:\\/\\/)?${pattern}\\/?`, 'u');
+  const pattern = escaped.replace("\\{\\{username\\}\\}", "(?<username>[a-z0-9-{}]+?)");
+  return new RegExp(`^(https?:\\/\\/)?${pattern}\\/?`, "u");
 };
 
-const enhanceHost = (host: z.infer<typeof schema>['hosts'][0]) => {
-  const isWildcard = host.url.includes('{{username}}');
+const enhanceHost = (host: z.infer<typeof schema>["hosts"][0]) => {
+  const isWildcard = host.url.includes("{{username}}");
   const normalised = HostService.normaliseHostUrl(host.url);
   const pattern = getWildcardPattern(host.url);
 
@@ -79,20 +96,20 @@ const enhanceHost = (host: z.infer<typeof schema>['hosts'][0]) => {
   };
 };
 
-export const config = result.data as Omit<z.infer<typeof schema>, 'hosts'>;
+export const config = result.data as Omit<z.infer<typeof schema>, "hosts">;
 export const hosts = result.data.hosts.map((host) => enhanceHost(host));
 export const rootHost = hosts[0];
 
 if (rootHost.isWildcard) {
-  throw new Error(`Root host cannot be a wildcard domain.`);
+  throw new Error("Root host cannot be a wildcard domain.");
 }
 
-const disallowed = new Set(['youshallnotpass', 'you_shall_not_pass', 'secret', 'test']);
+const disallowed = new Set(["youshallnotpass", "you_shall_not_pass", "secret", "test"]);
 if (disallowed.has(config.secret.toLowerCase())) {
-  const token = randomBytes(24).toString('hex');
+  const token = randomBytes(24).toString("hex");
   throw new Error(
     dedent`
-      ${c.redBright.bold('Do not use the default secret.')}
+      ${c.redBright.bold("Do not use the default secret.")}
       Please generate a random, secure secret or you risk anyone being able to impersonate you.
       If you're lazy, here is a random secret: ${c.underline(token)}
     `,

packages/api/src/helpers/get-stream-type.helper.ts

@@ -1,16 +1,16 @@
-import { fileTypeFromBuffer } from 'file-type';
-import * as mimeType from 'mime-types';
-import path from 'path';
-import type { PassThrough } from 'stream';
-import { isBinary } from 'istextorbinary';
+import { fileTypeFromBuffer } from "file-type";
+import * as mimeType from "mime-types";
+import path from "path";
+import type { PassThrough } from "stream";
+import { isBinary } from "istextorbinary";
 
-const DEFAULT_TYPE = 'application/octet-stream';
+const DEFAULT_TYPE = "application/octet-stream";
 // is-binary scans the first 1kb
 // file-type scans the first 4.2kb
 const SCAN_BYTE_COUNT = 4200;
 // overrides for types that are poorly mapped by sharex
-const EXT_TEXT_MAP = new Set(['ts', 'tsx', 'jsx', 'ejs', 'cjs', 'mjs']);
-const EXT_TEXT_TYPE = 'text/plain';
+const EXT_TEXT_MAP = new Set(["ts", "tsx", "jsx", "ejs", "cjs", "mjs"]);
+const EXT_TEXT_TYPE = "text/plain";
 
 async function readFirstBytes(stream: PassThrough) {
   let count = 0;