From 5620d0bed3c1714d15d1d45df8e54c3df31e11db Mon Sep 17 00:00:00 2001 From: xwashere Date: Wed, 25 Oct 2023 14:27:52 -0400 Subject: [PATCH 01/16] add: database class interface --- backend/app.ts | 5 +- backend/data.ts | 16 +++--- backend/db/database.ts | 105 +++++++++++++++++++++++++++++++++++ backend/{sql => db}/mysql.ts | 55 +++++++++--------- backend/routers/api.ts | 4 +- 5 files changed, 146 insertions(+), 39 deletions(-) create mode 100644 backend/db/database.ts rename backend/{sql => db}/mysql.ts (69%) diff --git a/backend/app.ts b/backend/app.ts index a0b2cc20..f213acca 100644 --- a/backend/app.ts +++ b/backend/app.ts @@ -11,8 +11,9 @@ import { epcss } from '@tycrek/express-postcss'; import { log } from './log'; import { ensureFiles, get } from './data'; import { UserConfig } from './UserConfig'; -import { MySql } from './sql/mysql'; +import { MySQLDatabase } from './db/mysql'; import { buildFrontendRouter } from './routers/_frontend'; +import { DBManager } from './db/database'; /** * Top-level metadata exports @@ -113,7 +114,7 @@ async function main() { // If user config is ready, try to configure SQL if (UserConfig.ready && UserConfig.config.sql?.mySql != null) - try { await MySql.configure(); } + try { await DBManager.use(new MySQLDatabase()); } catch (err) { throw new Error(`Failed to configure SQL`); } // Set up Express diff --git a/backend/data.ts b/backend/data.ts index 85ae6450..99c44d0b 100644 --- a/backend/data.ts +++ b/backend/data.ts @@ -6,7 +6,7 @@ import { path } from '@tycrek/joint'; import { log } from './log'; import { nanoid } from './generators'; import { UserConfig } from './UserConfig'; -import { MySql } from './sql/mysql'; +import { DBManager } from './db/database'; /** * Switcher type for exported functions @@ -107,7 +107,7 @@ export const setDataModeToSql = (): Promise => new Promise(async (resolve, export const put = (sector: DataSector, key: NID, data: AssFile | AssUser): Promise => new Promise(async (resolve, reject) => { try { - const useSql = MySql.ready; + const useSql = DBManager.ready; if (sector === 'files') { @@ -134,7 +134,7 @@ export const put = (sector: DataSector, key: NID, data: AssFile | AssUser): Prom } else { // ? SQL - if (!(await MySql.get('assfiles', key))) await MySql.put('assfiles', key, data); + if (!(await DBManager.get('assfiles', key))) await DBManager.put('assfiles', key, data); else return reject(new Error(`File key ${key} already exists`)); // todo: modify users SQL files property @@ -158,7 +158,7 @@ export const put = (sector: DataSector, key: NID, data: AssFile | AssUser): Prom } else { // ? SQL - if (!(await MySql.get('assusers', key))) await MySql.put('assusers', key, data); + if (!(await DBManager.get('assusers', key))) await DBManager.put('assusers', key, data); else return reject(new Error(`User key ${key} already exists`)); } } @@ -172,8 +172,8 @@ export const put = (sector: DataSector, key: NID, data: AssFile | AssUser): Prom export const get = (sector: DataSector, key: NID): Promise => new Promise(async (resolve, reject) => { try { - const data: AssFile | AssUser | undefined = (MySql.ready) - ? (await MySql.get(sector === 'files' ? 'assfiles' : 'assusers', key) as AssFile | AssUser | undefined) + const data: AssFile | AssUser | undefined = (DBManager.ready) + ? (await DBManager.get(sector === 'files' ? 'assfiles' : 'assusers', key) as AssFile | AssUser | undefined) : (await fs.readJson(PATHS[sector]))[sector][key]; (!data) ? resolve(false) : resolve(data); } catch (err) { @@ -183,9 +183,9 @@ export const get = (sector: DataSector, key: NID): Promise => new Promise(async (resolve, reject) => { try { - const data: { [key: string]: AssFile | AssUser } | undefined = (MySql.ready) + const data: { [key: string]: AssFile | AssUser } | undefined = (DBManager.ready) // todo: fix MySQL - ? (await MySql.getAll(sector === 'files' ? 'assfiles' : 'assusers') as /* AssFile[] | AssUser[] | */ undefined) + ? (await DBManager.getAll(sector === 'files' ? 'assfiles' : 'assusers') as /* AssFile[] | AssUser[] | */ []) : (await fs.readJson(PATHS[sector]))[sector]; (!data) ? resolve(false) : resolve(data); } catch (err) { diff --git a/backend/db/database.ts b/backend/db/database.ts new file mode 100644 index 00000000..a2bb8cb5 --- /dev/null +++ b/backend/db/database.ts @@ -0,0 +1,105 @@ +import { AssFile, AssUser, NID, UploadToken } from "ass"; + +export type DatabaseValue = AssFile | AssUser | UploadToken; +export type DatabaseTable = 'assfiles' | 'assusers' | 'asstokens'; + +/** + * interface for database classes + */ +export interface Database { + /** + * preform database initialization tasks + */ + open(): Promise; + + /** + * preform database suspension tasks + */ + close(): Promise; + + /** + * set up database + */ + configure(): Promise; + + /** + * put a value in the database + */ + put(table: DatabaseTable, key: NID, data: DatabaseValue): Promise; + + /** + * get a value from the database + */ + get(table: DatabaseTable, key: NID): Promise; + + /** + * get all values from the database + */ + getAll(table: DatabaseTable): Promise; +} + +export class DBManager { + private static _db: Database; + private static _dbReady: boolean = false; + public static get ready() { + return this._dbReady; + } + + static { + process.on('exit', () => { + if (DBManager._db) DBManager._db.close(); + }); + } + + /** + * activate a database + */ + public static use(db: Database): Promise { + return new Promise(async (resolve, reject) => { + if (this._db != undefined) { + await this._db.close(); + this._dbReady = false; + } + + this._db = db; + await this._db.open(); + await this._db.configure(); + + this._dbReady = true; + resolve(); + }); + } + + public static configure(): Promise { + if (this._db && this._dbReady) { + return this._db.configure(); + } else throw new Error("No database active"); + } + + /** + * put a value in the database + */ + public static put(table: DatabaseTable, key: NID, data: DatabaseValue): Promise { + if (this._db && this._dbReady) { + return this._db.put(table, key, data); + } else throw new Error("No database active"); + } + + /** + * get a value from the database + */ + public static get(table: DatabaseTable, key: NID): Promise { + if (this._db && this._dbReady) { + return this._db.get(table, key); + } else throw new Error("No database active"); + } + + /** + * get all values from the database + */ + public static getAll(table: DatabaseTable): Promise { + if (this._db && this._dbReady) { + return this._db.getAll(table); + } else throw new Error("No database active"); + } +} \ No newline at end of file diff --git a/backend/sql/mysql.ts b/backend/db/mysql.ts similarity index 69% rename from backend/sql/mysql.ts rename to backend/db/mysql.ts index 5d25a5a9..26e694ea 100644 --- a/backend/sql/mysql.ts +++ b/backend/db/mysql.ts @@ -4,21 +4,20 @@ import mysql, { Pool } from 'mysql2/promise'; import { log } from '../log'; import { UserConfig } from '../UserConfig'; +import { Database, DatabaseTable, DatabaseValue } from './database'; -type TableNamesType = 'assfiles' | 'assusers' | 'asstokens'; +export class MySQLDatabase implements Database { + private _pool: Pool; -export class MySql { - private static _pool: Pool; - - private static _ready: boolean = false; - public static get ready() { return MySql._ready; } + private _ready: boolean = false; + public get ready() { return this._ready; } /** * Quick function for creating a simple JSON table */ - private static _tableManager(mode: 'create' | 'drop', name: string, schema = '( NanoID varchar(255), Data JSON )'): Promise { + private _tableManager(mode: 'create' | 'drop', name: string, schema = '( NanoID varchar(255), Data JSON )'): Promise { return new Promise((resolve, reject) => - MySql._pool.query( + this._pool.query( mode === 'create' ? `CREATE TABLE ${name} ${schema};` : `DROP TABLE ${name};`) @@ -26,31 +25,33 @@ export class MySql { .catch((err) => reject(err))); } + public open() { return Promise.resolve(); } + public close() { return Promise.resolve(); } + /** * Build the MySQL client and create the tables */ - public static configure(): Promise { + public configure(): Promise { return new Promise(async (resolve, reject) => { try { - // Config check if (!UserConfig.ready) throw new Error('User configuration not ready'); if (!UserConfig.config.sql?.mySql) throw new Error('MySQL configuration missing'); // Create the pool - MySql._pool = mysql.createPool(UserConfig.config.sql.mySql); + this._pool = mysql.createPool(UserConfig.config.sql.mySql); // Check if the pool is usable - const [rowz, _fields] = await MySql._pool.query(`SHOW FULL TABLES WHERE Table_Type LIKE 'BASE TABLE';`); + const [rowz, _fields] = await this._pool.query(`SHOW FULL TABLES WHERE Table_Type LIKE 'BASE TABLE';`); const rows_tableData = rowz as unknown as { [key: string]: string }[]; // Create tables if needed if (rows_tableData.length === 0) { log.warn('MySQL', 'Tables do not exist, creating'); await Promise.all([ - MySql._tableManager('create', 'assfiles'), - MySql._tableManager('create', 'assusers'), - MySql._tableManager('create', 'asstokens') + this._tableManager('create', 'assfiles'), + this._tableManager('create', 'assusers'), + this._tableManager('create', 'asstokens') ]); log.success('MySQL', 'Tables created').callback(resolve); } else { @@ -61,7 +62,7 @@ export class MySql { // Check which tables ACTUALLY do exist for (let row of rows_tableData) { const table = row[`Tables_in_${UserConfig.config.sql!.mySql!.database}` - ] as TableNamesType; + ] as DatabaseTable; if (table === 'assfiles') tablesExist.files = true; if (table === 'assusers') tablesExist.users = true; if (table === 'asstokens') tablesExist.tokens = true; @@ -69,9 +70,9 @@ export class MySql { } // Mini-function for creating a one-off table - const createOneTable = async (name: TableNamesType) => { + const createOneTable = async (name: DatabaseTable) => { log.warn('MySQL', `Table '${name}' missing, creating`); - await MySql._tableManager('create', name); + await this._tableManager('create', name); log.success('MySQL', `Table '${name}' created`); } @@ -88,7 +89,7 @@ export class MySql { // Hopefully we are ready if (tablesExist.files && tablesExist.users) log.info('MySQL', 'Tables exist, ready').callback(() => { - MySql._ready = true; + this._ready = true; resolve(void 0); }); else throw new Error('Table(s) missing!'); @@ -101,26 +102,26 @@ export class MySql { }); } - public static put(table: TableNamesType, key: NID, data: UploadToken | AssFile | AssUser): Promise { + public put(table: DatabaseTable, key: NID, data: DatabaseValue): Promise { return new Promise(async (resolve, reject) => { - if (!MySql._ready) return reject(new Error('MySQL not ready')); + if (!this._ready) return reject(new Error('MySQL not ready')); const query = ` INSERT INTO ${table} ( NanoID, Data ) VALUES ('${key}', '${JSON.stringify(data)}'); `; - return MySql._pool.query(query) + return this._pool.query(query) .then(() => resolve(void 0)) .catch((err) => reject(err)); }); } - public static get(table: TableNamesType, key: NID): Promise { + public get(table: DatabaseTable, key: NID): Promise { return new Promise(async (resolve, reject) => { try { // Run query - const [rowz, _fields] = await MySql._pool.query(`SELECT Data FROM ${table} WHERE NanoID = '${key}';`); + const [rowz, _fields] = await this._pool.query(`SELECT Data FROM ${table} WHERE NanoID = '${key}';`); // Disgustingly interpret the query results const rows_tableData = (rowz as unknown as { [key: string]: string }[])[0] as unknown as ({ Data: UploadToken | AssFile | AssUser | undefined }); @@ -133,11 +134,11 @@ VALUES ('${key}', '${JSON.stringify(data)}'); } // todo: unknown if this works - public static getAll(table: TableNamesType): Promise { + public getAll(table: DatabaseTable): Promise { return new Promise(async (resolve, reject) => { try { // Run query // ! this may not work as expected - const [rowz, _fields] = await MySql._pool.query(`SELECT Data FROM ${table}`); + const [rowz, _fields] = await this._pool.query(`SELECT Data FROM ${table}`); // Interpret results this is pain const rows = (rowz as unknown as { [key: string]: string }[]); @@ -145,7 +146,7 @@ VALUES ('${key}', '${JSON.stringify(data)}'); // console.log(rows); // aaaaaaaaaaaa - resolve(undefined); + resolve([]); } catch (err) { reject(err); } diff --git a/backend/routers/api.ts b/backend/routers/api.ts index 9dd7bf04..21252343 100644 --- a/backend/routers/api.ts +++ b/backend/routers/api.ts @@ -7,8 +7,8 @@ import * as data from '../data'; import { log } from '../log'; import { nanoid } from '../generators'; import { UserConfig } from '../UserConfig'; -import { MySql } from '../sql/mysql'; import { rateLimiterMiddleware } from '../ratelimit'; +import { DBManager } from '../db/database'; const router = Router({ caseSensitive: true }); @@ -28,7 +28,7 @@ router.post('/setup', BodyParserJson(), async (req, res) => { // Set data storage (not files) to SQL if required if (UserConfig.config.sql?.mySql != null) - await Promise.all([MySql.configure(), data.setDataModeToSql()]); + await Promise.all([DBManager.configure(), data.setDataModeToSql()]); log.success('Setup', 'completed'); From ce3ad10281288ce68e540a27a39951587df92d32 Mon Sep 17 00:00:00 2001 From: xwashere Date: Wed, 25 Oct 2023 15:20:56 -0400 Subject: [PATCH 02/16] split json database out of data.ts --- backend/app.ts | 8 ++- backend/data.ts | 120 +++----------------------------- backend/db/database.ts | 4 +- backend/db/json.ts | 154 +++++++++++++++++++++++++++++++++++++++++ backend/db/mysql.ts | 6 +- 5 files changed, 176 insertions(+), 116 deletions(-) create mode 100644 backend/db/json.ts diff --git a/backend/app.ts b/backend/app.ts index f213acca..a12c7778 100644 --- a/backend/app.ts +++ b/backend/app.ts @@ -9,7 +9,8 @@ import { path, isProd } from '@tycrek/joint'; import { epcss } from '@tycrek/express-postcss'; import { log } from './log'; -import { ensureFiles, get } from './data'; +import { JSONDatabase, ensureFiles } from './db/json'; +import { get } from './data'; import { UserConfig } from './UserConfig'; import { MySQLDatabase } from './db/mysql'; import { buildFrontendRouter } from './routers/_frontend'; @@ -113,9 +114,12 @@ async function main() { .catch((err) => (err.code && err.code === 'ENOENT' ? {} : console.error(err), resolve(void 0)))); // If user config is ready, try to configure SQL - if (UserConfig.ready && UserConfig.config.sql?.mySql != null) + if (UserConfig.ready && UserConfig.config.sql?.mySql != null) { try { await DBManager.use(new MySQLDatabase()); } catch (err) { throw new Error(`Failed to configure SQL`); } + } else { + await DBManager.use(new JSONDatabase()); + } // Set up Express const app = express(); diff --git a/backend/data.ts b/backend/data.ts index 99c44d0b..5c1e046c 100644 --- a/backend/data.ts +++ b/backend/data.ts @@ -4,7 +4,6 @@ import fs from 'fs-extra'; import { path } from '@tycrek/joint'; import { log } from './log'; -import { nanoid } from './generators'; import { UserConfig } from './UserConfig'; import { DBManager } from './db/database'; @@ -17,8 +16,8 @@ type DataSector = 'files' | 'users'; * Absolute filepaths for JSON data files */ const PATHS = { - files: path.join('.ass-data/files.json'), - users: path.join('.ass-data/users.json') + files: path.join('.ass-data/files.json'), + users: path.join('.ass-data/users.json') }; const bothWriter = async (files: FilesSchema, users: UsersSchema) => { @@ -26,55 +25,6 @@ const bothWriter = async (files: FilesSchema, users: UsersSchema) => { await fs.writeJson(PATHS.users, users, { spaces: '\t' }); }; -/** - * Creates a JSON file with a given empty data template - */ -const createEmptyJson = (filepath: string, emptyData: any): Promise => new Promise(async (resolve, reject) => { - try { - if (!(await fs.pathExists(filepath))) { - await fs.ensureFile(filepath); - await fs.writeJson(filepath, emptyData, { spaces: '\t' }); - } - resolve(void 0); - } catch (err) { - reject(err); - } -}); - -/** - * Ensures the data files exist and creates them if required - */ -export const ensureFiles = (): Promise => new Promise(async (resolve, reject) => { - log.debug('Checking data files'); - - try { - // Create data directory - await fs.ensureDir(path.join('.ass-data')); - - // * Default files.json - await createEmptyJson(PATHS.files, { - files: {}, - useSql: false, - meta: {} - } as FilesSchema); - - // * Default users.json - await createEmptyJson(PATHS.users, { - tokens: [], - users: {}, - cliKey: nanoid(32), - useSql: false, - meta: {} - } as UsersSchema); - - log.debug('Data files exist'); - resolve(); - } catch (err) { - log.error('Failed to verify existence of data files'); - reject(err); - } -}); - export const setDataModeToSql = (): Promise => new Promise(async (resolve, reject) => { log.debug('Setting data mode to SQL'); @@ -107,60 +57,14 @@ export const setDataModeToSql = (): Promise => new Promise(async (resolve, export const put = (sector: DataSector, key: NID, data: AssFile | AssUser): Promise => new Promise(async (resolve, reject) => { try { - const useSql = DBManager.ready; + const useSql = UserConfig.config.sql != undefined; if (sector === 'files') { - // * 1: Save as files (image, video, etc) - data = data as AssFile; - if (!useSql) { - - // ? Local JSON - const filesJson = await fs.readJson(PATHS.files) as FilesSchema; - - // Check if key already exists - if (filesJson.files[key] != null) return reject(new Error(`File key ${key} already exists`)); - - // Otherwise add the data - filesJson.files[key] = data; - - // Also save the key to the users file - const usersJson = await fs.readJson(PATHS.users) as UsersSchema; - // todo: uncomment this once users are implemented - // usersJson.users[data.uploader].files.push(key); - - // Save the files - await bothWriter(filesJson, usersJson); - } else { - - // ? SQL - if (!(await DBManager.get('assfiles', key))) await DBManager.put('assfiles', key, data); - else return reject(new Error(`File key ${key} already exists`)); - - // todo: modify users SQL files property - } + await DBManager.put('assfiles', key, data as AssFile); } else { - // * 2: Save as users - data = data as AssUser; - if (!useSql) { - - // ? Local JSON - const usersJson = await fs.readJson(PATHS.users) as UsersSchema; - - // Check if key already exists - if (usersJson.users[key] != null) return reject(new Error(`User key ${key} already exists`)); - - // Otherwise add the data - usersJson.users[key] = data; - - await fs.writeJson(PATHS.users, usersJson, { spaces: '\t' }); - } else { - - // ? SQL - if (!(await DBManager.get('assusers', key))) await DBManager.put('assusers', key, data); - else return reject(new Error(`User key ${key} already exists`)); - } + await DBManager.put('assusers', key, data as AssUser); } log.info(`PUT ${sector} data`, `using ${useSql ? 'SQL' : 'local JSON'}`, key); @@ -172,22 +76,18 @@ export const put = (sector: DataSector, key: NID, data: AssFile | AssUser): Prom export const get = (sector: DataSector, key: NID): Promise => new Promise(async (resolve, reject) => { try { - const data: AssFile | AssUser | undefined = (DBManager.ready) - ? (await DBManager.get(sector === 'files' ? 'assfiles' : 'assusers', key) as AssFile | AssUser | undefined) - : (await fs.readJson(PATHS[sector]))[sector][key]; + const data: AssFile | AssUser | undefined = await DBManager.get(sector === 'files' ? 'assfiles' : 'assusers', key) as AssFile | AssUser | undefined (!data) ? resolve(false) : resolve(data); } catch (err) { reject(err); } }); -export const getAll = (sector: DataSector): Promise<{ [key: string]: AssFile | AssUser } | false> => new Promise(async (resolve, reject) => { +export const getAll = (sector: DataSector): Promise<{ [key: string]: AssFile | AssUser }> => new Promise(async (resolve, reject) => { try { - const data: { [key: string]: AssFile | AssUser } | undefined = (DBManager.ready) - // todo: fix MySQL - ? (await DBManager.getAll(sector === 'files' ? 'assfiles' : 'assusers') as /* AssFile[] | AssUser[] | */ []) - : (await fs.readJson(PATHS[sector]))[sector]; - (!data) ? resolve(false) : resolve(data); + // todo: fix MySQL + const data: { [key: string]: AssFile | AssUser } = await DBManager.getAll(sector === 'files' ? 'assfiles' : 'assusers') as /* AssFile[] | AssUser[] | */ {} + resolve(data); } catch (err) { reject(err); } diff --git a/backend/db/database.ts b/backend/db/database.ts index a2bb8cb5..66242458 100644 --- a/backend/db/database.ts +++ b/backend/db/database.ts @@ -35,7 +35,7 @@ export interface Database { /** * get all values from the database */ - getAll(table: DatabaseTable): Promise; + getAll(table: DatabaseTable): Promise<{ [index: string]: DatabaseValue }>; } export class DBManager { @@ -97,7 +97,7 @@ export class DBManager { /** * get all values from the database */ - public static getAll(table: DatabaseTable): Promise { + public static getAll(table: DatabaseTable): Promise<{ [index: string]: DatabaseValue }> { if (this._db && this._dbReady) { return this._db.getAll(table); } else throw new Error("No database active"); diff --git a/backend/db/json.ts b/backend/db/json.ts new file mode 100644 index 00000000..3ac634e7 --- /dev/null +++ b/backend/db/json.ts @@ -0,0 +1,154 @@ +import { AssFile, AssUser, FilesSchema, UsersSchema } from 'ass'; + +import path, { resolve } from 'path'; +import fs from 'fs-extra'; + +import { Database, DatabaseTable, DatabaseValue } from './database'; +import { log } from '../log'; +import { nanoid } from '../generators'; + +/** + * Absolute filepaths for JSON data files + */ +const PATHS = { + files: path.join('.ass-data/files.json'), + users: path.join('.ass-data/users.json') +}; + +/** + * map from tables to paths + */ +const PATHMAP = { + assfiles: PATHS.files, + assusers: PATHS.users +} as { [index: string]: string }; + +/** + * map from tables to sectors + */ +const SECTORMAP = { + assfiles: 'files', + assusers: 'users' +} as { [index: string]: string }; + +const bothWriter = async (files: FilesSchema, users: UsersSchema) => { + await fs.writeJson(PATHS.files, files, { spaces: '\t' }); + await fs.writeJson(PATHS.users, users, { spaces: '\t' }); +}; + +/** + * Creates a JSON file with a given empty data template + */ +const createEmptyJson = (filepath: string, emptyData: any): Promise => new Promise(async (resolve, reject) => { + try { + if (!(await fs.pathExists(filepath))) { + await fs.ensureFile(filepath); + await fs.writeJson(filepath, emptyData, { spaces: '\t' }); + } + resolve(void 0); + } catch (err) { + reject(err); + } +}); + +/** + * Ensures the data files exist and creates them if required + */ +export const ensureFiles = (): Promise => new Promise(async (resolve, reject) => { + log.debug('Checking data files'); + + try { + // Create data directory + await fs.ensureDir(path.join('.ass-data')); + + // * Default files.json + await createEmptyJson(PATHS.files, { + files: {}, + useSql: false, + meta: {} + } as FilesSchema); + + // * Default users.json + await createEmptyJson(PATHS.users, { + tokens: [], + users: {}, + cliKey: nanoid(32), + useSql: false, + meta: {} + } as UsersSchema); + + log.debug('Data files exist'); + resolve(); + } catch (err) { + log.error('Failed to verify existence of data files'); + reject(err); + } +}); + +/** + * JSON database + */ +export class JSONDatabase implements Database { + open(): Promise { return Promise.resolve() } + close(): Promise { return Promise.resolve() } + + configure(): Promise { + return new Promise((resolve, reject) => { + ensureFiles(); + + resolve(); + }); + } + + put(table: DatabaseTable, key: string, data: DatabaseValue): Promise { + return new Promise(async (resolve, reject) => { + if (table == 'assfiles') { + // ? Local JSON + const filesJson = await fs.readJson(PATHS.files) as FilesSchema; + + // Check if key already exists + if (filesJson.files[key] != null) return reject(new Error(`File key ${key} already exists`)); + + // Otherwise add the data + filesJson.files[key] = data as AssFile; + + // Also save the key to the users file + const usersJson = await fs.readJson(PATHS.users) as UsersSchema; + // todo: uncomment this once users are implemented + // usersJson.users[data.uploader].files.push(key); + + // Save the files + await bothWriter(filesJson, usersJson); + + resolve() + } else if (table == 'assusers') { + // ? Local JSON + const usersJson = await fs.readJson(PATHS.users) as UsersSchema; + + // Check if key already exists + if (usersJson.users[key] != null) return reject(new Error(`User key ${key} already exists`)); + + // Otherwise add the data + usersJson.users[key] = data as AssUser; + + await fs.writeJson(PATHS.users, usersJson, { spaces: '\t' }); + + resolve(); + } + }) + } + + get(table: DatabaseTable, key: string): Promise { + return new Promise(async (resolve, reject) => { + const data = (await fs.readJson(PATHMAP[table]))[SECTORMAP[table]][key]; + (!data) ? resolve(undefined) : resolve(data); + }); + } + + getAll(table: DatabaseTable): Promise<{ [index: string]: DatabaseValue }> { + return new Promise(async (resolve, reject) => { + const data = (await fs.readJson(PATHMAP[table]))[SECTORMAP[table]]; + (!data) ? resolve({}) : resolve(data); + }); + } +} \ No newline at end of file diff --git a/backend/db/mysql.ts b/backend/db/mysql.ts index 26e694ea..d2c9bfd1 100644 --- a/backend/db/mysql.ts +++ b/backend/db/mysql.ts @@ -106,6 +106,8 @@ export class MySQLDatabase implements Database { return new Promise(async (resolve, reject) => { if (!this._ready) return reject(new Error('MySQL not ready')); + if (await this.get(table, key)) reject(new Error(`${table == 'assfiles' ? 'File' : table == 'assusers' ? 'User' : 'Token'} key ${key} already exists`)); + const query = ` INSERT INTO ${table} ( NanoID, Data ) VALUES ('${key}', '${JSON.stringify(data)}'); @@ -134,7 +136,7 @@ VALUES ('${key}', '${JSON.stringify(data)}'); } // todo: unknown if this works - public getAll(table: DatabaseTable): Promise { + public getAll(table: DatabaseTable): Promise<{ [index: string]: DatabaseValue }> { return new Promise(async (resolve, reject) => { try { // Run query // ! this may not work as expected @@ -146,7 +148,7 @@ VALUES ('${key}', '${JSON.stringify(data)}'); // console.log(rows); // aaaaaaaaaaaa - resolve([]); + resolve({}); } catch (err) { reject(err); } From d36e5c53fee6dc7d34befb4753e5cca8a0ae2682 Mon Sep 17 00:00:00 2001 From: xwashere Date: Wed, 25 Oct 2023 15:47:37 -0400 Subject: [PATCH 03/16] nvm --- backend/app.ts | 6 +++--- backend/data.ts | 2 +- backend/routers/api.ts | 2 +- backend/{db => sql}/database.ts | 0 backend/{db => sql}/json.ts | 2 +- backend/{db => sql}/mysql.ts | 0 backend/sql/postgres.ts | 0 7 files changed, 6 insertions(+), 6 deletions(-) rename backend/{db => sql}/database.ts (100%) rename backend/{db => sql}/json.ts (98%) rename backend/{db => sql}/mysql.ts (100%) create mode 100644 backend/sql/postgres.ts diff --git a/backend/app.ts b/backend/app.ts index a12c7778..ff51707d 100644 --- a/backend/app.ts +++ b/backend/app.ts @@ -9,12 +9,12 @@ import { path, isProd } from '@tycrek/joint'; import { epcss } from '@tycrek/express-postcss'; import { log } from './log'; -import { JSONDatabase, ensureFiles } from './db/json'; +import { JSONDatabase, ensureFiles } from './sql/json'; import { get } from './data'; import { UserConfig } from './UserConfig'; -import { MySQLDatabase } from './db/mysql'; +import { MySQLDatabase } from './sql/mysql'; import { buildFrontendRouter } from './routers/_frontend'; -import { DBManager } from './db/database'; +import { DBManager } from './sql/database'; /** * Top-level metadata exports diff --git a/backend/data.ts b/backend/data.ts index 5c1e046c..61426e38 100644 --- a/backend/data.ts +++ b/backend/data.ts @@ -5,7 +5,7 @@ import { path } from '@tycrek/joint'; import { log } from './log'; import { UserConfig } from './UserConfig'; -import { DBManager } from './db/database'; +import { DBManager } from './sql/database'; /** * Switcher type for exported functions diff --git a/backend/routers/api.ts b/backend/routers/api.ts index fe6990e4..c43cfd25 100644 --- a/backend/routers/api.ts +++ b/backend/routers/api.ts @@ -8,7 +8,7 @@ import { log } from '../log'; import { nanoid } from '../generators'; import { UserConfig } from '../UserConfig'; import { rateLimiterMiddleware } from '../ratelimit'; -import { DBManager } from '../db/database'; +import { DBManager } from '../sql/database'; const router = Router({ caseSensitive: true }); diff --git a/backend/db/database.ts b/backend/sql/database.ts similarity index 100% rename from backend/db/database.ts rename to backend/sql/database.ts diff --git a/backend/db/json.ts b/backend/sql/json.ts similarity index 98% rename from backend/db/json.ts rename to backend/sql/json.ts index 3ac634e7..2d605453 100644 --- a/backend/db/json.ts +++ b/backend/sql/json.ts @@ -86,7 +86,7 @@ export const ensureFiles = (): Promise => new Promise(async (resolve, reje }); /** - * JSON database + * JSON database. i know json isnt sql, shut up. */ export class JSONDatabase implements Database { open(): Promise { return Promise.resolve() } diff --git a/backend/db/mysql.ts b/backend/sql/mysql.ts similarity index 100% rename from backend/db/mysql.ts rename to backend/sql/mysql.ts diff --git a/backend/sql/postgres.ts b/backend/sql/postgres.ts new file mode 100644 index 00000000..e69de29b From 26431b2982e2cc72ad9aec7e7102efe8d4dad3d8 Mon Sep 17 00:00:00 2001 From: xwashere Date: Thu, 26 Oct 2023 10:24:05 -0400 Subject: [PATCH 04/16] fix: oh god oh fuck oh god oh no oh --- backend/ratelimit.ts | 17 +++++++++++------ backend/routers/api.ts | 11 ++++++++--- backend/routers/index.ts | 2 +- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/backend/ratelimit.ts b/backend/ratelimit.ts index 0706ade2..7e6a31d8 100644 --- a/backend/ratelimit.ts +++ b/backend/ratelimit.ts @@ -7,12 +7,7 @@ import { rateLimit } from 'express-rate-limit'; */ const rateLimiterGroups = new Map void>(); -/** - * creates middleware for rate limiting - */ -export const rateLimiterMiddleware = (group: string, config: EndpointRateLimitConfiguration | undefined): (req: Request, res: Response, next: NextFunction) => void => { - if (rateLimiterGroups.has(group)) return rateLimiterGroups.get(group)!; - +export const setRateLimiter = (group: string, config: EndpointRateLimitConfiguration | undefined): (req: Request, res: Response, next: NextFunction) => void => { if (config == null) { // config might be null if the user doesnt want a rate limit rateLimiterGroups.set(group, (req, res, next) => { next(); @@ -38,4 +33,14 @@ export const rateLimiterMiddleware = (group: string, config: EndpointRateLimitCo return rateLimiterGroups.get(group)!; } +} +/** + * creates middleware for rate limiting + */ +export const rateLimiterMiddleware = (group: string, config: EndpointRateLimitConfiguration | undefined): (req: Request, res: Response, next: NextFunction) => void => { + if (rateLimiterGroups.has(group)) setRateLimiter(group, config); + + return (req, res, next) => { + return rateLimiterGroups.get(group)!(req, res, next); + }; }; \ No newline at end of file diff --git a/backend/routers/api.ts b/backend/routers/api.ts index c43cfd25..c0d1ca55 100644 --- a/backend/routers/api.ts +++ b/backend/routers/api.ts @@ -7,7 +7,7 @@ import * as data from '../data'; import { log } from '../log'; import { nanoid } from '../generators'; import { UserConfig } from '../UserConfig'; -import { rateLimiterMiddleware } from '../ratelimit'; +import { rateLimiterMiddleware, setRateLimiter } from '../ratelimit'; import { DBManager } from '../sql/database'; const router = Router({ caseSensitive: true }); @@ -30,6 +30,11 @@ router.post('/setup', BodyParserJson(), async (req, res) => { if (UserConfig.config.sql?.mySql != null) await Promise.all([DBManager.configure(), data.setDataModeToSql()]); + // set rate limits + if (UserConfig.config.rateLimit?.api) setRateLimiter('api', UserConfig.config.rateLimit.api); + if (UserConfig.config.rateLimit?.login) setRateLimiter('login', UserConfig.config.rateLimit.login); + if (UserConfig.config.rateLimit?.upload) setRateLimiter('upload', UserConfig.config.rateLimit.upload); + log.success('Setup', 'completed'); return res.json({ success: true }); @@ -39,7 +44,7 @@ router.post('/setup', BodyParserJson(), async (req, res) => { }); // User login -router.post('/login', rateLimiterMiddleware('login', UserConfig.config.rateLimit?.login), BodyParserJson(), (req, res) => { +router.post('/login', rateLimiterMiddleware('login', UserConfig.config?.rateLimit?.login), BodyParserJson(), (req, res) => { const { username, password } = req.body; data.getAll('users') @@ -69,7 +74,7 @@ router.post('/login', rateLimiterMiddleware('login', UserConfig.config.rateLimit }); // todo: authenticate API endpoints -router.post('/user', rateLimiterMiddleware('api', UserConfig.config.rateLimit?.api), BodyParserJson(), async (req, res) => { +router.post('/user', rateLimiterMiddleware('api', UserConfig.config?.rateLimit?.api), BodyParserJson(), async (req, res) => { if (!UserConfig.ready) return res.status(409).json({ success: false, message: 'User config not ready' }); diff --git a/backend/routers/index.ts b/backend/routers/index.ts index e49df5ef..eed8d13c 100644 --- a/backend/routers/index.ts +++ b/backend/routers/index.ts @@ -30,7 +30,7 @@ bb.extend(router, { router.get('/', (req, res) => UserConfig.ready ? res.render('index', { version: App.pkgVersion }) : res.redirect('/setup')); // Upload flow -router.post('/', rateLimiterMiddleware("upload", UserConfig.config.rateLimit?.upload), async (req, res) => { +router.post('/', rateLimiterMiddleware("upload", UserConfig.config?.rateLimit?.upload), async (req, res) => { // Check user config if (!UserConfig.ready) return res.status(500).type('text').send('Configuration missing!'); From 9c9f2ac768477759579140a2d2215930bce65dc8 Mon Sep 17 00:00:00 2001 From: xwashere Date: Thu, 26 Oct 2023 12:26:39 -0400 Subject: [PATCH 05/16] impr: move sql config to "database", add config for different db variants --- backend/UserConfig.ts | 17 +++++++++------ backend/app.ts | 20 ++++++++++++++---- backend/data.ts | 44 +++++++------------------------------- backend/routers/api.ts | 20 +++++++++++++++--- backend/sql/json.ts | 12 +++++------ backend/sql/mysql.ts | 33 +++++++++++++++++++++++++---- backend/sql/postgres.ts | 35 ++++++++++++++++++++++++++++++ common/types.d.ts | 28 +++++++++++++++--------- frontend/setup.mts | 47 +++++++++++++++++++++++++++++++++-------- package.json | 2 ++ views/setup.pug | 45 ++++++++++++++++++++++++++++----------- 11 files changed, 213 insertions(+), 90 deletions(-) diff --git a/backend/UserConfig.ts b/backend/UserConfig.ts index c31b9031..df4bce5e 100644 --- a/backend/UserConfig.ts +++ b/backend/UserConfig.ts @@ -102,12 +102,17 @@ export class UserConfig { if (!Checkers.s3.credentials.secretKey(config.s3.credentials.secretKey)) throw new Error('Invalid S3 Secret key'); } - // * Optional SQL config(s) (Currently only checks MySQL) - if (config.sql?.mySql != null) { - if (!Checkers.sql.mySql.host(config.sql.mySql.host)) throw new Error('Invalid MySql Host'); - if (!Checkers.sql.mySql.user(config.sql.mySql.user)) throw new Error('Invalid MySql User'); - if (!Checkers.sql.mySql.password(config.sql.mySql.password)) throw new Error('Invalid MySql Password'); - if (!Checkers.sql.mySql.database(config.sql.mySql.database)) throw new Error('Invalid MySql Database'); + // * Optional database config(s) + if (config.database != null) { + // these both have the same schema so we can just check both + if (config.database.kind == 'mysql' || config.database.kind == 'postgres') { + if (config.database.options != undefined) { + if (!Checkers.sql.mySql.host(config.database.options.host)) throw new Error('Invalid database host'); + if (!Checkers.sql.mySql.user(config.database.options.user)) throw new Error('Invalid databse user'); + if (!Checkers.sql.mySql.password(config.database.options.password)) throw new Error('Invalid database password'); + if (!Checkers.sql.mySql.database(config.database.options.database)) throw new Error('Invalid database'); + } else throw new Error('Database options missing'); + } } // * optional rate limit config diff --git a/backend/app.ts b/backend/app.ts index ff51707d..85165c83 100644 --- a/backend/app.ts +++ b/backend/app.ts @@ -15,6 +15,7 @@ import { UserConfig } from './UserConfig'; import { MySQLDatabase } from './sql/mysql'; import { buildFrontendRouter } from './routers/_frontend'; import { DBManager } from './sql/database'; +import { PostgreSQLDatabase } from './sql/postgres'; /** * Top-level metadata exports @@ -114,10 +115,21 @@ async function main() { .catch((err) => (err.code && err.code === 'ENOENT' ? {} : console.error(err), resolve(void 0)))); // If user config is ready, try to configure SQL - if (UserConfig.ready && UserConfig.config.sql?.mySql != null) { - try { await DBManager.use(new MySQLDatabase()); } - catch (err) { throw new Error(`Failed to configure SQL`); } - } else { + if (UserConfig.ready && UserConfig.config.database != null) { + try { + switch (UserConfig.config.database?.kind) { + case 'json': + await DBManager.use(new JSONDatabase()); + break; + case 'mysql': + await DBManager.use(new MySQLDatabase()); + break; + case 'postgres': + await DBManager.use(new PostgreSQLDatabase()); + break; + } + } catch (err) { throw new Error(`Failed to configure SQL`); } + } else { // default to json database await DBManager.use(new JSONDatabase()); } diff --git a/backend/data.ts b/backend/data.ts index 61426e38..72e2f8c0 100644 --- a/backend/data.ts +++ b/backend/data.ts @@ -20,45 +20,17 @@ const PATHS = { users: path.join('.ass-data/users.json') }; -const bothWriter = async (files: FilesSchema, users: UsersSchema) => { - await fs.writeJson(PATHS.files, files, { spaces: '\t' }); - await fs.writeJson(PATHS.users, users, { spaces: '\t' }); +/** + * database kind -> name mapping + */ +const DBNAMES = { + 'mysql': 'MySQL', + 'postgres': 'PostgreSQL', + 'json': 'JSON' }; -export const setDataModeToSql = (): Promise => new Promise(async (resolve, reject) => { - log.debug('Setting data mode to SQL'); - - // Main config check - if (!UserConfig.ready || !UserConfig.config.sql?.mySql) return reject(new Error('MySQL not configured')); - const mySqlConf = UserConfig.config.sql.mySql; - - // Read data files - const [files, users]: [FilesSchema, UsersSchema] = await Promise.all([fs.readJson(PATHS.files), fs.readJson(PATHS.users)]); - - // Check the MySQL configuration - const checker = (val: string) => val != null && val !== ''; - const issue = - !checker(mySqlConf.host) ? 'Missing MySQL Host' - : !checker(mySqlConf.user) ? 'Missing MySQL User' - : !checker(mySqlConf.password) ? 'Missing MySQL Password' - : !checker(mySqlConf.database) ? 'Missing MySQL Database' - - // ! Blame VS Code for this weird indentation - : undefined; - - // Set the vars - files.useSql = issue == null; - users.useSql = issue == null; - - // Write data & return - await bothWriter(files, users); - (issue) ? reject(new Error(issue)) : resolve(void 0); -}); - export const put = (sector: DataSector, key: NID, data: AssFile | AssUser): Promise => new Promise(async (resolve, reject) => { try { - const useSql = UserConfig.config.sql != undefined; - if (sector === 'files') { // * 1: Save as files (image, video, etc) await DBManager.put('assfiles', key, data as AssFile); @@ -67,7 +39,7 @@ export const put = (sector: DataSector, key: NID, data: AssFile | AssUser): Prom await DBManager.put('assusers', key, data as AssUser); } - log.info(`PUT ${sector} data`, `using ${useSql ? 'SQL' : 'local JSON'}`, key); + log.info(`PUT ${sector} data`, `using ${DBNAMES[UserConfig.config.database?.kind ?? 'json']}`, key); resolve(void 0); } catch (err) { reject(err); diff --git a/backend/routers/api.ts b/backend/routers/api.ts index 3383b5eb..e2aa2bc3 100644 --- a/backend/routers/api.ts +++ b/backend/routers/api.ts @@ -9,6 +9,9 @@ import { nanoid } from '../generators'; import { UserConfig } from '../UserConfig'; import { rateLimiterMiddleware, setRateLimiter } from '../ratelimit'; import { DBManager } from '../sql/database'; +import { JSONDatabase } from '../sql/json'; +import { MySQLDatabase } from '../sql/mysql'; +import { PostgreSQLDatabase } from '../sql/postgres'; const router = Router({ caseSensitive: true }); @@ -26,9 +29,20 @@ router.post('/setup', BodyParserJson(), async (req, res) => { // Save config await UserConfig.saveConfigFile(); - // Set data storage (not files) to SQL if required - if (UserConfig.config.sql?.mySql != null) - await Promise.all([DBManager.configure(), data.setDataModeToSql()]); + // set up new databases + if (UserConfig.config.database) { + switch (UserConfig.config.database.kind) { + case 'json': + await DBManager.use(new JSONDatabase()); + break; + case 'mysql': + await DBManager.use(new MySQLDatabase()); + break; + case 'postgres': + await DBManager.use(new PostgreSQLDatabase()); + break; + } + } // set rate limits if (UserConfig.config.rateLimit?.api) setRateLimiter('api', UserConfig.config.rateLimit.api); diff --git a/backend/sql/json.ts b/backend/sql/json.ts index 2d605453..6703ed2a 100644 --- a/backend/sql/json.ts +++ b/backend/sql/json.ts @@ -89,10 +89,10 @@ export const ensureFiles = (): Promise => new Promise(async (resolve, reje * JSON database. i know json isnt sql, shut up. */ export class JSONDatabase implements Database { - open(): Promise { return Promise.resolve() } - close(): Promise { return Promise.resolve() } + public open(): Promise { return Promise.resolve() } + public close(): Promise { return Promise.resolve() } - configure(): Promise { + public configure(): Promise { return new Promise((resolve, reject) => { ensureFiles(); @@ -100,7 +100,7 @@ export class JSONDatabase implements Database { }); } - put(table: DatabaseTable, key: string, data: DatabaseValue): Promise { + public put(table: DatabaseTable, key: string, data: DatabaseValue): Promise { return new Promise(async (resolve, reject) => { if (table == 'assfiles') { // ? Local JSON @@ -138,14 +138,14 @@ export class JSONDatabase implements Database { }) } - get(table: DatabaseTable, key: string): Promise { + public get(table: DatabaseTable, key: string): Promise { return new Promise(async (resolve, reject) => { const data = (await fs.readJson(PATHMAP[table]))[SECTORMAP[table]][key]; (!data) ? resolve(undefined) : resolve(data); }); } - getAll(table: DatabaseTable): Promise<{ [index: string]: DatabaseValue }> { + public getAll(table: DatabaseTable): Promise<{ [index: string]: DatabaseValue }> { return new Promise(async (resolve, reject) => { const data = (await fs.readJson(PATHMAP[table]))[SECTORMAP[table]]; (!data) ? resolve({}) : resolve(data); diff --git a/backend/sql/mysql.ts b/backend/sql/mysql.ts index d2c9bfd1..3e90b473 100644 --- a/backend/sql/mysql.ts +++ b/backend/sql/mysql.ts @@ -25,6 +25,31 @@ export class MySQLDatabase implements Database { .catch((err) => reject(err))); } + /** + * validate the mysql config + */ + private _validateConfig(): string | undefined { + // make sure the configuration exists + if (!UserConfig.ready) return 'User configuration not ready'; + if (typeof UserConfig.config.database != 'object') return 'MySQL configuration missing'; + if (UserConfig.config.database.kind != "mysql") return 'Database not set to MySQL, but MySQL is in use, something has gone terribly wrong'; + if (typeof UserConfig.config.database.options != 'object') return 'MySQL configuration missing'; + + let mySqlConf = UserConfig.config.database.options; + + // Check the MySQL configuration + const checker = (val: string) => val != null && val !== ''; + const issue = + !checker(mySqlConf.host) ? 'Missing MySQL Host' + : !checker(mySqlConf.user) ? 'Missing MySQL User' + : !checker(mySqlConf.password) ? 'Missing MySQL Password' + : !checker(mySqlConf.database) ? 'Missing MySQL Database' + // ! Blame VS Code for this weird indentation + : undefined; + + return issue; + } + public open() { return Promise.resolve(); } public close() { return Promise.resolve(); } @@ -35,11 +60,11 @@ export class MySQLDatabase implements Database { return new Promise(async (resolve, reject) => { try { // Config check - if (!UserConfig.ready) throw new Error('User configuration not ready'); - if (!UserConfig.config.sql?.mySql) throw new Error('MySQL configuration missing'); + let configError = this._validateConfig(); + if (configError) throw new Error(configError); // Create the pool - this._pool = mysql.createPool(UserConfig.config.sql.mySql); + this._pool = mysql.createPool(UserConfig.config.database!.options!); // Check if the pool is usable const [rowz, _fields] = await this._pool.query(`SHOW FULL TABLES WHERE Table_Type LIKE 'BASE TABLE';`); @@ -61,7 +86,7 @@ export class MySQLDatabase implements Database { // Check which tables ACTUALLY do exist for (let row of rows_tableData) { - const table = row[`Tables_in_${UserConfig.config.sql!.mySql!.database}` + const table = row[`Tables_in_${UserConfig.config.database!.options!.database}` ] as DatabaseTable; if (table === 'assfiles') tablesExist.files = true; if (table === 'assusers') tablesExist.users = true; diff --git a/backend/sql/postgres.ts b/backend/sql/postgres.ts index e69de29b..45a3ed09 100644 --- a/backend/sql/postgres.ts +++ b/backend/sql/postgres.ts @@ -0,0 +1,35 @@ +import { Client } from 'pg'; + +import { log } from '../log'; +import { Database, DatabaseTable, DatabaseValue } from './database'; + +export class PostgreSQLDatabase implements Database { + private _client: Client; + + public open(): Promise { return Promise.resolve(); } + public close(): Promise { return Promise.resolve(); } + + public configure(): Promise { + return new Promise((resolve, reject) => { + try { + + } catch (err) { + log.error('PostgreSQL', 'failed to initialize'); + console.error(err); + reject(err); + } + }); + } + + public put(table: DatabaseTable, key: string, data: DatabaseValue): Promise { + throw new Error("Method not implemented."); + } + + public get(table: DatabaseTable, key: string): Promise { + throw new Error("Method not implemented."); + } + + public getAll(table: DatabaseTable): Promise<{ [index: string]: DatabaseValue; }> { + throw new Error("Method not implemented."); + } +} \ No newline at end of file diff --git a/common/types.d.ts b/common/types.d.ts index af05700f..305f5e72 100644 --- a/common/types.d.ts +++ b/common/types.d.ts @@ -24,7 +24,7 @@ declare module 'ass' { maximumFileSize: number; s3?: S3Configuration; - sql?: SqlConfiguration; + database?: DatabaseConfiguration; rateLimit?: RateLimitConfiguration; } @@ -51,13 +51,23 @@ declare module 'ass' { } } - interface SqlConfiguration { - mySql?: { - host: string; - user: string; - password: string; - database: string; - } + interface DatabaseConfiguration { + kind: 'mysql' | 'postgres' | 'json'; + options?: MySQLConfiguration | PostgresConfiguration; + } + + interface MySQLConfiguration { + host: string; + user: string; + password: string; + database: string; + } + + interface PostgresConfiguration { + host: string; + user: string; + password: string; + database: string; } /** @@ -219,7 +229,6 @@ declare module 'ass' { files: { [key: NID]: AssFile; } - useSql: boolean; meta: { [key: string]: any }; } @@ -232,7 +241,6 @@ declare module 'ass' { [key: NID]: AssUser; }; cliKey: string; - useSql: boolean; meta: { [key: string]: any }; } } diff --git a/frontend/setup.mts b/frontend/setup.mts index 306e608a..ca33b7eb 100644 --- a/frontend/setup.mts +++ b/frontend/setup.mts @@ -1,4 +1,4 @@ -import { SlInput, SlButton } from '@shoelace-style/shoelace'; +import { SlInput, SlButton, SlTab } from '@shoelace-style/shoelace'; import { IdType, UserConfiguration } from 'ass'; const genericErrorAlert = () => alert('An error occured, please check the console for details'); @@ -41,11 +41,20 @@ document.addEventListener('DOMContentLoaded', () => { s3secretKey: document.querySelector('#s3-secretKey') as SlInput, s3region: document.querySelector('#s3-region') as SlInput, + jsonTab: document.querySelector('#json-tab') as SlTab, + + mySqlTab: document.querySelector('#mysql-tab') as SlTab, mySqlHost: document.querySelector('#mysql-host') as SlInput, mySqlUser: document.querySelector('#mysql-user') as SlInput, mySqlPassword: document.querySelector('#mysql-password') as SlInput, mySqlDatabase: document.querySelector('#mysql-database') as SlInput, + pgsqlTab: document.querySelector('#pgsql-tab') as SlTab, + pgsqlHost: document.querySelector('#pgsql-host') as SlInput, + pgsqlUser: document.querySelector('#pgsql-user') as SlInput, + pgsqlPassword: document.querySelector('#pgsql-password') as SlInput, + pgsqlDatabase: document.querySelector('#pgsql-database') as SlInput, + userUsername: document.querySelector('#user-username') as SlInput, userPassword: document.querySelector('#user-password') as SlInput, @@ -88,14 +97,34 @@ document.addEventListener('DOMContentLoaded', () => { config.s3.region = Elements.s3region.value; } - // Append MySQL to config, if specified - if (Elements.mySqlHost.value != null && Elements.mySqlHost.value !== '') { - if (!config.sql) config.sql = {}; - config.sql.mySql = { - host: Elements.mySqlHost.value, - user: Elements.mySqlUser.value, - password: Elements.mySqlPassword.value, - database: Elements.mySqlDatabase.value + // Append database to config, if specified + if (Elements.jsonTab.active) { + config.database = { + kind: 'json' + }; + } else if (Elements.mySqlTab.active) { + if (Elements.mySqlHost.value != null && Elements.mySqlHost.value != '') { + config.database = { + kind: 'mysql', + options: { + host: Elements.mySqlHost.value, + user: Elements.mySqlUser.value, + password: Elements.mySqlPassword.value, + database: Elements.mySqlDatabase.value + } + } + }; + } else if (Elements.pgsqlTab.active) { + if (Elements.pgsqlHost.value != null && Elements.pgsqlHost.value != '') { + config.database = { + kind: 'postgres', + options: { + host: Elements.pgsqlHost.value, + user: Elements.pgsqlUser.value, + password: Elements.pgsqlPassword.value, + database: Elements.pgsqlDatabase.value + } + } }; } diff --git a/package.json b/package.json index c8a56599..c24d47a5 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "mysql2": "^3.6.1", "node-fetch": "^2.6.7", "node-vibrant": "^3.1.6", + "pg": "^8.11.3", "pug": "^3.0.2", "sanitize-filename": "^1.6.3", "sharp": "^0.32.6", @@ -76,6 +77,7 @@ "@types/luxon": "^3.3.2", "@types/node": "^18.16.19", "@types/node-fetch": "^2.6.6", + "@types/pg": "^8.10.7", "@types/sharp": "^0.32.0", "@types/tailwindcss": "^3.1.0", "@types/uuid": "^8.3.1" diff --git a/views/setup.pug b/views/setup.pug index 6446068b..cd7cb6a9 100644 --- a/views/setup.pug +++ b/views/setup.pug @@ -30,6 +30,39 @@ block content h3.setup-text-item-title Password sl-input#user-password(type='password' placeholder='the-most-secure' clearable): sl-icon(slot='prefix' name='fas-lock' library='fa') + //- * Database + h2.setup-text-section-header.mt-4 Database + .setup-panel + sl-tab-group + //- * JSON + sl-tab#json-tab(slot='nav' panel='json') JSON + sl-tab-panel(name='json') + | you all good! + + //- * MySQL + sl-tab#mysql-tab(slot='nav' panel='mysql') MySQL + sl-tab-panel(name='mysql') + h3.setup-text-item-title Host + sl-input#mysql-host(type='text' placeholder='mysql.example.com' clearable): sl-icon(slot='prefix' name='fas-server' library='fa') + h3.setup-text-item-title User + sl-input#mysql-user(type='text' placeholder='myassql' clearable): sl-icon(slot='prefix' name='fas-user' library='fa') + h3.setup-text-item-title Password + sl-input#mysql-password(type='password' placeholder='super-secure' clearable): sl-icon(slot='prefix' name='fas-lock' library='fa') + h3.setup-text-item-title Database + sl-input#mysql-database(type='text' placeholder='assdb' clearable): sl-icon(slot='prefix' name='fas-database' library='fa') + + //- * PostgreSQL + sl-tab#pgsql-tab(slot='nav' panel='pgsql') PostgreSQL + sl-tab-panel(name='pgsql') + h3.setup-text-item-title Host + sl-input#pgsql-host(type='text' placeholder='postgres.example.com' clearable): sl-icon(slot='prefix' name='fas-server' library='fa') + h3.setup-text-item-title User + sl-input#pgsql-user(type='text' placeholder='posgrassql' clearable): sl-icon(slot='prefix' name='fas-user' library='fa') + h3.setup-text-item-title Password + sl-input#pgsql-password(type='password' placeholder='super-secure' clearable): sl-icon(slot='prefix' name='fas-lock' library='fa') + h3.setup-text-item-title Database + sl-input#pgsql-database(type='text' placeholder='assdb' clearable): sl-icon(slot='prefix' name='fas-database' library='fa') + //- * S3 h2.setup-text-section-header.mt-4 S3 #[span.setup-text-optional optional] .setup-panel @@ -43,18 +76,6 @@ block content sl-input#s3-secretKey(type='password' placeholder='EF56GH78IJ90KL12' clearable): sl-icon(slot='prefix' name='fas-user-secret' library='fa') h3.setup-text-item-title Region #[span.setup-text-optional optional] sl-input#s3-region(type='text' placeholder='us-east' clearable): sl-icon(slot='prefix' name='fas-map-location-dot' library='fa') - - //- * MySQL - h2.setup-text-section-header.mt-4 MySQL #[span.setup-text-optional optional] - .setup-panel - h3.setup-text-item-title Host - sl-input#mysql-host(type='text' placeholder='mysql.example.com' clearable): sl-icon(slot='prefix' name='fas-server' library='fa') - h3.setup-text-item-title User - sl-input#mysql-user(type='text' placeholder='myassql' clearable): sl-icon(slot='prefix' name='fas-user' library='fa') - h3.setup-text-item-title Password - sl-input#mysql-password(type='password' placeholder='super-secure' clearable): sl-icon(slot='prefix' name='fas-lock' library='fa') - h3.setup-text-item-title Database - sl-input#mysql-database(type='text' placeholder='assdb' clearable): sl-icon(slot='prefix' name='fas-database' library='fa') //- * Rate Limits h2.setup-text-section-header.mt-4 Rate Limits #[span.setup-text-optional optional] From 0d1172d332a59d8e4b694d89d98432d542ac82ca Mon Sep 17 00:00:00 2001 From: xwashere Date: Thu, 26 Oct 2023 14:52:56 -0400 Subject: [PATCH 06/16] add: postgresql! --- backend/UserConfig.ts | 12 ++- backend/sql/postgres.ts | 180 ++++++++++++++++++++++++++++++++++++++-- common/types.d.ts | 4 + frontend/setup.mts | 2 + views/setup.pug | 2 + 5 files changed, 191 insertions(+), 9 deletions(-) diff --git a/backend/UserConfig.ts b/backend/UserConfig.ts index df4bce5e..735088c6 100644 --- a/backend/UserConfig.ts +++ b/backend/UserConfig.ts @@ -1,4 +1,4 @@ -import { UserConfiguration, UserConfigTypeChecker } from 'ass'; +import { UserConfiguration, UserConfigTypeChecker, PostgresConfiguration } from 'ass'; import fs from 'fs-extra'; import { path } from '@tycrek/joint'; @@ -56,7 +56,10 @@ const Checkers: UserConfigTypeChecker = { host: basicStringChecker, user: basicStringChecker, password: basicStringChecker, - database: basicStringChecker + database: basicStringChecker, + }, + postgres: { + port: (val) => numChecker(val) && val >= 1 && val <= 65535 } }, @@ -111,6 +114,11 @@ export class UserConfig { if (!Checkers.sql.mySql.user(config.database.options.user)) throw new Error('Invalid databse user'); if (!Checkers.sql.mySql.password(config.database.options.password)) throw new Error('Invalid database password'); if (!Checkers.sql.mySql.database(config.database.options.database)) throw new Error('Invalid database'); + if (config.database.kind == 'postgres') { + if (!Checkers.sql.postgres.port((config.database.options as PostgresConfiguration).port)) { + throw new Error("Invalid database port"); + } + } } else throw new Error('Database options missing'); } } diff --git a/backend/sql/postgres.ts b/backend/sql/postgres.ts index 45a3ed09..a92703f8 100644 --- a/backend/sql/postgres.ts +++ b/backend/sql/postgres.ts @@ -1,20 +1,144 @@ +import { PostgresConfiguration } from 'ass'; + import { Client } from 'pg'; import { log } from '../log'; import { Database, DatabaseTable, DatabaseValue } from './database'; +import { UserConfig } from '../UserConfig'; +/** + * database adapter for postgresql + */ export class PostgreSQLDatabase implements Database { private _client: Client; - public open(): Promise { return Promise.resolve(); } - public close(): Promise { return Promise.resolve(); } + /** + * validate config + */ + private _validateConfig(): string | undefined { + // make sure the configuration exists + if (!UserConfig.ready) return 'User configuration not ready'; + if (typeof UserConfig.config.database != 'object') return 'PostgreSQL configuration missing'; + if (UserConfig.config.database.kind != "postgres") return 'Database not set to PostgreSQL, but PostgreSQL is in use, something has gone terribly wrong'; + if (typeof UserConfig.config.database.options != 'object') return 'PostgreSQL configuration missing'; + + let config = UserConfig.config.database.options; + + // check the postgres config + const checker = (val: string) => val != null && val !== ''; + const issue = + !checker(config.host) ? 'Missing PostgreSQL Host' + : !checker(config.user) ? 'Missing PostgreSQL User' + : !checker(config.password) ? 'Missing PostgreSQL Password' + : !checker(config.database) ? 'Missing PostgreSQL Database' + // ! Blame VS Code for this weird indentation + : undefined; + + return issue; + + } + + public open(): Promise { + return new Promise(async (resolve, reject) => { + try { + // config check + let configError = this._validateConfig(); + if (configError) throw new Error(configError); + + // grab the config + let config = UserConfig.config.database!.options! as PostgresConfiguration; + + // set up the client + this._client = new Client({ + host: config.host, + port: config.port, + user: config.user, + password: config.password, + database: config.database, + }); + + // connect to the database + log.info('PostgreSQL', `connecting to ${config.host}:${config.port}`); + await this._client.connect(); + log.success('PostgreSQL', 'ok'); + + resolve(); + } catch (err) { + log.error('PostgreSQL', 'failed to connect'); + console.error(err); + reject(err); + } + }); + } + + public close(): Promise { + return new Promise(async (resolve, reject) => { + try { + // gracefully disconnect + await this._client.end(); + + resolve(); + } catch (err) { + log.error('PostgreSQL', 'failed to disconnect'); + console.error(err); + reject(err); + } + }); + } public configure(): Promise { - return new Promise((resolve, reject) => { + return new Promise(async (resolve, reject) => { try { + await this._client.query( +`CREATE TABLE IF NOT EXISTS asstables ( + name TEXT PRIMARY KEY, + version INT NOT NULL +);`); + log.info('PostgreSQL', 'checking database'); + + // update tables + let seenRows = new Set(); + let versions = await this._client.query('SELECT * FROM asstables;'); + for (let row of versions.rows) { + seenRows.add(row.name); + } + + const assTableSchema = '(id TEXT PRIMARY KEY, data JSON NOT NULL)' + + // add missing tables + if (!seenRows.has('assfiles')) { + log.warn('PostgreSQL', 'assfiles missing, repairing...') + await this._client.query( + `CREATE TABLE assfiles ${assTableSchema};` + + `INSERT INTO asstables (name, version) VALUES ('assfiles', 1);` + ); + log.success('PostgreSQL', 'ok'); + } + + if (!seenRows.has('assusers')) { + log.warn('PostgreSQL', 'asstokens missing, repairing...') + await this._client.query( + `CREATE TABLE assusers ${assTableSchema};` + + `INSERT INTO asstables (name, version) VALUES ('assusers', 1);` + ); + log.success('PostgreSQL', 'ok'); + } + + if (!seenRows.has('asstokens')) { + log.warn('PostgreSQL', 'asstokens missing, repairing...') + await this._client.query( + `CREATE TABLE asstokens ${assTableSchema};` + + `INSERT INTO asstables (name, version) VALUES ('asstokens', 1);` + ); + log.success('PostgreSQL', 'ok'); + } + + log.success('PostgreSQL', 'database is ok').callback(() => { + resolve(); + }); } catch (err) { - log.error('PostgreSQL', 'failed to initialize'); + log.error('PostgreSQL', 'failed to set up'); console.error(err); reject(err); } @@ -22,14 +146,56 @@ export class PostgreSQLDatabase implements Database { } public put(table: DatabaseTable, key: string, data: DatabaseValue): Promise { - throw new Error("Method not implemented."); + return new Promise(async (resolve, reject) => { + try { + const queries = { + assfiles: 'INSERT INTO assfiles (id, data) VALUES ($1, $2)', + assusers: 'INSERT INTO assusers (id, data) VALUES ($1, $2)', + asstokens: 'INSERT INTO asstokens (id, data) VALUES ($1, $2)' + }; + + let result = await this._client.query(queries[table], [key, data]); + + resolve(); + } catch (err) { + reject(err); + } + }); } public get(table: DatabaseTable, key: string): Promise { - throw new Error("Method not implemented."); + return new Promise(async (resolve, reject) => { + try { + const queries = { + assfiles: 'SELECT data FROM assfiles WHERE id = $1::text', + assusers: 'SELECT data FROM assusers WHERE id = $1::text', + asstokens: 'SELECT data FROM asstokens WHERE id = $1::text' + }; + + let result = await this._client.query(queries[table], [key]); + + resolve(result.rowCount ? result.rows[0].data : void 0); + } catch (err) { + reject(err); + } + }); } public getAll(table: DatabaseTable): Promise<{ [index: string]: DatabaseValue; }> { - throw new Error("Method not implemented."); + return new Promise(async (resolve, reject) => { + try { + const queries = { + assfiles: 'SELECT json_object_agg(id, data) AS stuff FROM assfiles;', + assusers: 'SELECT json_object_agg(id, data) AS stuff FROM assusers;', + asstokens: 'SELECT json_object_agg(id, data) AS stuff FROM asstokens;' + }; + + let result = await this._client.query(queries[table]); + + resolve(result.rowCount ? result.rows[0].stuff : void 0); + } catch (err) { + reject(err); + } + }); } } \ No newline at end of file diff --git a/common/types.d.ts b/common/types.d.ts index 305f5e72..ddc3cd8b 100644 --- a/common/types.d.ts +++ b/common/types.d.ts @@ -65,6 +65,7 @@ declare module 'ass' { interface PostgresConfiguration { host: string; + port: number; user: string; password: string; database: string; @@ -129,6 +130,9 @@ declare module 'ass' { password: (val: any) => boolean; database: (val: any) => boolean; } + postgres: { + port: (val: any) => boolean; + } } rateLimit: { endpoint: (val: any) => boolean; diff --git a/frontend/setup.mts b/frontend/setup.mts index ca33b7eb..c4372d88 100644 --- a/frontend/setup.mts +++ b/frontend/setup.mts @@ -51,6 +51,7 @@ document.addEventListener('DOMContentLoaded', () => { pgsqlTab: document.querySelector('#pgsql-tab') as SlTab, pgsqlHost: document.querySelector('#pgsql-host') as SlInput, + pgsqlPort: document.querySelector('#pgsql-port') as SlInput, pgsqlUser: document.querySelector('#pgsql-user') as SlInput, pgsqlPassword: document.querySelector('#pgsql-password') as SlInput, pgsqlDatabase: document.querySelector('#pgsql-database') as SlInput, @@ -120,6 +121,7 @@ document.addEventListener('DOMContentLoaded', () => { kind: 'postgres', options: { host: Elements.pgsqlHost.value, + port: parseInt(Elements.pgsqlPort.value), user: Elements.pgsqlUser.value, password: Elements.pgsqlPassword.value, database: Elements.pgsqlDatabase.value diff --git a/views/setup.pug b/views/setup.pug index cd7cb6a9..7a86a7b8 100644 --- a/views/setup.pug +++ b/views/setup.pug @@ -56,6 +56,8 @@ block content sl-tab-panel(name='pgsql') h3.setup-text-item-title Host sl-input#pgsql-host(type='text' placeholder='postgres.example.com' clearable): sl-icon(slot='prefix' name='fas-server' library='fa') + h3.setup-text-item-title Port + sl-input#pgsql-port(type='number' placeholder='5432' min='1' max='65535' no-spin-buttons clearable): sl-icon(slot='prefix' name='fas-server' library='fa') h3.setup-text-item-title User sl-input#pgsql-user(type='text' placeholder='posgrassql' clearable): sl-icon(slot='prefix' name='fas-user' library='fa') h3.setup-text-item-title Password From b7ddc96b061a886aeecb1846a9f97e2f1a60f08a Mon Sep 17 00:00:00 2001 From: xwashere Date: Thu, 26 Oct 2023 15:01:01 -0400 Subject: [PATCH 07/16] mayb this should be a number --- views/setup.pug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/views/setup.pug b/views/setup.pug index 7a86a7b8..5426e4ec 100644 --- a/views/setup.pug +++ b/views/setup.pug @@ -57,7 +57,7 @@ block content h3.setup-text-item-title Host sl-input#pgsql-host(type='text' placeholder='postgres.example.com' clearable): sl-icon(slot='prefix' name='fas-server' library='fa') h3.setup-text-item-title Port - sl-input#pgsql-port(type='number' placeholder='5432' min='1' max='65535' no-spin-buttons clearable): sl-icon(slot='prefix' name='fas-server' library='fa') + sl-input#pgsql-port(type='number' placeholder='5432' min='1' max='65535' no-spin-buttons clearable): sl-icon(slot='prefix' name='fas-number' library='fa') h3.setup-text-item-title User sl-input#pgsql-user(type='text' placeholder='posgrassql' clearable): sl-icon(slot='prefix' name='fas-user' library='fa') h3.setup-text-item-title Password From 3c5094695a76950a1ec0abcdc75753517655f63c Mon Sep 17 00:00:00 2001 From: xwashere Date: Thu, 26 Oct 2023 15:01:51 -0400 Subject: [PATCH 08/16] semicolons --- backend/sql/postgres.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/sql/postgres.ts b/backend/sql/postgres.ts index a92703f8..7b879f7d 100644 --- a/backend/sql/postgres.ts +++ b/backend/sql/postgres.ts @@ -149,9 +149,9 @@ export class PostgreSQLDatabase implements Database { return new Promise(async (resolve, reject) => { try { const queries = { - assfiles: 'INSERT INTO assfiles (id, data) VALUES ($1, $2)', - assusers: 'INSERT INTO assusers (id, data) VALUES ($1, $2)', - asstokens: 'INSERT INTO asstokens (id, data) VALUES ($1, $2)' + assfiles: 'INSERT INTO assfiles (id, data) VALUES ($1, $2);', + assusers: 'INSERT INTO assusers (id, data) VALUES ($1, $2);', + asstokens: 'INSERT INTO asstokens (id, data) VALUES ($1, $2);' }; let result = await this._client.query(queries[table], [key, data]); @@ -167,9 +167,9 @@ export class PostgreSQLDatabase implements Database { return new Promise(async (resolve, reject) => { try { const queries = { - assfiles: 'SELECT data FROM assfiles WHERE id = $1::text', - assusers: 'SELECT data FROM assusers WHERE id = $1::text', - asstokens: 'SELECT data FROM asstokens WHERE id = $1::text' + assfiles: 'SELECT data FROM assfiles WHERE id = $1::text;', + assusers: 'SELECT data FROM assusers WHERE id = $1::text;', + asstokens: 'SELECT data FROM asstokens WHERE id = $1::text;' }; let result = await this._client.query(queries[table], [key]); From df5510d01eec930c4c480dd38eecbdc2007b6474 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Fri, 27 Oct 2023 17:26:46 -0600 Subject: [PATCH 09/16] fix: update pnpm lock --- pnpm-lock.yaml | 160 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3132fc03..30d02e7e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -77,6 +77,9 @@ dependencies: node-vibrant: specifier: ^3.1.6 version: 3.1.6 + pg: + specifier: ^8.11.3 + version: 8.11.3 pug: specifier: ^3.0.2 version: 3.0.2 @@ -121,6 +124,9 @@ devDependencies: '@types/node': specifier: ^20.8.9 version: 20.8.9 + '@types/pg': + specifier: ^8.10.7 + version: 8.10.7 packages: @@ -1877,6 +1883,14 @@ packages: dependencies: undici-types: 5.26.5 + /@types/pg@8.10.7: + resolution: {integrity: sha512-ksJqHipwYaSEHz9e1fr6H6erjoEdNNaOxwyJgPx9bNeaqOW3iWBQgVHfpwiSAoqGzchfc+ZyRLwEfeCcyYD3uQ==} + dependencies: + '@types/node': 20.8.9 + pg-protocol: 1.6.0 + pg-types: 4.0.1 + dev: true + /@types/prop-types@15.7.9: resolution: {integrity: sha512-n1yyPsugYNSmHgxDFjicaI2+gCNjsBck8UX9kuofAKlc0h1bL+20oSF72KeNaW2DUlesbEVCFgyV2dPGTiY42g==} dev: false @@ -2185,6 +2199,11 @@ packages: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} dev: false + /buffer-writer@2.0.0: + resolution: {integrity: sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==} + engines: {node: '>=4'} + dev: false + /buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} dependencies: @@ -3699,6 +3718,10 @@ packages: resolution: {integrity: sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==} dev: false + /obuf@1.1.2: + resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} + dev: true + /omggif@1.0.10: resolution: {integrity: sha512-LMJTtvgc/nugXj0Vcrrs68Mn2D1r0zf630VNtqtpI1FEO7e+O9FP4gqs9AcnBaSEeoHIPm28u6qgPR0oyEpGSw==} dev: false @@ -3721,6 +3744,10 @@ packages: wrappy: 1.0.2 dev: false + /packet-reader@1.0.0: + resolution: {integrity: sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==} + dev: false + /pako@1.0.11: resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==} dev: false @@ -3771,6 +3798,86 @@ packages: engines: {node: '>=8'} dev: false + /pg-cloudflare@1.1.1: + resolution: {integrity: sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==} + requiresBuild: true + dev: false + optional: true + + /pg-connection-string@2.6.2: + resolution: {integrity: sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==} + dev: false + + /pg-int8@1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} + engines: {node: '>=4.0.0'} + + /pg-numeric@1.0.2: + resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==} + engines: {node: '>=4'} + dev: true + + /pg-pool@3.6.1(pg@8.11.3): + resolution: {integrity: sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==} + peerDependencies: + pg: '>=8.0' + dependencies: + pg: 8.11.3 + dev: false + + /pg-protocol@1.6.0: + resolution: {integrity: sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==} + + /pg-types@2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} + engines: {node: '>=4'} + dependencies: + pg-int8: 1.0.1 + postgres-array: 2.0.0 + postgres-bytea: 1.0.0 + postgres-date: 1.0.7 + postgres-interval: 1.2.0 + dev: false + + /pg-types@4.0.1: + resolution: {integrity: sha512-hRCSDuLII9/LE3smys1hRHcu5QGcLs9ggT7I/TCs0IE+2Eesxi9+9RWAAwZ0yaGjxoWICF/YHLOEjydGujoJ+g==} + engines: {node: '>=10'} + dependencies: + pg-int8: 1.0.1 + pg-numeric: 1.0.2 + postgres-array: 3.0.2 + postgres-bytea: 3.0.0 + postgres-date: 2.0.1 + postgres-interval: 3.0.0 + postgres-range: 1.1.3 + dev: true + + /pg@8.11.3: + resolution: {integrity: sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==} + engines: {node: '>= 8.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + dependencies: + buffer-writer: 2.0.0 + packet-reader: 1.0.0 + pg-connection-string: 2.6.2 + pg-pool: 3.6.1(pg@8.11.3) + pg-protocol: 1.6.0 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.1.1 + dev: false + + /pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + dependencies: + split2: 4.2.0 + dev: false + /phin@2.9.3: resolution: {integrity: sha512-CzFr90qM24ju5f88quFC/6qohjC144rehe5n6DH900lgXmUe86+xCKc10ev56gRKC4/BkHUoG4uSiQgBiIXwDA==} dev: false @@ -4158,6 +4265,54 @@ packages: source-map-js: 1.0.2 dev: false + /postgres-array@2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} + engines: {node: '>=4'} + dev: false + + /postgres-array@3.0.2: + resolution: {integrity: sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==} + engines: {node: '>=12'} + dev: true + + /postgres-bytea@1.0.0: + resolution: {integrity: sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==} + engines: {node: '>=0.10.0'} + dev: false + + /postgres-bytea@3.0.0: + resolution: {integrity: sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==} + engines: {node: '>= 6'} + dependencies: + obuf: 1.1.2 + dev: true + + /postgres-date@1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} + engines: {node: '>=0.10.0'} + dev: false + + /postgres-date@2.0.1: + resolution: {integrity: sha512-YtMKdsDt5Ojv1wQRvUhnyDJNSr2dGIC96mQVKz7xufp07nfuFONzdaowrMHjlAzY6GDLd4f+LUHHAAM1h4MdUw==} + engines: {node: '>=12'} + dev: true + + /postgres-interval@1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} + engines: {node: '>=0.10.0'} + dependencies: + xtend: 4.0.2 + dev: false + + /postgres-interval@3.0.0: + resolution: {integrity: sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==} + engines: {node: '>=12'} + dev: true + + /postgres-range@1.1.3: + resolution: {integrity: sha512-VdlZoocy5lCP0c/t66xAfclglEapXPCIVhqqJRncYpvbCgImF0w67aPKfbqUMr72tO2k5q0TdTZwCLjPTI6C9g==} + dev: true + /prebuild-install@7.1.1: resolution: {integrity: sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==} engines: {node: '>=10'} @@ -4588,6 +4743,11 @@ packages: engines: {node: '>=0.10.0'} dev: false + /split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + dev: false + /sqlstring@2.3.3: resolution: {integrity: sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==} engines: {node: '>= 0.6'} From 6e7ac81ed48dbc994f777c802dedc79df9d60803 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 1 Nov 2023 23:39:07 -0600 Subject: [PATCH 10/16] chore: organize imports --- backend/app.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/app.ts b/backend/app.ts index 85165c83..02985da3 100644 --- a/backend/app.ts +++ b/backend/app.ts @@ -9,13 +9,13 @@ import { path, isProd } from '@tycrek/joint'; import { epcss } from '@tycrek/express-postcss'; import { log } from './log'; -import { JSONDatabase, ensureFiles } from './sql/json'; import { get } from './data'; import { UserConfig } from './UserConfig'; -import { MySQLDatabase } from './sql/mysql'; -import { buildFrontendRouter } from './routers/_frontend'; import { DBManager } from './sql/database'; +import { JSONDatabase, ensureFiles } from './sql/json'; +import { MySQLDatabase } from './sql/mysql'; import { PostgreSQLDatabase } from './sql/postgres'; +import { buildFrontendRouter } from './routers/_frontend'; /** * Top-level metadata exports From c81541b6679618186ea546f24bdf3753e1af6c77 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 1 Nov 2023 23:44:02 -0600 Subject: [PATCH 11/16] fix: only ensure dir on start; files can be done by JSON DB --- backend/app.ts | 7 ++++--- backend/sql/json.ts | 2 -- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/backend/app.ts b/backend/app.ts index 02985da3..ddd2e067 100644 --- a/backend/app.ts +++ b/backend/app.ts @@ -12,7 +12,7 @@ import { log } from './log'; import { get } from './data'; import { UserConfig } from './UserConfig'; import { DBManager } from './sql/database'; -import { JSONDatabase, ensureFiles } from './sql/json'; +import { JSONDatabase } from './sql/json'; import { MySQLDatabase } from './sql/mysql'; import { PostgreSQLDatabase } from './sql/postgres'; import { buildFrontendRouter } from './routers/_frontend'; @@ -79,8 +79,9 @@ async function main() { App.pkgVersion = pkg.version; - // Ensure data files exist - await ensureFiles(); + // Ensure data directory exists + log.debug('Check data dir') + await fs.ensureDir(path.join('.ass-data')); // Set default server configuration const serverConfig: ServerConfiguration = { diff --git a/backend/sql/json.ts b/backend/sql/json.ts index 6703ed2a..173058f9 100644 --- a/backend/sql/json.ts +++ b/backend/sql/json.ts @@ -58,8 +58,6 @@ export const ensureFiles = (): Promise => new Promise(async (resolve, reje log.debug('Checking data files'); try { - // Create data directory - await fs.ensureDir(path.join('.ass-data')); // * Default files.json await createEmptyJson(PATHS.files, { From ec97d0a3b646d8d6c6004c861082215540c7aa56 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 1 Nov 2023 23:46:46 -0600 Subject: [PATCH 12/16] feat: inform host that DB has not been set --- backend/app.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/app.ts b/backend/app.ts index ddd2e067..df3a7307 100644 --- a/backend/app.ts +++ b/backend/app.ts @@ -131,6 +131,7 @@ async function main() { } } catch (err) { throw new Error(`Failed to configure SQL`); } } else { // default to json database + log.debug('DB not set! Defaulting to JSON'); await DBManager.use(new JSONDatabase()); } From 4457d322053fd4ca73928f5a58a9e49e358f6cfa Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 1 Nov 2023 23:49:39 -0600 Subject: [PATCH 13/16] fix: move semicolons --- frontend/setup.mts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/setup.mts b/frontend/setup.mts index c4372d88..310d3e67 100644 --- a/frontend/setup.mts +++ b/frontend/setup.mts @@ -113,8 +113,8 @@ document.addEventListener('DOMContentLoaded', () => { password: Elements.mySqlPassword.value, database: Elements.mySqlDatabase.value } - } - }; + }; + } } else if (Elements.pgsqlTab.active) { if (Elements.pgsqlHost.value != null && Elements.pgsqlHost.value != '') { config.database = { @@ -126,8 +126,8 @@ document.addEventListener('DOMContentLoaded', () => { password: Elements.pgsqlPassword.value, database: Elements.pgsqlDatabase.value } - } - }; + }; + } } // append rate limit config, if specified From fd578d835902bfcc6d74c5e4bc879c7cfbe656db Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 1 Nov 2023 23:58:50 -0600 Subject: [PATCH 14/16] fix: remove duplicate sections --- backend/routers/api.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/backend/routers/api.ts b/backend/routers/api.ts index e2aa2bc3..d5bad9bd 100644 --- a/backend/routers/api.ts +++ b/backend/routers/api.ts @@ -45,14 +45,9 @@ router.post('/setup', BodyParserJson(), async (req, res) => { } // set rate limits - if (UserConfig.config.rateLimit?.api) setRateLimiter('api', UserConfig.config.rateLimit.api); - if (UserConfig.config.rateLimit?.login) setRateLimiter('login', UserConfig.config.rateLimit.login); - if (UserConfig.config.rateLimit?.upload) setRateLimiter('upload', UserConfig.config.rateLimit.upload); - - // set rate limits - if (UserConfig.config.rateLimit?.api) setRateLimiter('api', UserConfig.config.rateLimit.api); - if (UserConfig.config.rateLimit?.login) setRateLimiter('login', UserConfig.config.rateLimit.login); - if (UserConfig.config.rateLimit?.upload) setRateLimiter('upload', UserConfig.config.rateLimit.upload); + if (UserConfig.config.rateLimit?.api) setRateLimiter('api', UserConfig.config.rateLimit.api); + if (UserConfig.config.rateLimit?.login) setRateLimiter('login', UserConfig.config.rateLimit.login); + if (UserConfig.config.rateLimit?.upload) setRateLimiter('upload', UserConfig.config.rateLimit.upload);; log.success('Setup', 'completed'); From 3e26d4da6591c455936b2a5cfb8d1185e20ea4fd Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 1 Nov 2023 23:59:54 -0600 Subject: [PATCH 15/16] fix: remove unused PATHS variable (moved to other file) --- backend/data.ts | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/backend/data.ts b/backend/data.ts index 72e2f8c0..118105f4 100644 --- a/backend/data.ts +++ b/backend/data.ts @@ -1,7 +1,4 @@ -import { AssFile, AssUser, NID, FilesSchema, UsersSchema } from 'ass'; - -import fs from 'fs-extra'; -import { path } from '@tycrek/joint'; +import { AssFile, AssUser, NID } from 'ass'; import { log } from './log'; import { UserConfig } from './UserConfig'; @@ -12,14 +9,6 @@ import { DBManager } from './sql/database'; */ type DataSector = 'files' | 'users'; -/** - * Absolute filepaths for JSON data files - */ -const PATHS = { - files: path.join('.ass-data/files.json'), - users: path.join('.ass-data/users.json') -}; - /** * database kind -> name mapping */ From 9083df93157292fdc4bab91542d8b5de1422fd29 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 2 Nov 2023 00:26:53 -0600 Subject: [PATCH 16/16] chore: make consistent with "Checking data files" --- backend/app.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app.ts b/backend/app.ts index df3a7307..4c3619b2 100644 --- a/backend/app.ts +++ b/backend/app.ts @@ -80,7 +80,7 @@ async function main() { App.pkgVersion = pkg.version; // Ensure data directory exists - log.debug('Check data dir') + log.debug('Checking data dir') await fs.ensureDir(path.join('.ass-data')); // Set default server configuration