From 4ac866c7b6eab50ac875e71e45de45f2bf9000f5 Mon Sep 17 00:00:00 2001 From: Aitor Algorta Date: Thu, 8 Feb 2024 17:00:29 +0100 Subject: [PATCH 01/19] wip --- .../backend/src/app.ts | 163 ++++++++++++++++++ .../backend/src/package.json | 12 ++ .../backend/src/providers/karapace.ts | 121 +++++++++++++ .../backend/src/tsconfig.json | 10 ++ .../backend/src/types.ts | 8 + 5 files changed, 314 insertions(+) create mode 100644 backend/components/schema-registry-manager/backend/src/app.ts create mode 100644 backend/components/schema-registry-manager/backend/src/package.json create mode 100644 backend/components/schema-registry-manager/backend/src/providers/karapace.ts create mode 100644 backend/components/schema-registry-manager/backend/src/tsconfig.json create mode 100644 backend/components/schema-registry-manager/backend/src/types.ts diff --git a/backend/components/schema-registry-manager/backend/src/app.ts b/backend/components/schema-registry-manager/backend/src/app.ts new file mode 100644 index 000000000..d1eb2e629 --- /dev/null +++ b/backend/components/schema-registry-manager/backend/src/app.ts @@ -0,0 +1,163 @@ + +import dotenv from "dotenv"; +import express, { Express, Request, Response } from "express"; +import { SchemaProvider } from "./types"; +import { createSchema, getSchemaInfo, getSchemaVersions, getSchemas, updateSchema } from "./providers/karapace"; + +dotenv.config(); + +const app: Express = express(); +const port = process.env.PORT || 3000; +const bodyParser = require('body-parser'); +const currentProvider: SchemaProvider = SchemaProvider.karapace; + +// Middleware +app.use(bodyParser.json()); + +app.get('/schemas.provider', (req: Request, res: Response) => { + res.status(200).send(currentProvider); +}); + +app.get('/schemas.list', (req: Request, res: Response) => { + switch (currentProvider) { + case SchemaProvider.karapace: + getSchemas().then((response: string[]) => { + res.send(response); + }).catch((e: any) => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); + +app.get('/schemas.versions', (req: Request, res: Response) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + + switch (currentProvider) { + case SchemaProvider.karapace: + getSchemaVersions(req.query.topicName as string).then((response: any) => { + res.status(200).send(response); + }).catch((e: any) => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); + +app.get('/schemas.info', (req: Request, res: Response) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + + let version = 'latest'; + if (req.query.version) { + version = req.query.version as string; + } + + switch (currentProvider) { + case SchemaProvider.karapace: + getSchemaInfo(req.query.topicName as string, version).then((response: any) => { + res.status(200).send(response); + }).catch((e: any) => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); + +app.post('/schemas.update', (req: Request, res: Response) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + if (!req.body.schema) { + res.status(400).send('Missing schema'); + return; + } + + switch (currentProvider) { + case SchemaProvider.karapace: + updateSchema(req.query.topicName as string, req.body.schema as string).then((response: any) => { + res.status(200).send(response); + }).catch((e: any) => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); + +app.post('/schemas.create', (req: Request, res: Response) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + if (!req.body.schema) { + res.status(400).send('Missing schema'); + return; + } + + switch (currentProvider) { + case SchemaProvider.karapace: + createSchema(req.query.topicName as string, req.body.schema as string).then((response: any) => { + res.status(200).send(response); + }).catch((e: any) => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); + +app.post('/schemas.compatibility', (req: Request, res: Response) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + if (!req.query.version) { + res.status(400).send('Missing version'); + return; + } + if (!req.body.schema) { + res.status(400).send('Missing schema'); + return; + } + + switch (currentProvider) { + case SchemaProvider.karapace: + createSchema(req.query.topicName as string, req.body.schema as string).then((response: any) => { + res.status(200).send(response); + }).catch((e: any) => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); + +async function main() { + app.listen(port, () => { + console.log(`Server is running on http://localhost:${port}`); + }); +} + +main().catch(console.error); \ No newline at end of file diff --git a/backend/components/schema-registry-manager/backend/src/package.json b/backend/components/schema-registry-manager/backend/src/package.json new file mode 100644 index 000000000..00f591b08 --- /dev/null +++ b/backend/components/schema-registry-manager/backend/src/package.json @@ -0,0 +1,12 @@ +{ + "dependencies": { + "@types/express": "^4.17.21", + "@types/node": "^20.10.3", + "node-fetch": "^2.6.1" + }, + "devDependencies": { + "@types/express": "^4.17.21", + "@types/node": "^20.10.3", + "typescript": "^5.3.2" + } +} diff --git a/backend/components/schema-registry-manager/backend/src/providers/karapace.ts b/backend/components/schema-registry-manager/backend/src/providers/karapace.ts new file mode 100644 index 000000000..c8632725c --- /dev/null +++ b/backend/components/schema-registry-manager/backend/src/providers/karapace.ts @@ -0,0 +1,121 @@ +export async function getSchemas() { + return getData('subjects').then(response => { + return response; + }); +}; + +export async function getSchemaVersions(topicName: string) { + return getData(`subjects/${topicName}/versions`).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + return response; + }); +}; + +export async function getSchemaInfo(topicName: string, version: string) { + return getData(`subjects/${topicName}/versions/${version}`).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + return response; + }); +}; + +export async function updateSchema(topicName: string, schema: string) { + const body = { + schema: JSON.stringify({...JSON.parse(schema)}), + }; + return postData(`subjects/${topicName}/versions`, body).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + if (response.id) return response; + if (response.message) return Promise.reject(response.message); + return Promise.reject('Unknown Error'); + }); +}; + +export async function createSchema(topicName: string, schema: string) { + const body = { + schema: JSON.stringify({...JSON.parse(schema)}), + }; + return postData(`subjects/${topicName}/versions`, body) + .then(response => { + if (response.id) return response; + if (response.message) return Promise.reject(response.message); + return Promise.reject('Unknown Error'); + }) + .catch(e => { + return Promise.reject(e); + }); +}; + +export async function checkCompatibilityOfNewSchema(topicName: string, schema: string, version: number) { + const body = { + schema: JSON.stringify({...JSON.parse(schema)}), + }; + return postData(`compatibility/subjects/${topicName}/versions/${version}`, body) + .then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + if (response.is_compatible !== undefined) { + if (response.is_compatible === true) { + return Promise.resolve(true); + } + return Promise.reject('Schema Not Compatible'); + } + if (response.message) return Promise.reject(response.message); + return Promise.reject('Unknown Error'); + }) + .catch(e => { + return Promise.reject(e); + }); +}; + +// export const deleteSchema = (topicName: string) => async () => { +// return deleteData(`subjects/${topicName}`).then(response => { +// if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { +// return Promise.reject('404 Not Found'); +// } +// return Promise.resolve(true); +// }); +// }; + +// export const getLastMessage = (topicName: string) => async (dispatch: Dispatch) => { +// const body = { +// ksql: `PRINT '${topicName}' FROM BEGINNING LIMIT 1;`, +// streamsProperties: {}, +// }; +// return postData('query', body).then(response => { +// dispatch(setLastMessage(response)); +// return Promise.resolve(true); +// }); +// }; + + async function getData(url: string) { + const response = await fetch(process.env.URL + '/' + url, { + method: 'GET', + }); + return response.json(); + } + + async function deleteData(url: string) { + const response = await fetch(process.env.URL + '/' + url, { + method: 'DELETE', + }); + return response.json(); + } + + async function postData(url: string, body: any) { + const response = await fetch(process.env.URL + '/' + url, { + method: 'POST', + headers: { + 'Content-Type': 'application/vnd.schemaregistry.v1+json', + }, + body: JSON.stringify(body), + }); + + return response.json(); + } \ No newline at end of file diff --git a/backend/components/schema-registry-manager/backend/src/tsconfig.json b/backend/components/schema-registry-manager/backend/src/tsconfig.json new file mode 100644 index 000000000..8975f6604 --- /dev/null +++ b/backend/components/schema-registry-manager/backend/src/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "target": "ES2016", + "module": "commonjs", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + } +} \ No newline at end of file diff --git a/backend/components/schema-registry-manager/backend/src/types.ts b/backend/components/schema-registry-manager/backend/src/types.ts new file mode 100644 index 000000000..c4b80a6a5 --- /dev/null +++ b/backend/components/schema-registry-manager/backend/src/types.ts @@ -0,0 +1,8 @@ +export enum SchemaProvider { + karapace = 'karapace', + confluentCloud = 'confluent-cloud', +} + +module.exports = { + SchemaProvider: SchemaProvider +}; \ No newline at end of file From 0be3ca10bad2b7111c47c961e08aff50b236a568 Mon Sep 17 00:00:00 2001 From: Aitor Algorta Date: Thu, 8 Feb 2024 17:15:23 +0100 Subject: [PATCH 02/19] add rest of endpoints from Karapace --- .../backend/src/app.ts | 45 ++++++++++++++++++- .../backend/src/providers/karapace.ts | 40 ++++++++--------- 2 files changed, 63 insertions(+), 22 deletions(-) diff --git a/backend/components/schema-registry-manager/backend/src/app.ts b/backend/components/schema-registry-manager/backend/src/app.ts index d1eb2e629..fbf8e82ae 100644 --- a/backend/components/schema-registry-manager/backend/src/app.ts +++ b/backend/components/schema-registry-manager/backend/src/app.ts @@ -2,7 +2,7 @@ import dotenv from "dotenv"; import express, { Express, Request, Response } from "express"; import { SchemaProvider } from "./types"; -import { createSchema, getSchemaInfo, getSchemaVersions, getSchemas, updateSchema } from "./providers/karapace"; +import { checkCompatibilityOfNewSchema, createSchema, deleteSchema, getLastMessage, getSchemaInfo, getSchemaVersions, getSchemas, updateSchema } from "./providers/karapace"; dotenv.config(); @@ -142,7 +142,48 @@ app.post('/schemas.compatibility', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: - createSchema(req.query.topicName as string, req.body.schema as string).then((response: any) => { + checkCompatibilityOfNewSchema(req.query.topicName as string, req.body.schema as string, req.query.version as string).then((response: any) => { + res.status(200).send(response); + }).catch((e: any) => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); + +app.post('/schemas.delete', (req: Request, res: Response) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + + switch (currentProvider) { + case SchemaProvider.karapace: + deleteSchema(req.query.topicName as string).then((response: any) => { + res.status(200).send(response); + }).catch((e: any) => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); + +app.get('/schemas.lastMessage', (req: Request, res: Response) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + + switch (currentProvider) { + case SchemaProvider.karapace: + getLastMessage(req.query.topicName as string).then((response: any) => { + console.log(response); res.status(200).send(response); }).catch((e: any) => { res.status(500).send(e); diff --git a/backend/components/schema-registry-manager/backend/src/providers/karapace.ts b/backend/components/schema-registry-manager/backend/src/providers/karapace.ts index c8632725c..0885ca6aa 100644 --- a/backend/components/schema-registry-manager/backend/src/providers/karapace.ts +++ b/backend/components/schema-registry-manager/backend/src/providers/karapace.ts @@ -51,7 +51,7 @@ export async function createSchema(topicName: string, schema: string) { }); }; -export async function checkCompatibilityOfNewSchema(topicName: string, schema: string, version: number) { +export async function checkCompatibilityOfNewSchema(topicName: string, schema: string, version: string) { const body = { schema: JSON.stringify({...JSON.parse(schema)}), }; @@ -62,7 +62,7 @@ export async function checkCompatibilityOfNewSchema(topicName: string, schema: s } if (response.is_compatible !== undefined) { if (response.is_compatible === true) { - return Promise.resolve(true); + return response; } return Promise.reject('Schema Not Compatible'); } @@ -74,25 +74,25 @@ export async function checkCompatibilityOfNewSchema(topicName: string, schema: s }); }; -// export const deleteSchema = (topicName: string) => async () => { -// return deleteData(`subjects/${topicName}`).then(response => { -// if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { -// return Promise.reject('404 Not Found'); -// } -// return Promise.resolve(true); -// }); -// }; +export async function deleteSchema(topicName: string) { + return deleteData(`subjects/${topicName}`).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + return response; + }); +}; -// export const getLastMessage = (topicName: string) => async (dispatch: Dispatch) => { -// const body = { -// ksql: `PRINT '${topicName}' FROM BEGINNING LIMIT 1;`, -// streamsProperties: {}, -// }; -// return postData('query', body).then(response => { -// dispatch(setLastMessage(response)); -// return Promise.resolve(true); -// }); -// }; +export async function getLastMessage(topicName: string) { + const body = { + ksql: `PRINT '${topicName}' FROM BEGINNING LIMIT 1;`, + streamsProperties: {}, + }; + return postData('query', body).then(response => { + console.log(response); + return response; + }); +}; async function getData(url: string) { const response = await fetch(process.env.URL + '/' + url, { From 1ec3701072bb7ce91e3b22bebca73ac2dcc6497e Mon Sep 17 00:00:00 2001 From: Aitor Algorta Date: Fri, 9 Feb 2024 09:35:34 +0100 Subject: [PATCH 03/19] lint --- .../backend/src/app.ts | 156 +++++++++++------- .../backend/src/providers/karapace.ts | 86 +++++----- .../backend/src/types.ts | 6 +- 3 files changed, 138 insertions(+), 110 deletions(-) diff --git a/backend/components/schema-registry-manager/backend/src/app.ts b/backend/components/schema-registry-manager/backend/src/app.ts index fbf8e82ae..00857d77a 100644 --- a/backend/components/schema-registry-manager/backend/src/app.ts +++ b/backend/components/schema-registry-manager/backend/src/app.ts @@ -1,8 +1,16 @@ - -import dotenv from "dotenv"; -import express, { Express, Request, Response } from "express"; -import { SchemaProvider } from "./types"; -import { checkCompatibilityOfNewSchema, createSchema, deleteSchema, getLastMessage, getSchemaInfo, getSchemaVersions, getSchemas, updateSchema } from "./providers/karapace"; +import dotenv from 'dotenv'; +import express, {Express, Request, Response} from 'express'; +import {SchemaProvider} from './types'; +import { + checkCompatibilityOfNewSchema, + createSchema, + deleteSchema, + getLastMessage, + getSchemaInfo, + getSchemaVersions, + getSchemas, + updateSchema, +} from './providers/karapace'; dotenv.config(); @@ -15,22 +23,24 @@ const currentProvider: SchemaProvider = SchemaProvider.karapace; app.use(bodyParser.json()); app.get('/schemas.provider', (req: Request, res: Response) => { - res.status(200).send(currentProvider); + res.status(200).send(currentProvider); }); app.get('/schemas.list', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: - getSchemas().then((response: string[]) => { - res.send(response); - }).catch((e: any) => { - res.status(500).send(e); - }); - break; + getSchemas() + .then((response: string[]) => { + res.send(response); + }) + .catch((e: any) => { + res.status(500).send(e); + }); + break; default: res.status(404).send('Provider Not Found'); break; - } + } }); app.get('/schemas.versions', (req: Request, res: Response) => { @@ -41,16 +51,18 @@ app.get('/schemas.versions', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: - getSchemaVersions(req.query.topicName as string).then((response: any) => { - res.status(200).send(response); - }).catch((e: any) => { - res.status(500).send(e); - }); - break; + getSchemaVersions(req.query.topicName as string) + .then((response: any) => { + res.status(200).send(response); + }) + .catch((e: any) => { + res.status(500).send(e); + }); + break; default: res.status(404).send('Provider Not Found'); break; - } + } }); app.get('/schemas.info', (req: Request, res: Response) => { @@ -66,16 +78,18 @@ app.get('/schemas.info', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: - getSchemaInfo(req.query.topicName as string, version).then((response: any) => { - res.status(200).send(response); - }).catch((e: any) => { - res.status(500).send(e); - }); - break; + getSchemaInfo(req.query.topicName as string, version) + .then((response: any) => { + res.status(200).send(response); + }) + .catch((e: any) => { + res.status(500).send(e); + }); + break; default: res.status(404).send('Provider Not Found'); break; - } + } }); app.post('/schemas.update', (req: Request, res: Response) => { @@ -90,16 +104,18 @@ app.post('/schemas.update', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: - updateSchema(req.query.topicName as string, req.body.schema as string).then((response: any) => { - res.status(200).send(response); - }).catch((e: any) => { - res.status(500).send(e); - }); - break; + updateSchema(req.query.topicName as string, req.body.schema as string) + .then((response: any) => { + res.status(200).send(response); + }) + .catch((e: any) => { + res.status(500).send(e); + }); + break; default: res.status(404).send('Provider Not Found'); break; - } + } }); app.post('/schemas.create', (req: Request, res: Response) => { @@ -114,16 +130,18 @@ app.post('/schemas.create', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: - createSchema(req.query.topicName as string, req.body.schema as string).then((response: any) => { - res.status(200).send(response); - }).catch((e: any) => { - res.status(500).send(e); - }); - break; + createSchema(req.query.topicName as string, req.body.schema as string) + .then((response: any) => { + res.status(200).send(response); + }) + .catch((e: any) => { + res.status(500).send(e); + }); + break; default: res.status(404).send('Provider Not Found'); break; - } + } }); app.post('/schemas.compatibility', (req: Request, res: Response) => { @@ -142,16 +160,22 @@ app.post('/schemas.compatibility', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: - checkCompatibilityOfNewSchema(req.query.topicName as string, req.body.schema as string, req.query.version as string).then((response: any) => { - res.status(200).send(response); - }).catch((e: any) => { - res.status(500).send(e); - }); - break; + checkCompatibilityOfNewSchema( + req.query.topicName as string, + req.body.schema as string, + req.query.version as string + ) + .then((response: any) => { + res.status(200).send(response); + }) + .catch((e: any) => { + res.status(500).send(e); + }); + break; default: res.status(404).send('Provider Not Found'); break; - } + } }); app.post('/schemas.delete', (req: Request, res: Response) => { @@ -162,16 +186,18 @@ app.post('/schemas.delete', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: - deleteSchema(req.query.topicName as string).then((response: any) => { - res.status(200).send(response); - }).catch((e: any) => { - res.status(500).send(e); - }); - break; + deleteSchema(req.query.topicName as string) + .then((response: any) => { + res.status(200).send(response); + }) + .catch((e: any) => { + res.status(500).send(e); + }); + break; default: res.status(404).send('Provider Not Found'); break; - } + } }); app.get('/schemas.lastMessage', (req: Request, res: Response) => { @@ -182,17 +208,19 @@ app.get('/schemas.lastMessage', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: - getLastMessage(req.query.topicName as string).then((response: any) => { - console.log(response); - res.status(200).send(response); - }).catch((e: any) => { - res.status(500).send(e); - }); - break; + getLastMessage(req.query.topicName as string) + .then((response: any) => { + console.log(response); + res.status(200).send(response); + }) + .catch((e: any) => { + res.status(500).send(e); + }); + break; default: res.status(404).send('Provider Not Found'); break; - } + } }); async function main() { @@ -201,4 +229,4 @@ async function main() { }); } -main().catch(console.error); \ No newline at end of file +main().catch(console.error); diff --git a/backend/components/schema-registry-manager/backend/src/providers/karapace.ts b/backend/components/schema-registry-manager/backend/src/providers/karapace.ts index 0885ca6aa..1f1792dd1 100644 --- a/backend/components/schema-registry-manager/backend/src/providers/karapace.ts +++ b/backend/components/schema-registry-manager/backend/src/providers/karapace.ts @@ -1,27 +1,27 @@ export async function getSchemas() { - return getData('subjects').then(response => { + return getData('subjects').then(response => { return response; }); -}; - +} + export async function getSchemaVersions(topicName: string) { return getData(`subjects/${topicName}/versions`).then(response => { if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { return Promise.reject('404 Not Found'); } - return response; + return response; }); -}; +} -export async function getSchemaInfo(topicName: string, version: string) { +export async function getSchemaInfo(topicName: string, version: string) { return getData(`subjects/${topicName}/versions/${version}`).then(response => { if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { return Promise.reject('404 Not Found'); } return response; }); -}; - +} + export async function updateSchema(topicName: string, schema: string) { const body = { schema: JSON.stringify({...JSON.parse(schema)}), @@ -34,8 +34,8 @@ export async function updateSchema(topicName: string, schema: string) { if (response.message) return Promise.reject(response.message); return Promise.reject('Unknown Error'); }); -}; - +} + export async function createSchema(topicName: string, schema: string) { const body = { schema: JSON.stringify({...JSON.parse(schema)}), @@ -49,8 +49,8 @@ export async function createSchema(topicName: string, schema: string) { .catch(e => { return Promise.reject(e); }); -}; - +} + export async function checkCompatibilityOfNewSchema(topicName: string, schema: string, version: string) { const body = { schema: JSON.stringify({...JSON.parse(schema)}), @@ -72,8 +72,8 @@ export async function checkCompatibilityOfNewSchema(topicName: string, schema: s .catch(e => { return Promise.reject(e); }); -}; - +} + export async function deleteSchema(topicName: string) { return deleteData(`subjects/${topicName}`).then(response => { if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { @@ -81,8 +81,8 @@ export async function deleteSchema(topicName: string) { } return response; }); -}; - +} + export async function getLastMessage(topicName: string) { const body = { ksql: `PRINT '${topicName}' FROM BEGINNING LIMIT 1;`, @@ -92,30 +92,30 @@ export async function getLastMessage(topicName: string) { console.log(response); return response; }); -}; - - async function getData(url: string) { - const response = await fetch(process.env.URL + '/' + url, { - method: 'GET', - }); - return response.json(); - } - - async function deleteData(url: string) { - const response = await fetch(process.env.URL + '/' + url, { - method: 'DELETE', - }); - return response.json(); - } - - async function postData(url: string, body: any) { - const response = await fetch(process.env.URL + '/' + url, { - method: 'POST', - headers: { - 'Content-Type': 'application/vnd.schemaregistry.v1+json', - }, - body: JSON.stringify(body), - }); - - return response.json(); - } \ No newline at end of file +} + +async function getData(url: string) { + const response = await fetch(process.env.URL + '/' + url, { + method: 'GET', + }); + return response.json(); +} + +async function deleteData(url: string) { + const response = await fetch(process.env.URL + '/' + url, { + method: 'DELETE', + }); + return response.json(); +} + +async function postData(url: string, body: any) { + const response = await fetch(process.env.URL + '/' + url, { + method: 'POST', + headers: { + 'Content-Type': 'application/vnd.schemaregistry.v1+json', + }, + body: JSON.stringify(body), + }); + + return response.json(); +} diff --git a/backend/components/schema-registry-manager/backend/src/types.ts b/backend/components/schema-registry-manager/backend/src/types.ts index c4b80a6a5..b336be9b7 100644 --- a/backend/components/schema-registry-manager/backend/src/types.ts +++ b/backend/components/schema-registry-manager/backend/src/types.ts @@ -1,8 +1,8 @@ export enum SchemaProvider { - karapace = 'karapace', + karapace = 'karapace', confluentCloud = 'confluent-cloud', } module.exports = { - SchemaProvider: SchemaProvider -}; \ No newline at end of file + SchemaProvider: SchemaProvider, +}; From 4a7ccd36f286386e82cfba0acb59836504841341 Mon Sep 17 00:00:00 2001 From: Aitor Algorta Date: Fri, 9 Feb 2024 09:39:30 +0100 Subject: [PATCH 04/19] remove unnecessary import --- .../components/schema-registry-manager/backend/src/types.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/backend/components/schema-registry-manager/backend/src/types.ts b/backend/components/schema-registry-manager/backend/src/types.ts index b336be9b7..53776abb6 100644 --- a/backend/components/schema-registry-manager/backend/src/types.ts +++ b/backend/components/schema-registry-manager/backend/src/types.ts @@ -2,7 +2,3 @@ export enum SchemaProvider { karapace = 'karapace', confluentCloud = 'confluent-cloud', } - -module.exports = { - SchemaProvider: SchemaProvider, -}; From 551539cd2e0753c178c66c6cac81126807842328 Mon Sep 17 00:00:00 2001 From: Aitor Algorta Date: Fri, 9 Feb 2024 12:43:08 +0100 Subject: [PATCH 05/19] added Dockerfile and Makefile" --- .../schema-registry-manager/Dockerfile | 19 ++++++++ .../schema-registry-manager/Makefile | 6 +++ .../schema-registry-manager/helm/BUILD | 32 +++++++++++++ .../schema-registry-manager/helm/Chart.yaml | 5 ++ .../helm/templates/configmap.yaml | 10 ++++ .../helm/templates/deployment.yaml | 48 +++++++++++++++++++ .../helm/templates/service.yaml | 15 ++++++ .../schema-registry-manager/helm/values.yaml | 9 ++++ .../{backend => }/src/app.ts | 0 .../{backend => }/src/package.json | 0 .../{backend => }/src/providers/karapace.ts | 0 .../{backend => }/src/tsconfig.json | 0 .../{backend => }/src/types.ts | 0 13 files changed, 144 insertions(+) create mode 100644 backend/components/schema-registry-manager/Dockerfile create mode 100644 backend/components/schema-registry-manager/Makefile create mode 100644 backend/components/schema-registry-manager/helm/BUILD create mode 100644 backend/components/schema-registry-manager/helm/Chart.yaml create mode 100644 backend/components/schema-registry-manager/helm/templates/configmap.yaml create mode 100644 backend/components/schema-registry-manager/helm/templates/deployment.yaml create mode 100644 backend/components/schema-registry-manager/helm/templates/service.yaml create mode 100644 backend/components/schema-registry-manager/helm/values.yaml rename backend/components/schema-registry-manager/{backend => }/src/app.ts (100%) rename backend/components/schema-registry-manager/{backend => }/src/package.json (100%) rename backend/components/schema-registry-manager/{backend => }/src/providers/karapace.ts (100%) rename backend/components/schema-registry-manager/{backend => }/src/tsconfig.json (100%) rename backend/components/schema-registry-manager/{backend => }/src/types.ts (100%) diff --git a/backend/components/schema-registry-manager/Dockerfile b/backend/components/schema-registry-manager/Dockerfile new file mode 100644 index 000000000..49fdcebab --- /dev/null +++ b/backend/components/schema-registry-manager/Dockerfile @@ -0,0 +1,19 @@ +FROM node:18 + +WORKDIR /app + +COPY ./src/package*.json ./ + +COPY ./src/tsconfig*.json ./ + +RUN npm install + +RUN npm install -g typescript + +RUN npm install -g ts-node + +COPY ./src/app.ts ./ + +EXPOSE 3000 + +CMD [ "ts-node", "app.ts" ] diff --git a/backend/components/schema-registry-manager/Makefile b/backend/components/schema-registry-manager/Makefile new file mode 100644 index 000000000..3aa78401e --- /dev/null +++ b/backend/components/schema-registry-manager/Makefile @@ -0,0 +1,6 @@ +build: + docker build -t schema-registry-manager . + +release: build + docker tag schema-registry-manager ghcr.io/airyhq/backend/schema-registry-manager:release + docker push ghcr.io/airyhq/backend/schema-registry-manager:release diff --git a/backend/components/schema-registry-manager/helm/BUILD b/backend/components/schema-registry-manager/helm/BUILD new file mode 100644 index 000000000..5bb7f8f59 --- /dev/null +++ b/backend/components/schema-registry-manager/helm/BUILD @@ -0,0 +1,32 @@ +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@com_github_airyhq_bazel_tools//helm:helm.bzl", "helm_template_test") +load("//tools/build:helm.bzl", "helm_push_develop", "helm_push_release") + +filegroup( + name = "files", + srcs = glob( + ["**/*"], + exclude = ["BUILD"], + ), + visibility = ["//visibility:public"], +) + +pkg_tar( + name = "package", + srcs = [":files"], + extension = "tgz", + strip_prefix = "./", +) + +helm_template_test( + name = "template", + chart = ":package", +) + +helm_push_develop( + chart = ":package", +) + +helm_push_release( + chart = ":package", +) diff --git a/backend/components/schema-registry-manager/helm/Chart.yaml b/backend/components/schema-registry-manager/helm/Chart.yaml new file mode 100644 index 000000000..7205b553c --- /dev/null +++ b/backend/components/schema-registry-manager/helm/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +appVersion: "1.0" +description: Schema registry component to manage different providers +name: schema-registry-manager +version: 1.0 diff --git a/backend/components/schema-registry-manager/helm/templates/configmap.yaml b/backend/components/schema-registry-manager/helm/templates/configmap.yaml new file mode 100644 index 000000000..05de4d589 --- /dev/null +++ b/backend/components/schema-registry-manager/helm/templates/configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.component }} + labels: + core.airy.co/managed: "true" + core.airy.co/mandatory: "{{ .Values.mandatory }}" + core.airy.co/component: "{{ .Values.component }}" + annotations: + core.airy.co/enabled: "{{ .Values.enabled }}" diff --git a/backend/components/schema-registry-manager/helm/templates/deployment.yaml b/backend/components/schema-registry-manager/helm/templates/deployment.yaml new file mode 100644 index 000000000..b90214517 --- /dev/null +++ b/backend/components/schema-registry-manager/helm/templates/deployment.yaml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.component }} + labels: + app: {{ .Values.component }} + core.airy.co/managed: "true" + core.airy.co/mandatory: "{{ .Values.mandatory }}" + core.airy.co/component: {{ .Values.component }} +spec: + replicas: {{ if .Values.enabled }} 1 {{ else }} 0 {{ end }} + selector: + matchLabels: + app: {{ .Values.component }} + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: {{ .Values.component }} + spec: + containers: + - name: app + image: "ghcr.io/airyhq/{{ .Values.image }}:{{ .Values.imageTag }}" + imagePullPolicy: Always + envFrom: + - configMapRef: + name: security + - configMapRef: + name: kafka-config + - configMapRef: + name: {{ .Values.component }} + env: + - name: KAFKA_TOPIC_NAME + value: {{ .Values.kafka.topic }} + livenessProbe: + httpGet: + path: /actuator/health + port: {{ .Values.port }} + httpHeaders: + - name: Health-Check + value: health-check + initialDelaySeconds: 43200 + periodSeconds: 10 + failureThreshold: 3 diff --git a/backend/components/schema-registry-manager/helm/templates/service.yaml b/backend/components/schema-registry-manager/helm/templates/service.yaml new file mode 100644 index 000000000..4d636e8b2 --- /dev/null +++ b/backend/components/schema-registry-manager/helm/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.component }} + labels: + app: {{ .Values.component }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: {{ .Values.component }} + port: 80 + targetPort: {{ .Values.port }} + selector: + app: {{ .Values.component }} diff --git a/backend/components/schema-registry-manager/helm/values.yaml b/backend/components/schema-registry-manager/helm/values.yaml new file mode 100644 index 000000000..200ef3ca3 --- /dev/null +++ b/backend/components/schema-registry-manager/helm/values.yaml @@ -0,0 +1,9 @@ +component: schema-registry-manager +mandatory: false +enabled: false +image: backend/schema-registry-manager +imageTag: release +port: 3000 +resources: +kafka: + topic: application.communication.messages \ No newline at end of file diff --git a/backend/components/schema-registry-manager/backend/src/app.ts b/backend/components/schema-registry-manager/src/app.ts similarity index 100% rename from backend/components/schema-registry-manager/backend/src/app.ts rename to backend/components/schema-registry-manager/src/app.ts diff --git a/backend/components/schema-registry-manager/backend/src/package.json b/backend/components/schema-registry-manager/src/package.json similarity index 100% rename from backend/components/schema-registry-manager/backend/src/package.json rename to backend/components/schema-registry-manager/src/package.json diff --git a/backend/components/schema-registry-manager/backend/src/providers/karapace.ts b/backend/components/schema-registry-manager/src/providers/karapace.ts similarity index 100% rename from backend/components/schema-registry-manager/backend/src/providers/karapace.ts rename to backend/components/schema-registry-manager/src/providers/karapace.ts diff --git a/backend/components/schema-registry-manager/backend/src/tsconfig.json b/backend/components/schema-registry-manager/src/tsconfig.json similarity index 100% rename from backend/components/schema-registry-manager/backend/src/tsconfig.json rename to backend/components/schema-registry-manager/src/tsconfig.json diff --git a/backend/components/schema-registry-manager/backend/src/types.ts b/backend/components/schema-registry-manager/src/types.ts similarity index 100% rename from backend/components/schema-registry-manager/backend/src/types.ts rename to backend/components/schema-registry-manager/src/types.ts From 1ba800798216058453b0e7368d3b846137392fe9 Mon Sep 17 00:00:00 2001 From: Aitor Algorta Date: Mon, 12 Feb 2024 15:15:43 +0100 Subject: [PATCH 06/19] general improvements --- .../schema-registry-manager/Dockerfile | 12 +- .../schema-registry-manager/src/app.js | 237 ++++++++++++++++++ .../schema-registry-manager/src/package.json | 9 +- .../src/providers/karapace.js | 182 ++++++++++++++ .../schema-registry-manager/src/types.js | 8 + 5 files changed, 439 insertions(+), 9 deletions(-) create mode 100644 backend/components/schema-registry-manager/src/app.js create mode 100644 backend/components/schema-registry-manager/src/providers/karapace.js create mode 100644 backend/components/schema-registry-manager/src/types.js diff --git a/backend/components/schema-registry-manager/Dockerfile b/backend/components/schema-registry-manager/Dockerfile index 49fdcebab..e06995486 100644 --- a/backend/components/schema-registry-manager/Dockerfile +++ b/backend/components/schema-registry-manager/Dockerfile @@ -3,17 +3,17 @@ FROM node:18 WORKDIR /app COPY ./src/package*.json ./ - COPY ./src/tsconfig*.json ./ RUN npm install - -RUN npm install -g typescript - -RUN npm install -g ts-node +RUN npm install typescript -g COPY ./src/app.ts ./ +COPY ./src/types.ts ./ +COPY ./src/providers/karapace.ts ./providers/ + +RUN tsc EXPOSE 3000 -CMD [ "ts-node", "app.ts" ] +CMD [ "node", "app.js" ] diff --git a/backend/components/schema-registry-manager/src/app.js b/backend/components/schema-registry-manager/src/app.js new file mode 100644 index 000000000..4102e6203 --- /dev/null +++ b/backend/components/schema-registry-manager/src/app.js @@ -0,0 +1,237 @@ +'use strict'; +var __awaiter = + (this && this.__awaiter) || + function (thisArg, _arguments, P, generator) { + function adopt(value) { + return value instanceof P + ? value + : new P(function (resolve) { + resolve(value); + }); + } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { + try { + step(generator.next(value)); + } catch (e) { + reject(e); + } + } + function rejected(value) { + try { + step(generator['throw'](value)); + } catch (e) { + reject(e); + } + } + function step(result) { + result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); + } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); + }; +var __importDefault = + (this && this.__importDefault) || + function (mod) { + return mod && mod.__esModule ? mod : {default: mod}; + }; +Object.defineProperty(exports, '__esModule', {value: true}); +const dotenv_1 = __importDefault(require('dotenv')); +const express_1 = __importDefault(require('express')); +const types_1 = require('./types'); +const karapace_1 = require('./providers/karapace'); +dotenv_1.default.config(); +const app = (0, express_1.default)(); +const port = process.env.PORT || 3000; +const bodyParser = require('body-parser'); +const currentProvider = types_1.SchemaProvider.karapace; +// Middleware +app.use(bodyParser.json()); +app.get('/schemas.provider', (req, res) => { + res.status(200).send(currentProvider); +}); +app.get('/schemas.list', (req, res) => { + switch (currentProvider) { + case types_1.SchemaProvider.karapace: + (0, karapace_1.getSchemas)() + .then(response => { + res.send(response); + }) + .catch(e => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); +app.get('/schemas.versions', (req, res) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + switch (currentProvider) { + case types_1.SchemaProvider.karapace: + (0, karapace_1.getSchemaVersions)(req.query.topicName) + .then(response => { + res.status(200).send(response); + }) + .catch(e => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); +app.get('/schemas.info', (req, res) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + let version = 'latest'; + if (req.query.version) { + version = req.query.version; + } + switch (currentProvider) { + case types_1.SchemaProvider.karapace: + (0, karapace_1.getSchemaInfo)(req.query.topicName, version) + .then(response => { + res.status(200).send(response); + }) + .catch(e => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); +app.post('/schemas.update', (req, res) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + if (!req.body.schema) { + res.status(400).send('Missing schema'); + return; + } + switch (currentProvider) { + case types_1.SchemaProvider.karapace: + (0, karapace_1.updateSchema)(req.query.topicName, req.body.schema) + .then(response => { + res.status(200).send(response); + }) + .catch(e => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); +app.post('/schemas.create', (req, res) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + if (!req.body.schema) { + res.status(400).send('Missing schema'); + return; + } + switch (currentProvider) { + case types_1.SchemaProvider.karapace: + (0, karapace_1.createSchema)(req.query.topicName, req.body.schema) + .then(response => { + res.status(200).send(response); + }) + .catch(e => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); +app.post('/schemas.compatibility', (req, res) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + if (!req.query.version) { + res.status(400).send('Missing version'); + return; + } + if (!req.body.schema) { + res.status(400).send('Missing schema'); + return; + } + switch (currentProvider) { + case types_1.SchemaProvider.karapace: + (0, karapace_1.checkCompatibilityOfNewSchema)(req.query.topicName, req.body.schema, req.query.version) + .then(response => { + res.status(200).send(response); + }) + .catch(e => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); +app.post('/schemas.delete', (req, res) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + switch (currentProvider) { + case types_1.SchemaProvider.karapace: + (0, karapace_1.deleteSchema)(req.query.topicName) + .then(response => { + res.status(200).send(response); + }) + .catch(e => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); +app.get('/schemas.lastMessage', (req, res) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + switch (currentProvider) { + case types_1.SchemaProvider.karapace: + (0, karapace_1.getLastMessage)(req.query.topicName) + .then(response => { + console.log(response); + res.status(200).send(response); + }) + .catch(e => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); +function main() { + return __awaiter(this, void 0, void 0, function* () { + app.listen(port, () => { + console.log(`Server is running on http://localhost:${port}`); + }); + }); +} +main().catch(console.error); diff --git a/backend/components/schema-registry-manager/src/package.json b/backend/components/schema-registry-manager/src/package.json index 00f591b08..1cdd291b3 100644 --- a/backend/components/schema-registry-manager/src/package.json +++ b/backend/components/schema-registry-manager/src/package.json @@ -1,12 +1,15 @@ { - "dependencies": { + "dependencies": { "@types/express": "^4.17.21", "@types/node": "^20.10.3", - "node-fetch": "^2.6.1" + "dotenv": "^16.4.2", + "express": "^4.18.2", + "node-fetch": "^2.6.1" }, "devDependencies": { "@types/express": "^4.17.21", "@types/node": "^20.10.3", - "typescript": "^5.3.2" + "ts-node": "^10.9.2", + "typescript": "^5.3.3" } } diff --git a/backend/components/schema-registry-manager/src/providers/karapace.js b/backend/components/schema-registry-manager/src/providers/karapace.js new file mode 100644 index 000000000..32c74cddf --- /dev/null +++ b/backend/components/schema-registry-manager/src/providers/karapace.js @@ -0,0 +1,182 @@ +'use strict'; +var __awaiter = + (this && this.__awaiter) || + function (thisArg, _arguments, P, generator) { + function adopt(value) { + return value instanceof P + ? value + : new P(function (resolve) { + resolve(value); + }); + } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { + try { + step(generator.next(value)); + } catch (e) { + reject(e); + } + } + function rejected(value) { + try { + step(generator['throw'](value)); + } catch (e) { + reject(e); + } + } + function step(result) { + result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); + } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); + }; +Object.defineProperty(exports, '__esModule', {value: true}); +exports.getLastMessage = + exports.deleteSchema = + exports.checkCompatibilityOfNewSchema = + exports.createSchema = + exports.updateSchema = + exports.getSchemaInfo = + exports.getSchemaVersions = + exports.getSchemas = + void 0; +function getSchemas() { + return __awaiter(this, void 0, void 0, function* () { + return getData('subjects').then(response => { + return response; + }); + }); +} +exports.getSchemas = getSchemas; +function getSchemaVersions(topicName) { + return __awaiter(this, void 0, void 0, function* () { + return getData(`subjects/${topicName}/versions`).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + return response; + }); + }); +} +exports.getSchemaVersions = getSchemaVersions; +function getSchemaInfo(topicName, version) { + return __awaiter(this, void 0, void 0, function* () { + return getData(`subjects/${topicName}/versions/${version}`).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + return response; + }); + }); +} +exports.getSchemaInfo = getSchemaInfo; +function updateSchema(topicName, schema) { + return __awaiter(this, void 0, void 0, function* () { + const body = { + schema: JSON.stringify(Object.assign({}, JSON.parse(schema))), + }; + return postData(`subjects/${topicName}/versions`, body).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + if (response.id) return response; + if (response.message) return Promise.reject(response.message); + return Promise.reject('Unknown Error'); + }); + }); +} +exports.updateSchema = updateSchema; +function createSchema(topicName, schema) { + return __awaiter(this, void 0, void 0, function* () { + const body = { + schema: JSON.stringify(Object.assign({}, JSON.parse(schema))), + }; + return postData(`subjects/${topicName}/versions`, body) + .then(response => { + if (response.id) return response; + if (response.message) return Promise.reject(response.message); + return Promise.reject('Unknown Error'); + }) + .catch(e => { + return Promise.reject(e); + }); + }); +} +exports.createSchema = createSchema; +function checkCompatibilityOfNewSchema(topicName, schema, version) { + return __awaiter(this, void 0, void 0, function* () { + const body = { + schema: JSON.stringify(Object.assign({}, JSON.parse(schema))), + }; + return postData(`compatibility/subjects/${topicName}/versions/${version}`, body) + .then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + if (response.is_compatible !== undefined) { + if (response.is_compatible === true) { + return response; + } + return Promise.reject('Schema Not Compatible'); + } + if (response.message) return Promise.reject(response.message); + return Promise.reject('Unknown Error'); + }) + .catch(e => { + return Promise.reject(e); + }); + }); +} +exports.checkCompatibilityOfNewSchema = checkCompatibilityOfNewSchema; +function deleteSchema(topicName) { + return __awaiter(this, void 0, void 0, function* () { + return deleteData(`subjects/${topicName}`).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + return response; + }); + }); +} +exports.deleteSchema = deleteSchema; +function getLastMessage(topicName) { + return __awaiter(this, void 0, void 0, function* () { + const body = { + ksql: `PRINT '${topicName}' FROM BEGINNING LIMIT 1;`, + streamsProperties: {}, + }; + return postData('query', body).then(response => { + console.log(response); + return response; + }); + }); +} +exports.getLastMessage = getLastMessage; +function getData(url) { + return __awaiter(this, void 0, void 0, function* () { + const response = yield fetch(process.env.URL + '/' + url, { + method: 'GET', + }); + return response.json(); + }); +} +function deleteData(url) { + return __awaiter(this, void 0, void 0, function* () { + const response = yield fetch(process.env.URL + '/' + url, { + method: 'DELETE', + }); + return response.json(); + }); +} +function postData(url, body) { + return __awaiter(this, void 0, void 0, function* () { + const response = yield fetch(process.env.URL + '/' + url, { + method: 'POST', + headers: { + 'Content-Type': 'application/vnd.schemaregistry.v1+json', + }, + body: JSON.stringify(body), + }); + return response.json(); + }); +} diff --git a/backend/components/schema-registry-manager/src/types.js b/backend/components/schema-registry-manager/src/types.js new file mode 100644 index 000000000..3571994e8 --- /dev/null +++ b/backend/components/schema-registry-manager/src/types.js @@ -0,0 +1,8 @@ +'use strict'; +Object.defineProperty(exports, '__esModule', {value: true}); +exports.SchemaProvider = void 0; +var SchemaProvider; +(function (SchemaProvider) { + SchemaProvider['karapace'] = 'karapace'; + SchemaProvider['confluentCloud'] = 'confluent-cloud'; +})(SchemaProvider || (exports.SchemaProvider = SchemaProvider = {})); From 1df0d50f58d4c8d1da8f13cc7964f4a95b60861b Mon Sep 17 00:00:00 2001 From: Aitor Algorta Date: Mon, 12 Feb 2024 15:33:38 +0100 Subject: [PATCH 07/19] remove js --- .../schema-registry-manager/src/app.js | 237 ------------------ .../src/providers/karapace.js | 182 -------------- .../schema-registry-manager/src/types.js | 8 - 3 files changed, 427 deletions(-) delete mode 100644 backend/components/schema-registry-manager/src/app.js delete mode 100644 backend/components/schema-registry-manager/src/providers/karapace.js delete mode 100644 backend/components/schema-registry-manager/src/types.js diff --git a/backend/components/schema-registry-manager/src/app.js b/backend/components/schema-registry-manager/src/app.js deleted file mode 100644 index 4102e6203..000000000 --- a/backend/components/schema-registry-manager/src/app.js +++ /dev/null @@ -1,237 +0,0 @@ -'use strict'; -var __awaiter = - (this && this.__awaiter) || - function (thisArg, _arguments, P, generator) { - function adopt(value) { - return value instanceof P - ? value - : new P(function (resolve) { - resolve(value); - }); - } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { - try { - step(generator.next(value)); - } catch (e) { - reject(e); - } - } - function rejected(value) { - try { - step(generator['throw'](value)); - } catch (e) { - reject(e); - } - } - function step(result) { - result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); - } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); - }; -var __importDefault = - (this && this.__importDefault) || - function (mod) { - return mod && mod.__esModule ? mod : {default: mod}; - }; -Object.defineProperty(exports, '__esModule', {value: true}); -const dotenv_1 = __importDefault(require('dotenv')); -const express_1 = __importDefault(require('express')); -const types_1 = require('./types'); -const karapace_1 = require('./providers/karapace'); -dotenv_1.default.config(); -const app = (0, express_1.default)(); -const port = process.env.PORT || 3000; -const bodyParser = require('body-parser'); -const currentProvider = types_1.SchemaProvider.karapace; -// Middleware -app.use(bodyParser.json()); -app.get('/schemas.provider', (req, res) => { - res.status(200).send(currentProvider); -}); -app.get('/schemas.list', (req, res) => { - switch (currentProvider) { - case types_1.SchemaProvider.karapace: - (0, karapace_1.getSchemas)() - .then(response => { - res.send(response); - }) - .catch(e => { - res.status(500).send(e); - }); - break; - default: - res.status(404).send('Provider Not Found'); - break; - } -}); -app.get('/schemas.versions', (req, res) => { - if (!req.query.topicName) { - res.status(400).send('Missing topicName'); - return; - } - switch (currentProvider) { - case types_1.SchemaProvider.karapace: - (0, karapace_1.getSchemaVersions)(req.query.topicName) - .then(response => { - res.status(200).send(response); - }) - .catch(e => { - res.status(500).send(e); - }); - break; - default: - res.status(404).send('Provider Not Found'); - break; - } -}); -app.get('/schemas.info', (req, res) => { - if (!req.query.topicName) { - res.status(400).send('Missing topicName'); - return; - } - let version = 'latest'; - if (req.query.version) { - version = req.query.version; - } - switch (currentProvider) { - case types_1.SchemaProvider.karapace: - (0, karapace_1.getSchemaInfo)(req.query.topicName, version) - .then(response => { - res.status(200).send(response); - }) - .catch(e => { - res.status(500).send(e); - }); - break; - default: - res.status(404).send('Provider Not Found'); - break; - } -}); -app.post('/schemas.update', (req, res) => { - if (!req.query.topicName) { - res.status(400).send('Missing topicName'); - return; - } - if (!req.body.schema) { - res.status(400).send('Missing schema'); - return; - } - switch (currentProvider) { - case types_1.SchemaProvider.karapace: - (0, karapace_1.updateSchema)(req.query.topicName, req.body.schema) - .then(response => { - res.status(200).send(response); - }) - .catch(e => { - res.status(500).send(e); - }); - break; - default: - res.status(404).send('Provider Not Found'); - break; - } -}); -app.post('/schemas.create', (req, res) => { - if (!req.query.topicName) { - res.status(400).send('Missing topicName'); - return; - } - if (!req.body.schema) { - res.status(400).send('Missing schema'); - return; - } - switch (currentProvider) { - case types_1.SchemaProvider.karapace: - (0, karapace_1.createSchema)(req.query.topicName, req.body.schema) - .then(response => { - res.status(200).send(response); - }) - .catch(e => { - res.status(500).send(e); - }); - break; - default: - res.status(404).send('Provider Not Found'); - break; - } -}); -app.post('/schemas.compatibility', (req, res) => { - if (!req.query.topicName) { - res.status(400).send('Missing topicName'); - return; - } - if (!req.query.version) { - res.status(400).send('Missing version'); - return; - } - if (!req.body.schema) { - res.status(400).send('Missing schema'); - return; - } - switch (currentProvider) { - case types_1.SchemaProvider.karapace: - (0, karapace_1.checkCompatibilityOfNewSchema)(req.query.topicName, req.body.schema, req.query.version) - .then(response => { - res.status(200).send(response); - }) - .catch(e => { - res.status(500).send(e); - }); - break; - default: - res.status(404).send('Provider Not Found'); - break; - } -}); -app.post('/schemas.delete', (req, res) => { - if (!req.query.topicName) { - res.status(400).send('Missing topicName'); - return; - } - switch (currentProvider) { - case types_1.SchemaProvider.karapace: - (0, karapace_1.deleteSchema)(req.query.topicName) - .then(response => { - res.status(200).send(response); - }) - .catch(e => { - res.status(500).send(e); - }); - break; - default: - res.status(404).send('Provider Not Found'); - break; - } -}); -app.get('/schemas.lastMessage', (req, res) => { - if (!req.query.topicName) { - res.status(400).send('Missing topicName'); - return; - } - switch (currentProvider) { - case types_1.SchemaProvider.karapace: - (0, karapace_1.getLastMessage)(req.query.topicName) - .then(response => { - console.log(response); - res.status(200).send(response); - }) - .catch(e => { - res.status(500).send(e); - }); - break; - default: - res.status(404).send('Provider Not Found'); - break; - } -}); -function main() { - return __awaiter(this, void 0, void 0, function* () { - app.listen(port, () => { - console.log(`Server is running on http://localhost:${port}`); - }); - }); -} -main().catch(console.error); diff --git a/backend/components/schema-registry-manager/src/providers/karapace.js b/backend/components/schema-registry-manager/src/providers/karapace.js deleted file mode 100644 index 32c74cddf..000000000 --- a/backend/components/schema-registry-manager/src/providers/karapace.js +++ /dev/null @@ -1,182 +0,0 @@ -'use strict'; -var __awaiter = - (this && this.__awaiter) || - function (thisArg, _arguments, P, generator) { - function adopt(value) { - return value instanceof P - ? value - : new P(function (resolve) { - resolve(value); - }); - } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { - try { - step(generator.next(value)); - } catch (e) { - reject(e); - } - } - function rejected(value) { - try { - step(generator['throw'](value)); - } catch (e) { - reject(e); - } - } - function step(result) { - result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); - } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); - }; -Object.defineProperty(exports, '__esModule', {value: true}); -exports.getLastMessage = - exports.deleteSchema = - exports.checkCompatibilityOfNewSchema = - exports.createSchema = - exports.updateSchema = - exports.getSchemaInfo = - exports.getSchemaVersions = - exports.getSchemas = - void 0; -function getSchemas() { - return __awaiter(this, void 0, void 0, function* () { - return getData('subjects').then(response => { - return response; - }); - }); -} -exports.getSchemas = getSchemas; -function getSchemaVersions(topicName) { - return __awaiter(this, void 0, void 0, function* () { - return getData(`subjects/${topicName}/versions`).then(response => { - if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { - return Promise.reject('404 Not Found'); - } - return response; - }); - }); -} -exports.getSchemaVersions = getSchemaVersions; -function getSchemaInfo(topicName, version) { - return __awaiter(this, void 0, void 0, function* () { - return getData(`subjects/${topicName}/versions/${version}`).then(response => { - if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { - return Promise.reject('404 Not Found'); - } - return response; - }); - }); -} -exports.getSchemaInfo = getSchemaInfo; -function updateSchema(topicName, schema) { - return __awaiter(this, void 0, void 0, function* () { - const body = { - schema: JSON.stringify(Object.assign({}, JSON.parse(schema))), - }; - return postData(`subjects/${topicName}/versions`, body).then(response => { - if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { - return Promise.reject('404 Not Found'); - } - if (response.id) return response; - if (response.message) return Promise.reject(response.message); - return Promise.reject('Unknown Error'); - }); - }); -} -exports.updateSchema = updateSchema; -function createSchema(topicName, schema) { - return __awaiter(this, void 0, void 0, function* () { - const body = { - schema: JSON.stringify(Object.assign({}, JSON.parse(schema))), - }; - return postData(`subjects/${topicName}/versions`, body) - .then(response => { - if (response.id) return response; - if (response.message) return Promise.reject(response.message); - return Promise.reject('Unknown Error'); - }) - .catch(e => { - return Promise.reject(e); - }); - }); -} -exports.createSchema = createSchema; -function checkCompatibilityOfNewSchema(topicName, schema, version) { - return __awaiter(this, void 0, void 0, function* () { - const body = { - schema: JSON.stringify(Object.assign({}, JSON.parse(schema))), - }; - return postData(`compatibility/subjects/${topicName}/versions/${version}`, body) - .then(response => { - if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { - return Promise.reject('404 Not Found'); - } - if (response.is_compatible !== undefined) { - if (response.is_compatible === true) { - return response; - } - return Promise.reject('Schema Not Compatible'); - } - if (response.message) return Promise.reject(response.message); - return Promise.reject('Unknown Error'); - }) - .catch(e => { - return Promise.reject(e); - }); - }); -} -exports.checkCompatibilityOfNewSchema = checkCompatibilityOfNewSchema; -function deleteSchema(topicName) { - return __awaiter(this, void 0, void 0, function* () { - return deleteData(`subjects/${topicName}`).then(response => { - if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { - return Promise.reject('404 Not Found'); - } - return response; - }); - }); -} -exports.deleteSchema = deleteSchema; -function getLastMessage(topicName) { - return __awaiter(this, void 0, void 0, function* () { - const body = { - ksql: `PRINT '${topicName}' FROM BEGINNING LIMIT 1;`, - streamsProperties: {}, - }; - return postData('query', body).then(response => { - console.log(response); - return response; - }); - }); -} -exports.getLastMessage = getLastMessage; -function getData(url) { - return __awaiter(this, void 0, void 0, function* () { - const response = yield fetch(process.env.URL + '/' + url, { - method: 'GET', - }); - return response.json(); - }); -} -function deleteData(url) { - return __awaiter(this, void 0, void 0, function* () { - const response = yield fetch(process.env.URL + '/' + url, { - method: 'DELETE', - }); - return response.json(); - }); -} -function postData(url, body) { - return __awaiter(this, void 0, void 0, function* () { - const response = yield fetch(process.env.URL + '/' + url, { - method: 'POST', - headers: { - 'Content-Type': 'application/vnd.schemaregistry.v1+json', - }, - body: JSON.stringify(body), - }); - return response.json(); - }); -} diff --git a/backend/components/schema-registry-manager/src/types.js b/backend/components/schema-registry-manager/src/types.js deleted file mode 100644 index 3571994e8..000000000 --- a/backend/components/schema-registry-manager/src/types.js +++ /dev/null @@ -1,8 +0,0 @@ -'use strict'; -Object.defineProperty(exports, '__esModule', {value: true}); -exports.SchemaProvider = void 0; -var SchemaProvider; -(function (SchemaProvider) { - SchemaProvider['karapace'] = 'karapace'; - SchemaProvider['confluentCloud'] = 'confluent-cloud'; -})(SchemaProvider || (exports.SchemaProvider = SchemaProvider = {})); From 72fafb01779d5359fee8ee2e95a057e8f5fee2e1 Mon Sep 17 00:00:00 2001 From: Aitor Algorta Date: Tue, 13 Feb 2024 11:11:56 +0100 Subject: [PATCH 08/19] add cors --- backend/components/schema-registry-manager/src/app.ts | 10 ++++++++++ .../schema-registry-manager/src/package.json | 2 ++ 2 files changed, 12 insertions(+) diff --git a/backend/components/schema-registry-manager/src/app.ts b/backend/components/schema-registry-manager/src/app.ts index 00857d77a..d26e25188 100644 --- a/backend/components/schema-registry-manager/src/app.ts +++ b/backend/components/schema-registry-manager/src/app.ts @@ -1,5 +1,7 @@ import dotenv from 'dotenv'; import express, {Express, Request, Response} from 'express'; +import cors from 'cors'; + import {SchemaProvider} from './types'; import { checkCompatibilityOfNewSchema, @@ -22,6 +24,14 @@ const currentProvider: SchemaProvider = SchemaProvider.karapace; // Middleware app.use(bodyParser.json()); +// CORS options +const corsOptions = { + origin: 'http://localhost:8080', +}; + +// Use cors middleware with the specified options +app.use(cors(corsOptions)); + app.get('/schemas.provider', (req: Request, res: Response) => { res.status(200).send(currentProvider); }); diff --git a/backend/components/schema-registry-manager/src/package.json b/backend/components/schema-registry-manager/src/package.json index 1cdd291b3..970c2ea5f 100644 --- a/backend/components/schema-registry-manager/src/package.json +++ b/backend/components/schema-registry-manager/src/package.json @@ -2,11 +2,13 @@ "dependencies": { "@types/express": "^4.17.21", "@types/node": "^20.10.3", + "cors": "^2.8.5", "dotenv": "^16.4.2", "express": "^4.18.2", "node-fetch": "^2.6.1" }, "devDependencies": { + "@types/cors": "^2.8.17", "@types/express": "^4.17.21", "@types/node": "^20.10.3", "ts-node": "^10.9.2", From f98f9e6072e94c31090baf36758f7cb7e5abe59d Mon Sep 17 00:00:00 2001 From: Aitor Algorta Date: Tue, 13 Feb 2024 12:32:19 +0100 Subject: [PATCH 09/19] add host --- .../schema-registry-manager/src/app.ts | 6 +++--- .../src/providers/karapace.ts | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/backend/components/schema-registry-manager/src/app.ts b/backend/components/schema-registry-manager/src/app.ts index d26e25188..c52344cb3 100644 --- a/backend/components/schema-registry-manager/src/app.ts +++ b/backend/components/schema-registry-manager/src/app.ts @@ -39,7 +39,7 @@ app.get('/schemas.provider', (req: Request, res: Response) => { app.get('/schemas.list', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: - getSchemas() + getSchemas(req.get('host') as string) .then((response: string[]) => { res.send(response); }) @@ -61,7 +61,7 @@ app.get('/schemas.versions', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: - getSchemaVersions(req.query.topicName as string) + getSchemaVersions(req.get('host') as string, req.query.topicName as string) .then((response: any) => { res.status(200).send(response); }) @@ -88,7 +88,7 @@ app.get('/schemas.info', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: - getSchemaInfo(req.query.topicName as string, version) + getSchemaInfo(req.get('host') as string, req.query.topicName as string, version) .then((response: any) => { res.status(200).send(response); }) diff --git a/backend/components/schema-registry-manager/src/providers/karapace.ts b/backend/components/schema-registry-manager/src/providers/karapace.ts index 1f1792dd1..08de3bd10 100644 --- a/backend/components/schema-registry-manager/src/providers/karapace.ts +++ b/backend/components/schema-registry-manager/src/providers/karapace.ts @@ -1,11 +1,11 @@ -export async function getSchemas() { - return getData('subjects').then(response => { +export async function getSchemas(host: string) { + return getData(host, 'subjects').then(response => { return response; }); } -export async function getSchemaVersions(topicName: string) { - return getData(`subjects/${topicName}/versions`).then(response => { +export async function getSchemaVersions(host: string, topicName: string) { + return getData(host, `subjects/${topicName}/versions`).then(response => { if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { return Promise.reject('404 Not Found'); } @@ -13,8 +13,8 @@ export async function getSchemaVersions(topicName: string) { }); } -export async function getSchemaInfo(topicName: string, version: string) { - return getData(`subjects/${topicName}/versions/${version}`).then(response => { +export async function getSchemaInfo(host: string, topicName: string, version: string) { + return getData(host, `subjects/${topicName}/versions/${version}`).then(response => { if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { return Promise.reject('404 Not Found'); } @@ -94,8 +94,8 @@ export async function getLastMessage(topicName: string) { }); } -async function getData(url: string) { - const response = await fetch(process.env.URL + '/' + url, { +async function getData(host: string, url: string) { + const response = await fetch('https://' + host + '/' + url, { method: 'GET', }); return response.json(); From 622dfa7483ab74c8764eac073a6f9f2867612d39 Mon Sep 17 00:00:00 2001 From: Aitor Algorta Date: Wed, 14 Feb 2024 12:20:32 +0100 Subject: [PATCH 10/19] add logs --- backend/components/schema-registry-manager/src/app.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/components/schema-registry-manager/src/app.ts b/backend/components/schema-registry-manager/src/app.ts index c52344cb3..6a492d24b 100644 --- a/backend/components/schema-registry-manager/src/app.ts +++ b/backend/components/schema-registry-manager/src/app.ts @@ -107,6 +107,9 @@ app.post('/schemas.update', (req: Request, res: Response) => { res.status(400).send('Missing topicName'); return; } + + console.log(req.body); + if (!req.body.schema) { res.status(400).send('Missing schema'); return; @@ -220,7 +223,6 @@ app.get('/schemas.lastMessage', (req: Request, res: Response) => { case SchemaProvider.karapace: getLastMessage(req.query.topicName as string) .then((response: any) => { - console.log(response); res.status(200).send(response); }) .catch((e: any) => { From badf1a0f59a4bdae9bef835cbd9f53116e5f4a5c Mon Sep 17 00:00:00 2001 From: Aitor Algorta Date: Wed, 14 Feb 2024 13:11:04 +0100 Subject: [PATCH 11/19] remove logs --- backend/components/schema-registry-manager/src/app.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/backend/components/schema-registry-manager/src/app.ts b/backend/components/schema-registry-manager/src/app.ts index 6a492d24b..b78665ca4 100644 --- a/backend/components/schema-registry-manager/src/app.ts +++ b/backend/components/schema-registry-manager/src/app.ts @@ -107,9 +107,6 @@ app.post('/schemas.update', (req: Request, res: Response) => { res.status(400).send('Missing topicName'); return; } - - console.log(req.body); - if (!req.body.schema) { res.status(400).send('Missing schema'); return; From a5ab0a1c19b5bf5a64bba195d8f4596811dbd695 Mon Sep 17 00:00:00 2001 From: Aitor Algorta Date: Wed, 14 Feb 2024 15:37:07 +0100 Subject: [PATCH 12/19] fix issues --- .../schema-registry-manager/src/app.ts | 9 +++--- .../src/providers/karapace.ts | 30 +++++++++---------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/backend/components/schema-registry-manager/src/app.ts b/backend/components/schema-registry-manager/src/app.ts index b78665ca4..305b46173 100644 --- a/backend/components/schema-registry-manager/src/app.ts +++ b/backend/components/schema-registry-manager/src/app.ts @@ -114,7 +114,7 @@ app.post('/schemas.update', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: - updateSchema(req.query.topicName as string, req.body.schema as string) + updateSchema(req.get('host') as string, req.query.topicName as string, req.body.schema as string) .then((response: any) => { res.status(200).send(response); }) @@ -140,7 +140,7 @@ app.post('/schemas.create', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: - createSchema(req.query.topicName as string, req.body.schema as string) + createSchema(req.get('host') as string, req.query.topicName as string, req.body.schema as string) .then((response: any) => { res.status(200).send(response); }) @@ -171,6 +171,7 @@ app.post('/schemas.compatibility', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: checkCompatibilityOfNewSchema( + req.get('host') as string, req.query.topicName as string, req.body.schema as string, req.query.version as string @@ -196,7 +197,7 @@ app.post('/schemas.delete', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: - deleteSchema(req.query.topicName as string) + deleteSchema(req.get('host') as string, req.query.topicName as string) .then((response: any) => { res.status(200).send(response); }) @@ -218,7 +219,7 @@ app.get('/schemas.lastMessage', (req: Request, res: Response) => { switch (currentProvider) { case SchemaProvider.karapace: - getLastMessage(req.query.topicName as string) + getLastMessage(req.get('host') as string, req.query.topicName as string) .then((response: any) => { res.status(200).send(response); }) diff --git a/backend/components/schema-registry-manager/src/providers/karapace.ts b/backend/components/schema-registry-manager/src/providers/karapace.ts index 08de3bd10..487aa31a8 100644 --- a/backend/components/schema-registry-manager/src/providers/karapace.ts +++ b/backend/components/schema-registry-manager/src/providers/karapace.ts @@ -22,11 +22,11 @@ export async function getSchemaInfo(host: string, topicName: string, version: st }); } -export async function updateSchema(topicName: string, schema: string) { +export async function updateSchema(host: string, topicName: string, schema: string) { const body = { schema: JSON.stringify({...JSON.parse(schema)}), }; - return postData(`subjects/${topicName}/versions`, body).then(response => { + return postData(host, `subjects/${topicName}/versions`, body).then(response => { if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { return Promise.reject('404 Not Found'); } @@ -36,11 +36,11 @@ export async function updateSchema(topicName: string, schema: string) { }); } -export async function createSchema(topicName: string, schema: string) { +export async function createSchema(host: string, topicName: string, schema: string) { const body = { schema: JSON.stringify({...JSON.parse(schema)}), }; - return postData(`subjects/${topicName}/versions`, body) + return postData(host, `subjects/${topicName}/versions`, body) .then(response => { if (response.id) return response; if (response.message) return Promise.reject(response.message); @@ -51,11 +51,12 @@ export async function createSchema(topicName: string, schema: string) { }); } -export async function checkCompatibilityOfNewSchema(topicName: string, schema: string, version: string) { +export async function checkCompatibilityOfNewSchema(host: string, topicName: string, schema: string, version: string) { const body = { schema: JSON.stringify({...JSON.parse(schema)}), }; - return postData(`compatibility/subjects/${topicName}/versions/${version}`, body) + + return postData(host, `compatibility/subjects/${topicName}/versions/${version}`, body) .then(response => { if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { return Promise.reject('404 Not Found'); @@ -74,8 +75,8 @@ export async function checkCompatibilityOfNewSchema(topicName: string, schema: s }); } -export async function deleteSchema(topicName: string) { - return deleteData(`subjects/${topicName}`).then(response => { +export async function deleteSchema(host: string, topicName: string) { + return deleteData(host, `subjects/${topicName}`).then(response => { if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { return Promise.reject('404 Not Found'); } @@ -83,13 +84,12 @@ export async function deleteSchema(topicName: string) { }); } -export async function getLastMessage(topicName: string) { +export async function getLastMessage(host: string, topicName: string) { const body = { ksql: `PRINT '${topicName}' FROM BEGINNING LIMIT 1;`, streamsProperties: {}, }; - return postData('query', body).then(response => { - console.log(response); + return postData(host, 'query', body).then(response => { return response; }); } @@ -101,15 +101,15 @@ async function getData(host: string, url: string) { return response.json(); } -async function deleteData(url: string) { - const response = await fetch(process.env.URL + '/' + url, { +async function deleteData(host: string, url: string) { + const response = await fetch('https://' + host + '/' + url, { method: 'DELETE', }); return response.json(); } -async function postData(url: string, body: any) { - const response = await fetch(process.env.URL + '/' + url, { +async function postData(host: string, url: string, body: any) { + const response = await fetch('https://' + host + '/' + url, { method: 'POST', headers: { 'Content-Type': 'application/vnd.schemaregistry.v1+json', From d7191207c42988c8729be673636bff97e04f8676 Mon Sep 17 00:00:00 2001 From: Aitor Algorta Date: Wed, 14 Feb 2024 16:03:03 +0100 Subject: [PATCH 13/19] add health check --- .../schema-registry-manager/src/app.ts | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/backend/components/schema-registry-manager/src/app.ts b/backend/components/schema-registry-manager/src/app.ts index 305b46173..061c98b81 100644 --- a/backend/components/schema-registry-manager/src/app.ts +++ b/backend/components/schema-registry-manager/src/app.ts @@ -1,5 +1,6 @@ import dotenv from 'dotenv'; -import express, {Express, Request, Response} from 'express'; +import express, {Express, Request as ExpressRequest, Response as ExpressResponse} from 'express'; +import http from 'http'; import cors from 'cors'; import {SchemaProvider} from './types'; @@ -32,11 +33,11 @@ const corsOptions = { // Use cors middleware with the specified options app.use(cors(corsOptions)); -app.get('/schemas.provider', (req: Request, res: Response) => { +app.get('/schemas.provider', (req: ExpressRequest, res: ExpressResponse) => { res.status(200).send(currentProvider); }); -app.get('/schemas.list', (req: Request, res: Response) => { +app.get('/schemas.list', (req: ExpressRequest, res: ExpressResponse) => { switch (currentProvider) { case SchemaProvider.karapace: getSchemas(req.get('host') as string) @@ -53,7 +54,7 @@ app.get('/schemas.list', (req: Request, res: Response) => { } }); -app.get('/schemas.versions', (req: Request, res: Response) => { +app.get('/schemas.versions', (req: ExpressRequest, res: ExpressResponse) => { if (!req.query.topicName) { res.status(400).send('Missing topicName'); return; @@ -75,7 +76,7 @@ app.get('/schemas.versions', (req: Request, res: Response) => { } }); -app.get('/schemas.info', (req: Request, res: Response) => { +app.get('/schemas.info', (req: ExpressRequest, res: ExpressResponse) => { if (!req.query.topicName) { res.status(400).send('Missing topicName'); return; @@ -102,7 +103,7 @@ app.get('/schemas.info', (req: Request, res: Response) => { } }); -app.post('/schemas.update', (req: Request, res: Response) => { +app.post('/schemas.update', (req: ExpressRequest, res: ExpressResponse) => { if (!req.query.topicName) { res.status(400).send('Missing topicName'); return; @@ -128,7 +129,7 @@ app.post('/schemas.update', (req: Request, res: Response) => { } }); -app.post('/schemas.create', (req: Request, res: Response) => { +app.post('/schemas.create', (req: ExpressRequest, res: ExpressResponse) => { if (!req.query.topicName) { res.status(400).send('Missing topicName'); return; @@ -154,7 +155,7 @@ app.post('/schemas.create', (req: Request, res: Response) => { } }); -app.post('/schemas.compatibility', (req: Request, res: Response) => { +app.post('/schemas.compatibility', (req: ExpressRequest, res: ExpressResponse) => { if (!req.query.topicName) { res.status(400).send('Missing topicName'); return; @@ -189,7 +190,7 @@ app.post('/schemas.compatibility', (req: Request, res: Response) => { } }); -app.post('/schemas.delete', (req: Request, res: Response) => { +app.post('/schemas.delete', (req: ExpressRequest, res: ExpressResponse) => { if (!req.query.topicName) { res.status(400).send('Missing topicName'); return; @@ -211,7 +212,7 @@ app.post('/schemas.delete', (req: Request, res: Response) => { } }); -app.get('/schemas.lastMessage', (req: Request, res: Response) => { +app.get('/schemas.lastMessage', (req: ExpressRequest, res: ExpressResponse) => { if (!req.query.topicName) { res.status(400).send('Missing topicName'); return; @@ -233,7 +234,27 @@ app.get('/schemas.lastMessage', (req: Request, res: Response) => { } }); +async function startHealthcheck() { + const server = http.createServer((req: any, res: any) => { + if (req.url === '/actuator/health' && req.method === 'GET') { + const response = { status: "UP" }; + const jsonResponse = JSON.stringify(response); + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(jsonResponse); + } else { + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('Not Found'); + } + }); + + server.listen(80, () => { + console.log("Health-check started"); + }); +} + async function main() { + startHealthcheck(); app.listen(port, () => { console.log(`Server is running on http://localhost:${port}`); }); From e1f48945a458db3b0698b23efbe40927919904ec Mon Sep 17 00:00:00 2001 From: Aitor Algorta Date: Wed, 14 Feb 2024 16:05:08 +0100 Subject: [PATCH 14/19] lint --- .../components/schema-registry-manager/src/app.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/components/schema-registry-manager/src/app.ts b/backend/components/schema-registry-manager/src/app.ts index 061c98b81..fb7f384e3 100644 --- a/backend/components/schema-registry-manager/src/app.ts +++ b/backend/components/schema-registry-manager/src/app.ts @@ -237,19 +237,19 @@ app.get('/schemas.lastMessage', (req: ExpressRequest, res: ExpressResponse) => { async function startHealthcheck() { const server = http.createServer((req: any, res: any) => { if (req.url === '/actuator/health' && req.method === 'GET') { - const response = { status: "UP" }; + const response = {status: 'UP'}; const jsonResponse = JSON.stringify(response); - - res.writeHead(200, { 'Content-Type': 'application/json' }); + + res.writeHead(200, {'Content-Type': 'application/json'}); res.end(jsonResponse); } else { - res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.writeHead(404, {'Content-Type': 'text/plain'}); res.end('Not Found'); } }); - + server.listen(80, () => { - console.log("Health-check started"); + console.log('Health-check started'); }); } From 8f761e392f6de5e5fd4f1dae8a6f93df96ba4deb Mon Sep 17 00:00:00 2001 From: ljupcovangelski Date: Thu, 15 Feb 2024 16:56:03 +0100 Subject: [PATCH 15/19] Fix helm build --- .../schema-registry-manager/helm/BUILD | 33 ++----------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/backend/components/schema-registry-manager/helm/BUILD b/backend/components/schema-registry-manager/helm/BUILD index 5bb7f8f59..8d6495211 100644 --- a/backend/components/schema-registry-manager/helm/BUILD +++ b/backend/components/schema-registry-manager/helm/BUILD @@ -1,32 +1,3 @@ -load("@rules_pkg//:pkg.bzl", "pkg_tar") -load("@com_github_airyhq_bazel_tools//helm:helm.bzl", "helm_template_test") -load("//tools/build:helm.bzl", "helm_push_develop", "helm_push_release") +load("//tools/build:helm.bzl", "helm_ruleset_core_version") -filegroup( - name = "files", - srcs = glob( - ["**/*"], - exclude = ["BUILD"], - ), - visibility = ["//visibility:public"], -) - -pkg_tar( - name = "package", - srcs = [":files"], - extension = "tgz", - strip_prefix = "./", -) - -helm_template_test( - name = "template", - chart = ":package", -) - -helm_push_develop( - chart = ":package", -) - -helm_push_release( - chart = ":package", -) +helm_ruleset_core_version() \ No newline at end of file From 7f7459d178429f9d580501b8f9a64c8be1d61bea Mon Sep 17 00:00:00 2001 From: ljupcovangelski Date: Thu, 15 Feb 2024 18:40:55 +0100 Subject: [PATCH 16/19] Check disk while building --- .github/workflows/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9348a3f58..9e07b03e8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,6 +54,11 @@ jobs: run: | bazel test --test_tag_filters=-lint //... + - name: Check disk + run: | + df -h + cd / && du -d 1 -h 2>/dev/null + - name: Build all artifacts run: | bazel build //... From 7cca1437f14d67699292eb8a3581b82f29f9e156 Mon Sep 17 00:00:00 2001 From: ljupcovangelski Date: Thu, 15 Feb 2024 23:03:16 +0100 Subject: [PATCH 17/19] Try to cleanup space on the CI --- .github/workflows/main.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9e07b03e8..63768a337 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,10 +54,15 @@ jobs: run: | bazel test --test_tag_filters=-lint //... - - name: Check disk + - name: Cleanup space run: | df -h - cd / && du -d 1 -h 2>/dev/null + sudo apt-get autoremove -y + sudo apt-get clean + du -d 1 -h /usr/local/share/ + du -d 1 -h /opt/ + docker images prune -a + df -h - name: Build all artifacts run: | From 12e3041b444dc635408303781c83dce61098a46d Mon Sep 17 00:00:00 2001 From: ljupcovangelski Date: Fri, 16 Feb 2024 10:40:44 +0100 Subject: [PATCH 18/19] Remove large directories --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 63768a337..c0cd677e2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -59,9 +59,9 @@ jobs: df -h sudo apt-get autoremove -y sudo apt-get clean - du -d 1 -h /usr/local/share/ - du -d 1 -h /opt/ docker images prune -a + rm -rf /usr/local/share/powershell + rm -rf /opt/hostedtoolcache df -h - name: Build all artifacts From 828d97e61067ace1018b180753a5a963e807c2b8 Mon Sep 17 00:00:00 2001 From: ljupcovangelski Date: Fri, 16 Feb 2024 11:00:43 +0100 Subject: [PATCH 19/19] Fix permissions --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c0cd677e2..02c8436e4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -60,8 +60,8 @@ jobs: sudo apt-get autoremove -y sudo apt-get clean docker images prune -a - rm -rf /usr/local/share/powershell - rm -rf /opt/hostedtoolcache + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /opt/hostedtoolcache df -h - name: Build all artifacts