feat(endpoints): store and use "group“ (#2966)

## Describe your changes Fixes https://linear.app/nango/issue/NAN-1934/specify-script-endpoint-section-in-nangoyaml - Store entity - Use it in the UI to categorize endpoints *cf my own nango.yaml* <img width="1147" alt="Screenshot 2024-11-08 at 15 06 34" src="https://github.com/user-attachments/assets/3b3b506e-95c0-4c17-9f08-d28c1d666a9a">
NangoHQ · Nov 14, 2024 · 4b37eaa · 4b37eaa
1 parent 52fd88b
commit 4b37eaa
Show file tree

Hide file tree

Showing 14 changed files with 62 additions and 51 deletions.
diff --git a/docs-v2/reference/integration-configuration.mdx b/docs-v2/reference/integration-configuration.mdx
@@ -20,6 +20,7 @@ integrations:
         endpoint: # Generated endpoint to access the synced model.
           method: GET
           path: /tasks
+          group: Tasks
         sync_type: incremental # Data is replaced ('full') or upserted ('incremental') on each sync execution.
         runs: every 30min # Sync frequency.
         input:  AsanaProject # Necessary input to execute the sync.
@@ -41,6 +42,7 @@ integrations:
         endpoint: # Generated endpoint to trigger to action.
           method: POST
           path: /tasks
+          group: Tasks
         input: # Necessary input to execute the action.
         scopes: # Required scopes.
           - scope1
@@ -114,11 +116,13 @@ Sync configuration fields are under `integrations.<INTEGRATION-ID>.syncs.<SYNC-N
     endpoint:
       method: GET
       path: /tasks
+      group?: Tasks
     ```
 
     Possible method values are: `GET`
 
     The method/endpoint combination can be shared across syncs to unify the communication with external APIs.
+    The `group` is an optional value to group endpoints across scripts.
 
     Legacy format: `<METHOD> <URL-PATH>`.  e.g.: `GET /tasks`.
 </ResponseField>
@@ -195,6 +199,7 @@ Action configuration fields are under `integrations.<INTEGRATION-ID>.actions.<AC
     Possible method values are: `GET`, `POST`, `PUT`, `PATCH`, `DELETE`.
 
     The method/endpoint combination can be shared across actions to unify the communication with external APIs.
+    The `group` is an optional value to group endpoints across scripts.
 
     Legacy format: `<METHOD> <URL-PATH>`.  e.g.: `POST /tasks`.
 </ResponseField>

diff --git a/packages/cli/lib/nango.yaml.schema.v2.json b/packages/cli/lib/nango.yaml.schema.v2.json
@@ -304,7 +304,7 @@
                     "type": "string",
                     "pattern": "^/[a-zA-Z0-9-:{}./_]+$"
                 },
-                "entity": {
+                "group": {
                     "type": "string",
                     "pattern": "^[a-zA-Z _-]{1,64}$"
                 }
@@ -326,7 +326,7 @@
                     "type": "string",
                     "pattern": "^/[a-zA-Z0-9-:{}./_]+$"
                 },
-                "entity": {
+                "group": {
                     "type": "string",
                     "pattern": "^[a-zA-Z _-]{1,64}$"
                 }

diff --git a/packages/database/lib/migrations/20241108090909_endpoints_group.cjs b/packages/database/lib/migrations/20241108090909_endpoints_group.cjs
@@ -0,0 +1,15 @@
+exports.config = { transaction: false };
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.up = async function (knex) {
+    await knex.schema.raw(`ALTER TABLE "_nango_sync_endpoints" ADD COLUMN "group_name" varchar(64)`);
+};
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.down = async function (knex) {
+    await knex.schema.raw(`ALTER TABLE "_nango_sync_endpoints" DROP COLUMN "group_name"`);
+};
diff --git a/packages/nango-yaml/lib/helpers.ts b/packages/nango-yaml/lib/helpers.ts
@@ -205,5 +205,5 @@ export function parseEndpoint(rawEndpoint: string | NangoSyncEndpointV2 | NangoY
         return { method: defaultMethod, path: endpoint[0] as string };
     }
 
-    return { method: rawEndpoint.method || defaultMethod, path: rawEndpoint.path, entity: rawEndpoint.entity };
+    return { method: rawEndpoint.method || defaultMethod, path: rawEndpoint.path, group: rawEndpoint.group };
 }
diff --git a/packages/nango-yaml/lib/parser.v2.unit.test.ts b/packages/nango-yaml/lib/parser.v2.unit.test.ts
@@ -314,7 +314,7 @@ describe('parse', () => {
                                 output: ['Top', 'Tip'],
                                 endpoint: [
                                     { method: 'GET', path: '/provider/top' },
-                                    { path: '/provider/tip', entity: 'Record' }
+                                    { path: '/provider/tip', group: 'Record' }
                                 ]
                             }
                         }
@@ -329,7 +329,7 @@ describe('parse', () => {
                 {
                     endpoints: [
                         { method: 'GET', path: '/provider/top' },
-                        { method: 'GET', path: '/provider/tip', entity: 'Record' }
+                        { method: 'GET', path: '/provider/tip', group: 'Record' }
                     ]
                 }
             ]);

diff --git a/packages/server/lib/controllers/sync/deploy/postConfirmation.ts b/packages/server/lib/controllers/sync/deploy/postConfirmation.ts
@@ -62,7 +62,7 @@ export const flowConfig = z
                         .object({
                             method: z.enum(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']),
                             path: z.string(),
-                            entity: z.string().min(1).max(64).optional()
+                            group: z.string().min(1).max(64).optional()
                         })
                         .strict(),
                     z

diff --git a/packages/server/lib/controllers/sync/deploy/postDeploy.integration.test.ts b/packages/server/lib/controllers/sync/deploy/postDeploy.integration.test.ts
@@ -151,7 +151,7 @@ describe(`POST ${endpoint}`, () => {
                             auto_start: false,
                             description: 'a',
                             enabled: true,
-                            endpoints: [{ method: 'GET', path: '/path' }],
+                            endpoints: [{ method: 'GET', path: '/path', group: null }],
                             input: {
                                 fields: [{ array: false, name: 'id', optional: false, tsType: true, value: 'number' }],
                                 name: 'Input'

diff --git a/packages/shared/lib/models/Sync.ts b/packages/shared/lib/models/Sync.ts
@@ -1,5 +1,5 @@
 import type { JSONSchema7 } from 'json-schema';
-import type { HTTP_METHOD, Timestamps, TimestampsAndDeleted } from './Generic.js';
+import type { TimestampsAndDeleted } from './Generic.js';
 import type { NangoConfigMetadata, NangoModel, NangoSyncEndpointV2, ScriptTypeLiteral } from '@nangohq/types';
 
 export enum SyncStatus {
@@ -105,14 +105,6 @@ export interface SyncConfig extends TimestampsAndDeleted {
     models_json_schema?: JSONSchema7 | null;
 }
 
-export interface SyncEndpoint extends Timestamps {
-    id?: number;
-    sync_config_id: number;
-    method: HTTP_METHOD;
-    path: string;
-    model?: string;
-}
-
 export enum SyncCommand {
     PAUSE = 'PAUSE',
     UNPAUSE = 'UNPAUSE',

diff --git a/packages/shared/lib/services/sync/config/config.service.ts b/packages/shared/lib/services/sync/config/config.service.ts
@@ -693,7 +693,7 @@ export async function getSyncConfigsAsStandardConfig(
             '_nango_configs.provider',
             db.knex.raw(
                 `(
-                    SELECT json_agg(json_build_object('method', method, 'path', path))
+                    SELECT json_agg(json_build_object('method', method, 'path', path, 'group', group_name))
                     FROM _nango_sync_endpoints
                     WHERE _nango_sync_endpoints.sync_config_id = ${TABLE}.id
                 ) as endpoints_object`

diff --git a/packages/shared/lib/services/sync/config/deploy.service.ts b/packages/shared/lib/services/sync/config/deploy.service.ts
@@ -6,7 +6,7 @@ import { getSyncAndActionConfigByParams, increment, getSyncAndActionConfigsBySyn
 import connectionService from '../../connection.service.js';
 import { LogActionEnum } from '../../../models/Telemetry.js';
 import type { ServiceResponse } from '../../../models/Generic.js';
-import type { SyncModelSchema, SyncConfig, SyncEndpoint, SyncType, Sync } from '../../../models/Sync.js';
+import type { SyncModelSchema, SyncConfig, SyncType, Sync } from '../../../models/Sync.js';
 import type {
     DBEnvironment,
     DBTeam,
@@ -17,7 +17,9 @@ import type {
     NangoSyncEndpointV2,
     IncomingFlowConfig,
     HTTP_METHOD,
-    SyncDeploymentResult
+    SyncDeploymentResult,
+    DBSyncEndpointCreate,
+    DBSyncEndpoint
 } from '@nangohq/types';
 import { postConnectionScriptService } from '../post-connection.service.js';
 import { NangoError } from '../../../utils/error.js';
@@ -163,7 +165,7 @@ export async function deploy({
             )
             .returning('id');
 
-        const endpoints: SyncEndpoint[] = [];
+        const endpoints: DBSyncEndpointCreate[] = [];
         for (const [index, row] of flowIds.entries()) {
             const flow = flows[index];
             if (!flow) {
@@ -174,7 +176,7 @@ export async function deploy({
         }
 
         if (endpoints.length > 0) {
-            await db.knex.from<SyncEndpoint>(ENDPOINT_TABLE).insert(endpoints);
+            await db.knex.from<DBSyncEndpoint>(ENDPOINT_TABLE).insert(endpoints);
         }
 
         if (postConnectionScriptsByProvider.length > 0) {
@@ -301,18 +303,19 @@ export async function upgradePreBuilt({
             throw new NangoError('error_creating_sync_config');
         }
         const newSyncConfigId = newSyncConfig.id;
-        const endpoints: SyncEndpoint[] = [];
+        const endpoints: DBSyncEndpointCreate[] = [];
 
         // update sync_config_id in syncs table
         await db.knex.from<Sync>(SYNC_TABLE).update({ sync_config_id: newSyncConfigId }).where('sync_config_id', syncConfig.id);
 
         // update endpoints
         if (flow.endpoints) {
-            flow.endpoints.forEach(({ method, path }, endpointIndex) => {
-                const res: SyncEndpoint = {
+            flow.endpoints.forEach(({ method, path, group }, endpointIndex) => {
+                const res: DBSyncEndpointCreate = {
                     sync_config_id: newSyncConfigId,
                     method,
                     path,
+                    group_name: group || null,
                     created_at: now,
                     updated_at: now
                 };
@@ -325,7 +328,7 @@ export async function upgradePreBuilt({
         }
 
         if (endpoints.length > 0) {
-            await db.knex.from<SyncEndpoint>(ENDPOINT_TABLE).insert(endpoints);
+            await db.knex.from<DBSyncEndpoint>(ENDPOINT_TABLE).insert(endpoints);
         }
 
         await db.knex.from<SyncConfig>(TABLE).update({ active: false }).whereIn('id', [syncConfig.id]);
@@ -594,7 +597,7 @@ export async function deployPreBuilt({
             }
         });
 
-        const endpoints: SyncEndpoint[] = [];
+        const endpoints: DBSyncEndpointCreate[] = [];
         for (const [index, row] of syncConfigs.entries()) {
             const flow = configs[index];
             if (!flow) {
@@ -605,7 +608,7 @@ export async function deployPreBuilt({
         }
 
         if (endpoints.length > 0) {
-            await db.knex.from<SyncEndpoint>(ENDPOINT_TABLE).insert(endpoints);
+            await db.knex.from<DBSyncEndpoint>(ENDPOINT_TABLE).insert(endpoints);
         }
 
         for (const id of idsToMarkAsInactive) {
@@ -899,12 +902,13 @@ function findModelInModelSchema(fields: NangoModel['fields']) {
 }
 
 function endpointToSyncEndpoint(flow: Pick<CleanedIncomingFlowConfig, 'endpoints' | 'models'>, sync_config_id: number) {
-    const endpoints: SyncEndpoint[] = [];
+    const endpoints: DBSyncEndpointCreate[] = [];
     for (const [endpointIndex, endpoint] of flow.endpoints.entries()) {
-        const res: SyncEndpoint = {
+        const res: DBSyncEndpointCreate = {
             sync_config_id,
             method: endpoint.method,
             path: endpoint.path,
+            group_name: endpoint.group || null,
             created_at: new Date(),
             updated_at: new Date()
         };

diff --git a/packages/types/lib/endpoints/db.ts b/packages/types/lib/endpoints/db.ts
@@ -0,0 +1,13 @@
+import type { SetOptional } from 'type-fest';
+import type { Timestamps } from '../db';
+import type { HTTP_METHOD } from '../nangoYaml';
+
+export interface DBSyncEndpoint extends Timestamps {
+    id: number;
+    sync_config_id: number;
+    method: HTTP_METHOD;
+    path: string;
+    model: string | null;
+    group_name: string | null;
+}
+export type DBSyncEndpointCreate = SetOptional<Omit<DBSyncEndpoint, 'id'>, 'model' | 'group_name'>;
diff --git a/packages/types/lib/index.ts b/packages/types/lib/index.ts
@@ -40,6 +40,7 @@ export type * from './auth/http.api.js';
 export type * from './deploy/api.js';
 export type * from './deploy/index.js';
 export type * from './deploy/incomingFlow.js';
+export type * from './endpoints/db.js';
 export type * from './connect/api.js';
 export type * from './connect/session.js';
 export type * from './endUser/index.js';

diff --git a/packages/types/lib/nangoYaml/index.ts b/packages/types/lib/nangoYaml/index.ts
@@ -30,7 +30,7 @@ export interface NangoYamlV2 {
 export interface NangoYamlV2Endpoint {
     method?: HTTP_METHOD;
     path: string;
-    entity?: string | undefined;
+    group?: string | undefined;
 }
 export interface NangoYamlV2Integration {
     provider?: string;
@@ -141,5 +141,5 @@ export type NangoSyncEndpointOld = {
 export interface NangoSyncEndpointV2 {
     method: HTTP_METHOD;
     path: string;
-    entity?: string | undefined;
+    group?: string | undefined;
 }
diff --git a/packages/webapp/src/pages/Integrations/providerConfigKey/Endpoints/Show.tsx b/packages/webapp/src/pages/Integrations/providerConfigKey/Endpoints/Show.tsx
@@ -9,20 +9,6 @@ import { EndpointsList } from './components/List';
 import { EndpointOne } from './components/One';
 import PageNotFound from '../../../PageNotFound';
 
-const allowedGroup = [
-    'customers',
-    'invoices',
-    'payments',
-    'tickets',
-    'users',
-    'articles',
-    'accounts',
-    'contacts',
-    'employees',
-    'deals',
-    'opportunities',
-    'leads'
-];
 export const EndpointsShow: React.FC<{ integration: GetIntegration['Success']['data'] }> = ({ integration }) => {
     const env = useStore((state) => state.env);
     const { providerConfigKey } = useParams();
@@ -39,12 +25,7 @@ export const EndpointsShow: React.FC<{ integration: GetIntegration['Success']['d
         const tmp: Record<string, NangoSyncConfigWithEndpoint[]> = {};
         for (const flow of data.flows) {
             for (const endpoint of flow.endpoints) {
-                const paths = endpoint.path.split('/');
-                const firstPath = paths[1];
-                if (!firstPath) {
-                    continue;
-                }
-                const groupName = allowedGroup.includes(firstPath) ? firstPath : 'others';
+                const groupName = endpoint.group || 'others';
 
                 let group = tmp[groupName];
                 if (!group) {