feat: Hosea/ext 213 dropbox file sync (#99)

## Describe your changes

## Issue ticket number and link
EXT-213
https://linear.app/nango/issue/EXT-213/dropbox-file-sync

## Checklist before requesting a review (skip if just adding/editing
APIs & templates)
- [x] I added tests, otherwise the reason is:
- [x] External API requests have `retries`
- [x] Pagination is used where appropriate
- [ ] The built in `nango.paginate` call is used instead of a `while
(true)` loop
- [x] Third party requests are NOT parallelized (this can cause issues
with rate limits)
- [x] If a sync requires metadata the `nango.yaml` has `auto_start:
false`
- [x] If the sync is a `full` sync then `track_deletes: true` is set
- [x] I followed the best practices and guidelines from the [Writing
Integration
Scripts](/NangoHQ/integration-templates/blob/main/WRITING_INTEGRATION_SCRIPTS.md)
doc

---------

Co-authored-by: Khaliq <khaliqgant@gmail.com>

flows.yaml

@@ -1873,6 +1873,17 @@ integrations:
                 input: IdEntity
                 scopes:
                     - members.delete
+            fetch-file:
+                description: >-
+                    Fetches the content of a file given its ID, processes the data using a
+                    response stream, and encodes it into a base64 string. This
+                    base64-encoded string can be used to recreate the file in its original
+                    format using an external tool.
+                endpoint: GET /fetch-file
+                output: string
+                input: string
+                scopes:
+                    - files.content.read
         syncs:
             users:
                 runs: every day
@@ -1884,6 +1895,19 @@ integrations:
                 endpoint: GET /users
                 scopes:
                     - members.read
+            files:
+                runs: every day
+                description: >
+                    Sync the metadata of a specified files or folders paths from Dropbox.
+                    A file or folder id or path can be used.
+                input: DocumentMetadata
+                auto_start: false
+                output: Document
+                sync_type: full
+                track_deletes: true
+                endpoint: GET /files
+                scopes:
+                    - files.metadata.read
         models:
             SuccessResponse:
                 success: boolean
@@ -1898,6 +1922,13 @@ integrations:
                 firstName: string
                 lastName: string
                 email: string
+            DocumentMetadata:
+                files: string[] | undefined
+                folders: string[] | undefined
+            Document:
+                id: string
+                path: string
+                title: string
     evaluagent:
         syncs:
             users:

integrations/aws-iam/tests/aws-iam-create-user.test.ts

@@ -1,19 +1,19 @@
-import { vi, expect, it, describe } from "vitest";
+import { vi, expect, it, describe } from 'vitest';
 
-import runAction from "../actions/create-user.js";
+import runAction from '../actions/create-user.js';
 
-describe("aws-iam create-user tests", () => {
-  const nangoMock = new global.vitest.NangoActionMock({ 
-      dirname: __dirname,
-      name: "create-user",
-      Model: "User"
-  });
+describe('aws-iam create-user tests', () => {
+    const nangoMock = new global.vitest.NangoActionMock({
+        dirname: __dirname,
+        name: 'create-user',
+        Model: 'User'
+    });
 
-  it('should output the action output that is expected', async () => {
-      const input = await nangoMock.getInput();
-      const response = await runAction(nangoMock, input);
-      const output = await nangoMock.getOutput();
+    it('should output the action output that is expected', async () => {
+        const input = await nangoMock.getInput();
+        const response = await runAction(nangoMock, input);
+        const output = await nangoMock.getOutput();
 
-      expect(response).toEqual(output);
-  });
+        expect(response).toEqual(output);
+    });
 });

integrations/aws-iam/tests/aws-iam-delete-user.test.ts

@@ -1,19 +1,19 @@
-import { vi, expect, it, describe } from "vitest";
+import { vi, expect, it, describe } from 'vitest';
 
-import runAction from "../actions/delete-user.js";
+import runAction from '../actions/delete-user.js';
 
-describe("aws-iam delete-user tests", () => {
-  const nangoMock = new global.vitest.NangoActionMock({ 
-      dirname: __dirname,
-      name: "delete-user",
-      Model: "SuccessResponse"
-  });
+describe('aws-iam delete-user tests', () => {
+    const nangoMock = new global.vitest.NangoActionMock({
+        dirname: __dirname,
+        name: 'delete-user',
+        Model: 'SuccessResponse'
+    });
 
-  it('should output the action output that is expected', async () => {
-      const input = await nangoMock.getInput();
-      const response = await runAction(nangoMock, input);
-      const output = await nangoMock.getOutput();
+    it('should output the action output that is expected', async () => {
+        const input = await nangoMock.getInput();
+        const response = await runAction(nangoMock, input);
+        const output = await nangoMock.getOutput();
 
-      expect(response).toEqual(output);
-  });
+        expect(response).toEqual(output);
+    });
 });

integrations/dropbox/actions/fetch-file.ts

@@ -0,0 +1,43 @@
+import type { NangoAction, ProxyConfiguration } from '../../models';
+import type { DropboxTemporaryDownloadLink } from '../types.js';
+
+export default async function runAction(nango: NangoAction, input: string): Promise<string> {
+    if (!input || typeof input !== 'string') {
+        throw new Error('Missing or invalid input: a file ID is required and should be a string');
+    }
+
+    const proxyConfig: ProxyConfiguration = {
+        // https://www.dropbox.com/developers/documentation/http/documentation#files-get_temporary_link
+        endpoint: `/2/files/get_temporary_link`,
+        data: {
+            path: input
+        },
+        retries: 10
+    };
+
+    const { data } = await nango.post<DropboxTemporaryDownloadLink>(proxyConfig);
+
+    if (!data.metadata.is_downloadable) {
+        throw new nango.ActionError({
+            message: 'File is not downloadable',
+            data: data.metadata
+        });
+    }
+
+    const config: ProxyConfiguration = {
+        // https://www.dropbox.com/developers/documentation/http/documentation#files-get_temporary_link
+        endpoint: data.link,
+        responseType: 'arraybuffer',
+        retries: 10
+    };
+
+    const response = await nango.get(config);
+
+    const chunks: Buffer[] = [];
+    for await (const chunk of response.data) {
+        chunks.push(chunk);
+    }
+    const buffer = Buffer.concat(chunks);
+
+    return buffer.toString('base64');
+}

integrations/dropbox/mocks/files/Document/batchDelete.json

@@ -0,0 +1 @@
+[]

integrations/dropbox/mocks/files/Document/batchSave.json

@@ -0,0 +1,7 @@
+[
+    {
+        "id": "id:3fcG4EwxZfUAAAAAAAMQEg",
+        "title": "foo.com",
+        "path": "/nango/example/foo.com"
+    }
+]

integrations/dropbox/mocks/nango/getMetadata.json

@@ -0,0 +1,3 @@
+{
+    "files": ["foo"]
+}

integrations/dropbox/mocks/nango/getMetadata/files.json

@@ -0,0 +1,3 @@
+{
+    "files": ["foo"]
+}

integrations/dropbox/mocks/nango/post/proxy/2/files/get_metadata/files.json

@@ -0,0 +1,13 @@
+{
+    ".tag": "file",
+    "name": "foo.com",
+    "path_lower": "/nango/example/foo.com",
+    "path_display": "/nango/example/foo.com",
+    "id": "id:3fcG4EwxZfUAAAAAAAMQEg",
+    "client_modified": "2024-11-11T08:23:53Z",
+    "server_modified": "2024-11-11T08:23:57Z",
+    "rev": "6269ed26d20f900b0b30d",
+    "size": 460,
+    "is_downloadable": true,
+    "content_hash": "c4f28bc5220f4d5fc29f3a5f34da1ba4fe66210758041a4a07c74c1e98ff86d8"
+}

integrations/dropbox/mocks/nango/post/proxy/2/files/get_temporary_link/fetch-file.json

@@ -0,0 +1,15 @@
+{
+    "metadata": {
+        "name": "integrations.txt",
+        "path_lower": "/nango/test/integrations.txt",
+        "path_display": "/nango/test/integrations.txt",
+        "id": "id:3fcG4EwxZfUAAAAAAAMQEw",
+        "client_modified": "2024-11-11T08:24:06Z",
+        "server_modified": "2024-11-11T08:24:08Z",
+        "rev": "6269ed31f1e3e00b0b30d",
+        "size": 40,
+        "is_downloadable": true,
+        "content_hash": "00e050e1b9d380b3093fc46e96c7287fb1f3d3aafdc0eefca6a7d5c6c0cee542"
+    },
+    "link": "https://uc2877ef56e4d05243e9792126dd.dl.dropboxusercontent.com/cd/0/get/CeIdP4V5fqTP7YMldOx5LD4Z7ILhC4u2P4jYSMOQmomqK9EIU9IpaKryabn4nquqkNvg5j9c6X7fue6xPXCJw99ZT5xL1Pxc3L0OObZY68uug10SzlGD5NKg0f8qIhhSNdA61hs8pZ9hCyKlObr3IOcSLUIWadQ1YEGkwo0gaoPlQg/file"
+}

integrations/dropbox/mocks/nango/post/proxy/2/files/list_folder/continue/files.json

@@ -0,0 +1,39 @@
+{
+    "entries": [
+        {
+            ".tag": "folder",
+            "name": "example",
+            "path_lower": "/nango/example",
+            "path_display": "/nango/example",
+            "id": "id:3fcG4EwxZfUAAAAAAAMQEQ"
+        },
+        {
+            ".tag": "file",
+            "name": "foo.com",
+            "path_lower": "/nango/example/foo.com",
+            "path_display": "/nango/example/foo.com",
+            "id": "id:3fcG4EwxZfUAAAAAAAMQEg",
+            "client_modified": "2024-11-11T08:23:53Z",
+            "server_modified": "2024-11-11T08:23:57Z",
+            "rev": "6269ed26d20f900b0b30d",
+            "size": 460,
+            "is_downloadable": true,
+            "content_hash": "c4f28bc5220f4d5fc29f3a5f34da1ba4fe66210758041a4a07c74c1e98ff86d8"
+        },
+        {
+            ".tag": "file",
+            "name": "integrations.txt",
+            "path_lower": "/nango/test/integrations.txt",
+            "path_display": "/nango/test/integrations.txt",
+            "id": "id:3fcG4EwxZfUAAAAAAAMQEw",
+            "client_modified": "2024-11-11T08:24:06Z",
+            "server_modified": "2024-11-11T08:24:08Z",
+            "rev": "6269ed31f1e3e00b0b30d",
+            "size": 40,
+            "is_downloadable": true,
+            "content_hash": "00e050e1b9d380b3093fc46e96c7287fb1f3d3aafdc0eefca6a7d5c6c0cee542"
+        }
+    ],
+    "cursor": "AAGNZypUZDLtWS8HPGfP-G0t-dZlJ2m_nvX83JtvtuVKPJ7oKQ_rXMx1HLR9DZ8uijQ6AtAOQYhXDbeicP5CWs63m2YAwFHpZXO6srAvyev4UVRLelt2_18TZMWwz1d-xx90wkZbCAXsrFd9sdM3u1NcPiY8zy4kPCCLGKFwmZ7xhtfYF3PQd6tRtqtu-k89obsgdQddq2sVuzHgLQU0Eh2m",
+    "has_more": false
+}

integrations/dropbox/mocks/nango/post/proxy/2/files/list_folder/files.json

@@ -0,0 +1,20 @@
+{
+    "entries": [
+        {
+            ".tag": "folder",
+            "name": "nango",
+            "path_lower": "/nango",
+            "path_display": "/nango",
+            "id": "id:3fcG4EwxZfUAAAAAAAMQDw"
+        },
+        {
+            ".tag": "folder",
+            "name": "test",
+            "path_lower": "/nango/test",
+            "path_display": "/nango/test",
+            "id": "id:3fcG4EwxZfUAAAAAAAMQEA"
+        }
+    ],
+    "cursor": "AAHmqx25WE58QO_ia7UzBa8nm4TGIuBmHAVatpgvA1MCD3wI5enSbteq75P9cJkrX5GxM0z-5U2N23f82-UjdJHdrvuxUNrW8HTOw8JGAwlKs44SWldH2ATUjecoD-0xRQSLwEmI1mIS3GYbVL6yc_B1epFAr7ujIc8hSQdUSo60B_Z2jAIH9x1CbJr3xtjXSzd1lfYFVcEtMpKmK8Ia-txzj-xAlt6gFPnHM9RxzN43-A",
+    "has_more": true
+}

integrations/dropbox/nango.yaml

@@ -15,6 +15,15 @@ integrations:
                 input: IdEntity
                 scopes:
                     - members.delete
+            fetch-file:
+                description: Fetches the content of a file given its ID, processes the data using
+                    a response stream, and encodes it into a base64 string. This base64-encoded
+                    string can be used to recreate the file in its original format using an external tool.
+                endpoint: GET /fetch-file
+                output: string
+                input: string
+                scopes:
+                    - files.content.read
         syncs:
             users:
                 runs: every day
@@ -26,6 +35,18 @@ integrations:
                 endpoint: GET /users
                 scopes:
                     - members.read
+            files:
+                runs: every day
+                description: |
+                    Sync the metadata of a specified files or folders paths from Dropbox. A file or folder id or path can be used.
+                input: DocumentMetadata
+                auto_start: false
+                output: Document
+                sync_type: full
+                track_deletes: true
+                endpoint: GET /files
+                scopes:
+                    - files.metadata.read
 models:
     # Generic
     SuccessResponse:
@@ -44,3 +65,12 @@ models:
         firstName: string
         lastName: string
         email: string
+
+    DocumentMetadata:
+        files: string[] | undefined
+        folders: string[] | undefined
+
+    Document:
+        id: string
+        path: string
+        title: string