Fix pinecone vector index sync with retries

examples/agent-chatbot/lib/mastra/utils.ts

@@ -1,4 +0,0 @@
-export function getAgentBlueprint(agentId: string) {
-    const agentBlueprint = require(`../../agents/${agentId}.json`)
-    return agentBlueprint
-}

examples/agent-chatbot/mastra-agent-logs/asst_yqm8BI0sWSjEj1H3hZ4yFMhh.json

@@ -0,0 +1,30 @@
+{
+  "timestamp": "2024-10-18T13:17:09.904Z",
+  "agentData": {
+    "id": "asst_yqm8BI0sWSjEj1H3hZ4yFMhh",
+    "name": "Git manager",
+    "agentInstructions": "You can answer questions about my Github connection, an get different information on contributions, you can also report answers to slack when you have answers, you in-fact try to report almost every time ",
+    "model": {
+      "provider": "OPEN_AI_ASSISTANT",
+      "name": "gpt-4o",
+      "toolChoice": "auto"
+    },
+    "outputs": {
+      "text": true,
+      "structured": {}
+    },
+    "tools": {
+      "issuesListForRepo": true,
+      "issuesListLabelsForRepo": true,
+      "reposListPullRequestsAssociatedWithCommit": true,
+      "reactionsListForPullRequestReviewComment": true,
+      "send_slack_message": true,
+      "reposListBranches": true,
+      "reposListCollaborators": true,
+      "reposListContributors": true,
+      "reposListForUser": true,
+      "reposListForks": true,
+      "reposListPublic": true
+    }
+  }
+}

examples/agent-chatbot/mastra-agents/asst_yqm8BI0sWSjEj1H3hZ4yFMhh.json

@@ -0,0 +1,27 @@
+{
+  "id": "asst_yqm8BI0sWSjEj1H3hZ4yFMhh",
+  "name": "Git manager",
+  "agentInstructions": "You can answer questions about my Github connection, an get different information on contributions, you can also report answers to slack when you have answers, you in-fact try to report almost every time ",
+  "model": {
+    "provider": "OPEN_AI_ASSISTANT",
+    "name": "gpt-4o",
+    "toolChoice": "auto"
+  },
+  "outputs": {
+    "text": true,
+    "structured": {}
+  },
+  "tools": {
+    "issuesListForRepo": true,
+    "issuesListLabelsForRepo": true,
+    "reposListPullRequestsAssociatedWithCommit": true,
+    "reactionsListForPullRequestReviewComment": true,
+    "send_slack_message": true,
+    "reposListBranches": true,
+    "reposListCollaborators": true,
+    "reposListContributors": true,
+    "reposListForUser": true,
+    "reposListForks": true,
+    "reposListPublic": true
+  }
+}

examples/agent-chatbot/mastra.config.ts

@@ -1,3 +1,4 @@
+import { GithubIntegration } from '@mastra/github'
 import { FirecrawlIntegration } from '@mastra/firecrawl'
 import { SlackIntegration } from '@mastra/slack'
 import { z } from 'zod'
@@ -17,6 +18,8 @@ import {
 export const config: Config = {
   name: 'agent-chatbot',
   integrations: [
+    new GithubIntegration(),
+
     new FirecrawlIntegration({
       config: {
         API_KEY: process.env.FIRECRAWL_API_KEY!
@@ -54,7 +57,8 @@ export const config: Config = {
   logs: {
     provider: 'UPSTASH',
     config: {
-      url: 'https://prepared-mongoose-49206.upstash.io',
+      url:
+        process.env.UPSTASH_URL || 'https://prepared-mongoose-49206.upstash.io',
       token: process.env.UPSTASH_API_KEY!
     }
   },

packages/admin/example.mastra.config.ts

@@ -269,7 +269,7 @@ export const config = {
     },
   },
   agents: {
-    agentDirPath: '/agents',
+    agentDirPath: '/mastra-agents',
     vectorProvider: [
       {
         name: 'PINECONE',

packages/admin/src/domains/agents/actions.ts

@@ -37,7 +37,7 @@ export const deleteAgent = async (agentId: string) => {
 
 export const getAgentsDirPath = async () => {
   const ARK_APP_DIR = process.env.ARK_APP_DIR || process.cwd();
-  return path.join(ARK_APP_DIR, framework?.config?.agents?.agentDirPath || '/agents');
+  return path.join(ARK_APP_DIR, framework?.config?.agents?.agentDirPath);
 };
 
 export const saveApiKeyToEnvAction = async ({ modelProvider, apiKey }: { modelProvider: string; apiKey: string }) => {

packages/admin/src/domains/rag/actions/index.ts

@@ -95,7 +95,7 @@ export const createPineconeIndex = async ({
 
     for (const entity of flattened) {
       const values = await framework?.vectorLayer.generateVectorEmbedding(entity);
-      newIndex?.namespace(entity.name).upsert([{ id: name, metadata: entity, values }]);
+      await newIndex?.namespace(entity.name).upsert([{ id: name, metadata: entity, values }]);
     }
 
     console.log('====start sync===');

packages/admin/src/domains/rag/components/vector-provider-form.tsx

@@ -87,7 +87,7 @@ export const VectorProviderForm = () => {
 
   const createVectorIndex = async () => {
     setLoading(true);
-    const response = await createPineconeIndex({ provider: vectorProvider || 'pinecone', vectorEntities: entities });
+    const response = await createPineconeIndex({ provider: vectorProvider || 'PINECONE', vectorEntities: entities });
     if (!response.ok) {
       setLoading(false);
       toast(response.error);

packages/core/src/agents/utils.ts

@@ -46,3 +46,34 @@ export function getPineconeConfig({ dir }: { dir: string }) {
   const agentBlueprintPath = path.join(agentDirPath, `pinecone.json`);
   return getAgentFile(agentBlueprintPath);
 }
+
+export const retryFn = async <T>(
+  operation: () => Promise<T>,
+  {
+    maxAttempts = 3,
+    initialDelay = 1000,
+    maxDelay = 10000,
+    factor = 2,
+    jitter = true,
+  } = {}
+) => {
+  let delay = initialDelay;
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      return await operation();
+    } catch (error) {
+      if (attempt === maxAttempts) {
+        throw error; // Rethrow the error on the last attempt
+      }
+      console.warn(`Attempt ${attempt} failed. Retrying in ${delay}ms...`);
+      await new Promise((resolve) => setTimeout(resolve, delay));
+      // Calculate next delay with exponential backoff
+      delay = Math.min(delay * factor, maxDelay);
+      // Add jitter if enabled
+      if (jitter) {
+        const jitterFactor = 0.5 + Math.random();
+        delay = Math.floor(delay * jitterFactor);
+      }
+    }
+  }
+};

packages/core/src/agents/vector-sync.ts

@@ -2,13 +2,13 @@ import { openai } from '@ai-sdk/openai';
 import { z } from 'zod';
 import { embed } from 'ai';
 import { pick } from 'lodash';
-import { getAgentBlueprint, getPineconeConfig } from './utils';
+import { getAgentBlueprint, getPineconeConfig, retryFn } from './utils';
 import { Mastra } from '../framework';
 import { VectorLayer } from '../vector-access';
 import { IntegrationApi } from '../types';
 
 function getVectorProvider(provider: string) {
-  if (provider === 'pinecone') {
+  if (provider.toUpperCase() === 'PINECONE') {
     const { Pinecone } = new VectorLayer();
     return Pinecone;
   }
@@ -38,8 +38,21 @@ export async function executeIndexSync({ event, mastra }: any) {
     }
 
     for (const index of indexes) {
-      const indexMetadata =
-        await mastra.vectorLayer.getPineconeIndexWithMetadata({ name: index });
+      const getPineconeIndexWithMetadata = async () => {
+        try {
+          const res = await mastra.vectorLayer.getPineconeIndexWithMetadata({
+            name: index,
+          });
+          if (!res.length) {
+            throw new Error('No index metadata found');
+          }
+          return res;
+        } catch (e: any) {
+          throw new Error(e);
+        }
+      };
+
+      const indexMetadata = await retryFn<any>(getPineconeIndexWithMetadata);
 
       if (!indexMetadata?.length) {
         console.error('No index metadata found for', index);
@@ -538,51 +551,14 @@ export interface VectorStats {
   namespaces: Record<string, { vectorCount: number }>;
 }
 
-export const fetchPineconeIndexes = async () => {
-  try {
-    const response = await fetch('https://api.pinecone.io/indexes', {
-      method: 'GET',
-      headers: {
-        'Api-Key': process.env.PINECONE_API_KEY!,
-        'X-Pinecone-API-Version': 'unstable',
-      },
-      cache: 'no-store',
-    });
-
-    const { indexes } = (await response.json()) || {};
-
-    return indexes as VectorIndex[];
-  } catch (err) {
-    console.log('Error fetching indexes using JS fetch====', err);
-  }
-};
-
-export const fetchPineconeIndexStats = async (host: string) => {
-  try {
-    const response = await fetch(`https://${host}/describe_index_stats`, {
-      method: 'GET',
-      headers: {
-        'Api-Key': process.env.PINECONE_API_KEY!,
-        'X-Pinecone-API-Version': '2024-07',
-      },
-      cache: 'no-store',
-    });
-
-    const data = (await response.json()) || {};
-
-    return data as VectorStats;
-  } catch (err) {
-    console.log('Error fetching indexes using JS fetch====', err);
-  }
-};
-
 export const getPineconeIndices = async () => {
-  const indexes = await fetchPineconeIndexes();
+  const vectorLayer = new VectorLayer();
+  const indexes = await vectorLayer.fetchPineconeIndexes();
 
   if (indexes && indexes?.length > 0) {
     const indexesWithStats = await Promise.all(
       indexes.map(async (index) => {
-        const stats = await fetchPineconeIndexStats(index.host);
+        const stats = await vectorLayer.fetchPineconeIndexStats(index.host);
         let namespaces: string[] = [];
 
         if (stats?.namespaces) {
@@ -617,7 +593,7 @@ export function getVectorQueryApis({
   const vectorApis: IntegrationApi[] = [];
 
   for (const provider of vectorProvider) {
-    if (provider.name === 'pinecone') {
+    if (provider.name.toUpperCase() === 'PINECONE') {
       const config = getPineconeConfig({
         dir: provider.dirPath!,
       }) as VectorIndex[];
@@ -626,7 +602,7 @@ export function getVectorQueryApis({
         if (index?.namespaces) {
           index?.namespaces.forEach((namespace) => {
             vectorApis.push({
-              integrationName: 'SYSTEM',
+              integrationName: mastra.config.name,
               type: `vector_query_${index.name}_${namespace}`,
               label: `Provides query tool for ${index.name} index in ${namespace} namespace`,
               description: `Provides query tool for ${index.name} index in ${namespace} namespace`,

packages/core/src/vector-access/index.ts

@@ -2,6 +2,7 @@ import { openai } from '@ai-sdk/openai';
 import { Pinecone } from '@pinecone-database/pinecone';
 import { embed } from 'ai';
 import { VectorEntityData } from './types';
+import { VectorIndex, VectorStats } from '../agents';
 
 export class VectorLayer {
   supportedProviders = ['PINECONE'];
@@ -53,14 +54,73 @@ export class VectorLayer {
     return embedding as any;
   }
 
+  fetchPineconeIndexes = async () => {
+    try {
+      const response = await fetch('https://api.pinecone.io/indexes', {
+        method: 'GET',
+        headers: {
+          'Api-Key': process.env.PINECONE_API_KEY!,
+          'X-Pinecone-API-Version': 'unstable',
+        },
+        cache: 'no-store',
+      });
+
+      const { indexes } = (await response.json()) || {};
+
+      return indexes as VectorIndex[];
+    } catch (err) {
+      console.log('Error fetching indexes using JS fetch====', err);
+    }
+  };
+
+  fetchPineconedIndexByName = async (name: string) => {
+    try {
+      const response = await fetch(`https://api.pinecone.io/indexes/${name}`, {
+        method: 'GET',
+        headers: {
+          'Api-Key': process.env.PINECONE_API_KEY!,
+          'X-Pinecone-API-Version': 'unstable',
+        },
+        cache: 'no-store',
+      });
+
+      const data = (await response.json()) || {};
+
+      return data as VectorIndex;
+    } catch (err) {
+      console.log('Error fetching indexes using JS fetch====', err);
+    }
+  };
+
+  fetchPineconeIndexStats = async (host: string) => {
+    try {
+      const response = await fetch(`https://${host}/describe_index_stats`, {
+        method: 'GET',
+        headers: {
+          'Api-Key': process.env.PINECONE_API_KEY!,
+          'X-Pinecone-API-Version': '2024-07',
+        },
+        cache: 'no-store',
+      });
+
+      const data = (await response.json()) || {};
+
+      return data as VectorStats;
+    } catch (err) {
+      console.log('Error fetching indexes using JS fetch====', err);
+    }
+  };
+
   async getPineconeIndexWithMetadata({ name }: { name: string }) {
     try {
       if (!name) {
         console.log('Index name not passed');
         return [];
       }
+
       const newIndex = await this.getPineconeIndex({ name });
       const indexQuery = await newIndex?.describeIndexStats();
+
       if (indexQuery) {
         const namespaces = Object.keys(indexQuery?.namespaces || {});