diff --git a/examples/agent-chatbot/lib/mastra/utils.ts b/examples/agent-chatbot/lib/mastra/utils.ts index 5fe9d4b7..e69de29b 100644 --- a/examples/agent-chatbot/lib/mastra/utils.ts +++ b/examples/agent-chatbot/lib/mastra/utils.ts @@ -1,4 +0,0 @@ -export function getAgentBlueprint(agentId: string) { - const agentBlueprint = require(`../../agents/${agentId}.json`) - return agentBlueprint -} diff --git a/examples/agent-chatbot/mastra-agent-logs/asst_yqm8BI0sWSjEj1H3hZ4yFMhh.json b/examples/agent-chatbot/mastra-agent-logs/asst_yqm8BI0sWSjEj1H3hZ4yFMhh.json new file mode 100644 index 00000000..175d791d --- /dev/null +++ b/examples/agent-chatbot/mastra-agent-logs/asst_yqm8BI0sWSjEj1H3hZ4yFMhh.json @@ -0,0 +1,30 @@ +{ + "timestamp": "2024-10-18T13:17:09.904Z", + "agentData": { + "id": "asst_yqm8BI0sWSjEj1H3hZ4yFMhh", + "name": "Git manager", + "agentInstructions": "You can answer questions about my Github connection, an get different information on contributions, you can also report answers to slack when you have answers, you in-fact try to report almost every time ", + "model": { + "provider": "OPEN_AI_ASSISTANT", + "name": "gpt-4o", + "toolChoice": "auto" + }, + "outputs": { + "text": true, + "structured": {} + }, + "tools": { + "issuesListForRepo": true, + "issuesListLabelsForRepo": true, + "reposListPullRequestsAssociatedWithCommit": true, + "reactionsListForPullRequestReviewComment": true, + "send_slack_message": true, + "reposListBranches": true, + "reposListCollaborators": true, + "reposListContributors": true, + "reposListForUser": true, + "reposListForks": true, + "reposListPublic": true + } + } +} \ No newline at end of file diff --git a/examples/agent-chatbot/mastra-agents/asst_yqm8BI0sWSjEj1H3hZ4yFMhh.json b/examples/agent-chatbot/mastra-agents/asst_yqm8BI0sWSjEj1H3hZ4yFMhh.json new file mode 100644 index 00000000..d98db32f --- /dev/null +++ b/examples/agent-chatbot/mastra-agents/asst_yqm8BI0sWSjEj1H3hZ4yFMhh.json @@ -0,0 +1,27 @@ +{ + "id": "asst_yqm8BI0sWSjEj1H3hZ4yFMhh", + "name": "Git manager", + "agentInstructions": "You can answer questions about my Github connection, an get different information on contributions, you can also report answers to slack when you have answers, you in-fact try to report almost every time ", + "model": { + "provider": "OPEN_AI_ASSISTANT", + "name": "gpt-4o", + "toolChoice": "auto" + }, + "outputs": { + "text": true, + "structured": {} + }, + "tools": { + "issuesListForRepo": true, + "issuesListLabelsForRepo": true, + "reposListPullRequestsAssociatedWithCommit": true, + "reactionsListForPullRequestReviewComment": true, + "send_slack_message": true, + "reposListBranches": true, + "reposListCollaborators": true, + "reposListContributors": true, + "reposListForUser": true, + "reposListForks": true, + "reposListPublic": true + } +} \ No newline at end of file diff --git a/examples/agent-chatbot/mastra.config.ts b/examples/agent-chatbot/mastra.config.ts index 774209f0..fe7b24fc 100644 --- a/examples/agent-chatbot/mastra.config.ts +++ b/examples/agent-chatbot/mastra.config.ts @@ -1,3 +1,4 @@ +import { GithubIntegration } from '@mastra/github' import { FirecrawlIntegration } from '@mastra/firecrawl' import { SlackIntegration } from '@mastra/slack' import { z } from 'zod' @@ -17,6 +18,8 @@ import { export const config: Config = { name: 'agent-chatbot', integrations: [ + new GithubIntegration(), + new FirecrawlIntegration({ config: { API_KEY: process.env.FIRECRAWL_API_KEY! @@ -54,7 +57,8 @@ export const config: Config = { logs: { provider: 'UPSTASH', config: { - url: 'https://prepared-mongoose-49206.upstash.io', + url: + process.env.UPSTASH_URL || 'https://prepared-mongoose-49206.upstash.io', token: process.env.UPSTASH_API_KEY! } }, diff --git a/packages/admin/example.mastra.config.ts b/packages/admin/example.mastra.config.ts index 3c9efa91..f9fba2b9 100644 --- a/packages/admin/example.mastra.config.ts +++ b/packages/admin/example.mastra.config.ts @@ -269,7 +269,7 @@ export const config = { }, }, agents: { - agentDirPath: '/agents', + agentDirPath: '/mastra-agents', vectorProvider: [ { name: 'PINECONE', diff --git a/packages/admin/agents/3349f9af-13c4-4004-b15e-df67f78cb7ff.json b/packages/admin/mastra-agents/3349f9af-13c4-4004-b15e-df67f78cb7ff.json similarity index 100% rename from packages/admin/agents/3349f9af-13c4-4004-b15e-df67f78cb7ff.json rename to packages/admin/mastra-agents/3349f9af-13c4-4004-b15e-df67f78cb7ff.json diff --git a/packages/admin/agents/a3d4ff6b-766d-479f-8817-03bfd4ebd41c.json b/packages/admin/mastra-agents/a3d4ff6b-766d-479f-8817-03bfd4ebd41c.json similarity index 100% rename from packages/admin/agents/a3d4ff6b-766d-479f-8817-03bfd4ebd41c.json rename to packages/admin/mastra-agents/a3d4ff6b-766d-479f-8817-03bfd4ebd41c.json diff --git a/packages/admin/agents/fcb7a50f-d456-4130-bdcf-4e7fa02e4845.json b/packages/admin/mastra-agents/fcb7a50f-d456-4130-bdcf-4e7fa02e4845.json similarity index 100% rename from packages/admin/agents/fcb7a50f-d456-4130-bdcf-4e7fa02e4845.json rename to packages/admin/mastra-agents/fcb7a50f-d456-4130-bdcf-4e7fa02e4845.json diff --git a/packages/admin/src/domains/agents/actions.ts b/packages/admin/src/domains/agents/actions.ts index 14a7fec1..85b37945 100644 --- a/packages/admin/src/domains/agents/actions.ts +++ b/packages/admin/src/domains/agents/actions.ts @@ -37,7 +37,7 @@ export const deleteAgent = async (agentId: string) => { export const getAgentsDirPath = async () => { const ARK_APP_DIR = process.env.ARK_APP_DIR || process.cwd(); - return path.join(ARK_APP_DIR, framework?.config?.agents?.agentDirPath || '/agents'); + return path.join(ARK_APP_DIR, framework?.config?.agents?.agentDirPath); }; export const saveApiKeyToEnvAction = async ({ modelProvider, apiKey }: { modelProvider: string; apiKey: string }) => { diff --git a/packages/admin/src/domains/rag/actions/index.ts b/packages/admin/src/domains/rag/actions/index.ts index 2339beee..822ca78d 100644 --- a/packages/admin/src/domains/rag/actions/index.ts +++ b/packages/admin/src/domains/rag/actions/index.ts @@ -95,7 +95,7 @@ export const createPineconeIndex = async ({ for (const entity of flattened) { const values = await framework?.vectorLayer.generateVectorEmbedding(entity); - newIndex?.namespace(entity.name).upsert([{ id: name, metadata: entity, values }]); + await newIndex?.namespace(entity.name).upsert([{ id: name, metadata: entity, values }]); } console.log('====start sync==='); diff --git a/packages/admin/src/domains/rag/components/vector-provider-form.tsx b/packages/admin/src/domains/rag/components/vector-provider-form.tsx index 6f2a7e12..43ce9c0b 100644 --- a/packages/admin/src/domains/rag/components/vector-provider-form.tsx +++ b/packages/admin/src/domains/rag/components/vector-provider-form.tsx @@ -87,7 +87,7 @@ export const VectorProviderForm = () => { const createVectorIndex = async () => { setLoading(true); - const response = await createPineconeIndex({ provider: vectorProvider || 'pinecone', vectorEntities: entities }); + const response = await createPineconeIndex({ provider: vectorProvider || 'PINECONE', vectorEntities: entities }); if (!response.ok) { setLoading(false); toast(response.error); diff --git a/packages/core/src/agents/utils.ts b/packages/core/src/agents/utils.ts index aa1520ed..fe8bcb1d 100644 --- a/packages/core/src/agents/utils.ts +++ b/packages/core/src/agents/utils.ts @@ -46,3 +46,34 @@ export function getPineconeConfig({ dir }: { dir: string }) { const agentBlueprintPath = path.join(agentDirPath, `pinecone.json`); return getAgentFile(agentBlueprintPath); } + +export const retryFn = async ( + operation: () => Promise, + { + maxAttempts = 3, + initialDelay = 1000, + maxDelay = 10000, + factor = 2, + jitter = true, + } = {} +) => { + let delay = initialDelay; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await operation(); + } catch (error) { + if (attempt === maxAttempts) { + throw error; // Rethrow the error on the last attempt + } + console.warn(`Attempt ${attempt} failed. Retrying in ${delay}ms...`); + await new Promise((resolve) => setTimeout(resolve, delay)); + // Calculate next delay with exponential backoff + delay = Math.min(delay * factor, maxDelay); + // Add jitter if enabled + if (jitter) { + const jitterFactor = 0.5 + Math.random(); + delay = Math.floor(delay * jitterFactor); + } + } + } +}; diff --git a/packages/core/src/agents/vector-sync.ts b/packages/core/src/agents/vector-sync.ts index 5811775b..e104fb31 100644 --- a/packages/core/src/agents/vector-sync.ts +++ b/packages/core/src/agents/vector-sync.ts @@ -2,13 +2,13 @@ import { openai } from '@ai-sdk/openai'; import { z } from 'zod'; import { embed } from 'ai'; import { pick } from 'lodash'; -import { getAgentBlueprint, getPineconeConfig } from './utils'; +import { getAgentBlueprint, getPineconeConfig, retryFn } from './utils'; import { Mastra } from '../framework'; import { VectorLayer } from '../vector-access'; import { IntegrationApi } from '../types'; function getVectorProvider(provider: string) { - if (provider === 'pinecone') { + if (provider.toUpperCase() === 'PINECONE') { const { Pinecone } = new VectorLayer(); return Pinecone; } @@ -38,8 +38,21 @@ export async function executeIndexSync({ event, mastra }: any) { } for (const index of indexes) { - const indexMetadata = - await mastra.vectorLayer.getPineconeIndexWithMetadata({ name: index }); + const getPineconeIndexWithMetadata = async () => { + try { + const res = await mastra.vectorLayer.getPineconeIndexWithMetadata({ + name: index, + }); + if (!res.length) { + throw new Error('No index metadata found'); + } + return res; + } catch (e: any) { + throw new Error(e); + } + }; + + const indexMetadata = await retryFn(getPineconeIndexWithMetadata); if (!indexMetadata?.length) { console.error('No index metadata found for', index); @@ -538,51 +551,14 @@ export interface VectorStats { namespaces: Record; } -export const fetchPineconeIndexes = async () => { - try { - const response = await fetch('https://api.pinecone.io/indexes', { - method: 'GET', - headers: { - 'Api-Key': process.env.PINECONE_API_KEY!, - 'X-Pinecone-API-Version': 'unstable', - }, - cache: 'no-store', - }); - - const { indexes } = (await response.json()) || {}; - - return indexes as VectorIndex[]; - } catch (err) { - console.log('Error fetching indexes using JS fetch====', err); - } -}; - -export const fetchPineconeIndexStats = async (host: string) => { - try { - const response = await fetch(`https://${host}/describe_index_stats`, { - method: 'GET', - headers: { - 'Api-Key': process.env.PINECONE_API_KEY!, - 'X-Pinecone-API-Version': '2024-07', - }, - cache: 'no-store', - }); - - const data = (await response.json()) || {}; - - return data as VectorStats; - } catch (err) { - console.log('Error fetching indexes using JS fetch====', err); - } -}; - export const getPineconeIndices = async () => { - const indexes = await fetchPineconeIndexes(); + const vectorLayer = new VectorLayer(); + const indexes = await vectorLayer.fetchPineconeIndexes(); if (indexes && indexes?.length > 0) { const indexesWithStats = await Promise.all( indexes.map(async (index) => { - const stats = await fetchPineconeIndexStats(index.host); + const stats = await vectorLayer.fetchPineconeIndexStats(index.host); let namespaces: string[] = []; if (stats?.namespaces) { @@ -617,7 +593,7 @@ export function getVectorQueryApis({ const vectorApis: IntegrationApi[] = []; for (const provider of vectorProvider) { - if (provider.name === 'pinecone') { + if (provider.name.toUpperCase() === 'PINECONE') { const config = getPineconeConfig({ dir: provider.dirPath!, }) as VectorIndex[]; @@ -626,7 +602,7 @@ export function getVectorQueryApis({ if (index?.namespaces) { index?.namespaces.forEach((namespace) => { vectorApis.push({ - integrationName: 'SYSTEM', + integrationName: mastra.config.name, type: `vector_query_${index.name}_${namespace}`, label: `Provides query tool for ${index.name} index in ${namespace} namespace`, description: `Provides query tool for ${index.name} index in ${namespace} namespace`, diff --git a/packages/core/src/vector-access/index.ts b/packages/core/src/vector-access/index.ts index 3e753793..c2ae2a9a 100644 --- a/packages/core/src/vector-access/index.ts +++ b/packages/core/src/vector-access/index.ts @@ -2,6 +2,7 @@ import { openai } from '@ai-sdk/openai'; import { Pinecone } from '@pinecone-database/pinecone'; import { embed } from 'ai'; import { VectorEntityData } from './types'; +import { VectorIndex, VectorStats } from '../agents'; export class VectorLayer { supportedProviders = ['PINECONE']; @@ -53,14 +54,73 @@ export class VectorLayer { return embedding as any; } + fetchPineconeIndexes = async () => { + try { + const response = await fetch('https://api.pinecone.io/indexes', { + method: 'GET', + headers: { + 'Api-Key': process.env.PINECONE_API_KEY!, + 'X-Pinecone-API-Version': 'unstable', + }, + cache: 'no-store', + }); + + const { indexes } = (await response.json()) || {}; + + return indexes as VectorIndex[]; + } catch (err) { + console.log('Error fetching indexes using JS fetch====', err); + } + }; + + fetchPineconedIndexByName = async (name: string) => { + try { + const response = await fetch(`https://api.pinecone.io/indexes/${name}`, { + method: 'GET', + headers: { + 'Api-Key': process.env.PINECONE_API_KEY!, + 'X-Pinecone-API-Version': 'unstable', + }, + cache: 'no-store', + }); + + const data = (await response.json()) || {}; + + return data as VectorIndex; + } catch (err) { + console.log('Error fetching indexes using JS fetch====', err); + } + }; + + fetchPineconeIndexStats = async (host: string) => { + try { + const response = await fetch(`https://${host}/describe_index_stats`, { + method: 'GET', + headers: { + 'Api-Key': process.env.PINECONE_API_KEY!, + 'X-Pinecone-API-Version': '2024-07', + }, + cache: 'no-store', + }); + + const data = (await response.json()) || {}; + + return data as VectorStats; + } catch (err) { + console.log('Error fetching indexes using JS fetch====', err); + } + }; + async getPineconeIndexWithMetadata({ name }: { name: string }) { try { if (!name) { console.log('Index name not passed'); return []; } + const newIndex = await this.getPineconeIndex({ name }); const indexQuery = await newIndex?.describeIndexStats(); + if (indexQuery) { const namespaces = Object.keys(indexQuery?.namespaces || {});