diff --git a/.env.local.example b/.env.local.example index 6f7e8b6..4f82427 100644 --- a/.env.local.example +++ b/.env.local.example @@ -31,4 +31,7 @@ UPSTASH_REDIS_REST_TOKEN=AZ**** # Twilio related environment variables TWILIO_ACCOUNT_SID=AC*** -TWILIO_AUTH_TOKEN=***** \ No newline at end of file +TWILIO_AUTH_TOKEN=***** + +# Steamship related environment variables +STEAMSHIP_API_KEY=**** \ No newline at end of file diff --git a/.gitignore b/.gitignore index d489e57..f1c6735 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,10 @@ yarn-error.log* next-env.d.ts /.env.prod -/fly.toml \ No newline at end of file +/fly.toml + +# JetBrains +.idea + +# Yarn Lockfiles (since this project uses NPM) +yarn.lock \ No newline at end of file diff --git a/README.md b/README.md index 8b5352c..fb2a453 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ e. **Upstash API key** -e. **Supabase API key** (optional) +f. **Supabase API key** (optional) If you prefer to use Supabase, you will need to uncomment `VECTOR_DB=supabase` and fill out the Supabase credentials in `.env.local`. - Create a Supabase instance [here](https://supabase.com/dashboard/projects); then go to Project Settings -> API @@ -122,6 +122,16 @@ If you prefer to use Supabase, you will need to uncomment `VECTOR_DB=supabase` a - `SUPABASE_PRIVATE_KEY` is the key starts with `ey` under Project API Keys - Now, you should enable pgvector on Supabase and create a schema. You can do this easily by clicking on "SQL editor" on the left hand side on Supabase UI and then clicking on "+New Query". Copy paste [this code snippet](https://github.com/a16z-infra/ai-getting-started/blob/main/pgvector.sql) in the SQL editor and click "Run". +g. **Steamship API key** + +You can connect a Steamship agent instance as an LLM with personality, voice and image generation capabilities built in. It also includes its own vector storage and tools. To do so: + +- Create an account on [Steamship](https://steamship.com/account) +- Copy the API key from your account settings page +- Add it as the `STEAMSHIP_API_KEY` variable + +If you'd like to create your own character personality, add a custom voice, or use a different image model, visit [Steamship Agent Guidebook](https://www.steamship.com/learn/agent-guidebook), create your own instance and connect it in `companions.json` using the *Rick* example as a guide. + ### 4. Generate embeddings The `companions/` directory contains the "personalities" of the AIs in .txt files. To generate embeddings and load them into the vector database to draw from during the chat, run the following command: diff --git a/companions/companions.json b/companions/companions.json index 60cfbd5..b1f43e7 100644 --- a/companions/companions.json +++ b/companions/companions.json @@ -20,6 +20,15 @@ "llm": "vicuna13b", "phone": "OPTIONAL_COMPANION_PHONE_NUMBER" }, + { + "name": "Rick", + "title": "I can generate voice and pictures", + "imageUrl": "/rick.jpeg", + "llm": "steamship", + "generateEndpoint": "https://a16z.steamship.run/rick/ai-companion-59f5d9816b627a45856239ae9f83525e/answer", + "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", + "telegramLink": "https://t.me/rick_a16z_bot" + }, { "name": "Sebastian", "title": "I'm a travel blogger and a mystery novel writer", diff --git a/package-lock.json b/package-lock.json index 0abc3ea..5c3c98d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,6 +34,7 @@ "react-tooltip": "^5.16.1", "replicate": "^0.9.3", "tailwindcss": "3.3.2", + "ts-md5": "^1.3.1", "twilio": "^4.12.0", "typescript": "5.1.3" }, @@ -5929,6 +5930,14 @@ "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==" }, + "node_modules/ts-md5": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/ts-md5/-/ts-md5-1.3.1.tgz", + "integrity": "sha512-DiwiXfwvcTeZ5wCE0z+2A9EseZsztaiZtGrtSaY5JOD7ekPnR/GoIVD5gXZAlK9Na9Kvpo9Waz5rW64WKAWApg==", + "engines": { + "node": ">=12" + } + }, "node_modules/tsconfig-paths": { "version": "3.14.2", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz", diff --git a/package.json b/package.json index a69d80b..43058c1 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "react-tooltip": "^5.16.1", "replicate": "^0.9.3", "tailwindcss": "3.3.2", + "ts-md5": "^1.3.1", "twilio": "^4.12.0", "typescript": "5.1.3" }, diff --git a/public/rick.jpeg b/public/rick.jpeg new file mode 100644 index 0000000..0840ba3 Binary files /dev/null and b/public/rick.jpeg differ diff --git a/src/app/api/steamship/route.ts b/src/app/api/steamship/route.ts new file mode 100644 index 0000000..c68eff1 --- /dev/null +++ b/src/app/api/steamship/route.ts @@ -0,0 +1,105 @@ +import dotenv from "dotenv"; +import clerk from "@clerk/clerk-sdk-node"; +import { NextResponse } from "next/server"; +import { currentUser } from "@clerk/nextjs"; +import { rateLimit } from "@/app/utils/rateLimit"; +import {Md5} from 'ts-md5' +import ConfigManager from "@/app/utils/config"; + +dotenv.config({ path: `.env.local` }); + +function returnError(code: number, message: string) { + return new NextResponse( + JSON.stringify({ Message: message }), + { + status: code, + headers: { + "Content-Type": "application/json", + }, + } + ); +} + +export async function POST(req: Request) { + let clerkUserId; + let user; + let clerkUserName; + const { prompt, isText, userId, userName } = await req.json(); + const companionName = req.headers.get("name"); + + // Load the companion config + const configManager = ConfigManager.getInstance(); + const companionConfig = configManager.getConfig("name", companionName); + if (!companionConfig) { + return returnError(404, `Hi, we were unable to find the configuration for a companion named ${companionName}.`) + } + + // Make sure we're not rate limited + const identifier = req.url + "-" + (userId || "anonymous"); + const { success } = await rateLimit(identifier); + if (!success) { + console.log("INFO: rate limit exceeded"); + return returnError(429, `Hi, the companions can't talk this fast.`) + } + + if (!process.env.STEAMSHIP_API_KEY) { + return returnError(500, `Please set the STEAMSHIP_API_KEY env variable and make sure ${companionName} is connected to an Agent instance that you own.`) + } + + console.log(`Companion Name: ${companionName}`) + console.log(`Prompt: ${prompt}`); + + if (isText) { + clerkUserId = userId; + clerkUserName = userName; + } else { + user = await currentUser(); + clerkUserId = user?.id; + clerkUserName = user?.firstName; + } + + if (!clerkUserId || !!!(await clerk.users.getUser(clerkUserId))) { + console.log("user not authorized"); + return new NextResponse( + JSON.stringify({ Message: "User not authorized" }), + { + status: 401, + headers: { + "Content-Type": "application/json", + }, + } + ); + } + + // Create a chat session id for the user + const chatSessionId = Md5.hashStr(userId || "anonymous"); + + // Make sure we have a generate endpoint. + // TODO: Create a new instance of the agent per user if this proves advantageous. + const agentUrl = companionConfig.generateEndpoint + if (!agentUrl) { + return returnError(500, `Please add a Steamship 'generateEndpoint' to your ${companionName} configuration in companions.json.`) + } + + // Invoke the generation. Tool invocation, chat history management, backstory injection, etc is all done within this endpoint. + // To build, deploy, and host your own multi-tenant agent see: https://www.steamship.com/learn/agent-guidebook + const response = await fetch(agentUrl, { + method: "POST", + headers: { + "Content-Type": "application/json", + "Authorization": `Bearer ${process.env.STEAMSHIP_API_KEY}` + }, + body: JSON.stringify({ + question: prompt, + chat_session_id: chatSessionId + }) + }); + + if (response.ok) { + const responseText = await response.text() + const responseBlocks = JSON.parse(responseText) + return NextResponse.json(responseBlocks) + } else { + return returnError(500, await response.text()) + } +} diff --git a/src/components/ChatBlock.tsx b/src/components/ChatBlock.tsx new file mode 100644 index 0000000..9584f81 --- /dev/null +++ b/src/components/ChatBlock.tsx @@ -0,0 +1,69 @@ +/* + * Represents a unit of multimodal chat: text, video, audio, or image. + * + * For streaming responses, just update the `text` argument. + */ +export function ChatBlock({text, mimeType, url} : { + text?: string, + mimeType?: string, + url?: string +}) { + let internalComponent = <>> + if (text) { + internalComponent = {text} + } else if (mimeType && url) { + if (mimeType.startsWith("audio")) { + internalComponent = + } else if (mimeType.startsWith("video")) { + internalComponent = + } else if (mimeType.startsWith("image")) { + internalComponent = + } + } else if (url) { + internalComponent = Link + } + + return ( +
+ {internalComponent} +
+ ); +} + +/* + * Take a completion, which may be a string, JSON encoded as a string, or JSON object, + * and produce a list of ChatBlock objects. This is intended to be a one-size-fits-all + * method for funneling different LLM output into structure that supports different media + * types and can easily grow to support more metadata (such as speaker). + */ +export function responseToChatBlocks(completion: any) { + // First we try to parse completion as JSON in case we're dealing with an object. + console.log("got completoin", completion, typeof completion) + if (typeof completion == "string") { + try { + completion = JSON.parse(completion) + } catch { + // Do nothing; we'll just treat it as a string. + console.log("Couldn't parse") + } + } + let blocks = [] + if (typeof completion == "string") { + console.log("still string") + blocks.push({completion}
+ {blocks}