Skip to content

Commit

Permalink
Merge branch 'main' into add-tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@ejfox
ejfox committed Mar 28, 2024
2 parents 9f64370 + e360a38 commit 0802abf
Show file tree
Hide file tree
Showing 8 changed files with 238 additions and 267 deletions.
101 changes: 25 additions & 76 deletions api.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,45 +21,6 @@ const BOT_NAME = process.env.BOT_NAME;

app.use(express.json());


app.post("/api/message", async (req, res) => {
const { processMessageChain } = await require("./src/chain");
const message = req.body.message;
const username = req.body.username || "API User";

const processedMessage = await processMessageChain(
[
{
role: "user",
content: message,
},
],
username,
);

res.json({ response: processedMessage });
});

app.post("/api/message-image", async (req, res) => {
const { processMessageChain } = await require("./src/chain");
const message = req.body.message;
const image = req.body.image;
const username = req.body.username || "API User";

const processedMessage = await processMessageChain(
[
{
role: "user",
content: message,
image: image,
},
],
{ username },
);

res.json({ response: processedMessage });
});

const server = app.listen(port, "0.0.0.0", () => {
logger.info(`Server is running on port ${port}`);
});
Expand All @@ -72,31 +33,6 @@ server.on("error", (err) => {
}
});

/* These are endpoints to interact with Missive- we will have one endpoint that simply "receives" information, parses it and adds it to memory
Then we will also have a second endpoint that does all of the above, and then gets the message ID and uses POST to send the response to the location the webhook came from */

// app.post("/api/missive-read", async (req, res) => {
// const username = req.body.username || "API User";

// const body = req.body;
// const webhookDescription = `${body?.rule?.description}`;

// await processMessageChain(
// [
// {
// role: "user",
// content: `New data received from webhook: ${webhookDescription} \n ${req.body.message}`,
// },
// ],
// username
// );

// // res.json({ response: processedMessage });
// // just give a 200 response
// res.status(200).end();
// });

async function listMessages(emailMessageId) {
let url = `${apiFront}/conversations/${emailMessageId}/messages`;

Expand All @@ -113,14 +49,12 @@ async function listMessages(emailMessageId) {
const response = await fetch(url, options);
const data = await response.json();

logger.info(`Data: ${JSON.stringify(data)}`);
/*
{"messages":[{"id":"72f9162e-22cd-4f11-ae91-0b2223f688b1","subject":"[FYI] HubSpot is launching a new pricing model","preview":"Please read to learn what this means for your account as an existing customer.","type":"email","delivered_at":1709661736,"updated_at":1709661751,"created_at":1709661751,"email_message_id":"<1709661726112.203e1867-4f38-4c3e-b30f-18a4cca232e1@bf1.hubspot.com>","in_reply_to":[],"references":[],"from_field":{"name":"The HubSpot Team","address":"pricing@hubspot.com"},"to_fields":[{"name":null,"address":"ejfox@room302.studio"}],"cc_fields":[],"bcc_fields":[],"reply_to_fields":[{"name":null,"address":"pricing@hubspot.com"}],"attachments":[],"author":null}]}
*/
// logger.info(`Data: ${JSON.stringify(data)}`);
return data.messages;
}

function processWebhookPayload(payload) {
const userMessage = payload.userMessage;
const userMessage = payload.comment.body
const userName = payload.userName;
const userEmail = payload.userEmail;
const conversationId = payload.conversationId;
Expand Down Expand Up @@ -160,6 +94,7 @@ function processWebhookPayload(payload) {

return simplifiedPayload;
}

async function processMissiveRequest(body) {
// Require the processMessageChain function from the chain module
const { processMessageChain } = await require("./src/chain");
Expand Down Expand Up @@ -198,9 +133,11 @@ async function processMissiveRequest(body) {
// Fetch the attachment description using the vision API
const attachmentDescription = await vision.fetchImageDescription();


// Use the Missive conversationId as the channel
// Store the attachment description as a memory in the database
await storeUserMemory(
{ username, channel: conversationId, guild: "missive" },
{ username, channel: conversationId, guild: "missive", related_message_id: body.comment.id },
`Attachment ${body.comment.attachment.filename}: ${attachmentDescription}`,
"attachment",
resourceId
Expand Down Expand Up @@ -236,7 +173,7 @@ async function processMissiveRequest(body) {
}
} else {
// Log if no attachment is found in the comment
logger.info(`No attachment found in body.comment`);
// logger.info(`No attachment found in body.comment`);
}

// Fetch the context messages for the conversation from the database
Expand Down Expand Up @@ -290,7 +227,7 @@ async function processMissiveRequest(body) {
processedMessage = await processMessageChain(allMessages, {
username,
channel: conversationId,
guild: "missive",
guild: "missive"
});
} catch (error) {
// Log any errors that occur during message chain processing
Expand Down Expand Up @@ -366,11 +303,16 @@ app.post("/api/missive-reply", async (req, res) => {
});
});

app.post("/api/webhook-reply", async (req, res) => {
app.post("/api/webhook-prompt", async (req, res) => {
const { getPromptsFromSupabase } = require("./helpers.js");
const { processMessageChain } = await require('./src/chain.js');

// this is will be an authorized call from pgcron to send a request to the robot as if a user sent, but specifiying a prompt from the prompts table to use

const passphrase = process.env.WEBHOOK_PASSPHRASE; // Assuming PASSPHRASE is the environment variable name

logger.info(JSON.stringify(req.body));

// use basicauth to make sure passphrase in body matches passphrase in env
let payloadPassword = req.body.password;
if (payloadPassword !== passphrase) {
Expand All @@ -389,8 +331,14 @@ app.post("/api/webhook-reply", async (req, res) => {
// get the prompt from the prompt table
const allPrompts = await getPromptsFromSupabase();

logger.info(`All prompts: ${JSON.stringify(allPrompts)}`);

// look for the prompt slug in allPrompts
let prompt = allPrompts.find((p) => p.slug === promptSlug);
// let prompt = allPrompts.find((p) => p.slug === promptSlug);

let prompt = allPrompts[promptSlug];

logger.info(`Prompt: ${JSON.stringify(prompt)}`);

// if prompt is null / undefined make it an empty string
if (!prompt) {
Expand All @@ -402,10 +350,11 @@ app.post("/api/webhook-reply", async (req, res) => {
{
role: "user",
// content: message,
content: `${prompt.prompt} \n ${message}`
content: `${prompt.prompt_text} \n ${message}`
},
],
username,
{username},

);

logger.info(`Processed message: ${JSON.stringify(processedMessage)}`);
Expand Down
69 changes: 48 additions & 21 deletions capabilities/ingest.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ const cheerio = require("cheerio");
const dotenv = require("dotenv");
dotenv.config();
const { webPageToText, webpageToHTML } = require("./web.js"); // Adjust the path as necessary
const { destructureArgs, createChatCompletion } = require("../helpers");
const { storeUserMemory } = require("../src/remember");
const { destructureArgs, createChatCompletion, getPromptsFromSupabase } = require("../helpers");
const { storeUserMemory, hasMemoryOfResource, deleteMemoriesOfResource, getResourceMemories } = require("../src/remember");
const logger = require("../src/logger.js")("ingest-capability");
const { convert } = require("html-to-text");
const fs = require("fs");
Expand Down Expand Up @@ -37,33 +37,62 @@ async function deepDocumentIngest(url) {
// For testing:
// node capability-player.js --runCapability="ingest:deepDocumentIngest(https://docs.pdw.co/tachio-overview)"

// TODO: Cache the text-ified version of the URL for a certain amount of time
// Generate a hash for the URL
// const urlHash = crypto.createHash("md5").update(url).digest("hex");
// const cacheFilePath = path.join(cacheDir, `${urlHash}.json`);
const { PROMPT_DEEP_INGEST } = await getPromptsFromSupabase();

// // Check if cache exists and is recent (e.g., less than 1 hour old)
// if (
// fs.existsSync(cacheFilePath) &&
// Date.now() - fs.statSync(cacheFilePath).mtimeMs < 3600000
// ) {
// console.log("Using cached data");
// return fs.readFileSync(cacheFilePath, "utf8");
// }
// Generate a hash for the URL to use as a cache identifier
const urlHash = crypto.createHash("md5").update(url).digest("hex");
const cacheFilePath = path.join(cacheDir, `${urlHash}.json`);

// Check if cache exists and is recent (e.g., less than 1 hour old)
let cachedData = null;
if (
fs.existsSync(cacheFilePath) &&
Date.now() - fs.statSync(cacheFilePath).mtimeMs < 3600000
) {
console.log("Using cached data");
cachedData = JSON.parse(fs.readFileSync(cacheFilePath, "utf8"));
}

try {
const { html } = await webpageToHTML(url);
const document = convert(html, {
wordwrap: 130,
});

// check if we have memories about this URL *already*
const hasMemory = await hasMemoryOfResource(
url
);

// if we DO have memories, delete them
if (hasMemory) {
// get the date of the previous ingest from created_at
const resourceMemories = await getResourceMemories(url, 1);
const prevImportDate = resourceMemories[0].created_at;


// delete all the memories about this URL
const memoryDeleteResult = await deleteMemoriesOfResource(url);
logger.info(memoryDeleteResult);

// make a new memory that the document was re-ingested
const updateMessage = `We previously ingested this document on ${prevImportDate}. We re-ingested it at ${new Date().toISOString()} and removed our previous memories.`;
await storeUserMemory(
{ username: "capability-deepdocumentingest", guild: "" },
updateMessage,
"capability-deepdocumentingest",
url,
);

}

const messages = [
{
role: "user",
content: `Can you please write an extremely long and thorough reiteration of the following document:
${JSON.stringify(document, null, 2)}
When analyzing this document, your goal is to distill its content into concise, standalone facts, as many as you possibly can. Each fact should encapsulate a key piece of information, complete in itself, and easily understandable without needing further context. Pay special attention to precise details, especially if they involve code or search queries - accuracy in phrasing is crucial here. It's important to include relevant URLs, specific search queries, project IDs that are associated with these facts. Respond ONLY with the facts, do not greet me or confirm the request. Keep your response above 1000 words and below 5000 words, please.
${PROMPT_DEEP_INGEST}
Make separate sections of facts for each section of the document, using \`\`\`---\`\`\` between each section. Respond immediately, beginning with the first section, no introductions or confirmation.`,
},
Expand All @@ -72,18 +101,15 @@ Make separate sections of facts for each section of the document, using \`\`\`--
max_tokens: 4000,
});

// because the robot was instructed to deliniate the facts with '---' we can split the response into facts
// we need to be aware the first fact MAY be blank
const facts = completion.split("\n---\n");

// now that each fact is separated we can store them in the database
facts.forEach(async (fact, index) => {
const factAsMemory = `Memory about RESOURCE_ID: ${url}\n${fact}
(${index + 1}/${facts.length})
`;
await storeUserMemory(
{ username: "capability-deepdocumentingest", guild: "" },
fact,
factAsMemory,
"capability",
url,
);
Expand All @@ -104,20 +130,21 @@ Make separate sections of facts for each section of the document, using \`\`\`--
},
);

// Store the meta-summary in the database
await storeUserMemory(
{ username: "capability-deepdocumentingest", guild: "" },
metaSummaryCompletion,
"capability-deepdocumentingest",
url,
);

// Cache the current document for future reference
fs.writeFileSync(cacheFilePath, JSON.stringify({ document, facts, metaSummary: metaSummaryCompletion }), "utf8");

return `Document ingested successfully. ${facts.length} groups of facts were extracted from the ${url}.`;
} catch (error) {
throw new Error(`Error occurred while making external request: ${error}`);
}
}

module.exports = {
handleCapabilityMethod,
};
8 changes: 6 additions & 2 deletions capabilities/todo.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
const { supabase, destructureArgs } = require("../helpers");
const { logger } = require("../src/logger")('capability-supabasetodo')
const { destructureArgs } = require("../helpers");
const logger = require("../src/logger")('capability-supabasetodo')
/**
* Creates a new todo item in the database. This capability allows for the creation of a new todo item within a specified project. It supports optional details such as description, status, priority, due date, external URLs, and attachments, making it flexible for various use cases. The function defaults to setting the todo's status to "To Do" if not specified, ensuring a new todo is actionable immediately upon creation.
* When to Use: Use this capability when a new task arises that needs tracking within a project's context. It's suitable for user-driven todo creation based on input or automated task generation from project activities or milestones.
Expand All @@ -13,6 +13,7 @@ const { logger } = require("../src/logger")('capability-supabasetodo')
* @returns {Promise<string>} A promise that resolves to a success message.
*/
async function createTodo(name, description = "") {
const { supabase } = require("../src/supabaseclient");
const { data, error } = await supabase.from("todos").insert([
{
name,
Expand All @@ -38,6 +39,7 @@ async function createTodo(name, description = "") {
* @returns {Promise<boolean>} A promise that resolves to true if the deletion was successful, false otherwise.
*/
async function deleteTodo(todoId) {
const { supabase } = require("../src/supabaseclient");
const { data, error } = await supabase
.from("todos")
.delete()
Expand All @@ -61,6 +63,7 @@ async function deleteTodo(todoId) {
* @returns {Promise<Object>} A promise that resolves to the updated todo item.
*/
async function updateTodo(todoId, updates) {
const { supabase } = require("../src/supabaseclient");
const { data, error } = await supabase
.from("todos")
.update(updates)
Expand Down Expand Up @@ -95,6 +98,7 @@ Process Response: Use the updated todo item returned by the function to verify t
*
*/
async function listTodos() {
const { supabase } = require("../src/supabaseclient");
const { data, error } = await supabase.from("todos").select("*");

if (error) throw new Error(error.message);
Expand Down
Loading

0 comments on commit 0802abf

Please sign in to comment.