diff --git a/lib/chatbot-api/chatbot-s3-buckets/index.ts b/lib/chatbot-api/chatbot-s3-buckets/index.ts index 79ec464b..e7666d04 100644 --- a/lib/chatbot-api/chatbot-s3-buckets/index.ts +++ b/lib/chatbot-api/chatbot-s3-buckets/index.ts @@ -5,6 +5,7 @@ import { NagSuppressions } from "cdk-nag"; export class ChatBotS3Buckets extends Construct { public readonly filesBucket: s3.Bucket; + public readonly userFeedbackBucket: s3.Bucket; constructor(scope: Construct, id: string) { super(scope, id); @@ -39,7 +40,16 @@ export class ChatBotS3Buckets extends Construct { ], }); + const userFeedbackBucket = new s3.Bucket(this, "UserFeedbackBucket", { + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, + enforceSSL: true, + serverAccessLogsBucket: logsBucket, + }); + this.filesBucket = filesBucket; + this.userFeedbackBucket = userFeedbackBucket; /** * CDK NAG suppression diff --git a/lib/chatbot-api/functions/api-handler/index.py b/lib/chatbot-api/functions/api-handler/index.py index cfc2f888..8a7884e6 100644 --- a/lib/chatbot-api/functions/api-handler/index.py +++ b/lib/chatbot-api/functions/api-handler/index.py @@ -14,6 +14,7 @@ from routes.semantic_search import router as semantic_search_router from routes.documents import router as documents_router from routes.kendra import router as kendra_router +from routes.user_feedback import router as user_feedback_router tracer = Tracer() logger = Logger() @@ -30,6 +31,7 @@ app.include_router(semantic_search_router) app.include_router(documents_router) app.include_router(kendra_router) +app.include_router(user_feedback_router) @logger.inject_lambda_context( diff --git a/lib/chatbot-api/functions/api-handler/routes/user_feedback.py b/lib/chatbot-api/functions/api-handler/routes/user_feedback.py new file mode 100644 index 00000000..8098f8f1 --- /dev/null +++ b/lib/chatbot-api/functions/api-handler/routes/user_feedback.py @@ -0,0 +1,37 @@ +import genai_core.types +import genai_core.auth +import genai_core.user_feedback +from pydantic import BaseModel +from aws_lambda_powertools import Logger, Tracer +from aws_lambda_powertools.event_handler.appsync import Router + +tracer = Tracer() +router = Router() +logger = Logger() + + +class CreateUserFeedbackRequest(BaseModel): + sessionId: str + key: str + feedback: str + prompt: str + completion: str + model: str + + +@router.resolver(field_name="addUserFeedback") +@tracer.capture_method +def user_feedback(input: dict): + request = CreateUserFeedbackRequest(**input) + + userId = genai_core.auth.get_user_id(router) + + if userId is None: + raise genai_core.types.CommonError("User not found") + + result = genai_core.user_feedback.add_user_feedback( + request.sessionId, request.key, request.feedback, request.prompt, request.completion, request.model, userId) + + return { + "feedback_id": result["feedback_id"], + } diff --git a/lib/chatbot-api/index.ts b/lib/chatbot-api/index.ts index cf70c51c..55406b42 100644 --- a/lib/chatbot-api/index.ts +++ b/lib/chatbot-api/index.ts @@ -32,6 +32,7 @@ export class ChatBotApi extends Construct { public readonly sessionsTable: dynamodb.Table; public readonly byUserIdIndex: string; public readonly filesBucket: s3.Bucket; + public readonly userFeedbackBucket: s3.Bucket; public readonly graphqlApi: appsync.GraphqlApi; constructor(scope: Construct, id: string, props: ChatBotApiProps) { @@ -87,6 +88,7 @@ export class ChatBotApi extends Construct { sessionsTable: chatTables.sessionsTable, byUserIdIndex: chatTables.byUserIdIndex, api, + userFeedbackBucket: chatBuckets.userFeedbackBucket, }); const realtimeBackend = new RealtimeGraphqlApiBackend(this, "Realtime", { @@ -114,6 +116,7 @@ export class ChatBotApi extends Construct { this.messagesTopic = realtimeBackend.messagesTopic; this.sessionsTable = chatTables.sessionsTable; this.byUserIdIndex = chatTables.byUserIdIndex; + this.userFeedbackBucket = chatBuckets.userFeedbackBucket; this.filesBucket = chatBuckets.filesBucket; this.graphqlApi = api; diff --git a/lib/chatbot-api/rest-api.ts b/lib/chatbot-api/rest-api.ts index 0b09d910..a0ac6271 100644 --- a/lib/chatbot-api/rest-api.ts +++ b/lib/chatbot-api/rest-api.ts @@ -14,6 +14,7 @@ import { Shared } from "../shared"; import * as appsync from "aws-cdk-lib/aws-appsync"; import { parse } from "graphql"; import { readFileSync } from "fs"; +import * as s3 from "aws-cdk-lib/aws-s3"; export interface ApiResolversProps { readonly shared: Shared; @@ -22,6 +23,7 @@ export interface ApiResolversProps { readonly userPool: cognito.UserPool; readonly sessionsTable: dynamodb.Table; readonly byUserIdIndex: string; + readonly userFeedbackBucket: s3.Bucket; readonly modelsParameter: ssm.StringParameter; readonly models: SageMakerModelEndpoint[]; readonly api: appsync.GraphqlApi; @@ -62,6 +64,7 @@ export class ApiResolvers extends Construct { API_KEYS_SECRETS_ARN: props.shared.apiKeysSecret.secretArn, SESSIONS_TABLE_NAME: props.sessionsTable.tableName, SESSIONS_BY_USER_ID_INDEX_NAME: props.byUserIdIndex, + USER_FEEDBACK_BUCKET_NAME: props.userFeedbackBucket?.bucketName ?? "", UPLOAD_BUCKET_NAME: props.ragEngines?.uploadBucket?.bucketName ?? "", PROCESSING_BUCKET_NAME: props.ragEngines?.processingBucket?.bucketName ?? "", @@ -255,6 +258,7 @@ export class ApiResolvers extends Construct { props.shared.configParameter.grantRead(apiHandler); props.modelsParameter.grantRead(apiHandler); props.sessionsTable.grantReadWriteData(apiHandler); + props.userFeedbackBucket.grantReadWrite(apiHandler); props.ragEngines?.uploadBucket.grantReadWrite(apiHandler); props.ragEngines?.processingBucket.grantReadWrite(apiHandler); diff --git a/lib/chatbot-api/schema/schema.graphql b/lib/chatbot-api/schema/schema.graphql index bb10fd60..b9d7eee3 100644 --- a/lib/chatbot-api/schema/schema.graphql +++ b/lib/chatbot-api/schema/schema.graphql @@ -89,6 +89,10 @@ type DocumentResult @aws_cognito_user_pools { status: String } +type UserFeedbackResult @aws_cognito_user_pools { + feedback_id: String! +} + input DocumentSubscriptionStatusInput { workspaceId: String! documentId: String! @@ -235,6 +239,15 @@ type SessionHistoryItem @aws_cognito_user_pools { metadata: String } +input UserFeedbackInput { + sessionId: String! + key: Int! + feedback: String! + prompt: String! + completion: String! + model: String! +} + input TextDocumentInput { workspaceId: String! title: String! @@ -296,6 +309,7 @@ type Mutation { deleteWorkspace(workspaceId: String!): Boolean @aws_cognito_user_pools addTextDocument(input: TextDocumentInput!): DocumentResult @aws_cognito_user_pools + addUserFeedback(input: UserFeedbackInput!): UserFeedbackResult @aws_cognito_user_pools addQnADocument(input: QnADocumentInput!): DocumentResult @aws_cognito_user_pools setDocumentSubscriptionStatus( diff --git a/lib/shared/layers/python-sdk/python/genai_core/user_feedback.py b/lib/shared/layers/python-sdk/python/genai_core/user_feedback.py new file mode 100644 index 00000000..e0ce9f84 --- /dev/null +++ b/lib/shared/layers/python-sdk/python/genai_core/user_feedback.py @@ -0,0 +1,51 @@ +import os +import uuid +import boto3 +import json +from pydantic import BaseModel +from datetime import datetime + +dynamodb = boto3.resource("dynamodb") +s3_client = boto3.client("s3") + +USER_FEEDBACK_BUCKET_NAME = os.environ.get("USER_FEEDBACK_BUCKET_NAME") + + +def add_user_feedback( + sessionId: str, + key: str, + feedback: str, + prompt: str, + completion: str, + model: str, + userId: str +): + feedbackId = str(uuid.uuid4()) + timestamp = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + item = { + "feedbackId": feedbackId, + "sessionId": sessionId, + "userId": userId, + "key": key, + "prompt": prompt, + "completion": completion, + "model": model, + "feedback": feedback, + "createdAt": timestamp + } + + response = s3_client.put_object( + Bucket=USER_FEEDBACK_BUCKET_NAME, + Key=feedbackId, + Body=json.dumps(item), + ContentType="application/json", + StorageClass='STANDARD_IA', + ) + print(response) + + return { + "feedback_id": feedbackId + } + + \ No newline at end of file diff --git a/lib/user-interface/react-app/package-lock.json b/lib/user-interface/react-app/package-lock.json index ff818b70..71735f33 100644 --- a/lib/user-interface/react-app/package-lock.json +++ b/lib/user-interface/react-app/package-lock.json @@ -12,10 +12,14 @@ "@cloudscape-design/components": "^3.0.405", "@cloudscape-design/design-tokens": "^3.0.28", "@cloudscape-design/global-styles": "^1.0.13", + "@fortawesome/fontawesome-svg-core": "^6.4.2", + "@fortawesome/free-solid-svg-icons": "^6.4.2", + "@fortawesome/react-fontawesome": "^0.2.0", "aws-amplify": "^5.3.12", "luxon": "^3.4.3", "react": "^18.2.0", "react-dom": "^18.2.0", + "react-icons": "^4.12.0", "react-json-view-lite": "^0.9.8", "react-markdown": "^9.0.0", "react-router-dom": "^6.15.0", @@ -8012,6 +8016,51 @@ "tslib": "^2.4.0" } }, + "node_modules/@fortawesome/fontawesome-common-types": { + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.5.1.tgz", + "integrity": "sha512-GkWzv+L6d2bI5f/Vk6ikJ9xtl7dfXtoRu3YGE6nq0p/FFqA1ebMOAWg3XgRyb0I6LYyYkiAo+3/KrwuBp8xG7A==", + "hasInstallScript": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/@fortawesome/fontawesome-svg-core": { + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-svg-core/-/fontawesome-svg-core-6.5.1.tgz", + "integrity": "sha512-MfRCYlQPXoLlpem+egxjfkEuP9UQswTrlCOsknus/NcMoblTH2g0jPrapbcIb04KGA7E2GZxbAccGZfWoYgsrQ==", + "hasInstallScript": true, + "dependencies": { + "@fortawesome/fontawesome-common-types": "6.5.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@fortawesome/free-solid-svg-icons": { + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/@fortawesome/free-solid-svg-icons/-/free-solid-svg-icons-6.5.1.tgz", + "integrity": "sha512-S1PPfU3mIJa59biTtXJz1oI0+KAXW6bkAb31XKhxdxtuXDiUIFsih4JR1v5BbxY7hVHsD1RKq+jRkVRaf773NQ==", + "hasInstallScript": true, + "dependencies": { + "@fortawesome/fontawesome-common-types": "6.5.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@fortawesome/react-fontawesome": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@fortawesome/react-fontawesome/-/react-fontawesome-0.2.0.tgz", + "integrity": "sha512-uHg75Rb/XORTtVt7OS9WoK8uM276Ufi7gCzshVWkUJbHhh3svsUUeqXerrM96Wm7fRiDzfKRwSoahhMIkGAYHw==", + "dependencies": { + "prop-types": "^15.8.1" + }, + "peerDependencies": { + "@fortawesome/fontawesome-svg-core": "~1 || ~6", + "react": ">=16.3" + } + }, "node_modules/@hapi/hoek": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz", @@ -16589,6 +16638,14 @@ "react": ">=16" } }, + "node_modules/react-icons": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/react-icons/-/react-icons-4.12.0.tgz", + "integrity": "sha512-IBaDuHiShdZqmfc/TwHu6+d6k2ltNCf3AszxNmjJc1KUfXdEeRJOKyNvLmAHaarhzGmTSVygNdyu8/opXv2gaw==", + "peerDependencies": { + "react": "*" + } + }, "node_modules/react-is": { "version": "16.13.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", diff --git a/lib/user-interface/react-app/package.json b/lib/user-interface/react-app/package.json index 3d2d5b5e..463246a3 100644 --- a/lib/user-interface/react-app/package.json +++ b/lib/user-interface/react-app/package.json @@ -16,10 +16,14 @@ "@cloudscape-design/components": "^3.0.405", "@cloudscape-design/design-tokens": "^3.0.28", "@cloudscape-design/global-styles": "^1.0.13", + "@fortawesome/fontawesome-svg-core": "^6.4.2", + "@fortawesome/free-solid-svg-icons": "^6.4.2", + "@fortawesome/react-fontawesome": "^0.2.0", "aws-amplify": "^5.3.12", "luxon": "^3.4.3", "react": "^18.2.0", "react-dom": "^18.2.0", + "react-icons": "^4.12.0", "react-json-view-lite": "^0.9.8", "react-markdown": "^9.0.0", "remark-gfm": "^4.0.0", diff --git a/lib/user-interface/react-app/src/common/api-client/api-client.ts b/lib/user-interface/react-app/src/common/api-client/api-client.ts index fe68c544..c998e75a 100644 --- a/lib/user-interface/react-app/src/common/api-client/api-client.ts +++ b/lib/user-interface/react-app/src/common/api-client/api-client.ts @@ -9,6 +9,7 @@ import { SessionsClient } from "./sessions-client"; import { SemanticSearchClient } from "./semantic-search-client"; import { DocumentsClient } from "./documents-client"; import { KendraClient } from "./kendra-client"; +import { UserFeedbackClient } from "./user-feedback-client"; export class ApiClient { private _healthClient: HealthClient | undefined; @@ -21,6 +22,7 @@ export class ApiClient { private _semanticSearchClient: SemanticSearchClient | undefined; private _documentsClient: DocumentsClient | undefined; private _kendraClient: KendraClient | undefined; + private _userFeedbackClient: UserFeedbackClient | undefined; public get health() { if (!this._healthClient) { @@ -102,5 +104,13 @@ export class ApiClient { return this._kendraClient; } + public get userFeedback() { + if(!this._userFeedbackClient) { + this._userFeedbackClient = new UserFeedbackClient(this._appConfig); + } + + return this._userFeedbackClient; + } + constructor(protected _appConfig: AppConfig) {} } diff --git a/lib/user-interface/react-app/src/common/api-client/user-feedback-client.ts b/lib/user-interface/react-app/src/common/api-client/user-feedback-client.ts new file mode 100644 index 00000000..70452cb9 --- /dev/null +++ b/lib/user-interface/react-app/src/common/api-client/user-feedback-client.ts @@ -0,0 +1,31 @@ +import { GraphQLResult } from "@aws-amplify/api-graphql"; +import { API, GraphQLQuery } from "@aws-amplify/api"; +import { AddUserFeedbackMutation } from "../../API.ts"; +import { + addUserFeedback +} from "../../graphql/mutations.ts"; +import { FeedbackData } from "../../components/chatbot/types.ts"; + +export class UserFeedbackClient { + + async addUserFeedback(params: { + feedbackData: FeedbackData + } + ): Promise>> { + const result = API.graphql>({ + query: addUserFeedback, + variables: { + input: { + sessionId: params.feedbackData.sessionId, + key: params.feedbackData.key, + feedback: params.feedbackData.feedback, + prompt: params.feedbackData.prompt, + completion: params.feedbackData.completion, + model: params.feedbackData.model, + }, + }, + }); + return result; + } + +} diff --git a/lib/user-interface/react-app/src/components/chatbot/chat-message.tsx b/lib/user-interface/react-app/src/components/chatbot/chat-message.tsx index 4519c02e..2649bebc 100644 --- a/lib/user-interface/react-app/src/components/chatbot/chat-message.tsx +++ b/lib/user-interface/react-app/src/components/chatbot/chat-message.tsx @@ -27,11 +27,14 @@ import { getSignedUrl } from "./utils"; import "react-json-view-lite/dist/index.css"; import "../../styles/app.scss"; +import { FaThumbsUp, FaThumbsDown } from "react-icons/fa"; export interface ChatMessageProps { message: ChatBotHistoryItem; configuration?: ChatBotConfiguration; showMetadata?: boolean; + onThumbsUp: () => void; + onThumbsDown: () => void; } export default function ChatMessage(props: ChatMessageProps) { @@ -40,6 +43,7 @@ export default function ChatMessage(props: ChatMessageProps) { const [files, setFiles] = useState([] as ImageFile[]); const [documentIndex, setDocumentIndex] = useState("0"); const [promptIndex, setPromptIndex] = useState("0"); + const [selectedIcon, setSelectedIcon] = useState<1 | 0 | null>(null); useEffect(() => { const getSignedUrls = async () => { @@ -270,6 +274,26 @@ export default function ChatMessage(props: ChatMessageProps) { }, }} /> +
+ {(selectedIcon === 1 || selectedIcon === null) && ( + { + props.onThumbsUp(); + setSelectedIcon(1); + }} + /> + )} + {(selectedIcon === 0 || selectedIcon === null) && ( + { + props.onThumbsDown(); + setSelectedIcon(0); + }} + /> + )} +
)} {loading && ( diff --git a/lib/user-interface/react-app/src/components/chatbot/chat.tsx b/lib/user-interface/react-app/src/components/chatbot/chat.tsx index 1dd1701e..9f7b6bf8 100644 --- a/lib/user-interface/react-app/src/components/chatbot/chat.tsx +++ b/lib/user-interface/react-app/src/components/chatbot/chat.tsx @@ -3,6 +3,7 @@ import { ChatBotConfiguration, ChatBotHistoryItem, ChatBotMessageType, + FeedbackData, } from "./types"; import { SpaceBetween, StatusIndicator } from "@cloudscape-design/components"; import { v4 as uuidv4 } from "uuid"; @@ -79,6 +80,30 @@ export default function Chat(props: { sessionId?: string }) { })(); }, [appContext, props.sessionId]); + const handleFeedback = (feedbackType: 1 | 0, idx: number, message: ChatBotHistoryItem) => { + if (message.metadata.sessionId) { + const prompt = messageHistory[idx - 1]?.content; + const completion = message.content; + const model = message.metadata.modelId; + const feedbackData: FeedbackData = { + sessionId: message.metadata.sessionId as string, + key: idx, + feedback: feedbackType, + prompt: prompt, + completion: completion, + model: model as string + }; + addUserFeedback(feedbackData); + } + }; + + const addUserFeedback = async (feedbackData: FeedbackData) => { + if (!appContext) return; + + const apiClient = new ApiClient(appContext); + await apiClient.userFeedback.addUserFeedback({feedbackData}); + }; + return (
@@ -87,6 +112,8 @@ export default function Chat(props: { sessionId?: string }) { key={idx} message={message} showMetadata={configuration.showMetadata} + onThumbsUp={() => handleFeedback(1, idx, message)} + onThumbsDown={() => handleFeedback(0, idx, message)} /> ))} diff --git a/lib/user-interface/react-app/src/components/chatbot/multi-chat.tsx b/lib/user-interface/react-app/src/components/chatbot/multi-chat.tsx index 68ad1232..66b375c6 100644 --- a/lib/user-interface/react-app/src/components/chatbot/multi-chat.tsx +++ b/lib/user-interface/react-app/src/components/chatbot/multi-chat.tsx @@ -37,6 +37,7 @@ import { ChabotOutputModality, ChatBotHeartbeatRequest, ChatBotModelInterface, + FeedbackData, } from "./types"; import { LoadingStatus, ModelInterface } from "../../common/types"; import { getSelectedModelMetadata, updateMessageHistoryRef } from "./utils"; @@ -44,7 +45,7 @@ import LLMConfigDialog from "./llm-config-dialog"; import styles from "../../styles/chat.module.scss"; import { useNavigate } from "react-router-dom"; import { receiveMessages } from "../../graphql/subscriptions"; -import { sendQuery } from "../../graphql/mutations"; +import { sendQuery } from "../../graphql/mutations.ts"; import { Utils } from "../../common/utils"; export interface ChatSession { @@ -329,6 +330,31 @@ export default function MultiChat() { [ReadyState.UNINSTANTIATED]: "Uninstantiated", }[readyState]; + const handleFeedback = (feedbackType: 1 | 0, idx: number, message: ChatBotHistoryItem, messageHistory: ChatBotHistoryItem[]) => { + console.log("Message history: ", messageHistory); + if (message.metadata.sessionId) { + const prompt = messageHistory[idx - 1]?.content; + const completion = message.content; + const model = message.metadata.modelId; + const feedbackData: FeedbackData = { + sessionId: message.metadata.sessionId as string, + key: idx, + feedback: feedbackType, + prompt: prompt, + completion: completion, + model: model as string + }; + addUserFeedback(feedbackData); + } + }; + + const addUserFeedback = async (feedbackData: FeedbackData) => { + if (!appContext) return; + + const apiClient = new ApiClient(appContext); + await apiClient.userFeedback.addUserFeedback({feedbackData}); + }; + return (
@@ -484,6 +510,8 @@ export default function MultiChat() { key={idx} message={message} showMetadata={showMetadata} + onThumbsUp={() => handleFeedback(1, idx, message, val)} + onThumbsDown={() => handleFeedback(0, idx, message, val)} /> ))} diff --git a/lib/user-interface/react-app/src/components/chatbot/types.ts b/lib/user-interface/react-app/src/components/chatbot/types.ts index 747144ea..d61ffa2f 100644 --- a/lib/user-interface/react-app/src/components/chatbot/types.ts +++ b/lib/user-interface/react-app/src/components/chatbot/types.ts @@ -147,3 +147,12 @@ export enum ChabotOutputModality { Image = "IMAGE", Embedding = "EMBEDDING", } + +export interface FeedbackData { + sessionId: string; + key: number; + feedback: number; + prompt: string; + completion: string; + model: string; +} diff --git a/lib/user-interface/react-app/src/styles/chat.module.scss b/lib/user-interface/react-app/src/styles/chat.module.scss index 8a551623..83e75734 100644 --- a/lib/user-interface/react-app/src/styles/chat.module.scss +++ b/lib/user-interface/react-app/src/styles/chat.module.scss @@ -157,3 +157,37 @@ .markdownTable tr:nth-child(even) { background-color: awsui.$color-background-container-content; } + +.thumbsContainer { + display: flex; + align-items: center; + margin-top: 8px; +} + +.thumbsIcon { + cursor: pointer; + margin-right: 10px; + opacity: 0.5; +} + +/* Styles for thumbs up icon. Should be compatible with dark theme */ +.thumbsUp { + color: #539fe5; +} + +/* Styles for thumbs down icon. Should be compatible with dark theme */ +.thumbsDown { + color: #539fe5; +} + +/* Style for clicked state */ +.clicked { + opacity: 0.5; + pointer-events: none; /* Disable pointer events for a clicked icon */ +} + +/* Styles for selected icon */ +.thumbsIcon.selected { + opacity: 1 !important; + pointer-events: none; /* Disable pointer events for the selected icon */ +} \ No newline at end of file