Skip to content

Commit

Permalink
Video Chat の修正 (#781)
Browse files Browse the repository at this point in the history
  • Loading branch information
@maekawataiki
maekawataiki authored Dec 18, 2024
1 parent acd1e9f commit 062b506
Show file tree
Hide file tree
Showing 8 changed files with 91 additions and 52 deletions.
11 changes: 11 additions & 0 deletions packages/cdk/lambda/utils/models.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,17 @@ const createConverseCommandInput = (
},
},
} as ContentBlock.VideoMember);
} else if (extra.type === 'video' && extra.source.type === 's3') {
contentBlocks.push({
video: {
format: extra.source.mediaType.split('/')[1],
source: {
s3Location: {
uri: extra.source.data,
},
},
},
} as ContentBlock.VideoMember);
}
});
}
Expand Down
2 changes: 1 addition & 1 deletion packages/cdk/lib/construct/api.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ export class Api extends Construct {
],
},
});
fileBucket.grantWrite(predictStreamFunction);
fileBucket.grantReadWrite(predictStreamFunction);
predictStreamFunction.grantInvoke(idPool.authenticatedRole);

// Prompt Flow Lambda Function の追加
Expand Down
3 changes: 3 additions & 0 deletions packages/cdk/lib/generative-ai-use-cases-stack.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -306,5 +306,8 @@ export class GenerativeAiUseCasesStack extends Stack {

this.userPool = auth.userPool;
this.userPoolClient = auth.client;

this.exportValue(this.userPool.userPoolId);
this.exportValue(this.userPoolClient.userPoolClientId);
}
}
14 changes: 7 additions & 7 deletions packages/common/src/model.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,13 @@ export const modelFeatureFlags: Record<string, FeatureFlags> = {
// Amazon Titan
'amazon.titan-text-express-v1': MODEL_FEATURE.TEXT_DOC,
'amazon.titan-text-premier-v1:0': MODEL_FEATURE.TEXT_ONLY,
// Amazon Nova
'amazon.nova-pro-v1:0': MODEL_FEATURE.TEXT_DOC_IMAGE_VIDEO,
'amazon.nova-lite-v1:0': MODEL_FEATURE.TEXT_DOC_IMAGE_VIDEO,
'amazon.nova-micro-v1:0': MODEL_FEATURE.TEXT_ONLY,
'us.amazon.nova-pro-v1:0': MODEL_FEATURE.TEXT_DOC_IMAGE, // S3 Video アップロードが us-east-1 のみ対応のため。 Video を利用したい場合は us-east-1 の amazon.nova-pro-v1:0 で利用できます。（注意: リージョン変更の際 RAG を有効化している場合削除されます）
'us.amazon.nova-lite-v1:0': MODEL_FEATURE.TEXT_DOC_IMAGE, // 同上
'us.amazon.nova-micro-v1:0': MODEL_FEATURE.TEXT_ONLY,
// Meta
'meta.llama3-8b-instruct-v1:0': MODEL_FEATURE.TEXT_DOC,
'meta.llama3-70b-instruct-v1:0': MODEL_FEATURE.TEXT_DOC,
Expand All @@ -56,13 +63,6 @@ export const modelFeatureFlags: Record<string, FeatureFlags> = {
// Cohere
'cohere.command-r-v1:0': MODEL_FEATURE.TEXT_DOC,
'cohere.command-r-plus-v1:0': MODEL_FEATURE.TEXT_DOC,
// Amazon Nova
'amazon.nova-pro-v1:0': MODEL_FEATURE.TEXT_DOC_IMAGE_VIDEO,
'amazon.nova-lite-v1:0': MODEL_FEATURE.TEXT_DOC_IMAGE_VIDEO,
'amazon.nova-micro-v1:0': MODEL_FEATURE.TEXT_ONLY,
'us.amazon.nova-pro-v1:0': MODEL_FEATURE.TEXT_DOC_IMAGE_VIDEO,
'us.amazon.nova-lite-v1:0': MODEL_FEATURE.TEXT_DOC_IMAGE_VIDEO,
'us.amazon.nova-micro-v1:0': MODEL_FEATURE.TEXT_ONLY,
// Stability AI Image Gen
'stability.stable-diffusion-xl-v1': MODEL_FEATURE.IMAGE_GEN,
'stability.sd3-large-v1:0': MODEL_FEATURE.IMAGE_GEN,
Expand Down
6 changes: 3 additions & 3 deletions packages/types/src/message.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@ export type ExtraData = {
type: string; // 'image' | 'file'
name: string;
source: {
type: string; // 'S3'
mediaType: string; // file type
data: string;
type: string; // 'S3' | 'base64'
mediaType: string; // mime type (i.e. image/png)
data: string; // s3 location for s3, data for base64
};
};

Expand Down
48 changes: 33 additions & 15 deletions packages/web/src/hooks/useChat.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import useChatList from './useChatList';
// mutateListChat の本来の型は InfiniteKeyedMutator<ListChatsResponse[]>
import { getPrompter } from '../prompts';
import { findModelByModelId } from './useModel';
import useFileApi from './useFileApi';

const useChatState = create<{
chats: {
Expand Down Expand Up @@ -83,6 +84,7 @@ const useChatState = create<{
predictStream,
predictTitle,
} = useChatApi();
const { getS3Uri } = useFileApi();

const getModelId = (id: string) => {
return get().modelIds[id] || '';
Expand Down Expand Up @@ -241,21 +243,37 @@ const useChatState = create<{
// LLM で推論する形式に extraData を変換する
const extraData: ExtraData[] | undefined = m.extraData?.flatMap(
(data) => {
// 推論する際は"data:image/png..." のといった情報は必要ないため、削除する
const base64EncodedData = uploadedFiles
?.find((uploadedFile) => uploadedFile.s3Url === data.source.data)
?.base64EncodedData?.replace(/^data:(.*,)?/, '');

// Base64 エンコードされた画像情報を設定する
return {
type: data.type,
name: data.name,
source: {
type: 'base64',
mediaType: data.source.mediaType,
data: base64EncodedData ?? '',
},
};
if (data.type === 'video') {
// Send S3 location for video
// https:// 形式の S3 URL から s3:// 形式の S3 URI に変換する
const s3Uri = getS3Uri(data.source.data ?? '');
return {
type: data.type,
name: data.name,
source: {
type: 's3',
mediaType: data.source.mediaType,
data: s3Uri,
},
};
} else {
// Otherwise (image and file) send base64 encoded data
// 推論する際は"data:image/png..." のといった情報は必要ないため、削除する
const base64EncodedData = uploadedFiles
?.find((uploadedFile) => uploadedFile.s3Url === data.source.data)
?.base64EncodedData?.replace(/^data:(.*,)?/, '');

// Base64 エンコードされた画像情報を設定する
return {
type: data.type,
name: data.name,
source: {
type: 'base64',
mediaType: data.source.mediaType,
data: base64EncodedData ?? '',
},
};
}
}
);
return {
Expand Down
57 changes: 32 additions & 25 deletions packages/web/src/hooks/useFileApi.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,29 @@ import axios from 'axios';

const useFileApi = () => {
const http = useHttp();
const parseS3Url = (s3Url: string) => {
let result = /^s3:\/\/(?<bucketName>.+?)\/(?<prefix>.+)/.exec(s3Url);

if (!result) {
result =
/^https:\/\/s3.(?<region>.+?).amazonaws.com\/(?<bucketName>.+?)\/(?<prefix>.+)$/.exec(
s3Url
);

if (!result) {
result =
/^https:\/\/(?<bucketName>.+?).s3(|(\.|-)(?<region>.+?)).amazonaws.com\/(?<prefix>.+)$/.exec(
s3Url
);
}
}

return result?.groups as {
bucketName: string;
prefix: string;
region?: string;
};
};
return {
getSignedUrl: (req: GetFileUploadSignedUrlRequest) => {
return http.post<GetFileUploadSignedUrlResponse>('file/url', req);
Expand All @@ -23,36 +46,16 @@ const useFileApi = () => {
data: req.file,
});
},
getFileDownloadSignedUrl: async (s3Uri: string) => {
let result = /^s3:\/\/(?<bucketName>.+?)\/(?<prefix>.+)/.exec(s3Uri);
getFileDownloadSignedUrl: async (s3Url: string) => {
const { bucketName, prefix, region } = parseS3Url(s3Url);

if (!result) {
result =
/^https:\/\/s3.(?<region>.+?).amazonaws.com\/(?<bucketName>.+?)\/(?<prefix>.+)$/.exec(
s3Uri
);

if (!result) {
result =
/^https:\/\/(?<bucketName>.+?).s3(|(\.|-)(?<region>.+?)).amazonaws.com\/(?<prefix>.+)$/.exec(
s3Uri
);
}
}

const groups = result?.groups as {
bucketName: string;
prefix: string;
region?: string;
};

const [filePrefix, anchorLink] = groups.prefix.split('#');
const [filePrefix, anchorLink] = prefix.split('#');

// Signed URL を取得
const params: GetFileDownloadSignedUrlRequest = {
bucketName: groups.bucketName,
bucketName: bucketName,
filePrefix: decodeURIComponent(filePrefix),
region: groups.region,
region: region,
};
const { data: url } =
await http.api.get<GetFileDownloadSignedUrlResponse>('/file/url', {
Expand All @@ -63,6 +66,10 @@ const useFileApi = () => {
deleteUploadedFile: async (fileName: string) => {
return http.delete<DeleteFileResponse>(`file/${fileName}`);
},
getS3Uri: (s3Url: string) => {
const { bucketName, prefix } = parseS3Url(s3Url);
return `s3://${bucketName}/${prefix}`;
},
};
};

Expand Down
2 changes: 1 addition & 1 deletion packages/web/src/pages/ChatPage.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ const fileLimit: FileLimit = {
maxImageFileCount: 20,
maxImageFileSizeMB: 3.75,
maxVideoFileCount: 1,
maxVideoFileSizeMB: 25, // 25 MB for base64 input (TODO: up to 1 GB through S3)
maxVideoFileSizeMB: 1000, // 1 GB for S3 input
};

type StateType = {
Expand Down

0 comments on commit 062b506

Please sign in to comment.