From 4ec0465f9fe7761a284f514006b33d2f5e70f975 Mon Sep 17 00:00:00 2001 From: Yukinobu Mine Date: Thu, 12 Dec 2024 23:33:59 +0900 Subject: [PATCH] If model is Amazon Nova, set the upper limit of topK to 128. #629 --- backend/app/bedrock.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/app/bedrock.py b/backend/app/bedrock.py index 11689101..1255c8f0 100644 --- a/backend/app/bedrock.py +++ b/backend/app/bedrock.py @@ -80,7 +80,11 @@ def _prepare_nova_model_params( # Add top_k if specified in generation params if generation_params and generation_params.top_k is not None: - additional_fields["inferenceConfig"]["topK"] = generation_params.top_k + top_k = generation_params.top_k + if top_k > 128: + top_k = 128 + + additional_fields["inferenceConfig"]["topK"] = top_k return inference_config, additional_fields