If model is Amazon Nova, set the upper limit of topK to 128. #629

aws-samples · Dec 12, 2024 · 4ec0465 · 4ec0465
1 parent 8084a30
commit 4ec0465
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/backend/app/bedrock.py b/backend/app/bedrock.py
@@ -80,7 +80,11 @@ def _prepare_nova_model_params(
 
     # Add top_k if specified in generation params
     if generation_params and generation_params.top_k is not None:
-        additional_fields["inferenceConfig"]["topK"] = generation_params.top_k
+        top_k = generation_params.top_k
+        if top_k > 128:
+            top_k = 128
+
+        additional_fields["inferenceConfig"]["topK"] = top_k
 
     return inference_config, additional_fields