chore: add qwen2.5 32b awq

bentoml · Oct 17, 2024 · 813bcd2 · 813bcd2
1 parent ff11508
commit 813bcd2
Showing 1 changed file with 17 additions and 0 deletions.
diff --git a/src/recipe.yaml b/src/recipe.yaml
@@ -976,3 +976,20 @@
       memory: 60Gi
     traffic:
       timeout: 300
+'qwen2.5:32b-instruct-awq-4bit':
+  project: vllm-chat
+  extra_labels:
+    model_name: Qwen/Qwen2.5-32B-Instruct-AWQ
+    openllm_alias: 32b-4bit
+    platforms: linux
+  engine_config:
+    max_model_len: 20480
+    model: Qwen/Qwen2.5-32B-Instruct-AWQ
+  project: vllm-chat
+  service_config:
+    name: qwen2.5
+    resources:
+      gpu: 1
+      gpu_type: nvidia-tesla-a100
+    traffic:
+      timeout: 300