Add down_proj smoothing

neuralmagic · Oct 13, 2023 · 44d0b4c · 44d0b4c
1 parent cb4f6db
commit 44d0b4c
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/src/sparseml/experimental/sparsegpt/llama2.py b/src/sparseml/experimental/sparsegpt/llama2.py
@@ -38,6 +38,10 @@
         "module_to_balance": ["gate_proj", "up_proj"],
         "module_to_merge": "post_attention_layernorm",
     },
+    {
+        "module_to_balance": ["down_proj"],
+        "module_to_merge": "up_proj",
+    },
 ]