Add example deepspeed config json file

HHousen · Feb 26, 2021 · b2d81af · b2d81af
1 parent 2be444e
commit b2d81af
Showing 1 changed file with 30 additions and 0 deletions.
diff --git a/example_deepspeed_config.json b/example_deepspeed_config.json
@@ -0,0 +1,30 @@
+{
+    "zero_allow_untested_optimizer": true,
+    "optimizer": {
+        "type": "OneBitAdam",
+        "params": {
+            "lr": 2e-5,
+            "betas": [0.998, 0.999],
+            "eps": 1e-8,
+            "weight_decay": 1e-2,
+            "cuda_aware": true
+        }
+    },
+    "scheduler": {
+        "type": "WarmupLR",
+        "params": {
+            "last_batch_iteration": -1,
+            "warmup_min_lr": 0,
+            "warmup_max_lr": 2e-5,
+            "warmup_num_steps": 2000
+        }
+    },
+    "zero_optimization": {
+        "stage": 2,
+        "cpu_offload": true,
+        "contiguous_gradients": true,
+        "overlap_comm": true,
+        "allgather_bucket_size": 2e8,
+        "reduce_bucket_size": 2e8
+    }
+}