prep for training

Assets/Scripts/Agents/WormSimpleAgent.cs

@@ -1,27 +1,25 @@
-using CommandTerminal;
 using Unity.MLAgents;
 using Unity.MLAgents.Actuators;
 using Unity.MLAgents.Sensors;
-using UnityEngine;
 
 namespace DialogosEngine
 {
     public class WormSimpleAgent : Agent
     {
-        CommandLogger Logger;
+        //CommandLogger Logger;
         bool _IsInitialized = false;
         string _ExpectedString = "echo hello <eos>";
         string _CachedString;
 
         public override void Initialize()
         {
-            Logger = new CommandLogger("WormSimpleAgent_log.txt", 1000);
-            Logger.Log($"[{StepCount}] Initialize");
+            //Logger = new CommandLogger("WormSimpleAgent_log.txt");
+            //Logger.Log($"[{StepCount}] Initialize");
         }
 
         public override void OnEpisodeBegin()
         {
-            Logger.Log($"[{StepCount}] OnEpisodeBegin");
+            //Logger.Log($"[{StepCount}] OnEpisodeBegin");
             _IsInitialized = true;
         }
 
@@ -54,12 +52,12 @@ public override void OnActionReceived(ActionBuffers actions)
             float[] _actionArray = actions.ContinuousActions.Array;
             float _lengthControlValue = _actionArray[0];
 
-            Logger.Log($"[{StepCount}] Length Control Value: {_lengthControlValue}");
+            //Logger.Log($"[{StepCount}] Length Control Value: {_lengthControlValue}");
 
             int outputLength = Transformer.RoundMax(ref _lengthControlValue);
 
-            Logger.Log($"[{StepCount}] Rounded Output Length: {outputLength}");
-            
+            //Logger.Log($"[{StepCount}] Rounded Output Length: {outputLength}");
+
             for (int i = 1; i < _actionArray.Length; i++)
             {
                 _actionArray[i] = Transformer.Transform(ref _actionArray[i]);
@@ -69,9 +67,9 @@ public override void OnActionReceived(ActionBuffers actions)
 
             //Logger.Log($"[{StepCount}] Processed Action Array: {string.Join(", ", _actionArray)}");
 
-            _CachedString = Lexer.QuantizeUTF8(_actionArray);
+            //_CachedString = Lexer.QuantizeUTF8(_actionArray);
 
-            Logger.Log($"[{StepCount}] Quantized String: {_CachedString}");
+            //Logger.Log($"[{StepCount}] Quantized String: {_CachedString}");
         }
 
         void FixedUpdate()
@@ -81,9 +79,30 @@ void FixedUpdate()
                 return;
             }
 
-            var reward = Random.value;
-            AddReward(reward);
-            Logger.Log($"[{StepCount}] FixedUpdate.reward: {reward}");
+            if (_CachedString != null)
+            {
+                float reward = AgentUtils.CalculateEchoReward(_ExpectedString, _CachedString);
+
+                if (_CachedString.EndsWith(AgentUtils.k_EndOfSequence))
+                {
+                    string _commandLine = _CachedString.Replace(AgentUtils.k_EndOfSequence, "");
+                    //Logger.Log($"[{StepCount}] COMMAND: " + _commandLine);
+                }
+
+                if (_CachedString == _ExpectedString)
+                {
+                    //Logger.Log($"[{StepCount}] MATCH: " + _CachedString);
+                    _CachedString = null;
+                    SetReward(1f);
+                    EndEpisode();
+                }
+                else
+                {
+                    //Logger.Log($"[{StepCount}] {reward} | {_CachedString}");
+                    _CachedString = null;
+                    SetReward(reward);
+                }
+            }
         }
     }
 }

config/sac/WormSimple.yaml

@@ -3,25 +3,35 @@ behaviors:
     trainer_type: sac
     hyperparameters:
       learning_rate: 0.0003
-      learning_rate_schedule: constant
-      batch_size: 256
-      buffer_size: 500000
+      learning_rate_schedule: linear
+      batch_size: 1000
+      buffer_size: 1000000
       buffer_init_steps: 0
       tau: 0.005
-      steps_per_update: 20.0
+      steps_per_update: 30.0
       save_replay_buffer: false
       init_entcoef: 1.0
-      reward_signal_steps_per_update: 20.0
+      reward_signal_steps_per_update: 30.0
     network_settings:
       normalize: true
-      hidden_units: 512
+      hidden_units: 1024
       num_layers: 3
       vis_encode_type: simple
+      memory:
+        sequence_length: 1000
+        memory_size: 1024
     reward_signals:
       extrinsic:
         gamma: 0.995
         strength: 1.0
-    keep_checkpoints: 5
-    max_steps: 5000000
+      curiosity:
+        strength: 0.1
+        gamma: 0.995
+        network_settings:
+          hidden_units: 1024
+          learning_rate: 0.0001
+    keep_checkpoints: 100
+    checkpoint_interval: 10000
+    max_steps: 1000000
     time_horizon: 1000
-    summary_freq: 30000
+    summary_freq: 10000