Skip to content

Commit

Permalink
Fix REINFORCE continuous action (#146)
Browse files Browse the repository at this point in the history 
* add cont test for all algos

* fix reinforce cont action
  • Loading branch information
@kengz
kengz authored Aug 8, 2018
1 parent 5ec2a0f commit fb617ae
Show file tree
Hide file tree
Showing 8 changed files with 1,619 additions and 14 deletions.
2 changes: 2 additions & 0 deletions slm_lab/agent/algorithm/reinforce.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,8 @@ def calc_pdparam(self, x, evaluate=True, net=None):
else:
net.train()
pdparam = net(x)
if (not self.body.is_discrete) and len(pdparam) == 1:
pdparam = pdparam[0]
logger.debug(f'pdparam: {pdparam}')
return pdparam

Expand Down
334 changes: 334 additions & 0 deletions slm_lab/spec/a2c.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,340 @@
}]
}
},
"a2c_mlp_shared_pendulum": {
"agent": [{
"name": "A2C",
"algorithm": {
"name": "ActorCritic",
"action_pdtype": "default",
"action_policy": "default",
"action_policy_update": "no_update",
"explore_var_start": null,
"explore_var_end": null,
"explore_anneal_epi": null,
"gamma": 0.99,
"use_gae": true,
"lam": 1.0,
"use_nstep": false,
"num_step_returns": 100,
"add_entropy": true,
"entropy_coef": 0.01,
"policy_loss_coef": 1.0,
"val_loss_coef": 0.01,
"continuous_action_clip": 2.0,
"training_frequency": 1,
"training_epoch": 8
},
"memory": {
"name": "OnPolicyReplay"
},
"net": {
"type": "MLPNetShared",
"hid_layers": [64],
"hid_layers_activation": "relu",
"clip_grad": false,
"clip_grad_val": 1.0,
"use_same_optim": false,
"actor_optim_spec": {
"name": "Adam",
"lr": 0.02
},
"critic_optim_spec": {
"name": "Adam",
"lr": 0.02
},
"lr_decay": "rate_decay",
"lr_decay_frequency": 500,
"lr_decay_min_timestep": 1000,
"lr_anneal_timestep": 100000,
"gpu": true
}
}],
"env": [{
"name": "Pendulum-v0",
"max_timestep": null,
"max_episode": 500,
"save_epi_frequency": 1000
}],
"body": {
"product": "outer",
"num": 1
},
"meta": {
"max_session": 4,
"max_trial": 100,
"search": "RandomSearch",
"max_generation": null
},
"search": {
"agent": [{
"algorithm": {
"gamma__uniform": [0.90, 0.99],
"lam__uniform": [0.1, 1.0]
},
"net": {
"actor_optim_spec": {
"lr__uniform": [0.0001, 0.1]
},
"critic_optim_spec": {
"lr__uniform": [0.0001, 0.1]
}
}
}]
}
},
"a2c_mlp_separate_pendulum": {
"agent": [{
"name": "A2C",
"algorithm": {
"name": "ActorCritic",
"action_pdtype": "default",
"action_policy": "default",
"action_policy_update": "no_update",
"explore_var_start": null,
"explore_var_end": null,
"explore_anneal_epi": null,
"gamma": 0.99,
"use_gae": true,
"lam": 1.0,
"use_nstep": false,
"num_step_returns": 100,
"add_entropy": true,
"entropy_coef": 0.01,
"policy_loss_coef": 1.0,
"val_loss_coef": 0.01,
"continuous_action_clip": 2.0,
"training_frequency": 1,
"training_epoch": 8
},
"memory": {
"name": "OnPolicyReplay"
},
"net": {
"type": "MLPNetSeparate",
"hid_layers": [64],
"hid_layers_activation": "relu",
"clip_grad": false,
"clip_grad_val": 1.0,
"use_same_optim": false,
"actor_optim_spec": {
"name": "Adam",
"lr": 0.02
},
"critic_optim_spec": {
"name": "Adam",
"lr": 0.02
},
"lr_decay": "rate_decay",
"lr_decay_frequency": 500,
"lr_decay_min_timestep": 1000,
"lr_anneal_timestep": 100000,
"gpu": true
}
}],
"env": [{
"name": "Pendulum-v0",
"max_timestep": null,
"max_episode": 500,
"save_epi_frequency": 1000
}],
"body": {
"product": "outer",
"num": 1
},
"meta": {
"max_session": 4,
"max_trial": 100,
"search": "RandomSearch",
"max_generation": null
},
"search": {
"agent": [{
"algorithm": {
"gamma__uniform": [0.90, 0.99],
"lam__uniform": [0.1, 1.0]
},
"net": {
"actor_optim_spec": {
"lr__uniform": [0.0001, 0.1]
},
"critic_optim_spec": {
"lr__uniform": [0.0001, 0.1]
}
}
}]
}
},
"a2c_rnn_shared_pendulum": {
"agent": [{
"name": "A2C",
"algorithm": {
"name": "ActorCritic",
"action_pdtype": "default",
"action_policy": "default",
"action_policy_update": "no_update",
"explore_var_start": null,
"explore_var_end": null,
"explore_anneal_epi": null,
"gamma": 0.99,
"use_gae": true,
"lam": 1.0,
"use_nstep": false,
"num_step_returns": 100,
"add_entropy": true,
"entropy_coef": 0.01,
"policy_loss_coef": 1.0,
"val_loss_coef": 0.01,
"continuous_action_clip": 2.0,
"training_frequency": 1,
"training_epoch": 8
},
"memory": {
"name": "OnPolicySeqReplay"
},
"net": {
"type": "RecurrentNetShared",
"hid_layers": [],
"hid_layers_activation": "relu",
"rnn_hidden_size": 64,
"rnn_num_layers": 1,
"seq_len": 4,
"clip_grad": false,
"clip_grad_val": 1.0,
"use_same_optim": false,
"actor_optim_spec": {
"name": "Adam",
"lr": 0.02
},
"critic_optim_spec": {
"name": "Adam",
"lr": 0.02
},
"lr_decay": "rate_decay",
"lr_decay_frequency": 500,
"lr_decay_min_timestep": 1000,
"lr_anneal_timestep": 100000,
"gpu": true
}
}],
"env": [{
"name": "Pendulum-v0",
"max_timestep": null,
"max_episode": 500,
"save_epi_frequency": 1000
}],
"body": {
"product": "outer",
"num": 1
},
"meta": {
"max_session": 4,
"max_trial": 100,
"search": "RandomSearch",
"max_generation": null
},
"search": {
"agent": [{
"algorithm": {
"gamma__uniform": [0.90, 0.99],
"lam__uniform": [0.1, 1.0]
},
"net": {
"actor_optim_spec": {
"lr__uniform": [0.0001, 0.1]
},
"critic_optim_spec": {
"lr__uniform": [0.0001, 0.1]
}
}
}]
}
},
"a2c_rnn_separate_pendulum": {
"agent": [{
"name": "A2C",
"algorithm": {
"name": "ActorCritic",
"action_pdtype": "default",
"action_policy": "default",
"action_policy_update": "no_update",
"explore_var_start": null,
"explore_var_end": null,
"explore_anneal_epi": null,
"gamma": 0.99,
"use_gae": true,
"lam": 1.0,
"use_nstep": false,
"num_step_returns": 100,
"add_entropy": true,
"entropy_coef": 0.01,
"policy_loss_coef": 1.0,
"val_loss_coef": 0.01,
"continuous_action_clip": 2.0,
"training_frequency": 1,
"training_epoch": 8
},
"memory": {
"name": "OnPolicySeqReplay"
},
"net": {
"type": "RecurrentNetSeparate",
"hid_layers": [],
"hid_layers_activation": "relu",
"rnn_hidden_size": 64,
"rnn_num_layers": 1,
"seq_len": 4,
"clip_grad": false,
"clip_grad_val": 1.0,
"use_same_optim": false,
"actor_optim_spec": {
"name": "Adam",
"lr": 0.02
},
"critic_optim_spec": {
"name": "Adam",
"lr": 0.02
},
"lr_decay": "rate_decay",
"lr_decay_frequency": 500,
"lr_decay_min_timestep": 1000,
"lr_anneal_timestep": 100000,
"gpu": true
}
}],
"env": [{
"name": "Pendulum-v0",
"max_timestep": null,
"max_episode": 500,
"save_epi_frequency": 1000
}],
"body": {
"product": "outer",
"num": 1
},
"meta": {
"max_session": 4,
"max_trial": 100,
"search": "RandomSearch",
"max_generation": null
},
"search": {
"agent": [{
"algorithm": {
"gamma__uniform": [0.90, 0.99],
"lam__uniform": [0.1, 1.0]
},
"net": {
"actor_optim_spec": {
"lr__uniform": [0.0001, 0.1]
},
"critic_optim_spec": {
"lr__uniform": [0.0001, 0.1]
}
}
}]
}
},
"a2c_conv_shared_breakout": {
"agent": [{
"name": "A2C",
Expand Down
Loading

0 comments on commit fb617ae

Please sign in to comment.