add comments from PR46 (#60)

FluxML · Mar 7, 2022 · c1c8beb · c1c8beb · mcabbott · Mar 7, 2022
1 parent 3858611
commit c1c8beb
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 4 deletions.
diff --git a/Project.toml b/Project.toml
@@ -1,7 +1,7 @@
 name = "Optimisers"
 uuid = "3bd65402-5787-11e9-1adc-39752487f4e2"
 authors = ["Mike J Innes <mike.j.innes@gmail.com>"]
-version = "0.2.2"
+version = "0.2.1" 
 
 [deps]
 ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"

diff --git a/src/rules.jl b/src/rules.jl
@@ -442,20 +442,24 @@ end
 """
     WeightDecay(γ = 5f-4)
 
-Decay weights by `γ`.
+Decay weights by ``γ``, that is, add `γ .* x` to the gradient `x̄` which will be
+subtracted from `x`.
+
+Typically composed  with other optimisers as the first transformation in an [`OptimiserChain`](@ref).
+This is equivalent to adding ``L_2`` regularization with coefficient ``γ`` to the loss.
 
 # Parameters
 - Weight decay (`γ`): Decay applied to weights during optimisation.
 """
 struct WeightDecay{T}
-  wd::T
+  gamma::T
 end
 WeightDecay() = WeightDecay(5f-4)
 
 init(o::WeightDecay, x::AbstractArray) = nothing
 
 function apply!(o::WeightDecay, state, x, dx)
-  dx′ = @lazy dx + o.wd * x
+  dx′ = @lazy dx + o.gamma * x
 
   return state, dx′
 end