Feature add regularizer base class

neural_nets/regularizers/README.md

@@ -0,0 +1,34 @@
+## Usage of regularizers
+
+Regularizers allow to apply penalties on layer parameters or layer activity during optimization. These penalties are incorporated in the loss function that the network optimizes.
+
+The penalties are applied on a per-layer basis.
+
+## Example
+
+```python
+from neural_nets import regularizers
+
+```
+
+## Available penalties
+
+```python
+regularizers.l1(0.)
+regularizers.l2(0.)
+regularizers.l1_l2(l1=0.01, l2=0.01)
+```
+
+## Developing new regularizers
+
+Any function that takes in a weight matrix and returns a loss contribution tensor can be used as a regularizer, e.g.:
+
+```python
+import numpy as np
+
+def l1_reg(weight_matrix):
+    return 0.01 * np.sum(np.abs(weight_matrix))
+```
+
+Alternatively, you can write your regularizers in an object-oriented way;
+see the [neural_nets/regularizers.py](regularizers.py) module for examples.

neural_nets/regularizers/__init__.py

@@ -0,0 +1 @@
+from .regularizers import *

neural_nets/regularizers/regularizers.py

@@ -0,0 +1,57 @@
+"""
+Built-in regularizers.
+"""
+
+import numpy as np
+
+
+class Regularizer(object):
+    """
+    Regularizer base class.
+    """
+
+    def __call__(self, x):
+        return 0.
+
+    @classmethod
+    def from_config(cls, config):
+        return cls(**config)
+
+
+class L1L2(Regularizer):
+    """
+    Regularizer for L1 and L2 regularization.
+
+    Arguments
+    --------
+        l1: Float; L1 regularization factor.
+        l2: Float; L2 regularization factor.
+    """
+
+    def __init__(self, l1=0., l2=0.):
+        self.l1 = l1
+        self.l2 = l2
+
+    def __call__(self, x):
+        regularization = 0.
+        if self.l1:
+            regularization += np.sum(self.l1 * np.abs(x))
+        if self.l2:
+            regularization += np.sum(self.l2 * np.square(x))
+        return regularization
+
+    def get_config(self):
+        return {'l1': float(self.l1),
+                'l2': float(self.l2)}
+
+
+def l1(l=0.01):
+    return L1L2(l1=l)
+
+
+def l2(l=0.01):
+    return L1L2(l2=l)
+
+
+def l1_l2(l1=0.01, l2=0.01):
+    return L1L2(l1=l1, l2=l2)