* 0.4.10

CHANGELOG.txt

@@ -1,6 +1,13 @@
 Change Log
 ==========
 
+
+0.4.10 (04/02/2023)
+-----------------
+* TransformerEncoderStack to support activation as input
+* PositionalEncoding to support more than 3 dimensions input
+
+
 0.4.9 (22/01/2023)
 -----------------
 * Added assert to Attention class (from extensions) when mask is used

README.md

@@ -14,7 +14,7 @@ A Fast, Flexible Trainer with Callbacks and Extensions for PyTorch
 ![Liecense](https://img.shields.io/github/license/roysadaka/lpd)
 <!-- ![Follow](https://img.shields.io/twitter/follow/roysadaka?label=RoySadaka&style=social) -->
 
-There are 2 types of ``lpd`` packagaes available 
+There are 2 types of ``lpd`` packages available 
 * ``lpd`` which brings dependencies for pytorch, numpy and tensorboard
 ```sh
     pip install lpd
@@ -25,14 +25,16 @@ There are 2 types of ``lpd`` packagaes available
     pip install lpd-nodeps
 ```
 
-<b>[v0.4.9-beta](https://github.com/RoySadaka/lpd/releases) Release - contains the following:</b>
-* Added assert to Attention class (from extensions) when mask is used
-* Fixed confusion matrix cpu/gpu device error
-* Better handling on callbacks where apply_on_states=None (apply on all states)
-* Updated Pipfile
+<b>[v0.4.10-beta](https://github.com/RoySadaka/lpd/releases) Release - contains the following:</b> 
+
+* ``TransformerEncoderStack`` to support activation as input
+* ``PositionalEncoding`` to support more than 3 dimensions input
 
 
-Previously on lpd: 
+Previously on lpd:
+* Updated Pipfile
+* Fixed confusion matrix cpu/gpu device error
+* Better handling on callbacks where apply_on_states=None (apply on all states)
 * Bug fix in case validation samples are empty
 * Bug fix in verbosity level 2 in train
 * Verbosity change in torch_utils
@@ -62,7 +64,7 @@ The main usages are given below.
 
     seed_all(seed=42) # because its the answer to life and the universe
 
-    device = get_gpu_device_if_available() # with fallback to CPU if GPU not avilable
+    device = get_gpu_device_if_available() # with fallback to CPU if GPU not available
     model = MyModel().to(device) # this is your model class, and its being sent to the relevant device
     optimizer = torch.optim.SGD(params=model.parameters())
     scheduler = KerasDecay(optimizer, decay=0.01, last_step=-1) # decay scheduler using keras formula 
@@ -175,7 +177,7 @@ Here are some examples
 
 ## Callbacks
 Will be used to perform actions at various stages.  
-Some common callbacks are available under ``lpd.callbacks``, and you can also create your own, more detailes below.  
+Some common callbacks are available under ``lpd.callbacks``, and you can also create your own, more details below.  
 In a callback, ``apply_on_phase`` (``lpd.enums.Phase``) will determine the execution phase,  
 and ``apply_on_states`` (``lpd.enums.State`` or ``list(lpd.enums.State)``) will determine the execution states  
 These are the current available phases and states, more might be added in future releases

lpd/extensions/custom_layers.py

@@ -1,7 +1,11 @@
+from typing import Callable, Union
 import torch
+from torch import Tensor
 import torch.nn as nn
 import math
 
+nn.TransformerDecoderLayer
+
 class MatMul2D(nn.Module):
     def __init__(self, transpose_b, name=None):
         super(MatMul2D, self).__init__()
@@ -144,17 +148,19 @@ def __init__(self, in_dim,
                        out_dim,
                        drop_out_proba,
                        expansion_rate,
+                       activation,
                        name=None):
         super(TransformerEncoderFeedForward, self).__init__()
         #PARAMS
         self.in_dim = in_dim
         self.out_dim = out_dim
         self.drop_out_proba = drop_out_proba
         self.expansion_rate = expansion_rate
+        self.activation = activation
         self.name = name if name else 'Transformer-Encoder__Feed-Forward'
 
         #LAYERS
-        self.hidden_dense = Dense(in_dim=self.in_dim, out_dim=self.out_dim * self.expansion_rate, use_bias=True, activation=nn.ReLU(), name = f'{self.name}__Hidden-Dense')
+        self.hidden_dense = Dense(in_dim=self.in_dim, out_dim=self.out_dim * self.expansion_rate, use_bias=True, activation=activation, name = f'{self.name}__Hidden-Dense')
         self.output_dense = Dense(in_dim=self.out_dim * self.expansion_rate, out_dim=self.out_dim, use_bias=True, activation=None, name = f'{self.name}__Out-Dense')
         self.dropout = nn.Dropout(p=self.drop_out_proba)
 
@@ -167,13 +173,14 @@ def forward(self, inputs):                                              # (batch
         return self.norm(inputs + output)    #RESIDUAL & NORM               # (batch, num_elements, out_dim)
 
 class TransformerBlock(nn.Module):
-    def __init__(self, in_dim,
-                       key_dim,
-                       out_dim,
-                       num_heads,
-                       drop_out_proba,
-                       ff_expansion_rate,
-                       name = None):
+    def __init__(self, in_dim:int,
+                       key_dim:int,
+                       out_dim:int,
+                       num_heads:int,
+                       drop_out_proba:float,
+                       ff_expansion_rate:int,
+                       activation: Callable[[Tensor], Tensor]=nn.ReLU(),
+                       name:str = None):
         super(TransformerBlock, self).__init__()
         #PARAMS
         self.in_dim = in_dim
@@ -182,6 +189,7 @@ def __init__(self, in_dim,
         self.num_heads = num_heads
         self.drop_out_proba = drop_out_proba
         self.ff_expansion_rate = ff_expansion_rate
+        self.activation = activation
         self.name = name if name else 'Transformer-Encoder'
         #LAYERS
         self.multi_head_self_attention = MultiHeadAttention(self.in_dim,
@@ -195,6 +203,7 @@ def __init__(self, in_dim,
                                                           self.out_dim,
                                                           self.drop_out_proba,
                                                           self.ff_expansion_rate,
+                                                          self.activation,
                                                           name = f'{self.name}__FF')
 
 
@@ -216,17 +225,16 @@ def __init__(self, embedding_size, dropout_rate=0.1, maximum_position_encoding=5
         self._create_positional_encoding()
 
     def _create_positional_encoding(self):
-        if self.maximum_position_encoding:
-            position = torch.arange(self.maximum_position_encoding).unsqueeze(1)
-            div_term = torch.exp(torch.arange(0, self.embedding_size, 2) * (-math.log(10000.0) / self.embedding_size))
-            pe = torch.zeros(self.maximum_position_encoding, 1, self.embedding_size)
-            pe[:, 0, 0::2] = torch.sin(position * div_term)
-            pe[:, 0, 1::2] = torch.cos(position * div_term)
-            pe = pe.squeeze(1)
-            self.register_buffer('pe', pe)
+        position = torch.arange(self.maximum_position_encoding).unsqueeze(1)
+        div_term = torch.exp(torch.arange(0, self.embedding_size, 2) * (-math.log(10000.0) / self.embedding_size))
+        pe = torch.zeros(self.maximum_position_encoding, 1, self.embedding_size)
+        pe[:, 0, 0::2] = torch.sin(position * div_term)
+        pe[:, 0, 1::2] = torch.cos(position * div_term)
+        pe = pe.squeeze(1)
+        self.register_buffer('pe', pe)
 
     def forward(self, inputs):
-        inputs = inputs + self.pe[:inputs.size(1), :].unsqueeze(0)
+        inputs = inputs + self.pe[:inputs.size(-2), :].unsqueeze(0)
         return self.dropout(inputs)
 
 class TransformerEncoderStack(nn.Module):
@@ -238,6 +246,7 @@ def __init__(self, in_dim,
                        drop_out_proba,
                        ff_expansion_rate,
                        maximum_position_encoding=None,
+                       activation: Callable[[Tensor], Tensor]=nn.ReLU(),
                        name=None):
         super(TransformerEncoderStack, self).__init__()
         #PARAMS
@@ -249,6 +258,7 @@ def __init__(self, in_dim,
         self.drop_out_proba = drop_out_proba
         self.ff_expansion_rate = ff_expansion_rate
         self.maximum_position_encoding = maximum_position_encoding
+        self.activation = activation
         self.name = name if name else 'Transformer-Encoder-Stack'
 
         #LAYERS
@@ -258,9 +268,11 @@ def __init__(self, in_dim,
                                                     self.num_heads,
                                                     self.drop_out_proba,
                                                     self.ff_expansion_rate,
+                                                    self.activation,
                                                     name=f'{self.name}__E{i}')
                                                     for i in range(self.num_encoders)])
-        self.pos_encoder = PositionalEncoding(self.in_dim, self.drop_out_proba, self.maximum_position_encoding)
+        if self.maximum_position_encoding is not None:
+            self.pos_encoder = PositionalEncoding(self.in_dim, self.drop_out_proba, self.maximum_position_encoding)
 
     def forward(self, inputs, mask=None):
         outputs = inputs                                        # (batch, seq_len, emb_size)

misc.txt

@@ -11,5 +11,5 @@ Steps:
     >> python setup-nodeps.py bdist_wheel -d .
     Publish pypi package cmd:
     >> pipenv shell
-    >> twine upload lpd-0.4.9-py3-none-any.whl
-    >> twine upload lpd_nodeps-0.4.9-py3-none-any.whl
+    >> twine upload lpd-0.4.10-py3-none-any.whl
+    >> twine upload lpd_nodeps-0.4.10-py3-none-any.whl

setup-nodeps.py

@@ -34,7 +34,7 @@
 
 setup(
     name='lpd-nodeps',
-    version='0.4.9',
+    version='0.4.10',
     description='A Fast, Flexible Trainer with Callbacks and Extensions for PyTorch',
     long_description_content_type='text/markdown',
     long_description=README_md,

setup.py

@@ -36,7 +36,7 @@
 
 setup(
     name='lpd',
-    version='0.4.9',
+    version='0.4.10',
     description='A Fast, Flexible Trainer with Callbacks and Extensions for PyTorch',
     long_description_content_type='text/markdown',
     long_description=README_md,