Merge branch 'main' into 789-add-tabular-example-in-dashboard

.github/workflows/build.yml

@@ -26,8 +26,8 @@ jobs:
         with:
           python-version: '3.11'
 
-      - name: Run unit tests including downloader
-        run: pytest -v --downloader
+      - name: Run unit tests
+        run: pytest -v
 
       - name: Verify that we can build the package
         run: python setup.py sdist bdist_wheel
@@ -52,12 +52,25 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      #- name: Run unit tests
-      #  run: python -m pytest -v --downloader
+      - name: Run unit tests
+        run: python -m pytest -v
 
       #- name: Verify that we can build the package
       #  run: python setup.py sdist bdist_wheel
 
+  test_downloader:
+    name: Test file downloader
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/install-python-and-package
+        with:
+          python-version: '3.11'
+          extras-require: dev
+      - name: Run downloader test
+        run: python -m pytest -v --downloader -k downloader
+
   test_dashboard:
     name: Test dashboard
     if: always()

.github/workflows/notebooks.yml

@@ -17,17 +17,19 @@ on:
 jobs:
 
   notebooks:
-    name: Run notebooks on (3.10, ${{ matrix.os }})
+    name: Run notebooks on (${{ matrix.python-version }}, ${{ matrix.os }})
     if: github.event.pull_request.draft == false
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
+        python-version: ['3.10']
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/install-python-and-package
         with:
+          python-version: ${{ matrix.python-version }}
           extras-require: dev,notebooks
       - name: Run tutorial notebooks
         run: pytest --nbmake tutorials

dianna/methods/kernelshap_tabular.py

@@ -16,6 +16,7 @@ def __init__(
         mode: str = "classification",
         feature_names: List[int] = None,
         training_data_kmeans: Optional[int] = None,
+        silent: bool = False,
     ) -> None:
         """Initializer of KERNELSHAPTabular.
 
@@ -32,6 +33,7 @@ def __init__(
                                                  in the training data.
             training_data_kmeans(int, optional): summarize the whole training set with
                                                  weighted kmeans
+            silent (bool, optional): whether to print progress messages
         """
         if training_data_kmeans:
             self.training_data = shap.kmeans(training_data,
@@ -41,6 +43,7 @@ def __init__(
         self.feature_names = feature_names
         self.mode = mode
         self.explainer: KernelExplainer
+        self.silent = silent
 
     def explain(
         self,
@@ -73,8 +76,7 @@ def explain(
         explain_instance_kwargs = utils.get_kwargs_applicable_to_function(
             self.explainer.shap_values, kwargs)
 
-        saliency = self.explainer.shap_values(input_tabular,
-                                              **explain_instance_kwargs)
+        saliency = self.explainer.shap_values(input_tabular, silent=self.silent, **explain_instance_kwargs)
 
         if self.mode == 'regression':
             saliency = saliency[0]

dianna/utils/downloader.py

@@ -59,6 +59,10 @@
         "doi:10.5281/zenodo.10656613/apertif_frb_dynamic_spectrum_model.onnx",
         "sha256:3c87db3c6257d7f251a7bdceb3197d5bb482b8edc19870219fb7ca7c204dd257"
     ],
+    "stemmus_scope_emulator_model_LEtot.onnx": [
+        "doi:10.5281/zenodo.12623256/stemmus_scope_emulator_model_LEtot.onnx",
+        "sha256:8c8f34ad5a2c519b1f3c67a4eb0c645c96cac1de166277bfb24e7887c2ce83be"
+    ],
 }
 
 labels = {
@@ -141,6 +145,10 @@
         "doi:10.5281/zenodo.5071376/weather_prediction_dataset_light.csv",
         "sha256:945883204cae14251aec44e78845eb880ed3c04ac56a26fb5283c20f149be960"
     ],
+    "stemmus_scope_emulator_train_test_2014_data.csv": [
+        "doi:10.5281/zenodo.12623256/stemmus_scope_emulator_train_test_2014_data.csv",
+        "sha256:d30970c152f00dc45be7775361f4c88dfb084b13dd8e41fdb001c5306c136519"
+    ],
 }
 
 

dianna/utils/misc.py

@@ -1,6 +1,8 @@
 import inspect
 import warnings
 from pathlib import Path
+import numpy as np
+import numpy.typing
 
 
 def get_function(model_or_function, preprocess_function=None):
@@ -49,7 +51,7 @@ def get_kwargs_applicable_to_function(function, kwargs):
     }
 
 
-def to_xarray(data, axis_labels, required_labels=None):
+def to_xarray(data: numpy.typing.ArrayLike, axis_labels, required_labels=None):
     """Converts numpy data and axes labels to an xarray object."""
     if isinstance(axis_labels, dict):
         # key = axis index, value = label
@@ -59,10 +61,10 @@ def to_xarray(data, axis_labels, required_labels=None):
         indices = list(axis_labels.keys())
         for index in indices:
             if index < 0:
-                axis_labels[data.ndim + index] = axis_labels.pop(index)
+                axis_labels[np.ndim(data) + index] = axis_labels.pop(index)
         labels = [
             axis_labels[index] if index in axis_labels else f'dim_{index}'
-            for index in range(data.ndim)
+            for index in range(np.ndim(data))
         ]
     else:
         labels = list(axis_labels)

dianna/utils/onnx_runner.py

@@ -3,6 +3,7 @@
 
 class SimpleModelRunner:
     """Runs an onnx model with a set of inputs and outputs."""
+
     def __init__(self, filename, preprocess_function=None):
         """Generates function to run ONNX model with one set of inputs and outputs.
 
@@ -22,7 +23,9 @@ def __init__(self, filename, preprocess_function=None):
 
     def __call__(self, input_data):
         """Get ONNX predictions."""
-        sess = ort.InferenceSession(self.filename)
+        sess_options = ort.SessionOptions()
+        sess_options.enable_cpu_mem_arena = False  # disables pre-allocation of memory
+        sess = ort.InferenceSession(self.filename, sess_options=sess_options)
         input_name = sess.get_inputs()[0].name
         output_name = sess.get_outputs()[0].name
 

docs/CONTEXT.md

@@ -2,54 +2,74 @@
 
 # Why DIANNA?
 
-There are numerous other python OSS [eXplainable AI (XAI) libraries](https://github.com/wangyongjie-ntu/Awesome-explainable-AI#python-librariessort-in-alphabeta-order), the Awsome explainable AI features 57 of them, of which 9 have more than 2000 github stars:
-[SHAP](https://github.com/slundberg/shap)![](https://img.shields.io/github/stars/slundberg/shap.svg?style=social),
-[LIME](https://github.com/marcotcr/lime) ![](https://img.shields.io/github/stars/marcotcr/lime.svg?style=social),
-[pytorch-cnn-visualizations](https://github.com/utkuozbulak/pytorch-cnn-visualizations)![](https://img.shields.io/github/stars/utkuozbulak/pytorch-cnn-visualizations?style=social),
-[InterpretML](https://github.com/interpretml/interpret) ![](https://img.shields.io/github/stars/InterpretML/interpret.svg?style=social)
-[Lucid](https://github.com/tensorflow/lucid) ![](https://img.shields.io/github/stars/tensorflow/lucid.svg?style=social),
-[Pytorch-grad-cam](https://github.com/jacobgil/pytorch-grad-cam)![](https://img.shields.io/github/stars/jacobgil/pytorch-grad-cam?style=social),
-[Deep Visualization Toolbox](https://github.com/yosinski/deep-visualization-toolbox) ![](https://img.shields.io/github/stars/yosinski/deep-visualization-toolbox?style=social),
-[Captum](https://github.com/pytorch/captum) ![](https://img.shields.io/github/stars/pytorch/captum.svg?style=social), 
-[ELI5](https://github.com/TeamHG-Memex/eli5) ![](https://img.shields.io/github/stars/TeamHG-Memex/eli5.svg?style=social), etc.
-
-These libraries currently have serious limitations in respect to the usage by the various scientific communities:
+## Context
+
+There are numerous other python OSS [eXplainable AI (XAI) libraries](https://github.com/wangyongjie-ntu/Awesome-explainable-AI#python-librariessort-in-alphabeta-order), the Awsome explainable AI (XAI) features many of them, the most popular being:
+[SHAP](https://github.com/slundberg/shap),
+[LIME](https://github.com/marcotcr/lime),
+[pytorch-cnn-visualizations](https://github.com/utkuozbulak/pytorch-cnn-visualizations),
+[InterpretML](https://github.com/interpretml/interpret),
+[Lucid](https://github.com/tensorflow/lucid),
+[Pytorch-grad-cam](https://github.com/jacobgil/pytorch-grad-cam),
+[Deep Visualization Toolbox](https://github.com/yosinski/deep-visualization-toolbox),
+[Captum](https://github.com/pytorch/captum),
+[ELI5](https://github.com/TeamHG-Memex/eli5), etc.
+
+These libraries currently have serious limitations in respect to the usage by the various *scientific communities*:
 
 * **A single XAI method or single data modality is implemented**
 
-While SHAP, LIME, Pytorch-grad-cam, DeepLIFT, etc. enjoy enormous popularity, the specific methods are not always suitable for the research task and/or data modality (e.g. Grad cam is applicable only for images). Most importantly, different methods approach the AI explainability differently and so a single method provides limited explainability capacity. Which is the best method to use?
+While SHAP, LIME, Pytorch-grad-cam, DeepLIFT, etc. enjoy enormous popularity, the specific methods are not always suitable for the research task and/or data modality. Most importantly, different methods approach the AI explainability differently and so a single method provides limited explainability capacity. Which is the best method to use? 
+
+The scientific data are of different modalities and much more diverse than the image or text bencmarking datasets used in the Computer Vision ot Natural Language Processing research communities, where the explainability methods originate from. 
 
 * **A single DNN format/framework/architecture is supported**
 
-Lucid supports only [Tensoflow](https://www.tensorflow.org/), Captum is the library for [PyTorch](https://pytorch.org/) users, [iNNvstigate](https://github.com/albermax/innvestigate) has been used by Keras users. Pytorch-grad-cam supports a single method for a single format and Pytorch-cnn-visualizaiton even limits the choice to a single DNN type- CNN. While this is not an issue for the current most popular framework communities, not all are supported by mature libraries (e.g. ) and most importantly are not "future-proof", e.g. [Caffe](https://caffe.berkeleyvision.org/) has been the most polular framework by the computer vision comminuty in 2018, while now is the era of Tensorflow, Pytorch and Keras.  
+Lucid supports only [Tensoflow](https://www.tensorflow.org/), Captum is the library for [PyTorch](https://pytorch.org/) users, [iNNvstigate](https://github.com/albermax/innvestigate) has been used by Keras users. Pytorch-grad-cam supports a single method for a single format and Pytorch-cnn-visualizaiton even limits the choice to a single DNN type- CNN. While this is not an issue for the current most popular framework communities, not all are supported by mature libraries and most importantly are not "future-proof", e.g. [Caffe](https://caffe.berkeleyvision.org/) has been the most polular framework by the computer vision comminuty in 2018, while now is the era of Tensorflow and Pytorch.  
 
 * **The choice of supported XAI methods seems random**
 
 ELI5, for example, supports multiple frameworks/formats and XAI methods, but is not clear how the selection of these methods has been made. Futhermore the library is not maintained since 2020 and hence lacks any methods proposed since. Again, which are the best methods to use?
 
 * **Requires (X)AI expertise**
-For example the DeepVis Toolbox requires DNN knowledge and is used only by AI experts mostly within the computer vision community (which is large and can explain the number of stars). 
+
+For example the DeepVis Toolbox requires DNN knowledge and is used only by AI experts mostly within the computer vision community. 
+
+## Scientific communities and non (X)AI experts needs
 
-All of the above issues have a common reason: _the libraries have been developed by (X)AI researchers for the (X)AI researchers_.
+All of the above issues have a common reason: **_the libraries have been developed by (X)AI researchers for the (X)AI researchers_**.
 
-Even for the purpose of advancing (X)AI research, systematic study of the properties of the relevance scores is lacking and currently in the XAI literature often the human interpretation is intertwined with the one of the exlainer. Suitable datasets for such a study are lacking: the popular and simplest dataset used in publicaitons,  MNIST, is too complex for such a prurpose with 10 classes and no structural content variation. The XAI community does not publish work on simple scientific datasets, which would make the technology understandable and trustworthy by non-expert scientists on intuitive level. Another important show-stopper for reproducible AI-enabled science is the plethora of current formats and the speed in which they become obsolete. Some libraries are also becoming obsolete for the lack of support of the research projects which have sponsored their creation (e.g. iNNvestigate).
+Even for the purpose of advancing (X)AI research itself, systematic study of the properties of the relevance scores is lacking and currently in the XAI literature often the human interpretation is intertwined with the one of the explainer. Suitable datasets for such a study are lacking: the popular and simplest dataset used in publicaitons,  MNIST, is too complex for such a prurpose with 10 classes and no structural content variation. The XAI community does not publish work on simple scientific datasets, which would make the technology understandable and trustworthy by non-expert scientists on intuitive level. 
 
-DIANNA is designed to address the above challenges: _the library is developed by the Research Software enigneers with XAI knowledge for scientists: both the X(AI) and especially not X(AI) experts._ We have strived to create well-documented, well-tested software, conforming with the modern software engineering practices.
+Another important show-stopper for *reproducible AI-enabled science* is the plethora of current formats and the speed in which they become obsolete. Some libraries are also becoming obsolete for the lack research funding, being essential for their creation (e.g. iNNvestigate).
 
-*	**Simple datasets**
 
-To demonstrate the usefulness and enable studying the properties of the relevances on an intuitive level, DIANNA comes with two new datasets: [Simple Geometric](https://doi.org/10.5281/zenodo.5012824)<img width="20" alt="SimpleGeometric Logo" src="https://user-images.githubusercontent.com/3244249/151817429-80f38846-8a4b-4471-a4c9-df7d19b668e5.png">
- and [Simple Scientific (LeafSnap30)](https://doi.org/10.5281/zenodo.5061352)<img width="20" alt="LeafSnap30 Logo" src="https://user-images.githubusercontent.com/3244249/151817480-649ad3b7-2b4b-4aa3-a5d6-ebfaa6aa614a.png"> – subset of [LeafSnap dataset](http://leafsnap.com/dataset/). Tree species classification on the LeafSnap data is a good simple scientific example of a problem tackled with both classical Computer Vision in the past and the superior DL method. Enhanced with explainable tool, the DL approach is a clear winner. DIANNA also uses 2 well-established benchmarks - a (modified to binary) MNIST and the [Stanford Sentiment Treebank](https://nlp.stanford.edu/sentiment/index.html).
+## DIANNA for Open Science and non (X)AI experts
+
+DIANNA is designed to address the above challenges: **_the library is developed by the Research Software enigneers with XAI knowledge for scientists: both the X(AI) and especially not X(AI) experts_**. We have strived to create well-documented, well-tested software, conforming with the modern software engineering practices.
+
 
 *	**Systematically chosen generic explainers**
 
 DIANNA includes few XAI approaches which have been evaluated using [systematically defined criteria](https://arxiv.org/ftp/arxiv/papers/1912/1912.05100.pdf). Using a relevant subset of these criteria, we have included complementary state-of-the-art XAI methods: [LIME](https://www.kdd.org/kdd2016/papers/files/rfp0573-ribeiroA.pdf), [RISE](http://bmvc2018.org/contents/papers/1064.pdf) and [KernelSHAP](https://proceedings.neurips.cc/paper/2017/file/8a20a8621978632d76c43dfd28b67767-Paper.pdf). These methods are model architecture agnostic.
 
+* **Simple unified interface**
+
+DIANNA provided a simple unified interface to the chosen XAI methods (explainers). The library can be used both from the command-line and we are working on an easy to use [dashboard](https://github.com/dianna-ai/dianna/tree/main?tab=readme-ov-file#dashboard), which can already be run with some of our examples for demo purposes.
+
 *	**Standard DNN format**
 
 DIANNA is future-proof, the library is conforming with the [Open Neural Network eXchange (ONNX) standart](https://onnx.ai/). Using ONNX is not only very beneficial for interoperability, hence enabling reproducible-science, but ONNX-compatible runtimes and libraries are designed to maximize performance across hardware. 
 
 * **Multiple data modalities** 
 
-Most libraries support at most 2 modalities: images and text. While DIANNA has also started with these two, there are plans to include modalities, many scientists lack support for: time series, tabular data and embeddings. 
+Most libraries support at most 2 modalities: images and text. While DIANNA has also started with these two, now it also supports time series and tabular data with embeddings being work in progress. 
+
+*	**Simple datasets**
+
+To demonstrate the usefulness and enable studying the properties of the relevances on an intuitive level, DIANNA comes with new and existing [datasets](https://github.com/dianna-ai/dianna/tree/main?tab=readme-ov-file#datasets) and [models](https://github.com/dianna-ai/dianna/tree/main?tab=readme-ov-file#onnx-models) for both classification and regression tasks for image, text, time series and tabular data.
+
+* **Scientific use cases** 
+
+DIANNA explicitely shows how to use XAI in AI-enhanced scientific research. [Tutorials](https://github.com/dianna-ai/dianna/tree/main?tab=readme-ov-file#tutorials) for use-cases from the natural, envoronmental and social sciences are provided. Some of the explanability examples on both simple and scientific use-cases are also presented within the dashboard.
 

setup.cfg

@@ -39,7 +39,7 @@ install_requires =
     ipython
     lime
     matplotlib
-    numpy
+    numpy>=1.20
     onnx==1.14.1
     onnx_tf
     onnxruntime