Move most xfails to entries in tank/all_models.csv and temporarily re…

…move multiprocessing and TF gpu support. (#602)

* Move most xfails to entries in tank/all_models.csv

* enable usage of pytest without specifying tank/test_models.py

* add dict_configs.py to gitignore.

* Pin versions for runtimes and torch-mlir for setup.

.github/workflows/nightly.yml

@@ -122,7 +122,7 @@ jobs:
       continue-on-error: true
       run: |
         cd $GITHUB_WORKSPACE
-        USE_IREE=1 VENV_DIR=iree.venv ./setup_venv.sh
+        USE_IREE=1 VENV_DIR=iree.venv NIGHTLY=1 ./setup_venv.sh
         source iree.venv/bin/activate
         package_version="$(printf '%(%Y%m%d)T.${{ github.run_number }}')"
         SHARK_PACKAGE_VERSION=${package_version} \
@@ -146,7 +146,7 @@ jobs:
       if: ${{ matrix.backend == 'SHARK' }}
       run: |
         cd $GITHUB_WORKSPACE
-        ./setup_venv.sh
+        NIGHTLY=1 ./setup_venv.sh
         source shark.venv/bin/activate
         package_version="$(printf '%(%Y%m%d)T.${{ github.run_number }}')"
         SHARK_PACKAGE_VERSION=${package_version} \

.github/workflows/test-models.yml

@@ -123,7 +123,7 @@ jobs:
         echo "VULKAN SDK PATH with setup: $VULKAN_SDK"
         echo $PATH
         pip list | grep -E "torch|iree"
-        pytest --ci --ci_sha=${SHORT_SHA} --local_tank_cache="/Volumes/builder/anush/shark_cache" tank/test_models.py -k vulkan --update_tank
+        pytest -s --ci --ci_sha=${SHORT_SHA} --local_tank_cache="/Volumes/builder/anush/shark_cache" tank/test_models.py -k vulkan --update_tank
 
     - name: Validate Vulkan Models (a100)
       if: matrix.suite == 'vulkan' && matrix.os != 'MacStudio'

.gitignore

@@ -164,6 +164,7 @@ cython_debug/
 shark_tmp/
 *.vmfb
 .use-iree
+tank/dict_configs.py
 
 # ORT related artefacts
 cache_models/

build_tools/shark_versions.txt

@@ -0,0 +1,8 @@
+# IREE Compiler/ Runtime Version:
+20221207.350
+# SHARK Compiler/ Runtime Version:
+20221207.236
+# Torch-MLIR Version for IREE:
+20221207.680
+# Torch-MLIR Version for SHARK:
+20221207.680

conftest.py

@@ -38,7 +38,7 @@ def pytest_addoption(parser):
     )
     parser.addoption(
         "--update_tank",
-        action="store_true",
+        action="store_false",
         default="False",
         help="Update local shark tank with latest artifacts.",
     )

pytest.ini

@@ -1,3 +1,3 @@
 [pytest]
 addopts = --verbose -p no:warnings
-norecursedirs = inference tank/tflite 
+norecursedirs = inference tank/tflite examples benchmarks shark 

setup.py

@@ -6,12 +6,18 @@
 with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 
+with open("build_tools/shark_versions.txt", "r") as sv:
+    lines = [line.rstrip() for line in sv]
+    TM_VERSION = lines[7]
+    IREE_VERSION = lines[3]
+
+
 PACKAGE_VERSION = os.environ.get("SHARK_PACKAGE_VERSION") or "0.0.4"
 backend_deps = []
 if "NO_BACKEND" in os.environ.keys():
     backend_deps = [
-        "iree-compiler>=20221022.190",
-        "iree-runtime>=20221022.190",
+        f"iree-compiler=={IREE_VERSION}",
+        f"iree-runtime>={IREE_VERSION}",
     ]
 
 setup(
@@ -37,7 +43,7 @@
     install_requires=[
         "numpy",
         "PyYAML",
-        "torch-mlir>=20221021.633",
+        f"torch-mlir=={TM_VERSION}",
     ]
     + backend_deps,
 )

setup_venv.sh

@@ -76,12 +76,15 @@ fi
 $PYTHON -m pip install --upgrade pip || die "Could not upgrade pip"
 $PYTHON -m pip install --upgrade -r "$TD/requirements.txt"
 if [ "$torch_mlir_bin" = true ]; then
+  TM_VERSION=$(sed '8q;d' build_tools/shark_versions.txt)
   if [[ $(uname -s) = 'Darwin' ]]; then
     echo "MacOS detected. Installing torch-mlir from .whl, to avoid dependency problems with torch."
-    $PYTHON -m pip install --pre --no-cache-dir  torch-mlir -f https://llvm.github.io/torch-mlir/package-index/ -f https://download.pytorch.org/whl/nightly/torch/
-  else
+    $PYTHON -m pip install --pre --no-cache-dir  torch-mlir==${TM_VERSION} -f https://llvm.github.io/torch-mlir/package-index/ -f https://download.pytorch.org/whl/nightly/torch/
+  elif [[ ! -z "${NIGHTLY}" ]]; then
     $PYTHON -m pip install --pre torch-mlir -f https://llvm.github.io/torch-mlir/package-index/
-    if [ $? -eq 0 ];then
+  else
+    $PYTHON -m pip install --pre torch-mlir==${TM_VERSION} -f https://llvm.github.io/torch-mlir/package-index/
+    if [ $? -eq 0 ]; then
       echo "Successfully Installed torch-mlir"
     else
       echo "Could not install torch-mlir" >&2
@@ -96,13 +99,17 @@ fi
 if [[ -z "${USE_IREE}" ]]; then
   rm .use-iree
   RUNTIME="https://nod-ai.github.io/SHARK-Runtime/pip-release-links.html"
+  RUNTIME_VERSION=$(sed '4q;d' build_tools/shark_versions.txt)
+  TM_VERSION=$(sed '8q;d' build_tools/shark_versions.txt)
 else
   touch ./.use-iree
   RUNTIME="https://iree-org.github.io/iree/pip-release-links.html"
+  RUNTIME_VERSION=$(sed '2q;d' build_tools/shark_versions.txt)
+  TM_VERSION=$(sed '6q;d' build_tools/shark_versions.txt)
 fi
 if [[ -z "${NO_BACKEND}" ]]; then
   echo "Installing ${RUNTIME}..."
-  $PYTHON -m pip install --upgrade --find-links ${RUNTIME} iree-compiler iree-runtime
+  $PYTHON -m pip install --upgrade --find-links ${RUNTIME} iree-compiler==${RUNTIME_VERSION} iree-runtime==${RUNTIME_VERSION}
 else
   echo "Not installing a backend, please make sure to add your backend to PYTHONPATH"
 fi
@@ -122,6 +129,7 @@ fi
 
 $PYTHON -m pip install --no-warn-conflicts -e . -f https://llvm.github.io/torch-mlir/package-index/ -f ${RUNTIME} -f https://download.pytorch.org/whl/nightly/torch/
 
+
 if [[ $(uname -s) = 'Linux' && ! -z "${BENCHMARK}" ]]; then
   $PYTHON -m pip uninstall -y torch torchvision
   $PYTHON -m pip install --pre torch torchvision --extra-index-url https://download.pytorch.org/whl/nightly/cu117
@@ -142,6 +150,15 @@ if [[ ! -z "${ONNX}" ]]; then
   fi
 fi
 
+if [[ ! -z "${NIGHTLY}" ]]; then
+  $PYTHON -m pip install --upgrade --pre iree-compiler iree-runtime torch-mlir -f https://llvm.github.io/torch-mlir/package-index/ -f $RUNTIME -f https://download.pytorch.org/whl/nightly/torch/
+  if [ $? -eq 0 ];then
+    echo "Successfully Installed latest packages for nightly job."
+  else
+    echo "Could not install latest IREE and Torch-MLIR." >&2
+  fi
+fi
+
 if [[ -z "${CONDA_PREFIX}" ]]; then
   echo "${Green}Before running examples activate venv with:"
   echo "  ${Green}source $VENV_DIR/bin/activate"

shark/parser.py

@@ -108,7 +108,7 @@ def dir_file(path):
 parser.add_argument(
     "--enable_conv_transform",
     default=False,
-    action="store",
+    action="store_false",
     help="Enables the --iree-flow-enable-conv-nchw-to-nhwc-transform flag.",
 )
 

shark/shark_benchmark_runner.py

@@ -100,6 +100,7 @@ def setup_cl(self, input_tensors):
     def benchmark_frontend(self, modelname):
         if self.mlir_dialect in ["linalg", "torch"]:
             return self.benchmark_torch(modelname)
+
         elif self.mlir_dialect in ["mhlo", "tf"]:
             return self.benchmark_tf(modelname)
 
@@ -138,9 +139,21 @@ def benchmark_torch(self, modelname):
 
     def benchmark_tf(self, modelname):
         import tensorflow as tf
+
+        visible_default = tf.config.list_physical_devices("GPU")
+        try:
+            tf.config.set_visible_devices([], "GPU")
+            visible_devices = tf.config.get_visible_devices()
+            for device in visible_devices:
+                assert device.device_type != "GPU"
+        except:
+            # Invalid device or cannot modify virtual devices once initialized.
+            pass
+
         from tank.model_utils_tf import get_tf_model
 
-        tf_device = "/GPU:0" if self.device == "cuda" else "/CPU:0"
+        # tf_device = "/GPU:0" if self.device == "cuda" else "/CPU:0"
+        tf_device = "/CPU:0"
         with tf.device(tf_device):
             model, input, = get_tf_model(
                 modelname

shark/shark_downloader.py

@@ -118,9 +118,7 @@ def check_dir_exists(model_name, frontend="torch", dynamic=""):
             and os.path.isfile(os.path.join(model_dir, "golden_out.npz"))
             and os.path.isfile(os.path.join(model_dir, "hash.npy"))
         ):
-            print(
-                f"""Using cached models from {WORKDIR}..."""
-            )
+            print(f"""Using cached models from {WORKDIR}...""")
             return True
     return False
 

tank/all_models.csv

@@ -1,35 +1,34 @@
-resnet50,mhlo,tf,1e-2,1e-3,default,nhcw-nhwc
-albert-base-v2,mhlo,tf,1e-2,1e-2,default,None
-roberta-base,mhlo,tf,1e-02,1e-3,default,nhcw-nhwc
-bert-base-uncased,mhlo,tf,1e-2,1e-3,default,None
-camembert-base,mhlo,tf,1e-2,1e-3,default,None
-dbmdz/convbert-base-turkish-cased,mhlo,tf,1e-2,1e-3,default,nhcw-nhwc
-distilbert-base-uncased,mhlo,tf,1e-2,1e-3,default,None
-facebook/convnext-tiny-224,mhlo,tf,1e-2,1e-3,tf_vit,nhcw-nhwc,
-funnel-transformer/small,mhlo,tf,1e-2,1e-3,default,None
-google/electra-small-discriminator,mhlo,tf,1e-2,1e-3,default,None
-google/mobilebert-uncased,mhlo,tf,1e-2,1e-3,default,None
-google/vit-base-patch16-224,mhlo,tf,1e-2,1e-3,tf_vit,nhcw-nhwc
-hf-internal-testing/tiny-random-flaubert,mhlo,tf,1e-2,1e-3,default,None
-microsoft/MiniLM-L12-H384-uncased,mhlo,tf,1e-2,1e-3,tf_hf,None
-microsoft/layoutlm-base-uncased,mhlo,tf,1e-2,1e-3,default,None
-microsoft/mpnet-base,mhlo,tf,1e-2,1e-2,default,None
-albert-base-v2,linalg,torch,1e-2,1e-3,default,None
-alexnet,linalg,torch,1e-2,1e-3,default,None
-bert-base-cased,linalg,torch,1e-2,1e-3,default,None
-bert-base-uncased,linalg,torch,1e-2,1e-3,default,None
-facebook/deit-small-distilled-patch16-224,linalg,torch,1e-2,1e-3,default,nhcw-nhwc
-google/vit-base-patch16-224,linalg,torch,1e-2,1e-3,default,nhcw-nhwc
-microsoft/beit-base-patch16-224-pt22k-ft22k,linalg,torch,1e-2,1e-3,default,nhcw-nhwc
-microsoft/MiniLM-L12-H384-uncased,linalg,torch,1e-2,1e-3,default,None
-microsoft/resnet-50,linalg,torch,1e-2,1e-3,default,nhcw-nhwc
-google/mobilebert-uncased,linalg,torch,1e-2,1e-3,default,None
-mobilenet_v3_small,linalg,torch,1e-1,1e-2,default,nhcw-nhwc
-nvidia/mit-b0,linalg,torch,1e-2,1e-3,default,None
-resnet101,linalg,torch,1e-2,1e-3,default,nhcw-nhwc
-resnet18,linalg,torch,1e-2,1e-3,default,None
-resnet50,linalg,torch,1e-2,1e-3,default,nhcw-nhwc
-squeezenet1_0,linalg,torch,1e-2,1e-3,default,nhcw-nhwc
-wide_resnet50_2,linalg,torch,1e-2,1e-3,default,nhcw-nhwc
-efficientnet-v2-s,mhlo,tf,1e-02,1e-3,default,nhcw-nhwc
-mnasnet1_0,linalg,torch,1e-2,1e-3,default,nhcw-nhwc
+resnet50,mhlo,tf,1e-2,1e-3,default,nhcw-nhwc,False,False,True,"Vulkan Numerical Error: mostly conv"
+albert-base-v2,mhlo,tf,1e-2,1e-2,default,None,False,False,False,""
+roberta-base,mhlo,tf,1e-02,1e-3,default,nhcw-nhwc,False,False,False,""
+bert-base-uncased,mhlo,tf,1e-2,1e-3,default,None,False,False,False,""
+camembert-base,mhlo,tf,1e-2,1e-3,default,None,False,False,False,""
+dbmdz/convbert-base-turkish-cased,mhlo,tf,1e-2,1e-3,default,nhcw-nhwc,True,False,True,"https://github.com/iree-org/iree/issues/9971"
+distilbert-base-uncased,mhlo,tf,1e-2,1e-3,default,None,False,False,False,""
+facebook/convnext-tiny-224,mhlo,tf,1e-2,1e-3,tf_vit,nhcw-nhwc,False,True,True,"https://github.com/nod-ai/SHARK/issues/311 & https://github.com/nod-ai/SHARK/issues/342"
+funnel-transformer/small,mhlo,tf,1e-2,1e-3,default,None,False,True,True,"https://github.com/nod-ai/SHARK/issues/201"
+google/electra-small-discriminator,mhlo,tf,1e-2,1e-3,default,None,False,False,False,""
+google/mobilebert-uncased,mhlo,tf,1e-2,1e-3,default,None,False,False,False,""
+google/vit-base-patch16-224,mhlo,tf,1e-2,1e-3,tf_vit,nhcw-nhwc,False,False,True,"Vulkan Numerical Error (mostly conv)"
+microsoft/MiniLM-L12-H384-uncased,mhlo,tf,1e-2,1e-3,tf_hf,None,False,False,False,""
+microsoft/layoutlm-base-uncased,mhlo,tf,1e-2,1e-3,default,None,False,False,False,""
+microsoft/mpnet-base,mhlo,tf,1e-2,1e-2,default,None,False,False,False,""
+albert-base-v2,linalg,torch,1e-2,1e-3,default,None,False,False,False,""
+alexnet,linalg,torch,1e-2,1e-3,default,None,False,False,True,"Assertion Error: Zeros Output"
+bert-base-cased,linalg,torch,1e-2,1e-3,default,None,False,False,False,""
+bert-base-uncased,linalg,torch,1e-2,1e-3,default,None,False,False,False,""
+facebook/deit-small-distilled-patch16-224,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,True,False,"Fails during iree-compile."
+google/vit-base-patch16-224,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,True,False,"https://github.com/nod-ai/SHARK/issues/311"
+microsoft/beit-base-patch16-224-pt22k-ft22k,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,True,False,"https://github.com/nod-ai/SHARK/issues/390"
+microsoft/MiniLM-L12-H384-uncased,linalg,torch,1e-2,1e-3,default,None,False,False,True,""
+microsoft/resnet-50,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,False,True,"Vulkan Numerical Error (mostly conv)"
+google/mobilebert-uncased,linalg,torch,1e-2,1e-3,default,None,False,False,True,"https://github.com/nod-ai/SHARK/issues/344"
+mobilenet_v3_small,linalg,torch,1e-1,1e-2,default,nhcw-nhwc,False,True,True,"https://github.com/nod-ai/SHARK/issues/388"
+nvidia/mit-b0,linalg,torch,1e-2,1e-3,default,None,True,True,True,"https://github.com/nod-ai/SHARK/issues/343"
+resnet101,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,False,True,"Vulkan Numerical Error (mostly conv)"
+resnet18,linalg,torch,1e-2,1e-3,default,None,True,True,True,""
+resnet50,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,False,True,"Vulkan Numerical Error (mostly conv)"
+squeezenet1_0,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,False,True,"https://github.com/nod-ai/SHARK/issues/388"
+wide_resnet50_2,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,False,True,"Vulkan Numerical Error (mostly conv)"
+efficientnet-v2-s,mhlo,tf,1e-02,1e-3,default,nhcw-nhwc,False,False,True,"https://github.com/nod-ai/SHARK/issues/575"
+mnasnet1_0,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,False,True,"https://github.com/nod-ai/SHARK/issues/388"