Merge remote-tracking branch 'upstream/main' into iree-qol

e2eshark/onnx/.gitignore

@@ -1,2 +1,4 @@
 # Model artifacts
 operators/*/model.onnx
+models/*/model.onnx
+models/*/model.onnxsignature.json

e2eshark/onnx/combinations/QuantizeToMatMulInteger/model.py

@@ -0,0 +1,82 @@
+# Copyright 2024 Advanced Micro Devices, Inc.
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# run.py creates runmodel.py by concatenating this file model.py
+# and tools/stubs/onnxmodel.py
+
+# See https://onnx.ai/onnx/intro/python.html for intro on creating
+# onnx model using python onnx API
+import numpy, torch, sys
+import onnxruntime
+from onnx import numpy_helper, TensorProto, save_model
+from onnx.helper import make_model, make_node, make_graph, make_tensor_value_info
+from onnx.checker import check_model
+
+# import from e2eshark/tools to allow running in current dir, for run through
+# run.pl, commutils is symbolically linked to allow any rundir to work
+sys.path.insert(0, "../../../tools/stubs")
+from commonutils import E2ESHARK_CHECK_DEF
+
+# Create an instance of it for this test
+E2ESHARK_CHECK = dict(E2ESHARK_CHECK_DEF)
+
+# Create an input (ValueInfoProto)
+X = make_tensor_value_info("X", TensorProto.FLOAT, [2, 4, 5])
+Y = make_tensor_value_info("Y", TensorProto.FLOAT, [5, 3])
+
+# Create an output
+Z = make_tensor_value_info("Z", TensorProto.INT32, [2, 4, 3])
+
+# Create a node (NodeProto)
+qlxnode = make_node(
+    "DynamicQuantizeLinear", ["X"], ["QX", "SX", "ZPX"], "qlxnode"
+)
+qlynode = make_node(
+    "DynamicQuantizeLinear", ["Y"], ["QY", "SY", "ZPY"], "qlynode"
+)
+mminode = make_node(
+    "MatMulInteger", ["QX", "QY", "ZPX", "ZPY"], ["Z"], "mminode"  # node name  # inputs  # outputs
+)
+
+# Create the graph (GraphProto)
+graph = make_graph(
+    [qlxnode,qlynode, mminode],
+    "mmigraph",
+    [X, Y],
+    [Z],
+)
+
+# Create the model (ModelProto)
+onnx_model = make_model(graph)
+onnx_model.opset_import[0].version = 19
+
+# Save the model
+# save_model(onnx_model, "model.onnx")
+with open("model.onnx", "wb") as f:
+    f.write(onnx_model.SerializeToString())
+
+session = onnxruntime.InferenceSession("model.onnx", None)
+model_input_X = numpy.random.randn(2, 4, 5).astype(numpy.float32)
+model_input_Y = numpy.random.randn(5, 3).astype(numpy.float32)
+# gets X in inputs[0] and Y in inputs[1]
+inputs = session.get_inputs()
+# gets Z in outputs[0]
+outputs = session.get_outputs()
+
+model_output = session.run(
+    [outputs[0].name],
+    {inputs[0].name: model_input_X, inputs[1].name: model_input_Y},
+)
+
+# Moving to torch to handle bfloat16 as numpy does not support bfloat16
+E2ESHARK_CHECK["input"] = [
+    torch.from_numpy(model_input_X),
+    torch.from_numpy(model_input_Y),
+]
+E2ESHARK_CHECK["output"] = [torch.from_numpy(arr) for arr in model_output]
+
+print("Input:", E2ESHARK_CHECK["input"])
+print("Output:", E2ESHARK_CHECK["output"])

e2eshark/onnx/models/AlexNet_vaiq_int8/model.py

@@ -42,8 +42,8 @@
 # Post process output to do:
 # sort(topk(torch.nn.functional.softmax(output, 0), 2)[1])[0]
 # Top most probability
-E2ESHARK_CHECK["postprocess"] = [
-    (torch.nn.functional.softmax, [0], False, 0),
-    (torch.topk, [2], True, 1),
-    (torch.sort, [], True, 0),
-]
+# E2ESHARK_CHECK["postprocess"] = [
+#     (torch.nn.functional.softmax, [0], False, 0),
+#     (torch.topk, [2], True, 1),
+#     (torch.sort, [], True, 0),
+# ]

e2eshark/onnx/models/CSP-DarkNet_vaiq_int8/model.py

@@ -42,8 +42,8 @@
 # Post process output to do:
 # sort(topk(torch.nn.functional.softmax(output, 0), 2)[1])[0]
 # Top most probability
-E2ESHARK_CHECK["postprocess"] = [
-    (torch.nn.functional.softmax, [0], False, 0),
-    (torch.topk, [2], True, 1),
-    (torch.sort, [], True, 0),
-]
+# E2ESHARK_CHECK["postprocess"] = [
+#     (torch.nn.functional.softmax, [0], False, 0),
+#     (torch.topk, [2], True, 1),
+#     (torch.sort, [], True, 0),
+# ]

e2eshark/onnx/models/ConvNeXt_vaiq_int8/model.py

@@ -42,8 +42,8 @@
 # Post process output to do:
 # sort(topk(torch.nn.functional.softmax(output, 0), 2)[1])[0]
 # Top most probability
-E2ESHARK_CHECK["postprocess"] = [
-    (torch.nn.functional.softmax, [0], False, 0),
-    (torch.topk, [2], True, 1),
-    (torch.sort, [], True, 0),
-]
+# E2ESHARK_CHECK["postprocess"] = [
+#     (torch.nn.functional.softmax, [0], False, 0),
+#     (torch.topk, [2], True, 1),
+#     (torch.sort, [], True, 0),
+# ]

e2eshark/onnx/models/DarkNet53_vaiq_int8/model.py

@@ -42,8 +42,8 @@
 # Post process output to do:
 # sort(topk(torch.nn.functional.softmax(output, 0), 2)[1])[0]
 # Top most probability
-E2ESHARK_CHECK["postprocess"] = [
-    (torch.nn.functional.softmax, [0], False, 0),
-    (torch.topk, [2], True, 1),
-    (torch.sort, [], True, 0),
-]
+# E2ESHARK_CHECK["postprocess"] = [
+#     (torch.nn.functional.softmax, [0], False, 0),
+#     (torch.topk, [2], True, 1),
+#     (torch.sort, [], True, 0),
+# ]

e2eshark/onnx/models/DenseNet201_vaiq_int8/model.py

@@ -42,8 +42,8 @@
 # Post process output to do:
 # sort(topk(torch.nn.functional.softmax(output, 0), 2)[1])[0]
 # Top most probability
-E2ESHARK_CHECK["postprocess"] = [
-    (torch.nn.functional.softmax, [0], False, 0),
-    (torch.topk, [2], True, 1),
-    (torch.sort, [], True, 0),
-]
+# E2ESHARK_CHECK["postprocess"] = [
+#     (torch.nn.functional.softmax, [0], False, 0),
+#     (torch.topk, [2], True, 1),
+#     (torch.sort, [], True, 0),
+# ]

e2eshark/onnx/models/EfficientNet_v2_s_vaiq_int8/model.py

@@ -42,8 +42,8 @@
 # Post process output to do:
 # sort(topk(torch.nn.functional.softmax(output, 0), 2)[1])[0]
 # Top most probability
-E2ESHARK_CHECK["postprocess"] = [
-    (torch.nn.functional.softmax, [0], False, 0),
-    (torch.topk, [2], True, 1),
-    (torch.sort, [], True, 0),
-]
+# E2ESHARK_CHECK["postprocess"] = [
+#     (torch.nn.functional.softmax, [0], False, 0),
+#     (torch.topk, [2], True, 1),
+#     (torch.sort, [], True, 0),
+# ]

e2eshark/onnx/models/FCN_vaiq_int8/model.py

@@ -30,7 +30,7 @@
 
 
 model_output = session.run(
-    [outputs[0].name],
+    [outputs[0].name, outputs[1].name],
     {inputs[0].name: model_input_X},
 )[0]
 E2ESHARK_CHECK["input"] = [torch.from_numpy(model_input_X)]

e2eshark/onnx/models/GoogLeNet_vaiq_int8/model.py

@@ -42,8 +42,8 @@
 # Post process output to do:
 # sort(topk(torch.nn.functional.softmax(output, 0), 2)[1])[0]
 # Top most probability
-E2ESHARK_CHECK["postprocess"] = [
-    (torch.nn.functional.softmax, [0], False, 0),
-    (torch.topk, [2], True, 1),
-    (torch.sort, [], True, 0),
-]
+# E2ESHARK_CHECK["postprocess"] = [
+#     (torch.nn.functional.softmax, [0], False, 0),
+#     (torch.topk, [2], True, 1),
+#     (torch.sort, [], True, 0),
+# ]

e2eshark/onnx/models/Inception_v4_vaiq_int8/model.py

@@ -21,7 +21,7 @@
 
 # Even if model is quantized, the inputs and outputs are
 # not, so apply float32
-model_input_X = numpy.random.rand(1, 3, 224, 224).astype(numpy.float32)
+model_input_X = numpy.random.rand(32, 3, 224, 224).astype(numpy.float32)
 
 # gets X in inputs[0] and Y in inputs[1]
 inputs = session.get_inputs()
@@ -42,8 +42,8 @@
 # Post process output to do:
 # sort(topk(torch.nn.functional.softmax(output, 0), 2)[1])[0]
 # Top most probability
-E2ESHARK_CHECK["postprocess"] = [
-    (torch.nn.functional.softmax, [0], False, 0),
-    (torch.topk, [2], True, 1),
-    (torch.sort, [], True, 0),
-]
+# E2ESHARK_CHECK["postprocess"] = [
+#     (torch.nn.functional.softmax, [0], False, 0),
+#     (torch.topk, [2], True, 1),
+#     (torch.sort, [], True, 0),
+# ]

e2eshark/onnx/models/KeypointRCNN_vaiq_int8/model.py

@@ -21,7 +21,8 @@
 
 # Even if model is quantized, the inputs and outputs are
 # not, so apply float32
-model_input_X = numpy.random.rand(1, 3, 224, 224).astype(numpy.float32)
+model_input_X = numpy.random.rand(3, 300, 400).astype(numpy.float32)
+model_input_Y = numpy.random.rand(3, 500, 400).astype(numpy.float32)
 
 # gets X in inputs[0] and Y in inputs[1]
 inputs = session.get_inputs()
@@ -30,8 +31,22 @@
 
 
 model_output = session.run(
-    [outputs[0].name],
-    {inputs[0].name: model_input_X},
+    [
+        outputs[0].name,
+        outputs[1].name,
+        outputs[2].name,
+        outputs[3].name,
+        outputs[4].name,
+        outputs[5].name,
+        outputs[6].name,
+        outputs[7].name,
+        outputs[8].name,
+        outputs[9].name,
+    ],
+    {
+        inputs[0].name: model_input_X,
+        inputs[1].name: model_input_Y,
+    },
 )[0]
 E2ESHARK_CHECK["input"] = [torch.from_numpy(model_input_X)]
 E2ESHARK_CHECK["output"] = [torch.from_numpy(arr) for arr in model_output]
@@ -42,8 +57,8 @@
 # Post process output to do:
 # sort(topk(torch.nn.functional.softmax(output, 0), 2)[1])[0]
 # Top most probability
-E2ESHARK_CHECK["postprocess"] = [
-    (torch.nn.functional.softmax, [0], False, 0),
-    (torch.topk, [2], True, 1),
-    (torch.sort, [], True, 0),
-]
+# E2ESHARK_CHECK["postprocess"] = [
+#     (torch.nn.functional.softmax, [0], False, 0),
+#     (torch.topk, [2], True, 1),
+#     (torch.sort, [], True, 0),
+# ]

e2eshark/onnx/models/LRASPP_vaiq_int8/model.py

@@ -42,8 +42,8 @@
 # Post process output to do:
 # sort(topk(torch.nn.functional.softmax(output, 0), 2)[1])[0]
 # Top most probability
-E2ESHARK_CHECK["postprocess"] = [
-    (torch.nn.functional.softmax, [0], False, 0),
-    (torch.topk, [2], True, 1),
-    (torch.sort, [], True, 0),
-]
+# E2ESHARK_CHECK["postprocess"] = [
+#     (torch.nn.functional.softmax, [0], False, 0),
+#     (torch.topk, [2], True, 1),
+#     (torch.sort, [], True, 0),
+# ]

e2eshark/onnx/models/MNASNet_1_3_vaiq_int8/model.py

@@ -42,8 +42,8 @@
 # Post process output to do:
 # sort(topk(torch.nn.functional.softmax(output, 0), 2)[1])[0]
 # Top most probability
-E2ESHARK_CHECK["postprocess"] = [
-    (torch.nn.functional.softmax, [0], False, 0),
-    (torch.topk, [2], True, 1),
-    (torch.sort, [], True, 0),
-]
+# E2ESHARK_CHECK["postprocess"] = [
+#     (torch.nn.functional.softmax, [0], False, 0),
+#     (torch.topk, [2], True, 1),
+#     (torch.sort, [], True, 0),
+# ]

e2eshark/onnx/models/MobileNetV3_small_vaiq_int8/model.py

@@ -42,8 +42,8 @@
 # Post process output to do:
 # sort(topk(torch.nn.functional.softmax(output, 0), 2)[1])[0]
 # Top most probability
-E2ESHARK_CHECK["postprocess"] = [
-    (torch.nn.functional.softmax, [0], False, 0),
-    (torch.topk, [2], True, 1),
-    (torch.sort, [], True, 0),
-]
+# E2ESHARK_CHECK["postprocess"] = [
+#     (torch.nn.functional.softmax, [0], False, 0),
+#     (torch.topk, [2], True, 1),
+#     (torch.sort, [], True, 0),
+# ]

e2eshark/onnx/models/RAFT_vaiq_int8/model.py

@@ -30,13 +30,36 @@
 outputs = session.get_outputs()
 
 model_output = session.run(
-    [outputs[0].name],
-    {inputs[0].name: model_input_X, inputs[1].name: model_input_Y},
+    [
+        outputs[0].name,
+        outputs[1].name,
+        outputs[2].name,
+        outputs[3].name,
+        outputs[4].name,
+        outputs[5].name,
+        outputs[6].name,
+        outputs[7].name,
+        outputs[8].name,
+        outputs[9].name,
+        outputs[10].name,
+        outputs[11].name,
+    ],
+    {
+        inputs[0].name: model_input_X,
+        inputs[1].name: model_input_Y
+    },
 )[0]
 E2ESHARK_CHECK["inputs"] = [torch.from_numpy(model_input_X), torch.from_numpy(model_input_Y)]
 E2ESHARK_CHECK["outputs"] = [torch.from_numpy(arr) for arr in model_output]
 
 print("Input:", E2ESHARK_CHECK["inputs"])
 print("Output:", E2ESHARK_CHECK["outputs"])
 
-# may need to add postprocess here #
+# Post process output to do:
+# sort(topk(torch.nn.functional.softmax(output, 0), 2)[1])[0]
+# Top most probability
+# E2ESHARK_CHECK["postprocess"] = [
+#     (torch.nn.functional.softmax, [0], False, 0),
+#     (torch.topk, [2], True, 1),
+#     (torch.sort, [], True, 0),
+# ]

e2eshark/onnx/models/RDN_pytorch_vaiq_int8/model.py

@@ -42,8 +42,8 @@
 # Post process output to do:
 # sort(topk(torch.nn.functional.softmax(output, 0), 2)[1])[0]
 # Top most probability
-E2ESHARK_CHECK["postprocess"] = [
-    (torch.nn.functional.softmax, [0], False, 0),
-    (torch.topk, [2], True, 1),
-    (torch.sort, [], True, 0),
-]
+# E2ESHARK_CHECK["postprocess"] = [
+#     (torch.nn.functional.softmax, [0], False, 0),
+#     (torch.topk, [2], True, 1),
+#     (torch.sort, [], True, 0),
+# ]

e2eshark/onnx/models/RRDB_ESRGAN_vaiq_int8/model.py

@@ -42,8 +42,8 @@
 # Post process output to do:
 # sort(topk(torch.nn.functional.softmax(output, 0), 2)[1])[0]
 # Top most probability
-E2ESHARK_CHECK["postprocess"] = [
-    (torch.nn.functional.softmax, [0], False, 0),
-    (torch.topk, [2], True, 1),
-    (torch.sort, [], True, 0),
-]
+# E2ESHARK_CHECK["postprocess"] = [
+#     (torch.nn.functional.softmax, [0], False, 0),
+#     (torch.topk, [2], True, 1),
+#     (torch.sort, [], True, 0),
+# ]

e2eshark/onnx/models/RegNet_y_8gf_vaiq_int8/model.py

@@ -42,8 +42,8 @@
 # Post process output to do:
 # sort(topk(torch.nn.functional.softmax(output, 0), 2)[1])[0]
 # Top most probability
-E2ESHARK_CHECK["postprocess"] = [
-    (torch.nn.functional.softmax, [0], False, 0),
-    (torch.topk, [2], True, 1),
-    (torch.sort, [], True, 0),
-]
+# E2ESHARK_CHECK["postprocess"] = [
+#     (torch.nn.functional.softmax, [0], False, 0),
+#     (torch.topk, [2], True, 1),
+#     (torch.sort, [], True, 0),
+# ]