diff --git a/pyproject.toml b/pyproject.toml
index e47f7e55d8..bb0d458f4e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,7 +57,7 @@ core = [
     "open-clip-torch>=2.23.0,<2.26.1",
 ]
 openvino = ["openvino>=2024.0", "nncf>=2.10.0", "onnx>=1.16.0"]
-vlm = ["ollama", "openai", "python-dotenv","transformers"]
+vlm = ["ollama<0.4.0", "openai", "python-dotenv","transformers"]
 loggers = [
     "comet-ml>=3.31.7",
     "gradio>=4",