predict concur logging

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "truss"
-version = "0.9.53"
+version = "0.9.53rc1"
 description = "A seamless bridge from model development to model delivery"
 license = "MIT"
 readme = "README.md"

truss/templates/control/control/application.py

@@ -61,7 +61,10 @@ def create_app(base_config: Dict):
         app_logger=app_logger,
     )
 
-    limits = httpx.Limits(max_keepalive_connections=8, max_connections=32)
+    # max_connections = 1000
+    max_connections = 32
+    print(f"MAX CONNECTIONS: {max_connections}")
+    limits = httpx.Limits(max_keepalive_connections=8, max_connections=max_connections)
     app_state.proxy_client = httpx.AsyncClient(
         base_url=f"http://localhost:{app_state.inference_server_port}", limits=limits
     )

truss/templates/server/model_wrapper.py

@@ -17,14 +17,7 @@
 from multiprocessing import Lock
 from pathlib import Path
 from threading import Thread
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    Optional,
-    Tuple,
-    Union,
-)
+from typing import Any, Callable, Dict, Optional, Tuple, Union
 
 import opentelemetry.sdk.trace as sdk_trace
 import starlette.requests
@@ -655,6 +648,7 @@ async def __call__(
                 preprocess_result = await self.preprocess(inputs, request)
 
         span_predict = self._tracer.start_span("call-predict")
+        print(f"PREDICT SEMAPHORE: {self._predict_semaphore.value}")
         async with deferred_semaphore_and_span(
             self._predict_semaphore, span_predict
         ) as get_defer_fn: