Skip to content

Commit

Permalink
Set LitServer.stream using LitSpec.stream (#398)
Browse files Browse the repository at this point in the history
* set stream using method

* fix
  • Loading branch information
aniketmaurya authored Dec 17, 2024
1 parent fcce973 commit 636c9fd
Show file tree
Hide file tree
Showing 3 changed files with 10 additions and 3 deletions.
5 changes: 2 additions & 3 deletions src/litserve/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@
from litserve.loops import LitLoop, get_default_loop, inference_worker
from litserve.middlewares import MaxSizeMiddleware, RequestCountMiddleware
from litserve.python_client import client_template
from litserve.specs import OpenAISpec
from litserve.specs.base import LitSpec
from litserve.utils import LitAPIStatus, WorkerSetupStatus, call_after_stream

Expand Down Expand Up @@ -146,8 +145,8 @@ def __init__(
raise ValueError("batch_timeout must be less than timeout")
if max_batch_size <= 0:
raise ValueError("max_batch_size must be greater than 0")
if isinstance(spec, OpenAISpec):
stream = True
if isinstance(spec, LitSpec):
stream = spec.stream

if loop is None:
loop = "auto"
Expand Down
4 changes: 4 additions & 0 deletions src/litserve/specs/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ def __init__(self):

self._server: LitServer = None

@property
def stream(self):
return False

def pre_setup(self, lit_api: "LitAPI"):
pass

Expand Down
4 changes: 4 additions & 0 deletions src/litserve/specs/openai.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,10 @@ def __init__(
self.add_endpoint("/v1/chat/completions", self.chat_completion, ["POST"])
self.add_endpoint("/v1/chat/completions", self.options_chat_completions, ["OPTIONS"])

@property
def stream(self):
return True

def pre_setup(self, lit_api: "LitAPI"):
from litserve import LitAPI

Expand Down

0 comments on commit 636c9fd

Please sign in to comment.