diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 2f13ff83e9f..bb454f7f6ad 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -990,10 +990,12 @@ steps: - vllm/entrypoints - tests/entrypoints/unit_tests - tests/entrypoints/weight_transfer + - tests/entrypoints/launchers - vllm/platforms/rocm.py commands: - pytest -v -s entrypoints/unit_tests - pytest -v -s entrypoints/weight_transfer + - pytest -v -s entrypoints/launchers - label: Entrypoints Integration (LLM) # TBD timeout_in_minutes: 180 @@ -2320,9 +2322,9 @@ steps: - tests/benchmarks/test_serve_cli.py - tests/entrypoints/openai/chat_completion/test_chat_completion.py - tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py - - tests/entrypoints/openai/completion/test_shutdown.py + - tests/entrypoints/launchers/test_shutdown.py - tests/entrypoints/openai/test_return_token_ids.py - - tests/entrypoints/openai/test_uds.py + - tests/entrypoints/serve/instrumentator/test_uds.py - tests/v1/sample/test_logprobs_e2e.py - vllm/platforms/rocm.py commands: @@ -2331,9 +2333,9 @@ steps: - pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)" - pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex and not test_kv_transfer_prompt_token_ids_round_trip and not test_kv_transfer_prompt_token_ids_streaming" - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple" - - pytest -v -s entrypoints/openai/completion/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly" + - pytest -v -s entrypoints/launchers/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly" - pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison" - - pytest -v -s entrypoints/openai/test_uds.py + - pytest -v -s entrypoints/serve/instrumentator/test_uds.py - pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server" - label: Rust Frontend Serve Admin Coverage # TBD diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index 56d4e012581..b75a21ea119 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -11,9 +11,11 @@ steps: - vllm/entrypoints - tests/entrypoints/unit_tests - tests/entrypoints/weight_transfer + - tests/entrypoints/launchers commands: - pytest -v -s entrypoints/unit_tests - pytest -v -s entrypoints/weight_transfer + - pytest -v -s entrypoints/launchers mirror: amd: label: ":amd: (MI355) Entrypoints Unit" diff --git a/.buildkite/test_areas/rust_frontend.yaml b/.buildkite/test_areas/rust_frontend.yaml index 36b7d375d4f..21fcd127dbd 100644 --- a/.buildkite/test_areas/rust_frontend.yaml +++ b/.buildkite/test_areas/rust_frontend.yaml @@ -18,9 +18,9 @@ steps: - tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py # - tests/entrypoints/openai/completion/test_prompt_validation.py - - tests/entrypoints/openai/completion/test_shutdown.py + - tests/entrypoints/launchers/test_shutdown.py - tests/entrypoints/openai/test_return_token_ids.py - - tests/entrypoints/openai/test_uds.py + - tests/entrypoints/serve/instrumentator/test_uds.py - tests/v1/sample/test_logprobs_e2e.py commands: - export VLLM_USE_RUST_FRONTEND=1 @@ -30,11 +30,11 @@ steps: - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple" # - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds" - - pytest -v -s entrypoints/openai/completion/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly" + - pytest -v -s entrypoints/launchers/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly" # test_comparison streams differently: Rust emits a separate first (prompt_token_ids) chunk and # finish chunk without logprobs, while the test reads `logprobs.tokens` on every chunk. - pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison" - - pytest -v -s entrypoints/openai/test_uds.py + - pytest -v -s entrypoints/serve/instrumentator/test_uds.py - pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server" - label: ":nvidia: (H200) Rust Frontend Serve/Admin Coverage" diff --git a/docs/deployment/integrations/kthena.md b/docs/deployment/integrations/kthena.md index 7cc3f14a71e..0285b733c6b 100644 --- a/docs/deployment/integrations/kthena.md +++ b/docs/deployment/integrations/kthena.md @@ -117,7 +117,7 @@ Commands: - > bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh leader --ray_cluster_size=2; python3 -m - vllm.entrypoints.openai.api_server --port 8080 --model + vllm.entrypoints.launchers.api_server.entry --port 8080 --model meta-llama/Llama-3.1-405B-Instruct --tensor-parallel-size 8 --pipeline-parallel-size 2 ``` diff --git a/docs/design/arch_overview.md b/docs/design/arch_overview.md index c19ea49d96d..711e9a2668e 100644 --- a/docs/design/arch_overview.md +++ b/docs/design/arch_overview.md @@ -62,20 +62,6 @@ vllm serve The code for the `vllm` CLI can be found in [vllm/entrypoints/cli/main.py](../../vllm/entrypoints/cli/main.py). -Sometimes you may see the API server entrypoint used directly instead of via the -`vllm` CLI command. For example: - -```bash -python -m vllm.entrypoints.openai.api_server --model -``` - -!!! warning - - `python -m vllm.entrypoints.openai.api_server` is deprecated - and may become unsupported in a future release. - -That code can be found in [vllm/entrypoints/openai/api_server.py](../../vllm/entrypoints/openai/api_server.py). - More details on the API server can be found in the [Online Serving](../serving/online_serving/README.md) document. ## V1 Process Architecture @@ -88,7 +74,7 @@ The API server process handles HTTP requests (e.g., the OpenAI-compatible API), By default, there is **1 API server process**, but when data parallelism is used, the API server count automatically scales to match the data parallel size. This can also be manually configured with the `--api-server-count` flag. Each API server connects to **all** engine cores via ZMQ in a many-to-many topology, enabling any API server to route requests to any engine core. Each API server process uses multiple CPU threads for media loading (controlled by `VLLM_MEDIA_LOADING_THREAD_COUNT`, default 8). -The code can be found in [vllm/entrypoints/openai/api_server.py](../../vllm/entrypoints/openai/api_server.py) and [vllm/v1/utils.py](../../vllm/v1/utils.py). +The code can be found in [vllm/entrypoints/launchers/api_server](../../vllm/entrypoints/launchers/api_server) and [vllm/v1/utils.py](../../vllm/v1/utils.py). ### Engine Core Process diff --git a/docs/design/endpoint_plugins.md b/docs/design/endpoint_plugins.md index 9f38fe5da18..98b755788e2 100644 --- a/docs/design/endpoint_plugins.md +++ b/docs/design/endpoint_plugins.md @@ -1,6 +1,6 @@ # Endpoint Plugins -Endpoint plugins let out-of-tree packages add HTTP routes to the OpenAI compatible API server without editing `vllm/entrypoints/openai/api_server.py`. Their scope is +Endpoint plugins let out-of-tree packages add HTTP routes to the OpenAI compatible API server. Their scope is the **HTTP surface only** registering routes and optionally per app state used by those routes. A plugin reaches the engine the same way an in-tree serving handler does, through the `EngineClient` it is handed at startup (e.g. `engine_client.collective_rpc(...)`). No new engine access path is introduced. !!! warning "Security" diff --git a/examples/applications/api_server/server.py b/examples/applications/api_server/server.py index adac4133210..e56a7931f66 100644 --- a/examples/applications/api_server/server.py +++ b/examples/applications/api_server/server.py @@ -4,8 +4,7 @@ NOTE: This API server is used only for demonstrating usage of AsyncEngine and simple performance benchmarks. It is not intended for production use. For production use, we recommend using our OpenAI compatible server. -We are also not going to accept PRs modifying this file, please -change `vllm/entrypoints/openai/api_server.py` instead. +We are also not going to accept PRs modifying this file. """ import asyncio @@ -21,7 +20,7 @@ from fastapi.responses import JSONResponse, Response, StreamingResponse import vllm.envs as envs from vllm.engine.arg_utils import AsyncEngineArgs from vllm.engine.async_llm_engine import AsyncLLMEngine -from vllm.entrypoints.launcher import serve_http +from vllm.entrypoints.launchers.launcher import serve_http from vllm.entrypoints.serve.utils.api_utils import with_cancellation from vllm.logger import init_logger from vllm.sampling_params import SamplingParams diff --git a/tests/entrypoints/launchers/__init__.py b/tests/entrypoints/launchers/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/entrypoints/launchers/api_server/__init__.py b/tests/entrypoints/launchers/api_server/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/entrypoints/unit_tests/_api_server_spawn_workers.py b/tests/entrypoints/launchers/api_server/_api_server_spawn_workers.py similarity index 100% rename from tests/entrypoints/unit_tests/_api_server_spawn_workers.py rename to tests/entrypoints/launchers/api_server/_api_server_spawn_workers.py diff --git a/tests/entrypoints/unit_tests/test_api_server_process_manager.py b/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py similarity index 99% rename from tests/entrypoints/unit_tests/test_api_server_process_manager.py rename to tests/entrypoints/launchers/api_server/test_api_server_process_manager.py index 902d0ce2d74..93d0c497006 100644 --- a/tests/entrypoints/unit_tests/test_api_server_process_manager.py +++ b/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py @@ -10,7 +10,7 @@ from unittest.mock import patch import pytest import zmq -from tests.entrypoints.unit_tests._api_server_spawn_workers import ( +from tests.entrypoints.launchers.api_server._api_server_spawn_workers import ( exit_before_report_worker, ) from vllm.utils.network_utils import make_zmq_socket, split_zmq_path diff --git a/tests/entrypoints/openai/test_multi_api_servers.py b/tests/entrypoints/launchers/api_server/test_multi_api_servers.py similarity index 100% rename from tests/entrypoints/openai/test_multi_api_servers.py rename to tests/entrypoints/launchers/api_server/test_multi_api_servers.py diff --git a/tests/entrypoints/unit_tests/test_launch_cli.py b/tests/entrypoints/launchers/test_launch_cli.py similarity index 100% rename from tests/entrypoints/unit_tests/test_launch_cli.py rename to tests/entrypoints/launchers/test_launch_cli.py diff --git a/tests/entrypoints/openai/completion/test_shutdown.py b/tests/entrypoints/launchers/test_shutdown.py similarity index 99% rename from tests/entrypoints/openai/completion/test_shutdown.py rename to tests/entrypoints/launchers/test_shutdown.py index 4b62ce18226..e3be2919f5e 100644 --- a/tests/entrypoints/openai/completion/test_shutdown.py +++ b/tests/entrypoints/launchers/test_shutdown.py @@ -147,7 +147,7 @@ async def test_shutdown_on_engine_failure(tmp_path: Path): # dtype, max-len etc set so that this can run in CI sys.executable, "-m", - "vllm.entrypoints.openai.api_server", + "vllm.entrypoints.launchers.api_server.entry", "--model", MODEL_NAME, "--dtype", diff --git a/tests/entrypoints/serve/utils/test_ssl_cert_refresher.py b/tests/entrypoints/launchers/test_ssl_cert_refresher.py similarity index 97% rename from tests/entrypoints/serve/utils/test_ssl_cert_refresher.py rename to tests/entrypoints/launchers/test_ssl_cert_refresher.py index 8f5251374a6..77d2aa3f359 100644 --- a/tests/entrypoints/serve/utils/test_ssl_cert_refresher.py +++ b/tests/entrypoints/launchers/test_ssl_cert_refresher.py @@ -7,7 +7,7 @@ from ssl import SSLContext import pytest -from vllm.entrypoints.serve.utils.ssl import SSLCertRefresher +from vllm.entrypoints.launchers.utils.ssl import SSLCertRefresher class MockSSLContext(SSLContext): diff --git a/tests/entrypoints/serve/exception_handling/test_http_status_metrics.py b/tests/entrypoints/serve/exception_handling/test_http_status_metrics.py index 060b05bbaca..9fb1c34b879 100644 --- a/tests/entrypoints/serve/exception_handling/test_http_status_metrics.py +++ b/tests/entrypoints/serve/exception_handling/test_http_status_metrics.py @@ -14,7 +14,7 @@ import pytest from fastapi import HTTPException from prometheus_client import CollectorRegistry -from vllm.entrypoints.openai.api_server import build_app +from vllm.entrypoints.launchers.api_server.entry import build_app from vllm.exceptions import ( VLLMNotFoundError, VLLMServerError, diff --git a/tests/entrypoints/openai/test_uds.py b/tests/entrypoints/serve/instrumentator/test_uds.py similarity index 96% rename from tests/entrypoints/openai/test_uds.py rename to tests/entrypoints/serve/instrumentator/test_uds.py index 7c855714108..7dfb1f2cd12 100644 --- a/tests/entrypoints/openai/test_uds.py +++ b/tests/entrypoints/serve/instrumentator/test_uds.py @@ -6,11 +6,10 @@ from tempfile import TemporaryDirectory import httpx import pytest +from tests.utils import RemoteOpenAIServer from vllm import envs from vllm.version import __version__ as VLLM_VERSION -from ...utils import RemoteOpenAIServer - MODEL_NAME = "Qwen/Qwen3-0.6B" diff --git a/tests/lora/test_add_lora.py b/tests/lora/test_add_lora.py index 9a82ab99ea9..628f5bc28eb 100644 --- a/tests/lora/test_add_lora.py +++ b/tests/lora/test_add_lora.py @@ -6,7 +6,7 @@ import time import pytest from vllm.engine.arg_utils import AsyncEngineArgs -from vllm.entrypoints.openai.api_server import ( +from vllm.entrypoints.launchers.api_server.entry import ( build_async_engine_client_from_engine_args, ) from vllm.inputs import TextPrompt diff --git a/tests/lora/test_lora_functions.py b/tests/lora/test_lora_functions.py index 1c692630284..ecf2ad9ee65 100644 --- a/tests/lora/test_lora_functions.py +++ b/tests/lora/test_lora_functions.py @@ -7,7 +7,7 @@ Script to test add_lora, remove_lora, pin_lora, list_loras functions. import pytest from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs -from vllm.entrypoints.openai.api_server import ( +from vllm.entrypoints.launchers.api_server.entry import ( build_async_engine_client_from_engine_args, ) from vllm.lora.request import LoRARequest diff --git a/tests/plugins_tests/test_endpoint_plugins.py b/tests/plugins_tests/test_endpoint_plugins.py index 9385d98f2e3..5733333e575 100644 --- a/tests/plugins_tests/test_endpoint_plugins.py +++ b/tests/plugins_tests/test_endpoint_plugins.py @@ -17,14 +17,14 @@ import pytest from fastapi import FastAPI from vllm_add_dummy_endpoint_plugin import DummyAdminEndpointPlugin -from vllm.entrypoints.openai.api_server import ( - _attach_endpoint_plugins, - _init_endpoint_plugins_state, - build_app, -) +from vllm.entrypoints.launchers.app import build_app from vllm.entrypoints.openai.cli_args import make_arg_parser from vllm.plugins import load_endpoint_plugins -from vllm.plugins.endpoint_plugins.interface import EndpointPlugin +from vllm.plugins.endpoint_plugins.interface import ( + EndpointPlugin, + attach_endpoint_plugins, + init_endpoint_plugins_state, +) from vllm.utils.argparse_utils import FlexibleArgumentParser @@ -157,7 +157,7 @@ def test_attach_is_noop_when_nothing_discovered(monkeypatch: pytest.MonkeyPatch) monkeypatch.delenv("VLLM_PLUGINS", raising=False) app = FastAPI() - _attach_endpoint_plugins(app, ("generate",)) + attach_endpoint_plugins(app, ("generate",)) assert app.state.endpoint_plugins == [] @@ -172,7 +172,7 @@ async def test_init_state_is_noop_without_phase_a(monkeypatch: pytest.MonkeyPatc monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin") state = State() - await _init_endpoint_plugins_state(_FakeEngineClient(), state, _build_args()) + await init_endpoint_plugins_state(_FakeEngineClient(), state, _build_args()) assert not hasattr(state, "dummy_engine_client") @@ -196,7 +196,7 @@ async def test_render_server_attaches_endpoint_plugins_with_no_engine_client( for route in app.routes ) - await _init_endpoint_plugins_state(None, app.state, args) + await init_endpoint_plugins_state(None, app.state, args) assert app.state.dummy_engine_client is None @@ -223,7 +223,7 @@ async def test_endpoint_plugin_end_to_end(monkeypatch: pytest.MonkeyPatch): ) fake_engine_client = _FakeEngineClient(rpc_result=["cfg-a", "cfg-b"]) - await _init_endpoint_plugins_state(fake_engine_client, app.state, args) + await init_endpoint_plugins_state(fake_engine_client, app.state, args) assert app.state.dummy_engine_client is fake_engine_client diff --git a/vllm/benchmarks/throughput.py b/vllm/benchmarks/throughput.py index e8a0ccb3a0b..5b757d43d1c 100644 --- a/vllm/benchmarks/throughput.py +++ b/vllm/benchmarks/throughput.py @@ -305,7 +305,7 @@ async def run_vllm_async( disable_detokenize: bool = False, warmup_requests: list[SampleRequest] | None = None, ) -> float: - from vllm.entrypoints.openai.api_server import ( + from vllm.entrypoints.launchers.api_server.entry import ( build_async_engine_client_from_engine_args, ) diff --git a/vllm/entrypoints/cli/launch.py b/vllm/entrypoints/cli/launch.py index 91d4bf094b7..38df03fac6b 100644 --- a/vllm/entrypoints/cli/launch.py +++ b/vllm/entrypoints/cli/launch.py @@ -3,18 +3,11 @@ import argparse import inspect -import signal import uvloop -from vllm import envs -from vllm.config import VllmConfig -from vllm.engine.arg_utils import AsyncEngineArgs from vllm.entrypoints.cli.types import CLISubcommand -from vllm.entrypoints.openai.api_server import ( - build_and_serve_renderer, - setup_server, -) +from vllm.entrypoints.launchers.render.entry import run_launch_fastapi from vllm.entrypoints.openai.cli_args import ( make_arg_parser, validate_parsed_serve_args, @@ -120,38 +113,3 @@ class LaunchSubcommand(CLISubcommand): def cmd_init() -> list[CLISubcommand]: return [LaunchSubcommand()] - - -async def run_launch_fastapi(args: argparse.Namespace) -> None: - """Run the online serving layer with FastAPI (no GPU inference).""" - - # Interrupt initialization if SIGTERM arrives before uvicorn installs - # its own signal handlers. Once uvicorn is running it replaces this. - def _interrupt_init(*_) -> None: - raise KeyboardInterrupt("terminated") - - signal.signal(signal.SIGTERM, _interrupt_init) - - # 1. Socket binding - listen_address, sock = setup_server(args, reuse_port=False) - - # 2. Build and serve the API server - engine_args = AsyncEngineArgs.from_cli_args(args) - model_config = engine_args.create_model_config() - - # Render servers preprocess data only — no inference, no quantized kernels. - # Clear quantization so VllmConfig skips quant dtype/capability validation. - model_config.quantization = None - - # Render servers never allocate KV cache; suppress the spurious CPU KV - # cache space warning from CpuPlatform.check_and_update_config. - envs.VLLM_CPU_KVCACHE_SPACE = 0 - - vllm_config = VllmConfig(model_config=model_config) - shutdown_task = await build_and_serve_renderer( - vllm_config, listen_address, sock, args - ) - try: - await shutdown_task - finally: - sock.close() diff --git a/vllm/entrypoints/cli/serve.py b/vllm/entrypoints/cli/serve.py index 9f29cd545f3..1d289712abe 100644 --- a/vllm/entrypoints/cli/serve.py +++ b/vllm/entrypoints/cli/serve.py @@ -10,11 +10,9 @@ import uvloop import vllm import vllm.envs as envs from vllm.entrypoints.cli.types import CLISubcommand -from vllm.entrypoints.openai.api_server import run_server, setup_server +from vllm.entrypoints.launchers.api_server.entry import run_server, setup_server from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args -from vllm.entrypoints.openai.dp_supervisor import ( - run_dp_supervisor, -) +from vllm.entrypoints.openai.dp_supervisor import run_dp_supervisor from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG from vllm.logger import init_logger from vllm.usage.usage_lib import UsageContext diff --git a/vllm/entrypoints/launchers/api_server/app_state.py b/vllm/entrypoints/launchers/api_server/app_state.py new file mode 100644 index 00000000000..c896db43920 --- /dev/null +++ b/vllm/entrypoints/launchers/api_server/app_state.py @@ -0,0 +1,149 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import warnings +from argparse import Namespace +from typing import cast + +from starlette.datastructures import State + +from vllm.engine.protocol import EngineClient +from vllm.entrypoints.chat_utils import load_chat_template +from vllm.entrypoints.openai.models.protocol import BaseModelPath +from vllm.entrypoints.openai.models.serving import OpenAIServingModels +from vllm.entrypoints.serve.tokenize.serving import ServingTokenization +from vllm.entrypoints.serve.utils.api_utils import process_lora_modules +from vllm.entrypoints.serve.utils.request_logger import RequestLogger +from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state +from vllm.renderers.online_derenderer import OnlineDerenderer +from vllm.renderers.online_renderer import OnlineRenderer +from vllm.tasks import FALLBACK_SUPPORTED_TASKS, POOLING_TASKS, SupportedTask + + +async def init_app_state( + engine_client: EngineClient, + state: State, + args: Namespace, + supported_tasks: tuple["SupportedTask", ...] | None = None, +) -> None: + vllm_config = engine_client.vllm_config + + if args.tool_call_parser is not None: + from vllm.parser.metrics import init_parser_metrics + + init_parser_metrics( + model_name=cast(str, vllm_config.model_config.served_model_name) + ) + + if supported_tasks is None: + warnings.warn( + "The 'supported_tasks' parameter was not provided to " + "init_app_state and will be required in a future version. " + "Please pass 'supported_tasks' explicitly.", + DeprecationWarning, + stacklevel=2, + ) + supported_tasks = FALLBACK_SUPPORTED_TASKS + + if args.served_model_name is not None: + served_model_names = args.served_model_name + else: + served_model_names = [args.model] + + if args.enable_log_requests: + request_logger = RequestLogger(max_log_len=args.max_log_len) + else: + request_logger = None + + base_model_paths = [ + BaseModelPath(name=name, model_path=args.model) for name in served_model_names + ] + + state.engine_client = engine_client + state.log_stats = not args.disable_log_stats + state.vllm_config = vllm_config + state.args = args + resolved_chat_template = load_chat_template(args.chat_template) + + # Merge default_mm_loras into the static lora_modules + default_mm_loras = ( + vllm_config.lora_config.default_mm_loras + if vllm_config.lora_config is not None + else {} + ) + lora_modules = process_lora_modules(args.lora_modules, default_mm_loras) + + state.openai_serving_models = OpenAIServingModels( + engine_client=engine_client, + base_model_paths=base_model_paths, + lora_modules=lora_modules, + ) + await state.openai_serving_models.init_static_loras() + + state.online_renderer = OnlineRenderer( + model_config=engine_client.model_config, + renderer=engine_client.renderer, + request_logger=request_logger, + chat_template=resolved_chat_template, + chat_template_content_format=args.chat_template_content_format, + trust_request_chat_template=args.trust_request_chat_template, + enable_auto_tools=args.enable_auto_tool_choice, + exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, + tool_parser=args.tool_call_parser, + reasoning_parser=args.structured_outputs_config.reasoning_parser, + default_chat_template_kwargs=args.default_chat_template_kwargs, + log_error_stack=args.log_error_stack, + ) + state.online_renderer.warmup() + + state.online_derenderer = OnlineDerenderer( + model_config=engine_client.model_config, + renderer=engine_client.renderer, + request_logger=request_logger, + chat_template=resolved_chat_template, + chat_template_content_format=args.chat_template_content_format, + trust_request_chat_template=args.trust_request_chat_template, + enable_auto_tools=args.enable_auto_tool_choice, + exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, + tool_parser=args.tool_call_parser, + reasoning_parser=args.structured_outputs_config.reasoning_parser, + default_chat_template_kwargs=args.default_chat_template_kwargs, + log_error_stack=args.log_error_stack, + ) + + state.serving_tokenization = ServingTokenization( + state.openai_serving_models, + state.online_renderer, + request_logger=request_logger, + chat_template=resolved_chat_template, + chat_template_content_format=args.chat_template_content_format, + default_chat_template_kwargs=args.default_chat_template_kwargs, + trust_request_chat_template=args.trust_request_chat_template, + ) + + if "generate" in supported_tasks: + from vllm.entrypoints.generate.api_router import init_generate_state + + await init_generate_state( + engine_client, state, args, request_logger, supported_tasks + ) + + from vllm.entrypoints.scale_out.factories import init_scale_out_state + + init_scale_out_state(state, args, engine_client, request_logger) + + if "transcription" in supported_tasks or "realtime" in supported_tasks: + from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state + + init_speech_to_text_state( + engine_client, state, args, request_logger, supported_tasks + ) + + if any(task in POOLING_TASKS for task in supported_tasks): + from vllm.entrypoints.pooling.factories import init_pooling_state + + init_pooling_state(engine_client, state, args, request_logger, supported_tasks) + + await init_endpoint_plugins_state(engine_client, state, args) + + state.enable_server_load_tracking = args.enable_server_load_tracking + state.server_load_metrics = 0 diff --git a/vllm/entrypoints/launchers/api_server/entry.py b/vllm/entrypoints/launchers/api_server/entry.py new file mode 100644 index 00000000000..dadc91ec6e5 --- /dev/null +++ b/vllm/entrypoints/launchers/api_server/entry.py @@ -0,0 +1,229 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import asyncio +import multiprocessing +import multiprocessing.forkserver as forkserver +import os +import signal +import socket +import tempfile +from argparse import Namespace +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any + +import vllm.envs as envs +from vllm.engine.arg_utils import AsyncEngineArgs +from vllm.engine.protocol import EngineClient +from vllm.logger import init_logger +from vllm.reasoning import ReasoningParserManager +from vllm.tool_parsers import ToolParserManager +from vllm.usage.usage_lib import UsageContext +from vllm.utils.system_utils import decorate_logs + +from ..app import build_app +from ..launcher import serve_http, setup_server +from ..utils.server_utils import get_uvicorn_log_config +from .app_state import init_app_state + +prometheus_multiproc_dir: tempfile.TemporaryDirectory + +# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765) +logger = init_logger("vllm.entrypoints.launchers.api_server.entry") + + +@asynccontextmanager +async def build_async_engine_client( + args: Namespace, + *, + usage_context: UsageContext = UsageContext.OPENAI_API_SERVER, + client_config: dict[str, Any] | None = None, +) -> AsyncIterator[EngineClient]: + if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver": + # The executor is expected to be mp. + # Pre-import heavy modules in the forkserver process + logger.debug("Setup forkserver with pre-imports") + multiprocessing.set_start_method("forkserver") + multiprocessing.set_forkserver_preload(["vllm.v1.engine.async_llm"]) + forkserver.ensure_running() + logger.debug("Forkserver setup complete!") + + # Context manager to handle engine_client lifecycle + # Ensures everything is shutdown and cleaned up on error/exit + engine_args = AsyncEngineArgs.from_cli_args(args) + if client_config: + engine_args._api_process_count = client_config.get("client_count", 1) + engine_args._api_process_rank = client_config.get("client_index", 0) + + async with build_async_engine_client_from_engine_args( + engine_args, + usage_context=usage_context, + client_config=client_config, + ) as engine: + yield engine + + +@asynccontextmanager +async def build_async_engine_client_from_engine_args( + engine_args: AsyncEngineArgs, + *, + usage_context: UsageContext = UsageContext.OPENAI_API_SERVER, + client_config: dict[str, Any] | None = None, +) -> AsyncIterator[EngineClient]: + """ + Create EngineClient, either: + - in-process using the AsyncLLMEngine Directly + - multiprocess using AsyncLLMEngine RPC + + Returns the Client or None if the creation failed. + """ + + # Create the EngineConfig (determines if we can use V1). + vllm_config = engine_args.create_engine_config(usage_context=usage_context) + + from vllm.v1.engine.async_llm import AsyncLLM + + async_llm: AsyncLLM | None = None + + # Don't mutate the input client_config + client_config = dict(client_config) if client_config else {} + client_count = client_config.pop("client_count", 1) + client_index = client_config.pop("client_index", 0) + + try: + async_llm = AsyncLLM.from_vllm_config( + vllm_config=vllm_config, + usage_context=usage_context, + enable_log_requests=engine_args.enable_log_requests, + aggregate_engine_logging=engine_args.aggregate_engine_logging, + disable_log_stats=engine_args.disable_log_stats, + client_addresses=client_config, + client_count=client_count, + client_index=client_index, + ) + + # Don't keep the dummy data in memory + assert async_llm is not None + await async_llm.reset_mm_cache() + + yield async_llm + finally: + if async_llm: + async_llm.shutdown(timeout=vllm_config.shutdown_timeout) + + +async def build_and_serve( + engine_client: EngineClient, + listen_address: str, + sock: socket.socket, + args: Namespace, + **uvicorn_kwargs, +) -> asyncio.Task: + """Build FastAPI app, initialize state, and start serving. + + Returns the shutdown task for the caller to await. + """ + + # Get uvicorn log config (from file or with endpoint filter) + log_config = get_uvicorn_log_config(args) + if log_config is not None: + uvicorn_kwargs["log_config"] = log_config + + supported_tasks = await engine_client.get_supported_tasks() + model_config = engine_client.model_config + + logger.info("Supported tasks: %s", supported_tasks) + app = build_app(args, supported_tasks, model_config) + await init_app_state(engine_client, app.state, args, supported_tasks) + + logger.info("Starting vLLM server on %s", listen_address) + + return await serve_http( + app, + sock=sock, + enable_ssl_refresh=args.enable_ssl_refresh, + host=args.host, + port=args.port, + log_level=args.uvicorn_log_level, + # NOTE: When the 'disable_uvicorn_access_log' value is True, + # no access log will be output. + access_log=not args.disable_uvicorn_access_log, + timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE, + ssl_keyfile=args.ssl_keyfile, + ssl_certfile=args.ssl_certfile, + ssl_ca_certs=args.ssl_ca_certs, + ssl_cert_reqs=args.ssl_cert_reqs, + ssl_ciphers=args.ssl_ciphers, + h11_max_incomplete_event_size=args.h11_max_incomplete_event_size, + h11_max_header_count=args.h11_max_header_count, + **uvicorn_kwargs, + ) + + +async def run_server(args, **uvicorn_kwargs) -> None: + """Run a single-worker API server.""" + + decorate_logs("APIServer", skip_if_decorated=True) + + # Interrupt initialization if SIGTERM arrives before uvicorn installs its + # own signal handlers. Once uvicorn is running it replaces this. + def _interrupt_init(*_) -> None: + raise KeyboardInterrupt("terminated") + + signal.signal(signal.SIGTERM, _interrupt_init) + + listen_address, sock = setup_server(args, reuse_port=False) + await run_server_worker(listen_address, sock, args, **uvicorn_kwargs) + + +async def run_server_worker( + listen_address, sock, args, client_config=None, **uvicorn_kwargs +) -> None: + """Run a single API server worker.""" + + if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3: + ToolParserManager.import_tool_parser(args.tool_parser_plugin) + + if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3: + ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin) + + async with build_async_engine_client( + args, + client_config=client_config, + ) as engine_client: + shutdown_task = await build_and_serve( + engine_client, listen_address, sock, args, **uvicorn_kwargs + ) + # NB: Await server shutdown only after the backend context is exited + try: + await shutdown_task + finally: + sock.close() + + +def main(): + import uvloop + + from vllm.entrypoints.openai.cli_args import ( + make_arg_parser, + validate_parsed_serve_args, + ) + from vllm.entrypoints.serve.utils.api_utils import cli_env_setup + from vllm.utils.argparse_utils import FlexibleArgumentParser + + # NOTE(simon): + # This section should be in sync with vllm/entrypoints/cli/main.py for CLI + # entrypoints. + cli_env_setup() + parser = FlexibleArgumentParser( + description="vLLM OpenAI-Compatible RESTful API server." + ) + parser = make_arg_parser(parser) + args = parser.parse_args() + validate_parsed_serve_args(args) + + uvloop.run(run_server(args)) + + +if __name__ == "__main__": + main() diff --git a/vllm/entrypoints/launchers/app.py b/vllm/entrypoints/launchers/app.py new file mode 100644 index 00000000000..ddea7078105 --- /dev/null +++ b/vllm/entrypoints/launchers/app.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import warnings +from argparse import Namespace + +from fastapi import FastAPI + +from vllm.config import ModelConfig +from vllm.entrypoints.serve.exception_handling.register import init_exception_handler +from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware +from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap +from vllm.plugins.endpoint_plugins.interface import attach_endpoint_plugins +from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask + +from .api_server.routers import register_api_routers +from .utils.server_utils import lifespan + + +def build_app( + args: Namespace, + supported_tasks: tuple["SupportedTask", ...] | None = None, + model_config: ModelConfig | None = None, +) -> FastAPI: + if supported_tasks is None: + warnings.warn( + "The 'supported_tasks' parameter was not provided to " + "build_app and will be required in a future version. " + "Defaulting to ('generate',).", + DeprecationWarning, + stacklevel=2, + ) + supported_tasks = FALLBACK_SUPPORTED_TASKS + + if args.disable_fastapi_docs: + app = FastAPI( + openapi_url=None, docs_url=None, redoc_url=None, lifespan=lifespan + ) + elif args.enable_offline_docs: + app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan) + else: + app = FastAPI(lifespan=lifespan) + app.state.args = args + app.root_path = args.root_path + + register_api_routers(args, app, supported_tasks, model_config) + + # Endpoint plugins are attached last so their routes are registered after all core + # routers. This runs even for the CPU only render server. A plugin eligible for + # the `render` task still gets its routes registered. It receives + # `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`). + attach_endpoint_plugins(app, supported_tasks) + + init_exception_handler(app) + init_entrypoints_middleware(args, app, supported_tasks) + app = sagemaker_standards_bootstrap(app) + return app diff --git a/vllm/entrypoints/launcher.py b/vllm/entrypoints/launchers/launcher.py similarity index 64% rename from vllm/entrypoints/launcher.py rename to vllm/entrypoints/launchers/launcher.py index 80b21051633..aeacb9f8c22 100644 --- a/vllm/entrypoints/launcher.py +++ b/vllm/entrypoints/launchers/launcher.py @@ -14,13 +14,23 @@ from fastapi import FastAPI from vllm import envs from vllm.engine.protocol import EngineClient -from vllm.entrypoints.serve.utils.constants import ( +from vllm.entrypoints.launchers.utils.ssl import SSLCertRefresher +from vllm.entrypoints.serve.utils.api_utils import ( + log_non_default_args, + log_version_and_model, +) +from vllm.logger import init_logger +from vllm.reasoning import ReasoningParserManager +from vllm.tool_parsers import ToolParserManager +from vllm.tracing import instrument +from vllm.utils.network_utils import find_process_using_port, is_valid_ipv6_address +from vllm.utils.system_utils import set_ulimit +from vllm.version import __version__ as VLLM_VERSION + +from .utils.constants import ( H11_MAX_HEADER_COUNT_DEFAULT, H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT, ) -from vllm.entrypoints.serve.utils.ssl import SSLCertRefresher -from vllm.logger import init_logger -from vllm.utils.network_utils import find_process_using_port logger = init_logger(__name__) @@ -190,3 +200,83 @@ def terminate_if_errored(server: uvicorn.Server, engine: EngineClient): engine_errored = engine.errored and not engine.is_running if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH and engine_errored: server.should_exit = True + + +def create_server_socket( + addr: tuple[str, int], + *, + reuse_port: bool, +) -> socket.socket: + family = socket.AF_INET + if is_valid_ipv6_address(addr[0]): + family = socket.AF_INET6 + + sock = socket.socket(family=family, type=socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if reuse_port: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind(addr) + + return sock + + +def create_server_unix_socket(path: str) -> socket.socket: + sock = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM) + sock.bind(path) + return sock + + +def validate_api_server_args(args): + valid_tool_parses = ToolParserManager.list_registered() + if args.enable_auto_tool_choice and args.tool_call_parser not in valid_tool_parses: + raise KeyError( + f"invalid tool call parser: {args.tool_call_parser} " + f"(chose from {{ {','.join(valid_tool_parses)} }})" + ) + + valid_reasoning_parsers = ReasoningParserManager.list_registered() + if ( + reasoning_parser := args.structured_outputs_config.reasoning_parser + ) and reasoning_parser not in valid_reasoning_parsers: + raise KeyError( + f"invalid reasoning parser: {reasoning_parser} " + f"(chose from {{ {','.join(valid_reasoning_parsers)} }})" + ) + + +@instrument(span_name="API server setup") +def setup_server(args, *, reuse_port: bool): + """Validate API server args and create the server socket.""" + + log_version_and_model(logger, VLLM_VERSION, args.model) + log_non_default_args(args) + + if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3: + ToolParserManager.import_tool_parser(args.tool_parser_plugin) + + if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3: + ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin) + + validate_api_server_args(args) + + # workaround to make sure that we bind the port before the engine is set up. + # This avoids race conditions with ray. + # see https://github.com/vllm-project/vllm/issues/8204 + if args.uds: + sock = create_server_unix_socket(args.uds) + else: + sock_addr = (args.host or "", args.port) + sock = create_server_socket(sock_addr, reuse_port=reuse_port) + + # workaround to avoid footguns where uvicorn drops requests with too + # many concurrent requests active + set_ulimit() + + if args.uds: + listen_address = f"unix:{args.uds}" + else: + addr, port = sock_addr + is_ssl = args.ssl_keyfile and args.ssl_certfile + host_part = f"[{addr}]" if is_valid_ipv6_address(addr) else addr or "0.0.0.0" + listen_address = f"http{'s' if is_ssl else ''}://{host_part}:{port}" + return listen_address, sock diff --git a/vllm/entrypoints/launchers/render/__init__.py b/vllm/entrypoints/launchers/render/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/vllm/entrypoints/launchers/render/app_state.py b/vllm/entrypoints/launchers/render/app_state.py new file mode 100644 index 00000000000..2b0b030b4ed --- /dev/null +++ b/vllm/entrypoints/launchers/render/app_state.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from argparse import Namespace + +from starlette.datastructures import State + +from vllm.config import VllmConfig +from vllm.entrypoints.chat_utils import load_chat_template +from vllm.entrypoints.openai.models.protocol import BaseModelPath +from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry +from vllm.entrypoints.scale_out.factories import init_render_state +from vllm.entrypoints.serve.tokenize.serving import ServingTokenization +from vllm.entrypoints.serve.utils.request_logger import RequestLogger +from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state +from vllm.renderers import renderer_from_config +from vllm.renderers.online_derenderer import OnlineDerenderer +from vllm.renderers.online_renderer import OnlineRenderer + + +async def init_render_app_state( + vllm_config: VllmConfig, + state: State, + args: Namespace, +) -> None: + """Initialise FastAPI app state for a CPU-only render server. + + Unlike :func:`init_app_state` this function does not require an + :class:`~vllm.engine.protocol.EngineClient`; it bootstraps the + preprocessing pipeline (renderer, input_processor) + directly from the :class:`~vllm.config.VllmConfig`. + """ + + served_model_names = args.served_model_name or [args.model] + model_registry = OpenAIModelRegistry( + model_config=vllm_config.model_config, + base_model_paths=[ + BaseModelPath(name=name, model_path=args.model) + for name in served_model_names + ], + ) + + if args.enable_log_requests: + request_logger = RequestLogger(max_log_len=args.max_log_len) + else: + request_logger = None + + renderer = renderer_from_config(vllm_config) + resolved_chat_template = load_chat_template(args.chat_template) + + state.online_renderer = OnlineRenderer( + model_config=vllm_config.model_config, + renderer=renderer, + request_logger=request_logger, + chat_template=resolved_chat_template, + chat_template_content_format=args.chat_template_content_format, + trust_request_chat_template=args.trust_request_chat_template, + enable_auto_tools=args.enable_auto_tool_choice, + exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, + tool_parser=args.tool_call_parser, + reasoning_parser=args.reasoning_parser, + default_chat_template_kwargs=args.default_chat_template_kwargs, + log_error_stack=args.log_error_stack, + ) + state.online_renderer.warmup() + + state.online_derenderer = OnlineDerenderer( + model_config=vllm_config.model_config, + renderer=renderer, + request_logger=request_logger, + chat_template=resolved_chat_template, + chat_template_content_format=args.chat_template_content_format, + trust_request_chat_template=args.trust_request_chat_template, + enable_auto_tools=args.enable_auto_tool_choice, + exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, + tool_parser=args.tool_call_parser, + reasoning_parser=args.reasoning_parser, + default_chat_template_kwargs=args.default_chat_template_kwargs, + log_error_stack=args.log_error_stack, + ) + + state.openai_serving_models = model_registry + state.serving_tokenization = ServingTokenization( + model_registry, + state.online_renderer, + request_logger=request_logger, + chat_template=resolved_chat_template, + chat_template_content_format=args.chat_template_content_format, + default_chat_template_kwargs=args.default_chat_template_kwargs, + trust_request_chat_template=args.trust_request_chat_template, + ) + + init_render_state(state, request_logger) + + state.vllm_config = vllm_config + # Disable stats logging — there is no engine to poll. + state.log_stats = False + state.engine_client = None + state.args = args + state.enable_server_load_tracking = False + state.server_load_metrics = 0 + + # No `EngineClient` exists for the render server, so plugins get `None` and + # must handle it themselves (see `EndpointPlugin.init_state`). + await init_endpoint_plugins_state(None, state, args) diff --git a/vllm/entrypoints/launchers/render/entry.py b/vllm/entrypoints/launchers/render/entry.py new file mode 100644 index 00000000000..2c7c4b54259 --- /dev/null +++ b/vllm/entrypoints/launchers/render/entry.py @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import argparse +import asyncio +import signal +import socket +from argparse import Namespace + +from vllm import AsyncEngineArgs, envs +from vllm.config import VllmConfig +from vllm.logger import init_logger + +from ..app import build_app +from ..launcher import serve_http, setup_server +from ..utils.server_utils import get_uvicorn_log_config +from .app_state import init_render_app_state + +logger = init_logger("vllm.entrypoints.launchers.render.entry") + + +async def build_and_serve_renderer( + vllm_config: VllmConfig, + listen_address: str, + sock: socket.socket, + args: Namespace, + **uvicorn_kwargs, +) -> asyncio.Task: + """Build FastAPI app for a CPU-only render server, initialize state, and + start serving. + + Returns the shutdown task for the caller to await. + """ + + # Get uvicorn log config (from file or with endpoint filter) + log_config = get_uvicorn_log_config(args) + if log_config is not None: + uvicorn_kwargs["log_config"] = log_config + + app = build_app(args, ("render",)) + await init_render_app_state(vllm_config, app.state, args) + + logger.info("Starting vLLM server on %s", listen_address) + + return await serve_http( + app, + sock=sock, + enable_ssl_refresh=args.enable_ssl_refresh, + host=args.host, + port=args.port, + log_level=args.uvicorn_log_level, + # NOTE: When the 'disable_uvicorn_access_log' value is True, + # no access log will be output. + access_log=not args.disable_uvicorn_access_log, + timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE, + ssl_keyfile=args.ssl_keyfile, + ssl_certfile=args.ssl_certfile, + ssl_ca_certs=args.ssl_ca_certs, + ssl_cert_reqs=args.ssl_cert_reqs, + ssl_ciphers=args.ssl_ciphers, + h11_max_incomplete_event_size=args.h11_max_incomplete_event_size, + h11_max_header_count=args.h11_max_header_count, + **uvicorn_kwargs, + ) + + +async def run_launch_fastapi(args: argparse.Namespace) -> None: + """Run the online serving layer with FastAPI (no GPU inference).""" + + # Interrupt initialization if SIGTERM arrives before uvicorn installs + # its own signal handlers. Once uvicorn is running it replaces this. + def _interrupt_init(*_) -> None: + raise KeyboardInterrupt("terminated") + + signal.signal(signal.SIGTERM, _interrupt_init) + + # 1. Socket binding + listen_address, sock = setup_server(args, reuse_port=False) + + # 2. Build and serve the API server + engine_args = AsyncEngineArgs.from_cli_args(args) + model_config = engine_args.create_model_config() + + # Render servers preprocess data only — no inference, no quantized kernels. + # Clear quantization so VllmConfig skips quant dtype/capability validation. + model_config.quantization = None + + # Render servers never allocate KV cache; suppress the spurious CPU KV + # cache space warning from CpuPlatform.check_and_update_config. + envs.VLLM_CPU_KVCACHE_SPACE = 0 + + vllm_config = VllmConfig(model_config=model_config) + shutdown_task = await build_and_serve_renderer( + vllm_config, listen_address, sock, args + ) + try: + await shutdown_task + finally: + sock.close() + + +if __name__ == "__main__": + import uvloop + + from vllm.entrypoints.openai.cli_args import ( + make_arg_parser, + validate_parsed_serve_args, + ) + from vllm.entrypoints.serve.utils.api_utils import cli_env_setup + from vllm.utils.argparse_utils import FlexibleArgumentParser + + cli_env_setup() + parser = FlexibleArgumentParser( + description="Starts a GPU-less rendering server " + "for preprocessing and postprocessing only" + ) + parser = make_arg_parser(parser) + args = parser.parse_args() + validate_parsed_serve_args(args) + + uvloop.run(run_launch_fastapi(args)) diff --git a/vllm/entrypoints/launchers/utils/__init__.py b/vllm/entrypoints/launchers/utils/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/vllm/entrypoints/serve/utils/constants.py b/vllm/entrypoints/launchers/utils/constants.py similarity index 80% rename from vllm/entrypoints/serve/utils/constants.py rename to vllm/entrypoints/launchers/utils/constants.py index 5726ee0735d..99750c31749 100644 --- a/vllm/entrypoints/serve/utils/constants.py +++ b/vllm/entrypoints/launchers/utils/constants.py @@ -1,12 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Shared constants for vLLM entrypoints. -""" # HTTP header limits for h11 parser # These constants help mitigate header abuse attacks H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT = 4194304 # 4 MB H11_MAX_HEADER_COUNT_DEFAULT = 256 - -MCP_PREFIX = "mcp_" diff --git a/vllm/entrypoints/serve/utils/server_utils.py b/vllm/entrypoints/launchers/utils/server_utils.py similarity index 98% rename from vllm/entrypoints/serve/utils/server_utils.py rename to vllm/entrypoints/launchers/utils/server_utils.py index 97910d019ec..08258267238 100644 --- a/vllm/entrypoints/serve/utils/server_utils.py +++ b/vllm/entrypoints/launchers/utils/server_utils.py @@ -12,7 +12,7 @@ from vllm.engine.protocol import EngineClient from vllm.logger import init_logger from vllm.utils.gc_utils import freeze_gc_heap -logger = init_logger("vllm.entrypoints.openai.server_utils") +logger = init_logger(__name__) def load_log_config(log_config_file: str | None) -> dict | None: diff --git a/vllm/entrypoints/serve/utils/ssl.py b/vllm/entrypoints/launchers/utils/ssl.py similarity index 100% rename from vllm/entrypoints/serve/utils/ssl.py rename to vllm/entrypoints/launchers/utils/ssl.py diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index c541f03f445..a8e8b58c187 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -1,670 +1,59 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import asyncio -import multiprocessing -import multiprocessing.forkserver as forkserver -import os -import signal -import socket -import tempfile import warnings -from argparse import Namespace -from collections.abc import AsyncIterator -from contextlib import asynccontextmanager -from typing import Any, cast -import uvloop -from fastapi import FastAPI -from starlette.datastructures import State - -import vllm.envs as envs -from vllm.config import ModelConfig, VllmConfig -from vllm.engine.arg_utils import AsyncEngineArgs -from vllm.engine.protocol import EngineClient -from vllm.entrypoints.chat_utils import load_chat_template -from vllm.entrypoints.launcher import serve_http +from vllm.entrypoints.launchers.api_server.app_state import init_app_state +from vllm.entrypoints.launchers.api_server.entry import ( + build_and_serve, + build_async_engine_client, + build_async_engine_client_from_engine_args, + run_server, + run_server_worker, +) from vllm.entrypoints.launchers.api_server.routers import register_api_routers -from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args -from vllm.entrypoints.openai.models.protocol import BaseModelPath -from vllm.entrypoints.openai.models.serving import OpenAIServingModels -from vllm.entrypoints.serve.exception_handling.register import init_exception_handler -from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware -from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap -from vllm.entrypoints.serve.tokenize.serving import ServingTokenization -from vllm.entrypoints.serve.utils.api_utils import ( - cli_env_setup, - log_non_default_args, - log_version_and_model, - process_lora_modules, +from vllm.entrypoints.launchers.app import build_app +from vllm.entrypoints.launchers.launcher import ( + create_server_socket, + create_server_unix_socket, + setup_server, + validate_api_server_args, ) -from vllm.entrypoints.serve.utils.request_logger import RequestLogger -from vllm.entrypoints.serve.utils.server_utils import ( - get_uvicorn_log_config, - lifespan, +from vllm.entrypoints.launchers.render.app_state import init_render_app_state +from vllm.entrypoints.launchers.render.entry import build_and_serve_renderer + +warnings.warn( + "`vllm.entrypoints.openai.api_server` is deprecated and will likely be" + "unsupported in a future version. Use the corresponding function from " + "`vllm.entrypoints.launchers` instead.", + DeprecationWarning, + stacklevel=1, ) -from vllm.logger import init_logger -from vllm.reasoning import ReasoningParserManager -from vllm.renderers.online_derenderer import OnlineDerenderer -from vllm.renderers.online_renderer import OnlineRenderer -from vllm.tasks import POOLING_TASKS, SupportedTask -from vllm.tool_parsers import ToolParserManager -from vllm.tracing import instrument -from vllm.usage.usage_lib import UsageContext -from vllm.utils.argparse_utils import FlexibleArgumentParser -from vllm.utils.network_utils import is_valid_ipv6_address -from vllm.utils.system_utils import decorate_logs, set_ulimit -from vllm.version import __version__ as VLLM_VERSION -prometheus_multiproc_dir: tempfile.TemporaryDirectory - -# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765) -logger = init_logger("vllm.entrypoints.openai.api_server") - -_FALLBACK_SUPPORTED_TASKS: tuple[SupportedTask, ...] = ("generate",) - - -def _attach_endpoint_plugins( - app: FastAPI, supported_tasks: tuple["SupportedTask", ...] -) -> None: - """Phase A of endpoint plugin wiring: discover, gate and attach routes. - - Attached last after all core routers. This is so endpoint plugin routes can - shadow core routes with the same path (see `EndpointPlugin.attach_router` - docstring). No-ops when no plugins are discovered/allowlisted. - """ - from vllm.plugins import load_endpoint_plugins - - endpoint_plugins = load_endpoint_plugins(supported_tasks) - for plugin in endpoint_plugins: - plugin.attach_router(app) - app.state.endpoint_plugins = endpoint_plugins - - -async def _init_endpoint_plugins_state( - engine_client: EngineClient | None, state: State, args: Namespace -) -> None: - """Phase B of endpoint plugin wiring: initialize per app plugin state. - - `state.endpoint_plugins` is set by `_attach_endpoint_plugins` (Phase A) - in `build_app`. Some `init_app_state` callers (e.g. `run_batch.py`) - build their own bare `State` without going through `build_app`. As a result - `endpoint_plugins` may be absent and are treated that the same as "none attached". - - `engine_client` is `None` for the CPU only render server which has no - engine (see `init_render_app_state`). Plugins must handle a `None` - `engine_client` themselves (see `EndpointPlugin.init_state`). - """ - for plugin in getattr(state, "endpoint_plugins", []): - await plugin.init_state(engine_client, state, args) - - -@asynccontextmanager -async def build_async_engine_client( - args: Namespace, - *, - usage_context: UsageContext = UsageContext.OPENAI_API_SERVER, - client_config: dict[str, Any] | None = None, -) -> AsyncIterator[EngineClient]: - if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver": - # The executor is expected to be mp. - # Pre-import heavy modules in the forkserver process - logger.debug("Setup forkserver with pre-imports") - multiprocessing.set_start_method("forkserver") - multiprocessing.set_forkserver_preload(["vllm.v1.engine.async_llm"]) - forkserver.ensure_running() - logger.debug("Forkserver setup complete!") - - # Context manager to handle engine_client lifecycle - # Ensures everything is shutdown and cleaned up on error/exit - engine_args = AsyncEngineArgs.from_cli_args(args) - if client_config: - engine_args._api_process_count = client_config.get("client_count", 1) - engine_args._api_process_rank = client_config.get("client_index", 0) - - async with build_async_engine_client_from_engine_args( - engine_args, - usage_context=usage_context, - client_config=client_config, - ) as engine: - yield engine - - -@asynccontextmanager -async def build_async_engine_client_from_engine_args( - engine_args: AsyncEngineArgs, - *, - usage_context: UsageContext = UsageContext.OPENAI_API_SERVER, - client_config: dict[str, Any] | None = None, -) -> AsyncIterator[EngineClient]: - """ - Create EngineClient, either: - - in-process using the AsyncLLMEngine Directly - - multiprocess using AsyncLLMEngine RPC - - Returns the Client or None if the creation failed. - """ - - # Create the EngineConfig (determines if we can use V1). - vllm_config = engine_args.create_engine_config(usage_context=usage_context) - - from vllm.v1.engine.async_llm import AsyncLLM - - async_llm: AsyncLLM | None = None - - # Don't mutate the input client_config - client_config = dict(client_config) if client_config else {} - client_count = client_config.pop("client_count", 1) - client_index = client_config.pop("client_index", 0) - - try: - async_llm = AsyncLLM.from_vllm_config( - vllm_config=vllm_config, - usage_context=usage_context, - enable_log_requests=engine_args.enable_log_requests, - aggregate_engine_logging=engine_args.aggregate_engine_logging, - disable_log_stats=engine_args.disable_log_stats, - client_addresses=client_config, - client_count=client_count, - client_index=client_index, - ) - - # Don't keep the dummy data in memory - assert async_llm is not None - await async_llm.reset_mm_cache() - - yield async_llm - finally: - if async_llm: - async_llm.shutdown(timeout=vllm_config.shutdown_timeout) - - -def build_app( - args: Namespace, - supported_tasks: tuple["SupportedTask", ...] | None = None, - model_config: ModelConfig | None = None, -) -> FastAPI: - if supported_tasks is None: - warnings.warn( - "The 'supported_tasks' parameter was not provided to " - "build_app and will be required in a future version. " - "Defaulting to ('generate',).", - DeprecationWarning, - stacklevel=2, - ) - supported_tasks = _FALLBACK_SUPPORTED_TASKS - - if args.disable_fastapi_docs: - app = FastAPI( - openapi_url=None, docs_url=None, redoc_url=None, lifespan=lifespan - ) - elif args.enable_offline_docs: - app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan) - else: - app = FastAPI(lifespan=lifespan) - app.state.args = args - app.root_path = args.root_path - - register_api_routers(args, app, supported_tasks, model_config) - - # Endpoint plugins are attached last so their routes are registered after all core - # routers. This runs even for the CPU only render server. A plugin eligible for - # the `render` task still gets its routes registered. It receives - # `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`). - _attach_endpoint_plugins(app, supported_tasks) - - init_exception_handler(app) - init_entrypoints_middleware(args, app, supported_tasks) - app = sagemaker_standards_bootstrap(app) - return app - - -async def init_app_state( - engine_client: EngineClient, - state: State, - args: Namespace, - supported_tasks: tuple["SupportedTask", ...] | None = None, -) -> None: - vllm_config = engine_client.vllm_config - - if args.tool_call_parser is not None: - from vllm.parser.metrics import init_parser_metrics - - init_parser_metrics( - model_name=cast(str, vllm_config.model_config.served_model_name) - ) - - if supported_tasks is None: - warnings.warn( - "The 'supported_tasks' parameter was not provided to " - "init_app_state and will be required in a future version. " - "Please pass 'supported_tasks' explicitly.", - DeprecationWarning, - stacklevel=2, - ) - supported_tasks = _FALLBACK_SUPPORTED_TASKS - - if args.served_model_name is not None: - served_model_names = args.served_model_name - else: - served_model_names = [args.model] - - if args.enable_log_requests: - request_logger = RequestLogger(max_log_len=args.max_log_len) - else: - request_logger = None - - base_model_paths = [ - BaseModelPath(name=name, model_path=args.model) for name in served_model_names - ] - - state.engine_client = engine_client - state.log_stats = not args.disable_log_stats - state.vllm_config = vllm_config - state.args = args - resolved_chat_template = load_chat_template(args.chat_template) - - # Merge default_mm_loras into the static lora_modules - default_mm_loras = ( - vllm_config.lora_config.default_mm_loras - if vllm_config.lora_config is not None - else {} - ) - lora_modules = process_lora_modules(args.lora_modules, default_mm_loras) - - state.openai_serving_models = OpenAIServingModels( - engine_client=engine_client, - base_model_paths=base_model_paths, - lora_modules=lora_modules, - ) - await state.openai_serving_models.init_static_loras() - - state.online_renderer = OnlineRenderer( - model_config=engine_client.model_config, - renderer=engine_client.renderer, - request_logger=request_logger, - chat_template=resolved_chat_template, - chat_template_content_format=args.chat_template_content_format, - trust_request_chat_template=args.trust_request_chat_template, - enable_auto_tools=args.enable_auto_tool_choice, - exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, - tool_parser=args.tool_call_parser, - reasoning_parser=args.structured_outputs_config.reasoning_parser, - default_chat_template_kwargs=args.default_chat_template_kwargs, - log_error_stack=args.log_error_stack, - ) - state.online_renderer.warmup() - - state.online_derenderer = OnlineDerenderer( - model_config=engine_client.model_config, - renderer=engine_client.renderer, - request_logger=request_logger, - chat_template=resolved_chat_template, - chat_template_content_format=args.chat_template_content_format, - trust_request_chat_template=args.trust_request_chat_template, - enable_auto_tools=args.enable_auto_tool_choice, - exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, - tool_parser=args.tool_call_parser, - reasoning_parser=args.structured_outputs_config.reasoning_parser, - default_chat_template_kwargs=args.default_chat_template_kwargs, - log_error_stack=args.log_error_stack, - ) - - state.serving_tokenization = ServingTokenization( - state.openai_serving_models, - state.online_renderer, - request_logger=request_logger, - chat_template=resolved_chat_template, - chat_template_content_format=args.chat_template_content_format, - default_chat_template_kwargs=args.default_chat_template_kwargs, - trust_request_chat_template=args.trust_request_chat_template, - ) - - if "generate" in supported_tasks: - from vllm.entrypoints.generate.api_router import init_generate_state - - await init_generate_state( - engine_client, state, args, request_logger, supported_tasks - ) - - from vllm.entrypoints.scale_out.factories import init_scale_out_state - - init_scale_out_state(state, args, engine_client, request_logger) - - if "transcription" in supported_tasks or "realtime" in supported_tasks: - from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state - - init_speech_to_text_state( - engine_client, state, args, request_logger, supported_tasks - ) - - if any(task in POOLING_TASKS for task in supported_tasks): - from vllm.entrypoints.pooling.factories import init_pooling_state - - init_pooling_state(engine_client, state, args, request_logger, supported_tasks) - - await _init_endpoint_plugins_state(engine_client, state, args) - - state.enable_server_load_tracking = args.enable_server_load_tracking - state.server_load_metrics = 0 - - -async def init_render_app_state( - vllm_config: VllmConfig, - state: State, - args: Namespace, -) -> None: - """Initialise FastAPI app state for a CPU-only render server. - - Unlike :func:`init_app_state` this function does not require an - :class:`~vllm.engine.protocol.EngineClient`; it bootstraps the - preprocessing pipeline (renderer, input_processor) - directly from the :class:`~vllm.config.VllmConfig`. - """ - from vllm.entrypoints.chat_utils import load_chat_template - from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry - from vllm.renderers import renderer_from_config - from vllm.renderers.online_renderer import OnlineRenderer - - served_model_names = args.served_model_name or [args.model] - model_registry = OpenAIModelRegistry( - model_config=vllm_config.model_config, - base_model_paths=[ - BaseModelPath(name=name, model_path=args.model) - for name in served_model_names - ], - ) - - if args.enable_log_requests: - request_logger = RequestLogger(max_log_len=args.max_log_len) - else: - request_logger = None - - renderer = renderer_from_config(vllm_config) - resolved_chat_template = load_chat_template(args.chat_template) - - state.online_renderer = OnlineRenderer( - model_config=vllm_config.model_config, - renderer=renderer, - request_logger=request_logger, - chat_template=resolved_chat_template, - chat_template_content_format=args.chat_template_content_format, - trust_request_chat_template=args.trust_request_chat_template, - enable_auto_tools=args.enable_auto_tool_choice, - exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, - tool_parser=args.tool_call_parser, - reasoning_parser=args.reasoning_parser, - default_chat_template_kwargs=args.default_chat_template_kwargs, - log_error_stack=args.log_error_stack, - ) - state.online_renderer.warmup() - - state.online_derenderer = OnlineDerenderer( - model_config=vllm_config.model_config, - renderer=renderer, - request_logger=request_logger, - chat_template=resolved_chat_template, - chat_template_content_format=args.chat_template_content_format, - trust_request_chat_template=args.trust_request_chat_template, - enable_auto_tools=args.enable_auto_tool_choice, - exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, - tool_parser=args.tool_call_parser, - reasoning_parser=args.reasoning_parser, - default_chat_template_kwargs=args.default_chat_template_kwargs, - log_error_stack=args.log_error_stack, - ) - - state.openai_serving_models = model_registry - state.serving_tokenization = ServingTokenization( - model_registry, - state.online_renderer, - request_logger=request_logger, - chat_template=resolved_chat_template, - chat_template_content_format=args.chat_template_content_format, - default_chat_template_kwargs=args.default_chat_template_kwargs, - trust_request_chat_template=args.trust_request_chat_template, - ) - - from vllm.entrypoints.scale_out.factories import init_render_state - - init_render_state(state, request_logger) - - state.vllm_config = vllm_config - # Disable stats logging — there is no engine to poll. - state.log_stats = False - state.engine_client = None - state.args = args - state.enable_server_load_tracking = False - state.server_load_metrics = 0 - - # No `EngineClient` exists for the render server, so plugins get `None` and - # must handle it themselves (see `EndpointPlugin.init_state`). - await _init_endpoint_plugins_state(None, state, args) - - -def create_server_socket( - addr: tuple[str, int], - *, - reuse_port: bool, -) -> socket.socket: - family = socket.AF_INET - if is_valid_ipv6_address(addr[0]): - family = socket.AF_INET6 - - sock = socket.socket(family=family, type=socket.SOCK_STREAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if reuse_port: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - sock.bind(addr) - - return sock - - -def create_server_unix_socket(path: str) -> socket.socket: - sock = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM) - sock.bind(path) - return sock - - -def validate_api_server_args(args): - valid_tool_parses = ToolParserManager.list_registered() - if args.enable_auto_tool_choice and args.tool_call_parser not in valid_tool_parses: - raise KeyError( - f"invalid tool call parser: {args.tool_call_parser} " - f"(chose from {{ {','.join(valid_tool_parses)} }})" - ) - - valid_reasoning_parsers = ReasoningParserManager.list_registered() - if ( - reasoning_parser := args.structured_outputs_config.reasoning_parser - ) and reasoning_parser not in valid_reasoning_parsers: - raise KeyError( - f"invalid reasoning parser: {reasoning_parser} " - f"(chose from {{ {','.join(valid_reasoning_parsers)} }})" - ) - - -@instrument(span_name="API server setup") -def setup_server(args, *, reuse_port: bool): - """Validate API server args and create the server socket.""" - - log_version_and_model(logger, VLLM_VERSION, args.model) - log_non_default_args(args) - - if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3: - ToolParserManager.import_tool_parser(args.tool_parser_plugin) - - if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3: - ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin) - - validate_api_server_args(args) - - # workaround to make sure that we bind the port before the engine is set up. - # This avoids race conditions with ray. - # see https://github.com/vllm-project/vllm/issues/8204 - if args.uds: - sock = create_server_unix_socket(args.uds) - else: - sock_addr = (args.host or "", args.port) - sock = create_server_socket(sock_addr, reuse_port=reuse_port) - - # workaround to avoid footguns where uvicorn drops requests with too - # many concurrent requests active - set_ulimit() - - if args.uds: - listen_address = f"unix:{args.uds}" - else: - addr, port = sock_addr - is_ssl = args.ssl_keyfile and args.ssl_certfile - host_part = f"[{addr}]" if is_valid_ipv6_address(addr) else addr or "0.0.0.0" - listen_address = f"http{'s' if is_ssl else ''}://{host_part}:{port}" - return listen_address, sock - - -async def build_and_serve( - engine_client: EngineClient, - listen_address: str, - sock: socket.socket, - args: Namespace, - **uvicorn_kwargs, -) -> asyncio.Task: - """Build FastAPI app, initialize state, and start serving. - - Returns the shutdown task for the caller to await. - """ - - # Get uvicorn log config (from file or with endpoint filter) - log_config = get_uvicorn_log_config(args) - if log_config is not None: - uvicorn_kwargs["log_config"] = log_config - - supported_tasks = await engine_client.get_supported_tasks() - model_config = engine_client.model_config - - logger.info("Supported tasks: %s", supported_tasks) - app = build_app(args, supported_tasks, model_config) - await init_app_state(engine_client, app.state, args, supported_tasks) - - logger.info("Starting vLLM server on %s", listen_address) - - return await serve_http( - app, - sock=sock, - enable_ssl_refresh=args.enable_ssl_refresh, - host=args.host, - port=args.port, - log_level=args.uvicorn_log_level, - # NOTE: When the 'disable_uvicorn_access_log' value is True, - # no access log will be output. - access_log=not args.disable_uvicorn_access_log, - timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE, - ssl_keyfile=args.ssl_keyfile, - ssl_certfile=args.ssl_certfile, - ssl_ca_certs=args.ssl_ca_certs, - ssl_cert_reqs=args.ssl_cert_reqs, - ssl_ciphers=args.ssl_ciphers, - h11_max_incomplete_event_size=args.h11_max_incomplete_event_size, - h11_max_header_count=args.h11_max_header_count, - **uvicorn_kwargs, - ) - - -async def build_and_serve_renderer( - vllm_config: VllmConfig, - listen_address: str, - sock: socket.socket, - args: Namespace, - **uvicorn_kwargs, -) -> asyncio.Task: - """Build FastAPI app for a CPU-only render server, initialize state, and - start serving. - - Returns the shutdown task for the caller to await. - """ - - # Get uvicorn log config (from file or with endpoint filter) - log_config = get_uvicorn_log_config(args) - if log_config is not None: - uvicorn_kwargs["log_config"] = log_config - - app = build_app(args, ("render",)) - await init_render_app_state(vllm_config, app.state, args) - - logger.info("Starting vLLM server on %s", listen_address) - - return await serve_http( - app, - sock=sock, - enable_ssl_refresh=args.enable_ssl_refresh, - host=args.host, - port=args.port, - log_level=args.uvicorn_log_level, - # NOTE: When the 'disable_uvicorn_access_log' value is True, - # no access log will be output. - access_log=not args.disable_uvicorn_access_log, - timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE, - ssl_keyfile=args.ssl_keyfile, - ssl_certfile=args.ssl_certfile, - ssl_ca_certs=args.ssl_ca_certs, - ssl_cert_reqs=args.ssl_cert_reqs, - ssl_ciphers=args.ssl_ciphers, - h11_max_incomplete_event_size=args.h11_max_incomplete_event_size, - h11_max_header_count=args.h11_max_header_count, - **uvicorn_kwargs, - ) - - -async def run_server(args, **uvicorn_kwargs) -> None: - """Run a single-worker API server.""" - - decorate_logs("APIServer", skip_if_decorated=True) - - # Interrupt initialization if SIGTERM arrives before uvicorn installs its - # own signal handlers. Once uvicorn is running it replaces this. - def _interrupt_init(*_) -> None: - raise KeyboardInterrupt("terminated") - - signal.signal(signal.SIGTERM, _interrupt_init) - - listen_address, sock = setup_server(args, reuse_port=False) - await run_server_worker(listen_address, sock, args, **uvicorn_kwargs) - - -async def run_server_worker( - listen_address, sock, args, client_config=None, **uvicorn_kwargs -) -> None: - """Run a single API server worker.""" - - if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3: - ToolParserManager.import_tool_parser(args.tool_parser_plugin) - - if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3: - ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin) - - async with build_async_engine_client( - args, - client_config=client_config, - ) as engine_client: - shutdown_task = await build_and_serve( - engine_client, listen_address, sock, args, **uvicorn_kwargs - ) - # NB: Await server shutdown only after the backend context is exited - try: - await shutdown_task - finally: - sock.close() +__all__ = [ + "build_async_engine_client", + "build_async_engine_client_from_engine_args", + "build_app", + "init_app_state", + "init_render_app_state", + "create_server_socket", + "create_server_unix_socket", + "validate_api_server_args", + "setup_server", + "build_and_serve", + "build_and_serve_renderer", + "run_server", + "run_server_worker", + "register_api_routers", +] if __name__ == "__main__": - # NOTE(simon): - # This section should be in sync with vllm/entrypoints/cli/main.py for CLI - # entrypoints. - cli_env_setup() - parser = FlexibleArgumentParser( - description="vLLM OpenAI-Compatible RESTful API server." + warnings.warn( + "The `python -m vllm.entrypoints.openai.api_server` command is deprecated " + "and may be removed in a future release. Please use `vllm server` instead.", + DeprecationWarning, + stacklevel=1, ) - parser = make_arg_parser(parser) - args = parser.parse_args() - validate_parsed_serve_args(args) + from vllm.entrypoints.launchers.api_server.entry import main - uvloop.run(run_server(args)) + main() diff --git a/vllm/entrypoints/openai/cli_args.py b/vllm/entrypoints/openai/cli_args.py index e9284c5dd69..49730d34b81 100644 --- a/vllm/entrypoints/openai/cli_args.py +++ b/vllm/entrypoints/openai/cli_args.py @@ -20,11 +20,11 @@ from vllm.entrypoints.chat_utils import ( ChatTemplateContentFormatOption, validate_chat_template, ) -from vllm.entrypoints.openai.models.protocol import LoRAModulePath -from vllm.entrypoints.serve.utils.constants import ( +from vllm.entrypoints.launchers.utils.constants import ( H11_MAX_HEADER_COUNT_DEFAULT, H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT, ) +from vllm.entrypoints.openai.models.protocol import LoRAModulePath from vllm.tool_parsers import ToolParserManager from vllm.utils.argparse_utils import FlexibleArgumentParser @@ -444,6 +444,6 @@ def validate_parsed_serve_args(args: argparse.Namespace): def create_parser_for_docs() -> FlexibleArgumentParser: parser_for_docs = FlexibleArgumentParser( - prog="-m vllm.entrypoints.openai.api_server" + prog="-m vllm.entrypoints.launchers.api_server.entry" ) return make_arg_parser(parser_for_docs) diff --git a/vllm/entrypoints/openai/dp_supervisor.py b/vllm/entrypoints/openai/dp_supervisor.py index 4076e1ed324..c5efe916c02 100644 --- a/vllm/entrypoints/openai/dp_supervisor.py +++ b/vllm/entrypoints/openai/dp_supervisor.py @@ -23,7 +23,7 @@ import uvloop from fastapi import FastAPI, Response import vllm.envs as envs -from vllm.entrypoints.launcher import NoSignalServer +from vllm.entrypoints.launchers.launcher import NoSignalServer from vllm.logger import init_logger from vllm.utils.system_utils import ( decorate_logs, @@ -236,7 +236,7 @@ def _build_dp_supervisor_app(supervisor: DPSupervisor) -> FastAPI: def _run_python_vllm_dp_server(child_args: argparse.Namespace) -> None: - from vllm.entrypoints.openai.api_server import run_server + from vllm.entrypoints.launchers.api_server.entry import run_server uvloop.run(run_server(child_args)) diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py index 9bc3161f88d..72b3ea92b8b 100644 --- a/vllm/entrypoints/openai/responses/context.py +++ b/vllm/entrypoints/openai/responses/context.py @@ -39,7 +39,6 @@ from vllm.entrypoints.openai.responses.utils import ( build_response_output_items, construct_tool_dicts, ) -from vllm.entrypoints.serve.utils.constants import MCP_PREFIX from vllm.outputs import RequestOutput from vllm.parser.abstract_parser import Parser from vllm.tokenizers import TokenizerLike @@ -58,6 +57,7 @@ _TOOL_NAME_TO_TYPE_MAP = { "python": "code_interpreter", "container": "container", } +MCP_PREFIX = "mcp_" def _map_tool_name_to_tool_type(tool_name: str) -> str: diff --git a/vllm/entrypoints/openai/run_batch.py b/vllm/entrypoints/openai/run_batch.py index e9b7c582d6d..f4b7b35bf0e 100644 --- a/vllm/entrypoints/openai/run_batch.py +++ b/vllm/entrypoints/openai/run_batch.py @@ -31,7 +31,7 @@ from vllm.config import config from vllm.connections import global_http_connection from vllm.engine.arg_utils import AsyncEngineArgs from vllm.engine.protocol import EngineClient -from vllm.entrypoints.openai.api_server import init_app_state +from vllm.entrypoints.launchers.api_server.entry import init_app_state from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ChatCompletionResponse, @@ -836,8 +836,7 @@ async def run_batch( error_msg=f"URL {request.url} was used. " "Supported endpoints: /v1/chat/completions, /v1/embeddings," " /v1/audio/transcriptions, /v1/audio/translations, /score, " - " /rerank. See vllm/entrypoints/openai/api_server.py " - "for supported score/rerank versions.", + " /rerank.", ) ) @@ -848,7 +847,7 @@ async def run_batch( async def main(args: Namespace): - from vllm.entrypoints.openai.api_server import build_async_engine_client + from vllm.entrypoints.launchers.api_server.entry import build_async_engine_client from vllm.usage.usage_lib import UsageContext validate_run_batch_args(args) diff --git a/vllm/entrypoints/serve/exception_handling/handlers/vllm_error.py b/vllm/entrypoints/serve/exception_handling/handlers/vllm_error.py index 8ac9756f3d7..213c451167c 100644 --- a/vllm/entrypoints/serve/exception_handling/handlers/vllm_error.py +++ b/vllm/entrypoints/serve/exception_handling/handlers/vllm_error.py @@ -4,7 +4,7 @@ from fastapi import Request from starlette.responses import JSONResponse -from vllm.entrypoints.launcher import terminate_if_errored +from vllm.entrypoints.launchers.launcher import terminate_if_errored from vllm.entrypoints.openai.engine.protocol import GenerationError from vllm.exceptions import VLLMError from vllm.logger import init_logger diff --git a/vllm/plugins/endpoint_plugins/interface.py b/vllm/plugins/endpoint_plugins/interface.py index 99487f57b68..00451217d2f 100644 --- a/vllm/plugins/endpoint_plugins/interface.py +++ b/vllm/plugins/endpoint_plugins/interface.py @@ -20,7 +20,7 @@ for the loading/gating rules and `docs/usage/security.md` for the security posture of exposing plugin defined routes. The CPU only render server (see `build_and_serve_renderer` in -`vllm/entrypoints/openai/api_server.py`) has no `EngineClient`. A plugin +`vllm/entrypoints/launchers/render`) has no `EngineClient`. A plugin eligible for the `render` task (`required_tasks` is `None` or includes `"render"`) still gets `attach_router` called but `init_state` receives `engine_client=None`. Plugins that cannot function without an engine should @@ -34,8 +34,9 @@ from typing import TYPE_CHECKING, Protocol, runtime_checkable from fastapi import FastAPI from starlette.datastructures import State +from vllm.engine.protocol import EngineClient + if TYPE_CHECKING: - from vllm.engine.protocol import EngineClient from vllm.tasks import SupportedTask @@ -85,3 +86,38 @@ class EndpointPlugin(Protocol): cannot function without an engine. """ ... + + +def attach_endpoint_plugins( + app: FastAPI, supported_tasks: tuple["SupportedTask", ...] +) -> None: + """Phase A of endpoint plugin wiring: discover, gate and attach routes. + + Attached last after all core routers. This is so endpoint plugin routes can + shadow core routes with the same path (see `EndpointPlugin.attach_router` + docstring). No-ops when no plugins are discovered/allowlisted. + """ + from vllm.plugins import load_endpoint_plugins + + endpoint_plugins = load_endpoint_plugins(supported_tasks) + for plugin in endpoint_plugins: + plugin.attach_router(app) + app.state.endpoint_plugins = endpoint_plugins + + +async def init_endpoint_plugins_state( + engine_client: EngineClient | None, state: State, args: Namespace +) -> None: + """Phase B of endpoint plugin wiring: initialize per app plugin state. + + `state.endpoint_plugins` is set by `_attach_endpoint_plugins` (Phase A) + in `build_app`. Some `init_app_state` callers (e.g. `run_batch.py`) + build their own bare `State` without going through `build_app`. As a result + `endpoint_plugins` may be absent and are treated that the same as "none attached". + + `engine_client` is `None` for the CPU only render server which has no + engine (see `init_render_app_state`). Plugins must handle a `None` + `engine_client` themselves (see `EndpointPlugin.init_state`). + """ + for plugin in getattr(state, "endpoint_plugins", []): + await plugin.init_state(engine_client, state, args) diff --git a/vllm/tasks.py b/vllm/tasks.py index 366ef0c672f..d1de4249a10 100644 --- a/vllm/tasks.py +++ b/vllm/tasks.py @@ -41,3 +41,4 @@ FrontendTask = Literal["render"] FRONTEND_TASKS: tuple[FrontendTask, ...] = get_args(FrontendTask) SupportedTask = Literal[GenerationTask, PoolingTask, FrontendTask] +FALLBACK_SUPPORTED_TASKS: tuple[SupportedTask, ...] = ("generate",) diff --git a/vllm/v1/utils.py b/vllm/v1/utils.py index 8d482f58fdf..0501c2ae327 100644 --- a/vllm/v1/utils.py +++ b/vllm/v1/utils.py @@ -503,7 +503,7 @@ def run_api_server_worker_proc( ) -> None: """Entrypoint for individual API server worker processes.""" - from vllm.entrypoints.openai.api_server import run_server_worker + from vllm.entrypoints.launchers.api_server.entry import run_server_worker client_config = client_config or {} server_index = client_config.get("client_index", 0)