[Fix] http_server: Make SageMaker /ping reflect server readiness (#45445)

docs/docs/basic_usage/aws_sagemaker.mdx

@@ -138,5 +138,11 @@ model.tar.gz
   drivers for CUDA 13 images. See the
   [ProductionVariant API reference](https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_ProductionVariant.html)
   for valid values.
+- `/ping` returns 503 until server warmup completes, during shutdown, and after a failed `/health` or
+  `/health_generate` probe until a later probe succeeds. If loading and warmup take longer than the
+  default startup health check window, raise `ContainerStartupHealthCheckTimeoutInSeconds`
+  (60 to 3600 seconds) on the
+  [ProductionVariant](https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_ProductionVariant.html)
+  or [InferenceComponentStartupParameters](https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_InferenceComponentStartupParameters.html).
 - The endpoint exposes an OpenAI-compatible API, so the request body matches the SGLang server's
   `/v1/chat/completions` schema.

docs/docs/basic_usage/native_api.mdx

@@ -108,6 +108,8 @@ print_highlight(response.text)
 ## Health Check
 - `/health`: Check the health of the server.
 - `/health_generate`: Check the health of the server by generating one token.
+- `/ready`: Returns 200 when the server accepts new requests; 503 while starting, paused, unhealthy, or shutting down.
+- `/ping`: SageMaker health check. Returns 200 once the server is up; 503 while starting, unhealthy, or shutting down. Stays 200 while generation is paused. A failed `/health` or `/health_generate` probe marks the server unhealthy until a later probe succeeds.
 
 ```python Example
 url = f"http://localhost:{port}/health_generate"

examples/checkpoint_engine/update.py

@@ -46,7 +46,10 @@ def check_sglang_ready(
         while True:
             try:
                 response = client.get(f"{endpoint}/ping", timeout=10)
-                response.raise_for_status()
+                # /ping is 503 until warmup, which may wait on the weights this script sends;
+                # a 503 still means the server is listening, which is all this check needs.
+                if response.status_code != 503:
+                    response.raise_for_status()
                 break
             except (httpx.ConnectError, httpx.HTTPStatusError) as e:
                 if retry_num % 10 == 0:

python/sglang/srt/checkpoint_engine/update.py

@@ -60,7 +60,10 @@ def check_sglang_ready(
         while True:
             try:
                 response = client.get(f"{endpoint}/ping", timeout=10)
-                response.raise_for_status()
+                # /ping is 503 until warmup, which may wait on the weights this script sends;
+                # a 503 still means the server is listening, which is all this check needs.
+                if response.status_code != 503:
+                    response.raise_for_status()
                 break
             except (httpx.ConnectError, httpx.HTTPStatusError) as e:
                 if retry_num % 10 == 0:

python/sglang/srt/entrypoints/grpc_bridge.py

@@ -439,14 +439,7 @@ class RuntimeHandle:
         )
 
     def health_check(self) -> bool:
-        from sglang.srt.managers.tokenizer_manager import ServerStatus
-
-        if self.tokenizer_manager.gracefully_exit:
-            return False
-        return self.tokenizer_manager.server_status not in (
-            ServerStatus.Starting,
-            ServerStatus.UnHealthy,
-        )
+        return self.tokenizer_manager.is_healthy()
 
     def is_pause(self) -> bool:
         """Return the tokenizer manager's authoritative generation pause state."""

python/sglang/srt/entrypoints/http_server.py

@@ -2187,8 +2187,14 @@ async def systemone_decisions(request: SystemOneRequest, raw_request: Request):
 ## SageMaker API
 @app.get("/ping")
 async def sagemaker_health() -> Response:
-    """Check the health of the http server."""
-    return Response(status_code=200)
+    """SageMaker health check: 200 when is_healthy(), else 503.
+
+    Unlike /ready, a paused server still reports 200, so an intentional pause
+    (e.g. a weight sync) does not get the instance replaced. Stays a flag check
+    because SageMaker times out /ping after 2 seconds.
+    """
+    status_code = 200 if _global_state.tokenizer_manager.is_healthy() else 503
+    return Response(status_code=status_code)
 
 
 @app.post("/invocations")

python/sglang/srt/managers/tokenizer_manager.py

@@ -725,13 +725,13 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
         # Subprocess liveness watchdog — set by Engine or http_server after construction
         self._subprocess_watchdog = None
 
+    def is_healthy(self) -> bool:
+        """Return whether startup completed and the server is neither unhealthy nor shutting down."""
+        return not self.gracefully_exit and self.server_status == ServerStatus.Up
+
     def is_ready(self) -> bool:
         """Return whether this server should receive new requests."""
-        return (
-            not self.is_pause
-            and not self.gracefully_exit
-            and self.server_status == ServerStatus.Up
-        )
+        return not self.is_pause and self.is_healthy()
 
     def init_request_logging_and_dumping(self):
         # TODO: Refactor and organize the log export code.

test/registered/unit/entrypoints/test_ready.py

@@ -10,27 +10,29 @@ from sglang.test.test_utils import CustomTestCase
 register_cpu_ci(est_time=12, suite="base-a-test-cpu")
 
 
-class TestReadyEndpoint(CustomTestCase):
-    def call_ready(
-        self,
-        *,
-        is_pause: bool = False,
-        gracefully_exit: bool = False,
-        server_status: ServerStatus = ServerStatus.Up,
-    ):
-        tokenizer_manager = TokenizerManager.__new__(TokenizerManager)
-        tokenizer_manager.is_pause = is_pause
-        tokenizer_manager.gracefully_exit = gracefully_exit
-        tokenizer_manager.server_status = server_status
+def _call_endpoint(
+    handler,
+    *,
+    is_pause: bool = False,
+    gracefully_exit: bool = False,
+    server_status: ServerStatus = ServerStatus.Up,
+):
+    tokenizer_manager = TokenizerManager.__new__(TokenizerManager)
+    tokenizer_manager.is_pause = is_pause
+    tokenizer_manager.gracefully_exit = gracefully_exit
+    tokenizer_manager.server_status = server_status
 
-        prior_state = http_server.get_global_state()
-        http_server.set_global_state(
-            SimpleNamespace(tokenizer_manager=tokenizer_manager)
-        )
-        try:
-            return asyncio.run(http_server.ready())
-        finally:
-            http_server._global_state = prior_state
+    prior_state = http_server.get_global_state()
+    http_server.set_global_state(SimpleNamespace(tokenizer_manager=tokenizer_manager))
+    try:
+        return asyncio.run(handler())
+    finally:
+        http_server._global_state = prior_state
+
+
+class TestReadyEndpoint(CustomTestCase):
+    def call_ready(self, **state):
+        return _call_endpoint(http_server.ready, **state)
 
     def test_ready_while_accepting_requests(self):
         self.assertEqual(self.call_ready().status_code, 200)
@@ -52,5 +54,34 @@ class TestReadyEndpoint(CustomTestCase):
         self.assertEqual(self.call_ready(gracefully_exit=True).status_code, 503)
 
 
+class TestSageMakerPingEndpoint(CustomTestCase):
+    def call_ping(self, **state):
+        return _call_endpoint(http_server.sagemaker_health, **state)
+
+    def test_ping_ok_with_empty_body_when_up(self):
+        # SageMaker's container contract: HTTP 200 with an empty body.
+        response = self.call_ping()
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.body, b"")
+
+    def test_ping_ok_while_paused(self):
+        # Unlike /ready, pausing must not fail the health check and get the instance replaced.
+        self.assertEqual(self.call_ping(is_pause=True).status_code, 200)
+
+    def test_ping_unavailable_while_starting(self):
+        """/ping must not report healthy before warmup finishes."""
+        self.assertEqual(
+            self.call_ping(server_status=ServerStatus.Starting).status_code, 503
+        )
+
+    def test_ping_unavailable_while_unhealthy(self):
+        self.assertEqual(
+            self.call_ping(server_status=ServerStatus.UnHealthy).status_code, 503
+        )
+
+    def test_ping_unavailable_while_gracefully_exiting(self):
+        self.assertEqual(self.call_ping(gracefully_exit=True).status_code, 503)
+
+
 if __name__ == "__main__":
     unittest.main(verbosity=2)