diff --git a/.github/workflows/nightly-test-amd-miles-rocm10-mi30x.yml b/.github/workflows/nightly-test-amd-miles-rocm10-mi30x.yml index 05747a6e9b..822fb3e7ae 100644 --- a/.github/workflows/nightly-test-amd-miles-rocm10-mi30x.yml +++ b/.github/workflows/nightly-test-amd-miles-rocm10-mi30x.yml @@ -25,6 +25,7 @@ concurrency: jobs: test-miles-mi30x: if: github.repository == 'sgl-project/sglang' + name: test-miles-mi30x (${{ matrix.suite }}, ${{ matrix.runner }}${{ matrix.partition_size && format(', part {0}', matrix.part) || '' }}) strategy: fail-fast: false matrix: @@ -41,15 +42,26 @@ jobs: skip_tests: >- tests/e2e/sglang/test_session_server_multi_role/test_qwen38small.py tests/e2e/megatron/model_scripts/test_glm5_2_744b_a40b_5layer_ci.py - # The 4- and 8-GPU suites run past 180 min on MI300X; they take ~155 and ~112 min on MI355X. + # The 4- and 8-GPU suites run past 180 min on MI300X; the 4-GPU suite takes ~155 min on MI355X. - suite: nightly-stage-c-4-gpu-mi350 runner: linux-mi300-4gpu-sglang timeout: 300 skip_tests: "" + # The 8-GPU suite no longer finishes in 300 min on MI300X; split it in two. - suite: nightly-stage-c-8-gpu-mi350 runner: linux-mi300-8gpu-sglang timeout: 300 skip_tests: "" + partition_id: 0 + partition_size: 2 + part: 1/2 + - suite: nightly-stage-c-8-gpu-mi350 + runner: linux-mi300-8gpu-sglang + timeout: 300 + skip_tests: "" + partition_id: 1 + partition_size: 2 + part: 2/2 runs-on: ${{ matrix.runner }} steps: - name: Checkout code @@ -107,6 +119,7 @@ jobs: -e PYTHONPATH=/root/miles:/opt/tilelang \ -e GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \ python3 tests/ci/run_suite.py --hw rocm --suite ${{ matrix.suite }} --cadence nightly \ + ${{ matrix.partition_size && format('--auto-partition-id {0} --auto-partition-size {1}', matrix.partition_id, matrix.partition_size) || '' }} \ ${{ (github.event_name != 'workflow_dispatch' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? cat "${GITHUB_WORKSPACE}/github_summary.md" >> "$GITHUB_STEP_SUMMARY" || true exit ${TEST_EXIT_CODE} diff --git a/.github/workflows/nightly-test-amd-miles-rocm10.yml b/.github/workflows/nightly-test-amd-miles-rocm10.yml index 2c4964c831..3817c02d1d 100644 --- a/.github/workflows/nightly-test-amd-miles-rocm10.yml +++ b/.github/workflows/nightly-test-amd-miles-rocm10.yml @@ -46,8 +46,18 @@ jobs: partition_id: 1 partition_size: 2 part: 2/2 + # The 8-GPU suite grew to ~6.5 h of estimated test time and no longer + # finishes in the step timeout; split it the same way. - suite: nightly-stage-c-8-gpu-mi350 runner: linux-mi35x-gpu-8 + partition_id: 0 + partition_size: 2 + part: 1/2 + - suite: nightly-stage-c-8-gpu-mi350 + runner: linux-mi35x-gpu-8 + partition_id: 1 + partition_size: 2 + part: 2/2 runs-on: ${{ matrix.runner }} steps: - name: Checkout code