mirror of
https://gitee.com/trusted-list/sglang.git
synced 2026-10-28 08:34:56 +08:00
[Feature] Hybrid EP and TP (#8590)
This commit is contained in:
Cheng Wan authored and GitHub committed
1 parent
51c38163c1
commit
7a1f7fc504
14 files changed
+142
-39
No files matched your search
+1
-1
@@ -1 +1 @@
|
||||
<svg width="2392" height="729" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" overflow="hidden"><defs><filter id="fx0" x="-10%" y="-10%" width="120%" height="120%" filterUnits="userSpaceOnUse" primitiveUnits="userSpaceOnUse"><feComponentTransfer color-interpolation-filters="sRGB"><feFuncR type="discrete" tableValues="0.835294 0.835294"/><feFuncG type="discrete" tableValues="0.345098 0.345098"/><feFuncB type="discrete" tableValues="0.086275 0.086275"/><feFuncA type="linear" slope="0.400000" intercept="0.000000"/></feComponentTransfer><feGaussianBlur stdDeviation="7.638889 7.638889"/></filter><filter id="fx1" x="-10%" y="-10%" width="120%" height="120%" filterUnits="userSpaceOnUse" primitiveUnits="userSpaceOnUse"><feComponentTransfer color-interpolation-filters="sRGB"><feFuncR type="discrete" tableValues="0.835294 0.835294"/><feFuncG type="discrete" tableValues="0.345098 0.345098"/><feFuncB type="discrete" tableValues="0.086275 0.086275"/><feFuncA type="linear" slope="0.400000" intercept="0.000000"/></feComponentTransfer><feGaussianBlur stdDeviation="6.111111 6.111111"/></filter><filter id="fx2" x="-10%" y="-10%" width="120%" height="120%" filterUnits="userSpaceOnUse" primitiveUnits="userSpaceOnUse"><feComponentTransfer color-interpolation-filters="sRGB"><feFuncR type="discrete" tableValues="0.835294 0.835294"/><feFuncG type="discrete" tableValues="0.345098 0.345098"/><feFuncB type="discrete" tableValues="0.086275 0.086275"/><feFuncA type="linear" slope="0.400000" intercept="0.000000"/></feComponentTransfer><feGaussianBlur stdDeviation="7.638889 7.638889"/></filter><clipPath id="clip3"><path d="M1756.97 902.5C1708.88 902.5 1667.36 908.097 1632.43 919.291 1597.49 930.485 1568.73 945.491 1546.14 964.309 1523.56 983.128 1506.82 1004.62 1495.94 1028.8 1485.05 1052.97 1479.61 1078.03 1479.61 1103.99 1479.61 1125.73 1483.26 1144.63 1490.57 1160.69 1497.89 1176.75 1507.63 1190.86 1519.82 1203.03 1532.01 1215.2 1545.74 1225.74 1561.01 1234.66 1576.29 1243.59 1591.97 1251.86 1608.05 1259.48 1624.14 1267.11 1639.9 1274.49 1655.34 1281.63 1670.77 1288.77 1684.58 1296.39 1696.77 1304.5 1708.96 1312.61 1718.71 1321.62 1726.02 1331.51 1733.33 1341.41 1736.99 1353.17 1736.99 1366.8 1736.99 1378.48 1734.23 1389.59 1728.7 1400.14 1723.18 1410.68 1714.48 1420.01 1702.62 1428.12 1690.76 1436.23 1675.65 1442.64 1657.29 1447.35 1638.93 1452.05 1616.91 1454.4 1591.24 1454.4 1576.61 1454.4 1561.5 1453.59 1545.9 1451.97 1530.3 1450.35 1514.7 1448.08 1499.1 1445.16 1483.51 1442.23 1468.07 1438.67 1452.79 1434.45 1437.52 1430.23 1423.06 1425.69 1409.41 1420.82L1386.5 1533.25C1415.1 1542.33 1444.83 1549.14 1475.71 1553.69 1506.58 1558.23 1540.05 1560.5 1576.12 1560.5 1621.3 1560.5 1662.24 1555.47 1698.96 1545.41 1735.69 1535.35 1766.97 1520.92 1792.8 1502.1 1818.64 1483.28 1838.54 1460.57 1852.52 1433.96 1866.49 1407.36 1873.48 1377.67 1873.48 1344.9 1873.48 1323.48 1869.74 1304.58 1862.27 1288.2 1854.79 1271.81 1844.96 1257.38 1832.77 1244.88 1820.59 1232.39 1806.78 1221.44 1791.34 1212.03 1775.9 1202.62 1760.06 1193.94 1743.81 1185.99 1727.56 1178.05 1711.72 1170.5 1696.28 1163.36 1680.85 1156.23 1667.04 1148.76 1654.85 1140.98 1642.66 1133.19 1632.83 1124.67 1625.36 1115.42 1617.88 1106.18 1614.15 1095.39 1614.15 1083.06 1614.15 1072.35 1616.66 1062.21 1621.7 1052.64 1626.74 1043.07 1634.62 1034.8 1645.34 1027.82 1656.07 1020.85 1669.72 1015.33 1686.29 1011.27 1702.87 1007.22 1722.69 1005.19 1745.76 1005.19 1757.46 1005.19 1769.81 1005.76 1782.81 1006.89 1795.81 1008.03 1808.73 1009.65 1821.56 1011.76 1834.4 1013.87 1846.75 1016.3 1858.61 1019.06 1870.47 1021.82 1881.11 1024.82 1890.54 1028.06L1911.5 922.455C1901.75 919.859 1890.86 917.344 1878.84 914.91 1866.82 912.477 1854.06 910.368 1840.57 908.584 1827.09 906.799 1813.28 905.339 1799.14 904.203 1785 903.068 1770.95 902.5 1756.97 902.5ZM756 866 3148 866 3148 1595 756 1595Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath><clipPath id="clip4"><rect x="1.24353" y="1.08319" width="595.223" height="724.834"/></clipPath><clipPath id="clip5"><rect x="1.41663" y="1.66669" width="592.667" height="675.667"/></clipPath><clipPath id="clip6"><rect x="-2078.55" y="-2770.64" width="374073" height="374073"/></clipPath><image width="512" height="512" xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAYAAAD0eNT6AAAAAXNSR0IArs4c6QAAAIRlWElmTU0AKgAAAAgABQESAAMAAAABAAEAAAEaAAUAAAABAAAASgEbAAUAAAABAAAAUgEoAAMAAAABAAIAAIdpAAQAAAABAAAAWgAAAAAAAACQAAAAAQAAAJAAAAABAAOgAQADAAAAAQABAACgAgAEAAAAAQAAAgCgAwAEAAAAAQAAAgAAAAAAGcBUEAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAQABJREFUeAHt3QmcXFWd6PF/VXVWwq6OozAgIjoiDKQTEBjHwCij7Nla2Yc1Ks6Mz+d7b5aPb9pZ3rzZfM8NCKC4IWMnnYAs6jAIjEImJJ1gEFRkGQVxYSdk63TVnf+pTnf6dN3uruUu55z7u34wfW/de5bvuVX/f926iwgTAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIILine truncated
|
||||
<svg width="2392" height="729" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" overflow="hidden"><defs><filter id="fx0" x="-10%" y="-10%" width="120%" height="120%" filterUnits="userSpaceOnUse" primitiveUnits="userSpaceOnUse"><feComponentTransfer color-interpolation-filters="sRGB"><feFuncR type="discrete" tableValues="0.835294 0.835294"/><feFuncG type="discrete" tableValues="0.345098 0.345098"/><feFuncB type="discrete" tableValues="0.086275 0.086275"/><feFuncA type="linear" slope="0.400000" intercept="0.000000"/></feComponentTransfer><feGaussianBlur stdDeviation="7.638889 7.638889"/></filter><filter id="fx1" x="-10%" y="-10%" width="120%" height="120%" filterUnits="userSpaceOnUse" primitiveUnits="userSpaceOnUse"><feComponentTransfer color-interpolation-filters="sRGB"><feFuncR type="discrete" tableValues="0.835294 0.835294"/><feFuncG type="discrete" tableValues="0.345098 0.345098"/><feFuncB type="discrete" tableValues="0.086275 0.086275"/><feFuncA type="linear" slope="0.400000" intercept="0.000000"/></feComponentTransfer><feGaussianBlur stdDeviation="6.111111 6.111111"/></filter><filter id="fx2" x="-10%" y="-10%" width="120%" height="120%" filterUnits="userSpaceOnUse" primitiveUnits="userSpaceOnUse"><feComponentTransfer color-interpolation-filters="sRGB"><feFuncR type="discrete" tableValues="0.835294 0.835294"/><feFuncG type="discrete" tableValues="0.345098 0.345098"/><feFuncB type="discrete" tableValues="0.086275 0.086275"/><feFuncA type="linear" slope="0.400000" intercept="0.000000"/></feComponentTransfer><feGaussianBlur stdDeviation="7.638889 7.638889"/></filter><clipPath id="clip3"><path d="M1756.97 902.5C1708.88 902.5 1667.36 908.097 1632.43 919.291 1597.49 930.485 1568.73 945.491 1546.14 964.309 1523.56 983.128 1506.82 1004.62 1495.94 1028.8 1485.05 1052.97 1479.61 1078.03 1479.61 1103.99 1479.61 1125.73 1483.26 1144.63 1490.57 1160.69 1497.89 1176.75 1507.63 1190.86 1519.82 1203.03 1532.01 1215.2 1545.74 1225.74 1561.01 1234.66 1576.29 1243.59 1591.97 1251.86 1608.05 1259.48 1624.14 1267.11 1639.9 1274.49 1655.34 1281.63 1670.77 1288.77 1684.58 1296.39 1696.77 1304.5 1708.96 1312.61 1718.71 1321.62 1726.02 1331.51 1733.33 1341.41 1736.99 1353.17 1736.99 1366.8 1736.99 1378.48 1734.23 1389.59 1728.7 1400.14 1723.18 1410.68 1714.48 1420.01 1702.62 1428.12 1690.76 1436.23 1675.65 1442.64 1657.29 1447.35 1638.93 1452.05 1616.91 1454.4 1591.24 1454.4 1576.61 1454.4 1561.5 1453.59 1545.9 1451.97 1530.3 1450.35 1514.7 1448.08 1499.1 1445.16 1483.51 1442.23 1468.07 1438.67 1452.79 1434.45 1437.52 1430.23 1423.06 1425.69 1409.41 1420.82L1386.5 1533.25C1415.1 1542.33 1444.83 1549.14 1475.71 1553.69 1506.58 1558.23 1540.05 1560.5 1576.12 1560.5 1621.3 1560.5 1662.24 1555.47 1698.96 1545.41 1735.69 1535.35 1766.97 1520.92 1792.8 1502.1 1818.64 1483.28 1838.54 1460.57 1852.52 1433.96 1866.49 1407.36 1873.48 1377.67 1873.48 1344.9 1873.48 1323.48 1869.74 1304.58 1862.27 1288.2 1854.79 1271.81 1844.96 1257.38 1832.77 1244.88 1820.59 1232.39 1806.78 1221.44 1791.34 1212.03 1775.9 1202.62 1760.06 1193.94 1743.81 1185.99 1727.56 1178.05 1711.72 1170.5 1696.28 1163.36 1680.85 1156.23 1667.04 1148.76 1654.85 1140.98 1642.66 1133.19 1632.83 1124.67 1625.36 1115.42 1617.88 1106.18 1614.15 1095.39 1614.15 1083.06 1614.15 1072.35 1616.66 1062.21 1621.7 1052.64 1626.74 1043.07 1634.62 1034.8 1645.34 1027.82 1656.07 1020.85 1669.72 1015.33 1686.29 1011.27 1702.87 1007.22 1722.69 1005.19 1745.76 1005.19 1757.46 1005.19 1769.81 1005.76 1782.81 1006.89 1795.81 1008.03 1808.73 1009.65 1821.56 1011.76 1834.4 1013.87 1846.75 1016.3 1858.61 1019.06 1870.47 1021.82 1881.11 1024.82 1890.54 1028.06L1911.5 922.455C1901.75 919.859 1890.86 917.344 1878.84 914.91 1866.82 912.477 1854.06 910.368 1840.57 908.584 1827.09 906.799 1813.28 905.339 1799.14 904.203 1785 903.068 1770.95 902.5 1756.97 902.5ZM756 866 3148 866 3148 1595 756 1595Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath><clipPath id="clip4"><rect x="1.24353" y="1.08319" width="595.223" height="724.834"/></clipPath><clipPath id="clip5"><rect x="1.41663" y="1.66669" width="592.667" height="675.667"/></clipPath><clipPath id="clip6"><rect x="-2078.55" y="-2770.64" width="374073" height="374073"/></clipPath><image width="512" height="512" xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAYAAAD0eNT6AAAAAXNSR0IArs4c6QAAAIRlWElmTU0AKgAAAAgABQESAAMAAAABAAEAAAEaAAUAAAABAAAASgEbAAUAAAABAAAAUgEoAAMAAAABAAIAAIdpAAQAAAABAAAAWgAAAAAAAACQAAAAAQAAAJAAAAABAAOgAQADAAAAAQABAACgAgAEAAAAAQAAAgCgAwAEAAAAAQAAAgAAAAAAGcBUEAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAQABJREFUeAHt3QmcXFWd6PF/VXVWwq6OozAgIjoiDKQTEBjHwCij7Nla2Yc1Ks6Mz+d7b5aPb9pZ3rzZfM8NCKC4IWMnnYAs6jAIjEImJJ1gEFRkGQVxYSdk63TVnf+pTnf6dN3uruUu55z7u34wfW/de5bvuVX/f926iwgTAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIILine truncated
|
||||
|
Before Width: | Height: | Size: 65 KiB After Width: | Height: | Size: 65 KiB |
@@ -1 +1 @@
|
||||
<svg width="596" height="683" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" overflow="hidden"><defs><filter id="fx0" x="-10%" y="-10%" width="120%" height="120%" filterUnits="userSpaceOnUse" primitiveUnits="userSpaceOnUse"><feComponentTransfer color-interpolation-filters="sRGB"><feFuncR type="discrete" tableValues="0.835294 0.835294"/><feFuncG type="discrete" tableValues="0.345098 0.345098"/><feFuncB type="discrete" tableValues="0.086275 0.086275"/><feFuncA type="linear" slope="0.400000" intercept="0.000000"/></feComponentTransfer><feGaussianBlur stdDeviation="6.111111 6.111111"/></filter><clipPath id="clip1"><rect x="1.41663" y="1.66669" width="592.667" height="675.667"/></clipPath><clipPath id="clip2"><rect x="-2078.55" y="-2770.64" width="374073" height="374073"/></clipPath><image width="512" height="512" xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAYAAAD0eNT6AAAAAXNSR0IArs4c6QAAAIRlWElmTU0AKgAAAAgABQESAAMAAAABAAEAAAEaAAUAAAABAAAASgEbAAUAAAABAAAAUgEoAAMAAAABAAIAAIdpAAQAAAABAAAAWgAAAAAAAACQAAAAAQAAAJAAAAABAAOgAQADAAAAAQABAACgAgAEAAAAAQAAAgCgAwAEAAAAAQAAAgAAAAAAGcBUEAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAQABJREFUeAHt3QmcXFWd6PF/VXVWwq6OozAgIjoiDKQTEBjHwCij7Nla2Yc1Ks6Mz+d7b5aPb9pZ3rzZfM8NCKC4IWMnnYAs6jAIjEImJJ1gEFRkGQVxYSdk63TVnf+pTnf6dN3uruUu55z7u34wfW/de5bvuVX/f926iwgTAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBAwgKlhMujOAQQQCA1gUikdNsRss+OmbJvKZLZUUm2Vrtkx8Nr5Be9IrXUKqZgBAIUIAEIcFDpEgKhCNx8tLyuWpF3RZGcKCU5Uvv1Fv1vdkz/duiyJ/S/72uS8L1yTe5ctEF+GLMeixBAYJcACQC7AgIIOCVw+6EyY9u+crZEcoE27J36X7nNBm7S7W6oTpPretbI822WwWYIBCtAAhDs0NIxBPwSuH6BzNx7i3xYv+1/VFv+mwm2/hU9KnBtrSp/17NRnkmwXIpCwGsBEgCvh4/GIxCGQH+3nKqH+D+tvTkktR6V5AU9b+Djm9bLVb2cL5AaMwX7I0AC4M9Y0VIEghMw3/r33CJ/r4H5j7PqnH7o3V2pyrlnbpSns6qTehBwUYAEwMVRoU0IFECg/1g5QKpym3bVnNyX9fQrPbHgjIXr5f6sK6Y+BFwRaPfkGlfaTzsQQMBDgZXdejZ/Ve7VpucR/I3Yb+g1g3eu6JaTPeSjyQgkIsARgEQYKQQBBJoVuGm+HFiN6sH/wGa3SXG97bWanLx0g3w3xTooGgEnBUgAnBwWGoVAmAJ9x8l+lZ2yRnt3mDM9NCcHVuUE7hvgzIjQkIwE+AkgI2iqQaDoAnopXqkyKNepgzvB3wxKJPtGZVlxS3fsDYaKPmz0P2ABEoCAB5euIeCSgF7q9wG91G+hS20a05bDB8vyT2Pm+ROB4AX4CSD4IaaDCOQvsOpIeU00XX6sLdkn/9ZM2IKa3oTo+CUDsnbCNXgBgYAEugLqC12ZRMAcfr15vhygd0N7kx7ufL1+E5ulH3Z7l0rykmaBW/T1J6sij/Ssk19OUgwvIdCWQDRD/lYPtbsc/E2/yvp++JT++/a2OunBRnctkK5nXpaD9YP/t/RzYI7ef2GGfhYMVkvyin4ePCXb5Imeh2TQg67QxAQEOAKQAKKrRdQfpFKWJfrEtN/XNv6u/rdfE239pX4gmDOi/02fsraSe6g3IcYqkwrsut7/MV1p+qQrOvJiVJP3LNkg33akOR01o+9wmV6eLX+ghbxLE7Df0w/8w/XvaZMUOqSv/Vg/A/5d1/+OPkfhNv0M2DbJ+rzksQAJgMeDF9d0802/f568Vwf2I/r6SfpfJW69JpeZbwK362VSn+QyqSbFWK1BYOU8+aTuj/+t4QV3F9y1eH39veNuC6do2apj5DD9Rv8RDeLv01WbSfwnKnGzltFfLsunFq6TByZaieV+CpAA+Dlusa02NzXRQ5j/pIN6ROwKnS28Tz9QPsrvo50hFm1r/QY6pzJLDy2L7O1R3yP9Bnzo4nXyuEdtrje1b64cWinL/9UZc7Jl0id5f0uPjvwvPTqyyTcX2hsvQAIQ7+LV0puPld8YGpKr9EMr7TOs9eZpcv22inz0vLXysldINDYXgf75crl+g1yeS+WdVKoPDVo8IH/TSRFZbmsO9XfNkv+tRwA/pvXOSLFu8xPB1VsG5U8v2CRbUqyHojMQSDpDzKDJVDFWQD9g3zVU1UNz6Qd/U63ZXy6ZVZWB1XOle2w7+BuBWIFIlsUud31hSd7rehNH2rf67XJwZbbco8H/L3RZmsHfVGlOHP/wnOn6GTBfjjILmPwV4AiAv2MnGvz/UL9dXatdMG/KrKcdegbx+YsGZEXWFVOfHwL9c/Vs+nL9rn9+NNhu5ZAe6drf9SNd+hnwe/oZcLM2PY8rLLZpADl70fp6/bYec14IcATAi2FqbKSeWPUX+sa/Xl/JI/ibBs3Qqwtu1HZc3Ng6liCgAmX5oMcOXTN2yu+43H492fcM/QwwVyvkEfwNzSw96tCv7bjEZSfaNrEACcDENs6+om+4Xs28Xfh9sqLtuFbbs8RZLBqWi8CtR8i+WrHX+0Wlok8sdHRaMU8WaNO+rv/NzLmJ5iqja1fNkytybgfVtyFAAtAGWp6bmOCv9f9lnm0YV7fZh76ij3c9dtxyZgsssGN6/cjQbJ8J9NvtG11svwbbt2nUNYf98w7+Izz6a6B8ZpW51TOTVwIkAB4Nl4PBf0Rvpl5+uMI86W1kAf8WV0CDQUn/d7n3AvqQINf6cP0Cmam+N+h/eznWtpL+JHglRwIcG5UpmkMCMAWQKy87HPxHiA7UJ719emSGLine truncated
|
||||
<svg width="596" height="683" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" overflow="hidden"><defs><filter id="fx0" x="-10%" y="-10%" width="120%" height="120%" filterUnits="userSpaceOnUse" primitiveUnits="userSpaceOnUse"><feComponentTransfer color-interpolation-filters="sRGB"><feFuncR type="discrete" tableValues="0.835294 0.835294"/><feFuncG type="discrete" tableValues="0.345098 0.345098"/><feFuncB type="discrete" tableValues="0.086275 0.086275"/><feFuncA type="linear" slope="0.400000" intercept="0.000000"/></feComponentTransfer><feGaussianBlur stdDeviation="6.111111 6.111111"/></filter><clipPath id="clip1"><rect x="1.41663" y="1.66669" width="592.667" height="675.667"/></clipPath><clipPath id="clip2"><rect x="-2078.55" y="-2770.64" width="374073" height="374073"/></clipPath><image width="512" height="512" xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAYAAAD0eNT6AAAAAXNSR0IArs4c6QAAAIRlWElmTU0AKgAAAAgABQESAAMAAAABAAEAAAEaAAUAAAABAAAASgEbAAUAAAABAAAAUgEoAAMAAAABAAIAAIdpAAQAAAABAAAAWgAAAAAAAACQAAAAAQAAAJAAAAABAAOgAQADAAAAAQABAACgAgAEAAAAAQAAAgCgAwAEAAAAAQAAAgAAAAAAGcBUEAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAQABJREFUeAHt3QmcXFWd6PF/VXVWwq6OozAgIjoiDKQTEBjHwCij7Nla2Yc1Ks6Mz+d7b5aPb9pZ3rzZfM8NCKC4IWMnnYAs6jAIjEImJJ1gEFRkGQVxYSdk63TVnf+pTnf6dN3uruUu55z7u34wfW/de5bvuVX/f926iwgTAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBAwgKlhMujOAQQQCA1gUikdNsRss+OmbJvKZLZUUm2Vrtkx8Nr5Be9IrXUKqZgBAIUIAEIcFDpEgKhCNx8tLyuWpF3RZGcKCU5Uvv1Fv1vdkz/duiyJ/S/72uS8L1yTe5ctEF+GLMeixBAYJcACQC7AgIIOCVw+6EyY9u+crZEcoE27J36X7nNBm7S7W6oTpPretbI822WwWYIBCtAAhDs0NIxBPwSuH6BzNx7i3xYv+1/VFv+mwm2/hU9KnBtrSp/17NRnkmwXIpCwGsBEgCvh4/GIxCGQH+3nKqH+D+tvTkktR6V5AU9b+Djm9bLVb2cL5AaMwX7I0AC4M9Y0VIEghMw3/r33CJ/r4H5j7PqnH7o3V2pyrlnbpSns6qTehBwUYAEwMVRoU0IFECg/1g5QKpym3bVnNyX9fQrPbHgjIXr5f6sK6Y+BFwRaPfkGlfaTzsQQMBDgZXdejZ/Ve7VpucR/I3Yb+g1g3eu6JaTPeSjyQgkIsARgEQYKQQBBJoVuGm+HFiN6sH/wGa3SXG97bWanLx0g3w3xTooGgEnBUgAnBwWGoVAmAJ9x8l+lZ2yRnt3mDM9NCcHVuUE7hvgzIjQkIwE+AkgI2iqQaDoAnopXqkyKNepgzvB3wxKJPtGZVlxS3fsDYaKPmz0P2ABEoCAB5euIeCSgF7q9wG91G+hS20a05bDB8vyT2Pm+ROB4AX4CSD4IaaDCOQvsOpIeU00XX6sLdkn/9ZM2IKa3oTo+CUDsnbCNXgBgYAEugLqC12ZRMAcfr15vhygd0N7kx7ufL1+E5ulH3Z7l0rykmaBW/T1J6sij/Ssk19OUgwvIdCWQDRD/lYPtbsc/E2/yvp++JT++/a2OunBRnctkK5nXpaD9YP/t/RzYI7ef2GGfhYMVkvyin4ePCXb5Imeh2TQg67QxAQEOAKQAKKrRdQfpFKWJfrEtN/XNv6u/rdfE239pX4gmDOi/02fsraSe6g3IcYqkwrsut7/MV1p+qQrOvJiVJP3LNkg33akOR01o+9wmV6eLX+ghbxLE7Df0w/8w/XvaZMUOqSv/Vg/A/5d1/+OPkfhNv0M2DbJ+rzksQAJgMeDF9d0802/f568Vwf2I/r6SfpfJW69JpeZbwK362VSn+QyqSbFWK1BYOU8+aTuj/+t4QV3F9y1eH39veNuC6do2apj5DD9Rv8RDeLv01WbSfwnKnGzltFfLsunFq6TByZaieV+CpAA+Dlusa02NzXRQ5j/pIN6ROwKnS28Tz9QPsrvo50hFm1r/QY6pzJLDy2L7O1R3yP9Bnzo4nXyuEdtrje1b64cWinL/9UZc7Jl0id5f0uPjvwvPTqyyTcX2hsvQAIQ7+LV0puPld8YGpKr9EMr7TOs9eZpcv22inz0vLXysldINDYXgf75crl+g1yeS+WdVKoPDVo8IH/TSRFZbmsO9XfNkv+tRwA/pvXOSLFu8xPB1VsG5U8v2CRbUqyHojMQSDpDzKDJVDFWQD9g3zVU1UNz6Qd/U63ZXy6ZVZWB1XOle2w7+BuBWIFIlsUud31hSd7rehNH2rf67XJwZbbco8H/L3RZmsHfVGlOHP/wnOn6GTBfjjILmPwV4AiAv2MnGvz/UL9dXatdMG/KrKcdegbx+YsGZEXWFVOfHwL9c/Vs+nL9rn9+NNhu5ZAe6drf9SNd+hnwe/oZcLM2PY8rLLZpADl70fp6/bYec14IcATAi2FqbKSeWPUX+sa/Xl/JI/ibBs3Qqwtu1HZc3Ng6liCgAmX5oMcOXTN2yu+43H492fcM/QwwVyvkEfwNzSw96tCv7bjEZSfaNrEACcDENs6+om+4Xs28Xfh9sqLtuFbbs8RZLBqWi8CtR8i+WrHX+0Wlok8sdHRaMU8WaNO+rv/NzLmJ5iqja1fNkytybgfVtyFAAtAGWp6bmOCv9f9lnm0YV7fZh76ij3c9dtxyZgsssGN6/cjQbJ8J9NvtG11svwbbt2nUNYf98w7+Izz6a6B8ZpW51TOTVwIkAB4Nl4PBf0Rvpl5+uMI86W1kAf8WV0CDQUn/d7n3AvqQINf6cP0Cmam+N+h/eznWtpL+JHglRwIcG5UpmkMCMAWQKy87HPxHiA7UJ719emSGLine truncated
|
||||
|
Before Width: | Height: | Size: 51 KiB After Width: | Height: | Size: 51 KiB |
@@ -138,6 +138,7 @@ class BenchArgs:
|
||||
def load_model(server_args, port_args, tp_rank):
|
||||
suppress_other_loggers()
|
||||
rank_print = print if tp_rank == 0 else lambda *args, **kwargs: None
|
||||
moe_ep_rank = tp_rank // (server_args.tp_size // server_args.ep_size)
|
||||
|
||||
model_config = ModelConfig.from_server_args(server_args)
|
||||
model_runner = ModelRunner(
|
||||
@@ -146,6 +147,8 @@ def load_model(server_args, port_args, tp_rank):
|
||||
gpu_id=tp_rank,
|
||||
tp_rank=tp_rank,
|
||||
tp_size=server_args.tp_size,
|
||||
moe_ep_rank=moe_ep_rank,
|
||||
moe_ep_size=server_args.ep_size,
|
||||
pp_rank=0,
|
||||
pp_size=1,
|
||||
nccl_port=port_args.nccl_port,
|
||||
|
||||
@@ -354,6 +354,13 @@ class GroupCoordinator:
|
||||
self.cpu_group, 1 << 22, 6
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"ranks={self.ranks} rank={self.rank} local_rank={self.local_rank} use_pynccl={self.use_pynccl} "
|
||||
f"device_group={self.device_group} cpu_group={self.cpu_group} unique_name={self.unique_name} "
|
||||
f"world_size={self.world_size} rank_in_group={self.rank_in_group}"
|
||||
)
|
||||
|
||||
@property
|
||||
def first_rank(self):
|
||||
"""Return the global rank of the first process in the group"""
|
||||
@@ -1141,6 +1148,20 @@ def get_tp_group() -> GroupCoordinator:
|
||||
return _TP
|
||||
|
||||
|
||||
_MOE_EP: Optional[GroupCoordinator] = None
|
||||
_MOE_TP: Optional[GroupCoordinator] = None
|
||||
|
||||
|
||||
def get_moe_ep_group() -> GroupCoordinator:
|
||||
assert _MOE_EP is not None, "expert model parallel group is not initialized"
|
||||
return _MOE_EP
|
||||
|
||||
|
||||
def get_moe_tp_group() -> GroupCoordinator:
|
||||
assert _MOE_TP is not None, "expert model parallel group is not initialized"
|
||||
return _MOE_TP
|
||||
|
||||
|
||||
# kept for backward compatibility
|
||||
get_tensor_model_parallel_group = get_tp_group
|
||||
|
||||
@@ -1250,6 +1271,7 @@ def init_distributed_environment(
|
||||
|
||||
def initialize_model_parallel(
|
||||
tensor_model_parallel_size: int = 1,
|
||||
expert_model_parallel_size: int = 1,
|
||||
pipeline_model_parallel_size: int = 1,
|
||||
backend: Optional[str] = None,
|
||||
duplicate_tp_group: bool = False,
|
||||
@@ -1327,6 +1349,45 @@ def initialize_model_parallel(
|
||||
_TP.pynccl_comm.disabled = False
|
||||
_PDMUX_PREFILL_TP_GROUP.pynccl_comm.disabled = False
|
||||
|
||||
moe_ep_size = expert_model_parallel_size
|
||||
|
||||
moe_tp_size = tensor_model_parallel_size // moe_ep_size
|
||||
global _MOE_EP
|
||||
assert _MOE_EP is None, "expert model parallel group is already initialized"
|
||||
group_ranks = []
|
||||
for i in range(num_tensor_model_parallel_groups):
|
||||
for j in range(moe_tp_size):
|
||||
st = i * tensor_model_parallel_size + j
|
||||
en = (i + 1) * tensor_model_parallel_size + j
|
||||
ranks = list(range(st, en, moe_tp_size))
|
||||
group_ranks.append(ranks)
|
||||
|
||||
_MOE_EP = init_model_parallel_group(
|
||||
group_ranks,
|
||||
get_world_group().local_rank,
|
||||
backend,
|
||||
use_custom_allreduce=False,
|
||||
group_name="moe_ep",
|
||||
)
|
||||
|
||||
global _MOE_TP
|
||||
assert _MOE_TP is None, "expert model parallel group is already initialized"
|
||||
group_ranks = []
|
||||
for i in range(num_tensor_model_parallel_groups):
|
||||
for j in range(moe_ep_size):
|
||||
st = i * tensor_model_parallel_size + j * moe_tp_size
|
||||
en = i * tensor_model_parallel_size + (j + 1) * moe_tp_size
|
||||
ranks = list(range(st, en))
|
||||
group_ranks.append(ranks)
|
||||
|
||||
_MOE_TP = init_model_parallel_group(
|
||||
group_ranks,
|
||||
get_world_group().local_rank,
|
||||
backend,
|
||||
use_custom_allreduce=False,
|
||||
group_name="moe_tp",
|
||||
)
|
||||
|
||||
# Build the pipeline model-parallel groups.
|
||||
num_pipeline_model_parallel_groups: int = world_size // pipeline_model_parallel_size
|
||||
global _PP
|
||||
@@ -1347,6 +1408,7 @@ def initialize_model_parallel(
|
||||
|
||||
def ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size: int,
|
||||
expert_model_parallel_size: int,
|
||||
pipeline_model_parallel_size: int,
|
||||
backend: Optional[str] = None,
|
||||
) -> None:
|
||||
@@ -1357,7 +1419,10 @@ def ensure_model_parallel_initialized(
|
||||
backend = backend or torch.distributed.get_backend(get_world_group().device_group)
|
||||
if not model_parallel_is_initialized():
|
||||
initialize_model_parallel(
|
||||
tensor_model_parallel_size, pipeline_model_parallel_size, backend
|
||||
tensor_model_parallel_size,
|
||||
expert_model_parallel_size,
|
||||
pipeline_model_parallel_size,
|
||||
backend,
|
||||
)
|
||||
return
|
||||
|
||||
@@ -1417,6 +1482,26 @@ def get_tensor_model_parallel_rank():
|
||||
return get_tp_group().rank_in_group
|
||||
|
||||
|
||||
def get_moe_expert_parallel_world_size():
|
||||
"""Return world size for the moe expert parallel group."""
|
||||
return get_moe_ep_group().world_size
|
||||
|
||||
|
||||
def get_moe_expert_parallel_rank():
|
||||
"""Return my rank for the moe expert parallel group."""
|
||||
return get_moe_ep_group().rank_in_group
|
||||
|
||||
|
||||
def get_moe_tensor_parallel_world_size():
|
||||
"""Return world size for the moe tensor parallel group."""
|
||||
return get_moe_tp_group().world_size
|
||||
|
||||
|
||||
def get_moe_tensor_parallel_rank():
|
||||
"""Return my rank for the moe tensor parallel group."""
|
||||
return get_moe_tp_group().rank_in_group
|
||||
|
||||
|
||||
def destroy_model_parallel():
|
||||
"""Set the groups to none and destroy them."""
|
||||
global _TP
|
||||
|
||||
@@ -719,6 +719,7 @@ def _launch_subprocesses(
|
||||
+ ((pp_rank % pp_size_per_node) * tp_size_per_node)
|
||||
+ (tp_rank % tp_size_per_node) * server_args.gpu_id_step
|
||||
)
|
||||
moe_ep_rank = tp_rank // (server_args.tp_size // server_args.ep_size)
|
||||
proc = mp.Process(
|
||||
target=run_scheduler_process,
|
||||
args=(
|
||||
@@ -726,6 +727,7 @@ def _launch_subprocesses(
|
||||
port_args,
|
||||
gpu_id,
|
||||
tp_rank,
|
||||
moe_ep_rank,
|
||||
pp_rank,
|
||||
None,
|
||||
writer,
|
||||
|
||||
@@ -135,7 +135,7 @@ class EPMoE(FusedMoE):
|
||||
enable_ep_moe=True,
|
||||
)
|
||||
|
||||
self.start_expert_id = self.ep_rank * self.num_local_experts
|
||||
self.start_expert_id = self.moe_ep_rank * self.num_local_experts
|
||||
self.end_expert_id = self.start_expert_id + self.num_local_experts - 1
|
||||
|
||||
self.intermediate_size = intermediate_size
|
||||
|
||||
@@ -7,6 +7,10 @@ from typing import List, Optional, Tuple
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import (
|
||||
get_moe_expert_parallel_rank,
|
||||
get_moe_expert_parallel_world_size,
|
||||
get_moe_tensor_parallel_rank,
|
||||
get_moe_tensor_parallel_world_size,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_reduce,
|
||||
@@ -88,10 +92,6 @@ class FusedMoE(torch.nn.Module):
|
||||
self.layer_id = layer_id
|
||||
self.top_k = top_k
|
||||
self.hidden_size = hidden_size
|
||||
self.tp_size = (
|
||||
tp_size if tp_size is not None else get_tensor_model_parallel_world_size()
|
||||
)
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
self.num_experts = num_experts
|
||||
self.num_fused_shared_experts = num_fused_shared_experts
|
||||
self.expert_map_cpu = None
|
||||
@@ -103,30 +103,27 @@ class FusedMoE(torch.nn.Module):
|
||||
enable_ep_moe = False
|
||||
|
||||
self.enable_flashinfer_cutlass_moe = enable_flashinfer_cutlass_moe
|
||||
self.moe_ep_size = get_moe_expert_parallel_world_size()
|
||||
self.moe_ep_rank = get_moe_expert_parallel_rank()
|
||||
self.moe_tp_size = get_moe_tensor_parallel_world_size()
|
||||
self.moe_tp_rank = get_moe_tensor_parallel_rank()
|
||||
assert num_experts % self.moe_ep_size == 0
|
||||
self.num_local_experts = num_experts // self.moe_ep_size
|
||||
if enable_ep_moe:
|
||||
# TODO(ch-wan): support shared experts fusion
|
||||
self.ep_size = self.tp_size
|
||||
self.ep_rank = self.tp_rank
|
||||
self.tp_size = 1
|
||||
self.tp_rank = 0
|
||||
# Create a tensor of size num_experts filled with -1
|
||||
self.expert_map_cpu = torch.full((self.num_experts,), -1, dtype=torch.int32)
|
||||
# Create a expert map for the local experts
|
||||
assert num_experts % self.ep_size == 0
|
||||
self.num_local_experts = num_experts // self.ep_size
|
||||
self.expert_map_cpu[
|
||||
self.ep_rank
|
||||
* self.num_local_experts : (self.ep_rank + 1)
|
||||
self.moe_ep_rank
|
||||
* self.num_local_experts : (self.moe_ep_rank + 1)
|
||||
* self.num_local_experts
|
||||
] = torch.arange(0, self.num_local_experts, dtype=torch.int32, device="cpu")
|
||||
self.expert_map_gpu = self.expert_map_cpu.to(device="cuda")
|
||||
else:
|
||||
self.ep_size = 1
|
||||
self.ep_rank = 0
|
||||
self.num_local_experts = num_experts
|
||||
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
assert intermediate_size % self.tp_size == 0
|
||||
self.intermediate_size_per_partition = intermediate_size // self.tp_size
|
||||
assert intermediate_size % self.moe_tp_size == 0
|
||||
self.intermediate_size_per_partition = intermediate_size // self.moe_tp_size
|
||||
self.reduce_results = reduce_results
|
||||
self.activation = activation
|
||||
self.apply_router_weight_on_input = apply_router_weight_on_input
|
||||
@@ -437,8 +434,7 @@ class FusedMoE(torch.nn.Module):
|
||||
expert_id: int,
|
||||
) -> None:
|
||||
|
||||
# TP rank is set to 0 if EP is enabled
|
||||
tp_rank = 0 if self.ep_size > 1 else get_tensor_model_parallel_rank()
|
||||
tp_rank = self.moe_tp_rank
|
||||
|
||||
# compressed-tensors checkpoints with packed weights are stored flipped
|
||||
# TODO (mgoin): check self.quant_method.quant_config.quant_format
|
||||
@@ -630,17 +626,17 @@ class FusedMoE(torch.nn.Module):
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
**(
|
||||
dict(
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
ep_rank=self.ep_rank,
|
||||
ep_size=self.ep_size,
|
||||
tp_rank=self.moe_tp_rank,
|
||||
tp_size=self.moe_tp_size,
|
||||
ep_rank=self.moe_ep_rank,
|
||||
ep_size=self.moe_ep_size,
|
||||
)
|
||||
if self.quant_method.__class__.__name__ == "ModelOptNvFp4FusedMoEMethod"
|
||||
else {}
|
||||
),
|
||||
)
|
||||
|
||||
if self.reduce_results and (self.tp_size > 1 or self.ep_size > 1):
|
||||
if self.reduce_results and (self.moe_tp_size > 1 or self.moe_ep_size > 1):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
|
||||
return final_hidden_states
|
||||
|
||||
@@ -222,6 +222,7 @@ class DataParallelController:
|
||||
+ ((pp_rank % pp_size_per_node) * tp_size_per_node)
|
||||
+ (tp_rank % tp_size_per_node) * server_args.gpu_id_step
|
||||
)
|
||||
moe_ep_rank = tp_rank // (server_args.tp_size // server_args.ep_size)
|
||||
proc = mp.Process(
|
||||
target=run_scheduler_process,
|
||||
args=(
|
||||
@@ -229,6 +230,7 @@ class DataParallelController:
|
||||
rank_port_args,
|
||||
gpu_id,
|
||||
tp_rank,
|
||||
moe_ep_rank,
|
||||
pp_rank,
|
||||
dp_rank,
|
||||
writer,
|
||||
|
||||
@@ -200,15 +200,18 @@ class Scheduler(
|
||||
port_args: PortArgs,
|
||||
gpu_id: int,
|
||||
tp_rank: int,
|
||||
moe_ep_rank: int,
|
||||
pp_rank: int,
|
||||
dp_rank: Optional[int],
|
||||
):
|
||||
# Parse args
|
||||
self.server_args = server_args
|
||||
self.tp_rank = tp_rank
|
||||
self.moe_ep_rank = moe_ep_rank
|
||||
self.pp_rank = pp_rank
|
||||
self.dp_rank = dp_rank
|
||||
self.tp_size = server_args.tp_size
|
||||
self.moe_ep_size = server_args.ep_size
|
||||
self.pp_size = server_args.pp_size
|
||||
self.dp_size = server_args.dp_size
|
||||
self.schedule_policy = server_args.schedule_policy
|
||||
@@ -310,6 +313,7 @@ class Scheduler(
|
||||
server_args=server_args,
|
||||
gpu_id=gpu_id,
|
||||
tp_rank=tp_rank,
|
||||
moe_ep_rank=moe_ep_rank,
|
||||
pp_rank=pp_rank,
|
||||
dp_rank=dp_rank,
|
||||
nccl_port=port_args.nccl_port,
|
||||
@@ -322,6 +326,7 @@ class Scheduler(
|
||||
self.draft_worker = EAGLEWorker(
|
||||
gpu_id=gpu_id,
|
||||
tp_rank=tp_rank,
|
||||
moe_ep_rank=moe_ep_rank,
|
||||
server_args=server_args,
|
||||
nccl_port=port_args.nccl_port,
|
||||
target_worker=self.tp_worker,
|
||||
@@ -2358,6 +2363,7 @@ def run_scheduler_process(
|
||||
port_args: PortArgs,
|
||||
gpu_id: int,
|
||||
tp_rank: int,
|
||||
moe_ep_rank: int,
|
||||
pp_rank: int,
|
||||
dp_rank: Optional[int],
|
||||
pipe_writer,
|
||||
@@ -2368,6 +2374,8 @@ def run_scheduler_process(
|
||||
prefix += f" DP{dp_rank}"
|
||||
if server_args.tp_size > 1:
|
||||
prefix += f" TP{tp_rank}"
|
||||
if server_args.ep_size > 1:
|
||||
prefix += f" EP{moe_ep_rank}"
|
||||
if server_args.pp_size > 1:
|
||||
prefix += f" PP{pp_rank}"
|
||||
|
||||
@@ -2391,7 +2399,9 @@ def run_scheduler_process(
|
||||
|
||||
# Create a scheduler and run the event loop
|
||||
try:
|
||||
scheduler = Scheduler(server_args, port_args, gpu_id, tp_rank, pp_rank, dp_rank)
|
||||
scheduler = Scheduler(
|
||||
server_args, port_args, gpu_id, tp_rank, moe_ep_rank, pp_rank, dp_rank
|
||||
)
|
||||
pipe_writer.send(
|
||||
{
|
||||
"status": "ready",
|
||||
|
||||
@@ -56,6 +56,7 @@ class TpModelWorker:
|
||||
server_args: ServerArgs,
|
||||
gpu_id: int,
|
||||
tp_rank: int,
|
||||
moe_ep_rank: int,
|
||||
pp_rank: int,
|
||||
dp_rank: Optional[int],
|
||||
nccl_port: int,
|
||||
@@ -66,6 +67,7 @@ class TpModelWorker:
|
||||
# Parse args
|
||||
self.tp_size = server_args.tp_size
|
||||
self.tp_rank = tp_rank
|
||||
self.moe_ep_rank = moe_ep_rank
|
||||
self.pp_rank = pp_rank
|
||||
|
||||
# Init model and tokenizer
|
||||
@@ -85,6 +87,8 @@ class TpModelWorker:
|
||||
gpu_id=gpu_id,
|
||||
tp_rank=tp_rank,
|
||||
tp_size=server_args.tp_size,
|
||||
moe_ep_rank=moe_ep_rank,
|
||||
moe_ep_size=server_args.ep_size,
|
||||
pp_rank=pp_rank,
|
||||
pp_size=server_args.pp_size,
|
||||
nccl_port=nccl_port,
|
||||
|
||||
@@ -58,13 +58,14 @@ class TpModelWorkerClient:
|
||||
server_args: ServerArgs,
|
||||
gpu_id: int,
|
||||
tp_rank: int,
|
||||
moe_ep_rank: int,
|
||||
pp_rank: int,
|
||||
dp_rank: Optional[int],
|
||||
nccl_port: int,
|
||||
):
|
||||
# Load the model
|
||||
self.worker = TpModelWorker(
|
||||
server_args, gpu_id, tp_rank, pp_rank, dp_rank, nccl_port
|
||||
server_args, gpu_id, tp_rank, moe_ep_rank, pp_rank, dp_rank, nccl_port
|
||||
)
|
||||
self.max_running_requests = self.worker.max_running_requests
|
||||
self.device = self.worker.device
|
||||
|
||||
@@ -157,6 +157,8 @@ class ModelRunner:
|
||||
gpu_id: int,
|
||||
tp_rank: int,
|
||||
tp_size: int,
|
||||
moe_ep_rank: int,
|
||||
moe_ep_size: int,
|
||||
pp_rank: int,
|
||||
pp_size: int,
|
||||
nccl_port: int,
|
||||
@@ -175,6 +177,8 @@ class ModelRunner:
|
||||
logger.addFilter(RankZeroFilter(tp_rank == 0))
|
||||
self.tp_rank = tp_rank
|
||||
self.tp_size = tp_size
|
||||
self.moe_ep_rank = moe_ep_rank
|
||||
self.moe_ep_size = moe_ep_size
|
||||
self.dp_size = server_args.dp_size
|
||||
self.pp_rank = pp_rank
|
||||
self.pp_size = pp_size
|
||||
@@ -549,6 +553,7 @@ class ModelRunner:
|
||||
initialize_model_parallel(
|
||||
tensor_model_parallel_size=self.tp_size,
|
||||
pipeline_model_parallel_size=self.pp_size,
|
||||
expert_model_parallel_size=self.moe_ep_size,
|
||||
duplicate_tp_group=self.server_args.enable_pdmux,
|
||||
)
|
||||
initialize_dp_attention(
|
||||
|
||||
@@ -270,14 +270,6 @@ class ServerArgs:
|
||||
sm_group_num: int = 3
|
||||
|
||||
def __post_init__(self):
|
||||
# Expert parallelism
|
||||
# We put it here first due to some internal ckpt conversation issues.
|
||||
if self.enable_ep_moe:
|
||||
self.ep_size = self.tp_size
|
||||
logger.warning(
|
||||
f"EP MoE is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[{self.tp_size}]."
|
||||
)
|
||||
|
||||
# Set missing default values
|
||||
if self.tokenizer_path is None:
|
||||
self.tokenizer_path = self.model_path
|
||||
@@ -1335,6 +1327,7 @@ class ServerArgs:
|
||||
parser.add_argument(
|
||||
"--expert-parallel-size",
|
||||
"--ep-size",
|
||||
"--ep",
|
||||
type=int,
|
||||
default=ServerArgs.ep_size,
|
||||
help="The expert parallelism size.",
|
||||
|
||||
@@ -73,6 +73,7 @@ class EAGLEWorker(TpModelWorker):
|
||||
gpu_id: int,
|
||||
tp_rank: int,
|
||||
dp_rank: Optional[int],
|
||||
moe_ep_rank: int,
|
||||
nccl_port: int,
|
||||
target_worker: TpModelWorker,
|
||||
):
|
||||
@@ -127,6 +128,7 @@ class EAGLEWorker(TpModelWorker):
|
||||
tp_rank=tp_rank,
|
||||
pp_rank=0, # FIXME
|
||||
dp_rank=dp_rank,
|
||||
moe_ep_rank=moe_ep_rank,
|
||||
nccl_port=nccl_port,
|
||||
is_draft_worker=True,
|
||||
req_to_token_pool=self.req_to_token_pool,
|
||||
|
||||
Reference in new issue
Block a user