diff --git a/.github/scripts/ci-gate.mjs b/.github/scripts/ci-gate.mjs index 46ee5891e5..ba362113d5 100644 --- a/.github/scripts/ci-gate.mjs +++ b/.github/scripts/ci-gate.mjs @@ -14,11 +14,18 @@ export function gateFailures(needs, mode) { if (mode === "all" && !Object.values(routedJobs).every((output) => ["true", "false"].includes(needs.changes.outputs[output]))) { return ["missing or invalid job selection outputs"]; } + if (mode === "frontend" && !["true", "false"].includes(needs.changes.outputs.frontend)) { + return ["missing or invalid frontend selection output"]; + } const expected = mode === "rust" ? { "fast-checks": plan.fast, "rust-fmt-clippy": plan.rust, "rust-test": plan.rust, } : mode === "agents" ? { "fast-checks": plan.fast, "agent-checks": plan.agents, "agent-java": plan.agent_java, "agent-go": plan.agent_go_changed, "agent-rust": plan.agent_rust_changed, "agent-integration": plan.agent_integration_changed, + } : mode === "frontend" ? { + "frontend-checks": needs.changes.outputs.frontend === "true", + "frontend-typecheck": needs.changes.outputs.frontend === "true", + "frontend-test": needs.changes.outputs.frontend === "true", } : mode === "all" ? { rust: true, agents: true, "fast-checks": plan.fast, ...Object.fromEntries(Object.entries(routedJobs) diff --git a/.github/scripts/ci-plan.test.mjs b/.github/scripts/ci-plan.test.mjs index 6b6166d8de..604ce027ce 100644 --- a/.github/scripts/ci-plan.test.mjs +++ b/.github/scripts/ci-plan.test.mjs @@ -179,6 +179,22 @@ test("gates accept only successful selected jobs and intentionally skipped unsel assert.ok(gateFailures({ changes: { result: "success", outputs: { plan: "null" } } }, "rust").length); }); +test("the frontend gate checks every selected frontend job", () => { + const needs = results(["docs/README.md"]); + needs.changes.outputs.frontend = "true"; + for (const job of ["frontend-checks", "frontend-typecheck", "frontend-test"]) { + needs[job] = { result: "success" }; + } + assert.deepEqual(gateFailures(needs, "frontend"), []); + needs["frontend-test"].result = "failure"; + assert.ok(gateFailures(needs, "frontend").length); + needs.changes.outputs.frontend = "false"; + for (const job of ["frontend-checks", "frontend-typecheck", "frontend-test"]) { + needs[job] = { result: "skipped" }; + } + assert.deepEqual(gateFailures(needs, "frontend"), []); +}); + test("git diff routing includes both sides of renames, deleted files, and unusual filenames", () => { const directory = mkdtempSync(path.join(tmpdir(), "dbx-ci-plan-")); const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8", stdio: "pipe" }).trim(); diff --git a/.github/scripts/ci-vitest-file-timing-reporter.mjs b/.github/scripts/ci-vitest-file-timing-reporter.mjs new file mode 100644 index 0000000000..2d78517a49 --- /dev/null +++ b/.github/scripts/ci-vitest-file-timing-reporter.mjs @@ -0,0 +1,55 @@ +import { appendFile } from "node:fs/promises"; + +const CONSOLE_LIMIT = 20; + +export function collectFileTimings(testModules) { + return testModules + .map((testModule) => { + const diagnostic = testModule.diagnostic(); + const setupMs = diagnostic.environmentSetupDuration + diagnostic.prepareDuration + diagnostic.setupDuration; + return { + file: testModule.relativeModuleId, + state: testModule.state(), + totalMs: setupMs + diagnostic.collectDuration + diagnostic.duration, + testsMs: diagnostic.duration, + collectMs: diagnostic.collectDuration, + setupMs, + }; + }) + .sort((left, right) => right.totalMs - left.totalMs || left.file.localeCompare(right.file)); +} + +function seconds(milliseconds) { + return `${(milliseconds / 1000).toFixed(2)}s`; +} + +export function consoleReport(rows, limit = CONSOLE_LIMIT) { + const lines = ["Slowest test files:"]; + for (const row of rows.slice(0, limit)) { + lines.push(`${seconds(row.totalMs).padStart(8)} ${row.file}`); + } + return lines.join("\n"); +} + +function tableCell(value) { + return String(value).replaceAll("|", "\\|").replaceAll("\n", " "); +} + +export function markdownReport(rows, shard = "") { + const title = shard ? `## Vitest file timings (${tableCell(shard)})` : "## Vitest file timings"; + const lines = [title, "", "| Total | Tests | Collect | Setup | State | File |", "| ---: | ---: | ---: | ---: | --- | --- |"]; + for (const row of rows) { + lines.push(`| ${seconds(row.totalMs)} | ${seconds(row.testsMs)} | ${seconds(row.collectMs)} | ${seconds(row.setupMs)} | ${row.state} | \`${tableCell(row.file)}\` |`); + } + return `${lines.join("\n")}\n`; +} + +export default class FileTimingReporter { + async onTestRunEnd(testModules) { + const rows = collectFileTimings(testModules); + console.log(`\n${consoleReport(rows)}`); + if (process.env.GITHUB_STEP_SUMMARY) { + await appendFile(process.env.GITHUB_STEP_SUMMARY, markdownReport(rows, process.env.VITEST_SHARD), "utf8"); + } + } +} diff --git a/.github/scripts/ci-vitest-file-timing-reporter.test.mjs b/.github/scripts/ci-vitest-file-timing-reporter.test.mjs new file mode 100644 index 0000000000..734d3b9b48 --- /dev/null +++ b/.github/scripts/ci-vitest-file-timing-reporter.test.mjs @@ -0,0 +1,40 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { collectFileTimings, consoleReport, markdownReport } from "./ci-vitest-file-timing-reporter.mjs"; + +function testModule(file, state, diagnostic) { + return { + relativeModuleId: file, + state: () => state, + diagnostic: () => ({ + environmentSetupDuration: 0, + prepareDuration: 0, + setupDuration: 0, + collectDuration: 0, + duration: 0, + ...diagnostic, + }), + }; +} + +test("file timing reports sort total work and keep timing phases", () => { + const rows = collectFileTimings([ + testModule("fast.spec.ts", "passed", { duration: 100 }), + testModule("slow.spec.ts", "failed", { + environmentSetupDuration: 20, + prepareDuration: 30, + setupDuration: 40, + collectDuration: 50, + duration: 60, + }), + ]); + + assert.deepEqual(rows.map(({ file, totalMs, testsMs, collectMs, setupMs }) => ({ file, totalMs, testsMs, collectMs, setupMs })), [ + { file: "slow.spec.ts", totalMs: 200, testsMs: 60, collectMs: 50, setupMs: 90 }, + { file: "fast.spec.ts", totalMs: 100, testsMs: 100, collectMs: 0, setupMs: 0 }, + ]); + assert.match(consoleReport(rows, 1), /0\.20s slow\.spec\.ts/); + assert.doesNotMatch(consoleReport(rows, 1), /fast\.spec\.ts/); + assert.match(markdownReport(rows, "1\/2"), /Vitest file timings \(1\/2\)/); + assert.match(markdownReport(rows), /\| failed \| `slow\.spec\.ts` \|/); +}); diff --git a/.github/scripts/ci-workflow.test.mjs b/.github/scripts/ci-workflow.test.mjs index f8bd62f4ba..c0ece323d3 100644 --- a/.github/scripts/ci-workflow.test.mjs +++ b/.github/scripts/ci-workflow.test.mjs @@ -36,6 +36,7 @@ test("Agent and Rust matrices are bounded and do not cancel sibling failures", ( test("stable Rust, Agent and overall gates always inspect selected upstream results", () => { for (const [name, mode, dependencies] of [["rust", "rust", ["fast-checks", "rust-fmt-clippy", "rust-test"]], ["agents", "agents", ["fast-checks", "agent-checks", "agent-rust", "agent-go", "agent-integration", "agent-java"]], + ["frontend", "frontend", ["frontend-checks", "frontend-typecheck", "frontend-test"]], ["ci", "all", ["rust", "agents", "frontend", "packages", "windows-win7-bundle", "duckdb-windows-driver", "nix-packaging"]]]) { const content = job(name); assert.match(content, /if: always\(\)/); @@ -45,6 +46,16 @@ test("stable Rust, Agent and overall gates always inspect selected upstream resu } }); +test("frontend tests use two shards on separate runners", () => { + const content = job("frontend-test"); + assert.match(content, /fail-fast: false/); + assert.match(content, /shard: \[1, 2\]/); + assert.ok(content.includes("--shard=${{ matrix.shard }}/2")); + assert.ok(content.includes("--reporter=github-actions")); + assert.ok(content.includes("ci-vitest-file-timing-reporter.mjs")); + assert.doesNotMatch(job("frontend-typecheck"), /vitest|oxfmt|oxlint/); +}); + test("every old Agent stage has an independent owner and Java packaging remains strict", () => { assert.ok(job("agent-checks").includes("python3 -m unittest discover")); assert.ok(job("agent-checks").includes("python3 scripts/validate_agents.py")); @@ -61,6 +72,22 @@ test("every old Agent stage has an independent owner and Java packaging remains for (const name of ["agent-rust", "agent-go", "agent-integration", "agent-java"]) assert.doesNotMatch(job(name), /continue-on-error: true/); }); +test("native Rust driver caches exclude failed build artifacts", () => { + const content = job("agent-rust"); + assert.ok(content.includes('shared-key: ci-agent-rust-v2-${{ matrix.driver }}')); + assert.ok(content.includes("cache-on-failure: false")); +}); + +test("DuckDB Windows builds persist Rust and C++ compiler results", () => { + const content = job("duckdb-windows-driver"); + assert.ok(content.includes('SCCACHE_GHA_ENABLED: "true"')); + assert.ok(content.includes('CC: "sccache cl.exe"')); + assert.ok(content.includes('CXX: "sccache cl.exe"')); + assert.ok(content.includes("0b201ec74fa43914dc39ae48a89fd1d8cb592756")); + assert.ok(content.includes("fc920bf0ec8de6ee65d409111f7ec508035751ba")); + assert.ok(content.includes('version: "v0.16.0"')); +}); + test("the planner uses the exact event base and preserves a single workflow cancellation scope", () => { const changes = job("changes"); assert.ok(changes.includes("github.event.pull_request.base.sha || github.event.before")); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d96456d0c1..c4df280712 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ env: RUSTFLAGS: -C debuginfo=line-tables-only jobs: - frontend: + frontend-checks: needs: changes if: needs.changes.outputs.frontend == 'true' runs-on: ubuntu-22.04 @@ -38,8 +38,14 @@ jobs: - name: Install frontend dependencies run: pnpm --filter dbx... install --frozen-lockfile - - name: Frontend check - run: pnpm check + - name: Check generated connection types + run: pnpm check:connection-types + + - name: Check frontend formatting + run: pnpm exec oxfmt --check "apps/desktop/src/**/*.{ts,vue}" + + - name: Lint frontend + run: pnpm lint # Rebuild and diff the committed docs-export bundle after the frontend # checks so source changes cannot silently leave stale generated assets. @@ -56,6 +62,89 @@ jobs: run: git diff --exit-code -- crates/dbx-core/assets/ continue-on-error: true + frontend-typecheck: + needs: changes + if: needs.changes.outputs.frontend == 'true' + runs-on: ubuntu-22.04 + env: + NPM_CONFIG_LOGLEVEL: error + steps: + - uses: actions/checkout@v5 + + - name: Setup pnpm + uses: pnpm/action-setup@v6 + with: + version: 10.27.0 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: 22.13.0 + cache: pnpm + + - name: Install frontend dependencies + run: pnpm --filter dbx... install --frozen-lockfile + + - name: Type-check frontend + run: pnpm exec vue-tsc --noEmit --project apps/desktop/tsconfig.json + + frontend-test: + needs: changes + if: needs.changes.outputs.frontend == 'true' + runs-on: ubuntu-22.04 + env: + NPM_CONFIG_LOGLEVEL: error + strategy: + fail-fast: false + matrix: + shard: [1, 2] + name: frontend-test (${{ matrix.shard }}/2) + steps: + - uses: actions/checkout@v5 + + - name: Setup pnpm + uses: pnpm/action-setup@v6 + with: + version: 10.27.0 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: 22.13.0 + cache: pnpm + + - name: Install frontend dependencies + run: pnpm --filter dbx... install --frozen-lockfile + + - name: Run frontend test shard + env: + VITEST_SHARD: ${{ matrix.shard }}/2 + run: >- + pnpm exec vitest run + --shard=${{ matrix.shard }}/2 + --reporter=default + --reporter=github-actions + --reporter=./.github/scripts/ci-vitest-file-timing-reporter.mjs + + frontend: + needs: + - changes + - frontend-checks + - frontend-typecheck + - frontend-test + if: always() + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v5 + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: 22.13.0 + - name: Check selected frontend jobs + env: + NEEDS_JSON: ${{ toJSON(needs) }} + run: node .github/scripts/ci-gate.mjs frontend + github-scripts: needs: changes if: needs.changes.outputs.github_scripts == 'true' @@ -286,6 +375,11 @@ jobs: CARGO_TARGET_DIR: ${{ github.workspace }}/target/duckdb-driver RUSTFLAGS: -C debuginfo=line-tables-only -C target-feature=+crt-static RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + SCCACHE_GHA_VERSION: duckdb-windows-v1 + # libduckdb-sys uses cc-rs for its bundled C++ sources. + CC: "sccache cl.exe" + CXX: "sccache cl.exe" steps: - uses: actions/checkout@v5 @@ -295,25 +389,23 @@ jobs: toolchain: nightly-2026-07-22 components: rust-src + - name: Setup MSVC + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 + with: + arch: x64 + - uses: actions/setup-python@v5 with: python-version: "3.13" - - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 with: - version: "v0.10.0" + version: "v0.16.0" - name: Build DuckDB Windows driver - shell: bash - run: | - pushd agents/drivers/duckdb - cargo build \ - --locked \ - --release \ - --bin dbx-duckdb-driver \ - --target x86_64-win7-windows-msvc \ - -Z build-std=std,panic_abort - popd + shell: pwsh + working-directory: agents/drivers/duckdb + run: cargo build --locked --release --bin dbx-duckdb-driver --target x86_64-win7-windows-msvc -Z build-std=std,panic_abort - name: Validate DuckDB Windows driver shell: bash @@ -927,7 +1019,9 @@ jobs: uses: swatinem/rust-cache@v2 with: workspaces: agents/drivers/${{ matrix.driver }} -> target - cache-on-failure: true + # A failed bundled native build can leave link-compatible but invalid artifacts. + shared-key: ci-agent-rust-v2-${{ matrix.driver }} + cache-on-failure: false save-if: ${{ github.ref == 'refs/heads/main' }} # mold -run wraps the whole process tree (see the packages job note); # the duckdb sidecar links a large bundled C++ static library. diff --git a/apps/desktop/src/lib/plugins/pluginMarketplace.ts b/apps/desktop/src/lib/plugins/pluginMarketplace.ts index d3b7bd2fc6..1c2d98ed78 100644 --- a/apps/desktop/src/lib/plugins/pluginMarketplace.ts +++ b/apps/desktop/src/lib/plugins/pluginMarketplace.ts @@ -1,4 +1,5 @@ import type { InstalledPlugin, PluginMarketplaceArtifact, PluginMarketplacePlugin, PluginRepository, PluginRepositoryCatalogResult } from "@/types/database"; +import { uuid } from "@/lib/common/utils"; export type MarketplacePluginStatus = "install" | "installed" | "update" | "unsupported"; @@ -77,7 +78,7 @@ function installationClientId(): string { if (typeof localStorage === "undefined") return ""; let id = localStorage.getItem(INSTALLATION_ID_STORAGE_KEY); if (!id || !INSTALLATION_ID_PATTERN.test(id)) { - id = typeof crypto.randomUUID === "function" ? crypto.randomUUID() : randomUuidFallback(); + id = uuid(); localStorage.setItem(INSTALLATION_ID_STORAGE_KEY, id); } return id; @@ -86,14 +87,6 @@ function installationClientId(): string { } } -function randomUuidFallback(): string { - const hex = () => - Math.floor(Math.random() * 0x10000) - .toString(16) - .padStart(4, "0"); - return `${hex()}${hex()}-${hex()}-${hex()}-${hex()}-${hex()}${hex()}${hex()}`; -} - export type PluginInstallBeaconKind = "install" | "update"; // Fire-and-forget install beacon for marketplace statistics; never blocks or diff --git a/vitest.config.ts b/vitest.config.ts index eb8f9738b7..3381806cc0 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -22,8 +22,7 @@ export default defineConfig({ // import at once, CPU contention can stall a worker's event loop past the // old 5s default and flake deferred-promise tests. A 10s timeout absorbs // that without capping throughput, so workers can scale past 4. CI keeps - // 4 workers: its 4-vCPU runners already run vue-tsc/oxlint/oxfmt - // concurrently with vitest via `pnpm check`. + // 4 workers to match each runner's CPU count. testTimeout: 10_000, maxWorkers: process.env.CI ? 4 : 8, },