From b9fecb8982aa6b3615934eac11bd084bdbfddc95 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 25 Oct 2026 19:08:24 -0700 Subject: [PATCH] perf(plugins): reduce repeated catalog work when listing plugins (#184072) * perf(plugins): reuse bundled catalog identities in inventory reads Prepare shipped catalog identities once while retaining live install provenance and independent list arrays. Synthetic Gateway inventory proof measured 58% less allocation and 14% less main-thread CPU per RPC; latency remains inconclusive under shared-host A/A controls. * test(plugins): type inventory benchmark fixture before publication --- docs/plugins/architecture.md | 2 + scripts/perf/plugins-list-measure.ts | 346 ++++++++++++++++++ scripts/perf/plugins-list-owner-measure.ts | 185 ++++++++++ .../official-external-install-records.test.ts | 2 + ...l-external-plugin-bundled-catalogs.test.ts | 32 +- ...official-external-plugin-catalog-source.ts | 2 - .../official-external-plugin-catalog.test.ts | 33 +- .../official-external-plugin-catalog.ts | 54 +-- 8 files changed, 598 insertions(+), 58 deletions(-) create mode 100644 scripts/perf/plugins-list-measure.ts create mode 100644 scripts/perf/plugins-list-owner-measure.ts diff --git a/docs/plugins/architecture.md b/docs/plugins/architecture.md index 4b9f9c3d11e6..c0a3260f16b2 100644 --- a/docs/plugins/architecture.md +++ b/docs/plugins/architecture.md @@ -168,6 +168,8 @@ Plugin enablement readers can pass `rootConfig` to `resolveEffectiveEnableState` Plugin inventory responses share prepared enablement and package-ownership facts between install-health diagnostics and catalog presentation. Those facts belong to one synchronous response preparation, after hosted catalog loading and before optional category loading; maintenance operations still prepare their own current ownership checks. +Bundled official catalog identities are admitted and indexed once per process. Inventory reads reuse those static entries instead of rescanning the bundled catalog for each installed plugin. Hosted catalog refresh and installed-package provenance checks keep their own lifecycles. + Runs outside the agent's resolved workspace reuse the agent's prepared plugin generation instead of loading the plugins again. Rooted background runs, including skill workshop reviews and isolated cron jobs, qualify when their bootstrap workspace resolves to that agent's resolved workspace. In a Gateway, every other run workspace also qualifies because it resolves to the same boot inventory. Their execution directory and filesystem confinement remain at the task root, including during compaction. Hosts without that shared inventory, such as direct CLI runs in another workspace, retain their own workspace-scoped generation. Tool resolution treats plugins that the prepared generation recorded as disabled or failed (for example, a memory plugin outside the selected memory slot) as settled outcomes and does not load them again for each turn. diff --git a/scripts/perf/plugins-list-measure.ts b/scripts/perf/plugins-list-measure.ts new file mode 100644 index 000000000000..e764e76c05e3 --- /dev/null +++ b/scripts/perf/plugins-list-measure.ts @@ -0,0 +1,346 @@ +/** + * Real plugins.list RPCs against an owned dist Gateway with 10,800 synthetic + * sessions and 44 synthetic plugin manifests. Never connects to a running host. + * + * Prepare once, then run each already-built revision (serial AB/BA, same fixture): + * node --import ./scripts/tsx.mjs scripts/perf/plugins-list-measure.ts --state-dir .local/plugins-list --prepare + * node --import ./scripts/tsx.mjs scripts/perf/plugins-list-measure.ts --state-dir .local/plugins-list --output .local/plugins-list-base --revision BASE --mode timing + * Optional --start-file FILE pauses after warmup until FILE exists. + * Repeat with --mode allocation separately; allocation sampling distorts timing. + * Default 1,000 requests supplies enough samples for p99. The 64 sockets remain + * connected; this targeted list workload does not claim concurrent-turn stress. + */ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + createWriteStream, + existsSync, + mkdirSync, + readFileSync, + renameSync, + writeFileSync, +} from "node:fs"; +import { createServer } from "node:net"; +import { loadavg } from "node:os"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { parseArgs } from "node:util"; +import { PROTOCOL_VERSION } from "../../packages/gateway-protocol/src/version.ts"; +import type { PluginInstallRecord } from "../../src/config/types.plugins.js"; +import { stopChild } from "../lib/gateway-bench-child.ts"; +import { + controlGatewayProfile, + measureGatewayCpuUsage, + readGatewayHeapProfile, + readGatewayResources, + startGatewayDiagnostics, +} from "../lib/gateway-bench-profile.ts"; +import { + buildGatewayBenchChildArgs, + createGatewayBenchEnv, + writeGatewayBenchConfig, + writePluginFixtures, +} from "../lib/gateway-bench-runtime.ts"; +import { createGatewayWsClient } from "../lib/gateway-ws-client.ts"; +import { generateTeamShapeStressFixture } from "./team-shape-stress-fixture.ts"; + +const { values } = parseArgs({ + options: { + "state-dir": { type: "string" }, + output: { type: "string" }, + prepare: { type: "boolean" }, + entry: { type: "string", default: "dist/entry.js" }, + revision: { type: "string" }, + "start-file": { type: "string" }, + mode: { type: "string", default: "timing" }, + samples: { type: "string", default: "1000" }, + clients: { type: "string", default: "64" }, + "poll-ms": { type: "string", default: "20" }, + port: { type: "string", default: "41805" }, + }, +}); +assert(values["state-dir"], "--state-dir is required"); +const root = path.resolve(values["state-dir"]); +const port = Number(values.port); +const samples = Number(values.samples); +const clientCount = Number(values.clients); +const pollMs = Number(values["poll-ms"]); +assert(Number.isSafeInteger(port) && port >= 40000 && port <= 65535); +assert(Number.isSafeInteger(samples) && samples > 0); +assert(Number.isSafeInteger(clientCount) && clientCount > 0); +assert(Number.isSafeInteger(pollMs) && pollMs >= 0); +assert(values.mode === "timing" || values.mode === "allocation"); +if (values.prepare) { + const marker = path.join(root, "plugins-list-fixture.json"); + const prior = existsSync(marker) ? JSON.parse(readFileSync(marker, "utf8")) : undefined; + if (prior) { + assert.equal(prior.format, "plugins-list-v1"); + } + const fixture = prior + ? { + summary: JSON.parse(readFileSync(path.join(root, "team-shape-summary.json"), "utf8")), + env: { + ...createGatewayBenchEnv(path.join(root, "home"), path.join(root, "openclaw.json"), {}), + OPENCLAW_STATE_DIR: root, + }, + } + : await generateTeamShapeStressFixture({ + stateDir: root, + port, + seed: 1805, + now: Date.parse("2026-10-25T12:00:00Z"), + transcriptMiB: 16, + }); + if (!prior?.trackedNpm) { + const savedEnv = process.env; + process.env = { + ...fixture.env, + OPENCLAW_DISABLE_BUNDLED_PLUGINS: "1", + TMPDIR: path.join(root, "tmp"), + }; + try { + const { resolvePluginNpmProjectDir } = await import("../../src/plugins/install-paths.js"); + const { resolvePluginInstallRoots } = + await import("../../src/plugins/install-root-context.js"); + const { withPluginLifecycleLease } = + await import("../../src/plugins/plugin-lifecycle-lease.js"); + const { writePersistedInstalledPluginIndexInstallRecordsWithLease } = + await import("../../src/plugins/installed-plugin-index-records.js"); + const { npmDir } = resolvePluginInstallRoots(process.env); + const config = JSON.parse(readFileSync(path.join(root, "openclaw.json"), "utf8")); + const { pluginIds, pluginsDir } = writePluginFixtures(root, { + count: 44, + activationOnStartup: false, + }); + const records: Record = {}; + for (const id of pluginIds) { + const packageName = `@fixture/${id}`; + const project = resolvePluginNpmProjectDir({ npmDir, packageName }); + const installPath = path.join(project, "node_modules", "@fixture", id); + mkdirSync(path.dirname(installPath), { recursive: true }); + renameSync(path.join(pluginsDir, id), installPath); + writeFileSync( + path.join(project, "package.json"), + JSON.stringify({ private: true, dependencies: { [packageName]: "1.0.0" } }), + ); + writeFileSync( + path.join(installPath, "package.json"), + JSON.stringify({ + name: packageName, + version: "1.0.0", + openclaw: { extensions: ["./index.cjs"] }, + }), + ); + records[id] = { + source: "npm", + spec: `${packageName}@1.0.0`, + resolvedName: packageName, + resolvedSpec: `${packageName}@1.0.0`, + version: "1.0.0", + installPath, + }; + } + config.plugins = { enabled: true, allow: pluginIds, slots: { memory: "none" } }; + writeGatewayBenchConfig(root, config, {}); + await withPluginLifecycleLease({}, (lease) => + writePersistedInstalledPluginIndexInstallRecordsWithLease(records, { config, lease }), + ); + writeFileSync( + marker, + JSON.stringify({ + format: "plugins-list-v1", + plugins: 44, + sessions: fixture.summary.sessionCount, + trackedNpm: true, + }), + ); + } finally { + process.env = savedEnv; + } + } + console.log( + JSON.stringify({ + fixture: root, + plugins: 44, + sessions: fixture.summary.sessionCount, + trackedNpm: true, + }), + ); + process.exit(0); +} +assert(values.output && values.revision, "--output and --revision are required for measurement"); +assert(existsSync(path.resolve(values.entry)), "Build the Gateway before measurement"); +const fixture = JSON.parse(readFileSync(path.join(root, "plugins-list-fixture.json"), "utf8")); +assert.equal(fixture.format, "plugins-list-v1"); +const output = path.resolve(values.output); +mkdirSync(output, { recursive: true }); +const probe = createServer(); +await new Promise((resolve, reject) => { + probe.once("error", reject); + probe.listen(port, "127.0.0.1", resolve); +}); +await new Promise((resolve, reject) => { + probe.close((error) => (error ? reject(error) : resolve())); +}); +const log = createWriteStream(path.join(output, "gateway.log")); +const startupStarted = performance.now(); +const gateway = spawn( + process.execPath, + buildGatewayBenchChildArgs(path.resolve(values.entry), port, [ + "--import", + path.resolve("scripts/perf/team-shape-stress-loader.mjs"), + "--import", + path.resolve("scripts/lib/gateway-bench-profile-preload.ts"), + ]), + { + detached: true, + stdio: ["ignore", "pipe", "pipe", "ipc"], + env: createGatewayBenchEnv(path.join(root, "home"), path.join(root, "openclaw.json"), { + startupTrace: false, + caseEnv: { + OPENCLAW_STATE_DIR: root, + OPENCLAW_DISABLE_BUNDLED_PLUGINS: "1", + OPENCLAW_SKIP_CHANNELS: "1", + OPENCLAW_PROFILE: `plugins-list-${port}`, + OPENCLAW_SYSTEMD_UNIT: `openclaw-gateway-plugins-list-${port}`, + OPENCLAW_LAUNCHD_LABEL: `ai.openclaw.plugins-list-${port}`, + TMPDIR: path.join(root, "tmp"), + }, + }), + }, +); +gateway.stdout?.pipe(log); +gateway.stderr?.pipe(log); +const clients: ReturnType[] = []; +let closed = false; +async function cleanup() { + if (closed) { + return; + } + closed = true; + for (const client of clients) { + client.close(); + } + await stopChild(gateway, { teardownGraceMs: 10_000 }); + log.end(); +} +for (const signal of ["SIGINT", "SIGTERM"] as const) { + process.once(signal, () => { + void cleanup().finally(() => process.exit(1)); + }); +} +async function rpc(client: (typeof clients)[number], method: string, params: unknown) { + const response = await client.request(method, params, 60_000); + assert(response.ok, `${method}: ${JSON.stringify(response.error)}`); + return response.payload; +} +function summarize(numbers: number[]) { + const sorted = numbers.toSorted((a, b) => a - b); + const percentile = (fraction: number) => sorted[Math.ceil(sorted.length * fraction) - 1]; + return { + count: numbers.length, + p50: percentile(0.5), + p95: percentile(0.95), + p99: percentile(0.99), + max: sorted.at(-1), + }; +} +try { + const deadline = Date.now() + 300_000; + while ( + !(await fetch(`http://127.0.0.1:${port}/readyz`).then( + (r) => r.ok, + () => false, + )) + ) { + assert(gateway.exitCode === null && gateway.signalCode === null, `Gateway exited: ${output}`); + assert(Date.now() < deadline, `Gateway readiness timeout: ${output}`); + await delay(250); + } + const startupReadyMs = performance.now() - startupStarted; + for (let index = 0; index < clientCount; index++) { + const client = createGatewayWsClient({ url: `ws://127.0.0.1:${port}` }); + clients.push(client); + await client.waitOpen(); + await rpc(client, "connect", { + minProtocol: PROTOCOL_VERSION, + maxProtocol: PROTOCOL_VERSION, + client: { + id: "gateway-client", + displayName: "synthetic-plugins-list", + version: "1.0.0", + platform: process.platform, + mode: "backend", + }, + role: "operator", + scopes: ["operator.read", "operator.admin"], + caps: [], + }); + } + const client = clients[0]!; + const firstStart = performance.now(); + const first = (await rpc(client, "plugins.list", {})) as { + plugins: Array<{ id: string; installed: boolean }>; + }; + const firstRpcMs = performance.now() - firstStart; + writeFileSync(path.join(output, "response.json"), JSON.stringify(first, null, 2)); + const responseSha256 = createHash("sha256").update(JSON.stringify(first)).digest("hex"); + assert.equal( + first.plugins.filter((plugin) => plugin.installed && plugin.id.startsWith("bench-plugin-")) + .length, + 44, + ); + for (let index = 0; index < 20; index++) { + await rpc(client, "plugins.list", {}); + } + if (values["start-file"]) { + console.log(JSON.stringify({ ready: true, waitingFor: values["start-file"] })); + const startDeadline = Date.now() + 300_000; + while (!existsSync(values["start-file"])) { + assert(Date.now() < startDeadline, "Timed out waiting for --start-file"); + await delay(100); + } + } + const hostLoadBefore = loadavg(); + const before = await readGatewayResources(gateway); + const profilePath = path.join(output, "allocation.heapprofile"); + const stopDiagnostics = await startGatewayDiagnostics(gateway); + if (values.mode === "allocation") { + await controlGatewayProfile(gateway, "heap", "start", profilePath); + } + const times: number[] = []; + for (let index = 0; index < samples; index++) { + const started = performance.now(); + await rpc(clients[index % clients.length]!, "plugins.list", {}); + times.push(performance.now() - started); + if (pollMs) { + await delay(pollMs); + } + } + if (values.mode === "allocation") { + await controlGatewayProfile(gateway, "heap", "stop", profilePath); + } + const diagnostics = await stopDiagnostics(); + const after = await readGatewayResources(gateway); + const result = { + revision: values.revision, + mode: values.mode, + fixture, + clients: clientCount, + pollMs, + startupReadyMs, + firstRpcMs, + responseSha256, + latencyMs: summarize(times), + diagnostics, + cpu: measureGatewayCpuUsage(before, after), + memory: { before: before.memory, after: after.memory }, + hostLoad: { before: hostLoadBefore, after: loadavg() }, + ...(values.mode === "allocation" ? { allocation: readGatewayHeapProfile(profilePath) } : {}), + }; + writeFileSync(path.join(output, "result.json"), JSON.stringify(result, null, 2)); + console.log(JSON.stringify(result)); +} finally { + await cleanup(); +} diff --git a/scripts/perf/plugins-list-owner-measure.ts b/scripts/perf/plugins-list-owner-measure.ts new file mode 100644 index 000000000000..57c62ecd8e47 --- /dev/null +++ b/scripts/perf/plugins-list-owner-measure.ts @@ -0,0 +1,185 @@ +// Warm plugins.list inventory: 44 tracked synthetic packages, schemas and local metadata. +// OPENCLAW_STATE_DIR=$PWD/.local/plugins-list-owner node --expose-gc --import ./scripts/tsx.mjs \ +// scripts/perf/plugins-list-owner-measure.ts timing|allocation|counts 2000 +// Alternate baseline/candidate runs. CPU and latency use untraced timing; allocation +// includes collected objects. Counts separately instruments filesystem realpath calls. +// This owner workload complements real Gateway RPC proof; it excludes remote catalogs. +import assert from "node:assert/strict"; +import fs from "node:fs"; +import { Session } from "node:inspector/promises"; +import { syncBuiltinESMExports } from "node:module"; +import { loadavg } from "node:os"; +import path from "node:path"; +import { PerformanceObserver } from "node:perf_hooks"; +import { setImmediate as nextTurn } from "node:timers/promises"; +import type { PluginInstallRecord } from "../../src/config/types.plugins.js"; +import { prepareRuntimePluginsConfig } from "../../src/plugins/config-state.js"; +import { setGatewayPluginMetadataSnapshot } from "../../src/plugins/current-plugin-metadata-snapshot.js"; +import { clearCurrentPluginMetadataSnapshot } from "../../src/plugins/current-plugin-metadata-state.js"; +import { recordInstalledPluginIndexInstallOwner } from "../../src/plugins/installed-plugin-index-install-owner.js"; +import { resolveInstalledPluginIndexPolicyHash } from "../../src/plugins/installed-plugin-index-policy.js"; +import { listManagedPlugins } from "../../src/plugins/management-service.js"; +import type { PluginManifestRecord } from "../../src/plugins/manifest-registry.types.js"; +import { finalizePluginMetadataSnapshot } from "../../src/plugins/plugin-metadata-snapshot.js"; +import { createPluginMetadataSnapshotFixture } from "../../src/plugins/plugin-metadata.test-support.js"; + +const [mode = "timing", rawIterations = "2000"] = process.argv.slice(2); +const iterations = Number(rawIterations); +assert(["timing", "allocation", "counts"].includes(mode)); +assert(Number.isSafeInteger(iterations) && iterations > 0); +assert(global.gc, "Run with --expose-gc"); +const stateDir = process.env.OPENCLAW_STATE_DIR; +assert(stateDir, "Set an isolated OPENCLAW_STATE_DIR"); +fs.mkdirSync(stateDir, { recursive: true }); +const fixtureRoot = fs.mkdtempSync(path.join(stateDir, "inventory-")); +const plugins = Array.from({ length: 44 }, (_, index) => { + const id = `synthetic-${index}`; + const rootDir = path.join(fixtureRoot, "node_modules", id); + fs.mkdirSync(rootDir, { recursive: true }); + fs.writeFileSync( + path.join(rootDir, "package.json"), + JSON.stringify({ name: id, version: "1.0.0" }), + ); + return { + id, + name: `Synthetic plugin ${index}`, + rootDir, + origin: "global" as const, + categories: ["tools"], + configSchema: { type: "object", properties: { label: { type: "string" } } }, + } satisfies Partial; +}); +const config = { + plugins: { + allow: plugins.map(({ id }) => id), + entries: Object.fromEntries( + plugins.map(({ id }) => [id, { enabled: true, config: { label: id } }]), + ), + }, +}; +prepareRuntimePluginsConfig(config); +const metadata = createPluginMetadataSnapshotFixture( + { plugins }, + resolveInstalledPluginIndexPolicyHash(config), +); +const installRecords: Record = {}; +for (const record of metadata.index.plugins) { + recordInstalledPluginIndexInstallOwner(record, record.pluginId); + record.packageName = record.pluginId; + installRecords[record.pluginId] = { + source: "npm", + spec: record.pluginId, + installPath: record.rootDir, + version: "1.0.0", + }; +} +metadata.index.installRecords = installRecords; +finalizePluginMetadataSnapshot(metadata); +setGatewayPluginMetadataSnapshot(metadata, { config }); +const params = { config, metadata, officialCatalog: { entries: [] } }; +const durations: number[] = []; +let observed = 0; +async function run(count: number, measure = false) { + for (let index = 0; index < count; index++) { + const started = performance.now(); + const result = await listManagedPlugins(params); + assert.equal(result.plugins.length, plugins.length); + observed += result.plugins.length; + if (measure) { + durations.push(performance.now() - started); + } + } +} + +await run(20); +global.gc(); +await nextTurn(); +const inspector = new Session(); +const gcDurations: number[] = []; +const observer = new PerformanceObserver((list) => { + for (const entry of list.getEntries()) { + gcDurations.push(entry.duration); + } +}); +const originalRealpath = fs.realpathSync; +let realpathCalls = 0; +try { + if (mode === "allocation") { + inspector.connect(); + await inspector.post("HeapProfiler.startSampling", { + samplingInterval: 4096, + includeObjectsCollectedByMajorGC: true, + includeObjectsCollectedByMinorGC: true, + }); + } else if (mode === "counts") { + function counted(fn: T): T { + return new Proxy(fn, { + apply(target, receiver, args) { + realpathCalls++; + return Reflect.apply(target, receiver, args); + }, + }); + } + const native = counted(originalRealpath.native); + fs.realpathSync = counted( + new Proxy(originalRealpath, { + get(target, property, receiver) { + return property === "native" ? native : Reflect.get(target, property, receiver); + }, + }), + ); + syncBuiltinESMExports(); + } else { + observer.observe({ entryTypes: ["gc"] }); + } + const hostLoad = loadavg(); + const before = process.memoryUsage(); + const cpu = process.cpuUsage(); + const start = performance.now(); + await run(iterations, mode === "timing"); + const wallMs = performance.now() - start; + const used = process.cpuUsage(cpu); + const after = process.memoryUsage(); + await nextTurn(); + await nextTurn(); + observer.disconnect(); + let sampledBytes: number | undefined; + if (mode === "allocation") { + const { profile } = await inspector.post("HeapProfiler.stopSampling"); + sampledBytes = profile.samples.reduce((sum, sample) => sum + sample.size, 0); + fs.writeFileSync(path.join(stateDir, "allocation-profile.json"), JSON.stringify(profile)); + } + global.gc(); + durations.sort((a, b) => a - b); + assert.equal(observed, (iterations + 20) * plugins.length); + console.log( + JSON.stringify({ + mode, + iterations, + plugins: plugins.length, + node: process.version, + hostLoad, + wallMs, + cpuMs: (used.user + used.system) / 1000, + p50Ms: durations[Math.floor(durations.length * 0.5)], + p99Ms: durations[Math.floor(durations.length * 0.99)], + sampledBytes, + bytesPerOperation: sampledBytes === undefined ? undefined : sampledBytes / iterations, + realpathCalls: mode === "counts" ? realpathCalls : undefined, + gc: { + count: gcDurations.length, + totalMs: gcDurations.reduce((sum, ms) => sum + ms, 0), + maxMs: Math.max(0, ...gcDurations), + }, + memory: { before, after, retainedHeapBytes: process.memoryUsage().heapUsed }, + }), + ); +} finally { + fs.realpathSync = originalRealpath; + syncBuiltinESMExports(); + observer.disconnect(); + inspector.disconnect(); + clearCurrentPluginMetadataSnapshot(); + prepareRuntimePluginsConfig(null); + fs.rmSync(fixtureRoot, { recursive: true, force: true }); +} diff --git a/src/plugins/official-external-install-records.test.ts b/src/plugins/official-external-install-records.test.ts index f03a6043ba68..886c0768af4f 100644 --- a/src/plugins/official-external-install-records.test.ts +++ b/src/plugins/official-external-install-records.test.ts @@ -121,6 +121,8 @@ describe("trusted official npm install records", () => { it("preserves canonical official updates for resolved-spec-only evidence", () => { const record: PluginInstallRecord = { source: "npm", resolvedSpec: "@openclaw/acpx@2026.7.2" }; expect(resolveNpm({ pluginId: "acpx", record })?.npmSpec).toBe("@openclaw/acpx"); + record.resolvedSpec = "@synthetic/unrelated@1.0.0"; + expect(resolveNpm({ pluginId: "acpx", record })).toBeUndefined(); }); it("returns a replacement only for a catalog-declared legacy id", () => { diff --git a/src/plugins/official-external-plugin-bundled-catalogs.test.ts b/src/plugins/official-external-plugin-bundled-catalogs.test.ts index 40ae3895cac7..ef59b5148fec 100644 --- a/src/plugins/official-external-plugin-bundled-catalogs.test.ts +++ b/src/plugins/official-external-plugin-bundled-catalogs.test.ts @@ -64,17 +64,13 @@ const fixtures = vi.hoisted(() => { const otherKind = { ...pluginShadow, name: "@fixture/other-kind", kind: "channel" }; const lowercase = { name: "@fixture/shared", openclaw: { plugin: { id: "shared" } } }; const fallback = { id: "FallbackOnly", name: "@fixture/fallback" }; - const mutable: OfficialExternalPluginCatalogEntry = { - name: "@fixture/mutable", - openclaw: { plugin: { id: "Mutable" }, providers: [{ id: "OriginalProvider" }] }, - }; return { winner, accepted, lowercase, channelCatalog: { entries: [rejected, winner, otherKind] }, providerCatalog: { entries: [shadow, accepted, lowercase] }, - pluginCatalog: { entries: [pluginShadow, fallback, mutable] }, + pluginCatalog: { entries: [pluginShadow, fallback] }, }; }); @@ -123,32 +119,6 @@ it.each([ expect(list()[0]).toBe(fixtures.winner); }); -it("observes mutations to returned rows in later lookups and projections", () => { - const entry = entryFor("Mutable")!; - const { name: originalName, openclaw: originalManifest } = entry; - try { - entry.name = "@fixture/renamed"; - entry.openclaw = { - plugin: { id: "Renamed" }, - channel: { id: "RenamedChannel", label: "Renamed" }, - }; - - expect(entryFor("Mutable")).toBeUndefined(); - expect(entryFor("OriginalProvider")).toBeUndefined(); - expect(entryFor("Renamed")).toBe(entry); - expect(packageEntry("@fixture/mutable")).toBeUndefined(); - expect(packageEntry("@fixture/renamed")).toBe(entry); - expect([isOfficial("mutable"), isOfficial("renamed")]).toEqual([false, true]); - expect(listOfficialExternalChannelCatalogEntries()).toContain(entry); - expect(listOfficialExternalProviderCatalogEntries()).not.toContain(entry); - expect(channelTarget({ config: { channels: { renamedchannel: {} } }, env: {} })).toBe(true); - expect(providerTarget({ providerIds: ["OriginalProvider"], env: {} })).toBe(false); - } finally { - entry.name = originalName; - entry.openclaw = originalManifest; - } -}); - it.each(["shadow", "rejected"])("keeps repair targets from excluded %s rows", (prefix) => { const env = (kind: string) => ({ [`${prefix}_${kind}_KEY`]: "key" }); expect(entryFor(`${prefix}-model`)).toBeUndefined(); diff --git a/src/plugins/official-external-plugin-catalog-source.ts b/src/plugins/official-external-plugin-catalog-source.ts index 98e0710781f9..dac3bc5c6730 100644 --- a/src/plugins/official-external-plugin-catalog-source.ts +++ b/src/plugins/official-external-plugin-catalog-source.ts @@ -229,8 +229,6 @@ export function filterOfficialExternalPluginCatalogEntriesBySourceRefs( }); } -// Generated source structure is stable; rows remain shared and must be read on each query. - export function dedupeOfficialExternalPluginCatalogEntries( entries: OfficialExternalPluginCatalogEntry[], ): OfficialExternalPluginCatalogEntry[] { diff --git a/src/plugins/official-external-plugin-catalog.test.ts b/src/plugins/official-external-plugin-catalog.test.ts index 6eecc9f02a80..0ba4c0ba5bc0 100644 --- a/src/plugins/official-external-plugin-catalog.test.ts +++ b/src/plugins/official-external-plugin-catalog.test.ts @@ -1,10 +1,12 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; +import * as catalogSource from "./official-external-plugin-catalog-source.js"; import { getOfficialExternalChannelSecretContract as channelSecrets, getOfficialExternalPluginCatalogEntry as findEntry, getOfficialExternalPluginCatalogEntryForPackage as findPackage, isOfficialExternalPluginId as isOfficialId, isOfficialExternalPluginCatalogFeed as isFeed, + listOfficialExternalPluginCatalogEntries as listEntries, resolveOfficialExternalProviderContractPluginIds as capabilityOwners, resolveOfficialExternalProviderPluginIds as providerOwners, resolveOfficialExternalProviderPluginIdsForEnv as providerEnvOwners, @@ -21,6 +23,35 @@ import { const sha256 = "sha256:b355dda04403becaab8bbab069fd1e7b0578262e7459e598cc5b19615b5bdab9"; describe("official external plugin catalog", () => { + it("reuses admitted bundled identities across inventory reads without sharing the list array", () => { + const entries = listEntries(); + const entry = findEntry("fish-audio-speech"); + expect(entry).toBeDefined(); + const candidates = vi.spyOn(catalogSource, "getFeedEntryInstallCandidateRecords"); + const manifests = vi.spyOn(catalogSource, "getOfficialExternalPluginCatalogManifest"); + try { + for (let read = 0; read < 3; read++) { + const listed = listEntries(); + expect(listed).not.toBe(entries); + expect(listed).toEqual(entries); + listed.length = 0; + expect(findEntry(" fish-audio-speech ")).toBe(entry); + expect(findPackage(" @openclaw/fish-audio-speech ")).toBe(entry); + expect(isOfficialId(" FISH-AUDIO-SPEECH ")).toBe(true); + expect(findEntry("FISH-AUDIO-SPEECH")).toBeUndefined(); + expect(findPackage("@openclaw/FISH-AUDIO-SPEECH")).toBeUndefined(); + expect(findEntry("synthetic-community-plugin")).toBeUndefined(); + expect(findPackage("@synthetic/community-plugin")).toBeUndefined(); + expect(isOfficialId("synthetic-community-plugin")).toBe(false); + } + expect(candidates.mock.calls.length).toBe(0); + expect(manifests.mock.calls.length).toBe(0); + } finally { + candidates.mockRestore(); + manifests.mockRestore(); + } + }); + it("routes Abliteration and either credential variable to its official package", () => { const entry = findPackage("@openclaw/abliteration-provider"); expect(entry).toBeDefined(); diff --git a/src/plugins/official-external-plugin-catalog.ts b/src/plugins/official-external-plugin-catalog.ts index 5dfabc4c4bd0..2bc194d0ceb4 100644 --- a/src/plugins/official-external-plugin-catalog.ts +++ b/src/plugins/official-external-plugin-catalog.ts @@ -62,8 +62,12 @@ const BUNDLED_CATALOG_SOURCE_REFS = new Set( Object.keys(DEFAULT_OFFICIAL_EXTERNAL_PLUGIN_CATALOG_PROFILE_CONFIG.sources ?? {}), ); -function* bundledOfficialExternalPluginCatalogEntries(): Generator { +function prepareBundledOfficialExternalPluginCatalog() { const seen = new Set(); + const entries: OfficialExternalPluginCatalogEntry[] = []; + const byId = new Map(); + const byPackage = new Map(); + const canonicalIds = new Set(); for (const entry of BUNDLED_OFFICIAL_EXTERNAL_PLUGIN_CATALOG_ENTRIES) { if ( getFeedEntryInstallCandidateRecords(entry).some( @@ -75,20 +79,30 @@ function* bundledOfficialExternalPluginCatalogEntries(): Generator boolean, -): OfficialExternalPluginCatalogEntry | undefined { - for (const entry of bundledOfficialExternalPluginCatalogEntries()) { - if (matches(entry)) { - return entry; - } - } - return undefined; +let bundledCatalog: ReturnType | undefined; + +function getBundledOfficialExternalPluginCatalog() { + // Shipped JSON has process lifetime; hosted catalogs and install provenance remain live. + return (bundledCatalog ??= prepareBundledOfficialExternalPluginCatalog()); } function formatFeedInstallCandidateSpec( @@ -302,7 +316,7 @@ export async function loadConfiguredHostedOfficialExternalPluginCatalogEntries( } export function listOfficialExternalPluginCatalogEntries(): OfficialExternalPluginCatalogEntry[] { - return [...bundledOfficialExternalPluginCatalogEntries()]; + return [...getBundledOfficialExternalPluginCatalog().entries]; } /** Returns whether an id is the canonical id of an official external plugin. */ @@ -311,18 +325,14 @@ export function isOfficialExternalPluginId(pluginId: string): boolean { if (!normalized) { return false; } - return ( - findBundledOfficialExternalPluginCatalogEntry( - (entry) => resolveOfficialExternalPluginId(entry)?.toLowerCase() === normalized, - ) !== undefined - ); + return getBundledOfficialExternalPluginCatalog().canonicalIds.has(normalized); } function resolveOfficialExternalPluginOwners( matches: (manifest: OfficialExternalPluginCatalogManifest) => boolean | undefined, ): string[] { const pluginIds = new Set(); - for (const entry of bundledOfficialExternalPluginCatalogEntries()) { + for (const entry of getBundledOfficialExternalPluginCatalog().entries) { const pluginId = resolveOfficialExternalPluginId(entry); const manifest = getOfficialExternalPluginCatalogManifest(entry); if (pluginId && manifest && matches(manifest)) { @@ -504,9 +514,7 @@ export function getOfficialExternalPluginCatalogEntry( if (!normalized) { return undefined; } - return findBundledOfficialExternalPluginCatalogEntry((entry) => - resolveOfficialExternalPluginLookupIds(entry).includes(normalized), - ); + return getBundledOfficialExternalPluginCatalog().byId.get(normalized); } export function getOfficialExternalPluginCatalogEntryForPackage( @@ -516,9 +524,7 @@ export function getOfficialExternalPluginCatalogEntryForPackage( if (!normalized) { return undefined; } - return findBundledOfficialExternalPluginCatalogEntry( - (entry) => normalizeOptionalString(entry.name) === normalized, - ); + return getBundledOfficialExternalPluginCatalog().byPackage.get(normalized); } /** Source discovery alone does not make an external package part of the core distribution. */