From 2ffc59590115bb99d5252ceaeb3e73f06dfcd1ae Mon Sep 17 00:00:00 2001 From: CaIon Date: Wed, 30 Sep 2026 17:13:02 +0800 Subject: [PATCH] feat(jsplugin): run task plugins on moejs Task plugins now run on github.com/Calcium-Ion/moejs v0.1.0-alpha.3 instead of Sobek, and the host utils are ported to it. The plugin API is unchanged. Code compiled from strings is no longer allowed: eval and the Function constructors throw an EvalError. Hooks receive host values without defensive copies. The engine converts an object or array when the hook first reads it and never writes JavaScript changes back to the host value. Stored protocol state and task views are passed as raw JSON and parsed inside the runtime instead of being decoded into Go maps first. An idle runtime drops its references to the last call's arguments. Whether an optional hook is exported is decided once when the plugin loads, so checking for it no longer takes an execution slot. Usage validation returns the normalized facts in a new map instead of writing into the hook result, so a hook that returns the request body no longer changes it. The request descriptor is copied instead of being round-tripped through JSON. --- AGENTS.md | 2 +- THIRD-PARTY-LICENSES.md | 272 ++++----- controller/plugin_protocol.go | 20 +- docs/plugin-api/README.md | 8 +- docs/plugin-api/v1.md | 57 +- dto/task_plugin.go | 18 +- go.mod | 17 +- go.sum | 27 +- middleware/task_plugin.go | 21 +- pkg/jsplugin/engine.go | 516 +++++++++++------- pkg/jsplugin/engine_test.go | 123 +++++ pkg/jsplugin/json_state.go | 80 +-- pkg/jsplugin/registry.go | 12 +- pkg/jsplugin/routing.go | 50 +- pkg/jsplugin/routing_test.go | 52 +- pkg/jsplugin/utils.go | 235 ++++++-- plugins/builtin_plugins_test.go | 4 +- relay/channel/task/jsplugin/adaptor.go | 130 ++++- relay/channel/task/jsplugin/adaptor_test.go | 103 +++- .../channel/task/jsplugin/performance_test.go | 8 +- relay/channel/task/jsplugin/submit_stream.go | 16 +- relay/plugin_protocol.go | 13 +- relay/plugin_protocol_test.go | 8 +- service/task_plugin_view_test.go | 25 + 24 files changed, 1206 insertions(+), 611 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7d68eb9aee..c5a9793a32 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,7 @@ This is an AI API gateway/proxy built with Go. It aggregates 40+ upstream AI pro - **Databases**: SQLite, MySQL, PostgreSQL for the primary database (all three must be supported); a separately configured log database also supports ClickHouse - **Cache**: Redis (go-redis) + in-memory cache - **Auth**: Browser sessions, API tokens and personal access tokens, JWT, WebAuthn/Passkeys, TOTP, OAuth/OIDC; Casbin authorization in `service/authz/` -- **Extensions**: JavaScript task plugins executed by Sobek; Electron desktop wrapper +- **Extensions**: JavaScript task plugins executed by moejs; Electron desktop wrapper - **Frontend package manager**: Bun (preferred over npm/yarn/pnpm) ## Architecture diff --git a/THIRD-PARTY-LICENSES.md b/THIRD-PARTY-LICENSES.md index e67e61395f..841011b4f5 100644 --- a/THIRD-PARTY-LICENSES.md +++ b/THIRD-PARTY-LICENSES.md @@ -8,142 +8,142 @@ Transitive dependencies should be audited before a final external release. ## Dependency Inventory -| Area | Scope | Ecosystem | Dependency | Version | License | -|-------------|-------------|-----------|-------------------------------------------------------|--------------------------------------|----------------------------------------------------| -| backend | production | Go | `github.com/Calcium-Ion/go-epay` | `v0.0.4` | Proprietary/Internal - owned by project maintainer | -| backend | production | Go | `github.com/abema/go-mp4` | `v1.4.1` | MIT | -| backend | test | Go | `github.com/alicebob/miniredis/v2` | `v2.38.0` | MIT | -| backend | production | Go | `github.com/andybalholm/brotli` | `v1.1.1` | MIT | -| backend | production | Go | `github.com/anknown/ahocorasick` | `v0.0.0-20190904063843-d75dbd5169c0` | MIT | -| backend | production | Go | `github.com/aws/aws-sdk-go-v2` | `v1.41.5` | Apache-2.0 | -| backend | production | Go | `github.com/aws/aws-sdk-go-v2/credentials` | `v1.19.10` | Apache-2.0 | -| backend | production | Go | `github.com/aws/aws-sdk-go-v2/service/bedrockruntime` | `v1.50.4` | Apache-2.0 | -| backend | production | Go | `github.com/aws/smithy-go` | `v1.24.2` | Apache-2.0 | -| backend | production | Go | `github.com/bytedance/gopkg` | `v0.1.3` | Apache-2.0 | -| backend | production | Go | `github.com/gin-contrib/cors` | `v1.7.2` | MIT | -| backend | production | Go | `github.com/gin-contrib/gzip` | `v0.0.6` | MIT | -| backend | production | Go | `github.com/gin-contrib/static` | `v0.0.1` | MIT | -| backend | production | Go | `github.com/gin-gonic/gin` | `v1.9.1` | MIT | -| backend | production | Go | `github.com/glebarez/sqlite` | `v1.9.0` | MIT | -| backend | production | Go | `github.com/go-audio/aiff` | `v1.1.0` | Apache-2.0 | -| backend | production | Go | `github.com/go-audio/wav` | `v1.1.0` | Apache-2.0 | -| backend | production | Go | `github.com/go-playground/validator/v10` | `v10.20.0` | MIT | -| backend | production | Go | `github.com/go-redis/redis/v8` | `v8.11.5` | BSD-2-Clause | -| backend | production | Go | `github.com/go-webauthn/webauthn` | `v0.14.0` | BSD-3-Clause | -| backend | production | Go | `github.com/golang-jwt/jwt/v5` | `v5.3.0` | MIT | -| backend | production | Go | `github.com/google/uuid` | `v1.6.0` | BSD-3-Clause | -| backend | production | Go | `github.com/gorilla/websocket` | `v1.5.0` | BSD-2-Clause | -| backend | production | Go | `github.com/grafana/pyroscope-go` | `v1.2.7` | Apache-2.0 | -| backend | production | Go | `github.com/grafana/sobek` | `v0.0.0-20260708062710-267a0e055bb4` | MIT | -| backend | production | Go | `github.com/jfreymuth/oggvorbis` | `v1.0.5` | MIT | -| backend | production | Go | `github.com/jinzhu/copier` | `v0.4.0` | MIT | -| backend | production | Go | `github.com/joho/godotenv` | `v1.5.1` | MIT | -| backend | production | Go | `github.com/mewkiz/flac` | `v1.0.13` | Unlicense | -| backend | production | Go | `github.com/nicksnyder/go-i18n/v2` | `v2.6.1` | MIT | -| backend | test | Go | `github.com/openai/openai-go` | `v1.12.0` | Apache-2.0 | -| backend | production | Go | `github.com/pkg/errors` | `v0.9.1` | BSD-2-Clause | -| backend | production | Go | `github.com/pquerna/otp` | `v1.5.0` | Apache-2.0 | -| backend | production | Go | `github.com/samber/hot` | `v0.11.0` | MIT | -| backend | production | Go | `github.com/samber/lo` | `v1.52.0` | MIT | -| backend | production | Go | `github.com/shirou/gopsutil` | `v3.21.11+incompatible` | BSD-3-Clause | -| backend | production | Go | `github.com/shopspring/decimal` | `v1.4.0` | MIT | -| backend | production | Go | `github.com/stretchr/testify` | `v1.11.1` | MIT | -| backend | production | Go | `github.com/stripe/stripe-go/v81` | `v81.4.0` | MIT | -| backend | production | Go | `github.com/tcolgate/mp3` | `v0.0.0-20170426193717-e79c5a46d300` | MIT | -| backend | production | Go | `github.com/thanhpk/randstr` | `v1.0.6` | MIT | -| backend | production | Go | `github.com/tidwall/gjson` | `v1.18.0` | MIT | -| backend | production | Go | `github.com/tidwall/sjson` | `v1.2.5` | MIT | -| backend | production | Go | `github.com/tiktoken-go/tokenizer` | `v0.6.2` | MIT | -| backend | production | Go | `github.com/waffo-com/waffo-go` | `v1.3.1` | MIT | -| backend | production | Go | `github.com/yapingcat/gomedia` | `v0.0.0-20240906162731-17feea57090c` | MIT | -| backend | production | Go | `golang.org/x/crypto` | `v0.45.0` | BSD-3-Clause | -| backend | production | Go | `golang.org/x/image` | `v0.38.0` | BSD-3-Clause | -| backend | production | Go | `golang.org/x/net` | `v0.47.0` | BSD-3-Clause | -| backend | production | Go | `golang.org/x/sync` | `v0.20.0` | BSD-3-Clause | -| backend | production | Go | `golang.org/x/sys` | `v0.38.0` | BSD-3-Clause | -| backend | production | Go | `golang.org/x/text` | `v0.35.0` | BSD-3-Clause | -| backend | production | Go | `gopkg.in/yaml.v3` | `v3.0.1` | Apache-2.0 OR MIT | -| backend | production | Go | `gorm.io/driver/mysql` | `v1.4.3` | MIT | -| backend | production | Go | `gorm.io/driver/postgres` | `v1.5.2` | MIT | -| backend | production | Go | `gorm.io/gorm` | `v1.25.2` | MIT | -| backend | production | Go | `github.com/expr-lang/expr` | `v1.17.8` | MIT | -| web | production | npm | `@base-ui/react` | `1.6.0` | MIT | -| web | production | npm | `@codemirror/lang-javascript` | `6.2.5` | MIT | -| web | production | npm | `@codemirror/lang-markdown` | `6.5.1` | MIT | -| web | production | npm | `@codemirror/language` | `6.12.4` | MIT | -| web | production | npm | `@codemirror/state` | `6.7.1` | MIT | -| web | production | npm | `@codemirror/view` | `6.43.6` | MIT | -| web | production | npm | `@fontsource-variable/lora` | `5.3.0` | OFL-1.1 | -| web | production | npm | `@fontsource-variable/public-sans` | `5.3.0` | OFL-1.1 | -| web | production | npm | `@hookform/resolvers` | `5.4.0` | MIT | -| web | production | npm | `@hugeicons/core-free-icons` | `4.2.2` | MIT | -| web | production | npm | `@hugeicons/react` | `1.1.9` | MIT | -| web | production | npm | `@lezer/highlight` | `1.2.3` | MIT | -| web | production | npm | `@lobehub/icons` | `5.14.0` | MIT | -| web | production | npm | `@tanstack/react-query` | `5.101.2` | MIT | -| web | production | npm | `@tanstack/react-router` | `1.170.18` | MIT | -| web | production | npm | `@tanstack/react-table` | `8.21.3` | MIT | -| web | production | npm | `@tanstack/react-virtual` | `3.14.6` | MIT | -| web | production | npm | `@visactor/react-vchart` | `2.1.4` | MIT | -| web | production | npm | `@visactor/vchart` | `2.1.4` | MIT | -| web | production | npm | `ai` | `7.0.31` | Apache-2.0 | -| web | production | npm | `auto-skeleton-react` | `1.0.5` | MIT | -| web | production | npm | `axios` | `1.18.1` | MIT | -| web | production | npm | `class-variance-authority` | `0.7.1` | Apache-2.0 | -| web | production | npm | `clsx` | `2.1.1` | MIT | -| web | production | npm | `cmdk` | `1.1.1` | MIT | -| web | production | npm | `dayjs` | `1.11.21` | MIT | -| web | production | npm | `dompurify` | `3.4.11` | Apache-2.0 OR MPL-2.0 | -| web | production | npm | `i18next` | `26.3.6` | MIT | -| web | production | npm | `i18next-browser-languagedetector` | `8.2.1` | MIT | -| web | production | npm | `input-otp` | `1.4.2` | MIT | -| web | production | npm | `katex` | `0.17.0` | MIT | -| web | production | npm | `lucide-react` | `1.25.0` | ISC | -| web | production | npm | `marked` | `18.0.6` | MIT | -| web | production | npm | `motion` | `12.42.2` | MIT | -| web | production | npm | `nanoid` | `5.1.16` | MIT | -| web | production | npm | `next-themes` | `0.4.6` | MIT | -| web | production | npm | `qrcode.react` | `4.2.0` | ISC | -| web | production | npm | `react` | `19.2.7` | MIT | -| web | production | npm | `react-day-picker` | `10.0.1` | MIT | -| web | production | npm | `react-dom` | `19.2.7` | MIT | -| web | production | npm | `react-hook-form` | `7.82.0` | MIT | -| web | production | npm | `react-i18next` | `17.0.10` | MIT | -| web | production | npm | `react-icons` | `5.7.0` | MIT | -| web | production | npm | `react-resizable-panels` | `4.12.2` | MIT | -| web | production | npm | `react-top-loading-bar` | `3.0.2` | MIT | -| web | production | npm | `recharts` | `3.9.1` | MIT | -| web | production | npm | `shiki` | `4.3.1` | MIT | -| web | production | npm | `sonner` | `2.0.7` | MIT | -| web | production | npm | `sse.js` | `2.8.0` | Apache-2.0 | -| web | production | npm | `stream-markdown-parser` | `1.1.3` | MIT | -| web | production | npm | `tailwind-merge` | `3.6.0` | MIT | -| web | production | npm | `tailwindcss` | `4.3.3` | MIT | -| web | production | npm | `tokenlens` | `1.3.1` | MIT | -| web | production | npm | `tw-animate-css` | `1.4.0` | MIT | -| web | production | npm | `use-stick-to-bottom` | `1.1.6` | MIT | -| web | production | npm | `vaul` | `1.1.2` | MIT | -| web | production | npm | `zod` | `4.4.3` | MIT | -| web | production | npm | `zustand` | `5.0.14` | MIT | -| web | development | npm | `@rsbuild/core` | `2.1.6` | MIT | -| web | development | npm | `@rsbuild/plugin-react` | `2.1.0` | MIT | -| web | development | npm | `@rsbuild/plugin-tailwindcss` | `2.0.3` | MIT | -| web | development | npm | `@tanstack/react-query-devtools` | `5.101.2` | MIT | -| web | development | npm | `@tanstack/react-router-devtools` | `1.167.0` | MIT | -| web | development | npm | `@tanstack/router-plugin` | `1.168.23` | MIT | -| web | development | npm | `@types/node` | `26.1.1` | MIT | -| web | development | npm | `@types/react` | `19.2.17` | MIT | -| web | development | npm | `@types/react-dom` | `19.2.3` | MIT | -| web | development | npm | `@typescript/native-preview` | `7.0.0-dev.20260707.2` | Apache-2.0 | -| web | development | npm | `@xyflow/react` | `12.11.2` | MIT | -| web | development | npm | `embla-carousel-react` | `8.6.0` | MIT | -| web | development | npm | `knip` | `6.27.0` | ISC | -| web | development | npm | `oxfmt` | `0.57.0` | MIT | -| web | development | npm | `oxlint` | `1.74.0` | MIT | -| web | development | npm | `shadcn` | `4.13.1` | MIT | -| electron | development | npm | `cross-env` | `7.0.3` | MIT | -| electron | development | npm | `electron` | `39.8.5` | MIT | -| electron | development | npm | `electron-builder` | `26.7.0` | MIT | +| Area | Scope | Ecosystem | Dependency | Version | License | +|----------|-------------|-----------|-------------------------------------------------------|--------------------------------------|----------------------------------------------------| +| backend | production | Go | `github.com/Calcium-Ion/go-epay` | `v0.0.4` | Proprietary/Internal - owned by project maintainer | +| backend | production | Go | `github.com/Calcium-Ion/moejs` | `v0.1.0-alpha.3` | Apache-2.0 | +| backend | production | Go | `github.com/abema/go-mp4` | `v1.4.1` | MIT | +| backend | test | Go | `github.com/alicebob/miniredis/v2` | `v2.38.0` | MIT | +| backend | production | Go | `github.com/andybalholm/brotli` | `v1.1.1` | MIT | +| backend | production | Go | `github.com/anknown/ahocorasick` | `v0.0.0-20190904063843-d75dbd5169c0` | MIT | +| backend | production | Go | `github.com/aws/aws-sdk-go-v2` | `v1.41.5` | Apache-2.0 | +| backend | production | Go | `github.com/aws/aws-sdk-go-v2/credentials` | `v1.19.10` | Apache-2.0 | +| backend | production | Go | `github.com/aws/aws-sdk-go-v2/service/bedrockruntime` | `v1.50.4` | Apache-2.0 | +| backend | production | Go | `github.com/aws/smithy-go` | `v1.24.2` | Apache-2.0 | +| backend | production | Go | `github.com/bytedance/gopkg` | `v0.1.3` | Apache-2.0 | +| backend | production | Go | `github.com/gin-contrib/cors` | `v1.7.2` | MIT | +| backend | production | Go | `github.com/gin-contrib/gzip` | `v0.0.6` | MIT | +| backend | production | Go | `github.com/gin-contrib/static` | `v0.0.1` | MIT | +| backend | production | Go | `github.com/gin-gonic/gin` | `v1.9.1` | MIT | +| backend | production | Go | `github.com/glebarez/sqlite` | `v1.9.0` | MIT | +| backend | production | Go | `github.com/go-audio/aiff` | `v1.1.0` | Apache-2.0 | +| backend | production | Go | `github.com/go-audio/wav` | `v1.1.0` | Apache-2.0 | +| backend | production | Go | `github.com/go-playground/validator/v10` | `v10.20.0` | MIT | +| backend | production | Go | `github.com/go-redis/redis/v8` | `v8.11.5` | BSD-2-Clause | +| backend | production | Go | `github.com/go-webauthn/webauthn` | `v0.14.0` | BSD-3-Clause | +| backend | production | Go | `github.com/golang-jwt/jwt/v5` | `v5.3.0` | MIT | +| backend | production | Go | `github.com/google/uuid` | `v1.6.0` | BSD-3-Clause | +| backend | production | Go | `github.com/gorilla/websocket` | `v1.5.0` | BSD-2-Clause | +| backend | production | Go | `github.com/grafana/pyroscope-go` | `v1.2.7` | Apache-2.0 | +| backend | production | Go | `github.com/jfreymuth/oggvorbis` | `v1.0.5` | MIT | +| backend | production | Go | `github.com/jinzhu/copier` | `v0.4.0` | MIT | +| backend | production | Go | `github.com/joho/godotenv` | `v1.5.1` | MIT | +| backend | production | Go | `github.com/mewkiz/flac` | `v1.0.13` | Unlicense | +| backend | production | Go | `github.com/nicksnyder/go-i18n/v2` | `v2.6.1` | MIT | +| backend | test | Go | `github.com/openai/openai-go` | `v1.12.0` | Apache-2.0 | +| backend | production | Go | `github.com/pkg/errors` | `v0.9.1` | BSD-2-Clause | +| backend | production | Go | `github.com/pquerna/otp` | `v1.5.0` | Apache-2.0 | +| backend | production | Go | `github.com/samber/hot` | `v0.11.0` | MIT | +| backend | production | Go | `github.com/samber/lo` | `v1.52.0` | MIT | +| backend | production | Go | `github.com/shirou/gopsutil` | `v3.21.11+incompatible` | BSD-3-Clause | +| backend | production | Go | `github.com/shopspring/decimal` | `v1.4.0` | MIT | +| backend | production | Go | `github.com/stretchr/testify` | `v1.12.1` | MIT | +| backend | production | Go | `github.com/stripe/stripe-go/v81` | `v81.4.0` | MIT | +| backend | production | Go | `github.com/tcolgate/mp3` | `v0.0.0-20170426193717-e79c5a46d300` | MIT | +| backend | production | Go | `github.com/thanhpk/randstr` | `v1.0.6` | MIT | +| backend | production | Go | `github.com/tidwall/gjson` | `v1.18.0` | MIT | +| backend | production | Go | `github.com/tidwall/sjson` | `v1.2.5` | MIT | +| backend | production | Go | `github.com/tiktoken-go/tokenizer` | `v0.6.2` | MIT | +| backend | production | Go | `github.com/waffo-com/waffo-go` | `v1.3.1` | MIT | +| backend | production | Go | `github.com/yapingcat/gomedia` | `v0.0.0-20240906162731-17feea57090c` | MIT | +| backend | production | Go | `golang.org/x/crypto` | `v0.45.0` | BSD-3-Clause | +| backend | production | Go | `golang.org/x/image` | `v0.38.0` | BSD-3-Clause | +| backend | production | Go | `golang.org/x/net` | `v0.47.0` | BSD-3-Clause | +| backend | production | Go | `golang.org/x/sync` | `v0.20.0` | BSD-3-Clause | +| backend | production | Go | `golang.org/x/sys` | `v0.38.0` | BSD-3-Clause | +| backend | production | Go | `golang.org/x/text` | `v0.35.0` | BSD-3-Clause | +| backend | production | Go | `gopkg.in/yaml.v3` | `v3.0.1` | Apache-2.0 OR MIT | +| backend | production | Go | `gorm.io/driver/mysql` | `v1.4.3` | MIT | +| backend | production | Go | `gorm.io/driver/postgres` | `v1.5.2` | MIT | +| backend | production | Go | `gorm.io/gorm` | `v1.25.2` | MIT | +| backend | production | Go | `github.com/expr-lang/expr` | `v1.17.8` | MIT | +| web | production | npm | `@base-ui/react` | `1.6.0` | MIT | +| web | production | npm | `@codemirror/lang-javascript` | `6.2.5` | MIT | +| web | production | npm | `@codemirror/lang-markdown` | `6.5.1` | MIT | +| web | production | npm | `@codemirror/language` | `6.12.4` | MIT | +| web | production | npm | `@codemirror/state` | `6.7.1` | MIT | +| web | production | npm | `@codemirror/view` | `6.43.6` | MIT | +| web | production | npm | `@fontsource-variable/lora` | `5.3.0` | OFL-1.1 | +| web | production | npm | `@fontsource-variable/public-sans` | `5.3.0` | OFL-1.1 | +| web | production | npm | `@hookform/resolvers` | `5.4.0` | MIT | +| web | production | npm | `@hugeicons/core-free-icons` | `4.2.2` | MIT | +| web | production | npm | `@hugeicons/react` | `1.1.9` | MIT | +| web | production | npm | `@lezer/highlight` | `1.2.3` | MIT | +| web | production | npm | `@lobehub/icons` | `5.14.0` | MIT | +| web | production | npm | `@tanstack/react-query` | `5.101.2` | MIT | +| web | production | npm | `@tanstack/react-router` | `1.170.18` | MIT | +| web | production | npm | `@tanstack/react-table` | `8.21.3` | MIT | +| web | production | npm | `@tanstack/react-virtual` | `3.14.6` | MIT | +| web | production | npm | `@visactor/react-vchart` | `2.1.4` | MIT | +| web | production | npm | `@visactor/vchart` | `2.1.4` | MIT | +| web | production | npm | `ai` | `7.0.31` | Apache-2.0 | +| web | production | npm | `auto-skeleton-react` | `1.0.5` | MIT | +| web | production | npm | `axios` | `1.18.1` | MIT | +| web | production | npm | `class-variance-authority` | `0.7.1` | Apache-2.0 | +| web | production | npm | `clsx` | `2.1.1` | MIT | +| web | production | npm | `cmdk` | `1.1.1` | MIT | +| web | production | npm | `dayjs` | `1.11.21` | MIT | +| web | production | npm | `dompurify` | `3.4.11` | Apache-2.0 OR MPL-2.0 | +| web | production | npm | `i18next` | `26.3.6` | MIT | +| web | production | npm | `i18next-browser-languagedetector` | `8.2.1` | MIT | +| web | production | npm | `input-otp` | `1.4.2` | MIT | +| web | production | npm | `katex` | `0.17.0` | MIT | +| web | production | npm | `lucide-react` | `1.25.0` | ISC | +| web | production | npm | `marked` | `18.0.6` | MIT | +| web | production | npm | `motion` | `12.42.2` | MIT | +| web | production | npm | `nanoid` | `5.1.16` | MIT | +| web | production | npm | `next-themes` | `0.4.6` | MIT | +| web | production | npm | `qrcode.react` | `4.2.0` | ISC | +| web | production | npm | `react` | `19.2.7` | MIT | +| web | production | npm | `react-day-picker` | `10.0.1` | MIT | +| web | production | npm | `react-dom` | `19.2.7` | MIT | +| web | production | npm | `react-hook-form` | `7.82.0` | MIT | +| web | production | npm | `react-i18next` | `17.0.10` | MIT | +| web | production | npm | `react-icons` | `5.7.0` | MIT | +| web | production | npm | `react-resizable-panels` | `4.12.2` | MIT | +| web | production | npm | `react-top-loading-bar` | `3.0.2` | MIT | +| web | production | npm | `recharts` | `3.9.1` | MIT | +| web | production | npm | `shiki` | `4.3.1` | MIT | +| web | production | npm | `sonner` | `2.0.7` | MIT | +| web | production | npm | `sse.js` | `2.8.0` | Apache-2.0 | +| web | production | npm | `stream-markdown-parser` | `1.1.3` | MIT | +| web | production | npm | `tailwind-merge` | `3.6.0` | MIT | +| web | production | npm | `tailwindcss` | `4.3.3` | MIT | +| web | production | npm | `tokenlens` | `1.3.1` | MIT | +| web | production | npm | `tw-animate-css` | `1.4.0` | MIT | +| web | production | npm | `use-stick-to-bottom` | `1.1.6` | MIT | +| web | production | npm | `vaul` | `1.1.2` | MIT | +| web | production | npm | `zod` | `4.4.3` | MIT | +| web | production | npm | `zustand` | `5.0.14` | MIT | +| web | development | npm | `@rsbuild/core` | `2.1.6` | MIT | +| web | development | npm | `@rsbuild/plugin-react` | `2.1.0` | MIT | +| web | development | npm | `@rsbuild/plugin-tailwindcss` | `2.0.3` | MIT | +| web | development | npm | `@tanstack/react-query-devtools` | `5.101.2` | MIT | +| web | development | npm | `@tanstack/react-router-devtools` | `1.167.0` | MIT | +| web | development | npm | `@tanstack/router-plugin` | `1.168.23` | MIT | +| web | development | npm | `@types/node` | `26.1.1` | MIT | +| web | development | npm | `@types/react` | `19.2.17` | MIT | +| web | development | npm | `@types/react-dom` | `19.2.3` | MIT | +| web | development | npm | `@typescript/native-preview` | `7.0.0-dev.20260707.2` | Apache-2.0 | +| web | development | npm | `@xyflow/react` | `12.11.2` | MIT | +| web | development | npm | `embla-carousel-react` | `8.6.0` | MIT | +| web | development | npm | `knip` | `6.27.0` | ISC | +| web | development | npm | `oxfmt` | `0.57.0` | MIT | +| web | development | npm | `oxlint` | `1.74.0` | MIT | +| web | development | npm | `shadcn` | `4.13.1` | MIT | +| electron | development | npm | `cross-env` | `7.0.3` | MIT | +| electron | development | npm | `electron` | `39.8.5` | MIT | +| electron | development | npm | `electron-builder` | `26.7.0` | MIT | ## License Texts diff --git a/controller/plugin_protocol.go b/controller/plugin_protocol.go index 8a16c4748c..323c9bd4ea 100644 --- a/controller/plugin_protocol.go +++ b/controller/plugin_protocol.go @@ -527,13 +527,7 @@ func streamTaskPluginProtocol( } args := []any{rendererContext, viewValue} if previous.Present { - previousValue, stateErr := previous.PluginValue() - if stateErr != nil { - logger.LogError(c, "decode task protocol state failed: "+stateErr.Error()) - writeTaskPluginProtocolFailure(c, machine, lastStatus) - return - } - args = append(args, previousValue) + args = append(args, previous.PluginValue()) } value, callErr := pinned.Plugin.Engine.CallPathWithAdmissionTimeout(observationContext, deps.admissionTimeout, "protocols", []string{pinned.Protocol, "renderEvents"}, args...) hookElapsed := deps.now().Sub(hookStarted) @@ -1166,16 +1160,14 @@ func writeTaskPluginProtocolTimeoutResponse( c.JSON(http.StatusOK, response) } -func taskPluginProtocolJSONValue(value any) (any, error) { - encoded, err := common.Marshal(value) +// taskPluginProtocolJSONValue encodes a task view for a hook, which receives +// it parsed from the text. +func taskPluginProtocolJSONValue(view dto.TaskView) (any, error) { + encoded, err := common.Marshal(view) if err != nil { return nil, err } - var decoded any - if err = common.Unmarshal(encoded, &decoded); err != nil { - return nil, err - } - return decoded, nil + return pluginruntime.RawJSON(encoded), nil } func taskPluginProtocolRendererContext( diff --git a/docs/plugin-api/README.md b/docs/plugin-api/README.md index a29db084a7..4ea382742f 100644 --- a/docs/plugin-api/README.md +++ b/docs/plugin-api/README.md @@ -126,9 +126,11 @@ the plugin's declared native routes. The generic management surface remains ## Security boundary -Plugins have no `fetch`, filesystem, `require`, imports, async functions, or -environment access. The host limits execution time, concurrency, input size, -allowed request hosts, and resolves OAuth credentials outside JavaScript. +Plugins have no `fetch`, filesystem, `require`, imports, async functions, code +compiled from strings (`eval` and the `Function` constructors throw an +`EvalError`), or environment access. The host limits execution time, +concurrency, input size, allowed request hosts, and resolves OAuth credentials +outside JavaScript. Multipart files enter JavaScript only as opaque references. This is not a hard memory-isolation boundary. A plugin sees data needed for the diff --git a/docs/plugin-api/v1.md b/docs/plugin-api/v1.md index 80e6c0cf39..be6efe014d 100644 --- a/docs/plugin-api/v1.md +++ b/docs/plugin-api/v1.md @@ -243,6 +243,42 @@ Host functions are synchronous and must bound their own work: the JavaScript execution interrupt cannot preempt a running Go function. Request-scoped data must remain owned by the host request, not by pooled runtime globals. +## Hook arguments and results + +The host passes plain JSON values to hooks without copying them: objects, +arrays, strings, finite numbers, booleans and null. The engine converts each +object or array when the hook first reads it and never writes JavaScript +changes back into the host value. Other values (structs, byte slices, +`json.Number`, raw JSON, invalid UTF-8, non-finite numbers, nil slices, or +nesting deeper than 64 levels) are first normalized through the host JSON codec. + +Plugin rules: + +- Treat every argument as read-only input. Assigning to an argument changes + only the current call's view; the host, other hooks and later calls never see + it. Return the value the host should use, and use `utils.json.clone` + (`json-clone@1`) when a mutable copy is needed. +- Do not keep arguments, or objects and arrays read from them, in module + variables after the hook returns. Runtimes are pooled across requests, so a + kept value can surface in a later request, keeps the original request's data + alive, and is still read from the host value it came from. + +Host rules: + +- Once a value has been passed to a hook, host code must not change it. A + plugin may still hold it, and reading a Go map while another goroutine writes + it is a fatal runtime error that `recover` cannot catch and that stops the + whole process. Build a new value, or copy it first (`jsonValue` in the task + adaptor copies plain JSON and normalizes everything else). +- Hook results may be host values. A hook that returns an argument, or part of + one, unchanged hands back the original host value rather than a copy. Copy a + result before editing it in place; the request descriptor's file placeholders + are replaced on a copy for this reason. +- Integral numbers created in JavaScript come back as `int64` and other numbers + as `float64`; host values returned unchanged keep their original Go type. + Code reading numbers from hook results must accept every numeric type rather + than asserting `float64`. The JSON encoding is the same either way. + ## Request body Every decoder receives one host-parsed body: @@ -404,14 +440,23 @@ when accumulating, rather than mutating host-backed arrays in place. Idle runtimes survive garbage collection in a pool bounded by the plugin's configured concurrency. The host creates additional instances on demand and discards interrupted instances. Cancellation callbacks finish before an instance -is returned to the pool. Hook bindings remain live; the host does not freeze -exported functions or object members. No JavaScript execution slot is held while +is returned to the pool, and the engine then drops its references to the +finished call's arguments, so an idle instance does not keep the last request's +data alive; values a plugin kept in module variables still keep theirs. Hook +bindings remain live; the host does not freeze +exported functions or object members. Whether an optional hook (the usage +hooks, `listArtifacts`, `buildContentRequest`) is exported is decided once, +when the plugin loads: an export that is `undefined` then counts as absent, +and assigning it later does not make the host call it. No JavaScript execution slot is held while waiting for the next upstream event. -The host may copy plain JSON containers without re-encoding their string values; -hook inputs and successive states remain separate snapshots. State size is still -checked against its actual encoded JSON bytes after every event, including -intermediate state that a later event might replace with a smaller value. +The host passes the returned state to the next event as is (see +[Hook arguments and results](#hook-arguments-and-results)); changes a hook makes +to `previousState` without returning them are not kept. State that is not plain +JSON, such as a typed array, is first normalized through its JSON encoding. +State size is still checked against its actual encoded JSON bytes after every +event, including intermediate state that a later event might replace with a +smaller value. A complete, blank-line-delimited event whose hook returns `done: true` terminates the reader and closes the upstream response. The last state is passed as diff --git a/dto/task_plugin.go b/dto/task_plugin.go index 84c2a77234..bf241975dd 100644 --- a/dto/task_plugin.go +++ b/dto/task_plugin.go @@ -9,15 +9,17 @@ type TaskPluginError struct { // TaskView is the only persisted-task shape exposed to JavaScript plugins. // It deliberately excludes ownership, channel, quota, properties, and private -// upstream identifiers. +// upstream identifiers. Fields are sorted by JSON name so the encoded view +// orders its keys as an encoded map does; hooks receive it parsed from that +// text. type TaskView struct { - TaskID string `json:"task_id"` - Platform string `json:"platform"` - Status string `json:"status"` - Progress string `json:"progress"` - FailReason string `json:"fail_reason"` CreatedAt int64 `json:"created_at"` - UpdatedAt int64 `json:"updated_at,omitempty"` - FinishedAt int64 `json:"finished_at,omitempty"` Data any `json:"data,omitempty"` + FailReason string `json:"fail_reason"` + FinishedAt int64 `json:"finished_at,omitempty"` + Platform string `json:"platform"` + Progress string `json:"progress"` + Status string `json:"status"` + TaskID string `json:"task_id"` + UpdatedAt int64 `json:"updated_at,omitempty"` } diff --git a/go.mod b/go.mod index 05e01799f4..5198f75a45 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/samber/lo v1.53.0 github.com/shirou/gopsutil v3.21.11+incompatible github.com/shopspring/decimal v1.4.0 - github.com/stretchr/testify v1.11.1 + github.com/stretchr/testify v1.12.1 github.com/stripe/stripe-go/v81 v81.4.0 github.com/tcolgate/mp3 v0.0.0-20170426193717-e79c5a46d300 github.com/thanhpk/randstr v1.0.6 @@ -72,12 +72,9 @@ require ( github.com/ClickHouse/ch-go v0.71.0 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect github.com/casbin/govaluate v1.10.0 // indirect - github.com/dlclark/regexp2/v2 v2.2.2 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect - github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect - github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e // indirect github.com/hashicorp/go-version v1.8.0 // indirect github.com/paulmach/orb v0.12.0 // indirect github.com/pierrec/lz4/v4 v4.1.25 // indirect @@ -86,13 +83,12 @@ require ( github.com/yuin/gopher-lua v1.1.1 // indirect go.opentelemetry.io/otel v1.41.0 // indirect go.opentelemetry.io/otel/trace v1.41.0 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect ) require ( github.com/Azure/go-ntlmssp v0.1.1 github.com/alicebob/miniredis/v2 v2.38.0 - github.com/grafana/sobek v0.0.0-20260708062710-267a0e055bb4 github.com/openai/openai-go v1.12.0 ) @@ -108,12 +104,11 @@ require ( github.com/bytedance/sonic/loader v0.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/expr-lang/expr v1.17.8 - github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/glebarez/go-sqlite v1.21.2 @@ -135,7 +130,7 @@ require ( github.com/jfreymuth/vorbis v1.0.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/json-iterator/go v1.1.12 // indirect + github.com/json-iterator/go v1.1.12 github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -147,10 +142,9 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/pelletier/go-toml/v2 v2.2.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/common v0.62.0 github.com/prometheus/procfs v0.15.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/samber/go-singleflightx v0.3.2 // indirect @@ -172,6 +166,7 @@ require ( ) require ( + github.com/Calcium-Ion/moejs v0.1.0-alpha.3 github.com/ClickHouse/clickhouse-go/v2 v2.46.0 github.com/QuantumNous/new-api/relaykit v0.0.0 ) diff --git a/go.sum b/go.sum index cb4c676d09..f771350993 100644 --- a/go.sum +++ b/go.sum @@ -633,6 +633,8 @@ github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Calcium-Ion/go-epay v0.0.4 h1:C96M7WfRLadcIVscWzwLiYs8etI1wrDmtFMuK2zP22A= github.com/Calcium-Ion/go-epay v0.0.4/go.mod h1:cxo/ZOg8ClvE3VAnCmEzbuyAZINSq7kFEN9oHj5WQ2U= +github.com/Calcium-Ion/moejs v0.1.0-alpha.3 h1:7PPAVdwWb3opraKZ1gtFzxCxZFSYwVbOCgsyS8+hxn4= +github.com/Calcium-Ion/moejs v0.1.0-alpha.3/go.mod h1:9YqHj36EV7pzV/vlEYSw49f3oggeJFdmqcjM9TmDpHs= github.com/ClickHouse/ch-go v0.58.2/go.mod h1:Ap/0bEmiLa14gYjCiRkYGbXvbe8vwdrfTYWhsuQ99aw= github.com/ClickHouse/ch-go v0.71.0 h1:bUdZ/EZj/LcVHsMqaRUP2holqygrPWQKeMjc6nZoyRM= github.com/ClickHouse/ch-go v0.71.0/go.mod h1:NwbNc+7jaqfY58dmdDUbG4Jl22vThgx1cYjBw0vtgXw= @@ -643,8 +645,6 @@ github.com/ClickHouse/clickhouse-go/v2 v2.46.0/go.mod h1:giJfUVlMkcfUEPVfRpt51zZ github.com/DmitriyVTitov/size v1.5.0 h1:/PzqxYrOyOUX1BXj6J9OuVRVGe+66VL4D9FlUaW515g= github.com/DmitriyVTitov/size v1.5.0/go.mod h1:le6rNI4CoLQV1b9gzp1+3d7hMAD/uu2QcJ+aYbNgiU0= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= -github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= -github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= @@ -1014,8 +1014,6 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/distribution/distribution/v3 v3.0.0-20220526142353-ffbd94cbe269/go.mod h1:28YO/VJk9/64+sTGNuYaBjWxrXTPrj0C0XmgTIOjxX4= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dlclark/regexp2/v2 v2.2.2 h1:MYWvNYw8okuqNhwTYO587EZMiDruVa2vhV6fsGpfya0= -github.com/dlclark/regexp2/v2 v2.2.2/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU= github.com/dmarkham/enumer v1.5.8/go.mod h1:d10o8R3t/gROm2p3BXqTkMt2+HMuxEmWCXzorAruYak= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= @@ -1118,8 +1116,6 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= -github.com/glebarez/sqlite v1.9.0 h1:Aj6bPA12ZEx5GbSF6XADmCkYXlljPNUY+Zf1EQxynXs= -github.com/glebarez/sqlite v1.9.0/go.mod h1:YBYCoyupOao60lzp1MVBLEjZfgkq0tdB1voAQ09K9zw= github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= github.com/go-audio/aiff v1.1.0 h1:m2LYgu/2BarpF2yZnFPWtY3Tp41k0A4y51gDRZZsEuU= @@ -1206,8 +1202,6 @@ github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBEx github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= -github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= -github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= @@ -1222,8 +1216,6 @@ github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGF github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= -github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= @@ -1388,8 +1380,6 @@ github.com/grafana/pyroscope-go v1.2.7 h1:VWBBlqxjyR0Cwk2W6UrE8CdcdD80GOFNutj0Kb github.com/grafana/pyroscope-go v1.2.7/go.mod h1:o/bpSLiJYYP6HQtvcoVKiE9s5RiNgjYTj1DhiddP2Pc= github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasnuOY813tbMN8i/a3Og= github.com/grafana/pyroscope-go/godeltaprof v0.1.9/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= -github.com/grafana/sobek v0.0.0-20260708062710-267a0e055bb4 h1:LJ2pEOxFbfUIhrkpROwZ6hLhuCn6e5GSX08v5GFwN/4= -github.com/grafana/sobek v0.0.0-20260708062710-267a0e055bb4/go.mod h1:BL/2XROA/Wtlb+zGEhkSdSZiMFIgw+D2ZdDfrbccyVE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -1770,8 +1760,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= @@ -1929,8 +1917,9 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -1946,8 +1935,8 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/stripe/stripe-go/v81 v81.4.0 h1:AuD9XzdAvl193qUCSaLocf8H+nRopOouXhxqJUzCLbw= github.com/stripe/stripe-go/v81 v81.4.0/go.mod h1:C/F4jlmnGNacvYtBp/LUHCvVUJEZffFQCobkzwY1WOo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -2173,8 +2162,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/arch v0.21.0 h1:iTC9o7+wP6cPWpDWkivCvQFGAHDQ59SrSxsLPcnkArw= golang.org/x/arch v0.21.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= diff --git a/middleware/task_plugin.go b/middleware/task_plugin.go index f80a314f49..a0d6b00ede 100644 --- a/middleware/task_plugin.go +++ b/middleware/task_plugin.go @@ -1163,7 +1163,7 @@ func renderTaskPluginQuery( for _, task := range tasks { tasksByID[task.TaskID] = task } - views := make([]map[string]any, 0, len(taskIDs)) + views := make([]dto.TaskView, 0, len(taskIDs)) for _, taskID := range taskIDs { task := tasksByID[taskID] if task == nil || !task.ResultRetrievable() { @@ -1183,25 +1183,20 @@ func renderTaskPluginQuery( abortTaskPluginRouteError(c, http.StatusInternalServerError) return } - var viewValue map[string]any - encoded, marshalErr := common.Marshal(view) - if marshalErr != nil { - abortTaskPluginRouteError(c, http.StatusInternalServerError) - return - } - if unmarshalErr := common.Unmarshal(encoded, &viewValue); unmarshalErr != nil { - abortTaskPluginRouteError(c, http.StatusInternalServerError) - return - } - views = append(views, viewValue) + views = append(views, view) } var rendererInput any = views if !multiple { rendererInput = views[0] } + encoded, err := common.Marshal(rendererInput) + if err != nil { + abortTaskPluginRouteError(c, http.StatusInternalServerError) + return + } renderStarted := time.Now() - result, err := pinned.Plugin.Engine.CallPath(c.Request.Context(), "native", []string{renderer}, requestContext.JSValue(), rendererInput) + result, err := pinned.Plugin.Engine.CallPath(c.Request.Context(), "native", []string{renderer}, requestContext.JSValue(), pluginruntime.RawJSON(encoded)) if err != nil { logger.LogDebug( c, diff --git a/pkg/jsplugin/engine.go b/pkg/jsplugin/engine.go index 4b2a2f8fb7..cb25d005b9 100644 --- a/pkg/jsplugin/engine.go +++ b/pkg/jsplugin/engine.go @@ -2,16 +2,22 @@ package jsplugin import ( "context" + "encoding" + "encoding/json" "errors" "fmt" + "maps" + "math/big" + "reflect" "regexp" "slices" "strings" + "sync" "time" + "github.com/Calcium-Ion/moejs" + "github.com/QuantumNous/new-api/common" "github.com/QuantumNous/new-api/logger" - "github.com/grafana/sobek" - "github.com/grafana/sobek/parser" ) const ( @@ -67,37 +73,6 @@ func newHookError(hook, rawMessage string, wrapped error) *HookError { return &HookError{Hook: hook, Message: message, wrapped: wrapped} } -func hookErrorFromException(hook string, exc *sobek.Exception, wrapped error) (hookErr *HookError) { - // Reading message/toString executes plugin getters, which can throw again - // and panic sobek. By this point the caller's recover is already consumed, - // so a second panic would crash the process; fall back to a blank message. - defer func() { - if recover() != nil { - hookErr = newHookError(hook, "", wrapped) - } - }() - raw := "" - if exc != nil { - if val := exc.Value(); val != nil && !sobek.IsUndefined(val) && !sobek.IsNull(val) { - gotMessage := false - if obj, ok := val.(*sobek.Object); ok { - if msg := obj.Get("message"); msg != nil && !sobek.IsUndefined(msg) && !sobek.IsNull(msg) { - raw = msg.String() - gotMessage = true - } - } - if !gotMessage { - if exported, ok := val.Export().(string); ok { - raw = exported - } else { - raw = val.String() - } - } - } - } - return newHookError(hook, raw, wrapped) -} - var forbiddenSyntax = regexp.MustCompile(`(?m)(^|[^A-Za-z0-9_$])(async|await|import)([^A-Za-z0-9_$]|$)`) type Options struct { @@ -115,14 +90,30 @@ type Engine struct { timeout time.Duration now func() time.Time log func(string) - module *sobek.SourceTextModuleRecord + module *moejs.Module pool chan *runtimeInstance semaphore chan struct{} + hooksMu sync.RWMutex + hooks map[hookKey]moejs.Hook + // exports holds the exports that were not undefined once the module + // loaded; it is written only by Compile. + exports map[string]struct{} } +// hookKey names a cached hook handle. Contract hooks are an export and at +// most two members (protocols..); longer paths only come from +// fixture requests and are resolved on every call. +type hookKey struct { + export string + members [2]string + depth int +} + +// Fixture requests name arbitrary members, so the handle cache is bounded. +const maxCachedHooks = 256 + type runtimeInstance struct { - runtime *sobek.Runtime - module sobek.ModuleInstance + runtime *moejs.Runtime logContext *runtimeLogContext } @@ -131,23 +122,21 @@ type runtimeLogContext struct { } // Compile performs upload-time syntax checks and compiles an ESM plugin once. -// All Sobek-specific module and runtime handling is intentionally kept here. +// All engine-specific module and runtime handling is intentionally kept here. func Compile(source string, options Options) (*Engine, error) { if match := forbiddenSyntax.FindString(sourceWithoutCommentsAndStrings(source)); match != "" { return nil, fmt.Errorf("unsupported plugin syntax %q: plugins must be synchronous and cannot import modules", strings.TrimSpace(match)) } - resolve := func(_ any, specifier string) (sobek.ModuleRecord, error) { - return nil, fmt.Errorf("plugin imports are disabled: %s", specifier) - } - // Plugin source is untrusted; without this option a sourceMappingURL - // comment makes the parser read arbitrary server files via os.ReadFile. - module, err := sobek.ParseModule(options.Key+".js", source, resolve, parser.WithDisableSourceMaps) + // The compiler never reads files, so a sourceMappingURL comment in + // untrusted source stays inert. + module, err := moejs.Compile(options.Key+".js", source) if err != nil { return nil, fmt.Errorf("compile plugin: %w", err) } - if err = module.Link(); err != nil { - return nil, fmt.Errorf("link plugin: %w", err) + // export ... from has no import keyword; the host links no modules. + if requests := module.Requests(); len(requests) > 0 { + return nil, fmt.Errorf("unsupported plugin syntax: re-export from %q: plugins must be synchronous and cannot import modules", requests[0]) } timeout := options.Timeout @@ -172,19 +161,26 @@ func Compile(source string, options Options) (*Engine, error) { module: module, semaphore: make(chan struct{}, concurrency), pool: make(chan *runtimeInstance, concurrency), + hooks: make(map[hookKey]moejs.Hook), + exports: make(map[string]struct{}), } instance, err := engine.newRuntime(context.Background()) if err != nil { return nil, err } + for _, name := range module.Exports() { + if value, found := instance.runtime.Export(name); found && !value.IsUndefined() { + engine.exports[name] = struct{}{} + } + } instance.logContext.context = nil engine.putRuntime(instance) return engine, nil } -// Export returns one module export without exposing Sobek values outside the +// Export returns one module export without exposing engine values outside the // engine boundary. It is used for declarative exports such as meta. -func (e *Engine) Export(ctx context.Context, exportName string) (result any, err error) { +func (e *Engine) Export(ctx context.Context, exportName string) (any, error) { select { case e.semaphore <- struct{}{}: defer func() { <-e.semaphore }() @@ -207,56 +203,47 @@ func (e *Engine) Export(ctx context.Context, exportName string) (result any, err timedOut := errors.New("plugin export timed out") stopInterrupt := watchRuntimeContext(instance.runtime, ctx, e.timeout, timedOut) defer stopInterrupt() - defer func() { - if recovered := recover(); recovered != nil { - reusable = false - switch value := recovered.(type) { - case *sobek.InterruptedError: - err = fmt.Errorf("plugin %s@%s export %s interrupted: %v", e.key, e.version, exportName, value.Value()) - case *sobek.Exception: - err = fmt.Errorf("plugin %s@%s export %s failed: %v", e.key, e.version, exportName, value) - default: - panic(recovered) - } - } - }() - value := instance.module.GetBindingValue(exportName) - if value == nil || sobek.IsUndefined(value) { + value, found := instance.runtime.Export(exportName) + if !found || value.IsUndefined() { return nil, fmt.Errorf("plugin export %q not found", exportName) } - return value.Export(), nil + result, err := instance.runtime.ToGo(value) + if err == nil { + return result, nil + } + reusable = false + var interrupted *moejs.InterruptedError + if errors.As(err, &interrupted) { + return nil, fmt.Errorf("plugin %s@%s export %s interrupted: %w", e.key, e.version, exportName, interruptCause(interrupted)) + } + var exception *moejs.Exception + errors.As(err, &exception) + return nil, fmt.Errorf("plugin %s@%s export %s failed: %w%s", e.key, e.version, exportName, err, thrownAt(instance.runtime, exception)) } -// HasExport reports whether a module export exists. Optional contract hooks -// should be detected with this method instead of relying on engine errors. -func (e *Engine) HasExport(ctx context.Context, exportName string) (bool, error) { - select { - case e.semaphore <- struct{}{}: - defer func() { <-e.semaphore }() - case <-ctx.Done(): - return false, ctx.Err() - } - instance, err := e.getRuntime(ctx) - if err != nil { - return false, err - } - defer func() { - instance.logContext.context = nil - e.putRuntime(instance) - }() - value := instance.module.GetBindingValue(exportName) - return value != nil && !sobek.IsUndefined(value), nil +// HasExport reports whether a module export was defined when the plugin +// loaded: an export that was undefined then counts as absent, and a later +// assignment is not seen. Optional contract hooks should be detected with +// this method instead of relying on engine errors. It runs no JavaScript, so +// it neither waits for nor takes a call slot. +func (e *Engine) HasExport(exportName string) bool { + _, found := e.exports[exportName] + return found } // HasCallablePath reports whether an exported value, or a nested member below // it, exists and is callable. -func (e *Engine) HasCallablePath(ctx context.Context, exportName string, members ...string) (found bool, err error) { +func (e *Engine) HasCallablePath(ctx context.Context, exportName string, members ...string) (bool, error) { select { case e.semaphore <- struct{}{}: defer func() { <-e.semaphore }() case <-ctx.Done(): return false, ctx.Err() } + hook, err := e.hook(exportName, members) + if err != nil { + return false, nil + } instance, err := e.getRuntime(ctx) if err != nil { return false, err @@ -272,28 +259,26 @@ func (e *Engine) HasCallablePath(ctx context.Context, exportName string, members timedOut := errors.New("plugin inspection timed out") stopInterrupt := watchRuntimeContext(instance.runtime, ctx, e.timeout, timedOut) defer stopInterrupt() - defer func() { - if recovered := recover(); recovered != nil { - reusable = false - hookName := strings.Join(append([]string{exportName}, members...), ".") - switch value := recovered.(type) { - case *sobek.InterruptedError: - err = fmt.Errorf("plugin %s@%s hook %s inspection interrupted: %v", e.key, e.version, hookName, value.Value()) - case *sobek.Exception: - err = fmt.Errorf("plugin %s@%s hook %s inspection failed: %v", e.key, e.version, hookName, value) - default: - panic(recovered) - } - } - }() - value, _, found := resolveExportPath(instance, exportName, members) - if !found { - return false, nil + found, err := instance.runtime.Has(hook) + if err == nil { + return found, nil } - _, callable := sobek.AssertFunction(value) - return callable, nil + reusable = false + var interrupted *moejs.InterruptedError + if errors.As(err, &interrupted) { + return false, fmt.Errorf("plugin %s@%s hook %s inspection interrupted: %w", e.key, e.version, hook.Name(), interruptCause(interrupted)) + } + var exception *moejs.Exception + errors.As(err, &exception) + return false, fmt.Errorf("plugin %s@%s hook %s inspection failed: %w%s", e.key, e.version, hook.Name(), err, thrownAt(instance.runtime, exception)) } +// RawJSON is a hook argument given as JSON text. As a top-level argument the +// hook receives JSON.parse of it, with no Go values built in between; text +// nested too deep for JSON.parse, and a RawJSON inside another argument, are +// decoded with the host codec instead. +type RawJSON []byte + // Call invokes one named module export and returns its JSON-compatible value. func (e *Engine) Call(ctx context.Context, exportName string, args ...any) (result any, err error) { return e.call(ctx, 0, exportName, nil, args...) @@ -331,12 +316,19 @@ func (e *Engine) call( exportName string, members []string, args ...any, -) (result any, err error) { - if err = e.acquireCallSlot(ctx, admissionTimeout); err != nil { +) (any, error) { + if err := e.acquireCallSlot(ctx, admissionTimeout); err != nil { return nil, err } defer func() { <-e.semaphore }() + hook, err := e.hook(exportName, members) + if err != nil { + if len(members) == 0 { + return nil, fmt.Errorf("plugin export %q not found", exportName) + } + return nil, fmt.Errorf("plugin hook %q not found", exportName) + } instance, err := e.getRuntime(ctx) if err != nil { return nil, err @@ -346,65 +338,107 @@ func (e *Engine) call( instance.runtime.ClearInterrupt() instance.logContext.context = nil if reusable { + // An idle pooled runtime must not keep this call's request + // data alive. + instance.runtime.ReleaseCallData() e.putRuntime(instance) } }() - hookName := strings.Join(append([]string{exportName}, members...), ".") + callArgs := make([]moejs.Value, 0, 4) + for index, arg := range args { + var converted moejs.Value + if raw, ok := arg.(RawJSON); !ok { + value, _ := pluginValue(arg, 0) + converted, err = instance.runtime.FromGo(value) + } else if converted, err = instance.runtime.ParseJSON(raw); err != nil { + // JSON.parse stops at a lower nesting depth than the host codec. + var decoded any + if err = common.Unmarshal(raw, &decoded); err == nil { + converted, err = instance.runtime.FromGo(decoded) + } + } + if err != nil { + return nil, fmt.Errorf("plugin %s@%s hook %s argument %d: %w", e.key, e.version, hook.Name(), index+1, err) + } + callArgs = append(callArgs, converted) + } + timedOut := errors.New("plugin call timed out") stopInterrupt := watchRuntimeContext(instance.runtime, ctx, e.timeout, timedOut) defer stopInterrupt() - defer func() { - if recovered := recover(); recovered != nil { - reusable = false - switch value := recovered.(type) { - case *sobek.InterruptedError: - err = fmt.Errorf("plugin %s@%s hook %s interrupted: %v", e.key, e.version, hookName, value.Value()) - case *sobek.Exception: - wrapped := fmt.Errorf("plugin %s@%s hook %s failed: %v", e.key, e.version, hookName, value) - err = hookErrorFromException(hookName, value, wrapped) - default: - panic(recovered) - } - } - }() - value, resolvedHookName, found := resolveExportPath(instance, exportName, members) - hookName = resolvedHookName - if !found { + value, err := instance.runtime.Call(hook, callArgs...) + var interrupted *moejs.InterruptedError + var exception *moejs.Exception + if err == nil { + var result any + if result, err = instance.runtime.ToGo(value); err == nil { + return result, nil + } + if errors.As(err, &interrupted) { + reusable = false + return nil, fmt.Errorf("plugin %s@%s hook %s interrupted: %w", e.key, e.version, hook.Name(), interruptCause(interrupted)) + } + } + switch { + case errors.Is(err, moejs.ErrHookNotFound): if len(members) == 0 { return nil, fmt.Errorf("plugin export %q not found", exportName) } + // Name the path up to the first missing member, as far as data + // properties show it; getters on the path are not run again. + hookName := exportName + current, found := instance.runtime.Export(exportName) + for _, member := range members { + if !found || current.IsNullish() { + break + } + hookName += "." + member + if !current.IsObject() { + break + } + current, found = current.AsObject().GetOwnDataValue(instance.runtime.Realm().KeyFromGoString(member)) + } return nil, fmt.Errorf("plugin hook %q not found", hookName) + case errors.Is(err, moejs.ErrNotCallable): + return nil, fmt.Errorf("plugin hook %q is not a function", hook.Name()) + case errors.As(err, &interrupted): + reusable = false + return nil, fmt.Errorf("plugin %s@%s hook %s failed: %w", e.key, e.version, hook.Name(), interruptCause(interrupted)) + case errors.As(err, &exception): + wrapped := fmt.Errorf("plugin %s@%s hook %s failed: %w%s", e.key, e.version, hook.Name(), err, thrownAt(instance.runtime, exception)) + return nil, newHookError(hook.Name(), exception.Message(), wrapped) } - if value == nil || sobek.IsUndefined(value) { - return nil, fmt.Errorf("plugin export %q not found", exportName) - } - callable, ok := sobek.AssertFunction(value) - if !ok { - return nil, fmt.Errorf("plugin hook %q is not a function", hookName) - } + reusable = false + return nil, fmt.Errorf("plugin %s@%s hook %s failed: %w", e.key, e.version, hook.Name(), err) +} - callArgs := make([]sobek.Value, len(args)) - for i, arg := range args { - callArgs[i] = instance.runtime.ToValue(arg) - } - - value, err = callable(sobek.Undefined(), callArgs...) - if err != nil { - var interrupted *sobek.InterruptedError - if errors.As(err, &interrupted) { - reusable = false - return nil, fmt.Errorf("plugin %s@%s hook %s failed: %w", e.key, e.version, hookName, err) +// hook resolves a hook path against the module's export table. Only +// declared exports resolve; the handle is cached, while the bindings it +// names are read live on every call. +func (e *Engine) hook(exportName string, members []string) (moejs.Hook, error) { + key := hookKey{export: exportName, depth: len(members)} + cacheable := len(members) <= len(key.members) + if cacheable { + copy(key.members[:], members) + e.hooksMu.RLock() + hook, found := e.hooks[key] + e.hooksMu.RUnlock() + if found { + return hook, nil } - wrapped := fmt.Errorf("plugin %s@%s hook %s failed: %w", e.key, e.version, hookName, err) - var exc *sobek.Exception - if errors.As(err, &exc) { - return nil, hookErrorFromException(hookName, exc, wrapped) - } - return nil, wrapped } - return value.Export(), nil + hook, err := e.module.Hook(exportName, members...) + if err != nil || !cacheable { + return hook, err + } + e.hooksMu.Lock() + if len(e.hooks) < maxCachedHooks { + e.hooks[key] = hook + } + e.hooksMu.Unlock() + return hook, nil } func (e *Engine) acquireCallSlot(ctx context.Context, admissionTimeout time.Duration) error { @@ -432,25 +466,141 @@ func (e *Engine) acquireCallSlot(ctx context.Context, admissionTimeout time.Dura } } -func resolveExportPath(instance *runtimeInstance, exportName string, members []string) (sobek.Value, string, bool) { - value := instance.module.GetBindingValue(exportName) - hookName := exportName - if value == nil || sobek.IsUndefined(value) || sobek.IsNull(value) { - return nil, hookName, false +// interruptCause is what stopped a runtime: the context cause +// watchRuntimeContext interrupted it with, so errors.Is sees a caller's +// cancellation through the host error. +func interruptCause(interrupted *moejs.InterruptedError) error { + if cause, ok := interrupted.Value.(error); ok { + return cause } - for _, member := range members { - hookName += "." + member - object := value.ToObject(instance.runtime) - own := slices.Contains(object.GetOwnPropertyNames(), member) - if !own { - return nil, hookName, false + return interrupted +} + +// thrownAt is the frame an Error was thrown from, as " at fn (file:line:col)", +// for host logs; it is empty for thrown values that are not Errors and for +// errors that are not exceptions (exception is nil). +func thrownAt(runtime *moejs.Runtime, exception *moejs.Exception) string { + _, frames, found := strings.Cut(runtime.StackTrace(exception), "\n at ") + if !found { + return "" + } + frame, _, _ := strings.Cut(frames, "\n") + return " at " + frame +} + +// maxPluginValueDepth bounds the argument walk; deeper values, which no +// host-built argument reaches, are passed on as they are. +const maxPluginValueDepth = 512 + +// pluginValueContainers are the container types moejs converts; named types +// with one of these underlying types are only retyped. +var pluginValueContainers = []reflect.Type{ + reflect.TypeFor[map[string]any](), + reflect.TypeFor[map[string]string](), + reflect.TypeFor[map[string][]string](), + reflect.TypeFor[[]any](), + reflect.TypeFor[[]string](), + reflect.TypeFor[[]map[string]any](), +} + +// pluginValue returns a hook argument in the shapes moejs converts: nil, +// booleans, numbers, strings, JSON-shaped maps and slices and engine values. +// Other Go values (structs, pointers, maps and slices of other element +// types) take their JSON form through the configured codec, RawJSON is +// decoded by it, and named scalars and containers become their underlying +// type. changed is false when v has supported shapes throughout, the usual +// case, so an argument is passed on without a copy. +func pluginValue(v any, depth int) (converted any, changed bool) { + if depth > maxPluginValueDepth { + return v, false + } + switch typed := v.(type) { + case nil, bool, string, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, + float32, float64, json.Number, *big.Int, []byte, []string, map[string]string, map[string][]string, + moejs.Value, moejs.NativeFunc: + return v, false + case RawJSON: + var decoded any + if err := common.Unmarshal(typed, &decoded); err != nil { + return v, false } - value = object.Get(member) - if value == nil || sobek.IsUndefined(value) || sobek.IsNull(value) { - return nil, hookName, false + return decoded, true + case map[string]any: + var copied map[string]any + for key, item := range typed { + if item, changed = pluginValue(item, depth+1); changed { + if copied == nil { + copied = maps.Clone(typed) + } + copied[key] = item + } + } + if copied == nil { + return v, false + } + return copied, true + case []any: + var copied []any + for index, item := range typed { + if item, changed = pluginValue(item, depth+1); changed { + if copied == nil { + copied = slices.Clone(typed) + } + copied[index] = item + } + } + if copied == nil { + return v, false + } + return copied, true + case []map[string]any: + var copied []map[string]any + for index, item := range typed { + if next, changed := pluginValue(item, depth+1); changed { + if copied == nil { + copied = slices.Clone(typed) + } + copied[index] = next.(map[string]any) + } + } + if copied == nil { + return v, false + } + return copied, true + } + _, marshaler := v.(json.Marshaler) + _, textMarshaler := v.(encoding.TextMarshaler) + if !marshaler && !textMarshaler { + value := reflect.ValueOf(v) + switch value.Kind() { + case reflect.Bool: + return value.Bool(), true + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return value.Int(), true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return value.Uint(), true + case reflect.Float32, reflect.Float64: + return value.Float(), true + case reflect.String: + return value.String(), true + case reflect.Map, reflect.Slice: + for _, container := range pluginValueContainers { + if value.Type().ConvertibleTo(container) { + converted, _ = pluginValue(value.Convert(container).Interface(), depth) + return converted, true + } + } } } - return value, hookName, true + data, err := common.Marshal(v) + if err != nil { + return v, false + } + var decoded any + if err = common.Unmarshal(data, &decoded); err != nil { + return v, false + } + return decoded, true } // Idle runtimes are bounded by the execution limit and survive GC. Start with @@ -474,7 +624,7 @@ func (e *Engine) getRuntime(ctx context.Context) (*runtimeInstance, error) { // A timeout callback must finish before its runtime can be reused. Merely // stopping a timer does not wait for an already-started Interrupt call. -func watchRuntimeContext(runtime *sobek.Runtime, ctx context.Context, timeout time.Duration, timeoutError error) func() { +func watchRuntimeContext(runtime *moejs.Runtime, ctx context.Context, timeout time.Duration, timeoutError error) func() { callContext, cancel := context.WithTimeoutCause(ctx, timeout, timeoutError) interrupted := make(chan struct{}) stop := context.AfterFunc(callContext, func() { @@ -489,8 +639,10 @@ func watchRuntimeContext(runtime *sobek.Runtime, ctx context.Context, timeout ti } } -func (e *Engine) newRuntime(ctx context.Context) (instance *runtimeInstance, err error) { - runtime := sobek.New() +func (e *Engine) newRuntime(ctx context.Context) (*runtimeInstance, error) { + // Code compiled from strings would bypass the upload-time syntax checks, + // so eval and the Function constructors throw an EvalError. + runtime := moejs.NewRuntime(moejs.Options{DisableDynamicCode: true}) logContext := &runtimeLogContext{context: ctx} logOutput := e.log if logOutput == nil { @@ -505,31 +657,17 @@ func (e *Engine) newRuntime(ctx context.Context) (instance *runtimeInstance, err } timedOut := errors.New("plugin initialization timed out") stopInterrupt := watchRuntimeContext(runtime, ctx, e.timeout, timedOut) - defer func() { - stopInterrupt() - runtime.ClearInterrupt() - }() - defer func() { - if recovered := recover(); recovered != nil { - if interrupted, ok := recovered.(*sobek.InterruptedError); ok { - instance = nil - err = fmt.Errorf("initialize plugin %s@%s: %v", e.key, e.version, interrupted.Value()) - return - } - panic(recovered) - } - }() - promise := runtime.CyclicModuleRecordEvaluate(e.module, func(_ any, specifier string) (sobek.ModuleRecord, error) { - return nil, fmt.Errorf("plugin imports are disabled: %s", specifier) - }) - if promise.State() != sobek.PromiseStateFulfilled { - return nil, fmt.Errorf("evaluate plugin: %v", promise.Result().Export()) + err := runtime.Load(e.module) + stopInterrupt() + runtime.ClearInterrupt() + var interrupted *moejs.InterruptedError + if errors.As(err, &interrupted) { + return nil, fmt.Errorf("initialize plugin %s@%s: %w", e.key, e.version, interruptCause(interrupted)) } - return &runtimeInstance{ - runtime: runtime, - module: runtime.GetModuleInstance(e.module), - logContext: logContext, - }, nil + if err != nil { + return nil, fmt.Errorf("evaluate plugin: %w", err) + } + return &runtimeInstance{runtime: runtime, logContext: logContext}, nil } func sourceWithoutCommentsAndStrings(source string) string { diff --git a/pkg/jsplugin/engine_test.go b/pkg/jsplugin/engine_test.go index 5de737672c..91f19cb009 100644 --- a/pkg/jsplugin/engine_test.go +++ b/pkg/jsplugin/engine_test.go @@ -3,6 +3,7 @@ package jsplugin import ( "bytes" "context" + "encoding/json" "errors" "io" "math" @@ -14,6 +15,8 @@ import ( "testing" "time" "unicode/utf8" + "unsafe" + "weak" "github.com/QuantumNous/new-api/common" "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil" @@ -107,6 +110,72 @@ export function invalid(kind) { } } +func TestEngineRawJSONArgumentsMatchDecodedArguments(t *testing.T) { + engine, err := Compile(` +export function describe(...values) { + const seen = []; + const stack = values.reverse().map((v) => ({v})); + while (stack.length > 0) { + const {v, key, close} = stack.pop(); + if (close) { + seen.push(close); + continue; + } + if (key !== undefined) seen.push(key); + if (v === null || typeof v !== "object") { + seen.push(typeof v + ":" + (Object.is(v, -0) ? "-0" : JSON.stringify(v))); + continue; + } + const array = Array.isArray(v); + seen.push(array ? "[" : Object.getPrototypeOf(v) === Object.prototype ? "{" : "{?"); + stack.push({close: array ? "]" : "}"}); + const keys = Object.keys(v); + for (let i = keys.length - 1; i >= 0; i--) { + stack.push({v: v[keys[i]], key: array ? undefined : JSON.stringify(keys[i])}); + } + } + return seen.join(" "); +}`, Options{}) + require.NoError(t, err) + // Hosts pass text written by common.Marshal, where both paths agree on + // key order, numbers and strings. + values := map[string]any{ + "object": map[string]any{ + "__proto__": map[string]any{"polluted": true}, + "10": "ten", "2": "two", "a": []any{}, "b": map[string]any{}, + "numbers": []any{0, math.Copysign(0, -1), 1e21, int64(123456789012345678), 1.5e-7, -12, math.MaxInt64, math.SmallestNonzeroFloat64}, + "strings": []any{"", "<>&

", "é中😀", "\u0000\t\"\\", "a\xffb"}, + "nested": map[string]any{"z": nil, "y": []any{true, false, map[string]any{"x": []any{[]any{1}}}}}, + }, + "array": []any{map[string]any{"id": "x", "b": 1, "a": 2}, nil, "s"}, + "string": "plain", + "number": 42.5, + "null": nil, + "deep": json.RawMessage(strings.Repeat("[", 600) + strings.Repeat("]", 600)), + } + for name, value := range values { + t.Run(name, func(t *testing.T) { + text, err := common.Marshal(value) + require.NoError(t, err) + var decoded any + require.NoError(t, common.Unmarshal(text, &decoded)) + want, err := engine.Call(t.Context(), "describe", decoded) + require.NoError(t, err) + got, err := engine.Call(t.Context(), "describe", RawJSON(text)) + require.NoError(t, err) + assert.Equal(t, want, got) + nested, err := engine.Call(t.Context(), "describe", map[string]any{"value": RawJSON(text)}) + require.NoError(t, err) + wantNested, err := engine.Call(t.Context(), "describe", map[string]any{"value": decoded}) + require.NoError(t, err) + assert.Equal(t, wantNested, nested) + }) + } + + _, err = engine.Call(t.Context(), "describe", "first", RawJSON(`{"a":`)) + require.EqualError(t, err, "plugin @ hook describe argument 2: unexpected end of JSON input") +} + func TestJSONStateChangesAndEncodedLimits(t *testing.T) { const changes = `[ [{"op":"set","path":[],"value":{"text":"<","parts":[],"nested":{"value":null}}}], @@ -388,6 +457,8 @@ func TestCompileRejectsAsynchronousAndImportedPlugins(t *testing.T) { "static import": `import value from "dependency"; export function run() { return value; }`, "dynamic import": `export function run() { return import("dependency"); }`, "top-level await": `const value = await work(); export function run() { return value; }`, + "export from": `export { value as run } from "dependency";`, + "export star": `export * from "dependency"; export function run() {}`, } { t.Run(name, func(t *testing.T) { _, err := Compile(source, Options{Key: "invalid"}) @@ -398,6 +469,18 @@ func TestCompileRejectsAsynchronousAndImportedPlugins(t *testing.T) { _, err := Compile(`export function run() { return "import async await"; }`, Options{Key: "valid"}) require.NoError(t, err) + + for name, body := range map[string]string{ + "eval": `return eval("import('dependency')");`, + "Function constructor": `return Function("return async function () {}")();`, + } { + engine, err := Compile(`export function run() { `+body+` }`, Options{Key: "dynamic"}) + require.NoError(t, err, name) + _, err = engine.Call(context.Background(), "run") + assert.ErrorContains(t, err, "EvalError", name) + } + _, err = Compile(`eval("1"); export function run() {}`, Options{Key: "dynamic"}) + assert.ErrorContains(t, err, "EvalError") } func TestCompileIgnoresSourceMapDirectives(t *testing.T) { @@ -445,6 +528,22 @@ func TestEngineDoesNotReinitializeIdleRuntimeAfterGC(t *testing.T) { assert.Equal(t, int64(1), initialized.Load()) } +func TestEngineIdleRuntimeReleasesLastCallArguments(t *testing.T) { + engine, err := Compile(`export function run(body){return body.image.length;}`, Options{Concurrency: 1}) + require.NoError(t, err) + image := strings.Repeat("A", 1<<20) + released := weak.Make(unsafe.StringData(image)) + value, err := engine.Call(t.Context(), "run", map[string]any{"image": image}) + require.NoError(t, err) + assert.Equal(t, int64(1<<20), value) + runtime.GC() + runtime.GC() + assert.Nil(t, released.Value(), "an idle pooled runtime must not keep the last request body alive") + value, err = engine.Call(t.Context(), "run", map[string]any{"image": "AB"}) + require.NoError(t, err) + assert.Equal(t, int64(2), value) +} + func TestEngineBoundsConcurrentCallsAndCancelsAnOccupiedRuntime(t *testing.T) { entered := make(chan struct{}, 2) release := make(chan struct{}) @@ -495,6 +594,30 @@ export function replace(){run=function(){return 2;};native.render=function(){ret assert.Equal(t, int64(3), value) } +func TestEngineHasExportUsesBindingsAsLoaded(t *testing.T) { + engine, err := Compile(` +export let later; +export const unset = undefined; +export function run(){return 1;} +export function assign(){later=function(){return 2;};} +`, Options{Concurrency: 1}) + require.NoError(t, err) + + engine.semaphore <- struct{}{} + assert.True(t, engine.HasExport("run")) + assert.False(t, engine.HasExport("later")) + assert.False(t, engine.HasExport("unset")) + assert.False(t, engine.HasExport("missing")) + <-engine.semaphore + + _, err = engine.Call(t.Context(), "assign") + require.NoError(t, err) + assert.False(t, engine.HasExport("later")) + value, err := engine.Call(t.Context(), "later") + require.NoError(t, err) + assert.Equal(t, int64(2), value) +} + func TestEngineHookErrorExtractsSanitizedJSMessage(t *testing.T) { t.Parallel() tests := []struct { diff --git a/pkg/jsplugin/json_state.go b/pkg/jsplugin/json_state.go index 1961b9b63a..50978a1bc9 100644 --- a/pkg/jsplugin/json_state.go +++ b/pkg/jsplugin/json_state.go @@ -4,12 +4,11 @@ import ( "context" "fmt" "math" - "strconv" "strings" "unicode/utf8" + "github.com/Calcium-Ion/moejs/engine" "github.com/QuantumNous/new-api/common" - "github.com/grafana/sobek" ) const ( @@ -47,6 +46,8 @@ type jsonStateBudget struct { ctx context.Context bytes int nodes int + // realm reads JavaScript values; only json.clone passes them. + realm *engine.Realm } func (b *jsonStateBudget) spend(bytes, nodes int) error { @@ -65,41 +66,58 @@ func newJSONStateNode(value any, depth int, budget *jsonStateBudget) (*jsonState if depth > maxJSONToolDepth { return nil, fmt.Errorf("JSON state exceeds depth limit") } - if js, ok := value.(sobek.Value); ok { - if sobek.IsUndefined(js) { + if js, ok := value.(engine.Value); ok { + if js.IsUndefined() { return nil, fmt.Errorf("undefined is not a JSON value") } - if object, ok := js.(*sobek.Object); ok { - switch object.ClassName() { - case "Object": - keys := object.Keys() + if !js.IsObject() { + value = budget.realm.ToGo(js) + } else { + // Getters and proxy traps run while the value is read; what they + // throw is returned as it is, not as a JSON error. + object := js.AsObject() + switch { + case object.ClassName() == "Object" && !object.IsCallable(): + keys, err := budget.realm.EnumerableOwnKeys(object) + if err != nil { + return nil, err + } if len(keys) > budget.nodes { return nil, fmt.Errorf("JSON object exceeds node limit") } fields := make(map[string]any, len(keys)) for _, key := range keys { - fields[key] = object.Get(key) + if fields[key.GoString()], err = object.GetProp(budget.realm, key); err != nil { + return nil, err + } } value = fields - case "Array": - length := object.Get("length").ToInteger() - if length < 0 || length > int64(budget.nodes) { + case object.ClassName() == "Array": + length, err := budget.realm.LengthOfArrayLike(object) + if err != nil { + return nil, err + } + if length > int64(budget.nodes) { return nil, fmt.Errorf("JSON array exceeds node limit") } items := make([]any, int(length)) for index := range items { - item := object.Get(strconv.Itoa(index)) - if item == nil { + key := engine.IndexKey(uint32(index)) + present, err := budget.realm.HasOwn(object, key) + if err != nil { + return nil, err + } + if !present { return nil, fmt.Errorf("sparse arrays are not JSON values") } - items[index] = item + if items[index], err = object.GetProp(budget.realm, key); err != nil { + return nil, err + } } value = items default: return nil, fmt.Errorf("json.clone accepts only plain objects, arrays and JSON scalars") } - } else { - value = js.Export() } } beforeBytes, beforeNodes := budget.bytes, budget.nodes @@ -397,32 +415,32 @@ func (n *jsonStateNode) value() any { return n.scalar } -// Native JS containers are essential here: wrapping Go slices in ToValue would -// retain host-backed array mutation behavior instead of producing a JS clone. -func (n *jsonStateNode) jsValue(runtime *sobek.Runtime) (sobek.Value, error) { +// Native JS containers are essential here: a lazily converted Go map would +// read as a clone, but the result must be ordinary objects a plugin owns. +func (n *jsonStateNode) jsValue(realm *engine.Realm) (engine.Value, error) { if n.object != nil { - result := runtime.NewObject() + result := realm.NewObject() for key, child := range n.object { - value, err := child.jsValue(runtime) + value, err := child.jsValue(realm) if err != nil { - return nil, err + return engine.Undefined(), err } - if err = result.DefineDataProperty(key, value, sobek.FLAG_TRUE, sobek.FLAG_TRUE, sobek.FLAG_TRUE); err != nil { - return nil, err + if err = result.CreateDataPropertyOrThrow(realm, realm.KeyFromGoString(key), value); err != nil { + return engine.Undefined(), err } } - return result, nil + return engine.ObjectValue(result), nil } if n.array != nil { - items := make([]any, len(n.array)) + items := make([]engine.Value, len(n.array)) for index, child := range n.array { - value, err := child.jsValue(runtime) + value, err := child.jsValue(realm) if err != nil { - return nil, err + return engine.Undefined(), err } items[index] = value } - return runtime.NewArray(items...), nil + return engine.ObjectValue(realm.NewArrayFromSlice(items)), nil } - return runtime.ToValue(n.value()), nil + return realm.FromGo(n.value()) } diff --git a/pkg/jsplugin/registry.go b/pkg/jsplugin/registry.go index 521f5687d5..344f92c242 100644 --- a/pkg/jsplugin/registry.go +++ b/pkg/jsplugin/registry.go @@ -345,11 +345,7 @@ func CompilePlugin(source string, options Options) (*LoadedPlugin, error) { } artifactHooks := make(map[string]bool, 2) for _, hook := range []string{"listArtifacts", "buildContentRequest"} { - exported, exportErr := engine.HasExport(context.Background(), hook) - if exportErr != nil { - return nil, exportErr - } - if !exported { + if !engine.HasExport(hook) { continue } callable, callableErr := engine.HasCallablePath(context.Background(), hook) @@ -497,11 +493,7 @@ func CompilePlugin(source string, options Options) (*LoadedPlugin, error) { } } for _, removed := range []string{"resolveRequest", "renderError", "renderers"} { - has, e := engine.HasExport(context.Background(), removed) - if e != nil { - return nil, e - } - if has { + if engine.HasExport(removed) { return nil, fmt.Errorf("plugin %s export %q is no longer supported", meta.Key, removed) } } diff --git a/pkg/jsplugin/routing.go b/pkg/jsplugin/routing.go index 5d02638df2..567066d6d8 100644 --- a/pkg/jsplugin/routing.go +++ b/pkg/jsplugin/routing.go @@ -290,52 +290,24 @@ type RouteRequestContext struct { RequestBody any `json:"-"` } +// JSValue shares the request with hooks without copying it: the engine never +// writes JavaScript changes back into Go values, so the request must only stay +// unchanged while a hook runs. Missing params and query still reach hooks as +// empty objects rather than null. func (r RouteRequestContext) JSValue() map[string]any { - params := make(map[string]string, len(r.Params)) - maps.Copy(params, r.Params) - query := make(map[string][]string, len(r.Query)) - for key, values := range r.Query { - query[key] = append([]string(nil), values...) + params, query := r.Params, r.Query + if params == nil { + params = map[string]string{} + } + if query == nil { + query = map[string][]string{} } return map[string]any{ "path": r.Path, "method": r.Method, "params": params, "query": query, - "body": clonePluginRequestValue(r.Body), - } -} - -func clonePluginRequestValue(value any) any { - switch typed := value.(type) { - case map[string]any: - cloned := make(map[string]any, len(typed)) - for key, item := range typed { - cloned[key] = clonePluginRequestValue(item) - } - return cloned - case []any: - cloned := make([]any, len(typed)) - for index, item := range typed { - cloned[index] = clonePluginRequestValue(item) - } - return cloned - case []string: - return append([]string(nil), typed...) - case map[string][]string: - cloned := make(map[string][]string, len(typed)) - for key, values := range typed { - cloned[key] = append([]string(nil), values...) - } - return cloned - case []map[string]any: - cloned := make([]map[string]any, len(typed)) - for index, item := range typed { - cloned[index] = clonePluginRequestValue(item).(map[string]any) - } - return cloned - default: - return value + "body": r.Body, } } diff --git a/pkg/jsplugin/routing_test.go b/pkg/jsplugin/routing_test.go index 77a1e40d6c..fa3d950bc7 100644 --- a/pkg/jsplugin/routing_test.go +++ b/pkg/jsplugin/routing_test.go @@ -464,36 +464,30 @@ func TestRouteRetainResultDecodeAndValidation(t *testing.T) { } } -func TestRouteRequestContextClonesFormAndMultipartValuesPerDecoder(t *testing.T) { - tests := []struct { - name string - body any - mutate func(map[string]any) - }{ - { - name: "form fields", - body: map[string]any{"kind": "form", "fields": map[string][]string{"prompt": {"original"}}}, - mutate: func(value map[string]any) { - value["body"].(map[string]any)["fields"].(map[string][]string)["prompt"][0] = "mutated" - }, - }, - { - name: "multipart files", - body: map[string]any{"kind": "multipart", "files": []map[string]any{{"ref": "request_file:image", "filename": "safe.png"}}}, - mutate: func(value map[string]any) { - value["body"].(map[string]any)["files"].([]map[string]any)[0]["filename"] = "mutated.png" - }, - }, - } - for _, testCase := range tests { - t.Run(testCase.name, func(t *testing.T) { - request := RouteRequestContext{Body: testCase.body} - first := request.JSValue() - testCase.mutate(first) - second := request.JSValue() - assert.NotEqual(t, first, second) - }) +func TestRouteRequestContextKeepsFormAndMultipartValuesAcrossDecoderWrites(t *testing.T) { + plugin, err := CompilePlugin(routingTestPluginSource("route-request-writes", 0, `["gpt-5.5"]`, "", ` +export function decodeJob(ctx) { + const seen = { + prompt: ctx.body.fields.prompt[0], + filename: ctx.body.files[0].filename, + params: Object.keys(ctx.params).length, + query: Object.keys(ctx.query).length + }; + ctx.body.fields.prompt[0] = "mutated"; + ctx.body.files[0].filename = "mutated.png"; + return seen; +}`), Options{}) + require.NoError(t, err) + fields := map[string][]string{"prompt": {"original"}} + files := []map[string]any{{"ref": "request_file:image", "filename": "safe.png"}} + request := RouteRequestContext{Body: map[string]any{"kind": "multipart", "fields": fields, "files": files}} + for range 2 { + seen, err := plugin.Engine.Call(context.Background(), "decodeJob", request.JSValue()) + require.NoError(t, err) + assert.Equal(t, map[string]any{"prompt": "original", "filename": "safe.png", "params": int64(0), "query": int64(0)}, seen) } + assert.Equal(t, "original", fields["prompt"][0]) + assert.Equal(t, "safe.png", files[0]["filename"]) } func TestRegistryNoOpMutationsKeepCurrentGeneration(t *testing.T) { diff --git a/pkg/jsplugin/utils.go b/pkg/jsplugin/utils.go index c945babc7b..7bc541bf81 100644 --- a/pkg/jsplugin/utils.go +++ b/pkg/jsplugin/utils.go @@ -6,15 +6,17 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" + "errors" "fmt" + "math" "net/url" "sort" "strings" "time" + "github.com/Calcium-Ion/moejs" "github.com/golang-jwt/jwt/v5" "github.com/google/uuid" - "github.com/grafana/sobek" ) type volcSignRequest struct { @@ -29,65 +31,206 @@ type volcSignRequest struct { Timestamp int64 `json:"timestamp"` } -func injectGlobals(runtime *sobek.Runtime, identity func() string, now func() time.Time, logOutput func(string)) error { +func injectGlobals(runtime *moejs.Runtime, identity func() string, now func() time.Time, logOutput func(string)) error { utils := map[string]any{ - "hasCapability": HasCapability, + "hasCapability": runtime.Function("hasCapability", 1, func(r *moejs.Realm, _ moejs.Value, args []moejs.Value) (moejs.Value, error) { + name, err := stringValue(r, moejs.Arg(args, 0)) + if err != nil { + return moejs.Undefined(), err + } + return moejs.Bool(HasCapability(name)), nil + }), "json": map[string]any{ - "clone": func(call sobek.FunctionCall) sobek.Value { - if sobek.IsUndefined(call.Argument(0)) { - panic(runtime.NewTypeError("json.clone requires a JSON value")) + "clone": runtime.Function("clone", 1, func(r *moejs.Realm, _ moejs.Value, args []moejs.Value) (moejs.Value, error) { + if moejs.Arg(args, 0).IsUndefined() { + return moejs.Undefined(), r.TypeError("json.clone requires a JSON value") } - budget := &jsonStateBudget{ctx: context.Background(), bytes: MaxJSONToolBytes, nodes: maxJSONToolNodes} - value, err := newJSONStateNode(call.Argument(0), 0, budget) + budget := &jsonStateBudget{ctx: context.Background(), bytes: MaxJSONToolBytes, nodes: maxJSONToolNodes, realm: r} + value, err := newJSONStateNode(moejs.Arg(args, 0), 0, budget) if err != nil { - panic(runtime.NewTypeError("json.clone: %s", err)) + var exception *moejs.Exception + var interrupted *moejs.InterruptedError + if errors.As(err, &exception) || errors.As(err, &interrupted) { + return moejs.Undefined(), err + } + return moejs.Undefined(), r.TypeError("json.clone: %s", err) } - cloned, err := value.jsValue(runtime) + return value.jsValue(r) + }), + }, + "unixNow": runtime.Function("unixNow", 0, func(*moejs.Realm, moejs.Value, []moejs.Value) (moejs.Value, error) { + return moejs.Int(now().Unix()), nil + }), + "jwtSignHS256": runtime.Function("jwtSignHS256", 2, func(r *moejs.Realm, _ moejs.Value, args []moejs.Value) (moejs.Value, error) { + var claims map[string]any + if argument := moejs.Arg(args, 0); !argument.IsNullish() { + exported, err := r.ToGoStrict(argument) if err != nil { - panic(runtime.NewGoError(err)) + return moejs.Undefined(), err } - return cloned - }, - }, - "unixNow": func() int64 { return now().Unix() }, - "jwtSignHS256": func(claims map[string]any, secret string) (string, error) { - return jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims(claims)).SignedString([]byte(secret)) - }, - "hmacSHA256": func(message, secret string) string { + var ok bool + if claims, ok = exported.(map[string]any); !ok { + return moejs.Undefined(), r.TypeError("jwtSignHS256 claims must be an object") + } + } + secret, err := stringValue(r, moejs.Arg(args, 1)) + if err != nil { + return moejs.Undefined(), err + } + token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims(claims)).SignedString([]byte(secret)) + if err != nil { + return moejs.Undefined(), err + } + return moejs.String(token), nil + }), + "hmacSHA256": runtime.Function("hmacSHA256", 2, func(r *moejs.Realm, _ moejs.Value, args []moejs.Value) (moejs.Value, error) { + message, err := stringValue(r, moejs.Arg(args, 0)) + if err != nil { + return moejs.Undefined(), err + } + secret, err := stringValue(r, moejs.Arg(args, 1)) + if err != nil { + return moejs.Undefined(), err + } mac := hmac.New(sha256.New, []byte(secret)) _, _ = mac.Write([]byte(message)) - return hex.EncodeToString(mac.Sum(nil)) - }, - "base64": func(value string) string { return base64.StdEncoding.EncodeToString([]byte(value)) }, - "base64URL": func(value string) string { - return base64.RawURLEncoding.EncodeToString([]byte(value)) - }, - "base64URLDecode": func(value string) (string, error) { + return moejs.String(hex.EncodeToString(mac.Sum(nil))), nil + }), + "base64": runtime.Function("base64", 1, func(r *moejs.Realm, _ moejs.Value, args []moejs.Value) (moejs.Value, error) { + value, err := stringValue(r, moejs.Arg(args, 0)) + if err != nil { + return moejs.Undefined(), err + } + return moejs.String(base64.StdEncoding.EncodeToString([]byte(value))), nil + }), + "base64URL": runtime.Function("base64URL", 1, func(r *moejs.Realm, _ moejs.Value, args []moejs.Value) (moejs.Value, error) { + value, err := stringValue(r, moejs.Arg(args, 0)) + if err != nil { + return moejs.Undefined(), err + } + return moejs.String(base64.RawURLEncoding.EncodeToString([]byte(value))), nil + }), + "base64URLDecode": runtime.Function("base64URLDecode", 1, func(r *moejs.Realm, _ moejs.Value, args []moejs.Value) (moejs.Value, error) { + value, err := stringValue(r, moejs.Arg(args, 0)) + if err != nil { + return moejs.Undefined(), err + } decoded, err := base64.RawURLEncoding.DecodeString(value) - return string(decoded), err - }, - "uuid": func() string { return uuid.NewString() }, - "volcSignV4": func(request volcSignRequest) (map[string]string, error) { - return signVolcV4(request, now) - }, + if err != nil { + return moejs.Undefined(), err + } + return moejs.String(string(decoded)), nil + }), + "uuid": runtime.Function("uuid", 0, func(*moejs.Realm, moejs.Value, []moejs.Value) (moejs.Value, error) { + return moejs.String(uuid.NewString()), nil + }), + "volcSignV4": runtime.Function("volcSignV4", 1, func(r *moejs.Realm, _ moejs.Value, args []moejs.Value) (moejs.Value, error) { + // Fields are read by their Go names, as plugins have always + // passed them. + var request volcSignRequest + argument := moejs.Arg(args, 0) + if !argument.IsNullish() && !argument.IsObject() { + return moejs.Undefined(), r.TypeError("volcSignV4 request must be an object") + } + if argument.IsObject() { + object := argument.AsObject() + fields := []struct { + name string + target *string + }{ + {"Method", &request.Method}, {"URL", &request.URL}, {"Body", &request.Body}, + {"AccessKey", &request.AccessKey}, {"SecretKey", &request.SecretKey}, + {"Region", &request.Region}, {"Service", &request.Service}, + } + for _, field := range fields { + value, err := object.GetProp(r, r.KeyFromGoString(field.name)) + if err != nil { + return moejs.Undefined(), err + } + if *field.target, err = stringValue(r, value); err != nil { + return moejs.Undefined(), err + } + } + headers, err := object.GetProp(r, r.KeyFromGoString("Headers")) + if err != nil { + return moejs.Undefined(), err + } + if headers.IsObject() { + keys, err := r.EnumerableOwnKeys(headers.AsObject()) + if err != nil { + return moejs.Undefined(), err + } + request.Headers = make(map[string]string, len(keys)) + for _, key := range keys { + value, err := headers.AsObject().GetProp(r, key) + if err != nil { + return moejs.Undefined(), err + } + if request.Headers[key.GoString()], err = stringValue(r, value); err != nil { + return moejs.Undefined(), err + } + } + } + timestamp, err := object.GetProp(r, r.KeyFromGoString("Timestamp")) + if err != nil { + return moejs.Undefined(), err + } + if !timestamp.IsNullish() { + seconds, err := r.ToNumber(timestamp) + if err != nil { + return moejs.Undefined(), err + } + if !math.IsNaN(seconds) && !math.IsInf(seconds, 0) { + request.Timestamp = int64(seconds) + } + } + } + signed, err := signVolcV4(request, now) + if err != nil { + return moejs.Undefined(), err + } + return r.FromGo(signed) + }), } - if err := runtime.Set("utils", utils); err != nil { + if err := runtime.SetGlobal("utils", utils); err != nil { return err } - console := runtime.NewObject() - if err := console.Set("log", func(call sobek.FunctionCall) sobek.Value { - parts := make([]string, len(call.Arguments)) - for i, argument := range call.Arguments { - parts[i] = argument.String() - } - if logOutput != nil { - logOutput(identity() + " " + strings.Join(parts, " ")) - } - return sobek.Undefined() - }); err != nil { - return err + return runtime.SetGlobal("console", map[string]any{ + "log": runtime.Function("log", 0, func(r *moejs.Realm, _ moejs.Value, args []moejs.Value) (moejs.Value, error) { + parts := make([]string, len(args)) + for i, argument := range args { + if argument.IsSymbol() { + parts[i] = argument.String() + continue + } + text, err := r.ToString(argument) + if err != nil { + return moejs.Undefined(), err + } + parts[i] = text.GoString() + } + if logOutput != nil { + logOutput(identity() + " " + strings.Join(parts, " ")) + } + return moejs.Undefined(), nil + }), + }) +} + +// stringValue reads a host function's string parameter: undefined and null +// read as "", other values as JavaScript's String(value). +func stringValue(r *moejs.Realm, value moejs.Value) (string, error) { + if value.IsNullish() { + return "", nil } - return runtime.Set("console", console) + if value.IsString() { + return value.AsString().GoString(), nil + } + text, err := r.ToString(value) + if err != nil { + return "", err + } + return text.GoString(), nil } func signVolcV4(request volcSignRequest, now func() time.Time) (map[string]string, error) { diff --git a/plugins/builtin_plugins_test.go b/plugins/builtin_plugins_test.go index f743929b25..730f2c9637 100644 --- a/plugins/builtin_plugins_test.go +++ b/plugins/builtin_plugins_test.go @@ -121,9 +121,7 @@ func TestBuiltInTaskPluginResponsesAndUsageContracts(t *testing.T) { assert.True(t, callable, hook) } for _, hook := range []string{"extractUsage", "extractUsageOnComplete"} { - callable, callableErr := plugin.Engine.HasExport(t.Context(), hook) - require.NoError(t, callableErr) - assert.True(t, callable, hook) + assert.True(t, plugin.Engine.HasExport(hook), hook) } require.NotEmpty(t, plugin.Meta.UsageSchema) for usageKey, schema := range plugin.Meta.UsageSchema { diff --git a/relay/channel/task/jsplugin/adaptor.go b/relay/channel/task/jsplugin/adaptor.go index 43152fb00f..0c3bcffca3 100644 --- a/relay/channel/task/jsplugin/adaptor.go +++ b/relay/channel/task/jsplugin/adaptor.go @@ -184,10 +184,11 @@ func (a *TaskAdaptor) ExtractUsageFactsValidated(c *gin.Context, info *relaycomm if !ok { return nil, fmt.Errorf("plugin usage hook must return an object") } - if _, err = a.validatedUsageRatios(facts, info.UpstreamModelName, info.OriginModelName); err != nil { + validated, _, err := a.validatedUsageRatios(facts, info.UpstreamModelName, info.OriginModelName) + if err != nil { return nil, err } - return facts, nil + return validated, nil } func (a *TaskAdaptor) AdjustBillingOnSubmit(info *relaycommon.RelayInfo, taskData []byte) map[string]float64 { @@ -612,8 +613,8 @@ func (a *TaskAdaptor) FetchTask(baseURL, key string, task *model.Task, proxy str } func (a *TaskAdaptor) doFetchDescriptor(baseURL, proxy string, value any) (*http.Response, error) { - var descriptor requestDescriptor - if err := convert(value, &descriptor); err != nil { + descriptor, err := decodeRequestDescriptor(value) + if err != nil { return nil, err } if err := pluginruntime.ValidateRequestURL(descriptor.URL, baseURL, a.plugin.Meta.AllowedHosts); err != nil { @@ -862,7 +863,11 @@ func (a *TaskAdaptor) ConvertToOpenAIVideo(task *model.Task) ([]byte, error) { if err != nil { return nil, err } - value, err := a.plugin.Engine.CallPath(context.Background(), "protocols", []string{"openai_video", "render"}, map[string]any{"protocol": "openai_video", "operation": "retrieve"}, jsonValue(view)) + encodedView, err := common.Marshal(view) + if err != nil { + return nil, err + } + value, err := a.plugin.Engine.CallPath(context.Background(), "protocols", []string{"openai_video", "render"}, map[string]any{"protocol": "openai_video", "operation": "retrieve"}, pluginruntime.RawJSON(encodedView)) if err != nil { return nil, err } @@ -932,8 +937,8 @@ func (a *TaskAdaptor) BuildContentRequest(task *model.Task, artifactKey string, if err != nil { return nil, err } - var descriptor requestDescriptor - if err = convert(value, &descriptor); err != nil { + descriptor, err := decodeRequestDescriptor(value) + if err != nil { return nil, err } method := strings.ToUpper(strings.TrimSpace(descriptor.Method)) @@ -1208,8 +1213,8 @@ func (a *TaskAdaptor) buildSubmit(c *gin.Context, info *relaycommon.RelayInfo) ( ) return nil, err } - var descriptor requestDescriptor - if err = convert(value, &descriptor); err != nil { + descriptor, err := decodeRequestDescriptor(value) + if err != nil { logger.LogDebug( c, "task_plugin subsystem=adaptor event=build_submit_failed plugin=%q stage=build_submit_request reason=invalid_descriptor elapsed_ms=%d", @@ -1299,7 +1304,7 @@ func (a *TaskAdaptor) submitContext(c *gin.Context, info *relaycommon.RelayInfo) } } if taskRequest, exists := c.Get("task_request"); exists { - routeRequest.RequestBody = jsonValue(taskRequest) + routeRequest.RequestBody = taskRequest } if c.Request != nil { if routeRequest.Path == "" { @@ -1340,7 +1345,11 @@ func (a *TaskAdaptor) submitContext(c *gin.Context, info *relaycommon.RelayInfo) a.files = append(a.files[:0], files...) } ctx := routeRequest.JSValue() - ctx["requestBody"] = jsonValue(routeRequest.RequestBody) + requestBody := routeRequest.RequestBody + if !isPlainJSONValue(requestBody, 0) { + requestBody = jsonValue(requestBody) + } + ctx["requestBody"] = requestBody ctx["requestHeaders"] = requestHeaders ctx["files"] = files ctx["action"] = info.Action @@ -1393,7 +1402,7 @@ func (a *TaskAdaptor) usageRatios(ctx context.Context, models []string, hook str logger.LogDebug(ctx, "task_plugin subsystem=adaptor event=usage_hook_failed plugin=%q hook=%q reason=result_not_object elapsed_ms=%d", a.plugin.Meta.Key, hook, time.Since(started).Milliseconds()) return nil, fmt.Errorf("plugin usage hook must return an object") } - ratios, err := a.validatedUsageRatios(facts, models...) + _, ratios, err := a.validatedUsageRatios(facts, models...) if err != nil { logger.LogDebug(ctx, "task_plugin subsystem=adaptor event=usage_hook_failed plugin=%q hook=%q reason=invalid_usage elapsed_ms=%d", a.plugin.Meta.Key, hook, time.Since(started).Milliseconds()) return nil, err @@ -1442,17 +1451,22 @@ func (a *TaskAdaptor) validateResolvedUsageValue(value any, usageSchema map[stri return nil } -func (a *TaskAdaptor) validatedUsageRatios(facts map[string]any, models ...string) (map[string]float64, error) { +// validatedUsageRatios returns the normalized facts in a new map because the +// hook result may be a host value, such as the request body, that must stay +// unchanged. +func (a *TaskAdaptor) validatedUsageRatios(facts map[string]any, models ...string) (map[string]any, map[string]float64, error) { usageSchema, _ := a.plugin.Meta.UsageForModels(models...) + validated := make(map[string]any, len(facts)) ratios := make(map[string]float64) for key, value := range facts { + validated[key] = value if schema, declared := usageSchema[key]; declared { number, err := validateUsageValue(value, schema, false) if err != nil { - return nil, err + return nil, nil, err } if schema.Type == "number" { - facts[key] = number + validated[key] = number if number > 0 { ratios[key] = number } @@ -1471,16 +1485,16 @@ func (a *TaskAdaptor) validatedUsageRatios(facts map[string]any, models ...strin limit = relaycommon.MaxTaskDurationSeconds } if err := validateUsageNumberLimit(number, limit); err != nil { - return nil, err + return nil, nil, err } // Keep numeric facts usable by existing expressions even when the // selected profile no longer declares that legacy extension field. - facts[key] = number + validated[key] = number if number > 0 { ratios[key] = number } } - return ratios, nil + return validated, ratios, nil } func (a *TaskAdaptor) validatedCompletionUsageFacts(facts any, models ...string) (map[string]any, error) { @@ -1627,10 +1641,43 @@ func (a *TaskAdaptor) logRejectedUsage(hook string, _ error) { common.SysError(fmt.Sprintf("task plugin %s rejected invalid %s billing facts", a.plugin.Meta.Key, hook)) } -func (a *TaskAdaptor) hasHook(ctx context.Context, hook string) bool { - has, err := a.plugin.Engine.HasExport(ctx, hook) - return err == nil && has +func (a *TaskAdaptor) hasHook(_ context.Context, hook string) bool { + return a.plugin.Engine.HasExport(hook) } + +// decodeRequestDescriptor reads the result of a request-building hook. Fields +// other than body keep the codec's semantics: case-insensitive names, and type +// errors reject the descriptor. A plain JSON body is copied instead of encoded +// and decoded again; anything else takes the codec path unchanged. +func decodeRequestDescriptor(value any) (requestDescriptor, error) { + var descriptor requestDescriptor + fields, ok := value.(map[string]any) + if !ok { + return descriptor, convert(value, &descriptor) + } + body, plain := fields["body"] + for key := range fields { + // The codec picks among case variants by its own rule; keep it. + if key != "body" && strings.EqualFold(key, "body") { + plain = false + break + } + } + if plain { + body, plain = cloneJSONValue(body, 1) + } + if !plain { + return descriptor, convert(value, &descriptor) + } + rest := maps.Clone(fields) + delete(rest, "body") + if err := convert(rest, &descriptor); err != nil { + return descriptor, err + } + descriptor.Body = body + return descriptor, nil +} + func convert(value any, target any) error { data, err := common.Marshal(value) if err != nil { @@ -1654,8 +1701,45 @@ func jsonValue(value any) any { return normalized } -// Already-parsed JSON and Sobek's plain exported values do not need another -// encode/decode pass. Copy containers to isolate hooks, preserving the codec's +// isPlainJSONValue reports whether hooks can take value as is: it reaches +// JavaScript exactly as the codec's decoding of it would, and the engine never +// writes back into it. Integers count, since hooks see the same numbers; a nil +// slice does not, since it would reach hooks as [] instead of null. +func isPlainJSONValue(value any, depth int) bool { + if depth > 64 { + return false + } + switch typed := value.(type) { + case nil, bool, int64, int: + return true + case string: + return utf8.ValidString(typed) + case float64: + return !math.IsNaN(typed) && !math.IsInf(typed, 0) + case map[string]any: + for key, item := range typed { + if !utf8.ValidString(key) || !isPlainJSONValue(item, depth+1) { + return false + } + } + return true + case []any: + if typed == nil { + return false + } + for _, item := range typed { + if !isPlainJSONValue(item, depth+1) { + return false + } + } + return true + default: + return false + } +} + +// Already-parsed JSON and the engine's plain exported values do not need another +// encode/decode pass. Copy containers for host-side edits, preserving the codec's // float64 numbers and nulls. Other Go types still use the configured codec. func cloneJSONValue(value any, depth int) (any, bool) { if depth > 64 { diff --git a/relay/channel/task/jsplugin/adaptor_test.go b/relay/channel/task/jsplugin/adaptor_test.go index 846803c83b..55b01705fd 100644 --- a/relay/channel/task/jsplugin/adaptor_test.go +++ b/relay/channel/task/jsplugin/adaptor_test.go @@ -1757,22 +1757,27 @@ func TestTaskSubmitStreamIdleTimeout(t *testing.T) { } func TestPluginJSONValuesPreserveCodecNormalizationAndIsolation(t *testing.T) { - for _, value := range []any{ - map[string]any{"units": int64(3), "nested": []any{true, " / 图像", math.Copysign(0, -1), nil}}, - map[string]any{"empty": []any{}, "null": []any(nil), "object": map[string]any(nil)}, - map[string]any{"large": int64(math.MaxInt64), "invalid UTF-8": string([]byte{0xff, 0xfe})}, - map[string]any{string([]byte{0xff}): "invalid key"}, - map[string]any{"bytes": []byte{0, 1, 255}, "number": json.Number("9007199254740993")}, - json.RawMessage(`{"units":2,"enabled":false}`), - struct { + for _, tc := range []struct { + value any + plain bool + }{ + {map[string]any{"units": int64(3), "nested": []any{true, " / 图像", math.Copysign(0, -1), nil}}, true}, + {map[string]any{"empty": []any{}, "object": map[string]any(nil)}, true}, + {map[string]any{"null": []any(nil)}, false}, + {map[string]any{"large": int64(math.MaxInt64), "invalid UTF-8": string([]byte{0xff, 0xfe})}, false}, + {map[string]any{string([]byte{0xff}): "invalid key"}, false}, + {map[string]any{"bytes": []byte{0, 1, 255}, "number": json.Number("9007199254740993")}, false}, + {json.RawMessage(`{"units":2,"enabled":false}`), false}, + {struct { Units int `json:"units"` - }{Units: 0}, + }{Units: 0}, false}, } { - encoded, err := common.Marshal(value) + encoded, err := common.Marshal(tc.value) require.NoError(t, err) var expected any require.NoError(t, common.Unmarshal(encoded, &expected)) - assert.Equal(t, expected, jsonValue(value)) + assert.Equal(t, expected, jsonValue(tc.value)) + assert.Equal(t, tc.plain, isPlainJSONValue(tc.value, 0)) } source := map[string]any{"items": []any{map[string]any{"label": "original"}}} copy := jsonValue(source).(map[string]any) @@ -1780,6 +1785,49 @@ func TestPluginJSONValuesPreserveCodecNormalizationAndIsolation(t *testing.T) { assert.Equal(t, "original", source["items"].([]any)[0].(map[string]any)["label"]) } +func TestRequestDescriptorDecodingMatchesCodec(t *testing.T) { + deep := any("leaf") + for range 70 { + deep = []any{deep} + } + for name, value := range map[string]any{ + "plain body": map[string]any{"url": "https://provider.example/submit", "method": "POST", "headers": map[string]any{"X-Plugin": "submit"}, "body": map[string]any{ + "units": int64(3), "large": int64(math.MaxInt64), "zero": math.Copysign(0, -1), "text": " & 
 图像", + "items": []any{nil, true, 1.5, map[string]any(nil), []any(nil), []any{}}, + }}, + "string body": map[string]any{"url": "u", "body": `{"raw":true}`}, + "null body": map[string]any{"url": "u", "body": nil}, + "no body": map[string]any{"url": "u", "credentialless": true}, + "invalid UTF-8 body": map[string]any{"url": "u", "body": map[string]any{"text": string([]byte{0xff})}}, + "NaN body": map[string]any{"url": "u", "body": map[string]any{"ratio": math.NaN()}}, + "bytes body": map[string]any{"url": "u", "body": []byte{0, 1, 255}}, + "host typed body": map[string]any{"url": "u", "body": map[string]string{"prompt": "hello"}}, + "deep body": map[string]any{"url": "u", "body": deep}, + "capitalized body key": map[string]any{"url": "u", "Body": map[string]any{"from": "Body"}}, + "body and capitalized body": map[string]any{"url": "u", "body": map[string]any{"from": "body"}, "Body": map[string]any{"from": "Body"}, "BODY": "upper"}, + "invalid header with body": map[string]any{"url": "u", "headers": map[string]any{"x": int64(1)}, "body": map[string]any{}}, + "invalid parts with body": map[string]any{"url": "u", "parts": "none", "body": map[string]any{}}, + "unsupported field with body": map[string]any{"url": "u", "method": math.Inf(1), "body": map[string]any{}}, + "case-insensitive other field": map[string]any{"URL": "u", "Method": "PUT", "body": "text"}, + "not an object": "descriptor", + } { + var expected requestDescriptor + expectedErr := convert(value, &expected) + decoded, err := decodeRequestDescriptor(value) + if expectedErr != nil { + assert.EqualError(t, err, expectedErr.Error(), name) + continue + } + require.NoError(t, err, name) + assert.Equal(t, expected, decoded, name) + } + source := map[string]any{"url": "u", "body": map[string]any{"items": []any{map[string]any{"label": "original"}}}} + decoded, err := decodeRequestDescriptor(source) + require.NoError(t, err) + decoded.Body.(map[string]any)["items"].([]any)[0].(map[string]any)["label"] = "changed" + assert.Equal(t, "original", source["body"].(map[string]any)["items"].([]any)[0].(map[string]any)["label"]) +} + func TestTaskSubmitHooksReceiveIndependentRequestSnapshots(t *testing.T) { source := ` export const meta={apiVersion:1,key:"request-copy",name:"Request copy",version:"1.0.0",author:{name:"Test"},models:["copy"],fetchMode:"per_task",usageSchema:{units:{type:"number",unit:"count"}}}; @@ -1811,6 +1859,39 @@ export function parseTaskResult(){return {status:"SUCCESS"};} assert.Equal(t, "original", request["nested"].(map[string]any)["label"]) } +func TestTaskUsageValidationLeavesReturnedHostRequestUnchanged(t *testing.T) { + // Returning ctx.requestBody hands back the host request map itself, so + // normalized facts must be built in a new map. + source := ` +export const meta={apiVersion:1,key:"request-usage",name:"Request usage",version:"1.0.0",author:{name:"Test"},models:["usage"],fetchMode:"per_task",usageSchema:{units:{type:"number",unit:"count"}}}; +export function buildSubmitRequest(ctx){return {url:ctx.baseUrl+"/submit",body:ctx.requestBody};} +export function extractUsage(ctx){return ctx.requestBody;} +export function parseSubmitResponse(){return {taskId:"usage"};} +export function buildQueryRequest(ctx){return {url:ctx.baseUrl+"/query"};} +export function parseTaskResult(){return {status:"SUCCESS"};} +` + plugin, err := pluginruntime.CompilePlugin(source, pluginruntime.Options{}) + require.NoError(t, err) + request := map[string]any{"units": int64(2)} + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodPost, "/submit", nil) + c.Set("task_request", request) + info := &relaycommon.RelayInfo{OriginModelName: "usage", ChannelMeta: &relaycommon.ChannelMeta{UpstreamModelName: "usage", ChannelBaseUrl: "https://provider.example"}, TaskRelayInfo: &relaycommon.TaskRelayInfo{}} + adaptor := New(plugin) + adaptor.Init(info) + require.Nil(t, adaptor.ValidateRequestAndSetAction(c, info)) + facts, err := adaptor.ExtractUsageFactsValidated(c, info) + require.NoError(t, err) + assert.Equal(t, map[string]any{"units": float64(2)}, facts) + assert.Equal(t, int64(2), request["units"]) + estimateRequest := map[string]any{"units": int64(2)} + c.Set("task_request", estimateRequest) + ratios, err := adaptor.EstimateBillingValidated(c, info) + require.NoError(t, err) + assert.Equal(t, map[string]float64{"units": 2}, ratios) + assert.Equal(t, int64(2), estimateRequest["units"]) +} + func TestTaskSubmitDeltaStreamContract(t *testing.T) { source := strings.Replace(documentStreamPlugin, `submitResponseTypes:["json","sse"],`, `submitResponseTypes:["json","sse"],requiredCapabilities:["submit-sse-delta@1"],`, 1) + ` export function parseSubmitEventDelta(ctx,event,previous) { diff --git a/relay/channel/task/jsplugin/performance_test.go b/relay/channel/task/jsplugin/performance_test.go index f75dd7ecd0..0729e39987 100644 --- a/relay/channel/task/jsplugin/performance_test.go +++ b/relay/channel/task/jsplugin/performance_test.go @@ -60,9 +60,13 @@ func BenchmarkTaskPluginRuntime(b *testing.B) { var err error switch name { case "HasExport": - _, err = plugin.Engine.HasExport(ctx, "extractUsage") + if !plugin.Engine.HasExport("extractUsage") { + b.Fatal("extractUsage not exported") + } case "MissingExport": - _, err = plugin.Engine.HasExport(ctx, "absentHook") + if plugin.Engine.HasExport("absentHook") { + b.Fatal("absentHook exported") + } default: _, err = plugin.Engine.Call(ctx, "extractUsage", input) } diff --git a/relay/channel/task/jsplugin/submit_stream.go b/relay/channel/task/jsplugin/submit_stream.go index 84f0a88039..dc7b713144 100644 --- a/relay/channel/task/jsplugin/submit_stream.go +++ b/relay/channel/task/jsplugin/submit_stream.go @@ -117,15 +117,17 @@ func (a *TaskAdaptor) readSubmitEvents(parent context.Context, resp *http.Respon return nil, fmt.Errorf("invalid submit stream changes: %w", err) } } - // Keep the exact encoded-byte limit above. Plain JSON state can be - // isolated without parsing large strings again; codec-specific values - // (for example exported typed arrays) retain the old normalization. - var plainJSON bool - state, plainJSON = cloneJSONValue(nextState, 0) - if !plainJSON { - if err = common.Unmarshal(encoded, &state); err != nil { + // Keep the exact encoded-byte limit above. Plain JSON state goes back + // to the next event as is; codec-specific values (for example exported + // typed arrays) retain the old normalization. + if isPlainJSONValue(nextState, 0) { + state = nextState + } else { + var decoded any + if err = common.Unmarshal(encoded, &decoded); err != nil { return nil, err } + state = decoded } if done { if accumulated != nil { diff --git a/relay/plugin_protocol.go b/relay/plugin_protocol.go index 6cb90c51fc..aa5a0ebe8e 100644 --- a/relay/plugin_protocol.go +++ b/relay/plugin_protocol.go @@ -13,6 +13,7 @@ import ( "github.com/QuantumNous/new-api/common" "github.com/QuantumNous/new-api/dto" + pluginruntime "github.com/QuantumNous/new-api/pkg/jsplugin" ) const ( @@ -101,15 +102,13 @@ type ProtocolState struct { Value json.RawMessage } -func (s ProtocolState) PluginValue() (any, error) { +// PluginValue returns the state as a hook argument: nil when it is absent or +// null, otherwise its JSON text for the hook to receive parsed. +func (s ProtocolState) PluginValue() any { if !s.Present || s.Null { - return nil, nil + return nil } - var value any - if err := common.Unmarshal(s.Value, &value); err != nil { - return nil, err - } - return value, nil + return pluginruntime.RawJSON(s.Value) } type ProtocolSemanticEvent struct { diff --git a/relay/plugin_protocol_test.go b/relay/plugin_protocol_test.go index d0fc8f9d38..cf89b27311 100644 --- a/relay/plugin_protocol_test.go +++ b/relay/plugin_protocol_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/QuantumNous/new-api/common" + pluginruntime "github.com/QuantumNous/new-api/pkg/jsplugin" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -51,9 +52,10 @@ func TestDecodePluginProtocolEventResultPreservesStatePresence(t *testing.T) { assert.Empty(t, result.State.Value) } - if result.State.Present { - _, err = result.State.PluginValue() - require.NoError(t, err) + if testCase.wantPresent && !testCase.wantNull { + assert.Equal(t, pluginruntime.RawJSON(result.State.Value), result.State.PluginValue()) + } else { + assert.Nil(t, result.State.PluginValue()) } }) } diff --git a/service/task_plugin_view_test.go b/service/task_plugin_view_test.go index c1823bacef..10e38754a6 100644 --- a/service/task_plugin_view_test.go +++ b/service/task_plugin_view_test.go @@ -83,3 +83,28 @@ func TestBuildTaskPluginViewOmitsPrivatePollState(t *testing.T) { assert.NotContains(t, payload, "poll_failures") assert.NotContains(t, payload, "private_data") } + +func TestBuildTaskPluginViewEncodesKeysInMapOrder(t *testing.T) { + task := &model.Task{ + TaskID: "task_public_order", + Platform: "fixture", + Status: model.TaskStatusSuccess, + Progress: "100%", + FailReason: "none", + CreatedAt: 1700000000, + UpdatedAt: 1700000100, + FinishTime: 1700000200, + Data: []byte(`{"b":1,"a":{"10":true,"2":false}}`), + } + + view, err := BuildTaskPluginView(task) + require.NoError(t, err) + encoded, err := common.Marshal(view) + require.NoError(t, err) + var payload map[string]any + require.NoError(t, common.Unmarshal(encoded, &payload)) + require.Len(t, payload, 9) + reencoded, err := common.Marshal(payload) + require.NoError(t, err) + assert.Equal(t, string(reencoded), string(encoded)) +}