From a2886af92a66ca1d40b4857967e4e44e40596169 Mon Sep 17 00:00:00 2001 From: PhillLittlewood <68272904+PhillLittlewood@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:49:28 +0100 Subject: [PATCH] Added ComfyUi to has Image Provider (#850) * feat: integrate comfyui workflows into image provider selection * style: format code with prettier * fix: align i18n keys for russian locale * feat: implement requested changes and format code * feat: implement requested changes including renaming default workflow --------- Co-authored-by: wyuc --- app/api/comfyui-workflows/route.ts | 23 + app/api/generate/image/route.ts | 7 +- app/api/verify-image-provider/route.ts | 5 + comfyui-setup-instructions.md | 148 +++++ components/generation/media-popover.tsx | 41 +- components/settings/image-settings.tsx | 240 +++++-- components/settings/index.tsx | 2 + lib/i18n/locales/ar-SA.json | 9 + lib/i18n/locales/en-US.json | 9 + lib/i18n/locales/ja-JP.json | 9 + lib/i18n/locales/ko-KR.json | 9 + lib/i18n/locales/pt-BR.json | 9 + lib/i18n/locales/ru-RU.json | 9 + lib/i18n/locales/zh-CN.json | 9 + lib/i18n/locales/zh-TW.json | 9 + lib/media/adapters/comfyui-image-adapter.ts | 681 ++++++++++++++++++++ lib/media/comfyui-workflows.ts | 104 +++ lib/media/image-providers.ts | 28 + lib/media/types.ts | 1 + lib/store/settings.ts | 1 + public/comfyui-workflow.json | 221 +++++++ public/logos/comfyui.svg | 1 + 22 files changed, 1503 insertions(+), 72 deletions(-) create mode 100644 app/api/comfyui-workflows/route.ts create mode 100644 comfyui-setup-instructions.md create mode 100644 lib/media/adapters/comfyui-image-adapter.ts create mode 100644 lib/media/comfyui-workflows.ts create mode 100644 public/comfyui-workflow.json create mode 100644 public/logos/comfyui.svg diff --git a/app/api/comfyui-workflows/route.ts b/app/api/comfyui-workflows/route.ts new file mode 100644 index 000000000..92a45adab --- /dev/null +++ b/app/api/comfyui-workflows/route.ts @@ -0,0 +1,23 @@ +/** + * GET /api/comfyui-workflows + * + * Returns a list of ComfyUI workflow JSON files found in the Next.js + * public/ directory, with display names derived from their filenames. + * Discovery logic lives in lib/media/comfyui-workflows.ts and is shared + * with the ComfyUI image adapter, so the list returned here is always + * exactly what the adapter will accept as a workflow id. + * + * Response: { workflows: Array<{ id: string; name: string }> } + */ + +import { NextResponse } from 'next/server'; +import { listComfyuiWorkflows } from '@/lib/media/comfyui-workflows'; + +export async function GET() { + try { + return NextResponse.json({ workflows: await listComfyuiWorkflows() }); + } catch (err) { + console.error('[ComfyUI Workflows API] Failed to list workflows:', err); + return NextResponse.json({ workflows: [] }); + } +} diff --git a/app/api/generate/image/route.ts b/app/api/generate/image/route.ts index 79ae83b1a..4e78d6fdd 100644 --- a/app/api/generate/image/route.ts +++ b/app/api/generate/image/route.ts @@ -34,7 +34,12 @@ import { validateUrlForSSRF } from '@/lib/server/ssrf-guard'; const log = createLogger('ImageGeneration API'); -export const maxDuration = 60; +// The ComfyUI adapter polls up to GENERATION_TIMEOUT_MS (5 min) and real +// workflows can take 3–5 min. 60s would let platforms that enforce maxDuration +// (e.g. Vercel) kill the request ~4 min before the adapter finishes. 300s is +// the practical ceiling on most managed platforms and matches the poll budget. +// (Self-hosted Node servers ignore this value entirely.) +export const maxDuration = 300; export async function POST(request: NextRequest) { try { diff --git a/app/api/verify-image-provider/route.ts b/app/api/verify-image-provider/route.ts index 64fec54fc..af1e5c376 100644 --- a/app/api/verify-image-provider/route.ts +++ b/app/api/verify-image-provider/route.ts @@ -28,6 +28,11 @@ import { validateUrlForSSRF } from '@/lib/server/ssrf-guard'; const log = createLogger('VerifyImageProvider'); +// Connectivity probes are lightweight and each underlying request is bounded by +// its own AbortSignal, but the route had no ceiling at all — cap it so a stalled +// upstream can't tie up the function indefinitely. +export const maxDuration = 30; + export async function POST(request: NextRequest) { try { const providerId = (request.headers.get('x-image-provider') || 'seedream') as ImageProviderId; diff --git a/comfyui-setup-instructions.md b/comfyui-setup-instructions.md new file mode 100644 index 000000000..734d63a8f --- /dev/null +++ b/comfyui-setup-instructions.md @@ -0,0 +1,148 @@ +# ComfyUI Workflow Setup for OpenMAIC + +## Where to store workflows + +Place all workflow JSON files in your Next.js `public/` folder: + +``` +your-project/ + public/ + comfyui-workflow.json → displays as "Workflow" + comfyui-anime-style.json → displays as "Anime Style" + comfyui-line-art.json → displays as "Line Art" + comfyui-portrait.json → displays as "Portrait" +``` + +### Naming convention +- Filenames **must** start with `comfyui-` or contain `workflow` +- Use hyphens to separate words — they become the display name in the UI +- The `comfyui-` prefix is stripped automatically +- Example: `comfyui-anime-style.json` → **"Anime Style"** in the dropdown + +--- + +## Required nodes in your workflow + +The adapter finds nodes **by their title** (the `_meta.title` field in the JSON). +To set a node's title in ComfyUI: right-click the node → **Title** → type the name. + +### Required + +| Node Title | Recommended Type | Purpose | +|------------|-----------------|---------| +| `Input Prompt` | `PrimitiveStringMultiline` | The image description from OpenMAIC is injected here | + +### Recommended (automatically patched if present) + +| Node Title | Recommended Type | Purpose | +|------------|-----------------|---------| +| `Width` | `PrimitiveInt` | Output width in pixels — set from the requested aspect ratio | +| `Height` | `PrimitiveInt` | Output height in pixels — set from the requested aspect ratio | +| `KSampler` | `KSampler` | Seed is randomised on every generation for varied outputs | +| `Enable prompt enhancement?` | `PrimitiveBoolean` | Set to `false` to skip LLM prompt enhancement (recommended for speed) | + +### Fallback behaviour + +If `Width` and `Height` nodes are **not found**, the adapter automatically falls +back to patching the `Empty Flux 2 Latent` node's `width` and `height` inputs +directly — so existing workflows without dedicated dimension nodes still work. + +Both `Width` and `Height` must be present for the explicit node approach to +activate — if only one is found the adapter falls back to the latent node method +and logs a warning. + +### Prompt node fallback + +If `Input Prompt` is not found, the adapter falls back to a node titled +`String (Multiline - Prompt)` — so existing workflows still work without +renaming anything. + +--- + +## Connecting the nodes + +### Input Prompt +Connect the output of the `Input Prompt` node to wherever your prompt text +enters the pipeline — typically the `text` input of a `CLIPTextEncode` node, +or a `StringReplace` node if you use prompt templating. + +### Width and Height +Connect the output of each node to the corresponding `width` and `height` +inputs of your `Empty Flux 2 Latent` (or equivalent empty latent) node. + +Example wiring: +``` +[Input Prompt] ──→ CLIPTextEncode (text) +[Width] ──→ EmptyLatentImage (width) +[Height] ──→ EmptyLatentImage (height) +``` + +--- + +## How to export in API format + +The workflow JSON must be in **ComfyUI API format** (not the default save format). + +1. In ComfyUI, go to **Settings** → enable **Dev Mode Options** +2. A new **Save (API Format)** button appears in the toolbar +3. Click **Save (API Format)** — this produces the correct JSON +4. Place the file in your `public/` folder + +> ⚠️ The regular **Save** button produces a different format that will not work. + +--- + +## Settings in OpenMAIC + +1. Go to **Settings → Image Generation** +2. Select **ComfyUI Image** as the provider +3. Set **Base URL** to your ComfyUI address (default `http://localhost:8188`) +4. Select your workflow from the **Workflows** list +5. Click **Test Connection** to verify ComfyUI is reachable + +### Default workflow selection + +If no workflow is explicitly selected — for example on the autonomous +classroom-media generation path, or before you've clicked a workflow in +Settings — the adapter falls back to the **first workflow file discovered in +`public/`** (alphabetically by display name). It does **not** rely on any +hard-coded filename, so you don't need a file called `comfyui-workflow.json`; +any single `comfyui-*.json` you ship will be used as the default. If `public/` +contains **no** workflow files at all, generation fails with a clear error +asking you to add one. + +--- + +## Deployment topology (important for hosted/production) + +The default Base URL `http://localhost:8188` assumes **OpenMAIC and ComfyUI run +on the same host** (the typical local / self-hosted setup). + +When OpenMAIC runs with `NODE_ENV=production`, a **client-supplied** Base URL +(`x-base-url`) that points at `localhost`, `127.0.0.1`, or a private/internal IP +range is rejected with HTTP 403 by the SSRF guard (`validateUrlForSSRF`). This +is deliberate and matches the behaviour of the other local providers — it stops +a browser client from steering server-side requests at internal services. + +Practical implications: + +- **Same-host / self-hosted:** works out of the box. Server-resolved defaults + are not subject to the client-URL SSRF check, so the `localhost:8188` default + is fine when OpenMAIC and ComfyUI share a host. +- **ComfyUI on a different machine in production:** point OpenMAIC at ComfyUI + over a **routable, non-private** address (or terminate it behind a reverse + proxy on a public hostname). A `localhost`/private URL sent from the browser + in production will be refused. +- **Local development** (`NODE_ENV` ≠ `production`): the SSRF check is skipped, + so `localhost` works normally. + +--- + +## Performance tips + +- **Disable prompt enhancement** — if your workflow has an LLM-based prompt + enhancer, set its enable node to `false`. Enhancement can add 3–5 minutes + per image. The prompts generated by OpenMAIC are already descriptive enough. +- The adapter randomises the KSampler seed on every generation automatically. +- Output dimensions are calculated from the aspect ratio requested by OpenMAIC + and capped at the `maxResolution` set in `image-providers.ts` (default `1920×1920`). diff --git a/components/generation/media-popover.tsx b/components/generation/media-popover.tsx index e20b3e9b6..49fc3da6c 100644 --- a/components/generation/media-popover.tsx +++ b/components/generation/media-popover.tsx @@ -1,6 +1,6 @@ 'use client'; -import { useState, useCallback, useMemo, Fragment } from 'react'; +import { useState, useCallback, useMemo, Fragment, useEffect } from 'react'; import type { LucideIcon } from 'lucide-react'; import { Image as ImageIcon, @@ -45,6 +45,7 @@ const IMAGE_PROVIDER_ICONS: Record = { 'qwen-image': '/logos/bailian.svg', 'nano-banana': '/logos/gemini.svg', 'grok-image': '/logos/grok.svg', + 'comfyui-image': '/logos/comfyui.svg', }; const VIDEO_PROVIDER_ICONS: Record = { seedance: '/logos/doubao.svg', @@ -107,6 +108,14 @@ export function MediaPopover({ onSettingsOpen }: MediaPopoverProps) { const setASRProvider = useSettingsStore((s) => s.setASRProvider); const setASRLanguage = useSettingsStore((s) => s.setASRLanguage); + const [comfyWorkflows, setComfyWorkflows] = useState>([]); + useEffect(() => { + fetch('/api/comfyui-workflows') + .then((r) => r.json()) + .then((d) => setComfyWorkflows(d.workflows || [])) + .catch(() => setComfyWorkflows([])); + }, []); + const enabledMap: Record = { image: imageGenerationEnabled, video: videoGenerationEnabled, @@ -135,17 +144,25 @@ export function MediaPopover({ onSettingsOpen }: MediaPopoverProps) { () => Object.values(IMAGE_PROVIDERS) .filter((p) => cfgOk(imageProvidersConfig, p.id, p.requiresApiKey)) - .map((p) => ({ - groupId: p.id, - groupName: p.name, - groupIcon: IMAGE_PROVIDER_ICONS[p.id], - available: true, - items: providerModels(p.models, imageProvidersConfig[p.id]).map((m) => ({ - id: m.id, - name: m.name, - })), - })), - [cfgOk, imageProvidersConfig], + .map((p) => { + const items = + p.id === 'comfyui-image' + ? comfyWorkflows + : providerModels(p.models, imageProvidersConfig[p.id]); + + return { + groupId: p.id, + groupName: p.name, + groupIcon: IMAGE_PROVIDER_ICONS[p.id], + available: true, + // Map to a consistent format here + items: items.map((m) => ({ + id: m.id, + name: m.name, + })), + }; + }), + [cfgOk, imageProvidersConfig, comfyWorkflows], ); const videoGroups = useMemo( diff --git a/components/settings/image-settings.tsx b/components/settings/image-settings.tsx index 49b3b099a..66f280d35 100644 --- a/components/settings/image-settings.tsx +++ b/components/settings/image-settings.tsx @@ -1,6 +1,6 @@ 'use client'; -import { useState, useCallback, useMemo } from 'react'; +import { useState, useCallback, useMemo, useEffect } from 'react'; import { Label } from '@/components/ui/label'; import { Input } from '@/components/ui/input'; import { Button } from '@/components/ui/button'; @@ -18,6 +18,7 @@ import { Plus, Settings2, Trash2, + RefreshCw, } from 'lucide-react'; import { cn } from '@/lib/utils'; import type { ImageProviderId } from '@/lib/media/types'; @@ -26,12 +27,18 @@ interface ImageSettingsProps { selectedProviderId: ImageProviderId; } +interface WorkflowEntry { + id: string; + name: string; +} + export function ImageSettings({ selectedProviderId }: ImageSettingsProps) { const { t } = useI18n(); const imageModelId = useSettingsStore((state) => state.imageModelId); const imageProvidersConfig = useSettingsStore((state) => state.imageProvidersConfig); const _setImageModelId = useSettingsStore((state) => state.setImageModelId); + const setImageProvider = useSettingsStore((state) => state.setImageProvider); const setImageProviderConfig = useSettingsStore((state) => state.setImageProviderConfig); const [showApiKey, setShowApiKey] = useState(false); @@ -44,6 +51,13 @@ export function ImageSettings({ selectedProviderId }: ImageSettingsProps) { const [editingModelIndex, setEditingModelIndex] = useState(null); const [modelForm, setModelForm] = useState({ id: '', name: '' }); + // ComfyUI workflow list state + const [workflows, setWorkflows] = useState([]); + const [workflowsLoading, setWorkflowsLoading] = useState(false); + const [workflowsError, setWorkflowsError] = useState(null); + + const isComfyUI = selectedProviderId === 'comfyui-image'; + // Reset test state when provider changes (derived state pattern) const [prevSelectedProviderId, setPrevSelectedProviderId] = useState(selectedProviderId); if (selectedProviderId !== prevSelectedProviderId) { @@ -52,6 +66,28 @@ export function ImageSettings({ selectedProviderId }: ImageSettingsProps) { setTestMessage(''); } + // Fetch ComfyUI workflows when the provider is selected + const fetchWorkflows = useCallback(async () => { + setWorkflowsLoading(true); + setWorkflowsError(null); + try { + const res = await fetch('/api/comfyui-workflows'); + const data = await res.json(); + setWorkflows(data.workflows || []); + } catch (err) { + setWorkflowsError(t('settings.comfyuiLoadError').replace('{error}', String(err))); + setWorkflows([]); + } finally { + setWorkflowsLoading(false); + } + }, [t]); + + useEffect(() => { + if (isComfyUI) { + fetchWorkflows(); + } + }, [isComfyUI, fetchWorkflows]); + const currentConfig = imageProvidersConfig[selectedProviderId]; const currentProvider = IMAGE_PROVIDERS[selectedProviderId]; const builtInModels = currentProvider?.models || []; @@ -149,8 +185,9 @@ export function ImageSettings({ selectedProviderId }: ImageSettingsProps) { )} - {/* Managed providers are admin-owned: the operator's key and base URL are - authoritative and not overridable here, so the editing inputs are hidden. */} + {/* Managed providers are admin-owned: the operator's key and base URL + are authoritative and not overridable here, so the editing inputs + are hidden (server ignores client-sent overrides for these). */} {!isServerConfigured && ( <> {/* API Key + Test inline */} @@ -226,7 +263,11 @@ export function ImageSettings({ selectedProviderId }: ImageSettingsProps) { spellCheck={false} value={currentConfig?.baseUrl || ''} onChange={(e) => handleBaseUrlChange(e.target.value)} - placeholder={currentProvider?.defaultBaseUrl || t('settings.enterCustomBaseUrl')} + placeholder={ + currentConfig?.baseUrl || + currentProvider?.defaultBaseUrl || + t('settings.enterCustomBaseUrl') + } className="h-8" /> {(() => { @@ -243,66 +284,147 @@ export function ImageSettings({ selectedProviderId }: ImageSettingsProps) { )} - {/* Model list */} -
-
- - -
- -
- {/* Built-in models */} - {builtInModels.map((model) => ( -
+
+ +
- ))} + {workflowsLoading ? ( + + ) : ( + + )} + {t('settings.comfyuiRefresh')} + +
- {/* Custom models */} - {customModels.map((model, index) => ( -
-
-
{model.name}
-
{model.id}
-
-
- - -
+ {workflowsError && ( +
+ {workflowsError}
- ))} + )} + + {!workflowsLoading && !workflowsError && workflows.length === 0 && ( +
+ {t('settings.comfyuiNoWorkflowsFoundPrefix')}{' '} + public/. +
+ {t('settings.comfyuiAddWorkflowPrefix')}{' '} + comfyui-*.json{' '} + {t('settings.comfyuiAddWorkflowSuffix')} +
+ )} + +
+ {workflows.map((workflow) => ( +
{ + // Selecting a workflow here must also make ComfyUI the + // active image provider — otherwise the workflow filename + // gets written into imageModelId while a different + // provider (e.g. Seedream) stays active, and that + // provider's next generation call sends this filename as + // its model id. Mirrors the atomic setImageProvider + + // setImageModelId pairing used in media-popover.tsx. + setImageProvider(selectedProviderId); + _setImageModelId(workflow.id); + }} + > +
+
{workflow.name}
+
+ {workflow.id} +
+
+ {imageModelId === workflow.id && ( + + )} +
+ ))} +
+ +

+ {t('settings.comfyuiFolderHintPrefix')} public/{' '} + {t('settings.comfyuiFolderHintMiddle')}{' '} + comfyui-anime-style.json → "Anime Style". +

-
+ ) : ( + /* ── All other providers: standard model list ── */ +
+
+ + +
- {/* Add/Edit Model Dialog */} +
+ {/* Built-in models */} + {builtInModels.map((model) => ( +
+
+
{model.name}
+
{model.id}
+
+
+ ))} + + {/* Custom models */} + {customModels.map((model, index) => ( +
+
+
{model.name}
+
{model.id}
+
+
+ + +
+
+ ))} +
+
+ )} + + {/* Add/Edit Model Dialog — only used for non-ComfyUI providers */} diff --git a/components/settings/index.tsx b/components/settings/index.tsx index 07c48322c..a29738c3e 100644 --- a/components/settings/index.tsx +++ b/components/settings/index.tsx @@ -176,6 +176,7 @@ const IMAGE_PROVIDER_NAMES: Record = { 'nano-banana': 'providerNanoBanana', 'minimax-image': 'providerMiniMaxImage', 'grok-image': 'providerGrokImage', + 'comfyui-image': 'providerComfyUIImage', lemonade: 'providerLemonadeImage', }; @@ -186,6 +187,7 @@ const IMAGE_PROVIDER_ICONS: Record = { 'nano-banana': '/logos/gemini.svg', 'minimax-image': '/logos/minimax.svg', 'grok-image': '/logos/grok.svg', + 'comfyui-image': '/logos/comfyui.svg', lemonade: '/logos/lemonade.svg', }; diff --git a/lib/i18n/locales/ar-SA.json b/lib/i18n/locales/ar-SA.json index 075a012f8..4e46cf41a 100644 --- a/lib/i18n/locales/ar-SA.json +++ b/lib/i18n/locales/ar-SA.json @@ -1067,6 +1067,14 @@ }, "requestUrl": "عنوان الطلب", "models": "النماذج", + "comfyuiWorkflows": "سير العمل", + "comfyuiRefresh": "تحديث", + "comfyuiNoWorkflowsFoundPrefix": "لم يتم العثور على ملفات JSON لسير العمل في", + "comfyuiAddWorkflowPrefix": "أضف ملف", + "comfyuiAddWorkflowSuffix": "للبدء.", + "comfyuiFolderHintPrefix": "ضع ملفات JSON الخاصة بسير العمل في مجلد", + "comfyuiFolderHintMiddle": "الخاص بك. يصبح اسم الملف هو الاسم المعروض — على سبيل المثال", + "comfyuiLoadError": "فشل تحميل سير العمل: {error}", "addModel": "جديد", "reset": "إعادة تعيين", "fetch": "جلب", @@ -1488,6 +1496,7 @@ "providerNanoBanana": "Nano Banana (Gemini)", "providerMiniMaxImage": "MiniMax Image", "providerGrokImage": "Grok Image (xAI)", + "providerComfyUIImage": "ComfyUI Image", "providerLemonadeImage": "Lemonade Image (محلي)", "testImageGeneration": "اختبار توليد الصور", "testImageConnectivity": "اختبار الاتصال", diff --git a/lib/i18n/locales/en-US.json b/lib/i18n/locales/en-US.json index ff2222277..9126a90ed 100644 --- a/lib/i18n/locales/en-US.json +++ b/lib/i18n/locales/en-US.json @@ -1067,6 +1067,14 @@ }, "requestUrl": "Request URL", "models": "Models", + "comfyuiWorkflows": "Workflows", + "comfyuiRefresh": "Refresh", + "comfyuiNoWorkflowsFoundPrefix": "No workflow JSON files found in", + "comfyuiAddWorkflowPrefix": "Add a", + "comfyuiAddWorkflowSuffix": "file to get started.", + "comfyuiFolderHintPrefix": "Place workflow JSON files in your", + "comfyuiFolderHintMiddle": "folder. The filename becomes the display name — e.g.", + "comfyuiLoadError": "Failed to load workflows: {error}", "addModel": "Add", "reset": "Reset", "fetch": "Fetch", @@ -1488,6 +1496,7 @@ "providerNanoBanana": "Nano Banana (Gemini)", "providerMiniMaxImage": "MiniMax Image", "providerGrokImage": "Grok Image (xAI)", + "providerComfyUIImage": "ComfyUI Image", "providerLemonadeImage": "Lemonade Image (Local)", "testImageGeneration": "Test Image Generation", "testImageConnectivity": "Test Connection", diff --git a/lib/i18n/locales/ja-JP.json b/lib/i18n/locales/ja-JP.json index cd3d859fa..d1ce3fb59 100644 --- a/lib/i18n/locales/ja-JP.json +++ b/lib/i18n/locales/ja-JP.json @@ -1067,6 +1067,14 @@ }, "requestUrl": "リクエストURL", "models": "モデル", + "comfyuiWorkflows": "ワークフロー", + "comfyuiRefresh": "更新", + "comfyuiNoWorkflowsFoundPrefix": "ワークフローのJSONファイルが見つかりません：", + "comfyuiAddWorkflowPrefix": "追加してください：", + "comfyuiAddWorkflowSuffix": "ファイルを追加して開始します。", + "comfyuiFolderHintPrefix": "ワークフローのJSONファイルを", + "comfyuiFolderHintMiddle": "フォルダに配置してください。ファイル名が表示名になります。例：", + "comfyuiLoadError": "ワークフローの読み込みに失敗しました: {error}", "addModel": "追加", "reset": "リセット", "fetch": "取得", @@ -1488,6 +1496,7 @@ "providerNanoBanana": "Nano Banana（Gemini）", "providerMiniMaxImage": "MiniMax Image", "providerGrokImage": "Grok Image（xAI）", + "providerComfyUIImage": "ComfyUI Image", "providerLemonadeImage": "Lemonade Image（ローカル）", "testImageGeneration": "画像生成をテスト", "testImageConnectivity": "接続テスト", diff --git a/lib/i18n/locales/ko-KR.json b/lib/i18n/locales/ko-KR.json index e44fe4d71..c88ae1ea8 100644 --- a/lib/i18n/locales/ko-KR.json +++ b/lib/i18n/locales/ko-KR.json @@ -1067,6 +1067,14 @@ }, "requestUrl": "요청 URL", "models": "모델", + "comfyuiWorkflows": "워크플로", + "comfyuiRefresh": "새로고침", + "comfyuiNoWorkflowsFoundPrefix": "다음 위치에서 워크플로 JSON 파일을 찾을 수 없습니다:", + "comfyuiAddWorkflowPrefix": "다음 파일을 추가하여 시작하세요:", + "comfyuiAddWorkflowSuffix": "", + "comfyuiFolderHintPrefix": "워크플로 JSON 파일을", + "comfyuiFolderHintMiddle": "폴더에 넣으세요. 파일 이름이 표시 이름이 됩니다. 예:", + "comfyuiLoadError": "워크플로를 불러오지 못했습니다: {error}", "addModel": "추가", "reset": "초기화", "fetch": "불러오기", @@ -1488,6 +1496,7 @@ "providerNanoBanana": "Nano Banana (Gemini)", "providerMiniMaxImage": "MiniMax Image", "providerGrokImage": "Grok Image (xAI)", + "providerComfyUIImage": "ComfyUI Image", "providerLemonadeImage": "Lemonade Image (로컬)", "testImageGeneration": "이미지 생성 테스트", "testImageConnectivity": "연결 테스트", diff --git a/lib/i18n/locales/pt-BR.json b/lib/i18n/locales/pt-BR.json index d852715d5..bd75482d1 100644 --- a/lib/i18n/locales/pt-BR.json +++ b/lib/i18n/locales/pt-BR.json @@ -1067,6 +1067,14 @@ }, "requestUrl": "URL da Requisição", "models": "Modelos", + "comfyuiWorkflows": "Fluxos de trabalho", + "comfyuiRefresh": "Atualizar", + "comfyuiNoWorkflowsFoundPrefix": "Nenhum arquivo JSON de fluxo de trabalho encontrado em", + "comfyuiAddWorkflowPrefix": "Adicione um arquivo", + "comfyuiAddWorkflowSuffix": "para começar.", + "comfyuiFolderHintPrefix": "Coloque os arquivos JSON de fluxo de trabalho na sua pasta", + "comfyuiFolderHintMiddle": "O nome do arquivo se torna o nome exibido — ex.", + "comfyuiLoadError": "Falha ao carregar fluxos de trabalho: {error}", "addModel": "Adicionar", "reset": "Restaurar", "fetch": "Buscar", @@ -1488,6 +1496,7 @@ "providerNanoBanana": "Nano Banana (Gemini)", "providerMiniMaxImage": "MiniMax Image", "providerGrokImage": "Grok Image (xAI)", + "providerComfyUIImage": "ComfyUI Image", "providerLemonadeImage": "Lemonade Image (Local)", "testImageGeneration": "Testar Geração de Imagem", "testImageConnectivity": "Testar Conexão", diff --git a/lib/i18n/locales/ru-RU.json b/lib/i18n/locales/ru-RU.json index 978722010..a5c2ba60b 100644 --- a/lib/i18n/locales/ru-RU.json +++ b/lib/i18n/locales/ru-RU.json @@ -1067,6 +1067,14 @@ }, "requestUrl": "URL запроса", "models": "Модели", + "comfyuiWorkflows": "Рабочие процессы", + "comfyuiRefresh": "Обновить", + "comfyuiNoWorkflowsFoundPrefix": "JSON-файлы рабочих процессов не найдены в", + "comfyuiAddWorkflowPrefix": "Добавьте файл", + "comfyuiAddWorkflowSuffix": "чтобы начать.", + "comfyuiFolderHintPrefix": "Поместите JSON-файлы рабочих процессов в папку", + "comfyuiFolderHintMiddle": "Имя файла становится отображаемым именем — например", + "comfyuiLoadError": "Не удалось загрузить рабочие процессы: {error}", "addModel": "Добавить", "reset": "Сброс", "fetch": "Получить", @@ -1488,6 +1496,7 @@ "providerNanoBanana": "Nano Banana (Gemini)", "providerMiniMaxImage": "MiniMax Image", "providerGrokImage": "Grok Image (xAI)", + "providerComfyUIImage": "ComfyUI Image", "providerLemonadeImage": "Lemonade Image (Локальный)", "testImageGeneration": "Тест генерации изображений", "testImageConnectivity": "Тест подключения", diff --git a/lib/i18n/locales/zh-CN.json b/lib/i18n/locales/zh-CN.json index d7e74f806..3dccf3ec4 100644 --- a/lib/i18n/locales/zh-CN.json +++ b/lib/i18n/locales/zh-CN.json @@ -1067,6 +1067,14 @@ }, "requestUrl": "请求地址", "models": "模型", + "comfyuiWorkflows": "工作流", + "comfyuiRefresh": "刷新", + "comfyuiNoWorkflowsFoundPrefix": "未在", + "comfyuiAddWorkflowPrefix": "添加一个", + "comfyuiAddWorkflowSuffix": "文件即可开始。", + "comfyuiFolderHintPrefix": "请将工作流 JSON 文件放入", + "comfyuiFolderHintMiddle": "文件夹。文件名将作为显示名称，例如", + "comfyuiLoadError": "加载工作流失败：{error}", "addModel": "添加", "reset": "重置", "fetch": "获取", @@ -1488,6 +1496,7 @@ "providerNanoBanana": "Nano Banana（Gemini）", "providerMiniMaxImage": "MiniMax 图像", "providerGrokImage": "Grok Image（xAI）", + "providerComfyUIImage": "ComfyUI Image", "providerLemonadeImage": "Lemonade 图像（本地）", "testImageGeneration": "测试图像生成", "testImageConnectivity": "测试连接", diff --git a/lib/i18n/locales/zh-TW.json b/lib/i18n/locales/zh-TW.json index c1e395c72..3429421c0 100644 --- a/lib/i18n/locales/zh-TW.json +++ b/lib/i18n/locales/zh-TW.json @@ -1044,6 +1044,14 @@ "apiHost": "Base URL", "requestUrl": "請求位址", "models": "模型", + "comfyuiWorkflows": "工作流程", + "comfyuiRefresh": "重新整理", + "comfyuiNoWorkflowsFoundPrefix": "未在", + "comfyuiAddWorkflowPrefix": "新增一個", + "comfyuiAddWorkflowSuffix": "檔案即可開始。", + "comfyuiFolderHintPrefix": "請將工作流程 JSON 檔案放入", + "comfyuiFolderHintMiddle": "資料夾。檔案名稱將作為顯示名稱，例如", + "comfyuiLoadError": "載入工作流程失敗：{error}", "addModel": "新增", "reset": "重設", "fetch": "取得", @@ -1400,6 +1408,7 @@ "providerMiniMaxImage": "MiniMax 圖像", "providerLemonadeImage": "Lemonade 圖像（本機）", "providerGrokImage": "Grok Image（xAI）", + "providerComfyUIImage": "ComfyUI Image", "testImageGeneration": "測試圖像生成", "testImageConnectivity": "測試連線", "imageConnectivitySuccess": "圖像服務連線成功", diff --git a/lib/media/adapters/comfyui-image-adapter.ts b/lib/media/adapters/comfyui-image-adapter.ts new file mode 100644 index 000000000..6c84f2246 --- /dev/null +++ b/lib/media/adapters/comfyui-image-adapter.ts @@ -0,0 +1,681 @@ +/** + * ComfyUI Image Generation Adapter + * + * Submits a prompt to a local (or remote) ComfyUI instance via its + * REST API, polls for completion, and returns the image as base64. + * + * Endpoint: http://localhost:8188 (configurable via baseUrl) + * No API key required. + * + * Workflow loading strategy: + * 1. If config.workflowJson is set (an already-parsed object), use it. + * 2. Else if config.model names a workflow, load that file (validated against + * the public/ allowlist server-side). + * 3. Else default to the first workflow file discovered in public/. + * → Ship at least one comfyui-*.json in your Next.js public/ directory. + * + * Nodes patched at runtime: + * "String (Multiline - Prompt)" → inputs.value = prompt + * "Empty Flux 2 Latent" → inputs.width / height + * "KSampler" → inputs.seed = random int + */ + +import type { + ImageGenerationConfig, + ImageGenerationOptions, + ImageGenerationResult, +} from '../types'; +import { aspectRatioToDimensions, IMAGE_PROVIDERS } from '../image-providers'; + +// --------------------------------------------------------------------------- +// Logger (matches openmaic's [TIMESTAMP] [LEVEL] [Component] format) +// --------------------------------------------------------------------------- + +const COMPONENT = 'ComfyUI Image'; + +const log = { + info: (msg: string) => console.log(`[${new Date().toISOString()}] [INFO] [${COMPONENT}] ${msg}`), + warn: (msg: string) => + console.warn(`[${new Date().toISOString()}] [WARN] [${COMPONENT}] ${msg}`), + error: (msg: string) => + console.error(`[${new Date().toISOString()}] [ERROR] [${COMPONENT}] ${msg}`), + debug: (msg: string) => + console.debug(`[${new Date().toISOString()}] [DEBUG] [${COMPONENT}] ${msg}`), +}; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const DEFAULT_BASE_URL = 'http://localhost:8188'; +/** Default workflow filename (relative to Next.js public/) */ +const DEFAULT_WORKFLOW_FILENAME = 'comfyui-workflow.json'; +/** Default public path for the workflow JSON (relative to Next.js public/) */ +const DEFAULT_WORKFLOW_PUBLIC_PATH = `/${DEFAULT_WORKFLOW_FILENAME}`; +/** Polling interval while waiting for the queue to finish (ms) */ +const POLL_INTERVAL_MS = 1500; +/** Hard timeout for a single generation request (ms) */ +const GENERATION_TIMEOUT_MS = 300_000; // 5 minutes +/** + * Per-request timeout for individual ComfyUI HTTP calls (ms). The 5-minute + * bound above is on the *polling loop* only — without this, an awaited call + * whose socket connects but never responds (queuePrompt / pollHistory / + * fetchImageAsBase64) could hang indefinitely. + */ +const FETCH_TIMEOUT_MS = 30_000; +/** Timeout for the lightweight connectivity probe (ms) */ +const CONNECTIVITY_TIMEOUT_MS = 10_000; +/** Fallback max dimension if provider has no maxResolution defined */ +const DEFAULT_MAX_WIDTH = 1024; + +// --------------------------------------------------------------------------- +// Extended config type (avoids touching shared types.ts) +// --------------------------------------------------------------------------- + +interface ComfyUIImageGenerationConfig extends ImageGenerationConfig { + /** + * Pre-parsed workflow object. When supplied the adapter skips the + * fetch() call and uses this directly (deep-cloned on each request). + */ + workflowJson?: Record; + /** + * Public URL path to fetch the workflow JSON from. + * Defaults to "/comfyui-workflow.json" (served from Next.js public/). + */ + workflowPublicPath?: string; +} + +// --------------------------------------------------------------------------- +// Workflow helpers +// --------------------------------------------------------------------------- + +/** + * Load and deep-clone the workflow. + * - Browser: fetch() from the Next.js public/ origin + * - Server (API route): read directly from disk with fs — relative URLs + * don't work in Node fetch since there's no browser origin to resolve against + * + * Security: config.model is client-controlled (it flows straight from the + * x-image-model request header). It is never trusted as a raw path — it is + * checked against isComfyuiWorkflowFilename() (bare basename, no traversal) + * AND against the live directory listing from listComfyuiWorkflowFilenames() + * (the same list /api/comfyui-workflows shows the user), before it's used to + * build a filesystem path. See lib/media/comfyui-workflows.ts. + */ +async function loadWorkflow( + config: ComfyUIImageGenerationConfig, +): Promise> { + // Fast path: caller already supplied the parsed JSON. + if (config.workflowJson) { + log.debug('Using pre-supplied workflowJson (skipping fetch)'); + return JSON.parse(JSON.stringify(config.workflowJson)) as Record; + } + + // Server-side: read from disk (public/ directory relative to cwd) + if (typeof window === 'undefined') { + const fs = await import('fs'); + const path = await import('path'); + const { isComfyuiWorkflowFilename, listComfyuiWorkflowFilenames } = + await import('../comfyui-workflows'); + + let filename: string; + if (config.workflowPublicPath) { + // Server-set override (e.g. a caller passing workflowJson's sibling + // path directly) — not client-controlled, so a plain basename() is + // enough here rather than the full allowlist check below. + filename = path.basename(config.workflowPublicPath); + } else if (config.model) { + // Client-controlled (x-image-model header) — must be a safe basename + // AND a real file /api/comfyui-workflows would also list. Anything + // else is rejected outright rather than silently falling back, so a + // caller can't probe the filesystem via unexpected .json files. + if (!isComfyuiWorkflowFilename(config.model)) { + log.error(`Rejected unsafe workflow identifier: "${config.model}"`); + throw new Error(`ComfyUI: "${config.model}" is not a valid workflow filename.`); + } + const known = await listComfyuiWorkflowFilenames(); + if (!known.includes(config.model)) { + log.error(`Rejected unknown workflow identifier: "${config.model}"`); + throw new Error( + `ComfyUI: workflow "${config.model}" was not found. ` + + 'Choose one returned by /api/comfyui-workflows.', + ); + } + filename = config.model; + } else { + // No workflow specified. This happens on two real paths: + // 1. Classroom / autonomous media generation, which has no model id + // to pass for ComfyUI (IMAGE_PROVIDERS['comfyui-image'].models is []). + // 2. The provider is selected in Settings but no workflow has been + // clicked yet, so x-image-model is empty. + // Default to the first workflow actually discovered in public/ rather + // than a hard-coded name: the set of workflow files is user-supplied and + // nothing guarantees any particular filename (e.g. comfyui-workflow.json) + // exists. This is the same list /api/comfyui-workflows shows the user. + const known = await listComfyuiWorkflowFilenames(); + if (known.length === 0) { + log.error('No ComfyUI workflow files found in public/'); + throw new Error( + 'ComfyUI: no workflow JSON files found in the public/ folder. ' + + 'Add at least one comfyui-*.json workflow — see comfyui-setup-instructions.md.', + ); + } + filename = known[0]; + log.info(`No workflow specified — defaulting to first available: "${filename}"`); + } + + const publicDir = path.join(process.cwd(), 'public'); + const filePath = path.join(publicDir, filename); + + // Defense in depth: even after the allowlist check above, verify the + // resolved path is still inside public/ before reading it. path.join + // does NOT stop ".." segments from escaping a base directory, so this + // check — not the join above — is what actually prevents traversal. + const resolvedPublicDir = path.resolve(publicDir) + path.sep; + if (!path.resolve(filePath).startsWith(resolvedPublicDir)) { + log.error(`Refusing to read outside public/ directory: "${filePath}"`); + throw new Error('ComfyUI: resolved workflow path escapes the public/ directory.'); + } + + log.info(`Loading workflow from disk: "${filePath}"`); + if (!fs.existsSync(filePath)) { + log.error(`Workflow file not found at "${filePath}"`); + throw new Error( + `ComfyUI: workflow file not found at "${filePath}". ` + + 'Place comfyui-workflow.json in your Next.js public/ folder.', + ); + } + const raw = fs.readFileSync(filePath, 'utf-8'); + log.debug(`Workflow loaded from disk successfully`); + return JSON.parse(raw) as Record; + } + + // Browser-side: fetch from origin. + // + // NOTE: generation always runs server-side (via the API route), so this + // branch is currently unreachable for real generations. It enforces the + // basename/traversal check (isComfyuiWorkflowFilename) but NOT the live + // directory allowlist the server branch uses — listComfyuiWorkflowFilenames() + // reads the filesystem and deliberately returns [] in the browser, so there + // is nothing to check against here. The basename check is still the + // traversal guard; the missing allowlist is an intentional consequence of + // this path being client-side, not an oversight. + let publicPath = DEFAULT_WORKFLOW_PUBLIC_PATH; + if (config.workflowPublicPath) { + publicPath = config.workflowPublicPath; + } else if (config.model) { + const { isComfyuiWorkflowFilename } = await import('../comfyui-workflows'); + if (!isComfyuiWorkflowFilename(config.model)) { + log.error(`Rejected unsafe workflow identifier: "${config.model}"`); + throw new Error(`ComfyUI: "${config.model}" is not a valid workflow filename.`); + } + publicPath = `/${config.model}`; + } + + const url = `${window.location.origin}${publicPath}`; + log.info(`Loading workflow from "${url}"`); + const response = await fetch(url); + if (!response.ok) { + log.error(`Failed to load workflow from "${url}" (HTTP ${response.status})`); + throw new Error( + `ComfyUI: could not load workflow from "${url}" (HTTP ${response.status}). ` + + 'Place comfyui-workflow.json in your Next.js public/ folder.', + ); + } + + log.debug(`Workflow loaded from URL successfully`); + return (await response.json()) as Record; +} + +/** + * Resolve the output dimensions for a request, clamped to fit inside the + * provider's maxResolution bounding box while preserving aspect ratio. + * + * aspectRatioToDimensions() always pins width to maxWidth and scales height, + * so a portrait ratio overflows the declared max height (e.g. 9:16 at + * maxWidth 1920 → 1920×3413, well past a 1920×1920 cap). Feeding that + * straight into the latent-size node can OOM or fail generation. Scaling + * both axes down by the tighter of the width/height ratios keeps the result + * inside maxWidth × maxHeight without distorting the aspect ratio. + * + * Returns null when no dimensions can be resolved (caller uses workflow + * defaults in that case). + */ +function resolveDimensions( + options: ImageGenerationOptions, + maxWidth: number, + maxHeight: number, +): { width: number; height: number } | null { + const raw = options.aspectRatio + ? aspectRatioToDimensions(options.aspectRatio, maxWidth) + : options.width && options.height + ? { width: options.width, height: options.height } + : null; + if (!raw) return null; + + const scale = Math.min(1, maxWidth / raw.width, maxHeight / raw.height); + return { + width: Math.round(raw.width * scale), + height: Math.round(raw.height * scale), + }; +} + +/** + * Safely return a workflow node's `inputs` object, or undefined if the node + * is malformed (missing/!object inputs). Guards the direct `node.inputs[...]` + * assignments below against a TypeError on a hand-edited workflow file. + */ +function nodeInputs(node: unknown): Record | undefined { + const inputs = (node as Record | undefined)?.['inputs']; + return inputs && typeof inputs === 'object' ? (inputs as Record) : undefined; +} + +/** + * Walk every node in the workflow and return the id of the first node + * whose _meta.title matches title (case-insensitive). + */ +function findNodeIdByTitle(workflow: Record, title: string): string | undefined { + const lower = title.toLowerCase(); + for (const [id, node] of Object.entries(workflow)) { + const meta = (node as Record)['_meta'] as Record | undefined; + if (typeof meta?.title === 'string' && meta.title.toLowerCase() === lower) { + return id; + } + } + return undefined; +} + +/** + * Patch the workflow clone with the caller-supplied generation options. + * + * Prompt injection priority: + * 1. Node titled "Input Prompt" (preferred — explicit dedicated node) + * 2. Node titled "String (Multiline - Prompt)" (legacy fallback) + * + * Dimension injection priority: + * 1. Nodes titled "Width" and "Height" (preferred — explicit dedicated nodes) + * 2. Node titled "Empty Flux 2 Latent" (legacy fallback — patches inputs directly) + */ +function patchWorkflow( + workflow: Record, + options: ImageGenerationOptions, + maxWidth: number, + maxHeight: number, +): void { + // --- Resolve dimensions once (clamped to the provider's max resolution) ---- + const dims = resolveDimensions(options, maxWidth, maxHeight); + + // --- Prompt node ----------------------------------------------------------- + // Try "Input Prompt" first, fall back to "String (Multiline - Prompt)" + const promptNodeId = + findNodeIdByTitle(workflow, 'Input Prompt') ?? + findNodeIdByTitle(workflow, 'String (Multiline - Prompt)'); + + if (!promptNodeId) { + log.error('No prompt node found — add a node titled "Input Prompt" to your workflow'); + throw new Error( + 'ComfyUI workflow is missing a prompt input node. ' + + 'Add a node titled "Input Prompt" (or "String (Multiline - Prompt)") to your workflow.', + ); + } + const promptInputs = nodeInputs(workflow[promptNodeId]); + if (!promptInputs) { + log.error(`Prompt node (id: ${promptNodeId}) has no "inputs" object`); + throw new Error( + `ComfyUI workflow prompt node (id: ${promptNodeId}) is malformed — it has no "inputs" object. ` + + 'Re-export the workflow in API format (see comfyui-setup-instructions.md).', + ); + } + promptInputs['value'] = options.prompt; + log.debug( + `Patched prompt node (id: ${promptNodeId}) → "${options.prompt.slice(0, 80)}${options.prompt.length > 80 ? '…' : ''}"`, + ); + + // --- Width / Height nodes (preferred) ------------------------------------- + const widthNodeId = findNodeIdByTitle(workflow, 'Width'); + const heightNodeId = findNodeIdByTitle(workflow, 'Height'); + + if (widthNodeId && heightNodeId) { + // Explicit Width / Height primitive nodes found — patch them + if (dims) { + const widthInputs = nodeInputs(workflow[widthNodeId]); + const heightInputs = nodeInputs(workflow[heightNodeId]); + if (widthInputs && heightInputs) { + widthInputs['value'] = dims.width; + heightInputs['value'] = dims.height; + log.debug(`Patched Width node (id: ${widthNodeId}) → ${dims.width}`); + log.debug(`Patched Height node (id: ${heightNodeId}) → ${dims.height}`); + } else { + log.warn('Width/Height nodes are malformed (missing "inputs") — using workflow defaults'); + } + } else { + log.debug('Width/Height nodes found but no dimensions resolved — using workflow defaults'); + } + } else { + // Fall back to patching the latent size node directly + if (widthNodeId || heightNodeId) { + log.warn( + 'Only one of "Width"/"Height" nodes found — both are needed. Falling back to latent node.', + ); + } + const latentNodeId = findNodeIdByTitle(workflow, 'Empty Flux 2 Latent'); + if (latentNodeId) { + const latentInputs = nodeInputs(workflow[latentNodeId]); + if (dims && latentInputs) { + latentInputs['width'] = dims.width; + latentInputs['height'] = dims.height; + log.debug( + `Patched latent size node (id: ${latentNodeId}) → ${dims.width}×${dims.height} (aspectRatio: ${options.aspectRatio ?? 'none'})`, + ); + } else if (dims && !latentInputs) { + log.warn( + `Latent size node (id: ${latentNodeId}) is malformed (missing "inputs") — using workflow defaults`, + ); + } else { + log.debug( + `Latent size node (id: ${latentNodeId}) — no dimensions resolved, using workflow defaults`, + ); + } + } else { + log.warn( + 'No dimension nodes found ("Width"/"Height" or "Empty Flux 2 Latent") — using workflow defaults', + ); + } + } + + // --- KSampler seed --------------------------------------------------------- + const samplerNodeId = findNodeIdByTitle(workflow, 'KSampler'); + if (samplerNodeId) { + const samplerInputs = nodeInputs(workflow[samplerNodeId]); + if (samplerInputs) { + const seed = Math.floor(Math.random() * 1e15); + samplerInputs['seed'] = seed; + log.debug(`Patched KSampler seed (id: ${samplerNodeId}) → ${seed}`); + } else { + log.warn( + `KSampler node (id: ${samplerNodeId}) is malformed (missing "inputs") — seed not randomised`, + ); + } + } else { + log.warn('KSampler node not found — seed not randomised'); + } +} + +// --------------------------------------------------------------------------- +// ComfyUI REST helpers +// --------------------------------------------------------------------------- + +interface QueuePromptResponse { + prompt_id: string; + number: number; + node_errors: Record; +} + +interface HistoryEntry { + outputs: Record< + string, + { + images?: Array<{ filename: string; subfolder: string; type: string }>; + } + >; + status: { + status_str: string; + completed: boolean; + // ComfyUI records execution events here as [eventName, data] tuples. + // On failure an "execution_error" event carries the real reason. + messages?: Array<[string, Record]>; + }; +} + +/** + * Pull a human-readable reason out of a failed history entry's message log. + * Returns undefined if no execution_error detail is present. + */ +function extractExecutionError(entry: HistoryEntry): string | undefined { + const messages = entry.status?.messages; + if (!Array.isArray(messages)) return undefined; + for (const [event, data] of messages) { + if (event === 'execution_error' && data) { + const nodeType = typeof data['node_type'] === 'string' ? data['node_type'] : undefined; + const exception = + typeof data['exception_message'] === 'string' ? data['exception_message'] : undefined; + const parts = [nodeType, exception].filter(Boolean); + if (parts.length > 0) return parts.join(': '); + return 'execution_error'; + } + } + return undefined; +} + +async function queuePrompt( + baseUrl: string, + workflow: Record, + clientId: string, +): Promise { + log.info(`Submitting workflow to queue [client_id: ${clientId}]`); + const response = await fetch(`${baseUrl}/prompt`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ prompt: workflow, client_id: clientId }), + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + }); + + if (!response.ok) { + const text = await response.text(); + log.error(`/prompt request failed (HTTP ${response.status}): ${text}`); + throw new Error(`ComfyUI /prompt failed (${response.status}): ${text}`); + } + + const data = (await response.json()) as QueuePromptResponse; + + if (data.node_errors && Object.keys(data.node_errors).length > 0) { + log.error(`Node errors returned: ${JSON.stringify(data.node_errors)}`); + throw new Error(`ComfyUI reported node errors: ${JSON.stringify(data.node_errors)}`); + } + + log.info(`Queued successfully — prompt_id: ${data.prompt_id} (queue position: ${data.number})`); + return data.prompt_id; +} + +async function pollHistory(baseUrl: string, promptId: string): Promise { + // A single poll timing out or blipping must not abort the whole generation — + // return null so the caller's loop simply tries again on the next interval. + try { + const response = await fetch(`${baseUrl}/history/${promptId}`, { + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + }); + if (!response.ok) return null; + const data = (await response.json()) as Record; + return data[promptId] ?? null; + } catch (err) { + log.debug(`Poll request failed (will retry): ${err}`); + return null; + } +} + +async function fetchImageAsBase64( + baseUrl: string, + filename: string, + subfolder: string, + type: string, +): Promise { + const params = new URLSearchParams({ filename, subfolder, type }); + const response = await fetch(`${baseUrl}/view?${params.toString()}`, { + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + }); + + if (!response.ok) { + throw new Error(`ComfyUI /view failed (${response.status}) for image "${filename}"`); + } + + const buffer = await response.arrayBuffer(); + + // This runs server-side, so encode with Buffer (single native pass) rather + // than a multi-MB per-byte String.fromCharCode loop. The btoa fallback is + // retained only to keep this module import-safe in the browser bundle; it + // is never exercised in practice because generation runs server-side. + if (typeof Buffer !== 'undefined') { + return Buffer.from(buffer).toString('base64'); + } + const bytes = new Uint8Array(buffer); + let binary = ''; + for (const byte of bytes) binary += String.fromCharCode(byte); + return btoa(binary); +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * Lightweight connectivity test — calls GET /system_stats. + * Returns 200 when ComfyUI is running and reachable. + */ +export async function testComfyuiImageConnectivity( + config: ImageGenerationConfig, +): Promise<{ success: boolean; message: string }> { + const baseUrl = (config.baseUrl || DEFAULT_BASE_URL).replace(/\/$/, ''); + log.info(`Testing connectivity to ${baseUrl}`); + try { + const response = await fetch(`${baseUrl}/system_stats`, { + signal: AbortSignal.timeout(CONNECTIVITY_TIMEOUT_MS), + }); + if (response.ok) { + log.info(`Connectivity test passed — ComfyUI is reachable at ${baseUrl}`); + return { success: true, message: 'Connected to ComfyUI' }; + } + log.warn(`Connectivity test failed — HTTP ${response.status} from ${baseUrl}`); + return { + success: false, + message: `ComfyUI returned HTTP ${response.status}. Is it running at ${baseUrl}?`, + }; + } catch (err) { + log.error(`Connectivity test error: ${err}`); + return { + success: false, + message: `ComfyUI connectivity error: ${err}. Is it running at ${baseUrl}?`, + }; + } +} + +export async function generateWithComfyuiImage( + config: ImageGenerationConfig, + options: ImageGenerationOptions, +): Promise { + const baseUrl = (config.baseUrl || DEFAULT_BASE_URL).replace(/\/$/, ''); + const comfyConfig = config as ComfyUIImageGenerationConfig; + + log.info(`Starting image generation [baseUrl: ${baseUrl}] [model: ${config.model ?? 'default'}]`); + log.info(`Prompt: "${options.prompt.slice(0, 120)}${options.prompt.length > 120 ? '…' : ''}"`); + log.debug( + `Options: ${JSON.stringify({ width: options.width, height: options.height, aspectRatio: options.aspectRatio })}`, + ); + + const startTime = Date.now(); + + // Resolve the provider's max resolution (set in image-providers.ts). Both + // axes are needed so portrait ratios can be clamped to the bounding box. + const maxResolution = IMAGE_PROVIDERS[config.providerId]?.maxResolution; + const maxWidth = maxResolution?.width ?? DEFAULT_MAX_WIDTH; + const maxHeight = maxResolution?.height ?? DEFAULT_MAX_WIDTH; + + // 1. Load and patch the workflow ------------------------------------------- + const workflow = await loadWorkflow(comfyConfig); + patchWorkflow(workflow, options, maxWidth, maxHeight); + + // 2. Client ID for this request -------------------------------------------- + const clientId = `openmaic-${Date.now()}-${Math.random().toString(36).slice(2)}`; + + // 3. Submit to the queue --------------------------------------------------- + const promptId = await queuePrompt(baseUrl, workflow, clientId); + + // 4. Poll history until complete ------------------------------------------- + const deadline = Date.now() + GENERATION_TIMEOUT_MS; + let entry: HistoryEntry | null = null; + let pollCount = 0; + + log.info(`Polling for completion [prompt_id: ${promptId}]`); + while (Date.now() < deadline) { + await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS)); + pollCount++; + entry = await pollHistory(baseUrl, promptId); + + // Fail fast on a runtime execution error. A workflow that errors mid-run + // records completed:false with status_str:"error", so without this check + // the loop would poll the full timeout and then throw a misleading + // "timed out" instead of the real cause. + if (entry?.status?.status_str === 'error') { + const detail = extractExecutionError(entry); + log.error(`Workflow execution error [prompt_id: ${promptId}]${detail ? `: ${detail}` : ''}`); + throw new Error( + `ComfyUI workflow execution failed (prompt_id: ${promptId})` + + (detail ? `: ${detail}` : '. Check the ComfyUI server logs for details.'), + ); + } + + if (entry?.status?.completed) { + log.info( + `Generation complete after ${pollCount} poll(s) (${((Date.now() - startTime) / 1000).toFixed(1)}s)`, + ); + break; + } + + if (pollCount % 10 === 0) { + log.debug( + `Still waiting… ${pollCount} polls, ${((Date.now() - startTime) / 1000).toFixed(0)}s elapsed`, + ); + } + } + + if (!entry?.status?.completed) { + log.error( + `Generation timed out after ${GENERATION_TIMEOUT_MS / 1000}s [prompt_id: ${promptId}]`, + ); + throw new Error( + `ComfyUI generation timed out after ${GENERATION_TIMEOUT_MS / 1000}s ` + + `(prompt_id: ${promptId})`, + ); + } + + // 5. Extract the first output image ---------------------------------------- + let imageInfo: { filename: string; subfolder: string; type: string } | undefined; + + for (const nodeOutput of Object.values(entry.outputs)) { + if (nodeOutput.images && nodeOutput.images.length > 0) { + imageInfo = nodeOutput.images[0]; + break; + } + } + + if (!imageInfo) { + log.error('Generation finished but no images found in output nodes'); + throw new Error( + 'ComfyUI finished but returned no images. ' + + 'Check that your workflow includes a SaveImage node.', + ); + } + + log.info(`Fetching image "${imageInfo.filename}" from ComfyUI /view`); + + // 6. Download and encode the image ----------------------------------------- + const base64 = await fetchImageAsBase64( + baseUrl, + imageInfo.filename, + imageInfo.subfolder, + imageInfo.type, + ); + + const totalMs = Date.now() - startTime; + // Report the same clamped dimensions that were patched into the workflow. + const dims = resolveDimensions(options, maxWidth, maxHeight); + + log.info( + `Image generation complete — ${imageInfo.filename} (${dims?.width ?? options.width ?? 1024}×${dims?.height ?? options.height ?? 1024}) in ${(totalMs / 1000).toFixed(1)}s`, + ); + + return { + base64, + width: dims?.width ?? options.width ?? 1024, + height: dims?.height ?? options.height ?? 1024, + }; +} diff --git a/lib/media/comfyui-workflows.ts b/lib/media/comfyui-workflows.ts new file mode 100644 index 000000000..f7bdc7510 --- /dev/null +++ b/lib/media/comfyui-workflows.ts @@ -0,0 +1,104 @@ +/** + * ComfyUI Workflow Discovery + * + * Single source of truth for "which workflow JSON files exist and are + * selectable". Both `/api/comfyui-workflows` (lists workflows for the + * Settings UI) and the ComfyUI image adapter (validates a client-supplied + * workflow id before reading it from disk) import this module so the two + * can never drift apart — a filename the UI offers is always a filename + * the adapter will accept, and vice versa. + * + * Workflow files live flat in the Next.js `public/` directory and are + * recognised by filename: must end in `.json` and either start with + * "comfyui" or contain "workflow" (case-insensitive). + * + * IMPORTANT: this module is reachable from client bundles (imported by + * the adapter, which is imported by image-providers.ts, which is imported + * by the client-side settings store). `fs`/`path` are therefore imported + * dynamically inside each function rather than statically at the top of + * the file — a top-level `import ... from 'fs'` makes the bundler try to + * resolve `fs` for the browser build too, which fails ("Module not found: + * Can't resolve 'fs'"). The functions below are still only ever *called* + * server-side; the dynamic import just keeps the module import-safe. + */ + +export interface ComfyuiWorkflowEntry { + id: string; + name: string; +} + +/** Convert a workflow filename to a human-readable display name. */ +export function filenameToDisplayName(filename: string): string { + return ( + filename + .replace(/\.json$/i, '') // strip extension + .replace(/^comfyui[-_]?/i, '') // strip leading "comfyui-" or "comfyui_" + .replace(/[-_]+/g, ' ') // hyphens/underscores → spaces + .trim() + .replace(/\b\w/g, (c) => c.toUpperCase()) || // title-case + 'Default Workflow' + ); // fallback if name becomes empty +} + +/** + * Whether a filename looks like a ComfyUI workflow file, per the naming + * convention documented above. Also rejects anything that isn't a bare + * basename (no path separators, no "..") so this doubles as the safety + * check for client-supplied workflow ids. + */ +export function isComfyuiWorkflowFilename(filename: string): boolean { + if (filename.includes('/') || filename.includes('\\') || filename.includes('..')) { + return false; + } + const lower = filename.toLowerCase(); + return lower.endsWith('.json') && (lower.startsWith('comfyui') || lower.includes('workflow')); +} + +/** + * List the workflow JSON files actually present in `public/`, with display + * names. Returns `[]` if `public/` doesn't exist, on any read error, or if + * called from the browser (this is server-only — see file-level note). + * + * Async because fs/path are loaded dynamically — callers must `await` this. + * + * The `typeof window === 'undefined'` guard below isn't just a runtime + * safety check: Turbopack/webpack can statically eliminate code behind it + * for the client bundle, which is what lets the dynamic `import('fs')` + * resolve at all. Without the guard, the bundler can't prove this branch + * is server-only and tries to resolve 'fs' for the browser build too, + * which fails ("Module not found: Can't resolve 'fs'"). This matches the + * existing pattern in comfyui-image-adapter.ts's loadWorkflow(). + */ +export async function listComfyuiWorkflows(): Promise { + // Server-side only — see file-level note. The `typeof window === 'undefined'` + // guard must wrap the dynamic import directly (not an early-return above + // it) to match the shape Turbopack already dead-code-eliminates correctly + // in comfyui-image-adapter.ts's loadWorkflow(). + if (typeof window === 'undefined') { + try { + const fs = await import('fs'); + const path = await import('path'); + const publicDir = path.join(process.cwd(), 'public'); + if (!fs.existsSync(publicDir)) return []; + + return fs + .readdirSync(publicDir) + .filter( + (f) => isComfyuiWorkflowFilename(f) && fs.statSync(path.join(publicDir, f)).isFile(), + ) + .map((filename) => ({ id: filename, name: filenameToDisplayName(filename) })) + .sort((a, b) => a.name.localeCompare(b.name)); + } catch (err) { + console.error('[ComfyUI Workflows] Failed to list workflows:', err); + return []; + } + } + + return []; +} + +/** Just the filenames, for allowlist membership checks. */ +export async function listComfyuiWorkflowFilenames(): Promise { + const workflows = await listComfyuiWorkflows(); + return workflows.map((w) => w.id); +} diff --git a/lib/media/image-providers.ts b/lib/media/image-providers.ts index 4083dcfd3..593410535 100644 --- a/lib/media/image-providers.ts +++ b/lib/media/image-providers.ts @@ -21,6 +21,10 @@ import { testMiniMaxImageConnectivity, } from './adapters/minimax-image-adapter'; import { generateWithGrokImage, testGrokImageConnectivity } from './adapters/grok-image-adapter'; +import { + generateWithComfyuiImage, + testComfyuiImageConnectivity, +} from './adapters/comfyui-image-adapter'; import { generateWithLemonadeImage, testLemonadeImageConnectivity, @@ -121,6 +125,26 @@ export const IMAGE_PROVIDERS: Record = { ], supportedAspectRatios: ['16:9', '4:3', '1:1', '9:16'], }, + 'comfyui-image': { + id: 'comfyui-image', + name: 'ComfyUI Image', + requiresApiKey: false, + defaultBaseUrl: 'http://localhost:8188', + // No static models here — real selectable workflows are discovered at + // runtime from GET /api/comfyui-workflows (files in public/) and picked + // in Settings. A placeholder id like "comfyui-image" doesn't correspond + // to any real workflow file, so resolveSelectedModel() would resolve it + // to a dead path the first time this provider became active (#P2). + // With models: [], imageModelId resolves to '' when this provider is + // selected with no workflow chosen yet. In that case (and on the + // autonomous classroom-media path, which has no model id to pass) the + // adapter's loadWorkflow() defaults to the first workflow file discovered + // in public/ via listComfyuiWorkflowFilenames() — not a hard-coded + // filename, since no particular workflow name is guaranteed to exist. + models: [], + supportedAspectRatios: ['16:9', '4:3', '1:1', '9:16'], + maxResolution: { width: 1920, height: 1920 }, + }, lemonade: { id: 'lemonade', name: 'Lemonade', @@ -152,6 +176,8 @@ export async function testImageConnectivity( return testMiniMaxImageConnectivity(config); case 'grok-image': return testGrokImageConnectivity(config); + case 'comfyui-image': + return testComfyuiImageConnectivity(config); case 'lemonade': return testLemonadeImageConnectivity(config); default: @@ -179,6 +205,8 @@ export async function generateImage( return generateWithMiniMaxImage(config, options); case 'grok-image': return generateWithGrokImage(config, options); + case 'comfyui-image': + return generateWithComfyuiImage(config, options); case 'lemonade': return generateWithLemonadeImage(config, options); default: diff --git a/lib/media/types.ts b/lib/media/types.ts index 07dc37126..7d5c1c1dd 100644 --- a/lib/media/types.ts +++ b/lib/media/types.ts @@ -78,6 +78,7 @@ export type ImageProviderId = | 'nano-banana' | 'minimax-image' | 'grok-image' + | 'comfyui-image' | 'lemonade'; // Add new image providers below (uncomment and modify): // | 'dall-e' diff --git a/lib/store/settings.ts b/lib/store/settings.ts index d971163c1..29ddb67ca 100644 --- a/lib/store/settings.ts +++ b/lib/store/settings.ts @@ -485,6 +485,7 @@ const getDefaultImageConfig = () => ({ 'nano-banana': { apiKey: '', baseUrl: '', enabled: false }, 'minimax-image': { apiKey: '', baseUrl: '', enabled: false }, 'grok-image': { apiKey: '', baseUrl: '', enabled: false }, + 'comfyui-image': { apiKey: '', baseUrl: '', enabled: false }, lemonade: { apiKey: '', baseUrl: '', enabled: false }, } as Record, }); diff --git a/public/comfyui-workflow.json b/public/comfyui-workflow.json new file mode 100644 index 000000000..b4bc62c03 --- /dev/null +++ b/public/comfyui-workflow.json @@ -0,0 +1,221 @@ +{ + "73": { + "inputs": { + "filename_prefix": "Ernie-Image-Turbo", + "images": ["88:65", 0] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + }, + "88:71": { + "inputs": { + "width": 1024, + "height": 1024, + "batch_size": 1 + }, + "class_type": "EmptyFlux2LatentImage", + "_meta": { + "title": "Empty Flux 2 Latent" + } + }, + "88:66": { + "inputs": { + "unet_name": "ernie-image-turbo.safetensors", + "weight_dtype": "default" + }, + "class_type": "UNETLoader", + "_meta": { + "title": "Load Diffusion Model" + } + }, + "88:65": { + "inputs": { + "samples": ["88:70", 0], + "vae": ["88:63", 0] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "88:70": { + "inputs": { + "seed": 239071813140207, + "steps": 8, + "cfg": 1, + "sampler_name": "euler", + "scheduler": "simple", + "denoise": 1, + "model": ["88:66", 0], + "positive": ["88:67", 0], + "negative": ["88:91", 0], + "latent_image": ["88:71", 0] + }, + "class_type": "KSampler", + "_meta": { + "title": "KSampler" + } + }, + "88:67": { + "inputs": { + "text": ["88:97", 0], + "clip": ["88:62", 0] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "88:62": { + "inputs": { + "clip_name": "ministral-3-3b.safetensors", + "type": "flux2", + "device": "default" + }, + "class_type": "CLIPLoader", + "_meta": { + "title": "Load CLIP" + } + }, + "88:63": { + "inputs": { + "vae_name": "flux2-vae.safetensors" + }, + "class_type": "VAELoader", + "_meta": { + "title": "Load VAE" + } + }, + "88:91": { + "inputs": { + "conditioning": ["88:67", 0] + }, + "class_type": "ConditioningZeroOut", + "_meta": { + "title": "ConditioningZeroOut" + } + }, + "88:93": { + "inputs": { + "string": "[SYSTEM_PROMPT]你是一个专业的文生图 Prompt 增强助手。你将收到用户的简短图片描述及目标生成分辨率，请据此扩写为一段内容丰富、细节充分的视觉描述，以帮助文生图模型生成高质量的图片。仅输出增强后的描述，不要包含任何解释或前缀。[/SYSTEM_PROMPT][INST]{\"prompt\": \"{prompt}\", \"width\": {width}, \"height\": {height}}[/INST]", + "find": "{prompt}", + "replace": ["88:94", 0] + }, + "class_type": "StringReplace", + "_meta": { + "title": "Text Replace" + } + }, + "88:94": { + "inputs": { + "value": "" + }, + "class_type": "PrimitiveStringMultiline", + "_meta": { + "title": "String (Multiline - Prompt)" + } + }, + "88:95": { + "inputs": { + "prompt": ["88:102", 0], + "max_length": 2048, + "sampling_mode": "on", + "sampling_mode.temperature": 0.6, + "sampling_mode.top_k": 64, + "sampling_mode.top_p": 0.8, + "sampling_mode.min_p": 0.05, + "sampling_mode.repetition_penalty": 1.05, + "sampling_mode.seed": 0, + "sampling_mode.presence_penalty": 0, + "thinking": false, + "use_default_template": true, + "clip": ["88:98", 0] + }, + "class_type": "TextGenerate", + "_meta": { + "title": "TextGenerate" + } + }, + "88:96": { + "inputs": { + "value": false + }, + "class_type": "PrimitiveBoolean", + "_meta": { + "title": "Enable prompt enhancement?" + } + }, + "88:97": { + "inputs": { + "switch": ["88:96", 0], + "on_false": ["88:94", 0], + "on_true": ["88:95", 0] + }, + "class_type": "ComfySwitchNode", + "_meta": { + "title": "Switch" + } + }, + "88:98": { + "inputs": { + "clip_name": "ernie-image-prompt-enhancer.safetensors", + "type": "flux2", + "device": "default" + }, + "class_type": "CLIPLoader", + "_meta": { + "title": "Load CLIP (PE)" + } + }, + "88:99": { + "inputs": { + "source": 1024 + }, + "class_type": "PreviewAny", + "_meta": { + "title": "Preview as Text (Int to String)" + } + }, + "88:100": { + "inputs": { + "source": 1024 + }, + "class_type": "PreviewAny", + "_meta": { + "title": "Preview as Text (Int to String)" + } + }, + "88:101": { + "inputs": { + "string": ["88:93", 0], + "find": "{width}", + "replace": ["88:99", 0] + }, + "class_type": "StringReplace", + "_meta": { + "title": "Text Replace" + } + }, + "88:102": { + "inputs": { + "string": ["88:101", 0], + "find": "{height}", + "replace": ["88:100", 0] + }, + "class_type": "StringReplace", + "_meta": { + "title": "Text Replace" + } + }, + "88:103": { + "inputs": { + "source": ["88:97", 0] + }, + "class_type": "PreviewAny", + "_meta": { + "title": "Preview as Text (Int to String)" + } + } +} diff --git a/public/logos/comfyui.svg b/public/logos/comfyui.svg new file mode 100644 index 000000000..554a3bb53 --- /dev/null +++ b/public/logos/comfyui.svg @@ -0,0 +1 @@ +ComfyUI \ No newline at end of file