From 907302e1dac6904e568aa35e09c1e6f0f12b6371 Mon Sep 17 00:00:00 2001 From: mertmit Date: Mon, 4 May 2026 10:31:16 +0300 Subject: [PATCH] chore: sync changes Signed-off-by: mertmit --- .gitignore | 7 + packages/nc-gui/app.vue | 1 + .../nc-gui/components/account/License.vue | 5 +- .../components/cell/attachment/utils.ts | 54 +-- .../TreeView/Project/BaseSettingsMenu.vue | 10 + packages/nc-gui/components/dashboard/View.vue | 16 +- .../dashboard/settings/base/MigrateToV3.vue | 10 +- .../settings/base/Variables/index.vue | 3 + .../nc-gui/components/dlg/View/Create.vue | 92 +++-- packages/nc-gui/components/project/View.vue | 20 +- .../components/sandbox/ChangelogDrawer.vue | 3 + .../nc-gui/components/smartsheet/Topbar.vue | 3 + .../smartsheet/expanded-form/index.vue | 1 + .../smartsheet/toolbar/ViewActionMenu.vue | 8 + .../topbar/SandboxStatus/DiffPreview.vue | 3 + .../topbar/VariableSetupWarning.vue | 3 + .../workspace/project/AppMarket.vue | 28 +- .../nc-gui/composables/useBaseVariables.ts | 29 ++ .../composables/useBetaFeatureToggle.ts | 8 - packages/nc-gui/composables/useEeConfig.ts | 6 + packages/nc-gui/composables/useMetas.ts | 42 ++- .../nc-gui/composables/useSandboxChangelog.ts | 26 ++ packages/nc-gui/lang/en.json | 81 +++- packages/nc-gui/utils/settingsRouteUtils.ts | 1 + .../nocodb-sdk/src/lib/base-variable/index.ts | 26 ++ packages/nocodb-sdk/src/lib/enums.ts | 14 + .../src/lib/error-handler/nc-error-base.ts | 19 +- .../error-handler/preset-error-codex-map.ts | 11 + packages/nocodb-sdk/src/lib/globals.ts | 15 + packages/nocodb-sdk/src/lib/index.ts | 2 + .../nocodb-sdk/src/lib/managed-app/index.ts | 62 ++++ packages/nocodb-sdk/src/lib/payment/index.ts | 2 + packages/nocodb/src/cache/CacheMgr.ts | 47 ++- packages/nocodb/src/cache/NocoCache.ts | 53 ++- packages/nocodb/src/cache/cacheBypassScope.ts | 24 ++ .../nocodb/src/command-registry/op-names.ts | 122 ++++++ .../nocodb/src/command-registry/registry.ts | 29 ++ packages/nocodb/src/command-registry/types.ts | 109 ++++++ .../attachments-secure.controller.ts | 29 +- .../src/controllers/attachments.controller.ts | 28 +- .../modules/AttachmentGet.operations.ts | 44 +++ .../controllers/internal/operationScopes.ts | 4 + .../src/controllers/internal/provider.ts | 2 + .../controllers/v3/tables-v3.controller.ts | 1 + packages/nocodb/src/db/BaseModelSqlv2.ts | 309 ++++++++++----- .../nocodb/src/db/BaseModelSqlv2/insert.ts | 3 +- .../db/BaseModelSqlv2/ltar-cols-updater.ts | 2 +- .../src/db/BaseModelSqlv2/select-object.ts | 2 +- packages/nocodb/src/db/CustomKnex.ts | 14 +- packages/nocodb/src/db/IBaseModelSqlV2.ts | 22 +- .../formula/formula.general.handler.ts | 2 +- .../handlers/lookup/lookup.general.handler.ts | 4 +- .../src/db/formulav2/formulaQueryBuilderv2.ts | 4 +- packages/nocodb/src/db/relation-manager.ts | 2 +- .../src/db/sql-data-mapper/lib/BaseModel.ts | 13 +- packages/nocodb/src/dbQueryClient/generic.ts | 2 +- .../src/decorators/trace-command.decorator.ts | 12 + .../global-exception.filter.ts | 62 +--- packages/nocodb/src/helpers/columnHelpers.ts | 40 +- packages/nocodb/src/helpers/dbHelpers.ts | 6 + packages/nocodb/src/helpers/sandboxGuards.ts | 20 + packages/nocodb/src/helpers/tableHelpers.ts | 5 +- .../nocodb/src/helpers/throttlerLogger.ts | 99 +++++ packages/nocodb/src/helpers/webhookHelpers.ts | 7 +- packages/nocodb/src/interface/Jobs.ts | 24 ++ packages/nocodb/src/main.ts | 3 +- packages/nocodb/src/meta/meta.service.ts | 10 +- .../meta/migrations/XcMigrationSourcev0.ts | 4 + .../v0/nc_202604160000_docs_in_data.ts | 9 +- ...00_base_variables_and_sandbox_changelog.ts | 152 ++++++++ packages/nocodb/src/models/Base.ts | 6 +- packages/nocodb/src/models/BaseVariable.ts | 351 ++++++++++++++++++ .../nocodb/src/models/CalendarViewColumn.ts | 14 +- packages/nocodb/src/models/Column.ts | 10 +- packages/nocodb/src/models/Filter.ts | 60 +-- .../nocodb/src/models/GalleryViewColumn.ts | 14 +- packages/nocodb/src/models/GridViewColumn.ts | 14 +- packages/nocodb/src/models/Hook.ts | 10 + packages/nocodb/src/models/Model.ts | 6 + packages/nocodb/src/models/View.ts | 2 + packages/nocodb/src/models/index.ts | 1 + .../jobs/at-import/at-import.processor.ts | 7 + .../jobs/export-import/duplicate.processor.ts | 4 + .../jobs/export-import/duplicate.service.ts | 6 + .../jobs/jobs/export-import/import.service.ts | 67 +++- .../jobs/export-import/migrate.service.ts | 13 +- .../nc_job_010_soft_delete_column.ts | 32 +- packages/nocodb/src/run/docker.ts | 3 +- packages/nocodb/src/run/dockerRunMysql.ts | 3 +- packages/nocodb/src/run/dockerRunPG.ts | 3 +- .../nocodb/src/run/dockerRunPG_CyQuick.ts | 3 +- packages/nocodb/src/run/testDocker.ts | 3 +- packages/nocodb/src/schema/swagger-v2.json | 4 +- packages/nocodb/src/schema/swagger.json | 31 +- .../src/services/attachments.service.ts | 44 ++- .../nocodb/src/services/calendars.service.ts | 19 +- .../nocodb/src/services/columns.service.ts | 114 +++++- .../src/services/columns.service.type.ts | 37 ++ .../nocodb/src/services/data-table.service.ts | 4 +- .../nocodb/src/services/extensions.service.ts | 2 +- .../src/services/form-columns.service.ts | 16 +- packages/nocodb/src/services/forms.service.ts | 19 +- .../nocodb/src/services/galleries.service.ts | 19 +- .../src/services/grid-columns.service.ts | 16 +- packages/nocodb/src/services/grids.service.ts | 19 +- .../nocodb/src/services/kanbans.service.ts | 19 +- packages/nocodb/src/services/maps.service.ts | 7 +- ...pdate-kanban-groupby-dependency.handler.ts | 2 + .../services/model-visibilities.service.ts | 3 + .../src/services/shared-bases.service.ts | 12 + .../nocodb/src/services/tables.service.ts | 3 + .../nocodb/src/services/v3/data-v3.service.ts | 16 +- .../src/services/view-columns.service.ts | 2 +- packages/nocodb/src/services/views.service.ts | 2 +- packages/nocodb/src/utils/acl.ts | 10 + packages/nocodb/src/utils/globals.ts | 6 + packages/nocodb/src/utils/internal-type.ts | 10 +- packages/nocodb/src/utils/webhook-invoker.ts | 44 ++- tsconfig.json | 11 +- 119 files changed, 2552 insertions(+), 506 deletions(-) create mode 100644 packages/nc-gui/components/dashboard/settings/base/Variables/index.vue create mode 100644 packages/nc-gui/components/sandbox/ChangelogDrawer.vue create mode 100644 packages/nc-gui/components/smartsheet/topbar/SandboxStatus/DiffPreview.vue create mode 100644 packages/nc-gui/components/smartsheet/topbar/VariableSetupWarning.vue create mode 100644 packages/nc-gui/composables/useBaseVariables.ts create mode 100644 packages/nc-gui/composables/useSandboxChangelog.ts create mode 100644 packages/nocodb-sdk/src/lib/base-variable/index.ts create mode 100644 packages/nocodb-sdk/src/lib/managed-app/index.ts create mode 100644 packages/nocodb/src/cache/cacheBypassScope.ts create mode 100644 packages/nocodb/src/command-registry/op-names.ts create mode 100644 packages/nocodb/src/command-registry/registry.ts create mode 100644 packages/nocodb/src/command-registry/types.ts create mode 100644 packages/nocodb/src/controllers/internal/modules/AttachmentGet.operations.ts create mode 100644 packages/nocodb/src/decorators/trace-command.decorator.ts create mode 100644 packages/nocodb/src/helpers/sandboxGuards.ts create mode 100644 packages/nocodb/src/helpers/throttlerLogger.ts create mode 100644 packages/nocodb/src/meta/migrations/v0/nc_202604290000_base_variables_and_sandbox_changelog.ts create mode 100644 packages/nocodb/src/models/BaseVariable.ts diff --git a/.gitignore b/.gitignore index a9cd1aa0d3..d291bd50f3 100644 --- a/.gitignore +++ b/.gitignore @@ -116,3 +116,10 @@ antd.variable.css .playwright-mcp .playwright-cli .claude/skills/nocodb-dev-api/.credentials.json + +# nc-dev +.nc-dev-config.json +.nc-dev-registry.json +.nc-dev-proxy.pid +.nc-dev-proxy.log +.nc-dev-logs/ diff --git a/packages/nc-gui/app.vue b/packages/nc-gui/app.vue index 7f4484919a..3456e43539 100644 --- a/packages/nc-gui/app.vue +++ b/packages/nc-gui/app.vue @@ -163,4 +163,5 @@ const _ = (window as any).ResizeObserver + diff --git a/packages/nc-gui/components/account/License.vue b/packages/nc-gui/components/account/License.vue index 27caaeb770..fc24e738aa 100644 --- a/packages/nc-gui/components/account/License.vue +++ b/packages/nc-gui/components/account/License.vue @@ -272,7 +272,10 @@ loadLicense() -
+
{{ licenseStatus === 'none' ? $t('labels.buyLicense') : $t('labels.manageLicense') }} diff --git a/packages/nc-gui/components/cell/attachment/utils.ts b/packages/nc-gui/components/cell/attachment/utils.ts index 6c2f84e97d..5f81eb44d0 100644 --- a/packages/nc-gui/components/cell/attachment/utils.ts +++ b/packages/nc-gui/components/cell/attachment/utils.ts @@ -398,17 +398,20 @@ export const [useProvideAttachmentCell, useAttachmentCell] = useInjectionState( continue } - const apiPromise = isPublic.value - ? () => fetchSharedViewAttachment(columnId!, rowId!, src) - : () => - $api.dbDataTableRow.attachmentDownload(modelId!, columnId!, rowId!, { - urlOrPath: src, - }) - let res try { - res = await apiPromise() + if (isPublic.value) { + res = await fetchSharedViewAttachment(columnId!, rowId!, src) + } else { + res = await $api.internal.getOperation(meta.value!.fk_workspace_id!, meta.value!.base_id!, { + operation: 'attachmentDownload', + modelId: modelId!, + columnId: columnId!, + rowId: rowId!, + urlOrPath: src, + }) + } } catch {} if (!res) { @@ -490,22 +493,27 @@ export const [useProvideAttachmentCell, useAttachmentCell] = useInjectionState( const rowId = extractPkFromRow(unref(row).row, meta.value.columns!) const src = item.url || item.path if (modelId && columnId && rowId && src) { - const apiPromise = isPublic.value - ? () => fetchSharedViewAttachment(columnId, rowId, src) - : () => - $api.dbDataTableRow.attachmentDownload(modelId, columnId, rowId, { - urlOrPath: src, - }) + let res - await apiPromise().then((res) => { - if (res?.path) { - window.open(`${baseURL}/${res.path}`, '_self') - } else if (res?.url) { - window.open(res.url, '_self') - } else { - message.error('Failed to download file') - } - }) + if (isPublic.value) { + res = await fetchSharedViewAttachment(columnId, rowId, src) + } else { + res = await $api.internal.getOperation(meta.value.fk_workspace_id!, meta.value.base_id!, { + operation: 'attachmentDownload', + modelId, + columnId, + rowId, + urlOrPath: src, + }) + } + + if (res?.path) { + window.open(`${baseURL}/${res.path}`, '_self') + } else if (res?.url) { + window.open(res.url, '_self') + } else { + message.error('Failed to download file') + } } else { message.error('Failed to download file') } diff --git a/packages/nc-gui/components/dashboard/TreeView/Project/BaseSettingsMenu.vue b/packages/nc-gui/components/dashboard/TreeView/Project/BaseSettingsMenu.vue index ba79c7dcdb..564f51754e 100644 --- a/packages/nc-gui/components/dashboard/TreeView/Project/BaseSettingsMenu.vue +++ b/packages/nc-gui/components/dashboard/TreeView/Project/BaseSettingsMenu.vue @@ -205,6 +205,16 @@ onMounted(() => { /> + + {{ $t('title.baseVariables') }} + - + - + + + + + + + {{ $t(viewLockIcons[option.value].title) }} + + +
diff --git a/packages/nc-gui/components/project/View.vue b/packages/nc-gui/components/project/View.vue index 4db193628e..dc250160ab 100644 --- a/packages/nc-gui/components/project/View.vue +++ b/packages/nc-gui/components/project/View.vue @@ -179,6 +179,8 @@ watch( projectPageTab.value = 'workflows' } else if (newVal === 'mcp') { projectPageTab.value = 'mcp' + } else if (newVal === 'variables') { + projectPageTab.value = 'variables' } else if (newVal === 'snapshots' && isEeUI) { projectPageTab.value = 'snapshots' } else if (newVal === 'record-trash' && isEeUI) { @@ -208,6 +210,7 @@ const settingsPageTitle = computed(() => { 'permissions': t('labels.dataPermissions'), 'docs-permissions': t('labels.docsPermissions'), 'mcp': t('title.mcpServer'), + 'variables': t('title.baseVariables'), 'syncs': t('labels.manageSyncs'), 'snapshots': t('labels.manageSnapshots'), 'record-trash': t('trash.settings'), @@ -232,7 +235,10 @@ watch(projectPageTab, () => { const wsId = route.value.params.typeOrId const baseId = route.value.params.baseId - navigateTo(`/${wsId}/${baseId}/settings/${slug}`) + navigateTo({ + path: `/${wsId}/${baseId}/settings/${slug}`, + query: route.value.query, + }) return } @@ -383,6 +389,7 @@ watch(
+ @@ -553,6 +560,17 @@ watch(
+ + +
+ +
+
+ + diff --git a/packages/nc-gui/components/smartsheet/Topbar.vue b/packages/nc-gui/components/smartsheet/Topbar.vue index 3bd260720a..10c4086d8e 100644 --- a/packages/nc-gui/components/smartsheet/Topbar.vue +++ b/packages/nc-gui/components/smartsheet/Topbar.vue @@ -75,6 +75,9 @@ const topbarBreadcrumbItemWidth = computed(() => {
+ + + diff --git a/packages/nc-gui/components/smartsheet/expanded-form/index.vue b/packages/nc-gui/components/smartsheet/expanded-form/index.vue index 303a313624..0283ad30ea 100644 --- a/packages/nc-gui/components/smartsheet/expanded-form/index.vue +++ b/packages/nc-gui/components/smartsheet/expanded-form/index.vue @@ -332,6 +332,7 @@ const onDuplicateRow = () => { isUnsavedDuplicatedRecordExist.value = true const oldRow = { ..._row.value.row } delete oldRow.ncRecordId + delete oldRow.ncRecordHash const newRow = Object.assign( {}, { diff --git a/packages/nc-gui/components/smartsheet/toolbar/ViewActionMenu.vue b/packages/nc-gui/components/smartsheet/toolbar/ViewActionMenu.vue index 1c3e94c512..0e006ef611 100644 --- a/packages/nc-gui/components/smartsheet/toolbar/ViewActionMenu.vue +++ b/packages/nc-gui/components/smartsheet/toolbar/ViewActionMenu.vue @@ -281,9 +281,16 @@ const modifyViewDisabledReason = computed(() => { return '' }) +// Collaborative views cannot be deleted on a sandbox master base — backend guard mirrors this. +// Personal views (owned_by set) can still be deleted by their owner. +const isSandboxProductionCollaborativeDelete = computed(() => { + return !!base.value?.is_sandbox_production && !view.value?.owned_by +}) + // Tooltip shown when Delete is disabled. const deleteDisabledReason = computed(() => { if (isLockedView.value) return t('msg.info.disabledAsViewLocked') + if (isSandboxProductionCollaborativeDelete.value) return t('msg.info.disabledAsSandboxMasterCollabDelete') if (blockViewOperations.value && !isPersonalView.value) return t('msg.info.cantDeleteLastGridView') if (isPersonalView.value && !isPersonalViewOwner.value) return t('tooltip.onlyViewOwnerCanDeletePersonalView') return '' @@ -293,6 +300,7 @@ const deleteDisabledReason = computed(() => { // (which is enforced for everyone, including creators+). const isDeleteDisabled = computed(() => { if (!canDeleteView.value) return true + if (isSandboxProductionCollaborativeDelete.value) return true if (blockViewOperations.value && !isPersonalView.value) return true return false }) diff --git a/packages/nc-gui/components/smartsheet/topbar/SandboxStatus/DiffPreview.vue b/packages/nc-gui/components/smartsheet/topbar/SandboxStatus/DiffPreview.vue new file mode 100644 index 0000000000..a154d08eb5 --- /dev/null +++ b/packages/nc-gui/components/smartsheet/topbar/SandboxStatus/DiffPreview.vue @@ -0,0 +1,3 @@ + diff --git a/packages/nc-gui/components/smartsheet/topbar/VariableSetupWarning.vue b/packages/nc-gui/components/smartsheet/topbar/VariableSetupWarning.vue new file mode 100644 index 0000000000..a154d08eb5 --- /dev/null +++ b/packages/nc-gui/components/smartsheet/topbar/VariableSetupWarning.vue @@ -0,0 +1,3 @@ + diff --git a/packages/nc-gui/components/workspace/project/AppMarket.vue b/packages/nc-gui/components/workspace/project/AppMarket.vue index 988f9fbba8..129f38f0d1 100644 --- a/packages/nc-gui/components/workspace/project/AppMarket.vue +++ b/packages/nc-gui/components/workspace/project/AppMarket.vue @@ -28,6 +28,12 @@ const installing = ref(null) const searchQuery = ref('') const selectedCategory = ref(undefined) +// Variable setup state +const isVariableSetupVisible = ref(false) +const setupVariables = ref([]) +const setupBaseId = ref('') +const setupBaseTitle = ref('') + const categories = computed(() => { const cats = new Set() managedApps.value.forEach((ma) => { @@ -92,7 +98,7 @@ const loadManagedApps = async () => { const installManagedApp = async (managedApp: ManagedAppType) => { installing.value = managedApp.id try { - await $api.internal.postOperation( + const response = (await $api.internal.postOperation( props.workspaceId, NO_SCOPE, { @@ -102,11 +108,19 @@ const installManagedApp = async (managedApp: ManagedAppType) => { managedAppId: managedApp.id, target_workspace_id: props.workspaceId, }, - ) + )) as any message.success(t('msg.success.baseInstalled')) emit('installed', managedApp) visible.value = false + + // If setup is required, show variable configuration modal + if (response?.setupRequired && response?.setupVariables?.length) { + setupVariables.value = response.setupVariables + setupBaseId.value = response.installedBaseId + setupBaseTitle.value = managedApp.title + isVariableSetupVisible.value = true + } } catch (e: any) { message.error(await extractSdkResponseErrorMsg(e)) } finally { @@ -297,6 +311,16 @@ watch(
+ + diff --git a/packages/nc-gui/composables/useBaseVariables.ts b/packages/nc-gui/composables/useBaseVariables.ts new file mode 100644 index 0000000000..80945a3d15 --- /dev/null +++ b/packages/nc-gui/composables/useBaseVariables.ts @@ -0,0 +1,29 @@ +import type { BaseVariableReqType, BaseVariableType } from 'nocodb-sdk' + +export const useBaseVariables = createSharedComposable(() => { + const variables = ref([]) + const isLoading = ref(false) + + const unconfiguredRequiredVars = computed(() => []) + const hasUnconfiguredVars = computed(() => false) + + const listVariables = async () => {} + const createVariable = async (_body: BaseVariableReqType) => {} + const updateVariable = async (_variableId: string, _body: Partial) => {} + const deleteVariable = async (_variableId: string) => {} + const revertToDefault = async (_variableId: string) => {} + const bulkUpdateValues = async (_baseId: string, _updates: { id: string; value: string }[], _workspaceId?: string) => {} + + return { + variables, + isLoading, + unconfiguredRequiredVars, + hasUnconfiguredVars, + listVariables, + createVariable, + updateVariable, + deleteVariable, + revertToDefault, + bulkUpdateValues, + } +}) diff --git a/packages/nc-gui/composables/useBetaFeatureToggle.ts b/packages/nc-gui/composables/useBetaFeatureToggle.ts index d855b8de6a..4fc9eec2ac 100644 --- a/packages/nc-gui/composables/useBetaFeatureToggle.ts +++ b/packages/nc-gui/composables/useBetaFeatureToggle.ts @@ -2,14 +2,6 @@ import rfdc from 'rfdc' const deepClone = rfdc() const FEATURES = [ - { - id: 'sandbox', - title: 'Sandbox', - description: 'Allow users to create sandbox environments for testing schema changes before merging.', - enabled: false, - isEngineering: true, - isAdvanced: true, - }, { id: 'managed_apps', title: 'Managed Apps', diff --git a/packages/nc-gui/composables/useEeConfig.ts b/packages/nc-gui/composables/useEeConfig.ts index 01d460aad2..f9c1a9b215 100644 --- a/packages/nc-gui/composables/useEeConfig.ts +++ b/packages/nc-gui/composables/useEeConfig.ts @@ -260,6 +260,10 @@ export const useEeConfig = createSharedComposable(() => { const showUpgradeToUseViewSections = (..._args: any[]) => {} + const blockBaseVariables = computed(() => true) + + const showUpgradeToUseBaseVariables = (..._args: any[]) => {} + const showSandboxPlanLimitExceededModal = (..._args: any[]) => {} const showUpgradeToUseListView = (..._args: any[]) => {} @@ -439,6 +443,8 @@ export const useEeConfig = createSharedComposable(() => { showUpgradeToUseFormScheduling, blockViewSections, showUpgradeToUseViewSections, + blockBaseVariables, + showUpgradeToUseBaseVariables, blockListView, showUpgradeToUseListView, blockMapView, diff --git a/packages/nc-gui/composables/useMetas.ts b/packages/nc-gui/composables/useMetas.ts index 7b8ee5f1bf..88826a66a5 100644 --- a/packages/nc-gui/composables/useMetas.ts +++ b/packages/nc-gui/composables/useMetas.ts @@ -14,8 +14,17 @@ export const useMetas = createSharedComposable(() => { const { baseTables } = storeToRefs(useTablesStore()) - // keep a temporary state of deleted tables to avoid get api calls - const deletedTableIds = new Set() + // keep a temporary state of deleted tables per base to avoid get api calls + const deletedTableIdsByBase = new Map>() + + const getDeletedTableIds = (baseId: string): Set => { + let set = deletedTableIdsByBase.get(baseId) + if (!set) { + set = new Set() + deletedTableIdsByBase.set(baseId, set) + } + return set + } // Helper function to create composite key: baseId:tableIdOrTitle const getMetaKey = (baseId: string, tableIdOrTitle: string) => `${baseId}:${tableIdOrTitle}` @@ -89,9 +98,10 @@ export const useMetas = createSharedComposable(() => { // if marked as deleted, verify it's actually still gone // (e.g., table restored from trash re-appears in baseTables) - if (deletedTableIds.has(tableIdOrTitle)) { + const deletedSet = deletedTableIdsByBase.get(baseId) + if (deletedSet?.has(tableIdOrTitle)) { if (tables.some((t) => t.id === tableIdOrTitle)) { - deletedTableIds.delete(tableIdOrTitle) + deletedSet.delete(tableIdOrTitle) } else { return null } @@ -179,7 +189,17 @@ export const useMetas = createSharedComposable(() => { const clearAllMeta = () => { metas.value = {} - deletedTableIds.clear() + deletedTableIdsByBase.clear() + } + + /** Clear cached meta for a single base and reset deleted-table tracking. */ + const clearBaseMeta = (baseId: string) => { + deletedTableIdsByBase.delete(baseId) + for (const key of Object.keys(metas.value)) { + if (key.startsWith(`${baseId}:`)) { + delete metas.value[key] + } + } } const removeMeta = (baseId: string, idOrTitle: string, deleted = false) => { @@ -187,24 +207,25 @@ export const useMetas = createSharedComposable(() => { const meta = metas.value[metaKey] if (meta) { - if (deleted) deletedTableIds.add(meta.id) + if (deleted) getDeletedTableIds(baseId).add(meta.id) delete metas.value[getMetaKey(baseId, meta.id)] delete metas.value[getMetaKey(baseId, meta.title)] } } - const clearDeletedTableId = (tableId: string) => { - deletedTableIds.delete(tableId) + const clearDeletedTableId = (baseId: string, tableId: string) => { + deletedTableIdsByBase.get(baseId)?.delete(tableId) } // return partial metadata for related table of a meta service const getPartialMeta = async (baseId: string, linkColumnId: string, tableIdOrTitle: string): Promise => { if (!tableIdOrTitle || !linkColumnId) return null - if (deletedTableIds.has(tableIdOrTitle)) { + const deletedSet = deletedTableIdsByBase.get(baseId) + if (deletedSet?.has(tableIdOrTitle)) { const tables = baseTables.value.get(baseId) ?? [] if (tables.some((t) => t.id === tableIdOrTitle)) { - deletedTableIds.delete(tableIdOrTitle) + deletedSet.delete(tableIdOrTitle) } else { return null } @@ -245,6 +266,7 @@ export const useMetas = createSharedComposable(() => { return { getMeta, clearAllMeta, + clearBaseMeta, metas, metasWithIdAsKey, removeMeta, diff --git a/packages/nc-gui/composables/useSandboxChangelog.ts b/packages/nc-gui/composables/useSandboxChangelog.ts new file mode 100644 index 0000000000..99c724f60c --- /dev/null +++ b/packages/nc-gui/composables/useSandboxChangelog.ts @@ -0,0 +1,26 @@ +export const useSandboxChangelog = createSharedComposable(() => { + const isDrawerOpen = ref(false) + const isLoading = ref(false) + const data = ref<{ changelog: any[]; users: Record }>({ changelog: [], users: {} }) + const mergeStatus = ref<'idle' | 'loading' | 'success' | 'error'>('idle') + const mergeError = ref('') + const DRAWER_WIDTH = 480 + + const openDrawer = () => {} + const closeDrawer = () => {} + const mergeFromChangelog = async () => {} + const loadChangelog = async () => {} + + return { + DRAWER_WIDTH, + isDrawerOpen, + isLoading, + data, + mergeStatus, + mergeError, + openDrawer, + closeDrawer, + mergeFromChangelog, + loadChangelog, + } +}) diff --git a/packages/nc-gui/lang/en.json b/packages/nc-gui/lang/en.json index 43f717491b..bc4b80a29f 100644 --- a/packages/nc-gui/lang/en.json +++ b/packages/nc-gui/lang/en.json @@ -236,6 +236,8 @@ "upgradeToUseViewSections": "Upgrade to use view sections", "upgradeToUseViewSectionsSubtitle": "Upgrade to the {plan} plan to organize views into sections.", "upgradeToAccessViewSectionsSubtitle": "Upgrade to the {plan} plan to organize views into sections.", + "upgradeToUseBaseVariables": "Upgrade to use base variables", + "upgradeToUseBaseVariablesSubtitle": "Upgrade to the {plan} plan to define reusable variables scoped to a base.", "upgradeToUseSync": "Upgrade to use sync", "upgradeToUseSyncSubtitle": "Upgrade to the {plan} plan to sync data with internal or external sources.", "upgradeToEnableUnique": "Upgrade to enable unique", @@ -1067,6 +1069,7 @@ "setPassword": "Set Password", "externalIntegrations": "External Integrations", "mcpServer": "MCP Server", + "baseVariables": "Variables", "oauthClients": "OAuth Clients", "selectAttachmentField": "Select Attachment Field", "accessRestricted": "Access Restricted", @@ -1610,6 +1613,30 @@ "never": "Never", "expiresAt": "Expires At", "newMCPEndpoint": "New MCP Endpoint", + "newVariable": "New Variable", + "variableKey": "Key", + "variableValue": "Value", + "variableMode": "Mode", + "variableAccess": "Access", + "variableInheritance": "Inheritance", + "variableType": "Type", + "variableDescription": "Description", + "variableSensitive": "Sensitive", + "configureVariables": "Configure Variables", + "fixedByMaster": "Fixed by master — cannot be changed", + "fixedVariableReadonly": "This is a fixed variable set by the master base. Its value cannot be changed in the sandbox.", + "deleteVariableConfirmation": "This variable will be permanently deleted. This action cannot be undone.", + "usingDefault": "Using default", + "revertToDefault": "Revert to default", + "variablesSyncedFromMaster": "Variables inherited from master. Changes here won't affect the master base.", + "requiredVariablesMissing": "required variables need configuration", + "setupRequired": "Setup Required", + "inheritanceFixed": "Fixed", + "inheritanceFixedDesc": "Inherited as-is, cannot be changed", + "inheritanceEditable": "Editable", + "inheritanceEditableDesc": "Inherited with default, can be overridden", + "inheritanceRequired": "Required", + "inheritanceRequiredDesc": "Must be set by the child", "mcpSubText": "The NocoDB MCP (Model Context Protocol) can be added to any client that supports MCP. It's a unique endpoint that allows LLMs to directly perform actions (based on your role) within your base on your behalf.", "modelContextProtocol": "Model Context Protocol", "mcp": "MCP", @@ -1702,8 +1729,18 @@ "revertAllChangesToMatchMaster": "Revert all changes to match master", "publishSandboxChanges": "Publish Sandbox Changes", "sandboxChangesPublished": "Sandbox Changes Published", + "sandboxChanges": "Sandbox changes", + "viewSandboxChanges": "View changes", + "noSandboxChanges": "No changes yet", + "sandboxNoChangesHint": "Schema changes made in this sandbox will appear here.", + "sandboxChangesApplyDescription": "The following will be applied to production. We'll take a snapshot before applying changes.", + "applyNChanges": "Apply {count} change | Apply {count} changes", + "applyChangesToMaster": "Apply to master", + "yesterday": "Yesterday", + "newestFirst": "Newest first", + "oldestFirst": "Oldest first", "failedToPublishChanges": "Failed to Publish Changes", - "mergeSchemaChangesWarning": "This will merge all schema changes from this sandbox into the master base.", + "mergeSchemaChangesWarning": "This will apply all structure changes from this sandbox to the master base. Row data and variable values will not be transferred.", "sandboxChangesPublishedSuccess": "Sandbox changes have been published successfully.", "viewChangesOnMaster": "You can now go to the master base to view the changes.", "publishChangesAction": "Publish Changes", @@ -1715,7 +1752,6 @@ "enterSandboxTitle": "Enter sandbox title", "sandboxDescriptionPlaceholder": "What are you working on in this sandbox?", "creatingSandbox": "Creating Sandbox...", - "goToSandbox": "Go to Sandbox", "sandboxCreatedSuccess": "Sandbox \"{title}\" has been created successfully.", "sandboxCreatedInstructions": "You can now make changes to this sandbox. When ready, publish your changes back to the master base.", "allSandboxes": "All Sandboxes", @@ -1723,9 +1759,18 @@ "noSandboxesFound": "No sandboxes found. Create one to get started.", "noDescriptionProvided": "No description provided.", "deleteSandbox": "Delete Sandbox", + "deleteSandboxWarning": "This will permanently delete this sandbox and all its data. This action cannot be undone.", + "sandboxDeleted": "Sandbox deleted successfully", + "permanentlyRemoveSandbox": "Permanently remove this sandbox", "sandboxChangesReverted": "Sandbox changes reverted successfully", + "discardSandboxWarning": "This will permanently revert all schema changes in this sandbox to match the master base. This action cannot be undone.", "createNewSandbox": "Create new sandbox", - "viewAllSandboxes": "View all sandboxes", + "master": "Master", + "schemaLocked": "Schema Locked", + "schemaLockedActive": "Schema Editing Locked", + "schemaLockedDescription": "An active sandbox exists. Schema changes can only be made in the sandbox.", + "goToSandbox": "Go to Sandbox", + "openSandboxToEditSchema": "Open sandbox to make schema changes", "justNow": "Just now", "goLive": "Go live", "pleaseProvideVersion": "Please provide a version", @@ -2103,6 +2148,8 @@ "setupTwoFactorAuth": "Set Up 2FA", "requireTwoFactor": "Require two-factor authentication", "requireTwoFactorDescription": "All workspace members will be required to set up two-factor authentication before accessing this workspace.", + "requireTwoFactorSsoNote": "Note: Members who sign in via SSO are exempt from this requirement, as their authentication is managed by the identity provider.", + "twoFactorSsoNotAvailable": "Two-factor authentication is not available for accounts that sign in via SSO.", "controlAppearance": "Control your Appearance.", "accountEmailID": "Account Email ID", "backToWorkspace": "Back to Workspace", @@ -2841,10 +2888,18 @@ "goBackHome": "Go back home" }, "tooltip": { + "showExtensions": "Show extensions", + "hideExtensions": "Hide extensions", + "sandboxRequiresV3": "Sandbox requires a V3 base. Please migrate this base to V3 first.", + "sandboxLimitReached": "This base already has a sandbox. Only one sandbox per base is allowed.", + "lockedViewDisabledOnSandboxMaster": "Locked views can only be created in the sandbox", + "personalViewDisabledOnSandbox": "Personal views can only be created on the master base", + "noSchemaChanges": "No schema changes to publish", "scopeReadOnly": "View records, tables, fields, views, and comments. No modifications allowed.", "scopeReadWrite": "Create, read, update, and delete records and comments. Other resources are read-only.", "scopeFullAccess": "Full read and write access to all resources including tables, fields, views, webhooks, and users.", "scopeCustom": "Pick individual permissions per category for fine-grained control.", + "variableInheritance": "Controls how this variable behaves in managed apps and sandboxes derived from this base", "inheritedMembersInfo": "These members belong to a parent team and inherit access to resources assigned to this team.", "teamScopeThisOnly": "This team only — click to include sub-teams", "teamScopeIncludesSubTeams": "Includes sub-teams — click to match this team only", @@ -2994,6 +3049,9 @@ "placeholder": { "enterVerificationCode": "Enter 6-digit code", "enterBackupCode": "Enter backup code (e.g., xxxx-xxxx)", + "variableKey": "e.g. WEBHOOK_URL", + "variableValue": "Enter value", + "variableDescription": "Describe this variable", "askAnything": "Ask or build anything", "somethingElse": "Something else", "docEditor": "Start writing or type / for commands...", @@ -3134,6 +3192,9 @@ "publishingWillMakeAvailable": "Publishing version {version} will make it available in the App Store and automatically update all installations.", "createDraftDescription": "Create a new draft version to work on updates. Current published version {version} will remain unchanged.", "createVersionToMakeChanges": "Create {version} to make changes", + "allInstallationsUpdated": "All installations updated successfully", + "installationUpdatePartial": "{success} installations updated, {failed} failed", + "installationUpdateFailed": "Failed to update installations", "trackingInstallationsFor": "Tracking installations for", "currentlyOnVersion": "Currently on {version}", "ssoTokenTooltip": "Generated after SSO login and can be used for accessing APIs under SSO-enabled workspace", @@ -3320,10 +3381,15 @@ }, "reachedUploadLimit": "Reached upload limit", "youCanOnlyUploadMaxLimitFilesAtATime": "You can only upload {limit} {type} file at a time", - "youCanOnlyUploadMaxLimitFilesAtATimePlural": "You can only upload {limit} {type} files at a time" + "youCanOnlyUploadMaxLimitFilesAtATimePlural": "You can only upload {limit} {type} files at a time", + "variableKeyRequired": "Variable key is required", + "variableKeyFormat": "Key must be uppercase letters, numbers and underscores (e.g. WEBHOOK_URL)" }, "info": { "trashEmpty": "Trash is empty", + "variableKeyImmutable": "Key cannot be changed after creation", + "variableSetupHint": "Set values for the variables required by this base. Required fields must be configured for full functionality.", + "setupRequired": "This app requires configuration before use", "contactAdminForAccess": "You haven't been added to any workspace yet. Contact your administrator to get access.", "licenseRequiresPostgres": "Enterprise licensing requires PostgreSQL. Please migrate to PostgreSQL to activate your license key.", "licensesUpToDate": "All licenses are up to date", @@ -3370,6 +3436,7 @@ "noOIDC": "There are no configured OpenID authentications.", "disabledAsViewLocked": "Disabled as View is locked", "noFiltersApplied": "No filters applied", + "disabledAsSandboxMasterCollabDelete": "Collaborative views can only be deleted from the sandbox", "basesMigrated": "Bases are migrated. Please try again.", "pasteNotSupported": "Paste operation is not supported on the active cell", "pasteNotSupportedInSyncedCells": "Paste operation is not supported in synced fields", @@ -3605,6 +3672,8 @@ "workspaceTeamRoleUpdateNotice": "Changing a team’s workspace role will also update its access to bases where it doesn’t have an explicit role." }, "error": { + "variableKeyRequired": "Variable key is required", + "variableKeyFormat": "Key must be uppercase letters, numbers and underscores (e.g. WEBHOOK_URL)", "latitudeRange": "Latitude must be between -90 and 90", "longitudeRange": "Longitude must be between -180 and 180", "passwordChangeFailed": "Failed to change password", @@ -3816,6 +3885,10 @@ "mcpTokenDeleted": "MCP Token Deleted", "mcpTokenUpdated": "MCP Token Updated", "mcpTokenCreated": "MCP Token Created", + "variableCreated": "Variable created", + "variableUpdated": "Variable updated", + "variableDeleted": "Variable deleted", + "variablesConfigured": "Variables configured", "licenseKeyUpdated": "License Key Updated", "rlsPolicySaved": "Record-level security policy saved", "teamMoved": "Team moved successfully", diff --git a/packages/nc-gui/utils/settingsRouteUtils.ts b/packages/nc-gui/utils/settingsRouteUtils.ts index 23d7c8b590..c282aaa3aa 100644 --- a/packages/nc-gui/utils/settingsRouteUtils.ts +++ b/packages/nc-gui/utils/settingsRouteUtils.ts @@ -20,6 +20,7 @@ export const baseSettingsTabToSlug: Record = { 'overview': 'overview', 'mcp': 'mcp', 'record-trash': 'record-trash', + 'variables': 'variables', 'snapshots': 'snapshots', } diff --git a/packages/nocodb-sdk/src/lib/base-variable/index.ts b/packages/nocodb-sdk/src/lib/base-variable/index.ts new file mode 100644 index 0000000000..9378cb5e66 --- /dev/null +++ b/packages/nocodb-sdk/src/lib/base-variable/index.ts @@ -0,0 +1,26 @@ +import type { + BaseVariableInheritance, + BaseVariableValueType, +} from '~/lib/globals'; + +export interface BaseVariableType { + id?: string; + base_id?: string; + fk_workspace_id?: string; + key?: string; + value?: string; + description?: string; + inheritance?: BaseVariableInheritance; + type?: BaseVariableValueType; + order?: number; + is_overridden?: boolean; + is_inherited?: boolean; +} + +export interface BaseVariableReqType { + key: string; + value?: string; + description?: string; + inheritance: BaseVariableInheritance; + type?: BaseVariableValueType; +} diff --git a/packages/nocodb-sdk/src/lib/enums.ts b/packages/nocodb-sdk/src/lib/enums.ts index 6d0a36a485..16138bf650 100644 --- a/packages/nocodb-sdk/src/lib/enums.ts +++ b/packages/nocodb-sdk/src/lib/enums.ts @@ -364,6 +364,19 @@ export enum AppEvents { RESOURCE_RESTORE = 'resource.restore', RESOURCE_PERMANENT_DELETE = 'resource.permanent_delete', + + BASE_VARIABLE_CREATE = 'baseVariable.create', + BASE_VARIABLE_UPDATE = 'baseVariable.update', + BASE_VARIABLE_DELETE = 'baseVariable.delete', + + MANAGED_APP_CREATE = 'managedApp.create', + MANAGED_APP_UPDATE = 'managedApp.update', + MANAGED_APP_DELETE = 'managedApp.delete', + MANAGED_APP_PUBLISH = 'managedApp.publish', + MANAGED_APP_INSTALL = 'managedApp.install', + MANAGED_APP_UPDATE_START = 'managedApp.update.start', + MANAGED_APP_UPDATE_COMPLETE = 'managedApp.update.complete', + MANAGED_APP_UPDATE_FAIL = 'managedApp.update.fail', } export enum ClickhouseTables { @@ -719,5 +732,6 @@ export enum MapProvider { STADIAMAP_APIKEY = 'stadiamap_apikey', } + /** Default org ID for on-prem deployments */ export const NC_DEFAULT_ORG_ID = 'org_default'; diff --git a/packages/nocodb-sdk/src/lib/error-handler/nc-error-base.ts b/packages/nocodb-sdk/src/lib/error-handler/nc-error-base.ts index d5e9e6549b..d3d344ce92 100644 --- a/packages/nocodb-sdk/src/lib/error-handler/nc-error-base.ts +++ b/packages/nocodb-sdk/src/lib/error-handler/nc-error-base.ts @@ -1049,7 +1049,7 @@ export class NcErrorBase { { params: limit.toString(), ...args, - }, + } ); } @@ -1072,4 +1072,21 @@ export class NcErrorBase { ...args, }); } + + sandboxBlocked(message?: string, args?: NcErrorArgs): never { + throw this.errorCodex.generateError(NcErrorType.ERR_SANDBOX_BLOCKED, { + params: message || '', + ...args, + }); + } + + sandboxProductionBlocked(message?: string, args?: NcErrorArgs): never { + throw this.errorCodex.generateError( + NcErrorType.ERR_SANDBOX_PRODUCTION_BLOCKED, + { + params: message || '', + ...args, + } + ); + } } diff --git a/packages/nocodb-sdk/src/lib/error-handler/preset-error-codex-map.ts b/packages/nocodb-sdk/src/lib/error-handler/preset-error-codex-map.ts index d3ecbdebf0..f97f5740ac 100644 --- a/packages/nocodb-sdk/src/lib/error-handler/preset-error-codex-map.ts +++ b/packages/nocodb-sdk/src/lib/error-handler/preset-error-codex-map.ts @@ -483,4 +483,15 @@ export const presetErrorCodexMap: Partial< message || 'Too many requests. Please try again later.', code: 429, }, + [NcErrorType.ERR_SANDBOX_BLOCKED]: { + message: (message: string) => + message || 'This operation is not allowed in a sandbox base.', + code: 403, + }, + [NcErrorType.ERR_SANDBOX_PRODUCTION_BLOCKED]: { + message: (message: string) => + message || + 'This operation is not allowed while a sandbox is active. Make the change in the sandbox instead.', + code: 403, + }, }; diff --git a/packages/nocodb-sdk/src/lib/globals.ts b/packages/nocodb-sdk/src/lib/globals.ts index 713b940c48..bb2845ada5 100644 --- a/packages/nocodb-sdk/src/lib/globals.ts +++ b/packages/nocodb-sdk/src/lib/globals.ts @@ -291,6 +291,10 @@ export enum NcErrorType { // System configuration errors ERR_SYSTEM_MISCONFIGURED = 'ERR_SYSTEM_MISCONFIGURED', ERR_TOO_MANY_REQUESTS = 'ERR_TOO_MANY_REQUESTS', + + // Sandbox errors + ERR_SANDBOX_BLOCKED = 'ERR_SANDBOX_BLOCKED', + ERR_SANDBOX_PRODUCTION_BLOCKED = 'ERR_SANDBOX_PRODUCTION_BLOCKED', } export enum ROW_COLORING_MODE { @@ -438,3 +442,14 @@ export enum DeploymentType { INSTALL = 'install', UPDATE = 'update', } + +export enum BaseVariableInheritance { + FIXED = 'fixed', + EDITABLE = 'editable', + REQUIRED = 'required', +} + +export enum BaseVariableValueType { + TEXT = 'text', + SECRET = 'secret', +} diff --git a/packages/nocodb-sdk/src/lib/index.ts b/packages/nocodb-sdk/src/lib/index.ts index eea7d53ef8..d5b2522d87 100644 --- a/packages/nocodb-sdk/src/lib/index.ts +++ b/packages/nocodb-sdk/src/lib/index.ts @@ -18,6 +18,8 @@ export * from '~/lib/realtime'; export * from '~/lib/scripts'; export * from '~/lib/oauth'; export * from '~/lib/unifiedMeta'; +export * from '~/lib/base-variable'; +export * from '~/lib/managed-app'; export * from '~/lib/types'; export * from '~/lib/helpers/metaHelpers'; export * from '~/lib/circularReplacer'; diff --git a/packages/nocodb-sdk/src/lib/managed-app/index.ts b/packages/nocodb-sdk/src/lib/managed-app/index.ts new file mode 100644 index 0000000000..0927f31521 --- /dev/null +++ b/packages/nocodb-sdk/src/lib/managed-app/index.ts @@ -0,0 +1,62 @@ +import type { + DeploymentStatus, + DeploymentType, + ManagedAppVersionStatus, + ManagedAppVisibility, +} from '~/lib/globals'; + +export interface ManagedAppType { + id?: string; + base_id?: string; + fk_workspace_id?: string; + title?: string; + description?: string; + created_by?: string; + visibility?: ManagedAppVisibility; + category?: string; + install_count?: number; + meta?: Record; + deleted?: boolean; + created_at?: string; + updated_at?: string; + published_at?: string; +} + +export interface ManagedAppVersionType { + id?: string; + fk_managed_app_id?: string; + fk_workspace_id?: string; + version?: string; + version_number?: number; + status?: ManagedAppVersionStatus; + schema?: string; + release_notes?: string; + created_at?: string; + updated_at?: string; + published_at?: string; +} + +export interface ManagedAppDeploymentLogType { + id?: string; + fk_workspace_id?: string; + base_id?: string; + fk_managed_app_id?: string; + from_version_id?: string; + to_version_id?: string; + status?: DeploymentStatus; + deployment_type?: DeploymentType; + error_message?: string; + deployment_log?: string; + meta?: Record; + created_at?: string; + updated_at?: string; + started_at?: string; + completed_at?: string; +} + +export interface ManagedAppReqType { + title: string; + description?: string; + visibility?: ManagedAppVisibility; + category?: string; +} diff --git a/packages/nocodb-sdk/src/lib/payment/index.ts b/packages/nocodb-sdk/src/lib/payment/index.ts index b16b03e0f9..62d6824fe0 100644 --- a/packages/nocodb-sdk/src/lib/payment/index.ts +++ b/packages/nocodb-sdk/src/lib/payment/index.ts @@ -108,6 +108,7 @@ export enum PlanFeatureTypes { FEATURE_DATE_DEPENDENCY = 'feature_date_dependency', FEATURE_API_COMMENT_V3 = 'feature_api_comment_v3', FEATURE_API_WORKFLOW_MANAGEMENT = 'feature_api_workflow_management', + FEATURE_BASE_VARIABLES = 'feature_base_variables', /** On-prem: core EE capability flag — true for all paid plans, false for free */ FEATURE_EE_CORE = 'feature_ee_core', FEATURE_TRASH_SETTINGS = 'feature_trash_settings', @@ -386,6 +387,7 @@ export const PlanFeatureUpgradeMessages: Record = { [PlanFeatureTypes.FEATURE_DATE_DEPENDENCY]: 'to use date dependencies.', [PlanFeatureTypes.FEATURE_API_COMMENT_V3]: 'to use comment api.', [PlanFeatureTypes.FEATURE_API_WORKFLOW_MANAGEMENT]: 'to use workflow api.', + [PlanFeatureTypes.FEATURE_BASE_VARIABLES]: 'to use base variables.', [PlanFeatureTypes.FEATURE_EE_CORE]: 'to access enterprise features.', [PlanFeatureTypes.FEATURE_TRASH_SETTINGS]: 'to configure per-table trash settings.', diff --git a/packages/nocodb/src/cache/CacheMgr.ts b/packages/nocodb/src/cache/CacheMgr.ts index 89385636bb..9ce47b5b1d 100644 --- a/packages/nocodb/src/cache/CacheMgr.ts +++ b/packages/nocodb/src/cache/CacheMgr.ts @@ -233,6 +233,31 @@ export default abstract class CacheMgr { return this.client.incrby(key, value); } + async incrbyExpiring( + key: string, + value: number, + seconds: number, + ): Promise { + const results = await this.client + .pipeline() + .incrby(key, value) + .expire(key, seconds) + .exec(); + if (!results || !results[0]) return value; + const [err, total] = results[0]; + if (err) throw err; + return Number(total); + } + + async setIfNotExist( + key: string, + value: string, + seconds: number, + ): Promise { + const res = await this.client.set(key, value, 'EX', seconds, 'NX'); + return res === 'OK'; + } + async getList( scope: string, subKeys: string[], @@ -720,7 +745,27 @@ export default abstract class CacheMgr { async processPattern( pattern: string, - callback: (key: string | string[]) => Promise, + callback: (key: string) => Promise, + options?: { + count?: number; + type?: string; + raw?: boolean; + batch?: false; + }, + ): Promise; + async processPattern( + pattern: string, + callback: (keys: string[]) => Promise, + options: { + count?: number; + type?: string; + raw?: boolean; + batch: true; + }, + ): Promise; + async processPattern( + pattern: string, + callback: (key: any) => Promise, options: { count?: number; type?: string; diff --git a/packages/nocodb/src/cache/NocoCache.ts b/packages/nocodb/src/cache/NocoCache.ts index 5b49a1abdc..bdf649a511 100644 --- a/packages/nocodb/src/cache/NocoCache.ts +++ b/packages/nocodb/src/cache/NocoCache.ts @@ -1,5 +1,6 @@ import RedisCacheMgr from './RedisCacheMgr'; import RedisMockCacheMgr from './RedisMockCacheMgr'; +import { isCacheBypassed } from './cacheBypassScope'; import type { NcContext } from 'nocodb-sdk'; import type CacheMgr from './CacheMgr'; import { CACHE_PREFIX, CacheGetType } from '~/utils/globals'; @@ -45,7 +46,7 @@ export default class NocoCache { } public static async set(context: CacheContext, key, value): Promise { - if (this.cacheDisabled) return Promise.resolve(true); + if (this.cacheDisabled || isCacheBypassed()) return Promise.resolve(true); return this.client.set( `${this.prefix}:${cacheContext(context)}:${key}`, value, @@ -58,7 +59,7 @@ export default class NocoCache { value, expireSeconds, ): Promise { - if (this.cacheDisabled) return Promise.resolve(true); + if (this.cacheDisabled || isCacheBypassed()) return Promise.resolve(true); return this.client.setExpiring( `${this.prefix}:${cacheContext(context)}:${key}`, value, @@ -66,6 +67,20 @@ export default class NocoCache { ); } + public static async setIfNotExist( + context: CacheContext, + key: string, + value: string, + expireSeconds: number, + ): Promise { + if (this.cacheDisabled) return Promise.resolve(true); + return this.client.setIfNotExist( + `${this.prefix}:${cacheContext(context)}:${key}`, + value, + expireSeconds, + ); + } + public static async incrby( context: CacheContext, key, @@ -78,8 +93,24 @@ export default class NocoCache { ); } + // cache-disabled fallback returns the caller's own delta — for counters + // where callers expect a numeric running total, not a boolean success flag. + public static async incrbyExpiring( + context: CacheContext, + key: string, + value: number, + expireSeconds: number, + ): Promise { + if (this.cacheDisabled) return Promise.resolve(value); + return this.client.incrbyExpiring( + `${this.prefix}:${cacheContext(context)}:${key}`, + value, + expireSeconds, + ); + } + public static async get(context: CacheContext, key, type): Promise { - if (this.cacheDisabled) { + if (this.cacheDisabled || isCacheBypassed()) { if (type === CacheGetType.TYPE_ARRAY) return Promise.resolve([]); else if (type === CacheGetType.TYPE_OBJECT) return Promise.resolve(null); return Promise.resolve(null); @@ -112,7 +143,7 @@ export default class NocoCache { list: any[]; isNoneList: boolean; }> { - if (this.cacheDisabled) + if (this.cacheDisabled || isCacheBypassed()) return Promise.resolve({ list: [], isNoneList: false, @@ -131,7 +162,7 @@ export default class NocoCache { list: any[], props: string[] = [], ): Promise { - if (this.cacheDisabled) return Promise.resolve(true); + if (this.cacheDisabled || isCacheBypassed()) return Promise.resolve(true); return this.client.setList( `${this.prefix}:${cacheContext(context)}:${scope}`, subListKeys, @@ -159,7 +190,7 @@ export default class NocoCache { key: string, ): Promise { - if (this.cacheDisabled) return Promise.resolve(true); + if (this.cacheDisabled || isCacheBypassed()) return Promise.resolve(true); return this.client.appendToList( `${this.prefix}:${cacheContext(context)}:${scope}`, subListKeys, @@ -172,7 +203,7 @@ export default class NocoCache { key: string, updateObj: Record, ): Promise { - if (this.cacheDisabled) return Promise.resolve(true); + if (this.cacheDisabled || isCacheBypassed()) return Promise.resolve(true); return this.client.update( `${this.prefix}:${cacheContext(context)}:${key}`, updateObj, @@ -187,7 +218,7 @@ export default class NocoCache { ttl?: number; } = {}, ): Promise { - if (this.cacheDisabled) return Promise.resolve(true); + if (this.cacheDisabled || isCacheBypassed()) return Promise.resolve(true); if (Object.keys(hash).length === 0) { return; } @@ -202,7 +233,7 @@ export default class NocoCache { context: CacheContext, key: string, ): Promise> { - if (this.cacheDisabled) return Promise.resolve({}); + if (this.cacheDisabled || isCacheBypassed()) return Promise.resolve({}); return this.client.getHash( `${this.prefix}:${cacheContext(context)}:${key}`, ); @@ -213,7 +244,7 @@ export default class NocoCache { key: string, field: string, ): Promise { - if (this.cacheDisabled) return Promise.resolve(null); + if (this.cacheDisabled || isCacheBypassed()) return Promise.resolve(null); return this.client.getHashField( `${this.prefix}:${cacheContext(context)}:${key}`, field, @@ -226,7 +257,7 @@ export default class NocoCache { field: string, value: string | number, ): Promise { - if (this.cacheDisabled) return Promise.resolve(true); + if (this.cacheDisabled || isCacheBypassed()) return Promise.resolve(true); return !!this.client.setHashField( `${this.prefix}:${cacheContext(context)}:${key}`, field, diff --git a/packages/nocodb/src/cache/cacheBypassScope.ts b/packages/nocodb/src/cache/cacheBypassScope.ts new file mode 100644 index 0000000000..565ad7de15 --- /dev/null +++ b/packages/nocodb/src/cache/cacheBypassScope.ts @@ -0,0 +1,24 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; + +const scope = new AsyncLocalStorage(); + +/** + * Run `fn` in a scope where NocoCache read/write ops are bypassed. + * + * Inside the scope, `get`/`getList`/`getHash`/`getHashField` return empty + * values and `set`/`setList`/`setExpiring`/`setHash`/`setHashField`/ + * `appendToList`/`update` are no-ops — as if the cache were disabled. + * Invalidation ops (`del`/`deepDel`/`delHashField`/`clear`) and counters + * (`incrby`/`incrHashField`) pass through unchanged. + * + * Intended for use inside DB transactions where cache reads would return + * stale pre-transaction state and cache writes would leak uncommitted + * transaction-local data into the shared cache. + * + * The scope is per-async-tree via AsyncLocalStorage: concurrent requests + * outside the scope are unaffected. Nested calls are a safe no-op. + */ +export const runWithoutCache = (fn: () => Promise): Promise => + scope.run(true, fn); + +export const isCacheBypassed = (): boolean => scope.getStore() === true; diff --git a/packages/nocodb/src/command-registry/op-names.ts b/packages/nocodb/src/command-registry/op-names.ts new file mode 100644 index 0000000000..b5be65c880 --- /dev/null +++ b/packages/nocodb/src/command-registry/op-names.ts @@ -0,0 +1,122 @@ +/** + * Canonical names of every traced operation. Defined in CE so the + * `@TraceCommand` decorator can be applied directly on CE service methods + * without requiring EE-side overrides whose only purpose is to add tracing. + * + * Contract definitions (EE) use these values as their `name` field, and + * service decorators pass them to `@TraceCommand(...)`. The contract is + * resolved by name at invocation time via `OperationRegistry.contract(name)`. + * + * When adding a new traced operation, add the name here first. + */ +export const OperationName = { + // Tables + tableCreate: 'tableCreate', + tableUpdate: 'tableUpdate', + tableDelete: 'tableDelete', + + // Columns + columnAdd: 'columnAdd', + columnUpdate: 'columnUpdate', + columnDelete: 'columnDelete', + columnSetAsPrimary: 'columnSetAsPrimary', + + // Views (generic) + viewUpdate: 'viewUpdate', + viewDelete: 'viewDelete', + + // Per-type view ops + gridViewCreate: 'gridViewCreate', + gridViewUpdate: 'gridViewUpdate', + formViewCreate: 'formViewCreate', + formViewUpdate: 'formViewUpdate', + galleryViewCreate: 'galleryViewCreate', + galleryViewUpdate: 'galleryViewUpdate', + kanbanViewCreate: 'kanbanViewCreate', + kanbanViewUpdate: 'kanbanViewUpdate', + calendarViewCreate: 'calendarViewCreate', + calendarViewUpdate: 'calendarViewUpdate', + listViewCreate: 'listViewCreate', + listViewUpdate: 'listViewUpdate', + timelineViewCreate: 'timelineViewCreate', + timelineViewUpdate: 'timelineViewUpdate', + mapViewCreate: 'mapViewCreate', + mapViewUpdate: 'mapViewUpdate', + + // View columns + visibility + viewColumnUpdate: 'viewColumnUpdate', + gridColumnUpdate: 'gridColumnUpdate', + formColumnUpdate: 'formColumnUpdate', + + // Sorts + sortCreate: 'sortCreate', + sortUpdate: 'sortUpdate', + sortDelete: 'sortDelete', + + // Filters + filterCreate: 'filterCreate', + filterUpdate: 'filterUpdate', + filterDelete: 'filterDelete', + filterCreateV3: 'filterCreateV3', + filterReplaceV3: 'filterReplaceV3', + filterDeleteAllV3: 'filterDeleteAllV3', + linkFilterCreate: 'linkFilterCreate', + widgetFilterCreate: 'widgetFilterCreate', + rowColorConditionsCreate: 'rowColorConditionsCreate', + + // Hooks + hookCreate: 'hookCreate', + hookUpdate: 'hookUpdate', + hookDelete: 'hookDelete', + + // Extensions + extensionCreate: 'extensionCreate', + extensionUpdate: 'extensionUpdate', + extensionDelete: 'extensionDelete', + + // Dashboards + widgets + dashboardCreate: 'dashboardCreate', + dashboardUpdate: 'dashboardUpdate', + dashboardDelete: 'dashboardDelete', + widgetCreate: 'widgetCreate', + widgetUpdate: 'widgetUpdate', + widgetDelete: 'widgetDelete', + duplicateWidget: 'duplicateWidget', + + // Scripts + scriptCreate: 'scriptCreate', + scriptUpdate: 'scriptUpdate', + scriptDelete: 'scriptDelete', + + // Workflows + workflowCreate: 'workflowCreate', + workflowUpdate: 'workflowUpdate', + workflowDelete: 'workflowDelete', + workflowPublish: 'workflowPublish', + + // View sections + viewSectionCreate: 'viewSectionCreate', + viewSectionUpdate: 'viewSectionUpdate', + viewSectionDelete: 'viewSectionDelete', + + // Record templates + recordTemplateCreate: 'recordTemplateCreate', + recordTemplateUpdate: 'recordTemplateUpdate', + recordTemplateDelete: 'recordTemplateDelete', + + // Sync + syncCreate: 'syncCreate', + syncUpdate: 'syncUpdate', + syncDelete: 'syncDelete', + + // Date dependency + dateDependencyUpdate: 'dateDependencyUpdate', + dateDependencyDelete: 'dateDependencyDelete', + + // Base variables + baseVariableCreate: 'baseVariableCreate', + baseVariableUpdate: 'baseVariableUpdate', + baseVariableDelete: 'baseVariableDelete', +} as const; + +export type OperationName = (typeof OperationName)[keyof typeof OperationName]; diff --git a/packages/nocodb/src/command-registry/registry.ts b/packages/nocodb/src/command-registry/registry.ts new file mode 100644 index 0000000000..8c8bf54e10 --- /dev/null +++ b/packages/nocodb/src/command-registry/registry.ts @@ -0,0 +1,29 @@ +import type { CommandHandler, OperationContract } from './types'; + +// CE no-op stub. EE overrides with the real Map-backed singleton. +// `OperationContract` (not the default `OperationContract`) +// avoids a variance error: zod schema fields propagate `S` into function-type +// positions (`EntityRefFn`), and a specific `OperationContract` would otherwise be rejected as not assignable to the parameter. +class _OperationRegistryNoop { + register>(_c: C, _h: CommandHandler) {} + freeze() {} + resolve(_name: string, _version: number) { + return undefined; + } + describe(): Array<{ + name: string; + version: number; + entity: string; + idField?: string; + schemaHash: string; + extraSchemaHash?: string; + }> { + return []; + } + contract(_name: string, _version: number) { + return undefined; + } +} + +export const OperationRegistry = new _OperationRegistryNoop(); diff --git a/packages/nocodb/src/command-registry/types.ts b/packages/nocodb/src/command-registry/types.ts new file mode 100644 index 0000000000..7a73bc75f6 --- /dev/null +++ b/packages/nocodb/src/command-registry/types.ts @@ -0,0 +1,109 @@ +import type { z, ZodTypeAny } from 'zod'; +import type { MetaTable } from '~/utils/globals'; +import type { NcContext, NcRequest } from '~/interface/config'; +import type { OperationName } from './op-names'; + +/** + * Versioned, typed declaration of a single state-mutating operation. Each + * contract is the single source of truth for: + * + * - Identity — `name` + `version` (registry lookup key, also used + * as the `event` column in `nc_sandbox_changelog`). + * - Param shape — `schema` validates and shapes the persisted params at + * record time (strict — throws on mismatch). `extraSchema` + * validates handler-side metadata threaded via `extra`. + * - Entity reference — `entityId` / `entityTitle` / `parentId` / `parentTitle` + * extract identifying fields from `param` or the service + * result for changelog rows. Fall back to `idField` + * (a key in `param` whose value is the created entity). + * - Replay context — `resolveCtx` runs before the original method to capture + * pre-state (e.g. old title for rename detection). + * `skipIf` runs after, suppressing record when the call + * was a no-op. + * - Cross-refs — `deps` returns related entity IDs so replay can stitch + * foreign-key references across the merge boundary. + * `extraCommandMeta` packs additional payload (e.g. + * sandbox column IDs) onto the changelog row. + * - Description — human-readable summary for audit UIs (string or DescFn + * receiving entity/parent titles). + * + * Add new contracts under `src/ee/command-registry/operations/` and register + * a handler in the matching `handlers/` file — see `CLAUDE.md` for the full + * end-to-end checklist. + */ +export interface OperationContract { + readonly name: OperationName; + readonly version: number; + readonly entity: MetaTable; + + readonly schema: S; + readonly extraSchema?: ZodTypeAny; + + readonly idField?: string; + + readonly entityId?: string | EntityRefFn; + readonly entityTitle?: string | EntityRefFn; + readonly parentId?: string | EntityRefFn; + readonly parentTitle?: string | EntityRefFn; + readonly description?: string | DescFn; + + readonly resolveCtx?: (ctx: NcContext, p: z.infer) => Promise; + readonly skipIf?: ( + ctx: NcContext, + p: z.infer, + r: any, + resolved?: ResolvedCtx, + ) => Promise | boolean; + readonly deps?: (p: z.infer, r: any) => TraceCommandDep[]; + readonly extraCommandMeta?: ( + p: z.infer, + r: any, + ) => Record | undefined; +} + +export type EntityRefFn = ( + p: z.infer, + r: any, +) => string | undefined; + +export type ParamsOf = C extends OperationContract + ? z.infer + : never; + +export type CommandHandler< + C extends OperationContract = OperationContract, +> = (ctx: NcContext, params: ParamsOf, meta: HandlerMeta) => Promise; + +export interface HandlerMeta { + entryId: string; + entityId?: string; + originalReq: NcRequest; + createdBy: string; + extra?: Record; +} + +export interface ChangelogCommandPayload { + name: string; + version: number; + params: unknown; + extra?: Record; +} + +export interface TraceCommandDep { + entity: MetaTable; + id: string; +} + +export interface ResolvedCtx { + entityTitle?: string; + parentEntityTitle?: string; + extra?: Record; +} + +export interface DescCtx { + entityTitle?: string; + parentEntityTitle?: string; + operation: string; + extra?: Record; +} +export type DescFn = (ctx: DescCtx) => string; diff --git a/packages/nocodb/src/controllers/attachments-secure.controller.ts b/packages/nocodb/src/controllers/attachments-secure.controller.ts index 1d95f6bb0b..fa77b03a2e 100644 --- a/packages/nocodb/src/controllers/attachments-secure.controller.ts +++ b/packages/nocodb/src/controllers/attachments-secure.controller.ts @@ -21,14 +21,12 @@ import type { NcRequest } from '~/interface/config'; import { NcContext } from '~/interface/config'; import { GlobalGuard } from '~/guards/global/global.guard'; import { AttachmentsService } from '~/services/attachments.service'; -import { Column, PresignedUrl } from '~/models'; +import { PresignedUrl } from '~/models'; import { UploadAllowedInterceptor } from '~/interceptors/is-upload-allowed/is-upload-allowed.interceptor'; import { MetaApiLimiterGuard } from '~/guards/meta-api-limiter.guard'; -import { DataTableService } from '~/services/data-table.service'; import { DataApiLimiterGuard } from '~/guards/data-api-limiter.guard'; import { TenantContext } from '~/decorators/tenant-context.decorator'; import { Acl } from '~/middlewares/extract-ids/extract-ids.middleware'; -import { NcError } from '~/helpers/catchError'; import { ATTACHMENT_ROOTS, localFileExists } from '~/helpers/attachmentHelpers'; import { NC_DATA_IMPORT_FILE_SIZE } from '~/constants'; @@ -36,7 +34,6 @@ import { NC_DATA_IMPORT_FILE_SIZE } from '~/constants'; export class AttachmentsSecureController { constructor( private readonly attachmentsService: AttachmentsService, - private readonly dataTableService: DataTableService, ) {} @UseGuards(MetaApiLimiterGuard, GlobalGuard) @@ -152,30 +149,10 @@ export class AttachmentsSecureController { @Param('rowId') rowId: string, @Query('urlOrPath') urlOrPath: string, ) { - const column = await Column.get(context, { - colId: columnId, - }); - - if (!column) { - NcError.fieldNotFound(columnId); - } - - const record = await this.dataTableService.dataRead(context, { - baseId: context.base_id, + return this.attachmentsService.downloadAttachment(context, { modelId, + columnId, rowId, - query: { - fields: column.title, - }, - }); - - if (!record) { - NcError.recordNotFound(rowId); - } - - return this.attachmentsService.getAttachmentFromRecord({ - record, - column, urlOrPath, }); } diff --git a/packages/nocodb/src/controllers/attachments.controller.ts b/packages/nocodb/src/controllers/attachments.controller.ts index cba5e507cb..b7f8f02022 100644 --- a/packages/nocodb/src/controllers/attachments.controller.ts +++ b/packages/nocodb/src/controllers/attachments.controller.ts @@ -29,19 +29,15 @@ import { isPreviewAllowed, localFileExists, } from '~/helpers/attachmentHelpers'; -import { DataTableService } from '~/services/data-table.service'; import { TenantContext } from '~/decorators/tenant-context.decorator'; import { DataApiLimiterGuard } from '~/guards/data-api-limiter.guard'; import { Acl } from '~/middlewares/extract-ids/extract-ids.middleware'; -import { Column } from '~/models'; -import { NcError } from '~/helpers/catchError'; import { NC_DATA_IMPORT_FILE_SIZE } from '~/constants'; @Controller() export class AttachmentsController { constructor( private readonly attachmentsService: AttachmentsService, - private readonly dataTableService: DataTableService, ) {} @UseGuards(MetaApiLimiterGuard, GlobalGuard) @@ -251,30 +247,10 @@ export class AttachmentsController { @Param('rowId') rowId: string, @Query('urlOrPath') urlOrPath: string, ) { - const column = await Column.get(context, { - colId: columnId, - }); - - if (!column) { - NcError.fieldNotFound(columnId); - } - - const record = await this.dataTableService.dataRead(context, { - baseId: context.base_id, + return this.attachmentsService.downloadAttachment(context, { modelId, + columnId, rowId, - query: { - fields: column.title, - }, - }); - - if (!record) { - NcError.recordNotFound(rowId); - } - - return this.attachmentsService.getAttachmentFromRecord({ - record, - column, urlOrPath, }); } diff --git a/packages/nocodb/src/controllers/internal/modules/AttachmentGet.operations.ts b/packages/nocodb/src/controllers/internal/modules/AttachmentGet.operations.ts new file mode 100644 index 0000000000..41b4b2ba1a --- /dev/null +++ b/packages/nocodb/src/controllers/internal/modules/AttachmentGet.operations.ts @@ -0,0 +1,44 @@ +import { Injectable } from '@nestjs/common'; +import type { OPERATION_SCOPES } from '~/controllers/internal/operationScopes'; +import type { NcContext, NcRequest } from 'nocodb-sdk'; +import type { + InternalApiModule, + InternalGETResponseType, +} from '~/utils/internal-type'; +import { AttachmentsService } from '~/services/attachments.service'; + +@Injectable() +export class AttachmentGetOperations + implements InternalApiModule +{ + constructor( + protected readonly attachmentsService: AttachmentsService, + ) {} + + operations = ['attachmentDownload' as const]; + httpMethod = 'GET' as const; + + async handle( + context: NcContext, + { + req, + operation, + }: { + workspaceId: string; + baseId: string; + operation: keyof typeof OPERATION_SCOPES; + payload: any; + req: NcRequest; + }, + ): InternalGETResponseType { + switch (operation) { + case 'attachmentDownload': + return await this.attachmentsService.downloadAttachment(context, { + modelId: req.query.modelId as string, + columnId: req.query.columnId as string, + rowId: req.query.rowId as string, + urlOrPath: req.query.urlOrPath as string, + }); + } + } +} diff --git a/packages/nocodb/src/controllers/internal/operationScopes.ts b/packages/nocodb/src/controllers/internal/operationScopes.ts index d75e3504d3..d87a2bc08f 100644 --- a/packages/nocodb/src/controllers/internal/operationScopes.ts +++ b/packages/nocodb/src/controllers/internal/operationScopes.ts @@ -20,6 +20,9 @@ export const OPERATION_SCOPES = { oAuthClientRegenerateSecret: 'org', checkDependency: 'base', + // Attachment Operations + attachmentDownload: 'base', + // UI Get Operations tableGet: 'base', columnsHash: 'base', @@ -157,6 +160,7 @@ export const OPERATION_SCOPES = { managedAppDeployments: 'base', managedAppVersionDeployments: 'base', managedAppDeploymentLogs: 'base', + managedAppManualUpdate: 'base', // List View listViewDataList: 'base', diff --git a/packages/nocodb/src/controllers/internal/provider.ts b/packages/nocodb/src/controllers/internal/provider.ts index 6f23d38940..7ef3db3b01 100644 --- a/packages/nocodb/src/controllers/internal/provider.ts +++ b/packages/nocodb/src/controllers/internal/provider.ts @@ -11,6 +11,7 @@ import { BaseListAllGetOperations } from '~/controllers/internal/modules/BaseLis import { InstanceAdminGetOperations } from '~/controllers/internal/modules/InstanceAdminGet.operations'; import { IntegrationGetOperations } from '~/controllers/internal/modules/IntegrationGet.operations'; import { IntegrationPostOperations } from '~/controllers/internal/modules/IntegrationPost.operations'; +import { AttachmentGetOperations } from '~/controllers/internal/modules/AttachmentGet.operations'; import { INTERNAL_API_MODULE_PROVIDER_KEY } from '~/utils/internal-type'; export const InternalApiModules = [ @@ -26,6 +27,7 @@ export const InternalApiModules = [ InstanceAdminGetOperations, IntegrationGetOperations, IntegrationPostOperations, + AttachmentGetOperations, ]; export const InternalApiModuleProvider = { diff --git a/packages/nocodb/src/controllers/v3/tables-v3.controller.ts b/packages/nocodb/src/controllers/v3/tables-v3.controller.ts index d567545e68..62c636c12c 100644 --- a/packages/nocodb/src/controllers/v3/tables-v3.controller.ts +++ b/packages/nocodb/src/controllers/v3/tables-v3.controller.ts @@ -66,6 +66,7 @@ export class TablesV3Controller { sourceId: sourceId, table: body, user: req.user, + req, }); return result; diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index b2574ee6ff..f74ee70733 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -175,6 +175,31 @@ const MAX_RECURSION_DEPTH = 2; const SELECT_REGEX = /^(\(|)select/i; const INSERT_REGEX = /^(\(|)insert/i; +export interface ExecAndParseOptions { + skipDateConversion?: boolean; + skipAttachmentConversion?: boolean; + skipSubstitutingColumnIds?: boolean; + skipUserConversion?: boolean; + skipJsonConversion?: boolean; + raw?: boolean; + first?: boolean; + bulkAggregate?: boolean; + apiVersion?: NcApiVersion; +} + +/** Args stashed on DataLoader instances for relation queries (hm/mm/bt/oo). */ +interface RelationLoaderArgs { + limit?: number; + offset?: number; + fieldsSet?: Set; + fieldSet?: Set; +} + +/** DataLoader with a typed side-channel for query args. */ +class DataLoaderWithArgs extends DataLoader { + args?: RelationLoaderArgs; +} + /** * Base class for models * @@ -545,7 +570,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { validateFormula = false, ): Promise { const columns = await this.model.getColumns(this.context); - const { where, ...rest } = this._getListArgs(args as any); + const { where, ...rest } = this._getListArgs(args); const qb = this.dbDriver(this.tnPath); await this.selectObject({ ...args, qb, validateFormula, columns }); @@ -661,7 +686,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { const columns = await this.model.getColumns(this.context); - const { where, fields, ...rest } = this._getListArgs(args as any); + const { where, fields, ...rest } = this._getListArgs(args); const qb = this.dbDriver(this.tnPath); @@ -997,7 +1022,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { ) { const columns = await this.model.getColumns(this.context); - const { where, ...rest } = this._getListArgs(args as any); + const { where, ...rest } = this._getListArgs(args); const qb = this.dbDriver(this.tnPath); const aggregateStatement = `${aggregateColumnName} as ${aggregateFn}__${aggregateColumnName}`; @@ -1140,7 +1165,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { return {}; } - const { where, aggregation } = this._getListArgs(args as any); + const { where, aggregation } = this._getListArgs(args); const columns = await this.model.getColumns(this.context); @@ -1363,7 +1388,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { async aggregate(args: { filterArr?: Filter[]; where?: string }, view?: View) { try { - const { where, aggregation } = this._getListArgs(args as any); + const { where, aggregation } = this._getListArgs(args); const columns = await this.model.getColumns(this.context); @@ -1772,7 +1797,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { where: string; filters?: Filter[]; qb; - sort?: string; + sort?: string | string[]; onlySort?: boolean; skipViewFilter?: boolean; skipSort?: boolean; @@ -1980,18 +2005,18 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { // DataLoader collects all .load(id) calls from the same microtick // into a single batch. The batch callback is wrapped in _queryQueue.add() // to serialize actual DB execution across all relation types. - const listLoader = new DataLoader( - (ids: string[]) => + const listLoader = new DataLoaderWithArgs( + (ids: readonly string[]) => this._queryQueue.add(async () => { if (ids.length > 1) { const data = await this.multipleHmList( { colId: column.id, - ids, + ids: ids as string[], apiVersion, linksAsLtar, }, - (listLoader as any).args, + listLoader.args, ); return ids.map((id: string) => data[id] ? data[id] : [], @@ -2006,7 +2031,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { nested: true, linksAsLtar, }, - (listLoader as any).args, + listLoader.args, ), ]; } @@ -2021,8 +2046,8 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { column.uidt === UITypes.Links && !linksAsLtar ? `_nc_lk_${column.title}` : column.title - ] = async function (args): Promise { - (listLoader as any).args = args; + ] = async function (args?: RelationLoaderArgs): Promise { + listLoader.args = args; return listLoader.load( getCompositePkValue(self.model.primaryKeys, this), ); @@ -2030,8 +2055,8 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { } else if (isBtLikeV2Junction(column)) { // V2 MO/OO: single-record — return object (like BT) // Use multipleMmList for batching, take first record per parent - const readLoader = new DataLoader( - (ids: string[]) => + const readLoader = new DataLoaderWithArgs( + (ids: readonly string[]) => this._queryQueue.add(async () => { if (ids?.length > 1) { const lists = await this.multipleMmList( @@ -2039,14 +2064,14 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { parentIds: ids as string[], colId: column.id, }, - (readLoader as any).args, + readLoader.args, ); return lists.map((list) => list?.[0] ?? null); } else { return [ await this.mmRead( { parentId: ids[0], colId: column.id }, - (readLoader as any).args, + readLoader.args, ), ]; } @@ -2057,26 +2082,28 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { ); const self: BaseModelSqlv2 = this; - proto[column.title] = async function (args?: any) { - (readLoader as any).args = args; + proto[column.title] = async function ( + args?: RelationLoaderArgs, + ) { + readLoader.args = args; return await readLoader.load( getCompositePkValue(self.model.primaryKeys, this), ); }; } else if (colOptions.type === 'mm' || isMMLike) { - const listLoader = new DataLoader( - (ids: string[]) => + const listLoader = new DataLoaderWithArgs( + (ids: readonly string[]) => this._queryQueue.add(async () => { if (ids?.length > 1) { const data = await this.multipleMmList( { - parentIds: ids, + parentIds: ids as string[], colId: column.id, apiVersion, nested: true, linksAsLtar, }, - (listLoader as any).args, + listLoader.args, ); return data; @@ -2090,7 +2117,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { nested: true, linksAsLtar, }, - (listLoader as any).args, + listLoader.args, ), ]; } @@ -2105,8 +2132,8 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { column.uidt === UITypes.Links && !linksAsLtar ? `_nc_lk_${column.title}` : column.title - ] = async function (args): Promise { - (listLoader as any).args = args; + ] = async function (args?: RelationLoaderArgs): Promise { + listLoader.args = args; return await listLoader.load( getCompositePkValue(self.model.primaryKeys, this), ); @@ -2128,8 +2155,8 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { // it takes individual keys and callback is invoked with an array of values and we can get the // result for all those together and return the value in the same order as in the array // this way all parents data extracted together - const readLoader = new DataLoader( - (_ids: string[]) => + const readLoader = new DataLoaderWithArgs( + (_ids: readonly string[]) => this._queryQueue.add(async () => { // handle binary(16) foreign keys const ids = _ids.map((id) => { @@ -2163,7 +2190,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { }) ).list( { - fieldsSet: (readLoader as any).args?.fieldsSet, + fieldsSet: readLoader.args?.fieldsSet, filterArr: [ new Filter({ id: null, @@ -2191,14 +2218,16 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { ); // defining BelongsTo read resolver method - proto[column.title] = async function (args?: any) { + proto[column.title] = async function ( + args?: RelationLoaderArgs, + ) { if ( this?.[cCol?.title] === null || this?.[cCol?.title] === undefined ) return null; - (readLoader as any).args = args; + readLoader.args = args; return await readLoader.load(this?.[cCol?.title]); }; @@ -2221,8 +2250,8 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { // it takes individual keys and callback is invoked with an array of values and we can get the // result for all those together and return the value in the same order as in the array // this way all parents data extracted together - const readLoader = new DataLoader( - (_ids: string[]) => + const readLoader = new DataLoaderWithArgs( + (_ids: readonly string[]) => this._queryQueue.add(async () => { // handle binary(16) foreign keys const ids = _ids.map((id) => { @@ -2257,7 +2286,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { }) ).list( { - fieldsSet: (readLoader as any).args?.fieldsSet, + fieldsSet: readLoader.args?.fieldsSet, filterArr: [ new Filter({ id: null, @@ -2285,28 +2314,30 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { ); // defining BelongsTo read resolver method - proto[column.title] = async function (args?: any) { + proto[column.title] = async function ( + args?: RelationLoaderArgs, + ) { if ( this?.[cCol?.title] === null || this?.[cCol?.title] === undefined ) return null; - (readLoader as any).args = args; + readLoader.args = args; return await readLoader.load(this?.[cCol?.title]); }; } else { - const listLoader = new DataLoader( - (ids: string[]) => + const listLoader = new DataLoaderWithArgs( + (ids: readonly string[]) => this._queryQueue.add(async () => { if (ids.length > 1) { const data = await this.multipleHmList( { colId: column.id, - ids, + ids: ids as string[], }, - (listLoader as any).args, + listLoader.args, ); return ids.map((id: string) => data[id] ? data[id]?.[0] : null, @@ -2319,7 +2350,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { colId: column.id, id: ids[0], }, - (listLoader as any).args, + listLoader.args, ) )?.[0] ?? null, ]; @@ -2335,8 +2366,8 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { column.uidt === UITypes.Links && !linksAsLtar ? `_nc_lk_${column.title}` : column.title - ] = async function (args): Promise { - (listLoader as any).args = args; + ] = async function (args?: RelationLoaderArgs): Promise { + listLoader.args = args; return listLoader.load( getCompositePkValue(self.model.primaryKeys, this), ); @@ -2946,6 +2977,14 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { await this.prepareNocoData(updateObj, false, cookie, prevData); + // Reject empty payloads explicitly — knex would otherwise throw + // "Empty .update() call detected" with no usable context for the user. + if (!updateObj || Object.keys(updateObj).length === 0) { + NcError.get(this.context).invalidRequestBody( + 'No valid fields provided in update payload', + ); + } + const query = this.dbDriver(this.tnPath) .update(updateObj) .where(await this._wherePk(id, true)); @@ -3435,7 +3474,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { */ public async findByMergeFields( mergeColumns: Column[], - mergeValuesPerRecord: any[][], + mergeValuesPerRecord: unknown[][], ): Promise[]> { if (mergeValuesPerRecord.length === 0) return []; @@ -3445,7 +3484,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { // Deduplicate merge value tuples const seen = new Set(); - const uniqueTuples: any[][] = []; + const uniqueTuples: unknown[][] = []; for (const tuple of mergeValuesPerRecord) { const key = tuple .map((v) => (v === null ? '\0NULL\0' : String(v))) @@ -3982,7 +4021,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { } // Helper method to format date - private formatDate(val: string): any { + private formatDate(val: string): Knex.Raw | string { const { isMySQL, isSqlite, isPg } = this.clientMeta; if (val.indexOf('-') < 0 && val.indexOf('+') < 0 && val.slice(-1) !== 'Z') { // if no timezone is given, @@ -4980,17 +5019,17 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { * */ public async handleRichTextMentions( - _prevData, - _newData: Record | Array>, - _req, + _prevData: Record | Record[] | null, + _newData: Record | Record[], + _req: NcRequest, ) { return; } public async beforeInsert( - data: any, + data: Record, _trx: any, - req, + req: NcRequest, params?: { allowSystemColumn?: boolean; }, @@ -5008,9 +5047,9 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { } public async beforeBulkInsert( - data: any, + data: Record[], _trx: any, - req, + req: NcRequest, params?: { allowSystemColumn?: boolean; }, @@ -5033,8 +5072,8 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { trx: _trx, req, }: { - data: any; - insertData: any; + data: Record; + insertData: Record; trx: any; req: NcRequest; }): Promise { @@ -5076,7 +5115,11 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { await this.handleRichTextMentions(null, data, req); } - public async afterBulkInsert(data: any[], _trx: any, req): Promise { + public async afterBulkInsert( + data: Record[], + _trx: any, + req: NcRequest, + ): Promise { await this.handleHooks('after.bulkInsert', null, data, req); let parentAuditId; @@ -5150,9 +5193,9 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { } public async afterDelete( - data: any, + data: Record, _trx: any, - req, + req: NcRequest, eventType: AuditV1OperationTypes = AuditV1OperationTypes.DATA_DELETE, ): Promise { const id = this.extractPksValues(data); @@ -5180,9 +5223,9 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { } public async afterBulkDelete( - data: any, + data: Record[], _trx: any, - req, + req: NcRequest, isBulkAllOperation = false, bulkEventType: AuditV1OperationTypes = AuditV1OperationTypes.DATA_BULK_DELETE, rowEventType: AuditV1OperationTypes = AuditV1OperationTypes.DATA_DELETE, @@ -5322,17 +5365,19 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { } public async afterBulkUpdate( - prevData: any, - newData: any, + prevData: Record[] | null, + newData: Record[] | number, _trx: any, - req, + req: NcRequest, isBulkAllOperation = false, ): Promise { - if (!isBulkAllOperation) { + if (!isBulkAllOperation && Array.isArray(newData)) { await this.handleHooks('after.bulkUpdate', prevData, newData, req); } - if (newData && newData.length > 0) { + if (!Array.isArray(newData)) return; + + if (newData.length > 0) { const parentAuditId = await Noco.ncAudit.genNanoid(MetaTable.AUDIT); // disable external source audit in cloud @@ -5433,7 +5478,11 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { await this.handleRichTextMentions(prevData, newData, req); } - public async beforeUpdate(data: any, _trx: any, req): Promise { + public async beforeUpdate( + data: Record, + _trx: any, + req: NcRequest, + ): Promise { const ignoreWebhook = req.query?.ignoreWebhook; if (ignoreWebhook) { if (ignoreWebhook != 'true' && ignoreWebhook != 'false') { @@ -5448,10 +5497,10 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { } public async afterUpdate( - prevData: any, - newData: any, + prevData: Record, + newData: Record, _trx: any, - req, + req: NcRequest, updateObj?: Record, ): Promise { // TODO this is a temporary fix for the audit log / DOMPurify causes issue for long text @@ -5530,7 +5579,11 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { await this.handleRichTextMentions(prevData, newData, req); } - public async beforeDelete(data: any, _trx: any, req): Promise { + public async beforeDelete( + data: Record, + _trx: any, + req: NcRequest, + ): Promise { if (this.model.synced) { NcError.get(this.context).prohibitedSyncTableOperation({ modelName: this.model.title, @@ -5541,7 +5594,11 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { await this.handleHooks('before.delete', null, data, req); } - public async beforeBulkDelete(_data: any, _trx: any, _req): Promise { + public async beforeBulkDelete( + _data: Record[], + _trx: any, + _req: NcRequest, + ): Promise { if (this.model.synced) { NcError.get(this.context).prohibitedSyncTableOperation({ modelName: this.model.title, @@ -5550,7 +5607,12 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { } } - protected async handleHooks(hookName, prevData, newData, req): Promise { + protected async handleHooks( + hookName: string, + prevData: Record | Record[] | null, + newData: Record | Record[] | null, + req: NcRequest, + ): Promise { Noco.eventEmitter.emit(HANDLE_WEBHOOK, { context: { ...this.context, cache: false, cacheMap: undefined }, hookName, @@ -5563,16 +5625,31 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { }); } - public async errorInsert(_e, _data, _trx, _cookie) {} + public async errorInsert( + _e: Error, + _data: Record, + _trx: any, + _cookie: NcRequest, + ) {} - public async errorUpdate(_e, _data, _trx, _cookie) {} + public async errorUpdate( + _e: Error, + _data: Record, + _trx: any, + _cookie: NcRequest, + ) {} // todo: handle composite primary key - public extractPksValues(data: any, asString = false) { + public extractPksValues(data: Record, asString = false) { return dataWrapper(data).extractPksValue(this.model, asString); } - protected async errorDelete(_e, _id, _trx, _cookie) {} + protected async errorDelete( + _e: Error, + _id: Record, + _trx: any, + _cookie: NcRequest, + ) {} async validate( data: Record, @@ -6342,7 +6419,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { }[] > { try { - const { where, ...rest } = this._getListArgs(args as any); + const { where, ...rest } = this._getListArgs(args); const columns = await this.model.getColumns(this.context); const column = columns?.find((col) => col.id === args.groupColumnId); @@ -6638,7 +6715,10 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { return await this.execAndParse(qb); } - public async execAndGetRows(query: string, trx?: Knex | CustomKnex) { + public async execAndGetRows( + query: string, + trx?: Knex | CustomKnex, + ): Promise[]> { trx = trx || this.dbDriver; query = this.sanitizeQuery(query); @@ -6650,7 +6730,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { } else if (this.isMySQL && INSERT_REGEX.test(query)) { const res = await trx.raw(query); if (res?.[0] && res[0].insertId !== undefined) { - return res[0].insertId; + return [{ insertId: res[0].insertId }]; } return res; } else { @@ -6658,20 +6738,20 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { } } + public async execAndParse( + qb: Knex.QueryBuilder | string, + dependencyColumns: Column[] | undefined | null, + options: ExecAndParseOptions & { first: true }, + ): Promise>; + public async execAndParse( + qb: Knex.QueryBuilder | string, + dependencyColumns?: Column[] | null, + options?: ExecAndParseOptions, + ): Promise[]>; public async execAndParse( qb: Knex.QueryBuilder | string, dependencyColumns?: Column[], - options: { - skipDateConversion?: boolean; - skipAttachmentConversion?: boolean; - skipSubstitutingColumnIds?: boolean; - skipUserConversion?: boolean; - skipJsonConversion?: boolean; - raw?: boolean; // alias for skipDateConversion and skipAttachmentConversion - first?: boolean; - bulkAggregate?: boolean; - apiVersion?: NcApiVersion; - } = { + options: ExecAndParseOptions = { skipDateConversion: false, skipAttachmentConversion: false, skipSubstitutingColumnIds: false, @@ -6802,7 +6882,10 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { return data; } - sanitizeQuery(query: string | string[]) { + sanitizeQuery(query: string): string; + sanitizeQuery(query: string[]): string[]; + sanitizeQuery(query: string | string[]): string | string[]; + sanitizeQuery(query: string | string[]): string | string[] { const fn = (q: string) => { if (!this.isPg && !this.isSnowflake) { return unsanitize(q); @@ -6974,6 +7057,16 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { }); } + protected async convertUserFormat( + data: Record[], + dependencyColumns?: Column[], + apiVersion?: NcApiVersion, + ): Promise[]>; + protected async convertUserFormat( + data: Record, + dependencyColumns?: Column[], + apiVersion?: NcApiVersion, + ): Promise>; protected async convertUserFormat( data: Record, dependencyColumns?: Column[], @@ -7350,6 +7443,14 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { ); } + public async convertJsonTypes( + data: Record[], + dependencyColumns?: Column[], + ): Promise[]>; + public async convertJsonTypes( + data: Record, + dependencyColumns?: Column[], + ): Promise>; public async convertJsonTypes( data: Record, dependencyColumns?: Column[], @@ -7412,6 +7513,14 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { } } + public async convertMultiSelectTypes( + data: Record[], + dependencyColumns?: Column[], + ): Promise[]>; + public async convertMultiSelectTypes( + data: Record, + dependencyColumns?: Column[], + ): Promise>; public async convertMultiSelectTypes( data: Record, dependencyColumns?: Column[], @@ -7438,6 +7547,14 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { } } + public async convertAttachmentType( + data: Record[], + dependencyColumns?: Column[], + ): Promise[]>; + public async convertAttachmentType( + data: Record, + dependencyColumns?: Column[], + ): Promise>; public async convertAttachmentType( data: Record, dependencyColumns?: Column[], @@ -7648,6 +7765,14 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { return d; } + public convertDateFormat( + data: Record[], + dependencyColumns?: Column[], + ): Record[]; + public convertDateFormat( + data: Record, + dependencyColumns?: Column[], + ): Record; public convertDateFormat( data: Record, dependencyColumns?: Column[], @@ -7771,7 +7896,7 @@ class BaseModelSqlv2 implements IBaseModelSqlV2 { try { await this.model.getColumns(this.context); - const { where, sort } = this._getListArgs(args as any); + const { where, sort } = this._getListArgs(args); // todo: get only required fields const relColumn = this.model.columnsById[colId]; diff --git a/packages/nocodb/src/db/BaseModelSqlv2/insert.ts b/packages/nocodb/src/db/BaseModelSqlv2/insert.ts index 0dc3443554..b91aeee506 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2/insert.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2/insert.ts @@ -95,8 +95,9 @@ export const baseModelInsert = (baseModel: IBaseModelSqlV2) => { } else { const res = await baseModel.execAndParse(query, null, { raw: true, + first: true, }); - id = res?.id ?? res[0]?.insertId ?? res; + id = res?.id ?? res?.insertId ?? res; } if (ai) { diff --git a/packages/nocodb/src/db/BaseModelSqlv2/ltar-cols-updater.ts b/packages/nocodb/src/db/BaseModelSqlv2/ltar-cols-updater.ts index d39a425f99..2757b4c130 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2/ltar-cols-updater.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2/ltar-cols-updater.ts @@ -45,7 +45,7 @@ export const LTARColsUpdater = (param: { if (!(col.title in d)) continue; // extract existing link values to current record - let existingLinks = []; + let existingLinks: Record[] | Record = []; profiler.log(`${col.colOptions.type} list start`); if (isMMOrMMLike(col)) { diff --git a/packages/nocodb/src/db/BaseModelSqlv2/select-object.ts b/packages/nocodb/src/db/BaseModelSqlv2/select-object.ts index 4e57c0527a..972c48fb63 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2/select-object.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2/select-object.ts @@ -514,7 +514,7 @@ export const selectObject = (baseModel: IBaseModelSqlV2, logger: Logger) => { break; } case UITypes.LongText: { - if ((baseModel.dbDriver as any).isExternal) { + if (baseModel.dbDriver.isExternal) { const colPath = sanitize( `${alias || baseModel.tnPath}.${column.column_name}`, ); diff --git a/packages/nocodb/src/db/CustomKnex.ts b/packages/nocodb/src/db/CustomKnex.ts index 559cca83e6..b67b3bd9a1 100644 --- a/packages/nocodb/src/db/CustomKnex.ts +++ b/packages/nocodb/src/db/CustomKnex.ts @@ -1046,12 +1046,23 @@ function parseNestedCondition(obj, qb, pKey?, table?, tableAlias?) { return qb; } +interface ExtDbConfig { + dbMux?: string; + sourceId?: string; + client?: string; + upgrader?: boolean; + source?: { upgraderQueries?: string[] }; + [key: string]: any; +} + type CustomKnex = Knex & { attachToTransaction?: (fn: () => void) => void; ops?: (() => void)[]; _cteGenerator?: CTEGenerator; cteGenerator?: (context?: NcContext) => CTEGenerator; applyCte?: (qb: Knex.QueryInterface) => void; + extDb?: ExtDbConfig; + isExternal?: boolean; }; type CustomTransaction = Knex.Transaction & { @@ -1063,7 +1074,7 @@ type CustomTransaction = Knex.Transaction & { function CustomKnex( arg: string | Knex.Config | any, - extDb?: any, + extDb?: ExtDbConfig, ): CustomKnex { if (arg?.client === 'sqlite3') { arg.useNullAsDefault = true; @@ -1604,3 +1615,4 @@ const parseConditionv2 = (_obj: Filter | FilterType, qb: Knex.QueryBuilder) => { export default CustomKnex; export { Knex, CustomKnex as XKnex }; +export type { ExtDbConfig }; diff --git a/packages/nocodb/src/db/IBaseModelSqlV2.ts b/packages/nocodb/src/db/IBaseModelSqlV2.ts index beb649bcfd..00a1c078bc 100644 --- a/packages/nocodb/src/db/IBaseModelSqlV2.ts +++ b/packages/nocodb/src/db/IBaseModelSqlV2.ts @@ -22,6 +22,7 @@ import type { NestedLinkAuditEntry, NestedLinkLastModifiedEntry, } from '~/db/BaseModelSqlv2/nested-link-preparator'; +import type { ExecAndParseOptions } from 'src/db/BaseModelSqlv2'; export interface IBaseModelSqlV2 { context: NcContext; @@ -43,19 +44,14 @@ export interface IBaseModelSqlV2 { ): Promise; execAndParse( qb: Knex.QueryBuilder | string, - dependencyColumns?: Column[], - options?: { - skipDateConversion?: boolean; - skipAttachmentConversion?: boolean; - skipSubstitutingColumnIds?: boolean; - skipUserConversion?: boolean; - skipJsonConversion?: boolean; - raw?: boolean; // alias for skipDateConversion and skipAttachmentConversion - first?: boolean; - bulkAggregate?: boolean; - apiVersion?: NcApiVersion; - }, - ): Promise; + dependencyColumns: Column[] | undefined | null, + options: ExecAndParseOptions & { first: true }, + ): Promise>; + execAndParse( + qb: Knex.QueryBuilder | string, + dependencyColumns?: Column[] | null, + options?: ExecAndParseOptions, + ): Promise[]>; prepareNocoData( data, diff --git a/packages/nocodb/src/db/field-handler/handlers/formula/formula.general.handler.ts b/packages/nocodb/src/db/field-handler/handlers/formula/formula.general.handler.ts index d8e82cf899..11016f63af 100644 --- a/packages/nocodb/src/db/field-handler/handlers/formula/formula.general.handler.ts +++ b/packages/nocodb/src/db/field-handler/handlers/formula/formula.general.handler.ts @@ -37,7 +37,7 @@ export class FormulaGeneralHandler extends ComputedFieldHandler { ).builder; const parsedTree: ParsedFormulaNode = formula.getParsedTree(); const value = - parsedTree.dataType === FormulaDataTypes.DATE + parsedTree?.dataType === FormulaDataTypes.DATE ? filter.value : knex.raw('?', [ // convert value to number if formulaDataType if numeric diff --git a/packages/nocodb/src/db/field-handler/handlers/lookup/lookup.general.handler.ts b/packages/nocodb/src/db/field-handler/handlers/lookup/lookup.general.handler.ts index 7a14c3f83f..f204f5c1fc 100644 --- a/packages/nocodb/src/db/field-handler/handlers/lookup/lookup.general.handler.ts +++ b/packages/nocodb/src/db/field-handler/handlers/lookup/lookup.general.handler.ts @@ -53,7 +53,9 @@ export class LookupGeneralHandler extends ComputedFieldHandler { const colOptions = await column.getColOptions(context); - if (colOptions?.error) { + // Skip filter when colOptions is missing (orphaned lookup) or carries + // a parse error — emitting an empty clause keeps the query valid. + if (!colOptions || colOptions.error) { return { clause: (_qb) => {}, rootApply: undefined, diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts index 5c13ae9cab..5baa58cade 100644 --- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts @@ -525,10 +525,10 @@ export default async function formulaQueryBuilderv2({ TelemetryHandlerService.sendPriorityError(context, { trigger: 'formulaQueryBuilder', error_type: 'FORMULA_TOO_LONG_ERROR', - message: `Formula length too long for ${columnInfo.title}${columnInfo.id}`, + message: `Generated query too long for ${columnInfo.title}${columnInfo.id}`, }); NcError.get(context).formulaError( - `Formula length too long for ${columnInfo.title}`, + `The generated query for ${columnInfo.title} exceeds the maximum allowed length. Try simplifying the formula by reducing the number of referenced fields, lookup chains, or nested formula references.`, ); } if (!validateFormula) return qb; diff --git a/packages/nocodb/src/db/relation-manager.ts b/packages/nocodb/src/db/relation-manager.ts index d90283b6b6..09f2ca8f43 100644 --- a/packages/nocodb/src/db/relation-manager.ts +++ b/packages/nocodb/src/db/relation-manager.ts @@ -493,7 +493,7 @@ export class RelationManager { // Wrap cardinality enforcement + insert in a single transaction. // External mux sources don't support PG transactions over HTTP — skip // the transaction wrapper but still execute queries through execAndParse. - const isExternal = (baseModel.dbDriver as any).isExternal; + const isExternal = !!baseModel.dbDriver.isExternal; const trx: any = isExternal ? baseModel.dbDriver : await baseModel.dbDriver.transaction(); diff --git a/packages/nocodb/src/db/sql-data-mapper/lib/BaseModel.ts b/packages/nocodb/src/db/sql-data-mapper/lib/BaseModel.ts index d7b779d53e..bb015a492a 100644 --- a/packages/nocodb/src/db/sql-data-mapper/lib/BaseModel.ts +++ b/packages/nocodb/src/db/sql-data-mapper/lib/BaseModel.ts @@ -1251,12 +1251,17 @@ abstract class BaseModel { limit = 20, offset = 0, sort = '', - }: { limit?: number | string; offset?: number | string; sort?: string }, + }: { + limit?: number | string; + offset?: number | string; + sort?: string | string[]; + }, ) { query.offset(offset).limit(limit); if (sort) { - sort.split(',').forEach((o) => { + const sortItems = Array.isArray(sort) ? sort : sort.split(','); + sortItems.forEach((o) => { if (o[0] === '-') { query.orderBy(o.slice(1), 'desc'); } else { @@ -1521,7 +1526,7 @@ export interface XcFilter { limit?: string | number; shuffle?: string | number; offset?: string | number; - sort?: string; + sort?: string | string[]; fields?: string; filterArr?: Filter[]; sortArr?: Sort[]; @@ -1542,6 +1547,8 @@ export interface XcFilterWithAlias extends XcFilter { s?: string; f?: string; p?: string | number; + /** Allow extra properties — callers pass custom fields that _getListArgs ignores. */ + [key: string]: any; } export default BaseModel; diff --git a/packages/nocodb/src/dbQueryClient/generic.ts b/packages/nocodb/src/dbQueryClient/generic.ts index 564d0480d8..05589570b6 100644 --- a/packages/nocodb/src/dbQueryClient/generic.ts +++ b/packages/nocodb/src/dbQueryClient/generic.ts @@ -69,7 +69,7 @@ export abstract class GenericDBQueryClient implements DBQueryClient { async singleQueryRead( _context: any, _ctx: any, - ): Promise>> { + ): Promise> { throw new Error('Not implemented'); } diff --git a/packages/nocodb/src/decorators/trace-command.decorator.ts b/packages/nocodb/src/decorators/trace-command.decorator.ts new file mode 100644 index 0000000000..0b69cf70ba --- /dev/null +++ b/packages/nocodb/src/decorators/trace-command.decorator.ts @@ -0,0 +1,12 @@ +import type { OperationName } from '~/command-registry/op-names'; + +// CE no-op stub. EE overrides with the real implementation. +export function TraceCommand(_name: OperationName, _version: number = 1) { + return function ( + _target: unknown, + _propertyKey: string, + descriptor: PropertyDescriptor, + ) { + return descriptor; + }; +} diff --git a/packages/nocodb/src/filters/global-exception/global-exception.filter.ts b/packages/nocodb/src/filters/global-exception/global-exception.filter.ts index 97d4e9bda2..9c80f99e47 100644 --- a/packages/nocodb/src/filters/global-exception/global-exception.filter.ts +++ b/packages/nocodb/src/filters/global-exception/global-exception.filter.ts @@ -2,7 +2,6 @@ import { Catch, Logger, NotFoundException } from '@nestjs/common'; import * as Sentry from '@sentry/nestjs'; import { ThrottlerException } from '@nestjs/throttler'; -import hash from 'object-hash'; import { NcApiVersion, NcErrorType, @@ -12,8 +11,7 @@ import { } from 'nocodb-sdk'; import type { ArgumentsHost, ExceptionFilter } from '@nestjs/common'; import type { Request, Response } from 'express'; -import NocoCache from '~/cache/NocoCache'; -import { CacheGetType } from '~/utils/globals'; +import { throttlerLogger } from '~/helpers/throttlerLogger'; import { AjvError, BadRequest, @@ -90,64 +88,10 @@ export class GlobalExceptionFilter implements ExceptionFilter { this.logError(exception, request); if (exception instanceof ThrottlerException) { - const key = hash({ - ip: request.ip, - baseId: (request as any).ncBaseId, + throttlerLogger.record({ workspaceId: (request as any).ncWorkspaceId, - path: request.path, + baseId: (request as any).ncBaseId, }); - - const cacheKey = `throttler:${key}`; - - NocoCache.get('root', cacheKey, CacheGetType.TYPE_OBJECT) - .then((data) => { - if (!data) { - this.logger.warn( - `ThrottlerException: ${exception.message}, Path: ${ - request.path - }, Workspace ID: ${(request as any).ncWorkspaceId}, Base ID: ${ - (request as any).ncBaseId - }`, - ); - - NocoCache.setExpiring( - 'root', - cacheKey, - { value: true, count: 1, timestamp: Date.now() }, - 300, - ).catch((err) => { - this.logger.error(err); - }); - } else { - data.count += 1; - - const ttlInSeconds = Math.floor( - (data.timestamp + 300000 - Date.now()) / 1000, - ); - - if (ttlInSeconds > 0) { - NocoCache.setExpiring('root', cacheKey, data, ttlInSeconds).catch( - (err) => { - this.logger.error(err); - }, - ); - } - - // log every 50th request in last 5 minutes after first - if (data.count % 50 === 0) { - this.logger.warn( - `ThrottlerException: ${exception.message}, Path: ${ - request.path - }, Workspace ID: ${(request as any).ncWorkspaceId}, Base ID: ${ - (request as any).ncBaseId - }, Requests in last 5 minutes: ${data.count}`, - ); - } - } - }) - .catch((err) => { - this.logger.error(err); - }); } // if sso error then redirect to ui with error in query parameter diff --git a/packages/nocodb/src/helpers/columnHelpers.ts b/packages/nocodb/src/helpers/columnHelpers.ts index 47fc24160b..c18616034f 100644 --- a/packages/nocodb/src/helpers/columnHelpers.ts +++ b/packages/nocodb/src/helpers/columnHelpers.ts @@ -24,6 +24,7 @@ import type { NcContext } from '~/interface/config'; import type { RollupColumn, View } from '~/models'; import type { ColumnWebhookManager } from '~/utils/column-webhook-manager'; import type Model from '~/models/Model'; +import type { LtarHmBtIds } from '~/services/columns.service.type'; import { GridViewColumn } from '~/models'; import validateParams from '~/helpers/validateParams'; import { getUniqueColumnAliasName } from '~/helpers/getUniqueName'; @@ -55,6 +56,10 @@ export async function createHmAndBtColumn( parentColumn?: Column, isCustom = false, columnWebhookManager?: ColumnWebhookManager, + // Sandbox-replay only — pre-set IDs honored by `Column.insert` and ID + // capture for the recording side. CE callers leave both undefined. + idHints?: LtarHmBtIds, + out?: LtarHmBtIds, ) { let savedColumn: Column; let crossBaseProps: { @@ -80,6 +85,7 @@ export async function createHmAndBtColumn( const childRelCol = await Column.insert( { ...context, base_id: child.base_id }, { + ...(idHints?.childRelColId ? { id: idHints.childRelColId } : {}), title, fk_model_id: child.id, @@ -115,6 +121,7 @@ export async function createHmAndBtColumn( }, }, ); + if (out) out.childRelColId = childRelCol.id; await columnWebhookManager?.addNewColumnById({ columnId: childRelCol.id, action: WebhookActions.INSERT, @@ -157,6 +164,7 @@ export async function createHmAndBtColumn( savedColumn = await Column.insert( { ...context, base_id: parent.base_id }, { + ...(idHints?.savedColumnId ? { id: idHints.savedColumnId } : {}), title, fk_model_id: parent.id, uidt: isLinks ? UITypes.Links : UITypes.LinkToAnotherRecord, @@ -179,6 +187,7 @@ export async function createHmAndBtColumn( ...crossBaseProps, }, ); + if (out) out.savedColumnId = savedColumn.id; await columnWebhookManager?.addNewColumnById({ columnId: savedColumn.id, action: WebhookActions.INSERT, @@ -231,6 +240,9 @@ export async function createOOColumn( parentColumn?: Column, isCustom = false, columnWebhookManager?: ColumnWebhookManager, + // Sandbox-replay only — see `createHmAndBtColumn` for shape and rationale. + idHints?: LtarHmBtIds, + out?: LtarHmBtIds, ) { let savedColumn: Column; @@ -260,6 +272,7 @@ export async function createOOColumn( const childRelCol = await Column.insert( childContext, { + ...(idHints?.childRelColId ? { id: idHints.childRelColId } : {}), title, fk_model_id: child.id, // ref_db_alias @@ -291,6 +304,7 @@ export async function createOOColumn( ...crossBaseProps, }, ); + if (out) out.childRelColId = childRelCol.id; await columnWebhookManager?.addNewColumnById({ columnId: childRelCol.id, @@ -337,6 +351,7 @@ export async function createOOColumn( }; savedColumn = await Column.insert(parentContext, { + ...(idHints?.savedColumnId ? { id: idHints.savedColumnId } : {}), title, fk_model_id: parent.id, uidt: UITypes.LinkToAnotherRecord, @@ -358,6 +373,7 @@ export async function createOOColumn( ...crossBaseProps, ...(colExtra || {}), }); + if (out) out.savedColumnId = savedColumn.id; await columnWebhookManager?.addNewColumnById({ columnId: savedColumn.id, @@ -777,7 +793,14 @@ export const TableSystemColumns = (isMetaColSupport = false, isMeta = true) => [ : []), ]; -export const deleteColumnSystemPropsFromRequest = (col: any) => { +export enum OperationSource { + AT_IMPORT = 'at_import', +} + +export const deleteColumnSystemPropsFromRequest = ( + col: any, + opts?: { operationSource?: OperationSource }, +) => { // remove all properties not in documentations delete col.dt; delete col.np; @@ -795,7 +818,20 @@ export const deleteColumnSystemPropsFromRequest = (col: any) => { // delete col.dtxs; delete col.au; delete col.validate; - delete col.system; + switch (opts?.operationSource) { + case OperationSource.AT_IMPORT: { + const isNcRecordColumn = col.system && + ['ncRecordId', 'ncRecordHash'].includes(col.title); + if (!isNcRecordColumn) { + delete col.system; + } + break; + } + deafult: { + delete col.system; + } + } + }; // get the reverse type of the relation diff --git a/packages/nocodb/src/helpers/dbHelpers.ts b/packages/nocodb/src/helpers/dbHelpers.ts index eae2540942..477d7b5c48 100644 --- a/packages/nocodb/src/helpers/dbHelpers.ts +++ b/packages/nocodb/src/helpers/dbHelpers.ts @@ -99,6 +99,12 @@ export function _wherePk( ids = (id + '').split('___').map((val) => val.replaceAll('\\_', '_')); } + // Reject incomplete composite ids up-front — otherwise knex builds a + // WHERE with `undefined` bindings and throws a generic 500. + if (!skipPkValidation && (ids as unknown[]).length < primaryKeys.length) { + NcError.invalidPrimaryKey(id, primaryKeys.map((pk) => pk.title).join(',')); + } + for (let i = 0; i < primaryKeys.length; ++i) { if (primaryKeys[i].dt === 'bytea') { // if column is bytea, then we need to encode the id to hex based on format diff --git a/packages/nocodb/src/helpers/sandboxGuards.ts b/packages/nocodb/src/helpers/sandboxGuards.ts new file mode 100644 index 0000000000..f69af7e30e --- /dev/null +++ b/packages/nocodb/src/helpers/sandboxGuards.ts @@ -0,0 +1,20 @@ +import type { NcContext } from '~/interface/config'; + +// CE no-op stubs. Sandboxes are an EE feature; EE overrides these with +// the real impl in `src/ee/helpers/sandboxGuards.ts`. + +export async function assertNotSandbox( + _context: NcContext, + _message?: string, +): Promise {} + +export async function assertNotSandboxProduction( + _context: NcContext, + _message?: string, +): Promise {} + +export async function assertNotLockedViewOnSandboxProduction( + _context: NcContext, + _viewId: string, + _message?: string, +): Promise {} diff --git a/packages/nocodb/src/helpers/tableHelpers.ts b/packages/nocodb/src/helpers/tableHelpers.ts index ec0b05032b..b3d667ff62 100644 --- a/packages/nocodb/src/helpers/tableHelpers.ts +++ b/packages/nocodb/src/helpers/tableHelpers.ts @@ -14,6 +14,7 @@ import type { User } from '~/models'; import { Permission } from '~/models'; import { deleteColumnSystemPropsFromRequest, + type OperationSource, TableSystemColumns, } from '~/helpers/columnHelpers'; import { @@ -29,10 +30,12 @@ export const repopulateCreateTableSystemColumns = ( columns, clientType, isMeta = true, + operationSource, }: { columns: (ColumnType & { cn?: string })[]; clientType: DriverClient; isMeta?: boolean; + operationSource?: OperationSource; }, ) => { const tableSystemColumns = TableSystemColumns( @@ -85,7 +88,7 @@ export const repopulateCreateTableSystemColumns = ( col.cn = col.column_name; } } - deleteColumnSystemPropsFromRequest(col); + deleteColumnSystemPropsFromRequest(col, { operationSource }); } return result; }; diff --git a/packages/nocodb/src/helpers/throttlerLogger.ts b/packages/nocodb/src/helpers/throttlerLogger.ts new file mode 100644 index 0000000000..953ee1d5d6 --- /dev/null +++ b/packages/nocodb/src/helpers/throttlerLogger.ts @@ -0,0 +1,99 @@ +import { Logger } from '@nestjs/common'; +import NocoCache from '~/cache/NocoCache'; + +const FLUSH_INTERVAL_MS = 60_000; +// 1.5x window — absorbs clock skew between pods so a late-arriving leader +// can't emit a duplicate after the early leader's flag has expired. +const REDIS_DEDUP_TTL_SECONDS = 90; + +interface Bucket { + count: number; + firstSeenMs: number; + workspaceId?: string; + baseId?: string; +} + +export class ThrottlerLogger { + private readonly logger = new Logger('ThrottlerLogger'); + private buckets = new Map(); + + record({ workspaceId, baseId }: { workspaceId?: string; baseId?: string }) { + const key = `${workspaceId ?? 'anon'}:${baseId ?? '-'}`; + const existing = this.buckets.get(key); + if (existing) { + existing.count += 1; + return; + } + this.buckets.set(key, { + count: 1, + firstSeenMs: Date.now(), + workspaceId, + baseId, + }); + } + + async flush() { + if (this.buckets.size === 0) return; + + const snapshot = this.buckets; + this.buckets = new Map(); + + const windowId = Math.floor(Date.now() / FLUSH_INTERVAL_MS); + + for (const [key, bucket] of snapshot) { + const counterKey = `throttler:count:${windowId}:${key}`; + const leaderKey = `throttler:leader:${windowId}:${key}`; + + // Best-effort: every pod contributes to the shared counter, only the + // leader logs. The logged total reflects whatever pods had flushed + // before the leader's INCRBY — i.e. an approximate global count, not + // exact, but bounded above by the true total. + let total = bucket.count; + let isLeader = true; + try { + total = await NocoCache.incrbyExpiring( + 'root', + counterKey, + bucket.count, + REDIS_DEDUP_TTL_SECONDS, + ); + isLeader = await NocoCache.setIfNotExist( + 'root', + leaderKey, + '1', + REDIS_DEDUP_TTL_SECONDS, + ); + } catch { + // Cache unavailable — log local count so we don't lose the signal. + // Worst case is O(pods) duplicate lines for this window. + total = bucket.count; + isLeader = true; + } + + if (!isLeader) continue; + + this.logger.warn( + `ThrottlerException rollup: count=${total}, workspaceId=${ + bucket.workspaceId ?? 'anon' + }, baseId=${bucket.baseId ?? '-'}, windowMs=${FLUSH_INTERVAL_MS}`, + ); + } + } +} + +// Module-level singleton — guarded against re-evaluation under nest watch / +// hot reload to avoid leaking a setInterval per reload. +const SINGLETON_KEY = Symbol.for('nocodb.throttlerLogger'); +const g = globalThis as { [SINGLETON_KEY]?: ThrottlerLogger }; + +if (!g[SINGLETON_KEY]) { + const instance = new ThrottlerLogger(); + const fallbackLogger = new Logger('ThrottlerLogger'); + setInterval( + () => instance.flush().catch((err) => fallbackLogger.error(err)), + FLUSH_INTERVAL_MS, + ).unref?.(); + g[SINGLETON_KEY] = instance; +} + +export const throttlerLogger: ThrottlerLogger = g[SINGLETON_KEY]!; diff --git a/packages/nocodb/src/helpers/webhookHelpers.ts b/packages/nocodb/src/helpers/webhookHelpers.ts index a1c19e3031..a20f390920 100644 --- a/packages/nocodb/src/helpers/webhookHelpers.ts +++ b/packages/nocodb/src/helpers/webhookHelpers.ts @@ -42,7 +42,11 @@ Handlebars.registerHelper('json', function (context, pretty = false) { const logger = new Logger('webhookHelpers'); -export function parseBody(template: string, data: any): string { +export function parseBody( + template: string, + data: any, + vars?: Record, +): string { if (!template) { return template; } @@ -51,6 +55,7 @@ export function parseBody(template: string, data: any): string { return Handlebars.compile(template, { noEscape: true })({ data, event: data, + vars: vars || {}, }); } catch (e) { // if parsing fails then return the original template diff --git a/packages/nocodb/src/interface/Jobs.ts b/packages/nocodb/src/interface/Jobs.ts index b8d241d9df..15e09a9de1 100644 --- a/packages/nocodb/src/interface/Jobs.ts +++ b/packages/nocodb/src/interface/Jobs.ts @@ -77,6 +77,8 @@ export enum JobTypes { ChatApproval = 'chat-approval', BaseTrashCleanUp = 'base-trash-clean-up', DataImport = 'data-import', + SandboxMerge = 'sandbox-merge', + ManagedAppUpdate = 'managed-app-update', } export const SKIP_STORING_JOB_META = [ @@ -188,6 +190,10 @@ export interface DuplicateBaseJobData extends JobData { excludeScripts?: boolean; excludeDashboards?: boolean; excludeWorkflows?: boolean; + excludeDocuments?: boolean; + excludePersonalViews?: boolean; + excludePermissions?: boolean; + excludeRls?: boolean; }; } @@ -223,6 +229,24 @@ export interface DuplicateDashboardJobData extends JobData { options: never; } +export interface SandboxMergeJobData extends JobData { + sandboxBaseId: string; + productionBaseId: string; + sandboxId: string; + req: NcRequest; + selectedChangelogIds?: string[]; +} + +export interface ManagedAppUpdateJobData extends JobData { + managedAppId: string; + managedAppTitle: string; + masterBaseId: string; + masterWorkspaceId: string; + newVersionId: string; + newVersion: string; + req: NcRequest; +} + export interface HandleWebhookJobData extends JobData { hookId: string; modelId: string; diff --git a/packages/nocodb/src/main.ts b/packages/nocodb/src/main.ts index 981dfead49..7c6916767e 100644 --- a/packages/nocodb/src/main.ts +++ b/packages/nocodb/src/main.ts @@ -8,7 +8,8 @@ server.disable('etag'); server.disable('x-powered-by'); server.use( cors({ - exposedHeaders: 'xc-db-response', + exposedHeaders: + 'xc-db-response, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, X-RateLimit-Policy, Retry-After', }), ); diff --git a/packages/nocodb/src/meta/meta.service.ts b/packages/nocodb/src/meta/meta.service.ts index 80411ba17f..0b05493dc1 100644 --- a/packages/nocodb/src/meta/meta.service.ts +++ b/packages/nocodb/src/meta/meta.service.ts @@ -115,6 +115,7 @@ export class MetaService { [MetaTable.HOOK_LOGS]: 'hkl', [MetaTable.API_TOKENS]: 'tkn', [MetaTable.EXTENSIONS]: 'ext', + [MetaTable.BASE_VARIABLES]: 'bv', [MetaTable.COMMENTS]: 'com', [MetaTable.COMMENTS_REACTIONS]: 'cre', [MetaTable.USER_COMMENTS_NOTIFICATIONS_PREFERENCE]: 'cnp', @@ -143,6 +144,7 @@ export class MetaService { [MetaTable.MANAGED_APP_VERSIONS]: 'mav', [MetaTable.MANAGED_APP_DEPLOYMENT_LOGS]: 'madl', [MetaTable.SANDBOXES]: 'sb', + [MetaTable.SANDBOX_CHANGELOG]: 'scl', [MetaTable.SCIM_CONFIG]: 'scfg', [MetaTable.RLS_POLICIES]: 'rlp', [MetaTable.RLS_POLICY_SUBJECTS]: 'rlps', @@ -951,8 +953,12 @@ export class MetaService { ? this.connection : await this.connection.transaction(); - // todo: tobe done - return new MetaService( + // Instantiate via this.constructor so subclasses (e.g. EE MetaService) + // returned from startTransaction keep their overridden methods. + // Hard-coding `new MetaService(...)` here would always return a CE + // instance even when called on an EE/EE-Cloud subclass. + const Ctor = this.constructor as typeof MetaService; + return new Ctor( this.config, trx, // we need to keep track of the nested transaction level diff --git a/packages/nocodb/src/meta/migrations/XcMigrationSourcev0.ts b/packages/nocodb/src/meta/migrations/XcMigrationSourcev0.ts index affe80731f..85cb8f91cd 100644 --- a/packages/nocodb/src/meta/migrations/XcMigrationSourcev0.ts +++ b/packages/nocodb/src/meta/migrations/XcMigrationSourcev0.ts @@ -54,6 +54,7 @@ import * as nc_202604160000_docs_in_data from './v0/nc_202604160000_docs_in_data import * as nc_202604200002_trash_cleanup_due_at from './v0/nc_202604200002_trash_cleanup_due_at'; import * as nc_202604220000_uuid_readonly from './v0/nc_202604220000_uuid_readonly'; import * as nc_202604270000_base_trash from './v0/nc_202604270000_base_trash'; +import * as nc_202604290000_base_variables_and_sandbox_changelog from './v0/nc_202604290000_base_variables_and_sandbox_changelog'; // Create a custom migration source class export default class XcMigrationSourcev0 { @@ -119,6 +120,7 @@ export default class XcMigrationSourcev0 { 'nc_202603180000_user_mfa', 'nc_202604220000_uuid_readonly', 'nc_202604270000_base_trash', + 'nc_202604290000_base_variables_and_sandbox_changelog', ]); } @@ -240,6 +242,8 @@ export default class XcMigrationSourcev0 { return nc_202604220000_uuid_readonly; case 'nc_202604270000_base_trash': return nc_202604270000_base_trash; + case 'nc_202604290000_base_variables_and_sandbox_changelog': + return nc_202604290000_base_variables_and_sandbox_changelog; } } } diff --git a/packages/nocodb/src/meta/migrations/v0/nc_202604160000_docs_in_data.ts b/packages/nocodb/src/meta/migrations/v0/nc_202604160000_docs_in_data.ts index 331a2eb1ed..8ece70a379 100644 --- a/packages/nocodb/src/meta/migrations/v0/nc_202604160000_docs_in_data.ts +++ b/packages/nocodb/src/meta/migrations/v0/nc_202604160000_docs_in_data.ts @@ -47,11 +47,18 @@ const up = async (knex: Knex) => { const isRoot = !doc.parent_id; const offset = isRoot ? maxOrderByBase.get(doc.base_id) || 0 : 0; + // nc_docs_v2.title is varchar(512), nc_models_v2.title is varchar(255). + // Truncate to fit and preserve the overflow in description so nothing is lost. + const title = doc.title?.slice(0, 255); + const overflow = + doc.title && doc.title.length > 255 ? doc.title.slice(255) : null; + return { id: doc.id, base_id: doc.base_id, fk_workspace_id: doc.fk_workspace_id, - title: doc.title, + title, + description: overflow, meta: doc.meta, order: (Number(doc.order) || 0) + offset, parent_id: doc.parent_id, diff --git a/packages/nocodb/src/meta/migrations/v0/nc_202604290000_base_variables_and_sandbox_changelog.ts b/packages/nocodb/src/meta/migrations/v0/nc_202604290000_base_variables_and_sandbox_changelog.ts new file mode 100644 index 0000000000..bf5c59290d --- /dev/null +++ b/packages/nocodb/src/meta/migrations/v0/nc_202604290000_base_variables_and_sandbox_changelog.ts @@ -0,0 +1,152 @@ +import type { Knex } from 'knex'; +import { MetaTable } from '~/utils/globals'; + +const up = async (knex: Knex) => { + await knex.schema.createTable(MetaTable.BASE_VARIABLES, (table) => { + table.string('id', 20).notNullable(); + table.string('base_id', 20); + table.string('fk_workspace_id', 20); + table.string('key', 255); + table.text('value'); + table.text('description'); + table.string('inheritance', 20).defaultTo('fixed'); + table.string('type', 20).defaultTo('text'); + table.float('order'); + table.text('default_value'); + table.boolean('is_overridden').defaultTo(false); + table.boolean('is_inherited').defaultTo(false); + table.timestamps(true, true); + table.primary(['base_id', 'id']); + }); + + await knex.schema.alterTable(MetaTable.BASE_VARIABLES, (table) => { + table.unique( + ['fk_workspace_id', 'base_id', 'key'], + 'nc_base_variables_ws_base_key_unique', + ); + table.index( + ['base_id', 'fk_workspace_id'], + 'nc_base_variables_base_ws_index', + ); + }); + + await knex.schema.createTable(MetaTable.SANDBOX_CHANGELOG, (table) => { + table.string('id', 20).notNullable().primary(); + // App-managed monotonic counter (microseconds + intra-tick increment). + // Replay iterates in seq-ASC order. created_at has only second precision + // (meta.service.ts now()), so we cannot rely on it alone for ordering. + table.bigInteger('seq').notNullable(); + table.string('fk_sandbox_id', 20).notNullable(); + table.string('base_id', 20).notNullable(); + table.string('event', 80).notNullable(); + table.string('entity_type', 40).notNullable(); + table.string('entity_id', 20); + table.string('entity_title', 255); + table.string('parent_entity_id', 20); + table.string('parent_entity_title', 255); + table.string('created_by', 20).notNullable(); + table.text('description').nullable(); + table.text('meta'); + table.string('status', 20).notNullable().defaultTo('pending'); + table.timestamp('merged_at'); + table.timestamps(true, true); + }); + + await knex.schema.alterTable(MetaTable.SANDBOX_CHANGELOG, (table) => { + table.index(['fk_sandbox_id', 'seq'], 'nc_scl_sandbox_seq_index'); + table.index(['base_id'], 'nc_scl_base_id_index'); + table.index(['entity_type', 'entity_id'], 'nc_scl_entity_type_id_index'); + }); + + // Re-scope subscriber unique index to include base_id so sandbox and + // production can hold parallel subscriber rows for the same (workflow, user) + // pair. + await knex.schema.alterTable(MetaTable.AUTOMATION_SUBSCRIBERS, (table) => { + table.dropUnique( + ['fk_automation_id', 'fk_user_id'], + 'nc_automation_subscribers_unique_idx', + ); + }); + + await knex.schema.alterTable(MetaTable.AUTOMATION_SUBSCRIBERS, (table) => { + table.unique(['base_id', 'fk_automation_id', 'fk_user_id'], { + indexName: 'nc_automation_subscribers_unique_idx', + }); + }); + + // Sandbox terminology: rename "master" → "production" on sandbox-related + // tables. Managed-app `managed_app_master` is a separate concept (template + // vs installed instance) and is left unchanged. + await knex.schema.alterTable(MetaTable.SANDBOXES, (table) => { + table.dropIndex(['master_base_id'], 'nc_sandboxes_v2_master_base_id_idx'); + }); + await knex.schema.alterTable(MetaTable.SANDBOXES, (table) => { + table.renameColumn('master_base_id', 'production_base_id'); + }); + await knex.schema.alterTable(MetaTable.SANDBOXES, (table) => { + table.index( + ['production_base_id'], + 'nc_sandboxes_v2_production_base_id_idx', + ); + }); + + await knex.schema.alterTable(MetaTable.PROJECT, (table) => { + table.dropIndex(['is_sandbox_master'], 'nc_bases_is_sandbox_master_idx'); + }); + await knex.schema.alterTable(MetaTable.PROJECT, (table) => { + table.renameColumn('is_sandbox_master', 'is_sandbox_production'); + }); + await knex.schema.alterTable(MetaTable.PROJECT, (table) => { + table.index( + ['is_sandbox_production'], + 'nc_bases_is_sandbox_production_idx', + ); + }); +}; + +const down = async (knex: Knex) => { + // Reverse sandbox terminology rename + await knex.schema.alterTable(MetaTable.PROJECT, (table) => { + table.dropIndex( + ['is_sandbox_production'], + 'nc_bases_is_sandbox_production_idx', + ); + }); + await knex.schema.alterTable(MetaTable.PROJECT, (table) => { + table.renameColumn('is_sandbox_production', 'is_sandbox_master'); + }); + await knex.schema.alterTable(MetaTable.PROJECT, (table) => { + table.index(['is_sandbox_master'], 'nc_bases_is_sandbox_master_idx'); + }); + + await knex.schema.alterTable(MetaTable.SANDBOXES, (table) => { + table.dropIndex( + ['production_base_id'], + 'nc_sandboxes_v2_production_base_id_idx', + ); + }); + await knex.schema.alterTable(MetaTable.SANDBOXES, (table) => { + table.renameColumn('production_base_id', 'master_base_id'); + }); + await knex.schema.alterTable(MetaTable.SANDBOXES, (table) => { + table.index(['master_base_id'], 'nc_sandboxes_v2_master_base_id_idx'); + }); + + await knex.schema.alterTable(MetaTable.AUTOMATION_SUBSCRIBERS, (table) => { + table.dropUnique( + ['base_id', 'fk_automation_id', 'fk_user_id'], + 'nc_automation_subscribers_unique_idx', + ); + }); + + await knex.schema.alterTable(MetaTable.AUTOMATION_SUBSCRIBERS, (table) => { + table.unique(['fk_automation_id', 'fk_user_id'], { + indexName: 'nc_automation_subscribers_unique_idx', + }); + }); + + await knex.schema.dropTableIfExists(MetaTable.SANDBOX_CHANGELOG); + await knex.schema.dropTableIfExists(MetaTable.BASE_VARIABLES); +}; + +export { up, down }; diff --git a/packages/nocodb/src/models/Base.ts b/packages/nocodb/src/models/Base.ts index 7737a84837..6d1cec2f1e 100644 --- a/packages/nocodb/src/models/Base.ts +++ b/packages/nocodb/src/models/Base.ts @@ -65,7 +65,7 @@ export default class Base implements BaseType { managed_app_schema_locked?: boolean; // Computed: whether schema modifications are allowed // sandbox props - is_sandbox_master?: boolean; // Is this base a master base that has sandbox(es)? + is_sandbox_production?: boolean; // Is this base a production base that has sandbox(es) derived from it? is_sandbox?: boolean; // Is this base a sandbox base? constructor(base: Partial) { @@ -99,7 +99,7 @@ export default class Base implements BaseType { 'managed_app_id', 'managed_app_version_id', 'auto_update', - 'is_sandbox_master', + 'is_sandbox_production', 'is_sandbox', ]); @@ -482,7 +482,7 @@ export default class Base implements BaseType { 'managed_app_id', 'managed_app_version_id', 'auto_update', - 'is_sandbox_master', + 'is_sandbox_production', 'is_sandbox', ]); diff --git a/packages/nocodb/src/models/BaseVariable.ts b/packages/nocodb/src/models/BaseVariable.ts new file mode 100644 index 0000000000..a949142edb --- /dev/null +++ b/packages/nocodb/src/models/BaseVariable.ts @@ -0,0 +1,351 @@ +import CryptoJS from 'crypto-js'; +import { BaseVariableValueType } from 'nocodb-sdk'; +import type { BaseVariableInheritance, BaseVariableType } from 'nocodb-sdk'; +import type { NcContext } from '~/interface/config'; +import Noco from '~/Noco'; +import { extractProps } from '~/helpers/extractProps'; +import { NcError } from '~/helpers/ncError'; +import { + CacheDelDirection, + CacheGetType, + CacheScope, + MetaTable, +} from '~/utils/globals'; +import NocoCache from '~/cache/NocoCache'; +import { getCredentialEncryptSecret } from '~/utils/encryptDecrypt'; + +const KEY_REGEX = /^[A-Z][A-Z0-9_]*$/; +const MAX_VALUE_LENGTH = 65536; // 64KB + +export default class BaseVariable implements BaseVariableType { + id?: string; + base_id?: string; + fk_workspace_id?: string; + key?: string; + value?: string; + description?: string; + inheritance?: BaseVariableInheritance; + type?: BaseVariableValueType; + order?: number; + default_value?: string; + is_overridden?: boolean; + is_inherited?: boolean; + + constructor(data: Partial) { + Object.assign(this, data); + } + + get isSecret(): boolean { + return this.type === BaseVariableValueType.SECRET; + } + + private static isSecretType(data: Record): boolean { + return data.type === BaseVariableValueType.SECRET; + } + + private static encryptValue(value: string): string { + const secret = getCredentialEncryptSecret(); + if (!secret || !value) return value; + return CryptoJS.AES.encrypt(value, secret).toString(); + } + + private static decryptValue(value: string): string { + const secret = getCredentialEncryptSecret(); + if (!secret || !value) return value; + try { + return CryptoJS.AES.decrypt(value, secret).toString(CryptoJS.enc.Utf8); + } catch { + return value; + } + } + + private static prepareForDb( + data: Partial, + ): Record { + const obj = { ...data }; + if (BaseVariable.isSecretType(obj)) { + if (obj.value) { + obj.value = BaseVariable.encryptValue(obj.value); + } + if (obj.default_value) { + obj.default_value = BaseVariable.encryptValue(obj.default_value); + } + } + return obj; + } + + private static prepareForRead( + data: Record, + ): Record { + if (BaseVariable.isSecretType(data)) { + if (data.value) { + data.value = BaseVariable.decryptValue(data.value); + } + if (data.default_value) { + data.default_value = BaseVariable.decryptValue(data.default_value); + } + } + return data; + } + + public static async get( + context: NcContext, + variableId: string, + ncMeta = Noco.ncMeta, + ) { + let data = await NocoCache.get( + context, + `${CacheScope.BASE_VARIABLE}:${variableId}`, + CacheGetType.TYPE_OBJECT, + ); + + if (!data) { + data = await ncMeta.metaGet2( + context.workspace_id, + context.base_id, + MetaTable.BASE_VARIABLES, + variableId, + ); + + if (data) { + NocoCache.set( + context, + `${CacheScope.BASE_VARIABLE}:${variableId}`, + data, + ); + } + } + + if (data) { + data = BaseVariable.prepareForRead(data); + } + + return data && new BaseVariable(data); + } + + public static async list( + context: NcContext, + baseId: string, + ncMeta = Noco.ncMeta, + ) { + const cachedList = await NocoCache.getList( + context, + CacheScope.BASE_VARIABLE, + [baseId], + ); + let { list } = cachedList; + const { isNoneList } = cachedList; + + if (!isNoneList && !list.length) { + list = await ncMeta.metaList2( + context.workspace_id, + context.base_id, + MetaTable.BASE_VARIABLES, + { + condition: { base_id: baseId }, + orderBy: { order: 'asc' }, + }, + ); + + if (list) { + await NocoCache.setList( + context, + CacheScope.BASE_VARIABLE, + [baseId], + list, + ); + } + } + + return (list || []) + .sort((a, b) => (a?.order ?? Infinity) - (b?.order ?? Infinity)) + .map((item) => new BaseVariable(BaseVariable.prepareForRead(item))); + } + + /** + * Returns a flat key→value map for webhook template resolution. + * Secret values are decrypted. Empty values are skipped. + */ + public static async listAsMap( + context: NcContext, + baseId: string, + ncMeta = Noco.ncMeta, + ): Promise> { + const variables = await BaseVariable.list(context, baseId, ncMeta); + const map: Record = {}; + for (const v of variables) { + if (v.key && v.value) { + map[v.key] = v.value; + } + } + return map; + } + + public static async insert( + context: NcContext, + data: Partial, + ncMeta = Noco.ncMeta, + ) { + const insertObj = extractProps(data, [ + 'id', + 'base_id', + 'key', + 'value', + 'description', + 'inheritance', + 'type', + 'order', + 'default_value', + 'is_overridden', + 'is_inherited', + ]); + + // Validate key format + if (!insertObj.key || !KEY_REGEX.test(insertObj.key)) { + NcError.badRequest( + 'Variable key must be UPPER_SNAKE_CASE (e.g., MY_VARIABLE)', + ); + } + + if (insertObj.value && insertObj.value.length > MAX_VALUE_LENGTH) { + NcError.badRequest('Variable value exceeds 64KB limit'); + } + + if (insertObj.order === null || insertObj.order === undefined) { + insertObj.order = await ncMeta.metaGetNextOrder( + MetaTable.BASE_VARIABLES, + { base_id: insertObj.base_id }, + ); + } + + const { id } = await ncMeta.metaInsert2( + context.workspace_id, + context.base_id, + MetaTable.BASE_VARIABLES, + BaseVariable.prepareForDb(insertObj), + ); + + return this.get(context, id, ncMeta).then(async (res) => { + await NocoCache.appendToList( + context, + CacheScope.BASE_VARIABLE, + [data.base_id], + `${CacheScope.BASE_VARIABLE}:${id}`, + ); + return res; + }); + } + + public static async update( + context: NcContext, + variableId: string, + data: Partial, + ncMeta = Noco.ncMeta, + ) { + const updateObj = extractProps(data, [ + 'value', + 'description', + 'inheritance', + 'type', + 'order', + 'default_value', + 'is_overridden', + 'is_inherited', + ]); + + if (updateObj.value && updateObj.value.length > MAX_VALUE_LENGTH) { + NcError.badRequest('Variable value exceeds 64KB limit'); + } + + // Resolve effective type after the patch and the existing row so we can + // (a) encrypt new value/default_value when SECRET, and (b) re-encrypt or + // decrypt the persisted row when type flips between PLAIN and SECRET. + const current = await BaseVariable.get(context, variableId, ncMeta); + const previousType = current?.type; + const nextType = updateObj.type ?? previousType; + const willBeSecret = nextType === BaseVariableValueType.SECRET; + const typeFlipped = !!previousType && previousType !== nextType; + + // BaseVariable.get() returns decrypted values via prepareForRead, so when + // type flips and a field is not in the patch we pull it from `current` + // (already plaintext) and write it through under the new encryption state. + if (typeFlipped) { + if (updateObj.value === undefined && current?.value) { + updateObj.value = current.value; + } + if (updateObj.default_value === undefined && current?.default_value) { + updateObj.default_value = current.default_value; + } + } + + if (willBeSecret) { + if (updateObj.value) { + updateObj.value = BaseVariable.encryptValue(updateObj.value); + } + if (updateObj.default_value) { + updateObj.default_value = BaseVariable.encryptValue( + updateObj.default_value, + ); + } + } + + await ncMeta.metaUpdate( + context.workspace_id, + context.base_id, + MetaTable.BASE_VARIABLES, + updateObj, + variableId, + ); + + // Cache mirrors the DB row exactly: SECRET stores ciphertext, PLAIN stores + // plaintext. Reads route through prepareForRead, which decrypts based on + // the row's current `type`. + await NocoCache.update( + context, + `${CacheScope.BASE_VARIABLE}:${variableId}`, + updateObj, + ); + + return this.get(context, variableId, ncMeta); + } + + public static async delete( + context: NcContext, + variableId: string, + ncMeta = Noco.ncMeta, + ) { + const res = await ncMeta.metaDelete( + context.workspace_id, + context.base_id, + MetaTable.BASE_VARIABLES, + variableId, + ); + + await NocoCache.deepDel( + context, + `${CacheScope.BASE_VARIABLE}:${variableId}`, + CacheDelDirection.CHILD_TO_PARENT, + ); + + return res; + } + + public static async deleteByBaseId( + context: NcContext, + baseId: string, + ncMeta = Noco.ncMeta, + ) { + await ncMeta.metaDelete( + context.workspace_id, + context.base_id, + MetaTable.BASE_VARIABLES, + { base_id: baseId }, + ); + + await NocoCache.deepDel( + context, + `${CacheScope.BASE_VARIABLE}:${baseId}:list`, + CacheDelDirection.PARENT_TO_CHILD, + ); + } +} diff --git a/packages/nocodb/src/models/CalendarViewColumn.ts b/packages/nocodb/src/models/CalendarViewColumn.ts index 6c3e63bf95..00ad03100f 100644 --- a/packages/nocodb/src/models/CalendarViewColumn.ts +++ b/packages/nocodb/src/models/CalendarViewColumn.ts @@ -103,12 +103,14 @@ export default class CalendarViewColumn { { const view = await View.get(context, column.fk_view_id, false, ncMeta); - await View.clearSingleQueryCache( - context, - view.fk_model_id, - [view], - ncMeta, - ); + if (view) { + await View.clearSingleQueryCache( + context, + view.fk_model_id, + [view], + ncMeta, + ); + } } return this.get(context, id, ncMeta).then(async (viewColumn) => { diff --git a/packages/nocodb/src/models/Column.ts b/packages/nocodb/src/models/Column.ts index ac54a1658e..68dcc8d91c 100644 --- a/packages/nocodb/src/models/Column.ts +++ b/packages/nocodb/src/models/Column.ts @@ -1984,9 +1984,17 @@ export default class Column implements ColumnType { const columns = []; // add fk_model_id + const isReplay = !!context?.additionalContext?.is_replay; + const sandboxColumnIds: Record | undefined = isReplay + ? context.additionalContext?.sandboxColumnIds + : undefined; for (const column of param.columns) { // pre-populate column meta to use while inserting colOptions - const id = await ncMeta.genNanoid(MetaTable.COLUMNS); + const id = + (isReplay && column.id) || + sandboxColumnIds?.[column.column_name] || + sandboxColumnIds?.[column.title] || + (await ncMeta.genNanoid(MetaTable.COLUMNS)); const colWithId = { ...column, id, diff --git a/packages/nocodb/src/models/Filter.ts b/packages/nocodb/src/models/Filter.ts index a4d9f65e9a..7782490267 100644 --- a/packages/nocodb/src/models/Filter.ts +++ b/packages/nocodb/src/models/Filter.ts @@ -377,12 +377,16 @@ export default class Filter implements FilterType { if (filter.fk_view_id) { const view = await View.get(context, filter.fk_view_id, false, ncMeta); - await View.clearSingleQueryCache( - context, - view.fk_model_id, - [view], - ncMeta, - ); + // View may be missing if it was deleted concurrently or the filter + // is orphaned — skip cache invalidation rather than throwing. + if (view) { + await View.clearSingleQueryCache( + context, + view.fk_model_id, + [view], + ncMeta, + ); + } } } @@ -438,14 +442,16 @@ export default class Filter implements FilterType { { const filter = await this.get(context, id, ncMeta); // if not a view filter then no need to delete - if (filter.fk_view_id) { + if (filter?.fk_view_id) { const view = await View.get(context, filter.fk_view_id, false, ncMeta); - await View.clearSingleQueryCache( - context, - view.fk_model_id, - [{ id: filter.fk_view_id }], - ncMeta, - ); + if (view) { + await View.clearSingleQueryCache( + context, + view.fk_model_id, + [{ id: filter.fk_view_id }], + ncMeta, + ); + } } } @@ -482,12 +488,14 @@ export default class Filter implements FilterType { if (filter.fk_view_id) { const view = await View.get(context, filter.fk_view_id, false, ncMeta); - await View.clearSingleQueryCache( - context, - view.fk_model_id, - [{ id: filter.fk_view_id }], - ncMeta, - ); + if (view) { + await View.clearSingleQueryCache( + context, + view.fk_model_id, + [{ id: filter.fk_view_id }], + ncMeta, + ); + } } } } @@ -801,12 +809,14 @@ export default class Filter implements FilterType { // on update delete any optimised single query cache { const view = await View.get(context, viewId, false, ncMeta); - await View.clearSingleQueryCache( - context, - view.fk_model_id, - [view], - ncMeta, - ); + if (view) { + await View.clearSingleQueryCache( + context, + view.fk_model_id, + [view], + ncMeta, + ); + } } } diff --git a/packages/nocodb/src/models/GalleryViewColumn.ts b/packages/nocodb/src/models/GalleryViewColumn.ts index 076fd31609..bc9a23f8f4 100644 --- a/packages/nocodb/src/models/GalleryViewColumn.ts +++ b/packages/nocodb/src/models/GalleryViewColumn.ts @@ -91,12 +91,14 @@ export default class GalleryViewColumn { // on new view column, delete any optimised single query cache { const view = await View.get(context, column.fk_view_id, false, ncMeta); - await View.clearSingleQueryCache( - context, - view.fk_model_id, - [view], - ncMeta, - ); + if (view) { + await View.clearSingleQueryCache( + context, + view.fk_model_id, + [view], + ncMeta, + ); + } } return this.get(context, id, ncMeta).then(async (viewColumn) => { diff --git a/packages/nocodb/src/models/GridViewColumn.ts b/packages/nocodb/src/models/GridViewColumn.ts index af68ca898f..db643021df 100644 --- a/packages/nocodb/src/models/GridViewColumn.ts +++ b/packages/nocodb/src/models/GridViewColumn.ts @@ -155,12 +155,14 @@ export default class GridViewColumn implements GridColumnType { // on new view column, delete any optimised single query cache { const view = await View.get(context, column.fk_view_id, false, ncMeta); - await View.clearSingleQueryCache( - context, - view.fk_model_id, - [view], - ncMeta, - ); + if (view) { + await View.clearSingleQueryCache( + context, + view.fk_model_id, + [view], + ncMeta, + ); + } } return this.get(context, id, ncMeta).then(async (viewColumn) => { diff --git a/packages/nocodb/src/models/Hook.ts b/packages/nocodb/src/models/Hook.ts index 7f50736089..c6d12790ec 100644 --- a/packages/nocodb/src/models/Hook.ts +++ b/packages/nocodb/src/models/Hook.ts @@ -238,6 +238,11 @@ export default class Hook implements HookType { insertObj.notification = JSON.stringify(insertObj.notification); } + // Replay-only: preserve sandbox entity ID for idempotent merge + if (context?.additionalContext?.is_replay && hook.id) { + insertObj.id = hook.id; + } + const model = await Model.getByIdOrName( context, { id: hook.fk_model_id }, @@ -326,6 +331,11 @@ export default class Hook implements HookType { insertObj.notification = JSON.stringify(insertObj.notification); } + // Replay-only: preserve sandbox entity ID for idempotent merge + if (context?.additionalContext?.is_replay && hook.id) { + insertObj.id = hook.id; + } + const model = await Model.getByIdOrName( context, { id: hook.fk_model_id }, diff --git a/packages/nocodb/src/models/Model.ts b/packages/nocodb/src/models/Model.ts index 0553c38ebe..1f606ae2cf 100644 --- a/packages/nocodb/src/models/Model.ts +++ b/packages/nocodb/src/models/Model.ts @@ -216,6 +216,9 @@ export default class Model implements TableType { type?: ModelTypes; source_id?: string; user_id: string; + // Sandbox-replay only: preserve the master view ID when recreating the + // auto-created default view, so sorts/filters referencing it still resolve. + _sandboxDefaultViewId?: string; }, ncMeta = Noco.ncMeta, ) { @@ -290,6 +293,9 @@ export default class Model implements TableType { context, { view: { + ...(model._sandboxDefaultViewId + ? { id: model._sandboxDefaultViewId } + : {}), fk_model_id: id, title: model.title || model.table_name, type: ViewTypes.GRID, diff --git a/packages/nocodb/src/models/View.ts b/packages/nocodb/src/models/View.ts index 2adce23886..ed3f23310c 100644 --- a/packages/nocodb/src/models/View.ts +++ b/packages/nocodb/src/models/View.ts @@ -1263,6 +1263,7 @@ export default class View implements ViewType { ncMeta = Noco.ncMeta, ) { const view = await this.get(context, viewId, false, ncMeta); + if (!view) NcError.viewNotFound(viewId); let table; let cacheScope; switch (view.type) { @@ -1426,6 +1427,7 @@ export default class View implements ViewType { | any > { const view = await this.get(context, viewId, false, ncMeta); + if (!view) NcError.viewNotFound(viewId); const table = this.extractViewColumnsTableName(view); const existingCol = await ncMeta.metaGet2( diff --git a/packages/nocodb/src/models/index.ts b/packages/nocodb/src/models/index.ts index 6db413116d..aa42256655 100644 --- a/packages/nocodb/src/models/index.ts +++ b/packages/nocodb/src/models/index.ts @@ -48,6 +48,7 @@ export { default as Notification } from './Notification'; export { default as PresignedUrl } from './PresignedUrl'; export { default as UserRefreshToken } from './UserRefreshToken'; export { default as Extension } from './Extension'; +export { default as BaseVariable } from './BaseVariable'; export { default as Comment } from './Comment'; export { default as Job } from './Job'; export { default as Integration } from './Integration'; diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts index 356589263b..680951ef02 100644 --- a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts @@ -33,6 +33,7 @@ import { } from '~/utils'; import { type Base, Model, Source } from '~/models'; import { sanitizeColumnName } from '~/helpers'; +import { OperationSource } from '~/helpers/columnHelpers'; import { AttachmentsService } from '~/services/attachments.service'; import { ColumnsService } from '~/services/columns.service'; import { BulkDataAliasService } from '~/services/bulk-data-alias.service'; @@ -761,6 +762,7 @@ export class AtImportProcessor { table: tables[idx], user: syncDB.user, req, + operationSource: OperationSource.AT_IMPORT, }); recordPerfStats(_perfStart, 'dbTable.create'); @@ -914,6 +916,7 @@ export class AtImportProcessor { }, req, user: syncDB.user, + operationSource: OperationSource.AT_IMPORT, }); recordPerfStats(_perfStart, 'dbTableColumn.create'); @@ -1140,6 +1143,7 @@ export class AtImportProcessor { }, req, user: syncDB.user, + operationSource: OperationSource.AT_IMPORT, }); recordPerfStats(_perfStart, 'dbTableColumn.create'); @@ -1226,6 +1230,7 @@ export class AtImportProcessor { clientIp: '', } as any, user: syncDB.user, + operationSource: OperationSource.AT_IMPORT, }); recordPerfStats(_perfStart, 'dbTableColumn.create'); @@ -1390,6 +1395,7 @@ export class AtImportProcessor { }, req, user: syncDB.user, + operationSource: OperationSource.AT_IMPORT, }); recordPerfStats(_perfStart, 'dbTableColumn.create'); @@ -1457,6 +1463,7 @@ export class AtImportProcessor { }, req, user: syncDB.user, + operationSource: OperationSource.AT_IMPORT, }); recordPerfStats(_perfStart, 'dbTableColumn.create'); diff --git a/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts b/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts index 615c338d13..0c42e16ad2 100644 --- a/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts @@ -380,7 +380,11 @@ export class DuplicateProcessor { dataSource: source, req, context, + // Spread original options first so EE-only flags (excludeDocuments, + // excludePersonalViews, excludePermissions, excludeRls, etc.) reach the + // processor. Normalized CE flags override after. options: { + ...(options ?? {}), excludeData, excludeHooks, excludeViews, diff --git a/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.service.ts b/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.service.ts index 1c8174fd4e..337523d6ac 100644 --- a/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.service.ts +++ b/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.service.ts @@ -54,6 +54,12 @@ export class DuplicateService { NcError.get(context).baseNotFound(baseId); } + if (base.is_sandbox) { + NcError.get(context).badRequest( + 'Sandbox bases cannot be duplicated. Duplicate the master base instead.', + ); + } + const source = sourceId ? await Source.get(context, sourceId) : (await base.getSources())[0]; diff --git a/packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts b/packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts index c1aad35290..6594fbef73 100644 --- a/packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts +++ b/packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts @@ -1300,24 +1300,50 @@ export class ImportService { // create filters const filters = colOptions.filter?.children; + // Track filters that fail to create so descendants can be + // skipped too — otherwise a child whose fk_parent_id maps + // to undefined would be persisted at root level, leaving + // orphan filters that are hard to clean up later. + const failedFilterIds = new Set(); + for (const fl of filters) { - const fg = await this.filtersService.linkFilterCreate( - targetContext, - { - columnId: getIdOrExternalId(col.id), - filter: withoutId({ - ...fl, - fk_value_col_id: getIdOrExternalId(fl.fk_value_col_id), - fk_link_col_id: getIdOrExternalId(fl.fk_link_col_id), - fk_column_id: getIdOrExternalId(fl.fk_column_id), - fk_parent_id: getIdOrExternalId(fl.fk_parent_id), - }), - user: param.user, - req: param.req, - }, - ); - if (fg) { - idMap.set(fl.id, fg.id); + if (fl.fk_parent_id && failedFilterIds.has(fl.fk_parent_id)) { + failedFilterIds.add(fl.id); + this.logger.warn( + `Skipping link filter ${fl.id} on column ${col.id} — parent ${fl.fk_parent_id} failed earlier in this import`, + ); + continue; + } + + // A single broken link filter (e.g. column lookup resolves + // to undefined when the source export references something + // outside the duplicated scope) shouldn't abort the entire + // base duplication — log and skip. + try { + const fg = await this.filtersService.linkFilterCreate( + targetContext, + { + columnId: getIdOrExternalId(col.id), + filter: withoutId({ + ...fl, + fk_value_col_id: getIdOrExternalId(fl.fk_value_col_id), + fk_link_col_id: getIdOrExternalId(fl.fk_link_col_id), + fk_column_id: getIdOrExternalId(fl.fk_column_id), + fk_parent_id: getIdOrExternalId(fl.fk_parent_id), + }), + user: param.user, + req: param.req, + }, + ); + if (fg) { + idMap.set(fl.id, fg.id); + } + } catch (e) { + failedFilterIds.add(fl.id); + this.logger.warn( + `Skipping link filter ${fl.id} on column ${col.id} during import: ${e?.message}`, + e?.stack, + ); } } }); @@ -1406,6 +1432,12 @@ export class ImportService { !getIdOrExternalId(colOptions.fk_lookup_column_id) ) { if (colOptions.error) { + // NOTE: bypasses `columnsService.columnAdd` (and its `@TraceCommand`) + // intentionally — `columnAdd` validates references and would throw + // for an unmapped Lookup. Error-state columns are a degraded + // outcome of cross-base imports where the lookup target couldn't be + // resolved; sandbox-replay won't reproduce them, but the original + // column is already broken so this is acceptable. const tableId = getIdOrExternalId(getParentIdentifier(col.id)); const insertedColumn = await Column.insert(targetContext, { ...withoutId(flatCol), @@ -1460,6 +1492,7 @@ export class ImportService { !getIdOrExternalId(colOptions.fk_rollup_column_id) ) { if (colOptions.error) { + // See Lookup branch above — same rationale applies here. const tableId = getIdOrExternalId(getParentIdentifier(col.id)); const insertedColumn = await Column.insert(targetContext, { ...withoutId(flatCol), diff --git a/packages/nocodb/src/modules/jobs/jobs/export-import/migrate.service.ts b/packages/nocodb/src/modules/jobs/jobs/export-import/migrate.service.ts index c01da4da04..fbd034255f 100644 --- a/packages/nocodb/src/modules/jobs/jobs/export-import/migrate.service.ts +++ b/packages/nocodb/src/modules/jobs/jobs/export-import/migrate.service.ts @@ -2,9 +2,11 @@ import { Readable } from 'stream'; import debug from 'debug'; import { Injectable } from '@nestjs/common'; import axios from 'axios'; +import { useAgent } from 'request-filtering-agent'; import type { NcContext, NcRequest } from '~/interface/config'; import type { Base, Source } from '~/models'; import { NcError } from '~/helpers/ncError'; +import { assertNotSandbox } from '~/helpers/sandboxGuards'; import { ExportService } from '~/modules/jobs/jobs/export-import/export.service'; @Injectable() @@ -27,6 +29,11 @@ export class MigrateService { instanceUrl: string; req: NcRequest; }) { + await assertNotSandbox( + context, + 'Migrating a base is not allowed from a sandbox. Run the migration on the production base.', + ); + if (!base) { NcError.get(context).baseNotFound('Base not found!'); } @@ -65,12 +72,16 @@ export class MigrateService { stream.push(JSON.stringify(data)); }; + const targetUrl = `${instanceUrl}/api/v2/meta/duplicate/remote/${secret}`; + const axiosPromise = axios({ method: 'post', - url: `${instanceUrl}/api/v2/meta/duplicate/remote/${secret}`, + url: targetUrl, headers: { 'Content-Type': 'application/octet-stream', }, + httpAgent: useAgent(targetUrl), + httpsAgent: useAgent(targetUrl), data: stream, maxBodyLength: Infinity, }).catch((e) => { diff --git a/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_010_soft_delete_column.ts b/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_010_soft_delete_column.ts index bd3f42a85e..d1a924d23d 100644 --- a/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_010_soft_delete_column.ts +++ b/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_010_soft_delete_column.ts @@ -46,14 +46,6 @@ const PARALLEL_LIMIT = // TODO: Drop nc_temp_processed_soft_delete after migration is confirmed complete across all deployments const TEMP_TABLE = 'nc_temp_processed_soft_delete'; -// Comma-separated workspace IDs to target during dry-run. -// When set, the migration only processes models in these workspaces. -const TARGET_WORKSPACE_IDS = process.env.NC_TRASH_MIGRATION_WORKSPACE_IDS - ? process.env.NC_TRASH_MIGRATION_WORKSPACE_IDS.split(',') - .map((s) => s.trim()) - .filter(Boolean) - : null; - const propsByClientType = {}; const memoizedGetColumnPropsFromUIDT = async ( @@ -138,30 +130,16 @@ export class SoftDeleteColumnMigration { builder.where(`${MetaTable.SOURCES}.is_meta`, true); builder.orWhere({ is_local: true }); }) - .whereNotIn( - `${MetaTable.MODELS}.id`, - ncMeta.knexConnection(TEMP_TABLE).select('fk_model_id'), + .whereNotExists((qb) => + qb + .select(1) + .from(TEMP_TABLE) + .whereRaw(`${TEMP_TABLE}.fk_model_id = ${MetaTable.MODELS}.id`), ) - .modify((qb) => { - if (TARGET_WORKSPACE_IDS) { - qb.whereIn( - `${MetaTable.MODELS}.fk_workspace_id`, - TARGET_WORKSPACE_IDS, - ); - } - }) .count('*', { as: 'count' }) .first() )?.count; - if (TARGET_WORKSPACE_IDS) { - this.log( - `Dry-run mode: targeting ${ - TARGET_WORKSPACE_IDS.length - } workspace(s): ${TARGET_WORKSPACE_IDS.join(', ')}`, - ); - } - this.log( `Found ${numberOfModelsToBeProcessed} models to process for soft delete + meta column`, ); diff --git a/packages/nocodb/src/run/docker.ts b/packages/nocodb/src/run/docker.ts index 45e45ff25d..35e6dcc006 100644 --- a/packages/nocodb/src/run/docker.ts +++ b/packages/nocodb/src/run/docker.ts @@ -14,7 +14,8 @@ server.disable('etag'); server.disable('x-powered-by'); server.use( cors({ - exposedHeaders: 'xc-db-response', + exposedHeaders: + 'xc-db-response, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, X-RateLimit-Policy, Retry-After', }), ); diff --git a/packages/nocodb/src/run/dockerRunMysql.ts b/packages/nocodb/src/run/dockerRunMysql.ts index 62f397640b..f97b69caeb 100644 --- a/packages/nocodb/src/run/dockerRunMysql.ts +++ b/packages/nocodb/src/run/dockerRunMysql.ts @@ -14,7 +14,8 @@ server.disable('etag'); server.disable('x-powered-by'); server.use( cors({ - exposedHeaders: 'xc-db-response', + exposedHeaders: + 'xc-db-response, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, X-RateLimit-Policy, Retry-After', }), ); diff --git a/packages/nocodb/src/run/dockerRunPG.ts b/packages/nocodb/src/run/dockerRunPG.ts index 25d9148976..994b1b814f 100644 --- a/packages/nocodb/src/run/dockerRunPG.ts +++ b/packages/nocodb/src/run/dockerRunPG.ts @@ -14,7 +14,8 @@ server.disable('etag'); server.disable('x-powered-by'); server.use( cors({ - exposedHeaders: 'xc-db-response', + exposedHeaders: + 'xc-db-response, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, X-RateLimit-Policy, Retry-After', }), ); diff --git a/packages/nocodb/src/run/dockerRunPG_CyQuick.ts b/packages/nocodb/src/run/dockerRunPG_CyQuick.ts index d2cae52c98..f850e121dd 100644 --- a/packages/nocodb/src/run/dockerRunPG_CyQuick.ts +++ b/packages/nocodb/src/run/dockerRunPG_CyQuick.ts @@ -14,7 +14,8 @@ server.disable('etag'); server.disable('x-powered-by'); server.use( cors({ - exposedHeaders: 'xc-db-response', + exposedHeaders: + 'xc-db-response, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, X-RateLimit-Policy, Retry-After', }), ); diff --git a/packages/nocodb/src/run/testDocker.ts b/packages/nocodb/src/run/testDocker.ts index 4f45220de5..963cfec014 100644 --- a/packages/nocodb/src/run/testDocker.ts +++ b/packages/nocodb/src/run/testDocker.ts @@ -18,7 +18,8 @@ server.disable('etag'); server.disable('x-powered-by'); server.use( cors({ - exposedHeaders: 'xc-db-response', + exposedHeaders: + 'xc-db-response, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, X-RateLimit-Policy, Retry-After', }), ); diff --git a/packages/nocodb/src/schema/swagger-v2.json b/packages/nocodb/src/schema/swagger-v2.json index 5f1fcf09a1..1db1df1f2b 100644 --- a/packages/nocodb/src/schema/swagger-v2.json +++ b/packages/nocodb/src/schema/swagger-v2.json @@ -18845,9 +18845,9 @@ "$ref": "#/components/schemas/Bool", "description": "Indicates if the base is a sandbox" }, - "is_sandbox_master": { + "is_sandbox_production": { "$ref": "#/components/schemas/Bool", - "description": "Indicates if the base is a sandbox master" + "description": "Indicates if the base is a sandbox production base (has at least one sandbox derived from it)" } }, "x-stoplight": { diff --git a/packages/nocodb/src/schema/swagger.json b/packages/nocodb/src/schema/swagger.json index f651e76a54..d3a5f2e57b 100644 --- a/packages/nocodb/src/schema/swagger.json +++ b/packages/nocodb/src/schema/swagger.json @@ -21067,6 +21067,33 @@ "in": "query", "description": "Team ID", "required": false + }, + { + "schema": { + "type": "string" + }, + "name": "modelId", + "in": "query", + "description": "Model ID", + "required": false + }, + { + "schema": { + "type": "string" + }, + "name": "rowId", + "in": "query", + "description": "Row ID", + "required": false + }, + { + "schema": { + "type": "string" + }, + "name": "urlOrPath", + "in": "query", + "description": "URL or Path of the attachment", + "required": false } ] }, @@ -27170,9 +27197,9 @@ "$ref": "#/components/schemas/Bool", "description": "Indicates if the base is a sandbox" }, - "is_sandbox_master": { + "is_sandbox_production": { "$ref": "#/components/schemas/Bool", - "description": "Indicates if the base is a sandbox master" + "description": "Indicates if the base is a sandbox production base (has at least one sandbox derived from it)" } }, "x-stoplight": { diff --git a/packages/nocodb/src/services/attachments.service.ts b/packages/nocodb/src/services/attachments.service.ts index 5789dab24b..27271dcdde 100644 --- a/packages/nocodb/src/services/attachments.service.ts +++ b/packages/nocodb/src/services/attachments.service.ts @@ -11,12 +11,13 @@ import axios from 'axios'; import hash from 'object-hash'; import moment from 'moment'; import { useAgent } from 'request-filtering-agent'; -import type { AttachmentReqType, FileType } from 'nocodb-sdk'; +import type { AttachmentReqType, FileType, NcContext } from 'nocodb-sdk'; import type { NcRequest } from '~/interface/config'; import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; +import { DataTableService } from '~/services/data-table.service'; import NcPluginMgrv2 from '~/helpers/NcPluginMgrv2'; import { mimeIcons } from '~/utils/mimeTypes'; -import { FileReference, PresignedUrl } from '~/models'; +import { Column, FileReference, PresignedUrl } from '~/models'; import { utf8ify } from '~/helpers/stringHelpers'; import { NcBaseError, NcError } from '~/helpers/catchError'; import { IJobsService } from '~/modules/jobs/jobs-service.interface'; @@ -52,6 +53,8 @@ export class AttachmentsService { private readonly appHooksService: AppHooksService, @Inject(forwardRef(() => 'JobsService')) private readonly jobsService: IJobsService, + @Inject(forwardRef(() => DataTableService)) + private readonly dataTableService: DataTableService, ) {} async upload(param: { @@ -494,6 +497,43 @@ export class AttachmentsService { return { path: filePath, type }; } + async downloadAttachment( + context: NcContext, + param: { + modelId: string; + columnId: string; + rowId: string; + urlOrPath: string; + }, + ) { + const column = await Column.get(context, { + colId: param.columnId, + }); + + if (!column) { + NcError.fieldNotFound(param.columnId); + } + + const record = await this.dataTableService.dataRead(context, { + baseId: context.base_id, + modelId: param.modelId, + rowId: param.rowId, + query: { + fields: column.title, + }, + }); + + if (!record) { + NcError.recordNotFound(param.rowId); + } + + return this.getAttachmentFromRecord({ + record, + column, + urlOrPath: param.urlOrPath, + }); + } + async getAttachmentFromRecord(param: { record: any; column: { title: string }; diff --git a/packages/nocodb/src/services/calendars.service.ts b/packages/nocodb/src/services/calendars.service.ts index 0716d7848b..d4f55cd3ee 100644 --- a/packages/nocodb/src/services/calendars.service.ts +++ b/packages/nocodb/src/services/calendars.service.ts @@ -5,8 +5,9 @@ import type { UserType, ViewCreateReqType, } from 'nocodb-sdk'; -import type { NcContext, NcRequest } from '~/interface/config'; -import type { MetaService } from '~/meta/meta.service'; +import type { NcRequest } from '~/interface/config'; +import { NcContext } from '~/interface/config'; +import { MetaService } from '~/meta/meta.service'; import { type ViewWebhookManager, ViewWebhookManagerBuilder, @@ -14,7 +15,10 @@ import { import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; import { validatePayload } from '~/helpers'; import { assertPersonalViewAllowed } from '~/helpers/checkPersonalViewFeature'; +import { assertNotSandbox } from '~/helpers/sandboxGuards'; import { NcError } from '~/helpers/catchError'; +import { TraceCommand } from '~/decorators/trace-command.decorator'; +import { OperationName } from '~/command-registry/op-names'; import { CalendarView, Model, User, View } from '~/models'; import NocoCache from '~/cache/NocoCache'; import { CacheScope } from '~/utils/globals'; @@ -22,12 +26,13 @@ import NocoSocket from '~/socket/NocoSocket'; @Injectable() export class CalendarsService { - constructor(private readonly appHooksService: AppHooksService) {} + constructor(protected readonly appHooksService: AppHooksService) {} async calendarViewGet(context: NcContext, param: { calendarViewId: string }) { return await CalendarView.get(context, param.calendarViewId); } + @TraceCommand(OperationName.calendarViewCreate) async calendarViewCreate( context: NcContext, param: { @@ -40,6 +45,13 @@ export class CalendarsService { }, ncMeta?: MetaService, ) { + if (param?.ownedBy) { + await assertNotSandbox( + context, + 'Personal views cannot be created in a sandbox. Create them on the production base.', + ); + } + validatePayload( 'swagger.json#/components/schemas/ViewCreateReq', param.calendar, @@ -146,6 +158,7 @@ export class CalendarsService { return view; } + @TraceCommand(OperationName.calendarViewUpdate) async calendarViewUpdate( context: NcContext, param: { diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts index 1557e3c041..d104322f20 100644 --- a/packages/nocodb/src/services/columns.service.ts +++ b/packages/nocodb/src/services/columns.service.ts @@ -50,12 +50,14 @@ import type { import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2'; import type CustomKnex from '~/db/CustomKnex'; import type SqlMgrv2 from '~/db/sql-mgr/v2/SqlMgrv2'; -import type { NcContext, NcRequest } from '~/interface/config'; +import type { NcRequest } from '~/interface/config'; import type { Base, LinkToAnotherRecordColumn } from '~/models'; import type { IColumnsService, + LtarSideEffectIds, ReusableParams, } from '~/services/columns.service.type'; +import { NcContext } from '~/interface/config'; import { type ColumnWebhookManager, ColumnWebhookManagerBuilder, @@ -68,6 +70,7 @@ import { createHmAndBtColumn, createOOColumn, deleteColumnSystemPropsFromRequest, + type OperationSource, generateFkName, getMMColumnNames, getRevType, @@ -77,6 +80,8 @@ import { validateRequiredField, validateRollupPayload, } from '~/helpers'; +import { TraceCommand } from '~/decorators/trace-command.decorator'; +import { OperationName } from '~/command-registry/op-names'; import { NcError } from '~/helpers/catchError'; import { extractProps } from '~/helpers/extractProps'; import { pgQuoteLiteral } from '~/helpers/sqlSanitize'; @@ -125,7 +130,10 @@ import { getRelatedModelMap } from '~/utils/getRelatedModelMap'; import { validateColumnInternalMeta } from '~/types/column-internal-meta'; import { backfillAutoNumber } from '~/helpers/autonumberHelpers'; -export type { ReusableParams } from '~/services/columns.service.type'; +export type { + LtarSideEffectIds, + ReusableParams, +} from '~/services/columns.service.type'; const deepClone = rfdc(); @@ -1499,7 +1507,7 @@ export class ColumnsService implements IColumnsService { const existingOptions = colBody.colOptions.options.map( (el) => el.title, ); - const options = data.reduce((acc, el) => { + const options = data.reduce<{ title: string }[]>((acc, el) => { if (el[column.column_name]) { const values = String(el[column.column_name]).split(','); if (values.length > 1) { @@ -2802,6 +2810,7 @@ export class ColumnsService implements IColumnsService { return Column.get(context, { colId: param.columnId }); } + @TraceCommand(OperationName.columnSetAsPrimary) async columnSetAsPrimary( context: NcContext, param: { columnId: string; req: NcRequest }, @@ -2890,6 +2899,13 @@ export class ColumnsService implements IColumnsService { suppressFormulaError?: boolean; apiVersion?: T; columnWebhookManager?: ColumnWebhookManager; + operationSource?: OperationSource; + // Sandbox-replay LTAR side-effect IDs — set by the columnAdd handler + // on replay, threaded down into `createLTARColumn`. + _ltarReplayIds?: LtarSideEffectIds; + // Capture slot populated by `createLTARColumn` during recording; + // surfaced via `extraCommandMeta` on `ColumnAddContract`. + _ltarCapture?: LtarSideEffectIds; }, ncMeta = Noco.ncMeta, ): Promise { @@ -2941,7 +2957,9 @@ export class ColumnsService implements IColumnsService { `Cannot manually create system columns`, ); } else { - deleteColumnSystemPropsFromRequest(param.column); + deleteColumnSystemPropsFromRequest(param.column, { + operationSource: param.operationSource, + }); } const base = await reuseOrSave('base', reuse, async () => @@ -3110,14 +3128,27 @@ export class ColumnsService implements IColumnsService { break; case UITypes.Links: - case UITypes.LinkToAnotherRecord: + case UITypes.LinkToAnotherRecord: { + // Sandbox-replay capture slot — populated by `createLTARColumn` with + // side-effect IDs (assoc model, FK cols, back-link cols, reverse LTAR) + // and read by `extraCommandMeta` on `ColumnAddContract` to thread + // them into the changelog. Filtered from replay params via + // `NON_SERIALIZABLE_KEYS`. Object reference is shared so inner + // mutations are visible to the calling decorator. + const ltarCapture: LtarSideEffectIds = {}; + param._ltarCapture = ltarCapture; savedColumn = await this.createLTARColumn(context, { - ...param, + tableId: param.tableId, + column: param.column, + user: param.user, + req: param.req, + reuse: param.reuse, + columnWebhookManager, + _ltarReplayIds: param._ltarReplayIds, + _ltarCapture: ltarCapture, source, base, - reuse, colExtra, - columnWebhookManager, }); this.appHooksService.emit(AppEvents.RELATION_CREATE, { @@ -3131,6 +3162,7 @@ export class ColumnsService implements IColumnsService { context, }); break; + } case UITypes.QrCode: validateParams(['fk_qr_value_column_id'], param.column, context); @@ -5236,9 +5268,19 @@ export class ColumnsService implements IColumnsService { user: UserType; req: NcRequest; columnWebhookManager?: ColumnWebhookManager; + // Sandbox-replay only — set by the columnAdd handler when replaying a + // recorded LTAR create. Each insert site below honors the matching id + // so dependent ops (Lookup/Rollup/linkFilter) keep stable references. + _ltarReplayIds?: LtarSideEffectIds; + // Sandbox-replay only — capture slot populated during recording so + // `extraCommandMeta` on `ColumnAddContract` can thread the side-effect + // IDs into the changelog. Object ref shared with `columnAdd`'s param. + _ltarCapture?: LtarSideEffectIds; }, ) { let savedColumn: Column; + const replayIds = param._ltarReplayIds; + const capture = param._ltarCapture; if ((param.column as any).is_custom_link) { NcError.get(context).badRequest( @@ -5256,6 +5298,9 @@ export class ColumnsService implements IColumnsService { readonly?: boolean; meta?: Record; ref_base_id?: string; + // Sandbox-replay — pre-injected by `idField: 'column'` so each + // back-link/oo/mm `Column.insert` can honor the recorded id. + id?: string; }; if (!ltarReq.parentId) { @@ -5407,12 +5452,14 @@ export class ColumnsService implements IColumnsService { await refSqlMgr.sqlOpPlus(refSource, 'tableUpdate', tableUpdateBody); const { id } = await Column.insert(refContext, { + ...(replayIds?.fkColumnId ? { id: replayIds.fkColumnId } : {}), ...newColumn, uidt: UITypes.ForeignKey, fk_model_id: refTable.id, }); refColumn = await Column.get(refContext, { colId: id }); + if (capture) capture.fkColumnId = refColumn.id; // ignore relation creation if virtual if (!ltarReq.virtual) { @@ -5446,6 +5493,7 @@ export class ColumnsService implements IColumnsService { } } + const hmBtOut: { childRelColId?: string; savedColumnId?: string } = {}; savedColumn = await createHmAndBtColumn( context, param.req, @@ -5467,7 +5515,13 @@ export class ColumnsService implements IColumnsService { undefined, undefined, param.columnWebhookManager, + { + childRelColId: replayIds?.reverseColumnId, + savedColumnId: ltarReq.id, + }, + hmBtOut, ); + if (capture) capture.reverseColumnId = hmBtOut.childRelColId; } else if (!isMMLike && ltarReq.type === 'oo') { // populate fk column name const fkColName = getUniqueColumnName( @@ -5515,12 +5569,14 @@ export class ColumnsService implements IColumnsService { await sqlMgr.sqlOpPlus(refSource, 'tableUpdate', tableUpdateBody); const { id } = await Column.insert(refContext, { + ...(replayIds?.fkColumnId ? { id: replayIds.fkColumnId } : {}), ...newColumn, uidt: UITypes.ForeignKey, fk_model_id: refTable.id, }); refColumn = await Column.get(refContext, { colId: id }); + if (capture) capture.fkColumnId = refColumn.id; // ignore relation creation if virtual if (!ltarReq.virtual) { @@ -5553,6 +5609,7 @@ export class ColumnsService implements IColumnsService { }); } } + const ooOut: { childRelColId?: string; savedColumnId?: string } = {}; savedColumn = await createOOColumn( context, param.req, @@ -5573,7 +5630,13 @@ export class ColumnsService implements IColumnsService { undefined, undefined, param.columnWebhookManager, + { + childRelColId: replayIds?.reverseColumnId, + savedColumnId: ltarReq.id, + }, + ooOut, ); + if (capture) capture.reverseColumnId = ooOut.childRelColId; } else if (isMMLike || ltarReq.type === 'mm') { const aTn = await getJunctionTableName(param, table, refTable); const aTnAlias = aTn; @@ -5590,6 +5653,12 @@ export class ColumnsService implements IColumnsService { associateTableCols.push( { + // Pre-set ID on replay so `Column.bulkInsert` honors it (the + // `column_name` map lookup wouldn't match — assoc-table column + // names embed the source prefix which differs across bases). + ...(replayIds?.assocChildColId + ? { id: replayIds.assocChildColId } + : {}), cn: refColumnName, column_name: refColumnName, title: refColumnName, @@ -5605,6 +5674,9 @@ export class ColumnsService implements IColumnsService { uidt: UITypes.ForeignKey, }, { + ...(replayIds?.assocParentColId + ? { id: replayIds.assocParentColId } + : {}), cn: columnName, column_name: columnName, title: columnName, @@ -5632,6 +5704,10 @@ export class ColumnsService implements IColumnsService { param.base.id, param.source.id, { + ...(replayIds?.assocModelId ? { id: replayIds.assocModelId } : {}), + ...(replayIds?.assocDefaultViewId + ? { _sandboxDefaultViewId: replayIds.assocDefaultViewId } + : {}), table_name: aTn, title: aTnAlias, // todo: sanitize @@ -5679,6 +5755,7 @@ export class ColumnsService implements IColumnsService { ); // todo: skip hm and bt if new type + const hmBtRefOut: { childRelColId?: string; savedColumnId?: string } = {}; await createHmAndBtColumn( context, param.req, @@ -5698,7 +5775,11 @@ export class ColumnsService implements IColumnsService { undefined, // not need to pass columnWebhookManager here undefined, + replayIds?.hmBtCallRef, + hmBtRefOut, ); + const hmBtTableOut: { childRelColId?: string; savedColumnId?: string } = + {}; await createHmAndBtColumn( context, param.req, @@ -5718,6 +5799,8 @@ export class ColumnsService implements IColumnsService { undefined, // not need to pass columnWebhookManager here undefined, + replayIds?.hmBtCallTable, + hmBtTableOut, ); let refCrossBaseLinkProps: { @@ -5774,6 +5857,7 @@ export class ColumnsService implements IColumnsService { : pluralize(refTable.title); savedColumn = await Column.insert(context, { + ...(ltarReq.id ? { id: ltarReq.id } : {}), title: getUniqueColumnAliasName( await table.getColumns(context), param.column.title ?? defaultTitle, @@ -5823,6 +5907,9 @@ export class ColumnsService implements IColumnsService { : pluralize(table.title); const parentRelCol = await Column.insert(refContext, { + ...(replayIds?.reverseColumnId + ? { id: replayIds.reverseColumnId } + : {}), title: getUniqueColumnAliasName( [ ...(await refTable.getColumns(refContext)), @@ -5867,6 +5954,17 @@ export class ColumnsService implements IColumnsService { ...refCrossBaseLinkProps, }); + if (capture) { + capture.assocModelId = assocModel.id; + const assocViews = await assocModel.getViews(context); + capture.assocDefaultViewId = assocViews?.[0]?.id; + capture.reverseColumnId = parentRelCol.id; + capture.assocChildColId = childCol.id; + capture.assocParentColId = parentCol.id; + capture.hmBtCallRef = hmBtRefOut; + capture.hmBtCallTable = hmBtTableOut; + } + this.appHooksService.emit(AppEvents.COLUMN_CREATE, { table: refTable, column: parentRelCol, diff --git a/packages/nocodb/src/services/columns.service.type.ts b/packages/nocodb/src/services/columns.service.type.ts index 9cae662733..d73924cf2d 100644 --- a/packages/nocodb/src/services/columns.service.type.ts +++ b/packages/nocodb/src/services/columns.service.type.ts @@ -22,6 +22,43 @@ export interface ReusableParams { baseModel?: BaseModelSqlv2; } +/** + * IDs created by `createHmAndBtColumn` / `createOOColumn`. Used for both + * `idHints` (replay reads) and `out` (recording writes) in `columnHelpers`. + */ +export interface LtarHmBtIds { + childRelColId?: string; + savedColumnId?: string; +} + +/** + * Side-effect IDs created by `createLTARColumn`. Captured into + * `param._ltarCapture` during recording (surfaced via `extraCommandMeta` + * on `ColumnAddContract`) and read back as `param._ltarReplayIds` on + * replay so each insert site honors the recorded id. The savedColumn + * itself uses `idField: 'column'` and isn't in this shape. + */ +export interface LtarSideEffectIds { + /** FK column on `refTable` for hm/bt + oo paths. */ + fkColumnId?: string; + /** Hidden mm junction model (mm path only). */ + assocModelId?: string; + /** + * Auto-created default grid view on the assoc model. Threaded into + * `Model.insert` via `_sandboxDefaultViewId` so the production-side + * view keeps the sandbox-side id. + */ + assocDefaultViewId?: string; + /** Reverse LTAR column on `refTable`. */ + reverseColumnId?: string; + /** FK columns on the assoc table (mm path) — pre-set on `associateTableCols` so `Column.bulkInsert` honors them. */ + assocChildColId?: string; + assocParentColId?: string; + /** Two `createHmAndBtColumn` calls for mm — assoc→ref and assoc→table. */ + hmBtCallRef?: LtarHmBtIds; + hmBtCallTable?: LtarHmBtIds; +} + export interface IColumnsService { columnAdd( context: NcContext, diff --git a/packages/nocodb/src/services/data-table.service.ts b/packages/nocodb/src/services/data-table.service.ts index 4d973767e3..25cf04990d 100644 --- a/packages/nocodb/src/services/data-table.service.ts +++ b/packages/nocodb/src/services/data-table.service.ts @@ -489,7 +489,7 @@ export class DataTableService { ) { listArgs.nestedLimit = param.query.limit; } - let data: any[]; + let data: Record[] | Record; let count: number; // V2 single-target relations (MO/OO) — junction table with LIMIT 1 @@ -565,7 +565,7 @@ export class DataTableService { if (colOptions.type === RelationTypes.BELONGS_TO) return data; - return new PagedResponseImpl(data, { + return new PagedResponseImpl(data as Record[], { count, ...param.query, }); diff --git a/packages/nocodb/src/services/extensions.service.ts b/packages/nocodb/src/services/extensions.service.ts index 1efa933d3b..d60ed2e9b5 100644 --- a/packages/nocodb/src/services/extensions.service.ts +++ b/packages/nocodb/src/services/extensions.service.ts @@ -10,7 +10,7 @@ import { NcError } from '~/helpers/ncError'; @Injectable() export class ExtensionsService { - constructor(private readonly appHooksService: AppHooksService) {} + constructor(protected readonly appHooksService: AppHooksService) {} async extensionList(context: NcContext, param: { baseId: string }) { return await Extension.list(context, param.baseId); diff --git a/packages/nocodb/src/services/form-columns.service.ts b/packages/nocodb/src/services/form-columns.service.ts index 569c0a1eb7..be7f260632 100644 --- a/packages/nocodb/src/services/form-columns.service.ts +++ b/packages/nocodb/src/services/form-columns.service.ts @@ -1,13 +1,17 @@ import { Injectable } from '@nestjs/common'; import { AppEvents } from 'nocodb-sdk'; -import type { NcContext, NcRequest } from '~/interface/config'; -import type { MetaService } from '~/meta/meta.service'; +import type { NcRequest } from '~/interface/config'; +import { NcContext } from '~/interface/config'; +import { MetaService } from '~/meta/meta.service'; import { type ViewWebhookManager, ViewWebhookManagerBuilder, } from '~/utils/view-webhook-manager'; +import { TraceCommand } from '~/decorators/trace-command.decorator'; +import { OperationName } from '~/command-registry/op-names'; import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; import { validatePayload } from '~/helpers'; +import { assertNotLockedViewOnSandboxProduction } from '~/helpers/sandboxGuards'; import { Column, FormViewColumn, View } from '~/models'; import { extractProps } from '~/helpers/extractProps'; import { NcError } from '~/helpers/ncError'; @@ -16,6 +20,7 @@ import { NcError } from '~/helpers/ncError'; export class FormColumnsService { constructor(private readonly appHooksService: AppHooksService) {} + @TraceCommand(OperationName.formColumnUpdate) async columnUpdate( context: NcContext, param: { @@ -41,6 +46,13 @@ export class FormColumnsService { ncMeta, ); + if (oldFormViewColumn?.fk_view_id) { + await assertNotLockedViewOnSandboxProduction( + context, + oldFormViewColumn.fk_view_id, + ); + } + const view = await View.get( context, oldFormViewColumn.fk_view_id, diff --git a/packages/nocodb/src/services/forms.service.ts b/packages/nocodb/src/services/forms.service.ts index d67de56ffd..23fa282785 100644 --- a/packages/nocodb/src/services/forms.service.ts +++ b/packages/nocodb/src/services/forms.service.ts @@ -5,8 +5,9 @@ import type { UserType, ViewCreateReqType, } from 'nocodb-sdk'; -import type { MetaService } from '~/meta/meta.service'; -import type { NcContext, NcRequest } from '~/interface/config'; +import type { NcRequest } from '~/interface/config'; +import { MetaService } from '~/meta/meta.service'; +import { NcContext } from '~/interface/config'; import { type ViewWebhookManager, ViewWebhookManagerBuilder, @@ -14,7 +15,10 @@ import { import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; import { validatePayload } from '~/helpers'; import { assertPersonalViewAllowed } from '~/helpers/checkPersonalViewFeature'; +import { assertNotSandbox } from '~/helpers/sandboxGuards'; import { NcError } from '~/helpers/catchError'; +import { TraceCommand } from '~/decorators/trace-command.decorator'; +import { OperationName } from '~/command-registry/op-names'; import { FormView, Model, Source, User, View } from '~/models'; import NocoCache from '~/cache/NocoCache'; import { CacheScope } from '~/utils/globals'; @@ -22,12 +26,13 @@ import NocoSocket from '~/socket/NocoSocket'; @Injectable() export class FormsService { - constructor(private readonly appHooksService: AppHooksService) {} + constructor(protected readonly appHooksService: AppHooksService) {} async formViewGet(context: NcContext, param: { formViewId: string }) { return await FormView.getWithInfo(context, param.formViewId); } + @TraceCommand(OperationName.formViewCreate) async formViewCreate( context: NcContext, param: { @@ -40,6 +45,13 @@ export class FormsService { }, ncMeta?: MetaService, ) { + if (param?.ownedBy) { + await assertNotSandbox( + context, + 'Personal views cannot be created in a sandbox. Create them on the production base.', + ); + } + validatePayload( 'swagger.json#/components/schemas/ViewCreateReq', param.body, @@ -156,6 +168,7 @@ export class FormsService { return view; } + @TraceCommand(OperationName.formViewUpdate) async formViewUpdate( context: NcContext, param: { diff --git a/packages/nocodb/src/services/galleries.service.ts b/packages/nocodb/src/services/galleries.service.ts index 089b13958f..8e40c7a579 100644 --- a/packages/nocodb/src/services/galleries.service.ts +++ b/packages/nocodb/src/services/galleries.service.ts @@ -5,8 +5,9 @@ import type { UserType, ViewCreateReqType, } from 'nocodb-sdk'; -import type { NcContext, NcRequest } from '~/interface/config'; -import type { MetaService } from '~/meta/meta.service'; +import type { NcRequest } from '~/interface/config'; +import { NcContext } from '~/interface/config'; +import { MetaService } from '~/meta/meta.service'; import { type ViewWebhookManager, ViewWebhookManagerBuilder, @@ -14,7 +15,10 @@ import { import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; import { validatePayload } from '~/helpers'; import { assertPersonalViewAllowed } from '~/helpers/checkPersonalViewFeature'; +import { assertNotSandbox } from '~/helpers/sandboxGuards'; import { NcError } from '~/helpers/catchError'; +import { TraceCommand } from '~/decorators/trace-command.decorator'; +import { OperationName } from '~/command-registry/op-names'; import { GalleryView, Model, User, View } from '~/models'; import NocoCache from '~/cache/NocoCache'; import { CacheScope } from '~/utils/globals'; @@ -22,12 +26,13 @@ import NocoSocket from '~/socket/NocoSocket'; @Injectable() export class GalleriesService { - constructor(private readonly appHooksService: AppHooksService) {} + constructor(protected readonly appHooksService: AppHooksService) {} async galleryViewGet(context: NcContext, param: { galleryViewId: string }) { return await GalleryView.get(context, param.galleryViewId); } + @TraceCommand(OperationName.galleryViewCreate) async galleryViewCreate( context: NcContext, param: { @@ -40,6 +45,13 @@ export class GalleriesService { }, ncMeta?: MetaService, ) { + if (param?.ownedBy) { + await assertNotSandbox( + context, + 'Personal views cannot be created in a sandbox. Create them on the production base.', + ); + } + validatePayload( 'swagger.json#/components/schemas/ViewCreateReq', param.gallery, @@ -145,6 +157,7 @@ export class GalleriesService { return view; } + @TraceCommand(OperationName.galleryViewUpdate) async galleryViewUpdate( context: NcContext, param: { diff --git a/packages/nocodb/src/services/grid-columns.service.ts b/packages/nocodb/src/services/grid-columns.service.ts index 198c2f819b..537fdaeeaa 100644 --- a/packages/nocodb/src/services/grid-columns.service.ts +++ b/packages/nocodb/src/services/grid-columns.service.ts @@ -1,17 +1,21 @@ import { Injectable } from '@nestjs/common'; import { AppEvents, EventType } from 'nocodb-sdk'; import type { GridColumnReqType } from 'nocodb-sdk'; -import type { NcContext, NcRequest } from '~/interface/config'; -import type { MetaService } from '~/meta/meta.service'; +import type { NcRequest } from '~/interface/config'; +import { NcContext } from '~/interface/config'; +import { MetaService } from '~/meta/meta.service'; import { type ViewWebhookManager, ViewWebhookManagerBuilder, } from '~/utils/view-webhook-manager'; +import { TraceCommand } from '~/decorators/trace-command.decorator'; +import { OperationName } from '~/command-registry/op-names'; import { MetaTable } from '~/cli'; import NocoCache from '~/cache/NocoCache'; import { CacheDelDirection, CacheScope } from '~/utils/globals'; import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; import { validatePayload } from '~/helpers'; +import { assertNotLockedViewOnSandboxProduction } from '~/helpers/sandboxGuards'; import { Column, GridViewColumn, View } from '~/models'; import { extractProps } from '~/helpers/extractProps'; import Noco from '~/Noco'; @@ -29,6 +33,7 @@ export class GridColumnsService { return await GridViewColumn.list(context, param.gridViewId, ncMeta); } + @TraceCommand(OperationName.gridColumnUpdate) async gridColumnUpdate( context: NcContext, param: { @@ -50,6 +55,13 @@ export class GridColumnsService { ncMeta, ); + if (oldGridViewColumn?.fk_view_id) { + await assertNotLockedViewOnSandboxProduction( + context, + oldGridViewColumn.fk_view_id, + ); + } + const column = await Column.get( context, { diff --git a/packages/nocodb/src/services/grids.service.ts b/packages/nocodb/src/services/grids.service.ts index 2f2b285c29..19c193061d 100644 --- a/packages/nocodb/src/services/grids.service.ts +++ b/packages/nocodb/src/services/grids.service.ts @@ -1,8 +1,9 @@ import { Injectable } from '@nestjs/common'; import { AppEvents, EventType, ViewTypes } from 'nocodb-sdk'; import type { GridUpdateReqType, ViewCreateReqType } from 'nocodb-sdk'; -import type { NcContext, NcRequest } from '~/interface/config'; -import type { MetaService } from '~/meta/meta.service'; +import type { NcRequest } from '~/interface/config'; +import { NcContext } from '~/interface/config'; +import { MetaService } from '~/meta/meta.service'; import { type ViewWebhookManager, ViewWebhookManagerBuilder, @@ -10,7 +11,10 @@ import { import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; import { validatePayload } from '~/helpers'; import { assertPersonalViewAllowed } from '~/helpers/checkPersonalViewFeature'; +import { assertNotSandbox } from '~/helpers/sandboxGuards'; import { NcError } from '~/helpers/catchError'; +import { TraceCommand } from '~/decorators/trace-command.decorator'; +import { OperationName } from '~/command-registry/op-names'; import { GridView, Model, User, View } from '~/models'; import NocoCache from '~/cache/NocoCache'; import { CacheScope } from '~/utils/globals'; @@ -18,8 +22,9 @@ import NocoSocket from '~/socket/NocoSocket'; @Injectable() export class GridsService { - constructor(private readonly appHooksService: AppHooksService) {} + constructor(protected readonly appHooksService: AppHooksService) {} + @TraceCommand(OperationName.gridViewCreate) async gridViewCreate( context: NcContext, param: { @@ -31,6 +36,13 @@ export class GridsService { }, ncMeta?: MetaService, ) { + if (param?.ownedBy) { + await assertNotSandbox( + context, + 'Personal views cannot be created in a sandbox. Create them on the production base.', + ); + } + validatePayload( 'swagger.json#/components/schemas/ViewCreateReq', param.grid, @@ -135,6 +147,7 @@ export class GridsService { return view; } + @TraceCommand(OperationName.gridViewUpdate) async gridViewUpdate( context: NcContext, param: { diff --git a/packages/nocodb/src/services/kanbans.service.ts b/packages/nocodb/src/services/kanbans.service.ts index acea9d57bd..380a9bebe5 100644 --- a/packages/nocodb/src/services/kanbans.service.ts +++ b/packages/nocodb/src/services/kanbans.service.ts @@ -11,9 +11,10 @@ import type { UserType, ViewCreateReqType, } from 'nocodb-sdk'; -import type { NcContext, NcRequest } from '~/interface/config'; -import type { MetaService } from '~/meta/meta.service'; +import type { NcRequest } from '~/interface/config'; import type { SelectOption } from '~/models'; +import { NcContext } from '~/interface/config'; +import { MetaService } from '~/meta/meta.service'; import { type ViewWebhookManager, ViewWebhookManagerBuilder, @@ -21,7 +22,10 @@ import { import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; import { validatePayload } from '~/helpers'; import { assertPersonalViewAllowed } from '~/helpers/checkPersonalViewFeature'; +import { assertNotSandbox } from '~/helpers/sandboxGuards'; import { NcError } from '~/helpers/catchError'; +import { TraceCommand } from '~/decorators/trace-command.decorator'; +import { OperationName } from '~/command-registry/op-names'; import { KanbanView, Model, User, View } from '~/models'; import NocoCache from '~/cache/NocoCache'; import { CacheScope } from '~/utils/globals'; @@ -29,7 +33,7 @@ import NocoSocket from '~/socket/NocoSocket'; @Injectable() export class KanbansService { - constructor(private readonly appHooksService: AppHooksService) {} + constructor(protected readonly appHooksService: AppHooksService) {} async kanbanViewGet( context: NcContext, @@ -39,6 +43,7 @@ export class KanbansService { return await KanbanView.get(context, param.kanbanViewId, ncMeta); } + @TraceCommand(OperationName.kanbanViewCreate) async kanbanViewCreate( context: NcContext, param: { @@ -51,6 +56,13 @@ export class KanbansService { }, ncMeta?: MetaService, ) { + if (param?.ownedBy) { + await assertNotSandbox( + context, + 'Personal views cannot be created in a sandbox. Create them on the production base.', + ); + } + validatePayload( 'swagger.json#/components/schemas/ViewCreateReq', param.kanban, @@ -179,6 +191,7 @@ export class KanbansService { return view; } + @TraceCommand(OperationName.kanbanViewUpdate) async kanbanViewUpdate( context: NcContext, param: { diff --git a/packages/nocodb/src/services/maps.service.ts b/packages/nocodb/src/services/maps.service.ts index 41a4995416..e6f0a66bb3 100644 --- a/packages/nocodb/src/services/maps.service.ts +++ b/packages/nocodb/src/services/maps.service.ts @@ -1,7 +1,8 @@ import { Injectable } from '@nestjs/common'; import { AppEvents, ViewTypes } from 'nocodb-sdk'; import type { MapUpdateReqType, UserType, ViewCreateReqType } from 'nocodb-sdk'; -import type { NcContext, NcRequest } from '~/interface/config'; +import type { NcRequest } from '~/interface/config'; +import { NcContext } from '~/interface/config'; import NocoCache from '~/cache/NocoCache'; import { validatePayload } from '~/helpers'; import { assertPersonalViewAllowed } from '~/helpers/checkPersonalViewFeature'; @@ -9,6 +10,8 @@ import { NcError } from '~/helpers/catchError'; import { MapView, Model, User, View } from '~/models'; import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; import { CacheScope } from '~/utils/globals'; +import { TraceCommand } from '~/decorators/trace-command.decorator'; +import { OperationName } from '~/command-registry/op-names'; @Injectable() export class MapsService { @@ -18,6 +21,7 @@ export class MapsService { return await MapView.get(context, param.mapViewId); } + @TraceCommand(OperationName.mapViewCreate) async mapViewCreate( context: NcContext, param: { @@ -76,6 +80,7 @@ export class MapsService { return view; } + @TraceCommand(OperationName.mapViewUpdate) async mapViewUpdate( context: NcContext, param: { diff --git a/packages/nocodb/src/services/meta-dependency/handler/column/column-update-kanban-groupby-dependency.handler.ts b/packages/nocodb/src/services/meta-dependency/handler/column/column-update-kanban-groupby-dependency.handler.ts index 5f3b01d257..5204a85110 100644 --- a/packages/nocodb/src/services/meta-dependency/handler/column/column-update-kanban-groupby-dependency.handler.ts +++ b/packages/nocodb/src/services/meta-dependency/handler/column/column-update-kanban-groupby-dependency.handler.ts @@ -102,6 +102,8 @@ export class ColumnUpdateKanbanGroupByDependencyHandler false, ncMeta, ); + // Skip orphaned kanban entries whose parent view was deleted concurrently + if (!view) continue; view.meta = parseMetaProp(view); if (stillSingleSelect) { diff --git a/packages/nocodb/src/services/model-visibilities.service.ts b/packages/nocodb/src/services/model-visibilities.service.ts index 0f9eff5453..07e163c393 100644 --- a/packages/nocodb/src/services/model-visibilities.service.ts +++ b/packages/nocodb/src/services/model-visibilities.service.ts @@ -6,6 +6,7 @@ import type { UIAclEvent } from '~/services/app-hooks/interfaces'; import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; import { validatePayload } from '~/helpers'; import { NcError } from '~/helpers/catchError'; +import { assertNotSandbox } from '~/helpers/sandboxGuards'; import { Base, Model, ModelRoleVisibility, View } from '~/models'; @Injectable() @@ -20,6 +21,8 @@ export class ModelVisibilitiesService { req: NcRequest; }, ) { + await assertNotSandbox(context); + validatePayload( 'swagger.json#/components/schemas/VisibilityRuleReq', param.visibilityRule, diff --git a/packages/nocodb/src/services/shared-bases.service.ts b/packages/nocodb/src/services/shared-bases.service.ts index 5c2b3cecae..4009f49cfa 100644 --- a/packages/nocodb/src/services/shared-bases.service.ts +++ b/packages/nocodb/src/services/shared-bases.service.ts @@ -39,6 +39,12 @@ export class SharedBasesService { NcError.baseNotFound(param.baseId); } + if (base.is_sandbox) { + NcError.badRequest( + 'Shared links cannot be created on sandbox bases. Share the master base instead.', + ); + } + const data: any = { uuid: uuidv4(), password: param?.password, @@ -92,6 +98,12 @@ export class SharedBasesService { NcError.baseNotFound(param.baseId); } + if (base.is_sandbox) { + NcError.badRequest( + 'Shared links cannot be updated on sandbox bases. Share the master base instead.', + ); + } + if (roles === 'editor') { NcError.badRequest('Only viewer role is supported'); } diff --git a/packages/nocodb/src/services/tables.service.ts b/packages/nocodb/src/services/tables.service.ts index 29ca9b3fb3..fd0575852c 100644 --- a/packages/nocodb/src/services/tables.service.ts +++ b/packages/nocodb/src/services/tables.service.ts @@ -40,6 +40,7 @@ import { ColumnWebhookManagerBuilder } from '~/utils/column-webhook-manager'; import { Base, Column, Model, ModelRoleVisibility, Permission } from '~/models'; import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; import ProjectMgrv2 from '~/db/sql-mgr/v2/ProjectMgrv2'; +import type { OperationSource } from '~/helpers/columnHelpers'; import { NcError } from '~/helpers/catchError'; import getColumnPropsFromUIDT from '~/helpers/getColumnPropsFromUIDT'; import getColumnUiType from '~/helpers/getColumnUiType'; @@ -774,6 +775,7 @@ export class TablesService { synced?: boolean; apiVersion?: NcApiVersion; isDuplicateOperation?: boolean; + operationSource?: OperationSource; }, ) { // before validating add title for columns if only column name is present @@ -820,6 +822,7 @@ export class TablesService { columns: tableCreatePayLoad.columns, clientType: source.type, isMeta: !!source.isMeta(), + operationSource: param.operationSource, }); } diff --git a/packages/nocodb/src/services/v3/data-v3.service.ts b/packages/nocodb/src/services/v3/data-v3.service.ts index bf80c8e5f5..48caa1fa9a 100644 --- a/packages/nocodb/src/services/v3/data-v3.service.ts +++ b/packages/nocodb/src/services/v3/data-v3.service.ts @@ -25,6 +25,7 @@ import type { import type { NcContext } from '~/interface/config'; import type { LinkToAnotherRecordColumn } from '~/models'; import type { ReusableParams } from '~/utils'; +import type { PagedResponseImpl } from '~/helpers/PagedResponse'; import { dataWrapper } from '~/helpers/dbHelpers'; import { NcError } from '~/helpers/catchError'; import { Column, Model, Source } from '~/models'; @@ -1252,12 +1253,15 @@ export class DataV3Service { }; } - const pagedResponse = new PagedResponseV3Impl(response, { - context, - tableId: param.modelId, - baseUrl: param.req.ncSiteUrl, - queryParams: param.query, - }); + const pagedResponse = new PagedResponseV3Impl( + response as PagedResponseImpl>, + { + context, + tableId: param.modelId, + baseUrl: param.req.ncSiteUrl, + queryParams: param.query, + }, + ); // Extract requested fields from query parameters for nested data const requestedFields = this.getRequestedFields(param.query); diff --git a/packages/nocodb/src/services/view-columns.service.ts b/packages/nocodb/src/services/view-columns.service.ts index e194efc6e9..d065088445 100644 --- a/packages/nocodb/src/services/view-columns.service.ts +++ b/packages/nocodb/src/services/view-columns.service.ts @@ -35,7 +35,7 @@ import { ViewWebhookManagerBuilder } from '~/utils/view-webhook-manager'; @Injectable() export class ViewColumnsService { private logger = new Logger(ViewColumnsService.name); - constructor(private appHooksService: AppHooksService) {} + constructor(protected appHooksService: AppHooksService) {} async columnList( context: NcContext, diff --git a/packages/nocodb/src/services/views.service.ts b/packages/nocodb/src/services/views.service.ts index 7758f98804..e8cf73238c 100644 --- a/packages/nocodb/src/services/views.service.ts +++ b/packages/nocodb/src/services/views.service.ts @@ -92,7 +92,7 @@ async function xcVisibilityMetaGet( @Injectable() export class ViewsService { - constructor(private appHooksService: AppHooksService) {} + constructor(protected appHooksService: AppHooksService) {} async viewList( context: NcContext, diff --git a/packages/nocodb/src/utils/acl.ts b/packages/nocodb/src/utils/acl.ts index b909b62ff4..e846e1ab02 100644 --- a/packages/nocodb/src/utils/acl.ts +++ b/packages/nocodb/src/utils/acl.ts @@ -126,6 +126,7 @@ const permissionScopes = { 'formViewGet', 'baseGet', 'tableGet', + 'attachmentDownload', 'dataList', 'linkDataList', 'bulkDataList', @@ -253,6 +254,9 @@ const permissionScopes = { 'userInvite', + // Migration + 'migrateBase', + // AI 'aiUtils', 'aiData', @@ -387,6 +391,8 @@ const rolePermissions: baseGet: true, //table tableGet: true, + // attachment + attachmentDownload: true, // data dataList: true, linkDataList: true, @@ -556,6 +562,7 @@ const rolePermissions: [ProjectRoles.CREATOR]: { exclude: { baseDelete: true, + migrateBase: true, }, }, [ProjectRoles.OWNER]: { @@ -845,6 +852,7 @@ const permissionDescriptions: Record = { formViewGet: 'view forms', baseGet: 'view base details', tableGet: 'view table details', + attachmentDownload: 'download attachments', dataList: 'view data', linkDataList: 'view data', bulkDataList: 'view data', @@ -956,6 +964,8 @@ const permissionDescriptions: Record = { hookTrigger: 'trigger a webhook', + migrateBase: 'migrate a base to another instance', + mcpList: 'view list of MCP tokens', mcpCreate: 'create a new MCP token', mcpUpdate: 'update an MCP token', diff --git a/packages/nocodb/src/utils/globals.ts b/packages/nocodb/src/utils/globals.ts index e1727fc84a..bb00a06008 100644 --- a/packages/nocodb/src/utils/globals.ts +++ b/packages/nocodb/src/utils/globals.ts @@ -58,6 +58,7 @@ export enum MetaTable { NOTIFICATION = 'notification', USER_REFRESH_TOKENS = 'nc_user_refresh_tokens', EXTENSIONS = 'nc_extensions', + BASE_VARIABLES = 'nc_base_variables', COMMENTS = 'nc_comments', USER_COMMENTS_NOTIFICATIONS_PREFERENCE = 'nc_user_comment_notifications_preference', COMMENTS_REACTIONS = 'nc_comment_reactions', @@ -120,6 +121,7 @@ export enum MetaTable { MANAGED_APP_DEPLOYMENT_LOGS = 'nc_managed_app_deployment_logs', AUTOMATION_SUBSCRIBERS = 'nc_automation_subscribers', SANDBOXES = 'nc_sandboxes_v2', + SANDBOX_CHANGELOG = 'nc_sandbox_changelog', SCIM_CONFIG = 'nc_scim_config', RECORD_TEMPLATES = 'nc_record_templates', RLS_POLICIES = 'nc_rls_policies', @@ -189,6 +191,7 @@ export const BaseRelatedMetaTables = [ MetaTable.SYNC_SOURCE, MetaTable.VIEWS, MetaTable.WIDGETS, + MetaTable.BASE_VARIABLES, MetaTable.MODEL_STAT, MetaTable.AUTOMATIONS, MetaTable.AUTOMATION_EXECUTIONS, @@ -356,12 +359,14 @@ export const orderedMetaTables = [ MetaTable.SANDBOX_VERSIONS_OLD, MetaTable.SANDBOXES_OLD, MetaTable.SANDBOXES, + MetaTable.SANDBOX_CHANGELOG, MetaTable.INSTALLATIONS, MetaTable.GCP_MARKETPLACE_ACCOUNTS, MetaTable.GCP_MARKETPLACE_ENTITLEMENTS, // Misc standalone MetaTable.EXTENSIONS, + MetaTable.BASE_VARIABLES, MetaTable.JOBS, MetaTable.FILE_REFERENCES, MetaTable.SNAPSHOT, @@ -468,6 +473,7 @@ export enum CacheScope { VIEW_ALIAS = 'viewAlias', SSO_CLIENT = 'ssoClient', EXTENSION = 'uiExtension', + BASE_VARIABLE = 'baseVariable', INTEGRATION = 'integration', COL_BUTTON = 'colButton', CMD_PALETTE = 'cmdPalette', diff --git a/packages/nocodb/src/utils/internal-type.ts b/packages/nocodb/src/utils/internal-type.ts index c41b696262..0cf0617ba7 100644 --- a/packages/nocodb/src/utils/internal-type.ts +++ b/packages/nocodb/src/utils/internal-type.ts @@ -1,4 +1,9 @@ -import type { DependencyTableType, NcContext, NcRequest } from 'nocodb-sdk'; +import type { + BaseVariableType, + DependencyTableType, + NcContext, + NcRequest, +} from 'nocodb-sdk'; import type { PagedResponseImpl } from '~/helpers/PagedResponse'; import type { OPERATION_SCOPES } from '~/controllers/internal/operationScopes'; import type { Dashboard, Workflow } from '~/models'; @@ -36,6 +41,7 @@ export type InternalGETResponseType = Promise< | Hook[] | HookLog[] | { hash: string } + | { path?: string; url?: string } | OAuthClient | OAuthClient[] | Extension @@ -89,6 +95,8 @@ export type InternalPOSTResponseType = Promise< | Sort | Hook | Extension + | BaseVariableType + | BaseVariableType[] | { added: boolean; reaction: any } | { link: (string | number | Record)[]; diff --git a/packages/nocodb/src/utils/webhook-invoker.ts b/packages/nocodb/src/utils/webhook-invoker.ts index f47a2395e7..4a7c8aa8ff 100644 --- a/packages/nocodb/src/utils/webhook-invoker.ts +++ b/packages/nocodb/src/utils/webhook-invoker.ts @@ -24,6 +24,7 @@ import { } from '~/helpers/webhookHelpers'; import { JobTypes } from '~/interface/Jobs'; import { + BaseVariable, type Hook, HookLog, type Model, @@ -62,6 +63,7 @@ export class WebhookInvoker { prevData, newData, user, + vars, }: { apiMeta: any; user: UserType; @@ -70,6 +72,7 @@ export class WebhookInvoker { view?: ViewType; prevData: Record; newData: Record; + vars?: Record; }) { if (!_apiMeta) { _apiMeta = {}; @@ -112,13 +115,13 @@ export class WebhookInvoker { : JSON.stringify(apiMeta.body), (_key, value) => { return typeof value === 'string' - ? parseBody(value, webhookData) + ? parseBody(value, webhookData, vars) : value; }, ); } catch (e) { // if string parsing failed then directly apply the handlebar - apiMeta.body = parseBody(apiMeta.body, webhookData); + apiMeta.body = parseBody(apiMeta.body, webhookData, vars); } } if (apiMeta.auth) { @@ -129,22 +132,22 @@ export class WebhookInvoker { : JSON.stringify(apiMeta.auth), (_key, value) => { return typeof value === 'string' - ? parseBody(value, webhookData) + ? parseBody(value, webhookData, vars) : value; }, ); } catch (e) { - apiMeta.auth = parseBody(apiMeta.auth, webhookData); + apiMeta.auth = parseBody(apiMeta.auth, webhookData, vars); } } apiMeta.response = {}; - const url = parseBody(apiMeta.path, webhookData); + const url = parseBody(apiMeta.path, webhookData, vars); const reqPayload = { params: apiMeta.parameters ? apiMeta.parameters.reduce((paramsObj, param) => { if (param.name && param.enabled) { - paramsObj[param.name] = parseBody(param.value, webhookData); + paramsObj[param.name] = parseBody(param.value, webhookData, vars); } return paramsObj; }, {}) @@ -155,7 +158,11 @@ export class WebhookInvoker { headers: apiMeta.headers ? apiMeta.headers.reduce((headersObj, header) => { if (header.name && header.enabled) { - headersObj[header.name] = parseBody(header.value, webhookData); + headersObj[header.name] = parseBody( + header.value, + webhookData, + vars, + ); } return headersObj; }, {}) @@ -357,6 +364,10 @@ export class WebhookInvoker { let hookLog: HookLogType; const startTime = process.hrtime(); const source = await Source.get(context, model.source_id); + + // Load base variables for template resolution + const vars = await BaseVariable.listAsMap(context, model.base_id); + let notification, filters; let reqPayload; try { @@ -462,9 +473,13 @@ export class WebhookInvoker { }); const parsedPayload = { - to: parseBody(notification?.payload?.to, webhookData), - subject: parseBody(notification?.payload?.subject, webhookData), - html: parseBody(notification?.payload?.body, webhookData), + to: parseBody(notification?.payload?.to, webhookData, vars), + subject: parseBody( + notification?.payload?.subject, + webhookData, + vars, + ), + html: parseBody(notification?.payload?.body, webhookData, vars), }; const res = await ( await NcPluginMgrv2.emailAdapter(false) @@ -493,6 +508,7 @@ export class WebhookInvoker { view, prevData, newData, + vars, }); const { requestPayload, responsePayload } = await handleHttpWebHook( @@ -608,12 +624,12 @@ export class WebhookInvoker { const res = await ( await NcPluginMgrv2.webhookNotificationAdapters(notification.type) ).sendMessage( - parseBody(notification?.payload?.body, webhookData), + parseBody(notification?.payload?.body, webhookData, vars), JSON.parse( JSON.stringify(notification?.payload), (_key, value) => { return typeof value === 'string' - ? parseBody(value, webhookData) + ? parseBody(value, webhookData, vars) : value; }, ), @@ -647,14 +663,14 @@ export class WebhookInvoker { } } catch (e) { if (e.response) { - this.logger.error({ + this.logger.debug({ data: e.response.data, status: e.response.status, url: e.response.config?.url, message: e.message, }); } else { - this.logger.error(e.message, e.stack); + this.logger.debug(e.message, e.stack); } if (['ERROR', 'ALL'].includes(webhookLogLevel) || isEE) { hookLog = { diff --git a/tsconfig.json b/tsconfig.json index 353bd4c91e..302fcff636 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,7 +2,11 @@ /* Root tsconfig — "solution" style: no files of its own. Without "files": [], TS creates a catch-all project that includes the ENTIRE repo, causing every file to belong to 2+ projects and forcing VS Code's TS server - to reload when switching between files. */ + to reload when switching between files. + + Project references tell tsserver to treat each package as an isolated project. + This prevents one giant TS program from loading all 5000+ files at once. + Build is handled by rspack — composite/declaration are purely for IDE perf. */ "include": [], "compilerOptions": { "target": "ESNext", @@ -11,6 +15,11 @@ "noEmit": true, "skipLibCheck": true }, + "references": [ + { "path": "packages/nocodb-sdk" }, + { "path": "packages/nocodb" }, + { "path": "packages/nc-gui" } + ], "exclude": [ "node_modules", "packages/noco-integrations",