feat(grid): preview Oracle LOB columns with bounded substrings
This commit is contained in:
Conastin authored and GitHub committed
1 parent
a266cf0253
commit
b71823d784
12 files changed
+403
-25
No files matched your search
@@ -4405,7 +4405,7 @@ func (s *server) runPagedOracleSelect(sqlText string, opts queryOptions, pageSiz
|
||||
}
|
||||
return oraclePagedRetrySQL(sqlText, s.loadOracleColumnMeta, err)
|
||||
}
|
||||
rows, err := s.queryRowsWithOracleValueRewriteIfNeeded(sqlText, opts.TimeoutSecs, opts.DeferLOBs)
|
||||
rows, err := s.queryRowsWithOracleValueRewriteIfNeeded(sqlText, opts.TimeoutSecs, deferLobsForQuery(sqlText, opts.DeferLOBs))
|
||||
if err != nil {
|
||||
if rewritten, ok := retrySQL(err); ok {
|
||||
sqlText = rewritten
|
||||
@@ -4655,7 +4655,7 @@ func (s *server) executeQuery(opts queryOptions) (queryResult, error) {
|
||||
maxRows = defaultMaxRows
|
||||
}
|
||||
if isQuerySQL(sqlText) {
|
||||
result, err := s.executeSelect(sqlText, maxRows, opts.TimeoutSecs, opts.DeferLOBs)
|
||||
result, err := s.executeSelect(sqlText, maxRows, opts.TimeoutSecs, deferLobsForQuery(sqlText, opts.DeferLOBs))
|
||||
result.ExecutionTimeMS = time.Since(start).Milliseconds()
|
||||
return result, err
|
||||
}
|
||||
@@ -4803,6 +4803,16 @@ func columnTypeNames(rows *sql.Rows) []string {
|
||||
return result
|
||||
}
|
||||
|
||||
// deferLobsForQuery keeps SQL-level large-value previews in charge. When the
|
||||
// client already projected __DBX_LARGE_VALUE_BYTES_ marker columns (Oracle
|
||||
// table-data previews truncate LOBs with DBMS_LOB.SUBSTR in the SQL), those
|
||||
// expressions must pass through untouched and the nested value rewrite
|
||||
// (geometry/XMLType/opaque conversions) keeps running as usual, so the
|
||||
// deferred-LOB placeholder pass must not engage on top of them.
|
||||
func deferLobsForQuery(sqlText string, requested bool) bool {
|
||||
return requested && !strings.Contains(strings.ToUpper(sqlText), largeValueBytesColumnPrefix)
|
||||
}
|
||||
|
||||
func (s *server) queryRowsWithOracleValueRewriteIfNeeded(sqlText string, timeoutSecs int, deferLOBs bool) (resultRows *sql.Rows, queryErr error) {
|
||||
// A statement that keeps row locks while it runs is rewritten before the
|
||||
// first attempt: when such a statement panics after the rows were locked,
|
||||
|
||||
@@ -4956,6 +4956,22 @@ func TestRewriteOracleOpaqueObjectInDeferredProjection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeferLobsForQueryDefersToSQLPreviewMarkers(t *testing.T) {
|
||||
plain := `SELECT "ID", "BODY" FROM "MESPRD"."DOCUMENTS"`
|
||||
if !deferLobsForQuery(plain, true) {
|
||||
t.Fatal("defer should engage for plain projections")
|
||||
}
|
||||
if deferLobsForQuery(plain, false) {
|
||||
t.Fatal("defer must honor the request flag")
|
||||
}
|
||||
preview := `SELECT "ID", "BODY", "__DBX_LARGE_VALUE_BYTES_T_1" FROM (SELECT "ID", ` +
|
||||
`DBMS_LOB.SUBSTR("BODY", 201, 1) AS "BODY", 'T:200:' || DBMS_LOB.GETLENGTH("BODY") AS "__DBX_LARGE_VALUE_BYTES_T_1" ` +
|
||||
`FROM "MESPRD"."DOCUMENTS") WHERE ROWNUM <= 100`
|
||||
if deferLobsForQuery(preview, true) {
|
||||
t.Fatal("defer must not engage when SQL-level preview markers are present")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSingleOracleTableRefReadsDatabaseLinkAndAlias(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
|
||||
@@ -45,6 +45,7 @@ vi.mock("@/lib/backend/api", () => ({
|
||||
vi.mock("@/lib/table/tableSelectSql", () => ({
|
||||
buildTableSelectSql: mocks.buildTableSelectSql,
|
||||
quoteTableDataIdentifier: (_databaseType: string, name: string) => `"${name}"`,
|
||||
requiresEagerTableMetadataForDataOpen: (databaseType: string | undefined) => databaseType === "mysql" || databaseType === "postgres" || databaseType === "oracle",
|
||||
}));
|
||||
|
||||
vi.mock("@/stores/connectionStore", () => ({
|
||||
|
||||
@@ -368,10 +368,11 @@ export function useDataGridActions(activeTab: ComputedRef<QueryTab | undefined>)
|
||||
const connectionGeneration = connectionStore.metadataGenerationFor(tab.connectionId, tab.database);
|
||||
const lifecycleStale = isDataTabMetadataLifecycleStale(tab, connectionGeneration);
|
||||
const shouldRefreshMetadata = lifecycleStale || tab.tableMetaPending || !hasRealTableMetaColumns || metadataAgeMs > DATA_TAB_METADATA_TTL_MS;
|
||||
const effectiveDbType = effectiveDatabaseTypeForConnection(connectionStore.getConfig(tab.connectionId));
|
||||
// Dameng 元数据必须与数据查询串行（同 useSidebarDataOpenRuntime），
|
||||
// 延后到查询完成后再启动。主动刷新和跨生命周期重建除外：必须先拿到新列再
|
||||
// 构建 SQL，否则第一次 toolbar reload 仍会沿用断链前的显式列列表。
|
||||
const deferMetadataRefresh = intent !== "refresh" && !lifecycleStale && effectiveDatabaseTypeForConnection(connectionStore.getConfig(tab.connectionId)) === "dameng";
|
||||
const deferMetadataRefresh = intent !== "refresh" && !lifecycleStale && effectiveDbType === "dameng";
|
||||
const startMetadataRefresh = () => {
|
||||
console.info("[DBX][reloadData:metadata:background:start]", { traceId, elapsed: elapsed(), reason: hasRealTableMetaColumns ? "stale" : "missing", metadataAgeMs });
|
||||
void refreshDataTabTableMeta(tab, { force: lifecycleStale, trace: { traceId, elapsed } })
|
||||
@@ -442,7 +443,28 @@ export function useDataGridActions(activeTab: ComputedRef<QueryTab | undefined>)
|
||||
// 挂起等待，防止数据查询先返回后编辑/保存以空 primaryKeys 短暂可用，
|
||||
// 走整行 WHERE 保存路径（#3727）。真实元数据经 setTableMeta 落地后解除
|
||||
if (!hasRealTableMetaColumns) tab.tableMetaPending = true;
|
||||
if (!deferMetadataRefresh) startMetadataRefresh();
|
||||
// Oracle 的 LOB 预览投影依赖列清单：缺失时先等元数据落地再构建 SQL，
|
||||
// 否则恢复 tab 的翻页会退回无截断的 SELECT *。MySQL/PostgreSQL 维持
|
||||
// 既有语义——reload 先执行、元数据后台补（app-test
|
||||
// "data reload executes before slow metadata refresh completes"）。
|
||||
if (!hasRealTableMetaColumns && effectiveDbType === "oracle") {
|
||||
console.info("[DBX][reloadData:metadata:await:missing]", { traceId, elapsed: elapsed() });
|
||||
try {
|
||||
const rebuilt = await refreshDataTabTableMeta(tab, { force: false, trace: { traceId, elapsed } });
|
||||
console.info("[DBX][reloadData:metadata:await:done]", { traceId, elapsed: elapsed(), rebuilt });
|
||||
if (!rebuilt) {
|
||||
stopPreparing();
|
||||
return;
|
||||
}
|
||||
} catch (e: any) {
|
||||
console.warn("[DBX][reloadData:metadata:await:error]", { traceId, elapsed: elapsed(), error: e });
|
||||
stopPreparing();
|
||||
toast(e?.message || String(e), 5000);
|
||||
throw e;
|
||||
}
|
||||
} else if (!deferMetadataRefresh) {
|
||||
startMetadataRefresh();
|
||||
}
|
||||
} else {
|
||||
console.info("[DBX][reloadData:metadata:skip]", { traceId, elapsed: elapsed(), columnCount: tab.tableMeta!.columns.length, metadataAgeMs });
|
||||
}
|
||||
|
||||
@@ -333,6 +333,7 @@ async function openTableTarget(target: NavigationTarget, options: NavigationOpen
|
||||
whereInput: target.whereInput,
|
||||
primaryKeys,
|
||||
columns: columns.map((column) => column.name),
|
||||
...tableDataLargeValuePreviewOptions(effectiveDbType, columns, primaryKeys, pageLimit),
|
||||
includeRowId: true,
|
||||
orderBy: defaultSort.orderBy,
|
||||
limit: pageLimit,
|
||||
|
||||
@@ -57,6 +57,30 @@ describe("data grid large-value metadata", () => {
|
||||
expect(tableDataLargeValuePreviewOptions("mysql", [columns[0]!], ["id"], 100)).toEqual({});
|
||||
});
|
||||
|
||||
it("enables oracle previews for LOB columns and budgets over the LOB count only", () => {
|
||||
const lobColumns = [column("ID", "NUMBER", true), column("REQ_PAYLOAD", "CLOB"), column("RAW_DATA", "BLOB")];
|
||||
expect(canUseTableDataLargeValuePreview("oracle", lobColumns, ["ID"])).toBe(true);
|
||||
expect(tableDataLargeValuePreviewOptions("oracle", lobColumns, ["ID"], 1000)).toEqual({
|
||||
columnTypes: ["NUMBER", "CLOB", "BLOB"],
|
||||
// 24 MiB / 1000 rows / 2 LOB columns / 6 serialized bytes per character.
|
||||
largeValuePreviewSize: 2097,
|
||||
});
|
||||
|
||||
// 宽表只按 LOB 列摊预算：30 个标量列不影响每个 LOB 单元格的预览大小
|
||||
// （单 LOB 列 → 24 MiB / 1000 行 / 6 序列化字节 = 4194 字符）。
|
||||
const wideColumns = [column("ID", "NUMBER", true), ...Array.from({ length: 30 }, (_, index) => column(`SCALAR_${index}`, "VARCHAR2")), column("BODY", "NCLOB")];
|
||||
expect(tableDataLargeValuePreviewOptions("oracle", wideColumns, ["ID"], 1000)).toEqual({
|
||||
columnTypes: wideColumns.map((item) => item.data_type),
|
||||
largeValuePreviewSize: 4194,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps oracle previews off for tables without truncatable LOB columns", () => {
|
||||
expect(canUseTableDataLargeValuePreview("oracle", [column("ID", "NUMBER", true), column("DOC", "XMLTYPE")], ["ID"])).toBe(false);
|
||||
expect(canUseTableDataLargeValuePreview("oracle", [column("ID", "NUMBER", true), column("BODY", "CLOB")], [])).toBe(false);
|
||||
expect(tableDataLargeValuePreviewOptions("oracle", [column("ID", "NUMBER", true), column("DOC", "XMLTYPE")], ["ID"], 100)).toEqual({});
|
||||
});
|
||||
|
||||
it("disables marker parsing when a real column uses the reserved preview prefix", () => {
|
||||
const columns = [column("id", "bigint", true), column("__dbx_large_value_bytes_t_0", "text")];
|
||||
|
||||
|
||||
@@ -342,6 +342,7 @@ describe("quoteTableIdentifier", () => {
|
||||
expect(requiresEagerTableMetadataForDataOpen("salesforce")).toBe(true);
|
||||
expect(requiresEagerTableMetadataForDataOpen("mysql")).toBe(true);
|
||||
expect(requiresEagerTableMetadataForDataOpen("postgres")).toBe(true);
|
||||
expect(requiresEagerTableMetadataForDataOpen("oracle")).toBe(true);
|
||||
expect(requiresEagerTableMetadataForDataOpen("nebula")).toBe(true);
|
||||
expect(requiresEagerTableMetadataForDataOpen("neo4j")).toBe(true);
|
||||
// Drivers whose preview works from `SELECT *` keep loading metadata lazily.
|
||||
|
||||
@@ -177,12 +177,30 @@ function matchesMysqlUnboundedLargeValueType(base: string): boolean {
|
||||
return base === "blob" || base === "mediumblob" || base === "longblob" || base === "text" || base === "mediumtext" || base === "longtext" || base === "json";
|
||||
}
|
||||
|
||||
/**
|
||||
* Oracle table-data previews truncate LOBs with DBMS_LOB.SUBSTR in the SQL
|
||||
* itself; types without a SUBSTR overload (XMLTYPE, BFILE, LONG, object
|
||||
* columns) keep their existing passthrough behavior.
|
||||
*/
|
||||
function isOracleTableDataLobColumn(column: Pick<ColumnInfo, "data_type">): boolean {
|
||||
return /^(?:NCLOB|CLOB|BLOB)$/i.test(column.data_type.trim());
|
||||
}
|
||||
|
||||
export function canUseTableDataLargeValuePreview(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[]): boolean {
|
||||
return supportsTableDataLargeValuePreview(databaseType) && columns.length > 0 && primaryKeys.length > 0 && !columns.some((column) => column.name.toLocaleUpperCase().startsWith(TABLE_DATA_LARGE_VALUE_MARKER_PREFIX));
|
||||
return (
|
||||
supportsTableDataLargeValuePreview(databaseType) &&
|
||||
columns.length > 0 &&
|
||||
primaryKeys.length > 0 &&
|
||||
// Oracle opts in only when a CLOB/NCLOB/BLOB column actually exists: the
|
||||
// preview flag also reaches the Oracle agent as deferLobs, which must stay
|
||||
// off for tables the SQL preview cannot serve.
|
||||
(databaseType !== "oracle" || columns.some(isOracleTableDataLobColumn)) &&
|
||||
!columns.some((column) => column.name.toLocaleUpperCase().startsWith(TABLE_DATA_LARGE_VALUE_MARKER_PREFIX))
|
||||
);
|
||||
}
|
||||
|
||||
export function supportsTableDataLargeValuePreview(databaseType: DatabaseType | undefined): boolean {
|
||||
return databaseType === "mysql" || databaseType === "postgres" || databaseType === "db2";
|
||||
return databaseType === "mysql" || databaseType === "postgres" || databaseType === "db2" || databaseType === "oracle";
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -206,7 +224,15 @@ export function tableDataLargeValuePreviewOptions(databaseType: DatabaseType | u
|
||||
if (!canUseTableDataLargeValuePreview(databaseType, columns, primaryKeys)) return {};
|
||||
const keyColumns = new Set(primaryKeys.map((column) => column.toLocaleLowerCase()));
|
||||
const mysqlBudget = databaseType === "mysql" ? mysqlPreviewBudget(columns, keyColumns, pageSize) : null;
|
||||
const previewColumnCount = databaseType === "mysql" ? (mysqlBudget?.previewColumnCount ?? 0) : columns.filter((column) => !keyColumns.has(column.name.toLocaleLowerCase())).length;
|
||||
// Oracle only ever truncates CLOB/NCLOB/BLOB columns, so budgeting over the
|
||||
// exact LOB column count keeps the per-cell preview from being spread thin
|
||||
// across wide tables.
|
||||
const previewColumnCount =
|
||||
databaseType === "mysql"
|
||||
? (mysqlBudget?.previewColumnCount ?? 0)
|
||||
: databaseType === "oracle"
|
||||
? columns.filter((column) => !keyColumns.has(column.name.toLocaleLowerCase()) && isOracleTableDataLobColumn(column)).length
|
||||
: columns.filter((column) => !keyColumns.has(column.name.toLocaleLowerCase())).length;
|
||||
if (previewColumnCount === 0) return {};
|
||||
return {
|
||||
columnTypes: columns.map((column) => column.data_type),
|
||||
|
||||
@@ -443,8 +443,8 @@ export function normalizeWhereInput(whereInput?: string): string {
|
||||
* so the columns must be awaited before the statement is generated rather than
|
||||
* refreshed in the background.
|
||||
*
|
||||
* * MySQL / PostgreSQL: the large-value preview projection is derived from the
|
||||
* column types and primary keys.
|
||||
* * MySQL / PostgreSQL / Oracle: the large-value preview projection is derived
|
||||
* from the column types and primary keys.
|
||||
* * Salesforce: SOQL has no `SELECT *`. With no known fields the backend builder
|
||||
* falls back to the `FIELDS(ALL)` selector, which the org only accepts with
|
||||
* `LIMIT 200` or less — awaiting the describe keeps every page size working.
|
||||
@@ -452,7 +452,7 @@ export function normalizeWhereInput(whereInput?: string): string {
|
||||
* vertex/edge value instead of separate property columns.
|
||||
*/
|
||||
export function requiresEagerTableMetadataForDataOpen(databaseType: DatabaseType | undefined): boolean {
|
||||
return databaseType === "mysql" || databaseType === "postgres" || databaseType === "salesforce" || databaseType === "neo4j" || databaseType === "nebula";
|
||||
return databaseType === "mysql" || databaseType === "postgres" || databaseType === "oracle" || databaseType === "salesforce" || databaseType === "neo4j" || databaseType === "nebula";
|
||||
}
|
||||
|
||||
export async function buildTableSelectSql(options: BuildTableSelectSqlOptions): Promise<string> {
|
||||
|
||||
@@ -12941,6 +12941,63 @@ for line in sys.stdin:
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_oracle_rowid_preview_markers_indexed_by_result_position() {
|
||||
// 无主键 Oracle 基表的截断投影以 __DBX_ROWID 开头，marker 下标是结果集
|
||||
// 位置（源列在结果中的下标），不是表列下标——位置校验失败会整组拒识。
|
||||
let mut result = db::QueryResult {
|
||||
columns: vec![
|
||||
"__DBX_ROWID".to_string(),
|
||||
"ID".to_string(),
|
||||
"PAYLOAD".to_string(),
|
||||
format!("{}T_2", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
|
||||
],
|
||||
column_types: vec![
|
||||
"varchar2".to_string(),
|
||||
"number".to_string(),
|
||||
"varchar2".to_string(),
|
||||
"varchar2".to_string(),
|
||||
],
|
||||
column_sortables: vec![true, true, true, true],
|
||||
spatial_columns: Vec::new(),
|
||||
spatial_values: Vec::new(),
|
||||
rows: vec![
|
||||
vec![
|
||||
serde_json::json!("AAAR1yAAHAAAAFjAAA"),
|
||||
serde_json::json!(1),
|
||||
serde_json::json!("x".repeat(1000)),
|
||||
serde_json::json!("T:999:5231"),
|
||||
],
|
||||
vec![
|
||||
serde_json::json!("AAAR1yAAHAAAAFjAAB"),
|
||||
serde_json::json!(2),
|
||||
serde_json::Value::Null,
|
||||
serde_json::Value::Null,
|
||||
],
|
||||
],
|
||||
affected_rows: 0,
|
||||
execution_time_ms: 0,
|
||||
server_execute_time_us: None,
|
||||
query_timings_ms: None,
|
||||
truncated: false,
|
||||
session_id: None,
|
||||
has_more: false,
|
||||
elasticsearch_raw_body: None,
|
||||
messages: Vec::new(),
|
||||
};
|
||||
|
||||
let cells = extract_server_large_value_markers(&mut result);
|
||||
|
||||
assert_eq!(result.columns, vec!["__DBX_ROWID", "ID", "PAYLOAD"]);
|
||||
assert_eq!(result.rows.len(), 2);
|
||||
assert_eq!(result.rows[0][2], serde_json::json!(format!("{}...", "x".repeat(999))));
|
||||
assert_eq!(result.rows[1][2], serde_json::Value::Null);
|
||||
assert_eq!(
|
||||
cells,
|
||||
vec![db::LargeValueCell { row_index: 0, column_index: 2, original_bytes: 5231, value_ref: None }]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_lowercased_server_large_value_markers_from_case_folding_servers() {
|
||||
// KingbaseES instances with case-insensitive identifiers return even
|
||||
|
||||
@@ -105,11 +105,30 @@ fn large_value_preview_kind(
|
||||
None
|
||||
}
|
||||
}
|
||||
// XMLTYPE, BFILE, LONG and object columns stay untouched: DBMS_LOB.SUBSTR
|
||||
// has no overload for them, and the agent's value rewrite keeps
|
||||
// handling them exactly as before.
|
||||
Some(DatabaseType::Oracle) => {
|
||||
if matches!(base.as_str(), "clob" | "nclob") {
|
||||
Some(LargeValuePreviewKind::Text)
|
||||
} else if base == "blob" {
|
||||
Some(LargeValuePreviewKind::Binary)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_large_value_preview_columns(options: &TableDataSelectSqlOptions) -> Option<String> {
|
||||
struct LargeValuePreviewItem {
|
||||
projection: String,
|
||||
/// Output alias of the projection. ROWNUM pagination re-selects it from the
|
||||
/// inline view, so it must equal the expression's AS alias exactly.
|
||||
alias: String,
|
||||
}
|
||||
|
||||
fn build_large_value_preview_items(options: &TableDataSelectSqlOptions) -> Option<Vec<LargeValuePreviewItem>> {
|
||||
let database_type = options.database_type;
|
||||
let preview_size = options.large_value_preview_size?.max(1);
|
||||
if options.columns.is_empty()
|
||||
@@ -125,7 +144,17 @@ fn build_large_value_preview_columns(options: &TableDataSelectSqlOptions) -> Opt
|
||||
|
||||
let protected: std::collections::HashSet<String> =
|
||||
options.primary_keys.iter().map(|column| column.to_ascii_lowercase()).collect();
|
||||
let mut projections = Vec::with_capacity(options.columns.len() * 2);
|
||||
// Marker suffixes index the source column's position in the RESULT, which
|
||||
// the client-side extraction verifies (`expected_source_index`). Oracle
|
||||
// tables keyed by the synthetic ROWID prepend one projection, shifting
|
||||
// every column after it; MySQL/PostgreSQL/DB2 project exactly the column
|
||||
// list, so their offsets stay zero.
|
||||
let source_index_offset = usize::from(
|
||||
options.include_row_id
|
||||
&& uses_oracle_row_id(options.database_type)
|
||||
&& is_oracle_base_table_type(options.table_type.as_deref()),
|
||||
);
|
||||
let mut items = Vec::with_capacity(options.columns.len() * 2);
|
||||
let mut marker_count = 0;
|
||||
for (column_index, (column, data_type)) in options.columns.iter().zip(&options.column_types).enumerate() {
|
||||
let quoted = if uses_connection_identifier_quote(database_type, options.identifier_quote.as_deref()) {
|
||||
@@ -137,16 +166,27 @@ fn build_large_value_preview_columns(options: &TableDataSelectSqlOptions) -> Opt
|
||||
.then(|| large_value_preview_kind(database_type, data_type, preview_size))
|
||||
.flatten();
|
||||
let Some(kind) = kind else {
|
||||
projections.push(quoted);
|
||||
items.push(LargeValuePreviewItem { projection: quoted, alias: column.clone() });
|
||||
continue;
|
||||
};
|
||||
|
||||
let alias_kind = large_value_marker_alias_kind(kind, data_type);
|
||||
let marker_alias = quote_table_identifier(
|
||||
database_type,
|
||||
&format!("{DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX}{alias_kind}_{column_index}"),
|
||||
);
|
||||
// Oracle caps SQL-level results at 4000 bytes for VARCHAR2/NVARCHAR2
|
||||
// and 2000 bytes for RAW. DBMS_LOB.SUBSTR amounts are characters, so
|
||||
// 1000 characters of 4-byte UTF-8 and 1999 bytes (RAWTOHEX doubles to
|
||||
// 3998 characters plus the "0x" prefix) are the safe ceilings. The
|
||||
// recorded preview size stays below the fetch amount so oversized
|
||||
// values keep the +1-character truncation signal.
|
||||
let preview_size = match (database_type, kind) {
|
||||
(Some(DatabaseType::Oracle), LargeValuePreviewKind::Binary) => preview_size.min(1998),
|
||||
(Some(DatabaseType::Oracle), _) => preview_size.min(999),
|
||||
_ => preview_size,
|
||||
};
|
||||
let prefix_size = preview_size.saturating_add(1);
|
||||
|
||||
let alias_kind = large_value_marker_alias_kind(kind, data_type);
|
||||
let marker_name =
|
||||
format!("{DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX}{alias_kind}_{}", column_index + source_index_offset);
|
||||
let marker_alias = quote_table_identifier(database_type, &marker_name);
|
||||
let (preview, marker_kind) = match database_type {
|
||||
Some(DatabaseType::Mysql) if kind == LargeValuePreviewKind::Binary => {
|
||||
(format!("LEFT({quoted}, {prefix_size}) AS {quoted}"), "B")
|
||||
@@ -173,20 +213,41 @@ fn build_large_value_preview_columns(options: &TableDataSelectSqlOptions) -> Opt
|
||||
format!("SUBSTR({quoted}, 1, CASE WHEN LENGTH({quoted}) >= {prefix_size} THEN {prefix_size} ELSE LENGTH({quoted}) END) AS {quoted}"),
|
||||
"T",
|
||||
),
|
||||
Some(DatabaseType::Oracle) if kind == LargeValuePreviewKind::Binary => (
|
||||
// RAWTOHEX yields the 0x-prefixed hex shape the client-side
|
||||
// binary preview truncation expects.
|
||||
format!("'0x' || RAWTOHEX(DBMS_LOB.SUBSTR({quoted}, {prefix_size}, 1)) AS {quoted}"),
|
||||
"B",
|
||||
),
|
||||
Some(DatabaseType::Oracle) => (format!("DBMS_LOB.SUBSTR({quoted}, {prefix_size}, 1) AS {quoted}"), "T"),
|
||||
_ => return None,
|
||||
};
|
||||
let marker = if database_type == Some(DatabaseType::Mysql) {
|
||||
format!("CONCAT('{marker_kind}:{preview_size}:', LENGTH({quoted})) AS {marker_alias}")
|
||||
} else if database_type == Some(DatabaseType::Postgres) && kind == LargeValuePreviewKind::Binary {
|
||||
format!("'{marker_kind}:{preview_size}:' || octet_length({quoted})::text AS {marker_alias}")
|
||||
} else if database_type == Some(DatabaseType::Oracle) {
|
||||
// GETLENGTH reads LOB metadata only; a NULL LOB yields a NULL
|
||||
// marker, which the client treats as "nothing to load".
|
||||
format!("'{marker_kind}:{preview_size}:' || DBMS_LOB.GETLENGTH({quoted}) AS {marker_alias}")
|
||||
} else {
|
||||
format!("'{marker_kind}:{preview_size}' AS {marker_alias}")
|
||||
};
|
||||
projections.push(preview);
|
||||
projections.push(marker);
|
||||
items.push(LargeValuePreviewItem { projection: preview, alias: column.clone() });
|
||||
items.push(LargeValuePreviewItem { projection: marker, alias: marker_name });
|
||||
marker_count += 1;
|
||||
}
|
||||
(marker_count > 0).then(|| projections.join(", "))
|
||||
(marker_count > 0).then_some(items)
|
||||
}
|
||||
|
||||
fn build_large_value_preview_columns(options: &TableDataSelectSqlOptions) -> Option<String> {
|
||||
Some(
|
||||
build_large_value_preview_items(options)?
|
||||
.into_iter()
|
||||
.map(|item| item.projection)
|
||||
.collect::<Vec<_>>()
|
||||
.join(", "),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn build_count_table_sql(database_type: Option<DatabaseType>, schema: Option<&str>, table_name: &str) -> String {
|
||||
@@ -320,7 +381,19 @@ pub fn build_table_data_select_sql_with_database(
|
||||
&& is_postgres_base_table_type(options.table_type.as_deref())
|
||||
&& options.primary_keys.is_empty();
|
||||
let offset = options.offset.unwrap_or(0);
|
||||
let select_columns = if include_oracle_row_id {
|
||||
// Oracle previews ride the ROWNUM inline view: the truncated projections
|
||||
// and marker columns live in the inner SELECT, and the outer page SELECT
|
||||
// re-selects them by alias so markers survive pagination.
|
||||
let oracle_preview_items =
|
||||
build_large_value_preview_items(&options).filter(|_| database_type == Some(DatabaseType::Oracle));
|
||||
let select_columns = if let Some(items) = &oracle_preview_items {
|
||||
let mut projections = Vec::with_capacity(items.len() + 1);
|
||||
if include_oracle_row_id {
|
||||
projections.push(format!("ROWIDTOCHAR(t.ROWID) AS \"{DBX_ROWID_COLUMN}\""));
|
||||
}
|
||||
projections.extend(items.iter().map(|item| item.projection.clone()));
|
||||
projections.join(", ")
|
||||
} else if include_oracle_row_id {
|
||||
format!("ROWIDTOCHAR(t.ROWID) AS \"{DBX_ROWID_COLUMN}\", t.*")
|
||||
} else if include_postgres_ctid {
|
||||
// tableoid separates inherited relations; xmin keeps a recycled CTID from
|
||||
@@ -348,7 +421,14 @@ pub fn build_table_data_select_sql_with_database(
|
||||
)
|
||||
};
|
||||
let rownum_select_columns = quoted_table_columns_or_star(database_type, &options.columns);
|
||||
let page_select_columns = if include_oracle_row_id {
|
||||
let page_select_columns = if let Some(items) = &oracle_preview_items {
|
||||
let mut columns = Vec::with_capacity(items.len() + 1);
|
||||
if include_oracle_row_id {
|
||||
columns.push(format!("\"{DBX_ROWID_COLUMN}\""));
|
||||
}
|
||||
columns.extend(items.iter().map(|item| quote_table_identifier(database_type, &item.alias)));
|
||||
columns.join(", ")
|
||||
} else if include_oracle_row_id {
|
||||
if options.columns.is_empty() {
|
||||
"*".to_string()
|
||||
} else {
|
||||
@@ -410,8 +490,11 @@ pub fn build_table_data_select_sql_with_database(
|
||||
)
|
||||
}
|
||||
TablePaginationStrategy::Rownum => {
|
||||
let rownum_inner_select_columns =
|
||||
if include_oracle_row_id { &select_columns } else { &rownum_select_columns };
|
||||
let rownum_inner_select_columns = if include_oracle_row_id || oracle_preview_items.is_some() {
|
||||
&select_columns
|
||||
} else {
|
||||
&rownum_select_columns
|
||||
};
|
||||
build_rownum_table_select_sql(
|
||||
&table_alias,
|
||||
&where_clause,
|
||||
|
||||
@@ -1118,6 +1118,143 @@ fn builds_db2_table_data_large_value_previews_with_bounded_substr() {
|
||||
assert!(!sql.contains("SUBSTR(\"doc\", 1, 4097)"), "must not emit unbounded substr: {sql}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_oracle_table_data_large_value_previews_with_bounded_substr() {
|
||||
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||
database_type: Some(DatabaseType::Oracle),
|
||||
schema: Some("MESPRD".to_string()),
|
||||
table_name: "API_REQUEST_RESPONSE".to_string(),
|
||||
primary_keys: vec!["ID".to_string()],
|
||||
columns: vec!["ID".to_string(), "REQ_PAYLOAD".to_string(), "RESP_PAYLOAD".to_string(), "RAW_DATA".to_string()],
|
||||
column_types: vec!["NUMBER".to_string(), "CLOB".to_string(), "NCLOB".to_string(), "BLOB".to_string()],
|
||||
large_value_preview_size: Some(8192),
|
||||
limit: Some(100),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
// Oracle SQL 结果上限：VARCHAR2 4000 字节（4 字节/字符 → 999+1 字符）、RAW 2000 字节
|
||||
// （RAWTOHEX 翻倍 + "0x" → 1998+1 字节）。外层 ROWNUM 分页按别名重选内层视图的列，
|
||||
// marker 列必须透传到最外层，客户端才能剥离并登记大值单元格。
|
||||
assert!(
|
||||
sql.starts_with(
|
||||
"SELECT \"ID\", \"REQ_PAYLOAD\", \"__DBX_LARGE_VALUE_BYTES_T_1\", \"RESP_PAYLOAD\", \"__DBX_LARGE_VALUE_BYTES_T_2\", \"RAW_DATA\", \"__DBX_LARGE_VALUE_BYTES_B_3\" FROM"
|
||||
),
|
||||
"outer page projection: {sql}"
|
||||
);
|
||||
assert!(sql.contains("DBMS_LOB.SUBSTR(\"REQ_PAYLOAD\", 1000, 1) AS \"REQ_PAYLOAD\""), "clob preview sql: {sql}");
|
||||
assert!(
|
||||
sql.contains("'T:999:' || DBMS_LOB.GETLENGTH(\"REQ_PAYLOAD\") AS \"__DBX_LARGE_VALUE_BYTES_T_1\""),
|
||||
"clob marker sql: {sql}"
|
||||
);
|
||||
assert!(sql.contains("DBMS_LOB.SUBSTR(\"RESP_PAYLOAD\", 1000, 1) AS \"RESP_PAYLOAD\""), "nclob preview sql: {sql}");
|
||||
assert!(
|
||||
sql.contains("'T:999:' || DBMS_LOB.GETLENGTH(\"RESP_PAYLOAD\") AS \"__DBX_LARGE_VALUE_BYTES_T_2\""),
|
||||
"nclob marker sql: {sql}"
|
||||
);
|
||||
assert!(
|
||||
sql.contains("'0x' || RAWTOHEX(DBMS_LOB.SUBSTR(\"RAW_DATA\", 1999, 1)) AS \"RAW_DATA\""),
|
||||
"blob preview sql: {sql}"
|
||||
);
|
||||
assert!(
|
||||
sql.contains("'B:1998:' || DBMS_LOB.GETLENGTH(\"RAW_DATA\") AS \"__DBX_LARGE_VALUE_BYTES_B_3\""),
|
||||
"blob marker sql: {sql}"
|
||||
);
|
||||
assert!(sql.contains("ROWNUM <= 100"), "rownum page: {sql}");
|
||||
assert!(!sql.contains("SUBSTR(\"ID\""), "pk should not be previewed: {sql}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_oracle_table_data_previews_for_small_budgets() {
|
||||
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||
database_type: Some(DatabaseType::Oracle),
|
||||
table_name: "documents".to_string(),
|
||||
primary_keys: vec!["ID".to_string()],
|
||||
columns: vec!["ID".to_string(), "BODY".to_string()],
|
||||
column_types: vec!["NUMBER".to_string(), "CLOB".to_string()],
|
||||
large_value_preview_size: Some(200),
|
||||
limit: Some(50),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert!(sql.contains("DBMS_LOB.SUBSTR(\"BODY\", 201, 1) AS \"BODY\""), "preview sql: {sql}");
|
||||
assert!(
|
||||
sql.contains("'T:200:' || DBMS_LOB.GETLENGTH(\"BODY\") AS \"__DBX_LARGE_VALUE_BYTES_T_1\""),
|
||||
"marker sql: {sql}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_oracle_table_data_previews_for_rowid_keyed_tables() {
|
||||
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||
database_type: Some(DatabaseType::Oracle),
|
||||
table_name: "keyless_logs".to_string(),
|
||||
table_type: Some("TABLE".to_string()),
|
||||
primary_keys: vec!["__DBX_ROWID".to_string()],
|
||||
include_row_id: true,
|
||||
columns: vec!["PAYLOAD".to_string(), "CREATED_AT".to_string()],
|
||||
column_types: vec!["CLOB".to_string(), "DATE".to_string()],
|
||||
large_value_preview_size: Some(200),
|
||||
limit: Some(100),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
// 无主键基表：内层投影以合成键 ROWIDTOCHAR 开头、FROM 带别名 t，外层按别名重选
|
||||
// __DBX_ROWID 与 marker 列，取全值时按同一合成键回查。marker 下标是结果集位置：
|
||||
// __DBX_ROWID 占第 0 位，PAYLOAD 的 marker 因此是 _T_1（客户端剥离按此校验）。
|
||||
assert!(
|
||||
sql.contains(
|
||||
"SELECT ROWIDTOCHAR(t.ROWID) AS \"__DBX_ROWID\", DBMS_LOB.SUBSTR(\"PAYLOAD\", 201, 1) AS \"PAYLOAD\", 'T:200:' || DBMS_LOB.GETLENGTH(\"PAYLOAD\") AS \"__DBX_LARGE_VALUE_BYTES_T_1\", \"CREATED_AT\" FROM \"keyless_logs\" t"
|
||||
),
|
||||
"inner projection: {sql}"
|
||||
);
|
||||
assert!(
|
||||
sql.starts_with("SELECT \"__DBX_ROWID\", \"PAYLOAD\", \"__DBX_LARGE_VALUE_BYTES_T_1\", \"CREATED_AT\" FROM"),
|
||||
"outer projection: {sql}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_oracle_table_data_previews_on_later_pages() {
|
||||
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||
database_type: Some(DatabaseType::Oracle),
|
||||
table_name: "documents".to_string(),
|
||||
primary_keys: vec!["ID".to_string()],
|
||||
columns: vec!["ID".to_string(), "BODY".to_string()],
|
||||
column_types: vec!["NUMBER".to_string(), "CLOB".to_string()],
|
||||
large_value_preview_size: Some(200),
|
||||
limit: Some(100),
|
||||
offset: Some(200),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
// 第二页起是三层嵌套；中间层 dbx_inner.* 必须把 marker 带到最外层投影。
|
||||
assert!(sql.contains("\"__dbx_row_num\""), "row-number page: {sql}");
|
||||
assert!(
|
||||
sql.starts_with("SELECT \"ID\", \"BODY\", \"__DBX_LARGE_VALUE_BYTES_T_1\" FROM"),
|
||||
"outer projection: {sql}"
|
||||
);
|
||||
assert!(sql.contains("DBMS_LOB.SUBSTR(\"BODY\", 201, 1)"), "inner preview survives: {sql}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skips_oracle_previews_for_columns_without_substr_support() {
|
||||
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||
database_type: Some(DatabaseType::Oracle),
|
||||
table_name: "mixed_objects".to_string(),
|
||||
primary_keys: vec!["ID".to_string()],
|
||||
columns: vec!["ID".to_string(), "DOC".to_string(), "LEGACY".to_string()],
|
||||
column_types: vec!["NUMBER".to_string(), "XMLTYPE".to_string(), "LONG".to_string()],
|
||||
large_value_preview_size: Some(200),
|
||||
limit: Some(100),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
// XMLTYPE / LONG 没有 DBMS_LOB.SUBSTR 重载，保持原样投影（与旧行为一致）。
|
||||
assert!(!sql.contains("DBMS_LOB"), "no substr for xmltype/long: {sql}");
|
||||
assert!(!sql.contains("__DBX_LARGE_VALUE_BYTES_"), "no markers: {sql}");
|
||||
assert!(sql.contains("\"DOC\""), "xmltype column stays projected: {sql}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_postgres_array_types_in_large_value_previews() {
|
||||
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||
|
||||
Reference in new issue
Block a user