feat(sql-file): support ordered local SQL file includes

This commit is contained in:
CSXFanMeng authored and GitHub committed 2026-10-26 11:20:55 +08:00
1 parent 8b1f944248
commit b5b3c771b2
42 files changed
+1007 -8

No files matched your search

@@ -322,6 +322,66 @@ afterEach(() => {
root = undefined;
});
describe("SqlFileExecutionDialog includes", () => {
it("shows included source paths in live statement progress and retained failure details", async () => {
await mountReadyDialog();
const gate = deferred();
mocks.executeSqlFiles.mockImplementationOnce(async (request: { executionId: string }) => {
mocks.progressHandler?.(progress(request.executionId, "running", { statementIndex: 1, statementSummary: "[shared/seed.sql] INSERT INTO missing VALUES(1)" }));
await gate.promise;
mocks.progressHandler?.(progress(request.executionId, "statementFailed", { statementIndex: 1, failureCount: 1, statementSummary: "[shared/seed.sql] INSERT INTO missing VALUES(1)", error: "[shared/seed.sql] no such table: missing" }));
mocks.progressHandler?.(progress(request.executionId, "done", { statementIndex: 1, failureCount: 1 }));
});
findButton("sqlFile.execute").click();
try {
await vi.waitFor(() => expect(root!.textContent).toContain("[shared/seed.sql] INSERT INTO missing"));
} finally {
gate.resolve();
}
await vi.waitFor(() => expect(root!.textContent).toContain("[shared/seed.sql] no such table: missing"));
expect(mocks.trackerTask.sqlFileFailures[0].statementSummary).toContain("[shared/seed.sql]");
});
it("keeps includes opt-in and submits the checked option with the entry paths", async () => {
await mountReadyDialog();
const label = Array.from(root!.querySelectorAll("label")).find((item) => item.textContent?.includes("sqlFile.resolveIncludes"))!;
const input = label.querySelector<HTMLInputElement>('input[type="checkbox"]')!;
expect(input.checked).toBe(false);
expect(input.disabled).toBe(false);
input.click();
await nextTick();
expect(root!.textContent).toContain("sqlFile.resolveIncludesHint");
findButton("sqlFile.execute").click();
await vi.waitFor(() => expect(mocks.executeSqlFiles).toHaveBeenCalledWith(expect.objectContaining({ resolveIncludes: true }), ["/tmp/first.sql", "/tmp/second.sql"]));
expect(input.disabled).toBe(true);
});
it("does not offer server-local includes for prepared Web uploads", async () => {
mocks.desktop = false;
root = document.createElement("div");
document.body.append(root);
app = createApp(SqlFileExecutionDialog, { open: true, prefillPreview: { fileName: "main.sql", filePath: "/server/tmp/main.sql", sizeBytes: 9, preview: "SELECT 1;", canExecuteWithoutSelectedDatabase: true } });
app.mount(root);
await vi.waitFor(() => expect(findButton("sqlFile.execute").disabled).toBe(false));
expect(root.textContent).not.toContain("sqlFile.resolveIncludes");
findButton("sqlFile.execute").click();
await vi.waitFor(() => expect(mocks.executeSqlFiles).toHaveBeenCalledWith(expect.objectContaining({ resolveIncludes: false }), ["/server/tmp/main.sql"]));
});
it("disables includes for compressed SQL files", async () => {
mocks.openFileDialog.mockResolvedValue(["/tmp/backup.sql.gz"]);
root = document.createElement("div");
document.body.append(root);
app = createApp(SqlFileExecutionDialog, { open: true });
app.mount(root);
await vi.waitFor(() => expect(mocks.fetchSqlFileTargetOptions).toHaveBeenCalled());
findButton("sqlFile.browse").click();
await vi.waitFor(() => expect(mocks.previewSqlFile).toHaveBeenCalled());
const label = Array.from(root.querySelectorAll("label")).find((item) => item.textContent?.includes("sqlFile.resolveIncludes"))!;
expect(label.querySelector<HTMLInputElement>("input")!.disabled).toBe(true);
});
});
describe("SqlFileExecutionDialog retries", () => {
async function mountRestoredPreview() {
mocks.desktop = false;
@@ -77,6 +77,7 @@ const isDesktopRuntime = isTauriRuntime();
const fileInput = ref<HTMLInputElement | null>(null);
const previews = ref<SqlFilePreview[]>([]);
const encoding = ref<SqlFileEncoding>("auto");
const resolveIncludes = ref(false);
const previewError = ref("");
let previewGeneration = 0;
let selectedSources: Array<string | File> = [];
@@ -345,6 +346,10 @@ onBeforeUnmount(() => {
});
const restoreSelectedTables = ref(false);
const canResolveIncludes = computed(() => isDesktopRuntime && !restoreSelectedTables.value && previews.value.length > 0 && previews.value.every((item) => !item.packageFilePaths && item.fileName.toLowerCase().endsWith(".sql")));
watch(canResolveIncludes, (allowed) => {
if (!allowed) resolveIncludes.value = false;
});
const backupTables = ref<SqlFileTable[]>([]);
const selectedTableKeys = ref(new Set<string>());
const tableSearch = ref("");
@@ -569,6 +574,7 @@ function resetState() {
const initialTarget = resolveInitialTarget();
previews.value = [];
encoding.value = "auto";
resolveIncludes.value = false;
selectingFile.value = false;
loadingPreview.value = false;
applyTarget(initialTarget);
@@ -903,6 +909,7 @@ async function startExecution() {
...(target.schema ? { schema: target.schema } : {}),
filePath: executionPaths[0]!,
encoding: encoding.value,
resolveIncludes: canResolveIncludes.value && resolveIncludes.value,
continueOnError: continueOnError.value,
...(txnSessionId.value ? { txnSessionId: txnSessionId.value } : {}),
...(restoreSelectedTables.value ? { selectedTables: selectedTables.value.map((table) => ({ ...table })) } : {}),
@@ -1212,6 +1219,11 @@ watch(
{{ t("toolbar.manualTransaction") }}
</label>
<p v-if="manualTransaction" class="text-xs text-muted-foreground">{{ t("sqlFile.manualTransactionHint") }}</p>
<label v-if="isDesktopRuntime" class="flex items-center gap-2 text-xs">
<input v-model="resolveIncludes" type="checkbox" :disabled="executionLocked || !canResolveIncludes" />
{{ t("sqlFile.resolveIncludes") }}
</label>
<p v-if="resolveIncludes" class="text-xs text-muted-foreground">{{ t("sqlFile.resolveIncludesHint") }}</p>
<button type="button" class="flex items-center gap-2 text-xs text-left" :disabled="executionLocked || manualTransaction" @click="continueOnError = !continueOnError">
<CheckSquare v-if="continueOnError" class="w-3.5 h-3.5 text-primary shrink-0" />
<Square v-else class="w-3.5 h-3.5 text-muted-foreground/40 shrink-0" />
@@ -45,6 +45,7 @@ const under = (namespace: string, keys: string[]) => keys.map((key) => `${namesp
// leaks the raw key path, and a key missing from a locale silently shows
// English.
const FEATURE_MESSAGES: FeatureMessages[] = [
{ feature: "SQL file includes", keys: under("sqlFile", ["resolveIncludes", "resolveIncludesHint"]), translated: true },
{ feature: "AstraFlow provider", keys: ["ai.astraflowDescription"], translated: true },
{ feature: "AI HTML preview (#6467)", keys: under("ai", ["htmlPreviewLabel", "htmlExpandPreview", "htmlExpandPreviewHint", "htmlSaveSafe", "htmlSaveFailed", "htmlCopySource", "htmlCopyRiskBody", "htmlCopyRiskAccept", "htmlCopyRiskRemember", "htmlCopyRiskToast"]) },
{ feature: "AI conversation export (#6467)", keys: under("ai", ["exportConversation", "conversationExportMarkdown", "conversationExportHtml", "conversationRoleUser", "conversationRoleAssistant", "conversationFailedMarker", "conversationExportEmpty"]) },
@@ -231,6 +232,11 @@ describe("feature message rendering", () => {
});
describe("feature message content rules", () => {
it.each(ALL_LOCALES)("%s renders include syntax literally instead of resolving an i18n link", (locale) => {
const i18n = createI18n({ legacy: false, locale, messages: { [locale]: declared[locale] } });
expect(i18n.global.t("sqlFile.resolveIncludesHint")).toContain("-- @include");
i18n.dispose();
});
it.each(EXCEPT_AZ_TR)("%s keeps the Dameng validity labels as the database literals", (locale) => {
const objects = (locale === "en" ? en : declared[locale]).objects as Messages | undefined;
expect(objects?.validStatus ?? en.objects.validStatus).toBe("VALID");
+2
View File
@@ -6868,6 +6868,8 @@ export default withEnglishFallback({
exportData: "Məlumatları ixrac et",
},
sqlFile: {
resolveIncludes: "SQL fayl istinadlarını həll et",
resolveIncludesHint: "-- {'@'}include \"path.sql\" dəstəklənir; yalnız əsas skript qovluğundakı .sql fayllarına icazə verilir.",
encodingHint: "Bütün seçilmiş fayllara tətbiq olunur. BOM varsa, onun göstərdiyi kodlaşdırmaya üstünlük verilir.",
restoreScope: "Bərpa əhatəsi",
restoreAll: "Bütün məzmun",
+2
View File
@@ -8905,6 +8905,8 @@ export default {
exportData: "Export Data",
},
sqlFile: {
resolveIncludes: "Resolve SQL file includes",
resolveIncludesHint: "Supports -- {'@'}include \"path.sql\"; only .sql files within the entry script directory are allowed.",
encodingHint: "Applies to all selected files. A byte order mark (BOM), if present, takes precedence.",
manualTransactionHint: "Changes wait for Commit or Rollback. Errors roll back. DDL and COMMIT inside the file can still commit changes.",
manualCancelHint: "Cancel waits for the current statement, then rolls back uncommitted changes.",
+2
View File
@@ -7890,6 +7890,8 @@ export default withEnglishFallback({
exportData: "Exportar datos",
},
sqlFile: {
resolveIncludes: "Resolver inclusiones de archivos SQL",
resolveIncludesHint: "Admite -- {'@'}include \"ruta.sql\"; solo se permiten archivos .sql dentro del directorio del script principal.",
encodingHint: "Se aplica a todos los archivos seleccionados. Si hay una marca BOM, su codificación tiene prioridad.",
manualTransactionHint: "Los cambios esperan confirmación o reversión. Los errores provocan una reversión. DDL y COMMIT dentro del archivo aún pueden confirmar cambios.",
manualCancelHint: "Cancelar espera a que termine la sentencia actual y luego revierte los cambios sin confirmar.",
+2
View File
@@ -7555,6 +7555,8 @@ export default withEnglishFallback({
exportData: "Ekspor Data",
},
sqlFile: {
resolveIncludes: "Proses penyertaan file SQL",
resolveIncludesHint: "Mendukung -- {'@'}include \"path.sql\"; hanya file .sql dalam direktori skrip utama yang diizinkan.",
encodingHint: "Berlaku untuk semua file yang dipilih. Jika ada BOM, pengodean yang ditunjukkannya diutamakan.",
manualTransactionHint: "Perubahan menunggu Commit atau Rollback. Galat akan di-rollback. DDL dan COMMIT di dalam file tetap dapat melakukan commit perubahan.",
manualCancelHint: "Pembatalan menunggu pernyataan saat ini selesai, lalu me-rollback perubahan yang belum di-commit.",
+2
View File
@@ -7683,6 +7683,8 @@ export default withEnglishFallback({
exportData: "Esporta dati",
},
sqlFile: {
resolveIncludes: "Risolvi inclusioni di file SQL",
resolveIncludesHint: "Supporta -- {'@'}include \"percorso.sql\"; sono ammessi solo file .sql nella cartella dello script principale.",
encodingHint: "Si applica a tutti i file selezionati. Se presente, la codifica indicata dal BOM ha la precedenza.",
manualTransactionHint: "Le modifiche attendono commit o rollback. Gli errori causano il rollback. DDL e COMMIT nel file possono comunque confermare le modifiche.",
manualCancelHint: "L'annullamento attende la fine dell'istruzione corrente, poi esegue il rollback delle modifiche non confermate.",
+2
View File
@@ -7703,6 +7703,8 @@ export default withEnglishFallback({
exportData: "データをエクスポート",
},
sqlFile: {
resolveIncludes: "SQL ファイルの参照を解決",
resolveIncludesHint: "-- {'@'}include \"path.sql\" に対応。参照はエントリースクリプトのディレクトリ内の .sql ファイルに限定されます。",
encodingHint: "選択したすべてのファイルに適用します。BOM がある場合は、そのエンコーディングを優先します。",
manualTransactionHint: "変更は手動でコミットまたはロールバックするまで保留されます。エラー時はロールバックします。ファイル内の DDL や COMMIT により変更が確定する場合があります。",
manualCancelHint: "キャンセルすると現在の文の終了を待ち、未コミットの変更をロールバックします。",
+2
View File
@@ -7717,6 +7717,8 @@ export default withEnglishFallback({
exportData: "데이터 내보내기",
},
sqlFile: {
resolveIncludes: "SQL 파일 포함 처리",
resolveIncludesHint: "-- {'@'}include \"path.sql\" 지원. 시작 스크립트 디렉터리 내의 .sql 파일만 허용됩니다.",
encodingHint: "선택한 모든 파일에 적용됩니다. BOM이 있으면 해당 인코딩을 우선합니다.",
manualTransactionHint: "변경 사항은 수동 커밋 또는 롤백을 기다립니다. 오류가 발생하면 롤백합니다. 파일의 DDL과 COMMIT은 변경 사항을 커밋할 수 있습니다.",
manualCancelHint: "취소하면 현재 구문이 끝날 때까지 기다린 후 커밋되지 않은 변경 사항을 롤백합니다.",
+2
View File
@@ -7612,6 +7612,8 @@ export default withEnglishFallback({
exportData: "Eksportuj dane",
},
sqlFile: {
resolveIncludes: "Rozwiązuj dołączanie plików SQL",
resolveIncludesHint: "Obsługuje -- {'@'}include \"path.sql\"; dozwolone są tylko pliki .sql w katalogu skryptu wejściowego.",
manualTransactionHint: "Zmiany oczekują na Commit lub Rollback. W przypadku błędów następuje wycofanie. DDL i jawny COMMIT w pliku mogą nadal zatwierdzać zmiany.",
manualCancelHint: "Anulowanie czeka na bieżącą instrukcję, a następnie wycofuje niezatwierdzone zmiany.",
pendingTransaction: "Oczekiwanie na zatwierdzenie lub wycofanie",
+2
View File
@@ -7682,6 +7682,8 @@ export default withEnglishFallback({
exportData: "Exportar dados",
},
sqlFile: {
resolveIncludes: "Resolver inclusões de arquivos SQL",
resolveIncludesHint: "Suporta -- {'@'}include \"caminho.sql\"; permite apenas arquivos .sql no diretório do script principal.",
encodingHint: "Aplica-se a todos os arquivos selecionados. Se houver uma marca BOM, sua codificação terá prioridade.",
manualTransactionHint: "As alterações aguardam confirmação ou reversão. Erros provocam reversão. DDL e COMMIT no arquivo ainda podem confirmar alterações.",
manualCancelHint: "Cancelar aguarda o fim da instrução atual e depois reverte as alterações não confirmadas.",
+2
View File
@@ -9158,6 +9158,8 @@ export default withEnglishFallback({
exportData: "Экспорт данных",
},
sqlFile: {
resolveIncludes: "Обрабатывать включения SQL-файлов",
resolveIncludesHint: "Поддерживает -- {'@'}include \"path.sql\"; разрешены только файлы .sql в каталоге входного скрипта.",
encodingHint: "Применяется ко всем выбранным файлам. При наличии BOM используется указанная им кодировка.",
manualTransactionHint: "Изменения ожидают Commit или Rollback. При ошибках выполняется откат. DDL и COMMIT внутри файла всё равно могут зафиксировать изменения.",
manualCancelHint: "Отмена ожидает завершения текущего оператора, затем откатывает незафиксированные изменения.",
+2
View File
@@ -9510,6 +9510,8 @@ export default withEnglishFallback({
exportData: "ส่งออกข้อมูล",
},
sqlFile: {
resolveIncludes: "ประมวลผลการรวมไฟล์ SQL",
resolveIncludesHint: "รองรับ -- {'@'}include \"path.sql\" โดยอนุญาตเฉพาะไฟล์ .sql ภายในไดเรกทอรีของสคริปต์หลัก",
encodingHint: "ใช้กับไฟล์ทั้งหมดที่เลือกในครั้งนี้ หากไฟล์มีเครื่องหมายลำดับไบต์ (BOM) จะใช้การเข้ารหัสที่ระบุโดย BOM เป็นอันดับแรก",
manualTransactionHint: "การเปลี่ยนแปลงจะรอ Commit หรือ Rollback ข้อผิดพลาดจะ Rollback แต่ DDL และ COMMIT ภายในไฟล์ยังสามารถ Commit การเปลี่ยนแปลงได้",
manualCancelHint: "การยกเลิกจะรอ statement ปัจจุบัน แล้ว Rollback การเปลี่ยนแปลงที่ยังไม่ Commit",
+2
View File
@@ -6748,6 +6748,8 @@ export default withEnglishFallback({
exportData: "Veri Dışa Aktar",
},
sqlFile: {
resolveIncludes: "SQL dosyası eklemelerini çöz",
resolveIncludesHint: "-- {'@'}include \"path.sql\" desteklenir; yalnızca giriş betiği dizinindeki .sql dosyalarına izin verilir.",
encodingHint: "Seçilen tüm dosyalara uygulanır. BOM varsa belirttiği kodlama önceliklidir.",
restoreScope: "Geri yükleme kapsamı",
restoreAll: "Tüm içerik",
+2
View File
@@ -8872,6 +8872,8 @@ export default withEnglishFallback({
exportData: "导出数据",
},
sqlFile: {
resolveIncludes: "解析 SQL 文件引用",
resolveIncludesHint: "支持 -- {'@'}include \"路径.sql\";引用仅限入口脚本目录内的 .sql 文件。",
encodingHint: "应用于本次选择的所有文件;文件含字节序标记(BOM)时,优先采用 BOM 指定的编码。",
manualTransactionHint: "修改等待手动提交或回滚,出错时回滚。文件中的 DDL 和 COMMIT 仍可能提交修改。",
manualCancelHint: "取消会等待当前语句结束,再回滚未提交的修改。",
+2
View File
@@ -6819,6 +6819,8 @@ export default withEnglishFallback({
exportData: "匯出資料",
},
sqlFile: {
resolveIncludes: "解析 SQL 檔案引用",
resolveIncludesHint: "支援 -- {'@'}include \"路徑.sql\";引用僅限入口指令碼目錄內的 .sql 檔案。",
encodingHint: "套用至本次選取的所有檔案;檔案含位元組順序標記(BOM)時,優先採用 BOM 指定的編碼。",
manualTransactionHint: "修改等待手動提交或回復,出錯時回復。檔案中的 DDL 和 COMMIT 仍可能提交修改。",
manualCancelHint: "取消會等待目前陳述式結束,再回復未提交的修改。",
+1
View File
@@ -5894,6 +5894,7 @@ export type SqlFileStatus = "started" | "running" | "statementDone" | "statement
export type SqlFileEncoding = "auto" | "utf8" | "gbk" | "utf16Le" | "utf16Be";
export interface SqlFileRequest {
resolveIncludes?: boolean;
encoding?: SqlFileEncoding;
executionId: string;
connectionId: string;
+130 -6
View File
@@ -32,6 +32,7 @@ use crate::sql::{
};
use crate::types::QueryResult;
mod includes;
mod table_restore;
pub use table_restore::SqlFileTable;
use table_restore::TableRestoreFilter;
@@ -734,8 +735,13 @@ pub async fn execute_sql_file_content(
file_content: &str,
token: CancellationToken,
started_at: Instant,
emit: impl FnMut(SqlFileProgress),
mut emit: impl FnMut(SqlFileProgress),
) -> Result<(), String> {
if request.resolve_includes {
let error = "SQL includes require local .sql file execution, not inline content".to_string();
emit(sql_file_error_progress(&request.execution_id, started_at, error.clone()));
return Err(error);
}
with_sql_file_transaction(
state,
request,
@@ -878,9 +884,74 @@ async fn execute_sql_file_paths_with_options(
token: CancellationToken,
started_at: Instant,
zip_package: bool,
emit: impl FnMut(SqlFileProgress),
mut emit: impl FnMut(SqlFileProgress),
) -> Result<(), String> {
with_sql_file_transaction(
let prepared;
let mut expanded_request;
let expanded_paths;
let (request, file_paths, sources) = if request.resolve_includes {
let mut preparing =
sql_file_progress(&request.execution_id, SqlFileStatus::Running, 0, 0, 0, 0, started_at, "", None);
preparing.phase = Some(SqlFilePhase::Preparing);
emit(preparing);
// Erase preparation's future type so external callers keep their default recursion limit.
let prepare: futures::future::BoxFuture<'_, Result<includes::PreparedSqlIncludes, String>> = Box::pin(async {
if zip_package || request.selected_tables.is_some() {
return Err("SQL includes cannot be combined with ZIP packages or selected-table restore".to_string());
}
let target = sql_file_import_target(state, &request.connection_id, &request.database).await;
let options = target
.as_ref()
.map(|target| {
SqlParsingOptions::for_database_type_and_compatibility(
target.db_type,
target.compatibility_mode.as_mode_str(),
)
})
.unwrap_or_default();
let normalize_mysql_binary_literals = target.as_ref().is_some_and(|target| {
crate::sql::is_mysql_compatible_import_target(&target.db_type, target.driver_profile.as_deref())
});
includes::prepare_sql_file_includes(
file_paths,
request.encoding,
target.as_ref().map(|target| target.db_type),
options,
normalize_mysql_binary_literals,
&token,
)
.await
});
prepared = match prepare.await {
Ok(prepared) => prepared,
Err(error) => {
if token.is_cancelled() {
emit(sql_file_progress(
&request.execution_id,
SqlFileStatus::Cancelled,
0,
0,
0,
0,
started_at,
"",
None,
));
return Ok(());
}
emit(sql_file_error_progress(&request.execution_id, started_at, error.clone()));
return Err(error);
}
};
expanded_paths = prepared.paths.iter().map(|path| path.as_path()).collect::<Vec<_>>();
expanded_request = request.clone();
expanded_request.encoding = SqlFileEncoding::Utf8;
(&expanded_request, expanded_paths.as_slice(), Some(&prepared))
} else {
(request, file_paths, None)
};
let mut last_source = None;
let result = with_sql_file_transaction(
state,
request,
&token,
@@ -891,10 +962,26 @@ async fn execute_sql_file_paths_with_options(
token.clone(),
started_at,
zip_package,
emit,
|mut progress| {
if let Some(source) = sources.and_then(|sources| sources.source_for(&progress)) {
let prefix = format!("[{source}] ");
if !progress.statement_summary.is_empty() {
progress.statement_summary.insert_str(0, &prefix);
}
if let Some(error) = progress.error.as_mut() {
error.insert_str(0, &prefix);
}
last_source = Some(source.to_string());
}
emit(progress);
},
)),
)
.await
.await;
result.map_err(|error| match last_source {
Some(source) => format!("[{source}] {error}"),
None => error,
})
}
async fn execute_sql_file_paths_inner(
@@ -927,7 +1014,12 @@ async fn execute_sql_file_paths_inner(
})
.unwrap_or_default();
let database_type = import_target.as_ref().map(|target| target.db_type);
let max_insert_batch_statements = sql_file_import_max_insert_batch_statements(zip_package, database_type);
// Included statements retain individual source/error boundaries, even across adjacent INSERTs.
let max_insert_batch_statements = if request.resolve_includes {
Some(1)
} else {
sql_file_import_max_insert_batch_statements(zip_package, database_type)
};
let mut progress = SqlFileExecutionProgress::new();
// Validate the entire dump before any SQL reaches the database. This also
// identifies view placeholders whose final CREATE VIEW appears later.
@@ -1941,6 +2033,24 @@ impl StreamingSqlFileSplitter {
self.0.push_chunk(chunk)
}
fn can_start_file_directive(&self) -> bool {
match &self.0 {
StreamingSqlFileSplitterKind::Statements(splitter) => splitter.can_start_file_directive(),
StreamingSqlFileSplitterKind::SqlServerBatches(splitter) => {
matches!(splitter.lexical_state, SqlServerLexicalState::Normal)
&& splitter.partial_line.trim().is_empty()
}
}
}
fn is_at_file_directive_boundary(&self) -> bool {
self.can_start_file_directive()
&& match &self.0 {
StreamingSqlFileSplitterKind::Statements(splitter) => splitter.is_at_file_directive_boundary(),
StreamingSqlFileSplitterKind::SqlServerBatches(splitter) => !splitter.batch_has_sql,
}
}
pub(crate) fn finish(self) -> Vec<SqlStatementWithControl> {
self.0.finish()
}
@@ -1981,6 +2091,7 @@ struct SqlServerBatchSplitter {
partial_line: String,
lexical_state: SqlServerLexicalState,
flush_semicolons: bool,
batch_has_sql: bool,
}
impl Default for SqlServerBatchSplitter {
@@ -2007,6 +2118,7 @@ impl SqlServerBatchSplitter {
partial_line: String::new(),
lexical_state: SqlServerLexicalState::default(),
flush_semicolons,
batch_has_sql: false,
}
}
@@ -2038,6 +2150,13 @@ impl SqlServerBatchSplitter {
}
let mut chars = line.chars().peekable();
while let Some(ch) = chars.next() {
if matches!(self.lexical_state, SqlServerLexicalState::Normal)
&& !ch.is_whitespace()
&& !(ch == '-' && chars.peek() == Some(&'-'))
&& !(ch == '/' && chars.peek() == Some(&'*'))
{
self.batch_has_sql = true;
}
self.batch.push(ch);
match self.lexical_state {
SqlServerLexicalState::Normal => match ch {
@@ -2097,6 +2216,7 @@ impl SqlServerBatchSplitter {
batches.push(batch.to_string());
}
self.batch.clear();
self.batch_has_sql = false;
self.lexical_state = SqlServerLexicalState::Normal;
}
}
@@ -3066,6 +3186,7 @@ mod tests {
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
};
// Non-MySQL auto-commit files must keep BEGIN/COMMIT on one connection.
@@ -3094,6 +3215,7 @@ mod tests {
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
};
let mut events = Vec::new();
@@ -3450,6 +3572,7 @@ mod tests {
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
};
let token = CancellationToken::new();
@@ -4119,6 +4242,7 @@ mod tests {
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
};
let mut progress = Vec::new();
@@ -0,0 +1,572 @@
use std::path::{Component, Path, PathBuf};
use tokio::io::AsyncWriteExt;
use tokio_util::sync::CancellationToken;
use super::{
DatabaseType, SqlFileEncoding, SqlFileProgress, SqlFileStreamDecoder, SqlParsingOptions, StreamingSqlFileSplitter,
};
const MAX_DEPTH: usize = 16;
const MAX_FILES: usize = 256;
const MAX_BYTES: u64 = 1024 * 1024 * 1024;
const MAX_LINE_BYTES: usize = 16 * 1024 * 1024;
pub(super) struct PreparedSqlIncludes {
// Keep the private expanded scripts alive until execution has finished.
_directory: tempfile::TempDir,
pub paths: Vec<PathBuf>,
sources: Vec<SourceSpan>,
}
struct SourceSpan {
first: usize,
last: usize,
source: String,
}
impl PreparedSqlIncludes {
pub(super) fn source_for(&self, progress: &SqlFileProgress) -> Option<&str> {
if progress.file_index.is_some()
|| progress.statement_index == 0
|| (progress.statement_summary.is_empty() && progress.error.is_none())
{
return None;
}
let index = self.sources.partition_point(|span| span.last < progress.statement_index);
self.sources.get(index).filter(|span| span.first <= progress.statement_index).map(|span| span.source.as_str())
}
}
struct Expansion<'a> {
root: PathBuf,
active: Vec<PathBuf>,
files: usize,
bytes: u64,
encoding: SqlFileEncoding,
normalize_mysql_binary_literals: bool,
token: &'a CancellationToken,
splitter: StreamingSqlFileSplitter,
statement_count: usize,
sources: Vec<SourceSpan>,
}
fn directive(line: &str) -> Result<Option<&str>, String> {
let line = line.trim();
let Some(value) = line.strip_prefix("-- @include") else { return Ok(None) };
if !value.is_empty() && !value.starts_with(char::is_whitespace) {
return Ok(None);
}
let value = value.trim();
let path = if value.starts_with('"') || value.starts_with('\'') {
let quote = value.chars().next().unwrap();
value.strip_prefix(quote).and_then(|v| v.strip_suffix(quote)).unwrap_or("")
} else {
value
};
if path.is_empty() || path.contains(['\0', '\r', '\n']) {
return Err("Expected -- @include \"relative/path.sql\"".to_string());
}
Ok(Some(path))
}
impl Expansion<'_> {
fn record_statements(&mut self, count: usize) {
if count == 0 {
return;
}
let source = self
.active
.last()
.expect("SQL text is written from an active source file")
.strip_prefix(&self.root)
.expect("source is confined to the root")
.to_string_lossy()
.replace('\\', "/")
.chars()
.map(|ch| if ch.is_control() { '\u{fffd}' } else { ch })
.collect::<String>();
let first = self.statement_count + 1;
self.statement_count += count;
// Store ranges per file visit, not an unbounded filename per SQL statement.
if let Some(previous) = self.sources.last_mut().filter(|span| span.source == source && span.last + 1 == first) {
previous.last = self.statement_count;
} else {
self.sources.push(SourceSpan { first, last: self.statement_count, source });
}
}
async fn write(&mut self, output: &mut tokio::fs::File, text: &str) -> Result<(), String> {
self.bytes = self.bytes.saturating_add(text.len() as u64);
if self.bytes > MAX_BYTES {
return Err("SQL includes exceed the 1 GiB expanded size limit".to_string());
}
let count = self.splitter.push_chunk(text).len();
self.record_statements(count);
output.write_all(text.as_bytes()).await.map_err(|error| error.to_string())
}
async fn line(&mut self, source: &Path, output: &mut tokio::fs::File, line: &str) -> Result<(), String> {
if self.token.is_cancelled() {
return Err("SQL include preparation cancelled".to_string());
}
if self.splitter.can_start_file_directive() {
if let Some(relative) = directive(line)? {
if !self.splitter.is_at_file_directive_boundary() {
return Err("SQL include directives must appear between complete statements".to_string());
}
// Normalize Windows-style relative paths on every desktop platform.
let normalized = relative.replace('\\', "/");
let relative = Path::new(&normalized);
if relative.has_root()
|| relative.is_absolute()
|| normalized.contains(':')
|| relative.components().any(|part| matches!(part, Component::ParentDir | Component::Prefix(_)))
|| !relative
.extension()
.and_then(|ext| ext.to_str())
.is_some_and(|ext| ext.eq_ignore_ascii_case("sql"))
{
return Err("SQL includes require a relative .sql path without parent traversal".to_string());
}
let target = source.parent().unwrap().join(relative);
Box::pin(self.file(&target, output)).await?;
self.write(output, "\n").await?;
return Ok(());
}
}
self.write(output, line).await
}
async fn file(&mut self, source: &Path, output: &mut tokio::fs::File) -> Result<(), String> {
if self.token.is_cancelled() {
return Err("SQL include preparation cancelled".to_string());
}
let source = tokio::fs::canonicalize(source).await.map_err(|error| format!("{}: {error}", source.display()))?;
if !source.starts_with(&self.root) {
return Err(format!("SQL include escapes the entry script directory: {}", source.display()));
}
if self.active.contains(&source) {
return Err(format!("Cyclic SQL include: {}", source.display()));
}
self.files += 1;
if self.files > MAX_FILES || self.active.len() >= MAX_DEPTH {
return Err("SQL includes exceed the 256-file or 16-level limit".to_string());
}
let metadata = tokio::fs::metadata(&source).await.map_err(|error| error.to_string())?;
if !metadata.is_file() {
return Err(format!("SQL include is not a regular file: {}", source.display()));
}
if metadata.len() > MAX_BYTES {
return Err("SQL include input exceeds the 1 GiB limit".to_string());
}
self.active.push(source.clone());
let mut decoder = SqlFileStreamDecoder::open_for_import(
&source,
self.encoding.to_encoding(),
self.normalize_mysql_binary_literals,
None,
)
.await?;
let mut pending = String::new();
let mut line_number = 0;
while let Some(chunk) = decoder.next_chunk().await? {
if self.token.is_cancelled() {
return Err("SQL include preparation cancelled".to_string());
}
pending.push_str(&chunk);
let mut start = 0;
while let Some(end) = pending[start..].find('\n') {
let end = start + end + 1;
if end - start > MAX_LINE_BYTES {
return Err("SQL includes contain a line larger than 16 MiB".to_string());
}
line_number += 1;
self.line(&source, output, &pending[start..end])
.await
.map_err(|error| format!("{}:{line_number}: {error}", source.display()))?;
start = end;
}
pending.drain(..start);
if pending.len() > MAX_LINE_BYTES {
return Err("SQL includes contain a line larger than 16 MiB".to_string());
}
}
if !pending.is_empty() {
self.line(&source, output, &pending)
.await
.map_err(|error| format!("{}:{}: {error}", source.display(), line_number + 1))?;
}
// A trailing line comment ends here; unterminated quoted SQL does not.
self.write(output, "\n").await?;
if self.active.len() > 1 && !self.splitter.is_at_file_directive_boundary() {
return Err(format!("Included SQL must end with a complete statement delimiter: {}", source.display()));
}
self.active.pop();
Ok(())
}
}
pub(super) async fn prepare_sql_file_includes(
paths: &[&Path],
encoding: SqlFileEncoding,
database_type: Option<DatabaseType>,
options: SqlParsingOptions,
normalize_mysql_binary_literals: bool,
token: &CancellationToken,
) -> Result<PreparedSqlIncludes, String> {
if paths.is_empty() {
return Err("No SQL files selected".to_string());
}
let directory =
tempfile::Builder::new().prefix("dbx-sql-includes-").tempdir().map_err(|error| error.to_string())?;
let mut expanded = Vec::new();
let mut total_bytes: u64 = 0;
let mut total_files = 0;
let mut total_statements = 0;
let mut sources = Vec::new();
for (index, path) in paths.iter().enumerate() {
if !path.extension().and_then(|ext| ext.to_str()).is_some_and(|ext| ext.eq_ignore_ascii_case("sql")) {
return Err("SQL includes require uncompressed .sql entry files".to_string());
}
let source = tokio::fs::canonicalize(path).await.map_err(|error| error.to_string())?;
let folder = directory.path().join(index.to_string());
tokio::fs::create_dir(&folder).await.map_err(|error| error.to_string())?;
let target = folder.join(path.file_name().ok_or("Missing SQL file name")?);
let mut output = tokio::fs::File::create(&target).await.map_err(|error| error.to_string())?;
let mut expansion = Expansion {
root: source.parent().ok_or("Missing SQL file directory")?.to_path_buf(),
active: Vec::new(),
files: total_files,
bytes: total_bytes,
encoding,
normalize_mysql_binary_literals,
token,
splitter: StreamingSqlFileSplitter::new(database_type, options, false),
statement_count: total_statements,
sources: Vec::new(),
};
expansion.file(&source, &mut output).await?;
expansion.active.push(source.clone());
let splitter =
std::mem::replace(&mut expansion.splitter, StreamingSqlFileSplitter::new(database_type, options, false));
expansion.record_statements(splitter.finish().len());
output.flush().await.map_err(|error| error.to_string())?;
total_bytes = expansion.bytes;
total_files = expansion.files;
total_statements = expansion.statement_count;
sources.extend(expansion.sources);
expanded.push(target);
}
Ok(PreparedSqlIncludes { _directory: directory, paths: expanded, sources })
}
#[cfg(test)]
mod tests {
use super::*;
use crate::models::connection::DatabaseType;
#[tokio::test]
async fn executor_expands_before_dispatch_and_preserves_session_and_error_policy() {
use crate::connection::{AppState, PoolKind};
use crate::sql::{SqlFileRequest, SqlFileStatus};
use std::time::Instant;
let dir = tempfile::tempdir().unwrap();
let storage = crate::persistence::test_storage::open(&dir.path().join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let database_path = dir.path().join("queries.sqlite");
let config = serde_json::from_value(serde_json::json!({
"id": "include-conn", "name": "Includes", "db_type": "sqlite",
"host": database_path, "port": 0, "username": "", "password": "", "database": null
}))
.unwrap();
state.configs.write().await.insert("include-conn".into(), config);
let pool = crate::db::sqlite::connect_path_create_if_missing(database_path.to_str().unwrap()).await.unwrap();
let root = dir.path().join("main.sql");
let mut request: SqlFileRequest = serde_json::from_value(serde_json::json!({
"executionId": "includes-run", "connectionId": "include-conn", "database": "",
"filePath": root, "continueOnError": false, "resolveIncludes": true
}))
.unwrap();
for (script, continues, expected) in [
(
"CREATE TABLE shared(v INTEGER);\nCREATE TEMP TABLE session_values(v INTEGER);\n-- @include child.sql\nINSERT INTO shared SELECT v FROM session_values;\nINSERT INTO shared VALUES(2);",
false,
SqlFileStatus::Done,
),
("INSERT INTO shared VALUES(3);\n-- @include missing.sql", true, SqlFileStatus::Error),
("-- @include failure.sql\nINSERT INTO shared VALUES(4);", false, SqlFileStatus::Error),
("-- @include failure.sql\nINSERT INTO shared VALUES(5);", true, SqlFileStatus::Done),
] {
std::fs::write(&root, script).unwrap();
std::fs::write(dir.path().join("child.sql"), "INSERT INTO session_values VALUES(1);").unwrap();
std::fs::write(dir.path().join("failure.sql"), "INSERT INTO missing_table VALUES(0);").unwrap();
let session_key = state.session_pool_key_for_test("include-conn", None, "sql-file-includes-run").await;
state
.update_connection_pools(|pools| {
pools.insert("include-conn".into(), PoolKind::Sqlite(pool.clone()));
pools.insert(session_key, PoolKind::Sqlite(pool.clone()));
})
.await;
request.continue_on_error = continues;
let mut events = Vec::new();
let result = super::super::execute_sql_file_path(
&state,
&request,
&root,
CancellationToken::new(),
Instant::now(),
|event| events.push(event),
)
.await;
assert_eq!(result.is_err(), expected == SqlFileStatus::Error, "{result:?}");
assert_eq!(events.last().unwrap().status, expected);
if script.contains("failure.sql") {
let failure = events.iter().find(|event| event.status == SqlFileStatus::StatementFailed).unwrap();
assert!(failure.statement_summary.starts_with("[failure.sql] "), "{failure:?}");
assert!(failure.error.as_deref().unwrap().starts_with("[failure.sql] "));
if let Err(error) = &result { assert!(error.starts_with("[failure.sql] "), "{error}"); }
}
if continues && script.contains("failure.sql") {
assert_eq!(events.last().unwrap().failure_count, 1, "{events:?}");
assert_eq!(events.last().unwrap().success_count, 1, "{events:?}");
}
}
let rows = crate::db::sqlite::execute_query(&pool, "SELECT v FROM shared ORDER BY v").await.unwrap().rows;
assert_eq!(rows, vec![vec![serde_json::json!(1)], vec![serde_json::json!(2)], vec![serde_json::json!(5)]]);
}
async fn expand(path: &Path) -> Result<PreparedSqlIncludes, String> {
prepare_sql_file_includes(
&[path],
SqlFileEncoding::Auto,
Some(DatabaseType::Sqlite),
SqlParsingOptions::for_database_type_and_compatibility(DatabaseType::Sqlite, None),
false,
&CancellationToken::new(),
)
.await
}
#[tokio::test]
async fn sqlserver_includes_share_go_batch_boundaries_and_source_order() {
let dir = tempfile::tempdir().unwrap();
std::fs::create_dir(dir.path().join("shared")).unwrap();
let root = dir.path().join("main.sql");
std::fs::write(&root, "SELECT 0;\nGO\n-- @include shared/seed.sql\nSELECT 2;\nGO\n").unwrap();
std::fs::write(dir.path().join("shared/seed.sql"), "SELECT '-- @include absent.sql';\nGO").unwrap();
let options = SqlParsingOptions::for_database_type_and_compatibility(DatabaseType::SqlServer, None);
let result = prepare_sql_file_includes(
&[&root],
SqlFileEncoding::Auto,
Some(DatabaseType::SqlServer),
options,
false,
&CancellationToken::new(),
)
.await
.unwrap();
let expanded = std::fs::read_to_string(&result.paths[0]).unwrap();
let mut splitter = StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), options, false);
let mut statements = splitter.push_chunk(&expanded);
statements.extend(splitter.finish());
assert_eq!(statements.len(), 3);
assert!(statements[0].sql.contains("SELECT 0"));
assert!(statements[1].sql.contains("absent.sql"));
assert!(statements[2].sql.contains("SELECT 2"));
assert_eq!(
result.sources.iter().map(|span| span.source.as_str()).collect::<Vec<_>>(),
["main.sql", "shared/seed.sql", "main.sql"]
);
}
#[tokio::test]
async fn mysql_non_comment_dashes_cannot_hide_pending_sql_before_include() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().join("main.sql");
std::fs::write(&root, "--foo\n-- @include absent.sql\n").unwrap();
let options = SqlParsingOptions::for_database_type_and_compatibility(DatabaseType::Mysql, None);
let error = prepare_sql_file_includes(
&[&root],
SqlFileEncoding::Auto,
Some(DatabaseType::Mysql),
options,
true,
&CancellationToken::new(),
)
.await
.err()
.unwrap();
assert!(error.contains("between complete statements"), "{error}");
std::fs::write(&root, "-- real comment\n-- @include child.sql\n").unwrap();
std::fs::write(dir.path().join("child.sql"), "SELECT 1;\n").unwrap();
assert!(prepare_sql_file_includes(
&[&root],
SqlFileEncoding::Auto,
Some(DatabaseType::Mysql),
options,
true,
&CancellationToken::new()
)
.await
.is_ok());
}
#[tokio::test]
async fn provenance_ranges_stay_bounded_and_distinguish_repeated_sources() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().join("main.sql");
std::fs::write(
&root,
format!("{}-- @include child.sql\n{}", "SELECT 1;\n".repeat(1000), "SELECT 3;\n".repeat(1000)),
)
.unwrap();
std::fs::write(dir.path().join("child.sql"), "SELECT 2;\n".repeat(1000)).unwrap();
let result = expand(&root).await.unwrap();
assert_eq!(result.sources.len(), 3);
assert_eq!((result.sources[0].first, result.sources[0].last), (1, 1000));
assert_eq!((result.sources[1].first, result.sources[1].last), (1001, 2000));
assert_eq!((result.sources[2].first, result.sources[2].last), (2001, 3000));
assert_eq!(result.sources[1].source, "child.sql");
}
#[tokio::test]
async fn nested_repeated_includes_execute_in_place_and_cleanup_private_output() {
let dir = tempfile::tempdir().unwrap();
std::fs::create_dir(dir.path().join("shared")).unwrap();
let root = dir.path().join("main.sql");
std::fs::write(&root, "CREATE TABLE t(id INTEGER PRIMARY KEY, value TEXT);\n-- @include \"shared/insert row.sql\"\nINSERT INTO t(value) VALUES('middle');\n-- @include shared/insert row.sql\nINSERT INTO t(value) VALUES('last');").unwrap();
std::fs::write(dir.path().join("shared/insert row.sql"), "-- @include leaf.sql\n").unwrap();
std::fs::write(dir.path().join("shared/leaf.sql"), "INSERT INTO t(value) VALUES('included');").unwrap();
let result = expand(&root).await.unwrap();
let output = result.paths[0].clone();
let sql = std::fs::read_to_string(&output).unwrap();
let db = rusqlite::Connection::open_in_memory().unwrap();
db.execute_batch(&sql).unwrap();
let values = db
.prepare("SELECT value FROM t ORDER BY id")
.unwrap()
.query_map([], |row| row.get::<_, String>(0))
.unwrap()
.collect::<Result<Vec<_>, _>>()
.unwrap();
assert_eq!(values, ["included", "middle", "included", "last"]);
assert!(!sql.contains("@include"));
drop(result);
assert!(!output.exists());
assert!(root.exists());
}
#[tokio::test]
async fn text_inside_literals_and_block_comments_is_not_a_directive() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().join("main.sql");
let sql = "CREATE TABLE t(v TEXT);\n/*\n-- @include absent.sql\n*/\nINSERT INTO t VALUES('text\n-- @include absent.sql\nstill text');\n";
std::fs::write(&root, sql).unwrap();
let result = expand(&root).await.unwrap();
let expanded = std::fs::read_to_string(&result.paths[0]).unwrap();
let db = rusqlite::Connection::open_in_memory().unwrap();
db.execute_batch(&expanded).unwrap();
let value: String = db.query_row("SELECT v FROM t", [], |row| row.get(0)).unwrap();
assert!(value.contains("-- @include absent.sql"));
}
#[tokio::test]
async fn preflight_rejects_missing_cycles_traversal_and_invalid_directives() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().join("main.sql");
for (directive, error) in [
("-- @include missing.sql", "missing.sql"),
("-- @include main.sql", "Cyclic"),
("-- @include ../outside.sql", "relative .sql"),
("-- @include C:\\outside.sql", "relative .sql"),
("-- @include /outside.sql", "relative .sql"),
("-- @include secret.txt", "relative .sql"),
("-- @include \"unclosed.sql", "Expected"),
("-- @include", "Expected"),
] {
std::fs::write(&root, format!("CREATE TABLE never_executed(v);\n{directive}\n")).unwrap();
assert!(expand(&root).await.err().unwrap().contains(error), "{directive}");
}
std::fs::write(&root, "SELECT\n-- @include main.sql\n1;").unwrap();
assert!(expand(&root).await.err().unwrap().contains("between complete"));
std::fs::write(&root, "-- @include child.sql").unwrap();
std::fs::write(dir.path().join("child.sql"), "SELECT 1").unwrap();
assert!(expand(&root).await.err().unwrap().contains("statement delimiter"));
}
#[tokio::test]
async fn decodes_each_file_independently_and_preserves_unicode() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().join("main.sql");
std::fs::write(&root, "CREATE TABLE t(v TEXT);\n-- @include child.sql\n").unwrap();
let sql = "INSERT INTO t VALUES('中文');\n";
let mut bytes = vec![0xff, 0xfe];
bytes.extend(sql.encode_utf16().flat_map(u16::to_le_bytes));
std::fs::write(dir.path().join("child.sql"), bytes).unwrap();
let result = expand(&root).await.unwrap();
let db = rusqlite::Connection::open_in_memory().unwrap();
db.execute_batch(&std::fs::read_to_string(&result.paths[0]).unwrap()).unwrap();
let value: String = db.query_row("SELECT v FROM t", [], |row| row.get(0)).unwrap();
assert_eq!(value, "中文");
}
#[tokio::test]
async fn enforces_depth_file_limits_and_cancellation() {
let dir = tempfile::tempdir().unwrap();
for n in 0..=MAX_DEPTH {
std::fs::write(dir.path().join(format!("{n}.sql")), format!("-- @include {}.sql", n + 1)).unwrap();
}
let root = dir.path().join("0.sql");
assert!(expand(&root).await.err().unwrap().contains("limit"));
std::fs::write(&root, "-- @include child.sql\n".repeat(MAX_FILES)).unwrap();
std::fs::write(dir.path().join("child.sql"), "SELECT 1;").unwrap();
assert!(expand(&root).await.err().unwrap().contains("limit"));
let token = CancellationToken::new();
token.cancel();
assert!(prepare_sql_file_includes(
&[&root],
SqlFileEncoding::Auto,
Some(DatabaseType::Sqlite),
SqlParsingOptions::default(),
false,
&token
)
.await
.err()
.unwrap()
.contains("cancelled"));
}
#[tokio::test]
async fn explicit_utf8_keeps_mysql_binary_literal_normalization_in_included_files() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().join("main.sql");
std::fs::write(&root, "-- @include child.sql\n").unwrap();
std::fs::write(dir.path().join("child.sql"), b"INSERT INTO t VALUES (_binary '\xff\x00');\n").unwrap();
let result = prepare_sql_file_includes(
&[&root],
SqlFileEncoding::Utf8,
Some(DatabaseType::Mysql),
SqlParsingOptions::for_database_type_and_compatibility(DatabaseType::Mysql, None),
true,
&CancellationToken::new(),
)
.await
.unwrap();
let expanded = std::fs::read_to_string(&result.paths[0]).unwrap();
assert!(expanded.contains("X'FF00'"), "{expanded}");
}
#[cfg(unix)]
#[tokio::test]
async fn rejects_symlink_outside_entry_directory() {
let dir = tempfile::tempdir().unwrap();
let outside = tempfile::tempdir().unwrap();
std::fs::write(outside.path().join("secret.sql"), "SELECT 1;").unwrap();
std::os::unix::fs::symlink(outside.path().join("secret.sql"), dir.path().join("child.sql")).unwrap();
let root = dir.path().join("main.sql");
std::fs::write(&root, "-- @include child.sql").unwrap();
assert!(expand(&root).await.err().unwrap().contains("escapes"));
}
}
+1
View File
@@ -12872,6 +12872,7 @@ for line in sys.stdin:
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
}
}
+3
View File
@@ -1380,6 +1380,7 @@ INSERT INTO install_check (id) VALUES (1), (2);
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
};
@@ -1472,6 +1473,7 @@ INSERT INTO children (parent_id) VALUES (LAST_INSERT_ID());
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
};
@@ -1548,6 +1550,7 @@ async fn live_sql_file_import_preserves_raw_mysql_binary_literal_bytes() {
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
};
@@ -79,6 +79,7 @@ async fn live_mysql_selected_table_restore_preserves_unselected_tables() {
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
};
execute_sql_file_path(&state, &request, &path, CancellationToken::new(), std::time::Instant::now(), |_| {}).await?;
@@ -98,6 +99,7 @@ async fn live_mysql_selected_table_restore_preserves_unselected_tables() {
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
};
let mut events = Vec::new();
@@ -188,6 +190,7 @@ async fn live_mysql_database_export_restores_dependent_views() {
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
};
execute_sql_file_path(
@@ -68,6 +68,7 @@ async fn live_oceanbase_sql_files_commit_rollback_failure_and_cancel() {
database: database.clone(), schema: None, file_path: String::new(), continue_on_error: false,
selected_tables: None, part_cooldown_ms: 0, skip_relational_constraints: false, txn_session_id: None,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
};
let first = directory.path().join("first.sql");
@@ -140,6 +140,7 @@ async fn live_oracle_database_export_replays_trigger_sequences() {
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
};
execute_sql_file_path(
@@ -194,6 +194,7 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() {
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
},
&export_path,
@@ -342,6 +343,7 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() {
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
},
&path,
@@ -189,6 +189,7 @@ async fn live_postgres_export_with_foreign_key_cycle_restores_into_empty_databas
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
},
&export_path,
@@ -160,6 +160,7 @@ async fn live_postgres_export_restores_tables_that_depend_on_functions() {
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
},
&export_path,
tokio_util::sync::CancellationToken::new(),
@@ -78,6 +78,7 @@ fn request(database: &str, execution_id: &str) -> SqlFileRequest {
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
}
}
@@ -1328,6 +1328,7 @@ async fn live_sqlserver_sql_file_import_executes_go_batches() {
part_cooldown_ms: 0,
skip_relational_constraints: false,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
};
let done_seen = AtomicBool::new(false);
@@ -230,6 +230,7 @@ async fn manual_e2e_sqlserver_default_split_zip_export_then_import_skip_relation
// before the import and `WITH NOCHECK CHECK CONSTRAINT ALL` after.
skip_relational_constraints: true,
failure_log_enabled: true,
resolve_includes: false,
encoding: Default::default(),
};
@@ -0,0 +1,87 @@
use dbx_core::connection::AppState;
use dbx_core::sql::{SqlFileRequest, SqlFileStatus};
use dbx_core::sql_file_import::execute_sql_file_path;
use std::path::Path;
use std::time::Instant;
use tokio_util::sync::CancellationToken;
fn request(path: &Path, enabled: bool) -> SqlFileRequest {
serde_json::from_value(serde_json::json!({
"executionId": "external-include-preflight", "connectionId": "not-configured",
"database": "", "filePath": path, "continueOnError": false, "resolveIncludes": enabled
}))
.unwrap()
}
// No recursion_limit override: public async callers must compile with Rust's default.
#[tokio::test]
async fn missing_include_fails_before_dispatch_from_an_external_caller() {
let directory = tempfile::tempdir().unwrap();
let storage = dbx_core::persistence::test_storage::open(&directory.path().join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let path = directory.path().join("main.sql");
std::fs::write(&path, "CREATE TABLE never_dispatched(v INTEGER);\n-- @include missing.sql\n").unwrap();
let mut events = Vec::new();
let error = execute_sql_file_path(
&state,
&request(&path, true),
&path,
CancellationToken::new(),
Instant::now(),
|event| events.push(event),
)
.await
.unwrap_err();
assert!(error.contains("missing.sql"), "{error}");
assert_eq!(events.last().unwrap().status, SqlFileStatus::Error);
assert!(events.iter().all(|event| event.statement_index == 0));
assert!(!events
.iter()
.any(|event| matches!(event.status, SqlFileStatus::StatementDone | SqlFileStatus::StatementFailed)));
}
#[tokio::test]
async fn external_caller_preserves_legacy_comments_and_nested_includes() {
let directory = tempfile::tempdir().unwrap();
let storage = dbx_core::persistence::test_storage::open(&directory.path().join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let path = directory.path().join("main.sql");
std::fs::write(&path, "-- @include shared/child.sql\n").unwrap();
for enabled in [false, true] {
if enabled {
std::fs::create_dir(directory.path().join("shared")).unwrap();
std::fs::write(directory.path().join("shared/child.sql"), "-- no database statements").unwrap();
}
let mut events = Vec::new();
execute_sql_file_path(
&state,
&request(&path, enabled),
&path,
CancellationToken::new(),
Instant::now(),
|event| events.push(event),
)
.await
.unwrap();
let terminal = events.last().unwrap();
assert_eq!(terminal.status, SqlFileStatus::Done);
assert_eq!((terminal.success_count, terminal.failure_count), (0, 0));
}
}
#[tokio::test]
async fn external_caller_cancels_preparation_without_dispatch() {
let directory = tempfile::tempdir().unwrap();
let storage = dbx_core::persistence::test_storage::open(&directory.path().join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let path = directory.path().join("main.sql");
std::fs::write(&path, "-- @include missing.sql\n").unwrap();
let token = CancellationToken::new();
token.cancel();
let mut events = Vec::new();
execute_sql_file_path(&state, &request(&path, true), &path, token, Instant::now(), |event| events.push(event))
.await
.unwrap();
assert_eq!(events.last().unwrap().status, SqlFileStatus::Cancelled);
assert!(events.iter().all(|event| event.statement_index == 0 && event.error.is_none()));
}
+34 -1
View File
@@ -91,6 +91,9 @@ pub struct SqlFileRequest {
/// BOM/UTF-8/GBK detection behavior.
#[serde(default)]
pub encoding: SqlFileEncoding,
/// Explicitly allow desktop scripts to include SQL below the entry file's directory.
#[serde(default)]
pub resolve_includes: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
@@ -716,6 +719,22 @@ impl SqlStatementSplitter {
Self { options, ..Self::default() }
}
/// A whole-line file directive must not be interpreted inside SQL text.
pub fn can_start_file_directive(&self) -> bool {
!self.in_single_quote
&& !self.in_double_quote
&& !self.in_backtick
&& !self.in_line_comment
&& self.block_comment_depth == 0
&& self.dollar_quote_tag.is_none()
&& !self.oracle_q_quote.is_open()
&& self.pending_psql_command.is_none()
}
pub fn is_at_file_directive_boundary(&self) -> bool {
self.can_start_file_directive() && !has_executable_sql_with_options(&self.buffer, self.options)
}
pub fn push_chunk(&mut self, chunk: &str) -> Vec<String> {
self.push_chunk_with_control(chunk).into_iter().map(|statement| statement.sql).collect()
}
@@ -4119,7 +4138,7 @@ fn has_executable_sql_with_options(statement: &str, options: SqlParsingOptions)
continue;
}
if ch == '-' && next == Some('-') {
if ch == '-' && next == Some('-') && dash_dash_starts_line_comment(options.profile, chars.get(i + 2).copied()) {
in_line_comment = true;
previous = Some(ch);
i += 1;
@@ -4180,6 +4199,20 @@ mod tests {
SqlFileRequest, SqlFileStatementAction, SqlParsingOptions, SqlStatementSplitter,
};
#[test]
fn file_directive_boundaries_use_the_dialect_line_comment_rule() {
let mut mysql = SqlStatementSplitter::with_options(SqlParsingOptions::for_database_type(DatabaseType::Mysql));
mysql.push_chunk("--foo\n");
assert!(!mysql.is_at_file_directive_boundary());
let mut sqlite = SqlStatementSplitter::with_options(SqlParsingOptions::for_database_type(DatabaseType::Sqlite));
sqlite.push_chunk("--foo\n");
assert!(sqlite.is_at_file_directive_boundary());
let mut mysql_comment =
SqlStatementSplitter::with_options(SqlParsingOptions::for_database_type(DatabaseType::Mysql));
mysql_comment.push_chunk("-- real comment\n");
assert!(mysql_comment.is_at_file_directive_boundary());
}
#[test]
fn sql_file_request_keeps_failure_logging_on_when_the_field_is_absent() {
// Older frontends never send the flag; their imports must keep writing
+24
View File
@@ -326,6 +326,9 @@ pub async fn execute_sql_file(
Json(body): Json<SqlFileExecuteWrapper>,
) -> Result<Json<serde_json::Value>, AppError> {
let req = body.request;
if req.resolve_includes {
return Err(AppError::from("Local SQL includes are only available in the desktop application".to_string()));
}
// The failure log of this run belongs to the session that started it: the
// export/copy routes look it up by this owner, never by path (dbx#11619).
let log_owner = crate::auth::session_token_from_headers(&headers)
@@ -812,6 +815,27 @@ mod tests {
WebState::for_tests(Arc::new(dbx_core::connection::AppState::new(storage)), directory.to_path_buf())
}
#[tokio::test]
async fn web_execution_rejects_local_includes_before_reading_any_path() {
let directory = tempfile::tempdir().unwrap();
let state = Arc::new(restore_state(directory.path()).await);
let request = serde_json::from_value(serde_json::json!({
"executionId": "includes", "connectionId": "missing", "database": "app",
"filePath": "/not-an-upload/main.sql", "continueOnError": false, "resolveIncludes": true
}))
.unwrap();
let error = execute_sql_file(
State(state.clone()),
HeaderMap::new(),
Json(SqlFileExecuteWrapper { request, file_paths: vec![] }),
)
.await
.err()
.unwrap();
assert!(error.to_string().contains("only available in the desktop"));
assert!(state.sql_file_executions.read().await.is_empty());
}
#[tokio::test]
async fn restored_preview_uses_a_managed_copy_and_release_cannot_remove_the_backup() {
let directory = tempfile::tempdir().unwrap();
+18
View File
@@ -51,6 +51,24 @@ Docker/Web 中,脚本读取和执行发生在服务器端,文件完成或释
</Step>
</Steps>
## 引用其他 SQL 文件(桌面端)
执行本地 `.sql` 文件时,开启“解析 SQL 文件引用”,可以在完整语句之间插入单独一行引用:
```sql
CREATE TABLE example (id INTEGER PRIMARY KEY, name TEXT);
-- @include "shared/seed.sql"
INSERT INTO example VALUES (2, 'after include');
```
`shared/seed.sql` 在该行的位置执行,然后继续入口文件。被引用文件也可以引用其自身目录下的文件;多次引用同一文件会重复执行,而不是自动去重。只选择入口文件,不要额外选择已被引用的文件。引用边界使用与执行相同的分句器:SQL Server 以独立行 `GO` 结束批次,其他数据库使用对应的语句分隔符;被引用文件也必须结束完整语句或批次。
路径相对于包含该引用的文件,可以使用 `/` 或 `\`,含空格时使用单引号或双引号。只允许入口脚本目录及子目录中的 `.sql` 文件;绝对路径、`..`、非 SQL 文件、指向目录外的符号链接、循环引用均会被拒绝。字符串、带引号标识符和块注释中的引用文本不会展开,也不会执行 shell 命令。
DBX 在发送任何 SQL 前,流式解码并展开本批次全部引用到私有临时文件,结束、失败或取消后自动删除。每个文件分别识别 BOM;没有 BOM 时使用本次选择的编码或自动探测。总计最多 16 层、256 次文件读取、1 GiB 展开内容,单行最多 16 MiB。缺失文件或超限会中止整批执行;这不等于执行阶段自动回滚,事务和遇错继续仍遵循原有选项。
该选项默认关闭,不支持压缩文件、ZIP 分卷包、选表恢复及普通查询编辑器的“执行 SQL”。Docker/Web 不提供本地引用,并拒绝启用了引用的执行请求,防止脚本读取服务器上的其他文件。预览仍展示入口脚本,逐文件统计归属入口文件;当前语句、失败明细和返回错误以 `[shared/seed.sql]` 这样的相对来源路径标识实际脚本,失败日志也保留该信息。引用模式保留各语句的来源和错误边界,不合并相邻 INSERT。
## 多文件会话语义
多个文件属于同一个执行任务,按列表顺序运行。MySQL 兼容路径会在整个批次中复用同一条固定连接,因此前一个文件中的以下状态可以影响后续文件:
+18
View File
@@ -51,6 +51,24 @@ In Docker/Web, reading and execution happen on the server. Temporary resources a
</Step>
</Steps>
## Include Other SQL Files (Desktop)
For local `.sql` execution, enable **Resolve SQL file includes** and place a directive on its own line between complete statements:
```sql
CREATE TABLE example (id INTEGER PRIMARY KEY, name TEXT);
-- @include "shared/seed.sql"
INSERT INTO example VALUES (2, 'after include');
```
The referenced script runs at the directive's position, then execution resumes in its caller. Nested paths resolve relative to the containing file. Repeated references execute repeatedly; select only entry files, not their dependencies. Include boundaries use the same splitter as execution: SQL Server batches end with a standalone `GO` line, while other databases use their own statement delimiters. Included files must end a complete statement or batch.
Paths accept `/` or `\`; quote paths containing spaces with single or double quotes. Only `.sql` files beneath the entry script directory are allowed. Absolute paths, `..`, non-SQL files, symlinks escaping that directory, and cycles are rejected. Include-looking text inside strings, quoted identifiers, or block comments stays literal. No shell commands are executed.
Before sending SQL, DBX streams and decodes every reference in the batch into private temporary files, automatically removed after completion, failure, or cancellation. Each file's BOM takes precedence; otherwise the selected encoding or automatic detection applies. Limits are 16 levels, 256 file visits, 1 GiB expanded text, and 16 MiB per line. Missing files or exceeded limits stop preparation before execution; execution errors still follow the existing transaction and continue-on-error options, not an implicit batch rollback.
Includes are off by default and unavailable for compressed files, split ZIP packages, selected-table restore, or ordinary query-editor execution. Docker/Web hides the option and rejects include-enabled requests to prevent access to other server files. Preview shows the entry script and per-file statistics stay under that entry file. Current statements, failure details, returned errors, and failure logs identify the actual relative source with a prefix such as `[shared/seed.sql]`. Include mode preserves individual source/error boundaries instead of merging adjacent INSERT statements.
## Multi-File Session Semantics
Selected files form one execution task and run in list order. MySQL-compatible paths reuse one pinned connection across the complete batch, so state established by an earlier file can affect later files, including:
Binary file not shown.

After

Width:  |  Height:  |  Size: 62 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 38 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 59 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 53 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 32 KiB

+1 -1
View File
@@ -4,7 +4,7 @@
1. 确认实际工作目录、分支和远端。执行 `git status --short`、`git remote -v`、`git fetch origin main`,用 `git diff origin/main...HEAD` 检查本 PR 范围;本地未提交改动还要单独检查。工作区有改动时不要直接拉取覆盖。
2. 按 `.github/workflows/ci.yml` 和其他相关 workflow 选择检查。单元/组件测试、类型检查、格式、lint、生成文件、多语言完整性是不同检查,不能互相替代。重放到新基线后检查上游是否影响已验证路径。
3. 前端至少核对 CI 使用的格式与 lint 命令,以及当前变更相关测试和类型检查。Rust、Agent、发布文件按实际影响范围执行对应检查,不为无关路径增加发布关卡。
3. 前端至少核对 CI 使用的格式与 lint 命令,以及当前变更相关测试和类型检查。Rust、Agent、发布文件按实际影响范围执行对应检查,不为无关路径增加发布关卡。修改公共异步入口时还要编译外部集成测试 target;只跑 `--lib` 无法覆盖调用方 crate 的默认递归限制,不应以提高所有调用方的限制代替检查异步边界。
4. 新增简中词条时,先检查 `.github/scripts/i18n-autofill.mjs` 的 `TARGET_LOCALES`。补齐其要求的语言并保持 `{count}` 等占位符;不要把“英文可回退”当作自动翻译 CI 会成功的保证。
### 多语言的本地检查