sqlite: use shared-shape objects for result rows

`all()`, `get()` and `iterate()` built each row with the
`Object::New()` overload that takes names and values. That overload
always returns a dictionary-mode object, so no two rows shared a map
and every property read was a hash lookup.

Build rows from a `DictionaryTemplate` cached on the statement in place
of the column names, invalidated on re-prepare. Rows keep their null
prototype. Statements whose column names a template cannot express
(array indices, duplicates, non-ASCII names, which the template
interns as Latin-1) or with more than 64 columns keep the previous
path.

Add a benchmark that reads every column of each row, since the
existing ones only measure building rows.

Fixes: https://github.com/nodejs/node/issues/65799
Assisted-by: Claude
Signed-off-by: Guilherme Araújo <arauujogui@gmail.com>
PR-URL: https://github.com/nodejs/node/pull/66385
Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>

benchmark/sqlite/sqlite-prepare-select-read.js

@@ -0,0 +1,44 @@
+'use strict';
+const common = require('../common.js');
+const sqlite = require('node:sqlite');
+const assert = require('assert');
+
+const bench = common.createBenchmark(main, {
+  n: [1e4],
+  rows: [1, 100],
+  method: ['all', 'iterate', 'get'],
+  returnArrays: [0, 1],
+});
+
+function main(conf) {
+  const db = new sqlite.Database(':memory:');
+  db.exec(
+    'CREATE TABLE foo (text_column TEXT, integer_column INTEGER, real_column REAL, blob_column BLOB)',
+  );
+  const insert = db.prepare('INSERT INTO foo VALUES (?, ?, ?, ?)');
+  for (let i = 0; i < conf.rows; i++) {
+    insert.run(`text ${i}`, i, i / 3, Buffer.from('example blob data'));
+  }
+
+  const stmt = db.prepare(`SELECT * FROM foo LIMIT ${conf.rows}`);
+  stmt.setReturnArrays(conf.returnArrays === 1);
+
+  // Each row is built and then has every column read once.
+  const read = conf.returnArrays ?
+    (row) => row[0].length + row[1] + row[2] + row[3].length :
+    (row) => row.text_column.length + row.integer_column +
+             row.real_column + row.blob_column.length;
+
+  let sum = 0;
+  bench.start();
+  for (let i = 0; i < conf.n; i++) {
+    if (conf.method === 'get') {
+      sum += read(stmt.get());
+    } else {
+      for (const row of stmt[conf.method]()) sum += read(row);
+    }
+  }
+  bench.end(conf.n);
+
+  assert.ok(sum > 0);
+}

src/node_sqlite.cc

@@ -22,6 +22,7 @@
 #include <cstring>
 #include <limits>
 #include <string>
+#include <string_view>
 
 namespace node {
 namespace sqlite {
@@ -3812,6 +3813,7 @@ void Statement::Finalize() {
 
 void Statement::InvalidateColumnNameCache() {
   cached_column_names_.clear();
+  cached_row_template_.Reset();
   cached_column_names_reprepare_count_ = -1;
 }
 
@@ -4033,39 +4035,81 @@ MaybeLocal<Name> Statement::ColumnNameToName(const int column) {
       .As<Name>();
 }
 
-// Populates `keys` with cached column names, rebuilding the cache if the
-// statement was re-prepared.
-bool Statement::GetCachedColumnNames(LocalVector<Name>* keys) {
-  Isolate* isolate = env()->isolate();
+// Size of the stack array RowToObject passes to the template.
+static constexpr int kMaxRowTemplateColumns = 64;
 
+// Templates read names as Latin-1 and reject indices and duplicates.
+static bool CanUseInRowTemplate(Local<Context> context,
+                                Local<Name> key,
+                                std::string_view name,
+                                const std::vector<std::string_view>& seen) {
+  return simdutf::validate_ascii(name.data(), name.size()) &&
+         std::find(seen.begin(), seen.end(), name) == seen.end() &&
+         key->ToArrayIndex(context).IsEmpty();
+}
+
+// Fills either cached_row_template_ or cached_column_names_.
+bool Statement::UpdateRowCache() {
   const int reprepare_count =
       sqlite3_stmt_status(statement_.get(), SQLITE_STMTSTATUS_REPREPARE, false);
-  if (reprepare_count != cached_column_names_reprepare_count_) {
-    cached_column_names_.clear();
-    const int num_cols = sqlite3_column_count(statement_.get());
-    if (num_cols == 0) {
-      cached_column_names_reprepare_count_ = reprepare_count;
-      return true;
-    }
-    cached_column_names_.reserve(num_cols);
-    for (int i = 0; i < num_cols; ++i) {
-      Local<Name> key;
-      if (!ColumnNameToName(i).ToLocal(&key)) {
-        InvalidateColumnNameCache();
-        return false;
-      }
-      cached_column_names_.emplace_back(Global<Name>(isolate, key));
-    }
-    cached_column_names_reprepare_count_ = reprepare_count;
+  if (reprepare_count == cached_column_names_reprepare_count_) return true;
+
+  InvalidateColumnNameCache();
+  Isolate* isolate = env()->isolate();
+  Local<Context> context = env()->context();
+  const int num_cols = sqlite3_column_count(statement_.get());
+  LocalVector<Name> keys(isolate);
+  std::vector<std::string_view> names;
+  keys.reserve(num_cols);
+  names.reserve(num_cols);
+  bool use_template = num_cols <= kMaxRowTemplateColumns;
+  for (int i = 0; i < num_cols; ++i) {
+    Local<Name> key;
+    if (!ColumnNameToName(i).ToLocal(&key)) return false;
+    std::string_view name = sqlite3_column_name(statement_.get(), i);
+    use_template =
+        use_template && CanUseInRowTemplate(context, key, name, names);
+    keys.push_back(key);
+    names.push_back(name);
   }
 
-  keys->reserve(cached_column_names_.size());
-  for (const auto& name : cached_column_names_) {
-    keys->emplace_back(name.Get(isolate));
+  if (use_template) {
+    cached_row_template_.Reset(
+        isolate,
+        DictionaryTemplate::New(isolate, {names.data(), names.size()}));
+  } else {
+    cached_column_names_.reserve(num_cols);
+    for (Local<Name> key : keys) {
+      cached_column_names_.emplace_back(isolate, key);
+    }
   }
+  cached_column_names_reprepare_count_ = reprepare_count;
   return true;
 }
 
+MaybeLocal<Object> Statement::RowToObject(LocalVector<Value>* values) {
+  DCHECK_EQ(
+      sqlite3_stmt_status(statement_.get(), SQLITE_STMTSTATUS_REPREPARE, false),
+      cached_column_names_reprepare_count_);
+  Isolate* isolate = env()->isolate();
+  if (cached_row_template_.IsEmpty()) {
+    LocalVector<Name> keys(isolate);
+    keys.reserve(cached_column_names_.size());
+    for (const auto& name : cached_column_names_) {
+      keys.emplace_back(name.Get(isolate));
+    }
+    DCHECK_EQ(keys.size(), values->size());
+    return Object::New(
+        isolate, Null(isolate), keys.data(), values->data(), keys.size());
+  }
+  DCHECK_LE(values->size(), kMaxRowTemplateColumns);
+  MaybeLocal<Value> template_values[kMaxRowTemplateColumns];
+  std::copy(values->begin(), values->end(), template_values);
+  return NewDictionaryInstanceNullProto(env()->context(),
+                                        cached_row_template_.Get(isolate),
+                                        {template_values, values->size()});
+}
+
 MaybeLocal<Value> StatementExecutionHelper::ColumnToValue(Environment* env,
                                                           sqlite3_stmt* stmt,
                                                           const int column,
@@ -4108,12 +4152,14 @@ MaybeLocal<Value> StatementExecutionHelper::All(Environment* env,
   int num_cols = 0;
   LocalVector<Value> rows(isolate);
   LocalVector<Value> row_values(isolate);
-  LocalVector<Name> row_keys(isolate);
 
   SteppingStatementGuard stepping(db, stmt);
   while ((r = sqlite3_step(stmt)) == SQLITE_ROW) {
     if (num_cols == 0) {
       num_cols = sqlite3_column_count(stmt);
+      if (!return_arrays && !statement->UpdateRowCache()) {
+        return MaybeLocal<Value>();
+      }
     }
 
     if (ExtractRowValues(env, stmt, num_cols, use_big_ints, &row_values)
@@ -4126,16 +4172,10 @@ MaybeLocal<Value> StatementExecutionHelper::All(Environment* env,
           Array::New(isolate, row_values.data(), row_values.size());
       rows.emplace_back(row_array);
     } else {
-      if (row_keys.size() == 0) {
-        // Reuses the statement's internalized column names instead of
-        // re-interning them on every call.
-        if (!statement->GetCachedColumnNames(&row_keys)) {
-          return MaybeLocal<Value>();
-        }
+      Local<Object> row_obj;
+      if (!statement->RowToObject(&row_values).ToLocal(&row_obj)) {
+        return MaybeLocal<Value>();
       }
-      DCHECK_EQ(row_keys.size(), row_values.size());
-      Local<Object> row_obj = Object::New(
-          isolate, Null(isolate), row_keys.data(), row_values.data(), num_cols);
       rows.emplace_back(row_obj);
     }
   }
@@ -4272,17 +4312,9 @@ MaybeLocal<Value> StatementExecutionHelper::Get(Environment* env,
   Local<Value> result;
   if (return_arrays) {
     result = Array::New(isolate, row_values.data(), row_values.size());
-  } else {
-    LocalVector<Name> keys(isolate);
-    // Reuses the statement's internalized column names instead of
-    // re-interning them on every call.
-    if (!statement->GetCachedColumnNames(&keys)) {
-      return MaybeLocal<Value>();
-    }
-
-    DCHECK_EQ(keys.size(), row_values.size());
-    result = Object::New(
-        isolate, Null(isolate), keys.data(), row_values.data(), num_cols);
+  } else if (!statement->UpdateRowCache() ||
+             !statement->RowToObject(&row_values).ToLocal(&result)) {
+    return MaybeLocal<Value>();
   }
 
   RESET_AND_CHECK(isolate, db, stmt, needs_reset, MaybeLocal<Value>());
@@ -5083,7 +5115,6 @@ void StatementIterator::Next(const FunctionCallbackInfo<Value>& args) {
 
   int num_cols = sqlite3_column_count(iter->stmt_->statement_.get());
   Local<Value> row_value;
-  LocalVector<Name> row_keys(isolate);
   LocalVector<Value> row_values(isolate);
 
   if (ExtractRowValues(env,
@@ -5097,14 +5128,9 @@ void StatementIterator::Next(const FunctionCallbackInfo<Value>& args) {
 
   if (iter->stmt_->return_arrays_) {
     row_value = Array::New(isolate, row_values.data(), row_values.size());
-  } else {
-    // Use cached internalized column names to avoid repeated V8 string
-    // creation and enable hidden class sharing across row objects.
-    if (!iter->stmt_->GetCachedColumnNames(&row_keys)) return;
-
-    DCHECK_EQ(row_keys.size(), row_values.size());
-    row_value = Object::New(
-        isolate, Null(isolate), row_keys.data(), row_values.data(), num_cols);
+  } else if (!iter->stmt_->UpdateRowCache() ||
+             !iter->stmt_->RowToObject(&row_values).ToLocal(&row_value)) {
+    return;
   }
 
   MaybeLocal<Value> values[] = {Boolean::New(isolate, false), row_value};

src/node_sqlite.h

@@ -397,7 +397,9 @@ class Statement : public BaseObject {
   static void Dispose(const v8::FunctionCallbackInfo<v8::Value>& args);
   v8::MaybeLocal<v8::Value> ColumnToValue(const int column);
   v8::MaybeLocal<v8::Name> ColumnNameToName(const int column);
-  bool GetCachedColumnNames(v8::LocalVector<v8::Name>* keys);
+  // Call after sqlite3_step(), which may re-prepare, and before RowToObject.
+  bool UpdateRowCache();
+  v8::MaybeLocal<v8::Object> RowToObject(v8::LocalVector<v8::Value>* values);
   void Finalize();
   bool IsFinalized();
 
@@ -418,6 +420,8 @@ class Statement : public BaseObject {
   inline int ResetStatement();
   std::vector<v8::Global<v8::Name>> cached_column_names_;
   int cached_column_names_reprepare_count_ = -1;
+  // Set instead of cached_column_names_ when the names fit a template.
+  v8::Global<v8::DictionaryTemplate> cached_row_template_;
   void InvalidateColumnNameCache();
   bool BindParams(const v8::FunctionCallbackInfo<v8::Value>& args);
   bool BindValue(const v8::Local<v8::Value>& value, const int index);

test/parallel/test-sqlite-statement-row-shape.js

@@ -0,0 +1,79 @@
+// Flags: --allow-natives-syntax
+'use strict';
+const { skipIfSQLiteMissing } = require('../common');
+skipIfSQLiteMissing();
+const assert = require('node:assert');
+const { Database } = require('node:sqlite');
+const { suite, test } = require('node:test');
+
+function assertSharedShape(rows) {
+  for (const row of rows) {
+    assert.strictEqual(Object.getPrototypeOf(row), null);
+    assert(eval('%HasFastProperties(row)'));
+    assert(eval('%HaveSameMap(row, rows[0])'));
+  }
+}
+
+suite('result row objects', () => {
+  const db = new Database(':memory:');
+  db.exec('CREATE TABLE t (a INTEGER, b TEXT, c REAL)');
+  const insert = db.prepare('INSERT INTO t VALUES (?, ?, ?)');
+  insert.run(1, 'x', 1.5);
+  insert.run(2, 'y', 2.5);
+  insert.run(3, null, null);
+
+  test('all() rows share a fast map', () => {
+    const rows = db.prepare('SELECT * FROM t ORDER BY a').all();
+    assert.deepStrictEqual(rows, [
+      { __proto__: null, a: 1, b: 'x', c: 1.5 },
+      { __proto__: null, a: 2, b: 'y', c: 2.5 },
+      { __proto__: null, a: 3, b: null, c: null },
+    ]);
+    assertSharedShape(rows);
+  });
+
+  test('get() rows share a fast map', () => {
+    const stmt = db.prepare('SELECT * FROM t WHERE a = ?');
+    assertSharedShape([stmt.get(1), stmt.get(2), stmt.get(3)]);
+  });
+
+  test('iterate() rows share a fast map', () => {
+    assertSharedShape(db.prepare('SELECT * FROM t ORDER BY a').iterate().toArray());
+  });
+
+  test('a re-prepared statement picks up the new columns', () => {
+    using db = new Database(':memory:');
+    db.exec('CREATE TABLE s (a INTEGER)');
+    db.exec('INSERT INTO s VALUES (1)');
+    const stmt = db.prepare('SELECT * FROM s');
+    assert.deepStrictEqual(stmt.get(), { __proto__: null, a: 1 });
+    db.exec('ALTER TABLE s ADD COLUMN b INTEGER DEFAULT 2');
+    assert.deepStrictEqual(stmt.get(), { __proto__: null, a: 1, b: 2 });
+  });
+
+  test('array-index column names', () => {
+    const stmt = db.prepare('SELECT 1 AS "0", 2 AS b');
+    assert.deepStrictEqual(stmt.get(), { __proto__: null, 0: 1, b: 2 });
+    assert.deepStrictEqual(stmt.all(), [{ __proto__: null, 0: 1, b: 2 }]);
+  });
+
+  test('duplicate column names keep the last value', () => {
+    const stmt = db.prepare('SELECT 1 AS a, 2 AS a');
+    assert.deepStrictEqual(stmt.get(), { __proto__: null, a: 2 });
+    assert.deepStrictEqual(stmt.iterate().toArray(),
+                           [{ __proto__: null, a: 2 }]);
+  });
+
+  test('non-ASCII column names', () => {
+    const row = db.prepare('SELECT 1 AS "café", 2 AS "名前"').get();
+    assert.deepStrictEqual(row, { __proto__: null, café: 1, 名前: 2 });
+  });
+
+  test('rows wider than the template limit', () => {
+    const cols = Array.from({ length: 100 }, (_, i) => `${i} AS c${i}`);
+    const row = db.prepare(`SELECT ${cols.join(', ')}`).get();
+    const expected = { __proto__: null };
+    for (let i = 0; i < 100; i++) expected[`c${i}`] = i;
+    assert.deepStrictEqual(row, expected);
+  });
+});