crypto: validate inputEncoding in Cipher/Decipher update

Cipher.update(string, badEncoding, ...) and Decipher.update with
the same shape silently produced incorrect output: the binding
skipped the unrecognized encoding and fell back to a default,
giving the user wrong ciphertext or plaintext with no signal.

Sub-cases 1 and 2 from issue #45189 (bad output encoding to
update/final) were addressed in PR #45990. This commit completes
the fix for sub-case 3 (bad input encoding) per panva's comment
deferring it to a follow-up PR for CITGM testing. When `data` is
a string and `inputEncoding` is non-null but does not normalize
to a known encoding, throw ERR_UNKNOWN_ENCODING. Buffer /
TypedArray / DataView data paths are unaffected (the binding
ignores `inputEncoding` for non-string data anyway).

Fixes: https://github.com/nodejs/node/issues/45189
Refs: https://github.com/nodejs/node/pull/45990
Signed-off-by: Maruthan G <maruthang4@gmail.com>
PR-URL: https://github.com/nodejs/node/pull/66247
Reviewed-By: Xuguang Mei <meixuguang@gmail.com>
Reviewed-By: Rafael Gonzaga <rafael.nunu@hotmail.com>
Reviewed-By: Luigi Pinca <luigipinca@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>

lib/internal/crypto/cipher.js

@@ -185,6 +185,10 @@ function _flush(callback) {
 function update(data, inputEncoding, outputEncoding) {
   if (typeof data === 'string') {
     validateEncoding(data, inputEncoding);
+    if (inputEncoding != null &&
+        normalizeEncoding(inputEncoding) === undefined) {
+      throw new ERR_UNKNOWN_ENCODING(inputEncoding);
+    }
   } else if (!isArrayBufferView(data)) {
     throw new ERR_INVALID_ARG_TYPE(
       'data', ['string', 'Buffer', 'TypedArray', 'DataView'], data);

test/parallel/test-crypto-encoding-validation-error.js

@@ -56,3 +56,52 @@ const encodingChangeError = {
     { message: /^Unknown encoding: bad3$/, code: 'ERR_UNKNOWN_ENCODING' }
   );
 }
+
+// Regression tests for https://github.com/nodejs/node/issues/45189:
+// Unknown input encodings used to be silently accepted by Cipher/Decipher
+// `update`, producing incorrect (and silently non-deterministic) output.
+// They must now reject with ERR_UNKNOWN_ENCODING.
+
+{
+  const cipher = createCipher();
+
+  assert.throws(
+    () => cipher.update('test', 'bad', 'hex'),
+    { message: /^Unknown encoding: bad$/, code: 'ERR_UNKNOWN_ENCODING' }
+  );
+}
+
+{
+  const { createDecipheriv } = require('crypto');
+  const decipher = createDecipheriv(
+    'aes-256-cbc', randomBytes(32), randomBytes(16));
+
+  assert.throws(
+    () => decipher.update('test', 'bad', 'hex'),
+    { message: /^Unknown encoding: bad$/, code: 'ERR_UNKNOWN_ENCODING' }
+  );
+}
+
+// A buffer-like data argument should not trigger encoding validation,
+// because the input encoding is ignored when data is not a string.
+{
+  const cipher = createCipher();
+  // Should not throw.
+  cipher.update(Buffer.from('test'), 'bad-but-ignored', 'hex');
+}
+
+// Valid input encodings must continue to work.
+{
+  const cipher = createCipher();
+  let result = cipher.update('test', 'utf-8', 'hex');
+  result += cipher.final('hex');
+  assert.strictEqual(typeof result, 'string');
+}
+
+// Omitting the input encoding (undefined / null) is allowed; the
+// underlying binding falls back to its default behavior.
+{
+  const cipher = createCipher();
+  // Should not throw.
+  cipher.update(Buffer.from('test'));
+}