cksum: fix --check ambiguities in base64 detection

* src/cksum (valid_digits): Use '=' to disambiguate hex and base64.
(split_3): Likewise.
(digest_check): Likewise.
* tests/cksum/cksum-base64-untagged.sh: Add a test case.
* NEWS: Mention the fix.

NEWS

@@ -18,6 +18,12 @@ GNU coreutils NEWS                                    -*- outline -*-
   This also applies to the md5sum, sha*sum, and b2sum utilities.
   [bug introduced in coreutils-8.14]
 
+  'cksum --check' now accepts 16- and 32-bit hexadecimal BLAKE2b checksums,
+  and untagged BLAKE2b base64 checksums whose characters before the padding
+  are all hexadecimal digits.  Previously these could be rejected as
+  improperly formatted due to confusion between hex and base64 encodings.
+  [bugs introduced in coreutils-9.10]
+
   'cksum --check' now rejects length suffixes on fixed-length algorithm tags,
   such as SHA256-8.  Previously these caused only part of the digest to be
   checked, and could cause subsequent full-length untagged checksums to be

src/cksum.c

@@ -784,7 +784,10 @@ static bool
 valid_digits (unsigned char const *s, idx_t len)
 {
 #if HASH_ALGO_CKSUM
-  if (len == BASE64_LENGTH (digest_length >> 3))
+  /* For 16- and 32-bit digests, hex and base64 lengths coincide,
+     but base64 requires padding.  */
+  if (len == BASE64_LENGTH (digest_length >> 3)
+      && (len != digest_hex_bytes || s[len - 1] == '='))
     {
       idx_t i;
       for (i = 0; i < len - digest_length % 3; i++)
@@ -1069,7 +1072,7 @@ split_3 (char *s, idx_t s_len,
         return false;
     }
   else if (cksum_algorithm == blake2b
-           && digest_hex_bytes < digest_base64_bytes)
+           && (digest_hex_bytes < digest_base64_bytes || trailing_equals))
     {
       for (int j = 8; j <= DIGEST_MAX_LEN * 8; j += 8)
         {
@@ -1472,7 +1475,8 @@ digest_check (char const *checkfile_name)
             {
               bool match = false;
 #if HASH_ALGO_CKSUM
-              if (d_len == BASE64_LENGTH (digest_length >> 3))
+              if (d_len == BASE64_LENGTH (digest_length >> 3)
+                  && (d_len != digest_hex_bytes || digest[d_len - 1] == '='))
                 match = b64_equal (digest, bin_buffer);
               else
 #endif

tests/cksum/cksum-base64-untagged.sh

@@ -47,4 +47,23 @@ for length in 8 216 224 232 248 256 264 376 384 392 504 512; do
   compare expout out || fail=1
 done
 
+# Hex and base64 lengths coincide for 16- and 32-bit digests.
+# These inputs also produce base64 digests with only hex digits before
+# the padding: "26" at 16 bits gives f20=, "1027" at 32 bits gives c3ADDA==.
+for text in 26 1027; do
+  for length in 16 32; do
+    for encoding in '' --base64; do
+      for format in --tag --untagged; do
+        printf %s "$text" > inp || framework_failure_
+        cksum -a blake2b -l $length $encoding $format inp > check || fail=1
+        cksum -a blake2b --strict --check check > out || fail=1
+        compare expout out || fail=1
+        # Ensure the selected encoding is also used for comparison.
+        echo changed > inp || framework_failure_
+        returns_ 1 cksum -a blake2b --status --check check || fail=1
+      done
+    done
+  done
+done
+
 Exit $fail