libio: Ignore setbuf for open_memstream and open_wmemstream [BZ #34019]

open_memstream and open_wmemstream manage an internal growable buffer.
The default setbuf hook can reset that buffer, breaking the assumptions
used by the string stream overflow paths.

Install setbuf hooks that leave the internal buffer unchanged, and add
regression test cases for the narrow and wide cases, based on the
reproducer in BZ #34019.

Checked on x86_64 with no regression in the libio tests.

Reported-by: Rocket Ma <marocketbd@gmail.com>
Signed-off-by: Xiang Gao <gaoxiang@kylinos.cn>
Reviewed-by: Adhemerval Zanella  <adhemerval.zanella@linaro.org>

libio/Makefile

@@ -119,6 +119,7 @@ tests = \
   tst-ftell-partial-wide \
   tst-fwrite-error \
   tst-getdelim \
+  tst-memstream-setbuf \
   tst-memstream1 \
   tst-memstream2 \
   tst-memstream3 \
@@ -142,6 +143,7 @@ tests = \
   tst-wfile-sync \
   tst-wfiledoallocate-static \
   tst-widetext \
+  tst-wmemstream-setbuf \
   tst-wmemstream1 \
   tst-wmemstream2 \
   tst-wmemstream3 \

libio/libioP.h

@@ -741,8 +741,12 @@ extern size_t __IO_obstack_xsputn (FILE *fp, const void *data, size_t n)
 /* Jumptable functions for open_{w}memstream.  */
 extern int _IO_mem_sync (FILE* fp) __THROW attribute_hidden;
 extern void _IO_mem_finish (FILE* fp, int) __THROW attribute_hidden;
+extern FILE *_IO_mem_setbuf (FILE *fp, char *buf, ssize_t size)
+  __THROW attribute_hidden;
 extern int _IO_wmem_sync (FILE* fp) __THROW attribute_hidden;
 extern void _IO_wmem_finish (FILE* fp, int) __THROW attribute_hidden;
+extern FILE *_IO_wmem_setbuf (FILE *fp, char *buf, ssize_t size)
+  __THROW attribute_hidden;
 
 /* Other strfile functions */
 struct _IO_strfile_;

libio/memstream.c

@@ -112,3 +112,11 @@ _IO_mem_finish (FILE *fp, int dummy)
 
   _IO_str_finish (fp, 0);
 }
+
+
+FILE *
+_IO_mem_setbuf (FILE *fp, char *p, ssize_t len)
+{
+  /* Memstream manage a growable buffer internally.  */
+  return fp;
+}

libio/tst-memstream-setbuf.c

@@ -0,0 +1,41 @@
+/* Test for open_memstream BZ #34019.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <libio/tst-memstream.h>
+
+static int
+do_test (void)
+{
+  /* setbuf must not replace the internal growable buffer.  */
+  CHAR_T *buf = NULL;
+  size_t len = 0;
+  FILE *fp = OPEN_MEMSTREAM (&buf, &len);
+  setbuf (fp, NULL);
+  TEST_COMPARE (FPUTC (W('A'), fp), W('A'));
+  TEST_COMPARE (fclose (fp), 0);
+  TEST_COMPARE (len, 1);
+  TEST_VERIFY_EXIT (buf != NULL);
+  TEST_VERIFY (buf[0] == W('A'));
+  TEST_VERIFY (buf[1] == W('\0'));
+
+  free (buf);
+
+  return 0;
+}
+
+#include <support/test-driver.c>

libio/tst-wmemstream-setbuf.c

@@ -0,0 +1,20 @@
+/* Test for open_wmemstream BZ #34019.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#define TEST_WCHAR
+#include <libio/tst-memstream-setbuf.c>

libio/vtables.c

@@ -77,9 +77,11 @@
 # pragma weak _IO_cookie_write
 
 # pragma weak _IO_mem_finish
+# pragma weak _IO_mem_setbuf
 # pragma weak _IO_mem_sync
 
 # pragma weak _IO_wmem_finish
+# pragma weak _IO_wmem_setbuf
 # pragma weak _IO_wmem_sync
 
 # pragma weak __printf_buffer_as_file_overflow
@@ -334,7 +336,7 @@ const struct _IO_jump_t __io_vtables[] attribute_relro =
     JUMP_INIT (xsgetn, _IO_default_xsgetn),
     JUMP_INIT (seekoff, _IO_str_seekoff),
     JUMP_INIT (seekpos, _IO_default_seekpos),
-    JUMP_INIT (setbuf, _IO_default_setbuf),
+    JUMP_INIT (setbuf, _IO_mem_setbuf),
     JUMP_INIT (sync, _IO_mem_sync),
     JUMP_INIT (doallocate, _IO_default_doallocate),
     JUMP_INIT (read, _IO_default_read),
@@ -357,7 +359,7 @@ const struct _IO_jump_t __io_vtables[] attribute_relro =
     JUMP_INIT (xsgetn, _IO_wdefault_xsgetn),
     JUMP_INIT (seekoff, _IO_wstr_seekoff),
     JUMP_INIT (seekpos, _IO_default_seekpos),
-    JUMP_INIT (setbuf, _IO_default_setbuf),
+    JUMP_INIT (setbuf, _IO_wmem_setbuf),
     JUMP_INIT (sync, _IO_wmem_sync),
     JUMP_INIT (doallocate, _IO_wdefault_doallocate),
     JUMP_INIT (read, _IO_default_read),

libio/wmemstream.c

@@ -117,3 +117,11 @@ _IO_wmem_finish (FILE *fp, int dummy)
 
   _IO_wstr_finish (fp, 0);
 }
+
+
+FILE *
+_IO_wmem_setbuf (FILE *fp, char *p, ssize_t len)
+{
+  /* Wmemstreams manage a growable buffer internally.  */
+  return fp;
+}