stdlib: Work around MB_CUR_MAX bug on Solaris 10

[Bruno Haible]

Running the unit tests on Solaris 10, I got an abort() here:

FAIL: test-c32snrtombs-2.sh

The stack trace is:
#3  0xfffffd7fff286980 in abort () from /lib/64/libc.so.1
#4  0x0000000000402842 in wcsnrtombs (dest=0xfffffd7fffdff930 "süß", '_' 
<repeats 15 times>, "\177\375\377\377N.@", srcp=0xfffffd7fffdff950, srclen=3, 
len=1, 
    ps=0x414280 <_gl_wcsrtombs_state>) at ../../gllib/wcsnrtombs-impl.h:43
#5  0x0000000000402721 in c32snrtombs (dest=0xfffffd7fffdff930 "süß", '_' 
<repeats 15 times>, "\177\375\377\377N.@", srcp=0xfffffd7fffdff950, srclen=6, 
len=6, ps=0x0)
    at ../../gllib/c32snrtombs.c:55
#6  0x0000000000401b70 in main (argc=2, argv=0xfffffd7fffdff998) at 
../../gltests/test-c32snrtombs.c:102

The cause of the crash is that MB_CUR_MAX is 3, which is too small for UTF-8.

The following patch fixes it, by changing the MB_CUR_MAX value from 3 to 4.


2023-04-04  Bruno Haible  <bruno@clisp.org>

        stdlib: Work around MB_CUR_MAX bug on Solaris 10.
        * lib/stdlib.in.h (gl_MB_CUR_MAX): New function.
        (MB_CUR_MAX, GNULIB_defined_MB_CUR_MAX): New macros.
        * m4/stdlib_h.m4 (gl_STDLIB_H): Test whether MB_CUR_MAX is correct.
        (gl_STDLIB_H_DEFAULTS): Initialize REPLACE_MB_CUR_MAX.
        * modules/stdlib (Files): Add m4/locale-fr.m4.
        (Makefile.am): Substitute REPLACE_MB_CUR_MAX.
        * doc/posix-headers/stdlib.texi: Mention the Solaris 10 bug.

diff --git a/doc/posix-headers/stdlib.texi b/doc/posix-headers/stdlib.texi
index 612e794068..ac7efbda84 100644
--- a/doc/posix-headers/stdlib.texi
+++ b/doc/posix-headers/stdlib.texi
@@ -14,6 +14,10 @@
 Some platforms provide a @code{NULL} macro that cannot be used in arbitrary
 expressions:
 NetBSD 5.0
+@item
+The value of @code{MB_CUR_MAX} is too small (3 instead of 4) in UTF-8 locales
+on some platforms:
+Solaris 10.
 @end itemize
 
 Portability problems fixed by the Gnulib module @code{system-posix}:
diff --git a/lib/stdlib.in.h b/lib/stdlib.in.h
index 4ecfc96a6f..c18a16bacd 100644
--- a/lib/stdlib.in.h
+++ b/lib/stdlib.in.h
@@ -589,6 +589,21 @@ _GL_WARN_ON_USE (malloc, "malloc is not POSIX compliant 
everywhere - "
 # endif
 #endif
 
+/* Return maximum number of bytes of a multibyte character.  */
+#if @REPLACE_MB_CUR_MAX@
+# if !GNULIB_defined_MB_CUR_MAX
+static inline
+int gl_MB_CUR_MAX (void)
+{
+  /* Turn the value 3 to the value 4, as needed for the UTF-8 encoding.  */
+  return MB_CUR_MAX + (MB_CUR_MAX == 3);
+}
+#  undef MB_CUR_MAX
+#  define MB_CUR_MAX gl_MB_CUR_MAX ()
+#  define GNULIB_defined_MB_CUR_MAX 1
+# endif
+#endif
+
 /* Convert a string to a wide string.  */
 #if @GNULIB_MBSTOWCS@
 # if @REPLACE_MBSTOWCS@
diff --git a/m4/stdlib_h.m4 b/m4/stdlib_h.m4
index ac28ed9efc..cef133e051 100644
--- a/m4/stdlib_h.m4
+++ b/m4/stdlib_h.m4
@@ -1,4 +1,4 @@
-# stdlib_h.m4 serial 72
+# stdlib_h.m4 serial 73
 dnl Copyright (C) 2007-2023 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -32,6 +32,49 @@ AC_DEFUN_ONCE([gl_STDLIB_H]
 
   AC_REQUIRE([AC_C_RESTRICT])
 
+  dnl Test whether MB_CUR_MAX needs to be overridden.
+  dnl On Solaris 10, in UTF-8 locales, its value is 3 but needs to be 4.
+  dnl Fortunately, we can do this because on this platform MB_LEN_MAX is 5.
+  AC_REQUIRE([AC_CANONICAL_HOST])
+  AC_REQUIRE([gt_LOCALE_FR_UTF8])
+  AC_CACHE_CHECK([whether MB_CUR_MAX is correct],
+    [gl_cv_macro_MB_CUR_MAX_good],
+    [
+      dnl Initial guess, used when cross-compiling or when no suitable locale
+      dnl is present.
+changequote(,)dnl
+      case "$host_os" in
+                  # Guess no on Solaris.
+        solaris*) gl_cv_macro_MB_CUR_MAX_good="guessing no" ;;
+                  # Guess yes otherwise.
+        *)        gl_cv_macro_MB_CUR_MAX_good="guessing yes" ;;
+      esac
+changequote([,])dnl
+      if test $LOCALE_FR_UTF8 != none; then
+        AC_RUN_IFELSE(
+          [AC_LANG_SOURCE([[
+#include <locale.h>
+#include <stdlib.h>
+int main ()
+{
+  int result = 0;
+  if (setlocale (LC_ALL, "$LOCALE_FR_UTF8") != NULL)
+    {
+      if (MB_CUR_MAX < 4)
+        result |= 1;
+    }
+  return result;
+}]])],
+          [gl_cv_macro_MB_CUR_MAX_good=yes],
+          [gl_cv_macro_MB_CUR_MAX_good=no]
+          [:])
+      fi
+    ])
+  case "$gl_cv_macro_MB_CUR_MAX_good" in
+    *yes) ;;
+    *) REPLACE_MB_CUR_MAX=1 ;;
+  esac
+
   AC_CHECK_DECLS_ONCE([ecvt])
   if test $ac_cv_have_decl_ecvt = no; then
     HAVE_DECL_ECVT=0
@@ -181,6 +224,7 @@ AC_DEFUN([gl_STDLIB_H_DEFAULTS]
   REPLACE_INITSTATE=0;       AC_SUBST([REPLACE_INITSTATE])
   REPLACE_MALLOC_FOR_MALLOC_GNU=0;    AC_SUBST([REPLACE_MALLOC_FOR_MALLOC_GNU])
   REPLACE_MALLOC_FOR_MALLOC_POSIX=0;  
AC_SUBST([REPLACE_MALLOC_FOR_MALLOC_POSIX])
+  REPLACE_MB_CUR_MAX=0;      AC_SUBST([REPLACE_MB_CUR_MAX])
   REPLACE_MBSTOWCS=0;        AC_SUBST([REPLACE_MBSTOWCS])
   REPLACE_MBTOWC=0;          AC_SUBST([REPLACE_MBTOWC])
   REPLACE_MKOSTEMP=0;        AC_SUBST([REPLACE_MKOSTEMP])
diff --git a/modules/stdlib b/modules/stdlib
index efea327794..25593a8a5d 100644
--- a/modules/stdlib
+++ b/modules/stdlib
@@ -4,6 +4,7 @@ A GNU-like <stdlib.h>.
 Files:
 lib/stdlib.in.h
 m4/stdlib_h.m4
+m4/locale-fr.m4
 
 Depends-on:
 gen-header
@@ -141,6 +142,7 @@ stdlib.h: stdlib.in.h $(top_builddir)/config.status 
$(CXXDEFS_H) \
              -e 's|@''REPLACE_INITSTATE''@|$(REPLACE_INITSTATE)|g' \
              -e 
's|@''REPLACE_MALLOC_FOR_MALLOC_GNU''@|$(REPLACE_MALLOC_FOR_MALLOC_GNU)|g' \
              -e 
's|@''REPLACE_MALLOC_FOR_MALLOC_POSIX''@|$(REPLACE_MALLOC_FOR_MALLOC_POSIX)|g' \
+             -e 's|@''REPLACE_MB_CUR_MAX''@|$(REPLACE_MB_CUR_MAX)|g' \
              -e 's|@''REPLACE_MBSTOWCS''@|$(REPLACE_MBSTOWCS)|g' \
              -e 's|@''REPLACE_MBTOWC''@|$(REPLACE_MBTOWC)|g' \
              -e 's|@''REPLACE_MKOSTEMP''@|$(REPLACE_MKOSTEMP)|g' \