branch master updated: * tp/Texinfo/XS/main/debug.c (debug_protect_eol): protect all the newlines and do not truncate the string at the first newline.

[Patrice Dumas]

This is an automated email from the git hooks/post-receive script.

pertusus pushed a commit to branch master
in repository texinfo.

The following commit(s) were added to refs/heads/master by this push:
     new f5c4442415 * tp/Texinfo/XS/main/debug.c (debug_protect_eol): protect 
all the newlines and do not truncate the string at the first newline.
f5c4442415 is described below

commit f5c4442415d71e6b828cac21c922c16bb9edbb87
Author: Patrice Dumas <pertusus@free.fr>
AuthorDate: Wed Mar 27 23:32:33 2024 +0100

    * tp/Texinfo/XS/main/debug.c (debug_protect_eol): protect all the
    newlines and do not truncate the string at the first newline.
    
    * tp/Texinfo/XS/parsetexi/source_marks.c (source_marks_names): fix
    source marks name table.
    
    * tp/Texinfo/XS/parsetexi/Parsetexi.pm (parser): call parser_set_debug
    even if DEBUG is undef.
    
    * tp/Texinfo/XS/parsetexi/parser.c (parse_texi),
    tp/Texinfo/XS/parsetexi/input.c (next_text): add after_end_fetch_nr to
    diagnose cases of text being demanded after having reached once the
    end of input, as in Perl.
    
    * tp/Texinfo/XS/main/translations.c (replace_convert_substrings): add
    same error message as in Perl.
---
 ChangeLog                              | 19 +++++++++++++++++
 tp/TODO                                |  2 +-
 tp/Texinfo/Convert/Text.pm             |  2 +-
 tp/Texinfo/Convert/Utils.pm            |  8 ++------
 tp/Texinfo/XS/main/debug.c             | 37 +++++++++++++++++++++++-----------
 tp/Texinfo/XS/main/translations.c      | 15 +++++++-------
 tp/Texinfo/XS/parsetexi/Parsetexi.pm   |  9 +++++----
 tp/Texinfo/XS/parsetexi/input.c        | 22 ++++++++++++++++++++
 tp/Texinfo/XS/parsetexi/input.h        |  1 +
 tp/Texinfo/XS/parsetexi/parser.c       |  3 +++
 tp/Texinfo/XS/parsetexi/source_marks.c |  1 +
 11 files changed, 88 insertions(+), 31 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index ce75aef021..0522410604 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,22 @@
+2024-03-27  Patrice Dumas  <pertusus@free.fr>
+
+       * tp/Texinfo/XS/main/debug.c (debug_protect_eol): protect all the
+       newlines and do not truncate the string at the first newline.
+
+       * tp/Texinfo/XS/parsetexi/source_marks.c (source_marks_names): fix
+       source marks name table.
+
+       * tp/Texinfo/XS/parsetexi/Parsetexi.pm (parser): call parser_set_debug
+       even if DEBUG is undef.
+
+       * tp/Texinfo/XS/parsetexi/parser.c (parse_texi),
+       tp/Texinfo/XS/parsetexi/input.c (next_text): add after_end_fetch_nr to
+       diagnose cases of text being demanded after having reached once the
+       end of input, as in Perl.
+
+       * tp/Texinfo/XS/main/translations.c (replace_convert_substrings): add
+       same error message as in Perl.
+
 2024-03-26  Patrice Dumas  <pertusus@free.fr>
 
        * doc/texinfo.texi (HTML Customization Variables List): change in
diff --git a/tp/TODO b/tp/TODO
index d5ed75fda9..3309b9654b 100644
--- a/tp/TODO
+++ b/tp/TODO
@@ -754,7 +754,7 @@ mkdir t/check_debug_differences/
 for file in t/*.t ; do bfile=`basename $file .t`; perl -w $file -d 1 
2>t/check_debug_differences/XS_$bfile.err ; done
 export TEXINFO_XS_PARSER=0
 for file in t/*.t ; do bfile=`basename $file .t`; perl -w $file -d 1 
2>t/check_debug_differences/PL_$bfile.err ; done
-for file in t/*.t ; do bfile=`basename $file .t`; diff -u 
t/check_debug_differences/PL_$bfile.err t/check_debug_differences/XS_$bfile.err 
> t/check_debug_differences/debug_$bfile.diff; done
+for file in t/*.t ; do bfile=`basename $file .t`; sed 's/^XS|//' 
t/check_debug_differences/XS_$bfile.err | diff -u 
t/check_debug_differences/PL_$bfile.err - > 
t/check_debug_differences/debug_$bfile.diff; done
 
 Setting flags
 our_CFLAGS='-g -Wformat-security -Wall -Wno-parentheses -Wno-missing-braces'
diff --git a/tp/Texinfo/Convert/Text.pm b/tp/Texinfo/Convert/Text.pm
index 188b9124f3..3d22580064 100644
--- a/tp/Texinfo/Convert/Text.pm
+++ b/tp/Texinfo/Convert/Text.pm
@@ -708,7 +708,7 @@ sub _convert($$)
     }
   }
   if ($element->{'type'} and $element->{'type'} eq 'def_line') {
-    #print STDERR "$element->{'extra'}->{'def_command'}\n";
+    #print STDERR "DEF: $element->{'extra'}->{'def_command'}\n";
     my ($category_element, $class_element,
         $type_element, $name_element, $arguments)
          = Texinfo::Convert::Utils::definition_arguments_content($element);
diff --git a/tp/Texinfo/Convert/Utils.pm b/tp/Texinfo/Convert/Utils.pm
index 45eeb42c4e..dae44d596c 100644
--- a/tp/Texinfo/Convert/Utils.pm
+++ b/tp/Texinfo/Convert/Utils.pm
@@ -175,14 +175,10 @@ sub definition_category_tree($$)
       # in descriptions of object-oriented programming methods or operations.
       return $self->cdt('{category} on @code{{class}}', $substrings);
     } else {
-      return Texinfo::Translations::gdt('{category} on @code{{class}}',
+      my $tree = Texinfo::Translations::gdt('{category} on @code{{class}}',
                                  $current->{'extra'}->{'documentlanguage'},
                                  $substrings);
-      #my $result = {};
-      #$result->{'contents'}
-      #  = [$arg_category, {'text' => ' on ', 'parent' => $result},
-      #     $arg_class_code];
-      #return $result;
+      return $tree;
     }
   } elsif ($def_command eq 'defivar'
            or $def_command eq 'deftypeivar'
diff --git a/tp/Texinfo/XS/main/debug.c b/tp/Texinfo/XS/main/debug.c
index 39a2a70794..4f236b09ef 100644
--- a/tp/Texinfo/XS/main/debug.c
+++ b/tp/Texinfo/XS/main/debug.c
@@ -41,7 +41,6 @@ char *
 debug_protect_eol (char *input_string, int *allocated)
 {
   char *end_of_line;
-  char *protected_string = input_string;
   *allocated = 0;
 
   if (!input_string)
@@ -49,17 +48,31 @@ debug_protect_eol (char *input_string, int *allocated)
 
   end_of_line = strchr (input_string, '\n');
 
-  if (end_of_line) {
-    char *p;
-    protected_string = malloc ((strlen(input_string) + 2) * sizeof(char));
-    *allocated = 1;
-    memcpy (protected_string, input_string, strlen(input_string));
-    p = protected_string + (end_of_line - input_string);
-    *p = '\\';
-    *(p+1) = 'n';
-    *(p+2) = '\0';
-  }
-  return protected_string;
+  if (end_of_line)
+    {
+      char *p = input_string;
+      TEXT text;
+      *allocated = 1;
+      text_init (&text);
+      while (end_of_line)
+        {
+          size_t add_len = end_of_line - p;
+          if (add_len)
+            {
+              text_append_n (&text, p, add_len);
+              p += add_len;
+            }
+          text_append_n (&text, "\\n", 2);
+          p += 1;
+          end_of_line = strchr (p, '\n');
+        }
+      if (*p)
+        {
+          text_append (&text, p);
+        }
+      return text.text;
+    }
+  return input_string;
 }
 
 char *
diff --git a/tp/Texinfo/XS/main/translations.c 
b/tp/Texinfo/XS/main/translations.c
index 21179c9994..8d9c842dce 100644
--- a/tp/Texinfo/XS/main/translations.c
+++ b/tp/Texinfo/XS/main/translations.c
@@ -502,14 +502,15 @@ replace_convert_substrings (char *translated_string,
       free (texinfo_line);
     }
 
+  if (debug_level > 0)
+    {
+      char *result_texi = convert_to_texinfo (document->tree);
+      fprintf (stderr, "XS|RESULT GDT: '%s'\n", result_texi);
+      free (result_texi);
+    }
 /*
-  {
-    char *result_texi = convert_to_texinfo (document->tree);
-    if (debug_level > 0)
-      fprintf (stderr, "XS|RESULT GDT %d: '%s'\n", document_descriptor,
-               result_texi);
-    free (result_texi);
-  }
+  if (debug_level > 0)
+    fprintf (stderr, "GDT doc descriptor: %d\n", document_descriptor);
 */
 
   return document_descriptor;
diff --git a/tp/Texinfo/XS/parsetexi/Parsetexi.pm 
b/tp/Texinfo/XS/parsetexi/Parsetexi.pm
index f38de7fbb7..e7f9a65779 100644
--- a/tp/Texinfo/XS/parsetexi/Parsetexi.pm
+++ b/tp/Texinfo/XS/parsetexi/Parsetexi.pm
@@ -116,6 +116,8 @@ sub parser (;$$)
   my $debug = 0;
   $debug = $parser->{'DEBUG'} if ($parser->{'DEBUG'});
   reset_parser ($debug);
+  # (re)set debug in any case, assuming that undef DEBUG is no debug
+  parser_set_debug ($debug);
 
   if (defined($conf)) {
     foreach my $key (keys (%$conf)) {
@@ -156,8 +158,6 @@ sub parser (;$$)
         conf_set_CPP_LINE_DIRECTIVES($conf->{$key});
       } elsif ($key eq 'MAX_MACRO_CALL_NESTING') {
         conf_set_MAX_MACRO_CALL_NESTING($conf->{$key});
-      } elsif ($key eq 'DEBUG') {
-        parser_set_debug($conf->{$key}) if (defined($conf->{$key}));
       } elsif ($key eq 'DOC_ENCODING_FOR_INPUT_FILE_NAME') {
         parser_set_DOC_ENCODING_FOR_INPUT_FILE_NAME ($conf->{$key});
       } elsif ($key eq 'INPUT_FILE_NAME_ENCODING' and defined($conf->{$key})) {
@@ -170,8 +170,9 @@ sub parser (;$$)
         parser_set_accept_internalvalue(1);
       } elsif ($key eq 'restricted' and $conf->{$key}) {
         parser_set_restricted(1);
-      } elsif ($key eq 'registrar' or $key eq 'COMMAND_LINE_ENCODING') {
-        # no action needed, only used in perl code
+      } elsif ($key eq 'registrar' or $key eq 'COMMAND_LINE_ENCODING'
+               or $key eq 'DEBUG') {
+        # no action needed, already taken into account or only for Perl code
       } else {
         warn "ignoring parser configuration value \"$key\"\n";
       }
diff --git a/tp/Texinfo/XS/parsetexi/input.c b/tp/Texinfo/XS/parsetexi/input.c
index 705abf0be0..f15f751a41 100644
--- a/tp/Texinfo/XS/parsetexi/input.c
+++ b/tp/Texinfo/XS/parsetexi/input.c
@@ -92,6 +92,11 @@ int input_number = 0;
 int input_space = 0;
 int macro_expansion_nr = 0;
 int value_expansion_nr = 0;
+/* used for debugging only to be able to diagnose cases of text being
+   demanded after having reached once the end of input, see the comment
+   in next_text where after_end_fetch_nr is increased for an explanation
+   of cases where this happens */
+int after_end_fetch_nr = -1;
 
 /* Collect text from the input sources until a newline is found.  This is used 
    instead of next_text when we need to be sure we get an entire line of 
@@ -442,6 +447,23 @@ next_text (ELEMENT *current)
         }
       input_number--;
     }
+
+  if (after_end_fetch_nr < 0)
+    after_end_fetch_nr = 0;
+  else
+    {
+     /* At the end of the input, when some text is demanded, for instance
+        to get new input in case an @include added more input, but there
+        is nothing, we get here.  Also macro arguments ending on the last
+        line will lead to the consumption of the last text, then macro
+        expansion can readd more text, and the end of input will be reached
+        again.  With numerous macros expansions on the last line, this
+        place can be reached more than twice.
+      */
+      after_end_fetch_nr++;
+      if (after_end_fetch_nr > 1)
+        debug ("AFTER END FETCHED INPUT NR: %d", after_end_fetch_nr);
+    }
   debug ("INPUT FINISHED");
   return 0;
 }
diff --git a/tp/Texinfo/XS/parsetexi/input.h b/tp/Texinfo/XS/parsetexi/input.h
index 58583d4302..02957fb8ad 100644
--- a/tp/Texinfo/XS/parsetexi/input.h
+++ b/tp/Texinfo/XS/parsetexi/input.h
@@ -38,6 +38,7 @@ void forget_small_strings (void);
 extern int input_number;
 extern int macro_expansion_nr;
 extern int value_expansion_nr;
+extern int after_end_fetch_nr;
 
 void set_input_file_name_encoding (const char *value);
 void set_locale_encoding (const char *value);
diff --git a/tp/Texinfo/XS/parsetexi/parser.c b/tp/Texinfo/XS/parsetexi/parser.c
index 1bf72bb757..aaba1ab326 100644
--- a/tp/Texinfo/XS/parsetexi/parser.c
+++ b/tp/Texinfo/XS/parsetexi/parser.c
@@ -2770,6 +2770,9 @@ parse_texi (ELEMENT *root_elt, ELEMENT *current_elt)
   if (input_number > 0)
     fprintf (stderr, "BUG: at end, input_number > 0: %d\n", input_number);
 
+  /* reinitialize */
+  after_end_fetch_nr = -1;
+
   /* update merged_in.  Only needed for merging happening after first
      index merge */
   resolve_indices_merged_in ();
diff --git a/tp/Texinfo/XS/parsetexi/source_marks.c 
b/tp/Texinfo/XS/parsetexi/source_marks.c
index e5c13c86f5..1e5938c415 100644
--- a/tp/Texinfo/XS/parsetexi/source_marks.c
+++ b/tp/Texinfo/XS/parsetexi/source_marks.c
@@ -33,6 +33,7 @@ static int name##_counter = 0;
 
 static const char *source_marks_names[SM_type_expanded_conditional_command + 
1] =
 {
+  0,
   #define sm_type(name) #name,
     SM_TYPES_LIST
   #undef sm_type