branch master updated: * contrib/mass_test/*.sh: add more manuals, separate script to output HTML, separate generating HTML and checking the differences.

[Patrice Dumas]

This is an automated email from the git hooks/post-receive script.

pertusus pushed a commit to branch master
in repository texinfo.

The following commit(s) were added to refs/heads/master by this push:
     new ce8a963dc5 * contrib/mass_test/*.sh: add more manuals, separate script 
to output HTML, separate generating HTML and checking the differences.
ce8a963dc5 is described below

commit ce8a963dc55f6fcffcfcaaf5b1fb60df557e7f44
Author: Patrice Dumas <pertusus@free.fr>
AuthorDate: Tue Jan 23 04:08:28 2024 +0100

    * contrib/mass_test/*.sh: add more manuals, separate script to
    output HTML, separate generating HTML and checking the differences.
---
 .gitignore                                   |   3 +-
 ChangeLog                                    |   5 ++
 contrib/mass_test/check_perlVSC.sh           |  63 +++++----------
 contrib/mass_test/extract_manuals.sh         | 115 ++++++++++++++++++++++++---
 contrib/mass_test/generate_perlVSC.sh        |  67 ++++++++++++++++
 contrib/mass_test/get_manuals.sh             |  32 +++++++-
 contrib/mass_test/output_perlVSC.sh          |  20 +++++
 contrib/mass_test/setup_reference_perlVSC.sh |  27 +------
 8 files changed, 251 insertions(+), 81 deletions(-)

diff --git a/.gitignore b/.gitignore
index 537e1c833b..f48c26c215 100644
--- a/.gitignore
+++ b/.gitignore
@@ -397,11 +397,12 @@ contrib/perldoc-all/*.aux
 contrib/perldoc-all/perldoc-all.??
 contrib/perldoc-all/perldoc-all.???
 
+contrib/mass_test/download
 contrib/mass_test/compare_C_HTML
 contrib/mass_test/manuals
+contrib/mass_test/normalized
 contrib/mass_test/perl_HTML_refs
 contrib/mass_test/result_check_perlVSC
-contrib/mass_test/www.gnu.org
 
 js/build-aux/install-sh
 js/build-aux/missing
diff --git a/ChangeLog b/ChangeLog
index 9ae29fcd04..cac7e0a6fb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+2024-01-21  Patrice Dumas  <pertusus@free.fr>
+
+       * contrib/mass_test/*.sh: add more manuals, separate script to
+       output HTML, separate generating HTML and checking the differences.
+
 2024-01-21  Patrice Dumas  <pertusus@free.fr>
 
        * tp/Texinfo/Convert/DocBook.pm: comments covering, hopefully, all
diff --git a/contrib/mass_test/check_perlVSC.sh 
b/contrib/mass_test/check_perlVSC.sh
index affedf70b3..d47c2ae4f2 100755
--- a/contrib/mass_test/check_perlVSC.sh
+++ b/contrib/mass_test/check_perlVSC.sh
@@ -16,54 +16,33 @@
 # if set -e is used, diff call should be protected
 #set -e
 
-dir=compare_C_HTML
-mkdir -p $dir
-
-one_test=no
-if test -n "$1"; then
-  one_test=yes
-  the_test=$1
-else
-  rm -rf result_check_perlVSC
-fi
+dir=result_check_perlVSC
 
+rm -rf result_check_perlVSC
 mkdir -p result_check_perlVSC
 
-TEXINFO_XS_CONVERT=1
-export TEXINFO_XS_CONVERT
-
 #set -x
 
-for manual_dir in manuals/*/ ; do
-  manual_name=`basename $manual_dir`
-  one_manual_found=no
-  for file in $manual_dir/*.texi* ; do
-    if grep -q -s '^ *@node \+[tT][Oo][Pp] *\(,.*\)\?$' $file; then
-      one_manual_found=yes
-      bfile_ext=`basename $file`
-      bfile=`echo $bfile_ext | sed 's/\.texi.*$//'`
-      # if we're only doing one test, skip it unless this is the one.
-      if test $one_test = 'yes' && test "z$bfile" != "z$the_test" ; then
-        continue
-      fi
-
-      echo "doing $file"
-      rm -rf $dir/$bfile/
-      mkdir $dir/$bfile/
-      err_file=$dir/$bfile/${bfile}-html_nodes.err
-      ../../tp/texi2any.pl --force --error-limit=10000 -c TEST=1 --html -o 
$dir/$bfile/html_nodes/ $file 2>$err_file
-      if test -s $err_file ; then :
-      else rm -f $err_file
-      fi
-      
-      diff_file=result_check_perlVSC/${manual_name}-${bfile}.diff
-      diff -u -r perl_HTML_refs/$bfile/ $dir/$bfile/ > $diff_file
-      if test -s $diff_file ; then :
-      else rm -f $diff_file
+for manual_proj_dir in manuals/*/ ; do
+  proj_dir=`basename $manual_proj_dir`
+  for manual_dir in $manual_proj_dir/*/ ; do
+    manual_name=`basename $manual_dir`
+    one_manual_found=no
+    for file in $manual_dir/*.texi* ; do
+      if grep -q -s '^ *@node \+[tT][Oo][Pp] *\(,.*\)\?$' $file; then
+        one_manual_found=yes
+        bfile_ext=`basename $file`
+        bfile=`echo $bfile_ext | sed 's/\.texi.*$//'`
+
+        diff_file=result_check_perlVSC/${proj_dir}-${manual_name}-${bfile}.diff
+        diff -u -r perl_HTML_refs/$proj_dir/$bfile/ 
compare_C_HTML/$proj_dir/$bfile/ > $diff_file
+        if test -s $diff_file ; then :
+        else rm -f $diff_file
+        fi
       fi
+    done
+    if test $one_manual_found = 'no' ; then
+      echo "WARNING: $manual_dir: no manual" 1>&2
     fi
   done
-  if test $one_manual_found = 'no' ; then
-    echo "WARNING: $manual_dir: no manual" 1>&2
-  fi
 done
diff --git a/contrib/mass_test/extract_manuals.sh 
b/contrib/mass_test/extract_manuals.sh
index a8a3e75ec1..7833d04e2b 100755
--- a/contrib/mass_test/extract_manuals.sh
+++ b/contrib/mass_test/extract_manuals.sh
@@ -12,22 +12,107 @@
 
 set -e
 
+rm -rf normalized
+mkdir -p normalized
+
+# disambiguate manuals
+for dir in download/www.gnu.org/software/plotutils/manual/*/; do
+  dir_name=`basename $dir`
+  mkdir -p normalized/plotutils_${dir_name}
+  for file in 
download/www.gnu.org/software/plotutils/manual/$dir_name/*.texi.tar.gz ; do
+    cp -p $file normalized/plotutils_${dir_name}/
+  done
+done
+
 mkdir -p manuals
 
-for manual in www.gnu.org/software/*/manual/*.texi.tar.gz; do
+extract_manual() {
+  manual=$1
+  mdir=$2
   bfile=`basename $manual .texi.tar.gz`
-  echo "$bfile"
-  rm -rf manuals/$bfile
-  mkdir manuals/$bfile
-  cp -p $manual manuals/$bfile/
+  mdir_bfile="$mdir/$bfile"
+  echo "$mdir_bfile"
+  rm -rf manuals/$mdir_bfile
+  mkdir -p manuals/$mdir_bfile
+  cp -p $manual manuals/$mdir/$bfile/
   (
-    cd manuals/$bfile/
+    cd manuals/$mdir/$bfile/
     tar xzf ${bfile}.texi.tar.gz
     rm -f ${bfile}.texi.tar.gz
   )
+}
+
+for manual in download/www.gnu.org/software/*/manual/*.texi.tar.gz 
download/www.gnu.org/software/*/manual/*/*.texi.tar.gz ; do
+  mdir=`echo $manual | sed 's;download/www.gnu.org/software/\([^/]\+\)/.*;\1;'`
+  extract_manual $manual $mdir
 done
 
-for dir in www.gnu.org/software/*/manual/ ; do
+mv manuals/rottlog//rottlog/tmp/rottlog-*/doc/*.texi manuals/rottlog//rottlog
+rm -rf manuals/rottlog//rottlog/tmp/
+
+rm -rf manuals/plotutils
+# all the manuals are in each of the texinfo sub-manuals
+rm -rf manuals/texinfo/info-stnd manuals/texinfo/texi2any_*
+# already in auctex
+rm -rf manuals/auctex/preview-latex
+
+for manual in download/www.gnu.org/software/emacs/manual/texi/*.texi.tar.gz 
download/www.gnu.org/software/guile-gnome/docs/*/*.texi.tar.gz ; do
+  mdir=`echo $manual | sed 's;download/www.gnu.org/software/\([^/]\+\)/.*;\1;'`
+  extract_manual $manual $mdir
+  bfile=`basename $manual .texi.tar.gz`
+  mv manuals/$mdir/$bfile/*-*/* manuals/$mdir/$bfile/
+  rmdir manuals/$mdir/$bfile/*-*/
+done
+
+for manual in download/www.gnu.org/prep/*/*.texi.tar.gz ; do
+  mdir=`echo $manual | sed 's;download/www.gnu.org/prep/\([^/]\+\)/.*;\1;'`
+  extract_manual $manual $mdir
+done
+
+for dir in normalized/*/; do
+  dir_name=`basename $dir`
+  for manual in $dir/*.texi.tar.gz ; do
+    extract_manual $manual $dir_name
+  done
+done
+
+rm -rf tmp manuals/gcc
+mkdir tmp
+(
+cd tmp
+tar xzf ../download/gcc.gnu.org/onlinedocs/gcc-*/docs-sources.tar.gz
+# special case
+mv gcc/gcc/jit/docs/_build/texinfo/libgccjit.texi gcc/gcc/jit/
+find gcc -name *.texi | while read manual ; do
+  dirname=`dirname $manual`
+  bfile=`basename $dirname`
+  echo "gcc/$bfile $manual"
+  mkdir -p ../manuals/gcc/$bfile/
+  cp -p $manual ../manuals/gcc/$bfile/
+done
+)
+rm -rf tmp
+
+mkdir -p manuals/glibc/libc/
+echo glibc/libc
+cp -p download/sourceware.org/glibc/manual/texi/libc-texi.tar.gz 
manuals/glibc/libc/
+(
+  cd manuals/glibc/libc/
+  tar xzf *-texi.tar.gz
+  rm -f *-texi.tar.gz
+)
+
+rm -rf manuals/groff/groff/
+mkdir -p manuals/groff/groff/
+echo groff/groff
+cp -p download/www.gnu.org/software/groff/manual/groff.texi.gz 
manuals/groff/groff
+(
+  cd manuals/groff/groff
+  gunzip groff.texi.gz
+)
+
+
+for dir in download/www.gnu.org/software/*/manual/ ; do
   one_manual_found=no
   for file in $dir/*.texi.tar.gz; do
     if test -e "$file" ; then
@@ -36,7 +121,15 @@ for dir in www.gnu.org/software/*/manual/ ; do
     fi
   done
   if test $one_manual_found = 'no' ; then
-    echo "REMARK: $dir: no manual" 1>&2
+    for file in $dir/*/*.texi.tar.gz; do
+      if test -e "$file" ; then
+        one_manual_found=yes
+        break
+      fi
+    done
+    if test $one_manual_found = 'no' ; then
+      echo "REMARK: $dir: no manual" 1>&2
+    fi
   fi
 done
 
@@ -46,13 +139,13 @@ done
 echo '@macro FIXME {arg}
 \\arg\\
 @end macro
-' > manuals/anubis/rendition.texi
+' > manuals/anubis/anubis/rendition.texi
 
 # files with CRLF EOL found with
-# file manuals/*/*.texi | grep CRLF
+# file manuals/*/*/*.texi | grep CRLF
 
 # remove CR in end of lines.
-for file in manuals/orgadoc/*.texi manuals/remotecontrol/version.texi; do
+for file in manuals/orgadoc/*/*.texi manuals/remotecontrol/*/version.texi; do
   sed 's/\r$//' $file > $file.$$
   rm -f $file
   mv $file.$$ $file
diff --git a/contrib/mass_test/generate_perlVSC.sh 
b/contrib/mass_test/generate_perlVSC.sh
new file mode 100755
index 0000000000..2dc952de5c
--- /dev/null
+++ b/contrib/mass_test/generate_perlVSC.sh
@@ -0,0 +1,67 @@
+#! /bin/sh
+# Output HTML
+#
+# Copyright 2024 Free Software Foundation, Inc.
+#
+# This file is free software; as a special exception the author gives
+# unlimited permission to copy and/or distribute it, with or without
+# modifications, as long as this notice is preserved.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY, to the extent permitted by law; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+
+set -e
+
+dir=$1
+
+test -z $dir && exit 1
+
+shift
+
+one_test=no
+if test -n "$1"; then
+  one_test=yes
+  the_test=$1
+fi
+
+mkdir -p $dir
+
+#set -x
+
+for manual_proj_dir in manuals/*/ ; do
+  proj_dir=`basename $manual_proj_dir`
+  rm -rf $dir/$proj_dir
+  mkdir $dir/$proj_dir
+  for manual_dir in $manual_proj_dir/*/ ; do
+    one_manual_found=no
+    for file in $manual_dir/*.texi* ; do
+      if grep -q -s '^ *@node \+[tT][Oo][Pp] *\(,.*\)\?$' $file; then
+        one_manual_found=yes
+        bfile_ext=`basename $file`
+        bfile=`echo $bfile_ext | sed 's/\.texi.*$//'`
+
+        if test $one_test = 'yes' && test "z$bfile" != "z$the_test" ; then
+          continue
+        fi
+
+        echo "doing $file"
+
+        out_dir=$dir/$proj_dir/$bfile
+        rm -rf $out_dir
+        mkdir $out_dir
+        err_file=${out_dir}/${bfile}-html_nodes.err
+        # the -I directory is for gcc, could add more
+        #echo "../../tp/texi2any.pl -I manuals/$proj_dir/include/ --force 
--error-limit=10000 -c TEST=1 --html -o ${out_dir}/html_nodes/ $file 
2>$err_file"
+        ../../tp/texi2any.pl -I manuals/$proj_dir/include/ --force 
--error-limit=10000 -c TEST=1 --html -o ${out_dir}/html_nodes/ $file 2>$err_file
+        if test -s $err_file ; then :
+        else rm -f $err_file
+        fi
+      fi
+    done
+    if test $one_manual_found = 'no' ; then
+      echo "WARNING: $manual_dir: no manual" 1>&2
+    fi
+  done
+done
diff --git a/contrib/mass_test/get_manuals.sh b/contrib/mass_test/get_manuals.sh
index b528ad1e7b..0cc6ce766e 100755
--- a/contrib/mass_test/get_manuals.sh
+++ b/contrib/mass_test/get_manuals.sh
@@ -5,6 +5,36 @@
 # are permitted in any medium without royalty provided the copyright
 # notice and this notice are preserved.
 
+mkdir -p download
 
-wget -v -N -r 
--accept-regex='www.gnu.org/software/[^/]+/manual/(.+.texi.tar.gz)?$' 
https://www.gnu.org/manual/manual.html
+(
+cd download
 
+# -l 3 is needed for manuals with indirections, plotutils or texinfo for
+# example
+wget -v -N -r -l 3 
--accept-regex='www.gnu.org/software/[^/]+/manual/(([^/]+/)?(.+.texi.tar.gz)?|([^/.]+)|([^/.]+/[^/.]+))$'
 https://www.gnu.org/manual/manual.html
+
+# looks superficially the same but actually different, be it only
+# because the .texi is gzipped, but not in an archive
+wget -v -N -r -l 3 
--accept-regex='www.gnu.org/software/groff/manual/(([^/]+/)?(.+.texi.(tar.)?gz)?|([^/.]+)|([^/.]+/[^/.]+))$'
 https://www.gnu.org/software/groff/manual/
+
+# we do not want to get all manuals in html to get the main grub manuals
+# url, so we do a separate call to wget
+wget -v -N -r -l 3 
--accept-regex='www.gnu.org/software/grub/manual/[^/]+/(.+.texi.tar.gz)?$' 
https://www.gnu.org/software/grub/grub-documentation.html
+
+#https://www.gnu.org/software/emacs/manual/index.html
+#https://www.gnu.org/software/emacs/manual/texi/eintr.texi.tar.gz
+wget -v -N -r 
--accept-regex='www.gnu.org/software/emacs/manual/texi/(.+.texi.tar.gz)?$' 
https://www.gnu.org/software/emacs/manual/texi/
+
+wget -v -N -r -l 3 
--accept-regex='https://www.gnu.org/software/guile-gnome/docs/(([^/]+/)?(.+.texi.tar.gz)?|([^/.]+))$'
 https://www.gnu.org/software/guile-gnome/docs/
+
+wget -v -N -r --accept-regex='www.gnu.org/prep/[^/]+/(.+.texi.tar.gz)?$' 
https://www.gnu.org/prep/
+
+wget -v -N -r https://gcc.gnu.org/onlinedocs/gcc-13.2.0/docs-sources.tar.gz
+
+wget -v -N -r -l 3 
--accept-regex='sourceware.org/glibc/manual/texi/(.+.texi.tar.gz)?$' 
https://sourceware.org/glibc/manual/
+
+)
+
+# May be relevant to ask projects for *.texi*
+# gnupg, lilypond, octave
diff --git a/contrib/mass_test/output_perlVSC.sh 
b/contrib/mass_test/output_perlVSC.sh
new file mode 100755
index 0000000000..114563fb73
--- /dev/null
+++ b/contrib/mass_test/output_perlVSC.sh
@@ -0,0 +1,20 @@
+#! /bin/sh
+# Setup HTML output using C in conversion
+#
+# Copyright 2024 Free Software Foundation, Inc.
+#
+# This file is free software; as a special exception the author gives
+# unlimited permission to copy and/or distribute it, with or without
+# modifications, as long as this notice is preserved.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY, to the extent permitted by law; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+
+set -e
+
+TEXINFO_XS_CONVERT=1
+export TEXINFO_XS_CONVERT
+
+./generate_perlVSC.sh compare_C_HTML $1
diff --git a/contrib/mass_test/setup_reference_perlVSC.sh 
b/contrib/mass_test/setup_reference_perlVSC.sh
index 1249a0a9a8..a11b2a121e 100755
--- a/contrib/mass_test/setup_reference_perlVSC.sh
+++ b/contrib/mass_test/setup_reference_perlVSC.sh
@@ -14,32 +14,7 @@
 
 set -e
 
-dir=perl_HTML_refs
-mkdir -p $dir
-
 TEXINFO_XS_CONVERT=0
 export TEXINFO_XS_CONVERT
 
-#set -x
-
-for manual_dir in manuals/*/ ; do
-  one_manual_found=no
-  for file in $manual_dir/*.texi* ; do
-    if grep -q -s '^ *@node \+[tT][Oo][Pp] *\(,.*\)\?$' $file; then
-      one_manual_found=yes
-      echo "doing $file"
-      bfile_ext=`basename $file`
-      bfile=`echo $bfile_ext | sed 's/\.texi.*$//'`
-      rm -rf $dir/$bfile/
-      mkdir $dir/$bfile/
-      err_file=$dir/$bfile/${bfile}-html_nodes.err
-      ../../tp/texi2any.pl --force --error-limit=10000 -c TEST=1 --html -o 
$dir/$bfile/html_nodes/ $file 2>$err_file
-      if test -s $err_file ; then :
-      else rm -f $err_file
-      fi
-    fi
-  done
-  if test $one_manual_found = 'no' ; then
-    echo "WARNING: $manual_dir: no manual" 1>&2
-  fi
-done
+./generate_perlVSC.sh perl_HTML_refs $1