Re: fast make install (was: Introduce and use the %KEY?TRUE:FALSE% template token)

[Ralf Wildenhues]

Here's the proposed patch (generated files not shown), still missing
Autoconf version requirement.  It does the fast install for HEADERS,
DATA, PYTHON, LIBRARIES, and TEXINFOS outputs.

Comments, reviews, etc., greatly appreciated.

Cheers,
Ralf

    Exploit `install' with multiple files at once for most file types.
    
    With nobase targets, at most 50 files are installed at once,
    to avoid quadratic string concatenation and line length limits.
    This isn't yet done with base targets.  One hope is that there,
    the typical file name length is lower.  If this turns out to be
    a problem, it should be revisited.
    
    * lib/am/inst-vars.am (am__nobase_list): New helper macro for
    installation of multiple nobase files at once: sorts and lists
    files keyed by their target directory.
    * lib/am/data.am (%DIR%%PRIMARY%_INSTALL): No need to set it to
    $(install_sh_DATA) any more, not even for nobase targets.
    (install-%DIR%%PRIMARY%): Split base and nobase cases.
    Rewrite to pass lists of files which all go in the same
    directory to `install'.  In the nobase case, make use of
    $(am__nobase_list) and $(MKDIR_P) each directory before
    populating it.  Take care of files in source and build trees,
    and avoid creating directories where no files are to be
    installed.
    * lib/am/libs.am (%DIR%LIBRARIES_INSTALL): Likewise.
    (install-%DIR%LIBRARIES): Likewise.
    * lib/am/python.am (%DIR%PYTHON_INSTALL): Likewise.
    (install-%DIR%PYTHON): Likewise.  Also take care to keep note of
    a list for byte compilation after installation.
    * lib/am/texinfos.am (install-dvi-am, install-html-am)
    (install-info-am, install-pdf-am, install-ps-am): Use file lists
    to install multiple files at once.
    * NEWS: Update.
    Suggested by Akim Demaille.

diff --git a/NEWS b/NEWS
index e701578..c2c7e53 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,9 @@
 New in 1.10a:
 
+* Version requirements:
+
+  - Autoconf 2.61a-341 or greater is required.
+
 * Changes to aclocal:
 
   - The autoconf version check implemented by aclocal in aclocal.m4
@@ -44,6 +48,9 @@ New in 1.10a:
   - install-sh supports -C, which does not update the installed file
     (and its time stamps) if the contents did not change.
 
+  - `make install' now installs multiple files in one `install' invocation
+    for files with DATA, HEADERS, PYTHON, LIBRARIES and TEXINFOS primaries.
+
   - The "deleted header file problem" for *.m4 files is avoided by
     stub rules.  This allows `make' to trigger a rerun of `aclocal'
     also if some previously needed macro file has been removed.
diff --git a/lib/am/data.am b/lib/am/data.am
index 33b0613..627301b 100644
--- a/lib/am/data.am
+++ b/lib/am/data.am
@@ -1,6 +1,6 @@
 ## automake - create Makefile.in from Makefile.am
-## Copyright (C) 1994, 1995, 1996, 1998, 1999, 2001, 2003, 2004, 2006, 2007
-## Free Software Foundation, Inc.
+## Copyright (C) 1994, 1995, 1996, 1998, 1999, 2001, 2003, 2004, 2006,
+## 2007, 2008  Free Software Foundation, Inc.
 
 ## This program is free software; you can redistribute it and/or modify
 ## it under the terms of the GNU General Public License as published by
@@ -25,25 +25,39 @@ endif %?INSTALL%
 
 if %?INSTALL%
 am__installdirs += "$(DESTDIR)$(%NDIR%dir)"
-?BASE?%DIR%%PRIMARY%_INSTALL = $(INSTALL_%ONE_PRIMARY%)
-?!BASE?%DIR%%PRIMARY%_INSTALL = $(install_sh_DATA)
+%DIR%%PRIMARY%_INSTALL = $(INSTALL_%ONE_PRIMARY%)
 .PHONY install-%EXEC?exec:data%-am: install-%DIR%%PRIMARY%
 install-%DIR%%PRIMARY%: $(%DIR%_%PRIMARY%)
        @$(NORMAL_INSTALL)
        test -z "$(%NDIR%dir)" || $(MKDIR_P) "$(DESTDIR)$(%NDIR%dir)"
-?!BASE?        @$(am__vpath_adj_setup) \
+if %?BASE%
 ## Funny invocation because Makefile variable can be empty, leading to
 ## a syntax error in sh.
-       address@hidden'$(%DIR%_%PRIMARY%)'; for p in $$list; do \
+       @list='$(%DIR%_%PRIMARY%)'; list2=; for p in $$list; do \
 ## A file can be in the source directory or the build directory.
          if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
 ## If the _%PRIMARY% variable has an entry like foo/bar, install it as
 ## $(destdir)/bar, not $(destdir)/foo/bar.  The user can make a
 ## new dir variable or use a nobase_ target for the latter case.
-         %BASE?$(am__strip_dir):$(am__vpath_adj)% \
-         echo " $(%DIR%%PRIMARY%_INSTALL) '$$d$$p' 
'$(DESTDIR)$(%NDIR%dir)/$$f'"; \
-         $(%DIR%%PRIMARY%_INSTALL) "$$d$$p" "$(DESTDIR)$(%NDIR%dir)/$$f"; \
+         list2="$$list2 $$d$$p"; \
+       done; \
+       test -z "$$list2" || { \
+         echo " $(%DIR%%PRIMARY%_INSTALL) $$list2 '$(DESTDIR)$(%NDIR%dir)'"; \
+         $(%DIR%%PRIMARY%_INSTALL) $$list2 "$(DESTDIR)$(%NDIR%dir)"; }
+else !%?BASE%
+       @list='$(%DIR%_%PRIMARY%)'; $(am__nobase_list) | \
+       while read dir files; do \
+         xfiles=; for file in $$files; do \
+           if test -f "$$file"; then xfiles="$$xfiles $$file"; \
+           else xfiles="$$xfiles $(srcdir)/$$file"; fi; done; \
+         test -z "$$xfiles" || { \
+           test "x$$dir" = x. || { \
+             echo "$(MKDIR_P) '$(DESTDIR)$(%NDIR%dir)/$$dir'"; \
+             $(MKDIR_P) "$(DESTDIR)$(%NDIR%dir)/$$dir"; }; \
+           echo " $(%DIR%%PRIMARY%_INSTALL) $$xfiles 
'$(DESTDIR)$(%NDIR%dir)/$$dir'"; \
+           $(%DIR%%PRIMARY%_INSTALL) $$xfiles "$(DESTDIR)$(%NDIR%dir)/$$dir"; 
}; \
        done
+endif !%?BASE%
 endif %?INSTALL%
 
 
diff --git a/lib/am/inst-vars.am b/lib/am/inst-vars.am
index 7ddf850..acdfc9a 100644
--- a/lib/am/inst-vars.am
+++ b/lib/am/inst-vars.am
@@ -1,5 +1,5 @@
 ## automake - create Makefile.in from Makefile.am
-## Copyright (C) 2004, 2007  Free Software Foundation, Inc.
+## Copyright (C) 2004, 2007, 2008  Free Software Foundation, Inc.
 
 ## This program is free software; you can redistribute it and/or modify
 ## it under the terms of the GNU General Public License as published by
@@ -25,4 +25,18 @@ am__vpath_adj = case $$p in \
   esac;
 ## Strip all directories.
 am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+## Take a $list of nobase files, collect them, indexed by their
+## srcdir-stripped dirnames.  For up to 50 files, output a line
+## containing the dirname and the files, space-separated.
+## The arbitrary limit helps avoid the quadratic scaling exhibited by
+## string concatenation in most shells, and should avoid line length
+## limitations, while still offering only negligible performance impact
+## through spawning more install commands than absolutely needed.
+am__nobase_list = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`; \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == 50) { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } 
} \
+    END { for (dir in files) print dir " " files[dir] }'
 endif %?FIRST%
diff --git a/lib/am/libs.am b/lib/am/libs.am
index aecee2b..18d9642 100644
--- a/lib/am/libs.am
+++ b/lib/am/libs.am
@@ -1,6 +1,6 @@
 ## automake - create Makefile.in from Makefile.am
-## Copyright (C) 1994, 1995, 1996, 1998, 1999, 2001, 2002, 2003, 2004, 2006, 
2007
-## Free Software Foundation, Inc.
+## Copyright (C) 1994, 1995, 1996, 1998, 1999, 2001, 2002, 2003, 2004, 2006,
+## 2007, 2008  Free Software Foundation, Inc.
 
 ## This program is free software; you can redistribute it and/or modify
 ## it under the terms of the GNU General Public License as published by
@@ -25,23 +25,37 @@ endif %?INSTALL%
 
 if %?INSTALL%
 am__installdirs += "$(DESTDIR)$(%NDIR%dir)"
-%DIR%LIBRARIES_INSTALL = %BASE?$(INSTALL_DATA):$(install_sh_DATA)%
+%DIR%LIBRARIES_INSTALL = $(INSTALL_DATA)
 .PHONY install-%EXEC?exec:data%-am: install-%DIR%LIBRARIES
 install-%DIR%LIBRARIES: $(%DIR%_LIBRARIES)
        @$(NORMAL_INSTALL)
        test -z "$(%NDIR%dir)" || $(MKDIR_P) "$(DESTDIR)$(%NDIR%dir)"
+if %?BASE%
 ## Funny invocation because Makefile variable can be empty, leading to
 ## a syntax error in sh.
-       @list='$(%DIR%_LIBRARIES)'; for p in $$list; do \
+       @list='$(%DIR%_LIBRARIES)'; list2=; for p in $$list; do \
          if test -f $$p; then \
-## Compute basename of source file.  Unless this is a nobase_ target, we
-## want to install 'python/foo.py' as '$(DESTDIR)$(%NDIR%dir)/foo.yo',
-## not '$(DESTDIR)$(%NDIR%dir)/python/foo.yo'.
-           %BASE?$(am__strip_dir):f=$$p;% \
-           echo " $(%DIR%LIBRARIES_INSTALL) '$$p' 
'$(DESTDIR)$(%NDIR%dir)/$$f'"; \
-           $(%DIR%LIBRARIES_INSTALL) "$$p" "$(DESTDIR)$(%NDIR%dir)/$$f"; \
+           list2="$$list2 $$p"; \
          else :; fi; \
+       done; \
+       test -z "$$list2" || { \
+         echo " $(%DIR%LIBRARIES_INSTALL) $$list2 '$(DESTDIR)$(%NDIR%dir)'"; \
+         $(%DIR%LIBRARIES_INSTALL) $$list2 "$(DESTDIR)$(%NDIR%dir)"; }
+else !%?BASE%
+## Funny invocation because Makefile variable can be empty, leading to
+## a syntax error in sh.
+       @list='$(%DIR%_LIBRARIES)'; $(am__nobase_list) | \
+       while read dir files; do \
+         xfiles=; for p in $$files; do \
+           if test -f "$$p"; then xfiles="$$xfiles $$p"; else :; fi; done; \
+         test -z "$$xfiles" || { \
+           test "x$$dir" = x. || { \
+             echo "$(MKDIR_P) '$(DESTDIR)$(%NDIR%dir)/$$dir'"; \
+             $(MKDIR_P) "$(DESTDIR)$(%NDIR%dir)/$$dir"; }; \
+           echo " $(%DIR%%PRIMARY%_INSTALL) $$xfiles 
'$(DESTDIR)$(%NDIR%dir)/$$dir'"; \
+           $(%DIR%%PRIMARY%_INSTALL) $$xfiles "$(DESTDIR)$(%NDIR%dir)/$$dir"; 
}; \
        done
+endif !%?BASE%
 ## We do two loops here so that $(POST_INSTALL) can be empty.  If we
 ## merge the two loops, we get a syntax error from sh.  Anyway, having
 ## $(POST_INSTALL) in the middle of the loop essentially renders it
diff --git a/lib/am/python.am b/lib/am/python.am
index 2140dd2..2c02275 100644
--- a/lib/am/python.am
+++ b/lib/am/python.am
@@ -1,5 +1,6 @@
 ## automake - create Makefile.in from Makefile.am
-## Copyright (C) 1999, 2001, 2003, 2004, 2006, 2007 Free Software Foundation, 
Inc.
+## Copyright (C) 1999, 2001, 2003, 2004, 2006, 2007, 2008  Free Software
+## Foundation, Inc.
 
 ## This program is free software; you can redistribute it and/or modify
 ## it under the terms of the GNU General Public License as published by
@@ -24,26 +25,28 @@ endif %?INSTALL%
 
 if %?INSTALL%
 am__installdirs += "$(DESTDIR)$(%NDIR%dir)"
-%DIR%PYTHON_INSTALL = %BASE?$(INSTALL_DATA):$(install_sh_DATA)%
+%DIR%PYTHON_INSTALL = $(INSTALL_DATA)
 .PHONY install-%EXEC?exec:data%-am: install-%DIR%PYTHON
 install-%DIR%PYTHON: $(%DIR%_PYTHON)
        @$(NORMAL_INSTALL)
        test -z "$(%NDIR%dir)" || $(MKDIR_P) "$(DESTDIR)$(%NDIR%dir)"
-?!BASE?        @$(am__vpath_adj_setup) \
-       address@hidden'$(%DIR%_PYTHON)'; dlist=''; for p in $$list; do\
+if %?BASE%
+       @list='$(%DIR%_PYTHON)'; dlist=''; list2=''; for p in $$list; do\
 ## A file can be in the source directory or the build directory.
          if test -f "$$p"; then b=; else b="$(srcdir)/"; fi; \
          if test -f $$b$$p; then \
 ## Compute basename of source file.  Unless this is a nobase_ target, we
 ## want to install 'python/foo.py' as '$(DESTDIR)$(%NDIR%dir)/foo.py',
 ## not '$(DESTDIR)$(%NDIR%dir)/python/foo.py'.
-           %BASE?$(am__strip_dir):$(am__vpath_adj)% \
+           $(am__strip_dir) \
            dlist="$$dlist $$f"; \
-## Don't perform translation, since script name is important.
-           echo " $(%DIR%PYTHON_INSTALL) '$$b$$p' 
'$(DESTDIR)$(%NDIR%dir)/$$f'"; \
-           $(%DIR%PYTHON_INSTALL) "$$b$$p" "$(DESTDIR)$(%NDIR%dir)/$$f"; \
+           list2="$$list2 $$b$$p"; \
          else :; fi; \
        done; \
+       test -z "$$list2" || { \
+## Don't perform translation, since script name is important.
+         echo " $(%DIR%PYTHON_INSTALL) $$list2 '$(DESTDIR)$(%NDIR%dir)'"; \
+         $(%DIR%PYTHON_INSTALL) $$list2 "$(DESTDIR)$(%NDIR%dir)"; }; \
 ## Byte-compile must be done at install time, since file times are
 ## encoded in the actual files.
        if test -n "$$dlist"; then \
@@ -53,6 +56,32 @@ install-%DIR%PYTHON: $(%DIR%_PYTHON)
            PYTHON=$(PYTHON) $(py_compile) --destdir "$(DESTDIR)" --basedir 
"$(%NDIR%dir)" $$dlist; \
          fi; \
        else :; fi
+else !%?BASE%
+       @list='$(%DIR%_PYTHON)'; $(am__nobase_list) | \
+       while read dir files; do \
+         xfiles=; for p in $$files; do \
+## A file can be in the source directory or the build directory.
+           if test -f "$$p"; then b=; else b="$(srcdir)/"; fi; \
+           if test -f "$$b$$p"; then xfiles="$$xfiles $$b$$p"; dlist="$$dlist 
$$p"; \
+           else :; fi; done; \
+         test -z "$$xfiles" || { \
+           test "x$$dir" = x. || { \
+             echo "$(MKDIR_P) '$(DESTDIR)$(%NDIR%dir)/$$dir'"; \
+             $(MKDIR_P) "$(DESTDIR)$(%NDIR%dir)/$$dir"; }; \
+## Don't perform translation, since script name is important.
+           echo " $(%DIR%PYTHON_INSTALL) $$xfiles 
'$(DESTDIR)$(%NDIR%dir)/$$dir'"; \
+           $(%DIR%PYTHON_INSTALL) $$xfiles "$(DESTDIR)$(%NDIR%dir)/$$dir"; }; \
+## Byte-compile must be done at install time, since file times are
+## encoded in the actual files.
+         if test -n "$$dlist"; then \
+           if test -z "$(DESTDIR)"; then \
+             PYTHON=$(PYTHON) $(py_compile) --basedir "$(%NDIR%dir)" $$dlist; \
+           else \
+             PYTHON=$(PYTHON) $(py_compile) --destdir "$(DESTDIR)" --basedir 
"$(%NDIR%dir)" $$dlist; \
+           fi; \
+         else :; fi \
+       done
+endif !%?BASE%
 endif %?INSTALL%
 
 
diff --git a/lib/am/texinfos.am b/lib/am/texinfos.am
index 8bf594c..6bd65af 100644
--- a/lib/am/texinfos.am
+++ b/lib/am/texinfos.am
@@ -1,7 +1,7 @@
 ## automake - create Makefile.in from Makefile.am
 
 ## Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
-## 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc.
+## 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
 
 ## This program is free software; you can redistribute it and/or modify
 ## it under the terms of the GNU General Public License as published by
@@ -144,17 +144,18 @@ include inst-vars.am
 install-dvi-am: $(DVIS)
        @$(NORMAL_INSTALL)
        test -z "$(dvidir)" || $(MKDIR_P) "$(DESTDIR)$(dvidir)"
-       @list='$(DVIS)'; for p in $$list; do \
+       @list='$(DVIS)'; list2=; for p in $$list; do \
          if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
-         $(am__strip_dir) \
-         echo " $(INSTALL_DATA) '$$d$$p' '$(DESTDIR)$(dvidir)/$$f'"; \
-         $(INSTALL_DATA) "$$d$$p" "$(DESTDIR)$(dvidir)/$$f"; \
-       done
+         list2="$$list2 $$d$$p"; \
+       done; \
+       test -z "$$list2" || { \
+         echo " $(INSTALL_DATA) $$list2 '$(DESTDIR)$(dvidir)'"; \
+         $(INSTALL_DATA) $$list2 "$(DESTDIR)$(dvidir)"; }
 
 install-html-am: $(HTMLS)
        @$(NORMAL_INSTALL)
        test -z "$(htmldir)" || $(MKDIR_P) "$(DESTDIR)$(htmldir)"
-       @list='$(HTMLS)'; for p in $$list; do \
+       @list='$(HTMLS)'; list2=; for p in $$list; do \
          if test -f "$$p" || test -d "$$p"; then d=; else d="$(srcdir)/"; fi; \
          $(am__strip_dir) \
          if test -d "$$d$$p"; then \
@@ -163,16 +164,18 @@ install-html-am: $(HTMLS)
            echo " $(INSTALL_DATA) '$$d$$p'/* '$(DESTDIR)$(htmldir)/$$f'"; \
            $(INSTALL_DATA) "$$d$$p"/* "$(DESTDIR)$(htmldir)/$$f"; \
          else \
-           echo " $(INSTALL_DATA) '$$d$$p' '$(DESTDIR)$(htmldir)/$$f'"; \
-           $(INSTALL_DATA) "$$d$$p" "$(DESTDIR)$(htmldir)/$$f"; \
+           list2="$$list2 $$d$$p"; \
          fi; \
-       done
+       done; \
+       test -z "$$list2" || { \
+           echo " $(INSTALL_DATA) $$list2 '$(DESTDIR)$(htmldir)'"; \
+           $(INSTALL_DATA) $$list2 "$(DESTDIR)$(htmldir)"; }
 
 install-info-am: $(INFO_DEPS)
        @$(NORMAL_INSTALL)
        test -z "$(infodir)" || $(MKDIR_P) "$(DESTDIR)$(infodir)"
        @srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; \
-       list='$(INFO_DEPS)'; \
+       list='$(INFO_DEPS)'; list2=; \
        for file in $$list; do \
 ## Strip possible $(srcdir) prefix.
          case $$file in \
@@ -192,13 +195,13 @@ install-info-am: $(INFO_DEPS)
          for ifile in $$d/$$file $$d/$$file-[0-9] $$d/$$file-[0-9][0-9] \
                        $$d/$$file_i[0-9] $$d/$$file_i[0-9][0-9] ; do \
            if test -f $$ifile; then \
-## Strip directory
-             relfile=`echo "$$ifile" | sed 's|^.*/||'`; \
-             echo " $(INSTALL_DATA) '$$ifile' 
'$(DESTDIR)$(infodir)/$$relfile'"; \
-             $(INSTALL_DATA) "$$ifile" "$(DESTDIR)$(infodir)/$$relfile"; \
+             list2="$$list2 $$ifile"; \
            else : ; fi; \
          done; \
-       done
+       done; \
+       test -z "$$list2" || { \
+         echo " $(INSTALL_DATA) $$list2 '$(DESTDIR)$(infodir)'"; \
+         $(INSTALL_DATA) $$list2 "$(DESTDIR)$(infodir)"; }
        @$(POST_INSTALL)
 ## Only run this code if install-info actually exists, and it is not
 ## the Debian install-info.  FIXME: once Debian install-info goes
@@ -231,22 +234,24 @@ install-info-am: $(INFO_DEPS)
 install-pdf-am: $(PDFS)
        @$(NORMAL_INSTALL)
        test -z "$(pdfdir)" || $(MKDIR_P) "$(DESTDIR)$(pdfdir)"
-       @list='$(PDFS)'; for p in $$list; do \
+       @list='$(PDFS)'; list2=; for p in $$list; do \
          if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
-         $(am__strip_dir) \
-         echo " $(INSTALL_DATA) '$$d$$p' '$(DESTDIR)$(pdfdir)/$$f'"; \
-         $(INSTALL_DATA) "$$d$$p" "$(DESTDIR)$(pdfdir)/$$f"; \
-       done
+         list2="$$list2 $$d$$p"; \
+       done; \
+       test -z "$$list2" || { \
+         echo " $(INSTALL_DATA) $$list2 '$(DESTDIR)$(pdfdir)'"; \
+         $(INSTALL_DATA) $$list2 "$(DESTDIR)$(pdfdir)"; }
 
 install-ps-am: $(PSS)
        @$(NORMAL_INSTALL)
        test -z "$(psdir)" || $(MKDIR_P) "$(DESTDIR)$(psdir)"
-       @list='$(PSS)'; for p in $$list; do \
+       @list='$(PSS)'; list2=; for p in $$list; do \
          if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
-         $(am__strip_dir) \
-         echo " $(INSTALL_DATA) '$$d$$p' '$(DESTDIR)$(psdir)/$$f'"; \
-         $(INSTALL_DATA) "$$d$$p" "$(DESTDIR)$(psdir)/$$f"; \
-       done
+         list2="$$list2 $$d$$p"; \
+       done; \
+       test -z "$$list2" || { \
+         echo " $(INSTALL_DATA) $$list2 '$(DESTDIR)$(psdir)'"; \
+         $(INSTALL_DATA) $$list2 "$(DESTDIR)$(psdir)"; }
 
 else ! %?LOCAL-TEXIS%
 install-dvi-am: