scratch/faster-loaddefs 1c13a9331d: Update documentation and package calls for loaddefs-gen

[Lars Ingebrigtsen]

branch: scratch/faster-loaddefs
commit 1c13a9331d1d26833caec3a6274fa0c99dcaa11e
Author: Lars Ingebrigtsen <larsi@gnus.org>
Commit: Lars Ingebrigtsen <larsi@gnus.org>

    Update documentation and package calls for loaddefs-gen
---
 doc/lispref/loading.texi        |  25 +++++-----
 lisp/allout.el                  |   1 +
 lisp/emacs-lisp/autoload.el     |   3 ++
 lisp/emacs-lisp/loaddefs-gen.el | 101 +++++++++++++++++++++++-----------------
 lisp/emacs-lisp/package.el      |   8 +++-
 test/lisp/vc/vc-bzr-tests.el    |   2 +-
 6 files changed, 79 insertions(+), 61 deletions(-)

diff --git a/doc/lispref/loading.texi b/doc/lispref/loading.texi
index 68cd74c7d1..8a2bb5fa2d 100644
--- a/doc/lispref/loading.texi
+++ b/doc/lispref/loading.texi
@@ -529,7 +529,7 @@ primitive for autoloading; any Lisp program can call 
@code{autoload} at
 any time.  Magic comments are the most convenient way to make a function
 autoload, for packages installed along with Emacs.  These comments do
 nothing on their own, but they serve as a guide for the command
-@code{update-file-autoloads}, which constructs calls to @code{autoload}
+@code{loaddefs-generate}, which constructs calls to @code{autoload}
 and arranges to execute them when Emacs is built.
 
 @defun autoload function filename &optional docstring interactive type
@@ -627,22 +627,19 @@ subroutines not loaded successfully because they come 
later in the file.
 macro, then an error is signaled with data @code{"Autoloading failed to
 define function @var{function-name}"}.
 
-@findex update-file-autoloads
-@findex make-directory-autoloads
+@findex loaddefs-generate
 @cindex magic autoload comment
 @cindex autoload cookie
 @anchor{autoload cookie}
   A magic autoload comment (often called an @dfn{autoload cookie})
 consists of @samp{;;;###autoload}, on a line by itself,
 just before the real definition of the function in its
-autoloadable source file.  The command @kbd{M-x update-file-autoloads}
+autoloadable source file.  The function @code{loaddefs-generate}
 writes a corresponding @code{autoload} call into @file{loaddefs.el}.
 (The string that serves as the autoload cookie and the name of the
-file generated by @code{update-file-autoloads} can be changed from the
+file generated by @code{loaddefs-generate} can be changed from the
 above defaults, see below.)
 Building Emacs loads @file{loaddefs.el} and thus calls @code{autoload}.
-@kbd{M-x make-directory-autoloads} is even more powerful; it updates
-autoloads for all files in the current directory.
 
   The same magic comment can copy any kind of form into
 @file{loaddefs.el}.  The form following the magic comment is copied
@@ -675,7 +672,7 @@ and @code{define-global-minor-mode}.
 @emph{without} executing it when the file itself is loaded.  To do this,
 write the form @emph{on the same line} as the magic comment.  Since it
 is in a comment, it does nothing when you load the source file; but
-@kbd{M-x update-file-autoloads} copies it to @file{loaddefs.el}, where
+@code{loaddefs-generate} copies it to @file{loaddefs.el}, where
 it is executed while building Emacs.
 
   The following example shows how @code{doctor} is prepared for
@@ -728,11 +725,11 @@ corresponding autoload calls written into a file whose 
name is
 different from the default @file{loaddefs.el}.  Emacs provides two
 variables to control this:
 
-@defvar generate-autoload-cookie
-The value of this variable should be a string whose syntax is a Lisp
-comment.  @kbd{M-x update-file-autoloads} copies the Lisp form that
-follows the cookie into the autoload file it generates.  The default
-value of this variable is @code{";;;###autoload"}.
+@defvar lisp-mode-autoload-regexp
+The value of this constant is a regexp that matches autoload cookies.
+@code{loaddefs-generate} copies the Lisp form that follows the
+cookie into the autoload file it generates.  This will match comments
+like like @samp{;;;###autoload} and @samp{;;;###calc-autoload}.
 @end defvar
 
 @defvar generated-autoload-file
@@ -769,7 +766,7 @@ contain definitions matching the prefix being completed.  
The variable
 @code{definition-prefixes} holds a hashtable which maps a prefix to
 the corresponding list of files to load for it.  Entries to this
 mapping are added by calls to @code{register-definition-prefixes}
-which are generated by @code{update-file-autoloads}
+which are generated by @code{loaddefs-generate}
 (@pxref{Autoload}).  Files which don't contain any definitions worth
 loading (test files, for examples), should set
 @code{autoload-compute-prefixes} to @code{nil} as a file-local
diff --git a/lisp/allout.el b/lisp/allout.el
index 4624c236f5..4a61f6464d 100644
--- a/lisp/allout.el
+++ b/lisp/allout.el
@@ -489,6 +489,7 @@ calls."
   :type 'boolean :group 'allout)
 
 ;;;_  = allout-header-prefix
+;;;###autoload
 (defcustom allout-header-prefix "."
 ;; this string is treated as literal match.  it will be `regexp-quote'd, so
 ;; one cannot use regular expressions to match varying header prefixes.
diff --git a/lisp/emacs-lisp/autoload.el b/lisp/emacs-lisp/autoload.el
index 9c86e58ce8..4453b6a75f 100644
--- a/lisp/emacs-lisp/autoload.el
+++ b/lisp/emacs-lisp/autoload.el
@@ -28,6 +28,9 @@
 ;; Lisp source files in various useful ways.  To learn more, read the
 ;; source; if you're going to use this, you'd better be able to.
 
+;; The functions in this file have been largely superseded by
+;; loaddefs-gen.el.
+
 ;;; Code:
 
 (require 'lisp-mode)                   ;for `doc-string-elt' properties.
diff --git a/lisp/emacs-lisp/loaddefs-gen.el b/lisp/emacs-lisp/loaddefs-gen.el
index e6c908f05a..d97b2c640a 100644
--- a/lisp/emacs-lisp/loaddefs-gen.el
+++ b/lisp/emacs-lisp/loaddefs-gen.el
@@ -25,8 +25,8 @@
 ;; This package generates the main lisp/loaddefs.el file, as well as
 ;; all the other loaddefs files, like calendar/diary-loaddefs.el, etc.
 
-;; The main entry point is `loaddefs-gen--generate' (normally called
-;; from batch-loaddefs-gen via lisp/Makefile).
+;; The main entry point is `loaddefs-generate' (normally called
+;; from loaddefs-gen-batch via lisp/Makefile).
 ;;
 ;; The "other" loaddefs files are specified either via a file-local
 ;; setting of `generated-autoload-file', or by specifying
@@ -43,12 +43,12 @@
 
 (defvar autoload-compute-prefixes t
   "If non-nil, autoload will add code to register the prefixes used in a file.
-Standard prefixes won't be registered anyway.  I.e. if a file \"foo.el\" 
defines
-variables or functions that use \"foo-\" as prefix, that will not be 
registered.
-But all other prefixes will be included.")
+Standard prefixes won't be registered anyway.  I.e. if a file
+\"foo.el\" defines variables or functions that use \"foo-\" as
+prefix, that will not be registered.  But all other prefixes will
+be included.")
 (put 'autoload-compute-prefixes 'safe-local-variable #'booleanp)
 
-
 (defvar autoload-ignored-definitions
   '("define-obsolete-function-alias"
     "define-obsolete-variable-alias"
@@ -303,14 +303,16 @@ expression, in which case we want to handle forms 
differently."
                                                     'string<))))))
 
 
-(defun loaddefs-gen--parse-file (file main-outfile &optional package-only)
+(defun loaddefs-gen--parse-file (file main-outfile &optional package-data)
   "Examing FILE for ;;;###autoload statements.
 MAIN-OUTFILE is the main loaddefs file these statements are
 destined for, but this can be overriden by the buffer-local
 setting of `generated-autoload-file' in FILE, and
 by ;;;###foo-autoload statements.
 
-If PACKAGE-ONLY, only return the package info."
+If PACKAGE-DATA is `only', return only the package data.  If t,
+include the package data with the rest of the data.  Otherwise,
+don't include."
   (let ((defs nil)
         (load-name (loaddefs-gen--file-load-name file main-outfile))
         (compute-prefixes t)
@@ -339,21 +341,22 @@ If PACKAGE-ONLY, only return the package info."
 
       ;; We always return the package version (even for pre-dumped
       ;; files).
-      (let ((version (lm-header "version"))
-            package)
-        (when (and version
-                   (setq version (ignore-errors (version-to-list version)))
-                   (setq package (or (lm-header "package")
-                                     (file-name-sans-extension
-                                      (file-name-nondirectory file)))))
-          (push (list (or local-outfile main-outfile) file
-                      `(push (purecopy ',(cons (intern package) version))
-                             package--builtin-versions))
-                defs)))
+      (when package-data
+        (let ((version (lm-header "version"))
+              package)
+          (when (and version
+                     (setq version (ignore-errors (version-to-list version)))
+                     (setq package (or (lm-header "package")
+                                       (file-name-sans-extension
+                                        (file-name-nondirectory file)))))
+            (push (list (or local-outfile main-outfile) file
+                        `(push (purecopy ',(cons (intern package) version))
+                               package--builtin-versions))
+                  defs))))
 
       ;; Obey the `no-update-autoloads' file local variable.
       (when (and (not inhibit-autoloads)
-                 (not package-only))
+                 (not (eq package-data 'only)))
         (goto-char (point-min))
         ;; The cookie might be like ;;;###tramp-autoload...
         (while (re-search-forward lisp-mode-autoload-regexp nil t)
@@ -459,16 +462,6 @@ FILE's name."
 The header line lists the file name, its \"load name\", its autoloads,
 and the time the FILE was last updated (the time is inserted only
 if `autoload-timestamps' is non-nil, otherwise a fixed fake time is inserted)."
-  ;; (cl-assert ;Make sure we don't insert it in the middle of another section.
-  ;;  (save-excursion
-  ;;    (or (not (re-search-backward
-  ;;              (concat "\\("
-  ;;                      (regexp-quote generate-autoload-section-header)
-  ;;                      "\\)\\|\\("
-  ;;                      (regexp-quote generate-autoload-section-trailer)
-  ;;                      "\\)")
-  ;;              nil t))
-  ;;        (match-end 2))))
   (insert "\f\n;;;### ")
   (prin1 `(autoloads ,autoloads ,load-name ,file ,time)
         outbuf)
@@ -484,17 +477,22 @@ if `autoload-timestamps' is non-nil, otherwise a fixed 
fake time is inserted)."
        (or (eolp)
            (insert "\n" ";;;;;; "))))))
 
-(defun loaddefs-gen--generate (dir output-file &optional excluded-files)
+;;;###autoload
+(defun loaddefs-generate (dir output-file &optional excluded-files
+                              extra-data include-package-version)
   "Generate loaddefs files for Lisp files in the directories DIRS.
-DIR can be either a single directory or a list of
-directories.
+DIR can be either a single directory or a list of directories.
 
 The autoloads will be written to OUTPUT-FILE.  If any Lisp file
 binds `generated-autoload-file' as a file-local variable, write
 its autoloads into the specified file instead.
 
 The function does NOT recursively descend into subdirectories of the
-directory or directories specified."
+directory or directories specified.
+
+If EXTRA-DATA, include this string at the start of the generated file.
+
+If INCLUDE-PACKAGE-VERSION, include package version data."
   (let* ((files-re (let ((tmp nil))
                     (dolist (suf (get-load-suffixes))
                        ;; We don't use module-file-suffix below because
@@ -514,7 +512,7 @@ directory or directories specified."
     ;; Collect all the autoload data.
     (let ((progress (make-progress-reporter
                      (byte-compile-info
-                      (concat "Scraping files for autoloads"))
+                      (concat "Scraping files for loaddefs"))
                      0 (length files) nil 10))
           (file-count 0))
       (dolist (file files)
@@ -523,7 +521,12 @@ directory or directories specified."
         (setq defs (nconc
                    (loaddefs-gen--parse-file
                      file output-file
-                     (member (expand-file-name file) excluded-files))
+                     ;; We only want the package name from the
+                     ;; excluded files.
+                     (and include-package-version
+                          (if (member (expand-file-name file) excluded-files)
+                              'only
+                            t)))
                     defs)))
       (progress-reporter-done progress))
 
@@ -532,6 +535,9 @@ directory or directories specified."
       (with-temp-buffer
         (insert (loaddefs-gen--rubric (car fdefs) nil t))
         (search-backward "\f")
+        (when extra-data
+          (insert extra-data)
+          (ensure-empty-lines 1))
         ;; The group by source file (and sort alphabetically).
         (dolist (section (sort (seq-group-by #'cadr (cdr fdefs))
                                (lambda (e1 e2)
@@ -544,7 +550,7 @@ directory or directories specified."
           (let ((relfile (file-relative-name
                           (cadar section)
                           (file-name-directory (car fdefs)))))
-            (insert "\f\n;;; Generated autoloads from " relfile "\n")
+            (insert "\f\n;;; Generated autoloads from " relfile "\n\n")
             (dolist (def (reverse section))
               (setq def (caddr def))
               (if (stringp def)
@@ -604,15 +610,22 @@ directory or directories specified."
 
 ;;;###autoload
 (defun loaddefs-gen-batch ()
-  "Generate lisp/loaddefs.el autoloads in batch mode.
+  "Generate loaddefs.el files in batch mode.
+This scans for ;;;###autoload forms and related things.
+
 The first element on the command line should be the (main)
-loaddefs.el file, and the rest are the directories to use."
-  ;; For use during the Emacs build process only.
-  (let ((args command-line-args-left))
+loaddefs.el output file, and the rest are the directories to
+use."
+  (let* ((args command-line-args-left)
+         (output-file (expand-file-name (car args) lisp-directory)))
     (setq command-line-args-left nil)
-    (loaddefs-gen--generate
-     (cdr args) (expand-file-name (car args) lisp-directory)
-     (loaddefs-gen--excluded-files))))
+    (loaddefs-generate
+     (cdr args) output-file
+     (loaddefs-gen--excluded-files)
+     nil
+     ;; When generating the top-level Emacs loaddefs file, we want to
+     ;; include the `package--builtin-versions' things.
+     (equal (file-name-directory output-file) lisp-directory))))
 
 (provide 'loaddefs-gen)
 
diff --git a/lisp/emacs-lisp/package.el b/lisp/emacs-lisp/package.el
index b340848a6f..48551f59b4 100644
--- a/lisp/emacs-lisp/package.el
+++ b/lisp/emacs-lisp/package.el
@@ -1003,6 +1003,7 @@ untar into a directory named DIR; otherwise, signal an 
error."
 
 (defun package-autoload-ensure-default-file (file)
   "Make sure that the autoload file FILE exists and if not create it."
+  (declare (obsolete nil "29.1"))
   (unless (file-exists-p file)
     (require 'autoload)
     (let ((coding-system-for-write 'utf-8-emacs-unix))
@@ -1021,8 +1022,11 @@ untar into a directory named DIR; otherwise, signal an 
error."
          (autoload-timestamps nil)
          (backup-inhibited t)
          (version-control 'never))
-    (package-autoload-ensure-default-file output-file)
-    (make-directory-autoloads pkg-dir output-file)
+    (loaddefs-generate
+     pkg-dir output-file
+     nil
+     "(add-to-list 'load-path (directory-file-name
+                         (or (file-name-directory #$) (car load-path))))")
     (let ((buf (find-buffer-visiting output-file)))
       (when buf (kill-buffer buf)))
     auto-name))
diff --git a/test/lisp/vc/vc-bzr-tests.el b/test/lisp/vc/vc-bzr-tests.el
index 12f1e9034c..52f06df5bc 100644
--- a/test/lisp/vc/vc-bzr-tests.el
+++ b/test/lisp/vc/vc-bzr-tests.el
@@ -140,7 +140,7 @@
       ;; causes bzr status to fail.  This simulates a broken bzr
       ;; installation.
       (delete-file ".bzr/checkout/dirstate")
-      (should (progn (make-directory-autoloads
+      (should (progn (loaddefs-generate
                       default-directory
                       (expand-file-name "loaddefs.el" bzrdir))
                      t)))))