master 3217ae6: Add aid for finding missing dynamic variable declarations

[Mattias Engdegård]

branch: master
commit 3217ae6e05c5d99f5d2d364b8f631001ee1d29c9
Author: Mattias Engdegård <mattiase@acm.org>
Commit: Mattias Engdegård <mattiase@acm.org>

    Add aid for finding missing dynamic variable declarations
    
    Find lexical use of variables that are dynamically declared in other
    files by recording 'defvar' declarations in files that can be read
    in by the compiler in a second compilation.  This is particularly
    useful when converting code to use lexical-binding.
    
    The facility is controlled by setting environment variables:
    
     EMACS_GENERATE_DYNVARS -- set to non-empty to generate a .dynvars file
                               corresponding to each .elc.
     EMACS_DYNVARS_FILE     -- set to the name of a .dynvars file to use
                               as defvar information during compilation,
                               enabling the new warnings.
    
    * lisp/emacs-lisp/bytecomp.el (byte-compile--known-dynamic-vars)
    (byte-compile--seen-defvars): New variables.
    (byte-compile-warning-types): Add lexical-dynamic warning.
    (byte-compile--load-dynvars, byte-compile--warn-lexical-dynamic):
    New functions.
    * lisp/emacs-lisp/bytecomp.el (byte-compile-file, byte-compile--declare-var)
    (byte-compile-lambda, byte-compile-bind): Add dynamic variable loads,
    dumps and checks.
    * doc/lispref/variables.texi (Converting to Lexical Binding): Document.
---
 doc/lispref/variables.texi  | 39 +++++++++++++++++++++++++++++++++
 lisp/emacs-lisp/bytecomp.el | 53 +++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 90 insertions(+), 2 deletions(-)

diff --git a/doc/lispref/variables.texi b/doc/lispref/variables.texi
index acbc8df..6c0b3b5 100644
--- a/doc/lispref/variables.texi
+++ b/doc/lispref/variables.texi
@@ -1283,6 +1283,45 @@ you can also add a leading underscore to the variable's 
name to
 indicate to the compiler that this is a variable known not to
 be used.)
 
+@subsubheading Cross-file variable checking
+
+@strong{Note:} This is an experimental feature that may change or
+disappear without prior notice.
+
+The byte-compiler can also warn about lexical variables that are
+special in other Emacs Lisp files, often indicating a missing
+@code{defvar} declaration.  This useful but somewhat specialised check
+requires three steps:
+
+@enumerate
+@item
+Byte-compile all files whose special variable declarations may be of
+interest, with the environment variable @env{EMACS_GENERATE_DYNVARS}
+set to a nonempty string.  These are typically all the files in the
+same package or related packages or Emacs subsystems.  The process
+will generate a file whose name ends in @file{.dynvars} for each
+compiled Emacs Lisp file.
+
+@item
+Concatenate the @file{.dynvars} files into a single file.
+
+@item
+Byte-compile the files that need to be checked, this time with
+the environment variable @env{EMACS_DYNVARS_FILE} set to the name
+of the aggregated file created in step 2.
+@end enumerate
+
+Here is an example illustrating how this could be done, assuming that
+a Unix shell and @command{make} are used for byte-compilation:
+
+@example
+$ rm *.elc                                # force recompilation
+$ EMACS_GENERATE_DYNVARS=1 make           # generate .dynvars
+$ cat *.dynvars > ~/my.dynvars            # combine .dynvars
+$ rm *.elc                                # force recompilation
+$ EMACS_DYNVARS_FILE=~/my.dynvars make    # perform checks
+@end example
+
 @node Buffer-Local Variables
 @section Buffer-Local Variables
 @cindex variable, buffer-local
diff --git a/lisp/emacs-lisp/bytecomp.el b/lisp/emacs-lisp/bytecomp.el
index f4b9139..90809a9 100644
--- a/lisp/emacs-lisp/bytecomp.el
+++ b/lisp/emacs-lisp/bytecomp.el
@@ -268,6 +268,13 @@ This option is enabled by default because it reduces Emacs 
memory usage."
 (defconst byte-compile-log-buffer "*Compile-Log*"
   "Name of the byte-compiler's log buffer.")
 
+(defvar byte-compile--known-dynamic-vars nil
+  "Variables known to be declared as dynamic, for warning purposes.
+Each element is (VAR . FILE), indicating that VAR is declared in FILE.")
+
+(defvar byte-compile--seen-defvars nil
+  "All dynamic variable declarations seen so far.")
+
 (defcustom byte-optimize-log nil
   "If non-nil, the byte-compiler will log its optimizations.
 If this is `source', then only source-level optimizations will be logged.
@@ -290,7 +297,7 @@ The information is logged to `byte-compile-log-buffer'."
 (defconst byte-compile-warning-types
   '(redefine callargs free-vars unresolved
             obsolete noruntime cl-functions interactive-only
-            make-local mapcar constants suspicious lexical)
+            make-local mapcar constants suspicious lexical lexical-dynamic)
   "The list of warning types used when `byte-compile-warnings' is t.")
 (defcustom byte-compile-warnings t
   "List of warnings that the byte-compiler should issue (t for all).
@@ -310,6 +317,8 @@ Elements of the list may be:
   interactive-only
              commands that normally shouldn't be called from Lisp code.
   lexical     global/dynamic variables lacking a prefix.
+  lexical-dynamic
+              lexically bound variable declared dynamic elsewhere
   make-local  calls to make-variable-buffer-local that may be incorrect.
   mapcar      mapcar called for effect.
   constants   let-binding of, or assignment to, constants/nonvariables.
@@ -1873,6 +1882,17 @@ If compilation is needed, this functions returns the 
result of
        (load (if (file-exists-p dest) dest filename)))
       'no-byte-compile)))
 
+(defun byte-compile--load-dynvars (file)
+  (and file (not (equal file ""))
+       (with-temp-buffer
+         (insert-file-contents file)
+         (goto-char (point-min))
+         (let ((vars nil)
+               var)
+           (while (ignore-errors (setq var (read (current-buffer))))
+             (push var vars))
+           vars))))
+
 (defvar byte-compile-level 0           ; bug#13787
   "Depth of a recursive byte compilation.")
 
@@ -1911,6 +1931,9 @@ The value is non-nil if there were no errors, nil if 
errors."
   (let ((byte-compile-current-file filename)
         (byte-compile-current-group nil)
        (set-auto-coding-for-load t)
+        (byte-compile--seen-defvars nil)
+        (byte-compile--known-dynamic-vars
+         (byte-compile--load-dynvars (getenv "EMACS_DYNVARS_FILE")))
        target-file input-buffer output-buffer
        byte-compile-dest-file)
     (setq target-file (byte-compile-dest-file filename))
@@ -2035,6 +2058,15 @@ The value is non-nil if there were no errors, nil if 
errors."
                                        filename))))
            (save-excursion
              (display-call-tree filename)))
+        (let ((gen-dynvars (getenv "EMACS_GENERATE_DYNVARS")))
+          (when (and gen-dynvars (not (equal gen-dynvars ""))
+                     byte-compile--seen-defvars)
+            (let ((dynvar-file (concat target-file ".dynvars")))
+              (message "Generating %s" dynvar-file)
+              (with-temp-buffer
+                (dolist (var (delete-dups byte-compile--seen-defvars))
+                  (insert (format "%S\n" (cons var filename))))
+               (write-region (point-min) (point-max) dynvar-file)))))
        (if load
            (load target-file))
        t))))
@@ -2425,7 +2457,8 @@ list that represents a doc string reference.
     (setq byte-compile-lexical-variables
           (delq sym byte-compile-lexical-variables))
     (byte-compile-warn "Variable `%S' declared after its first use" sym))
-  (push sym byte-compile-bound-variables))
+  (push sym byte-compile-bound-variables)
+  (push sym byte-compile--seen-defvars))
 
 (defun byte-compile-file-form-defvar (form)
   (let ((sym (nth 1 form)))
@@ -2831,6 +2864,16 @@ If FORM is a lambda or a macro, byte-compile it as a 
function."
               (ash nonrest 8)
               (ash rest 7)))))
 
+(defun byte-compile--warn-lexical-dynamic (var context)
+  (when (byte-compile-warning-enabled-p 'lexical-dynamic var)
+    (byte-compile-warn
+     "`%s' lexically bound in %s here but declared dynamic in: %s"
+     var context
+     (mapconcat #'identity
+                (mapcan (lambda (v) (and (eq var (car v))
+                                         (list (cdr v))))
+                        byte-compile--known-dynamic-vars)
+                ", "))))
 
 (defun byte-compile-lambda (fun &optional add-lambda reserved-csts)
   "Byte-compile a lambda-expression and return a valid function.
@@ -2859,6 +2902,10 @@ for symbols generated by the byte compiler itself."
                     (if (cdr body)
                         (setq body (cdr body))))))
         (int (assq 'interactive body)))
+    (when lexical-binding
+      (dolist (var arglistvars)
+        (when (assq var byte-compile--known-dynamic-vars)
+          (byte-compile--warn-lexical-dynamic var 'lambda))))
     ;; Process the interactive spec.
     (when int
       (byte-compile-set-symbol-position 'interactive)
@@ -4379,6 +4426,8 @@ Return non-nil if the TOS value was popped."
       ;; VAR is a simple stack-allocated lexical variable.
       (progn (push (assq var init-lexenv)
                    byte-compile--lexical-environment)
+             (when (assq var byte-compile--known-dynamic-vars)
+               (byte-compile--warn-lexical-dynamic var 'let))
              nil)
     ;; VAR should be dynamically bound.
     (while (assq var byte-compile--lexical-environment)