master f2a3d6e: Support using ripgrep in project-find-regexp and friends

[Dmitry Gutov]

branch: master
commit f2a3d6e28d81dc2890e99afd550b59ff01fac5d9
Author: Dmitry Gutov <dgutov@yandex.ru>
Commit: Dmitry Gutov <dgutov@yandex.ru>

    Support using ripgrep in project-find-regexp and friends
    
    Performance results vary here.  Some projects and search terms
    don't see much of a change, but for some (including Emacs sources
    checkout and case-insensitive search) the switch to ripgrep shows
    ~2-3x speed improvement.  Another piece of anecdata here:
    https://lists.gnu.org/archive/html/emacs-devel/2020-06/msg00802.html
    
    * lisp/progmodes/xref.el (xref-search-program-alist)
    (xref-search-program): New user options.
    (xref-matches-in-files): Use them.
---
 etc/NEWS               |  3 +++
 lisp/progmodes/xref.el | 62 ++++++++++++++++++++++++++++++++++++++++++++------
 2 files changed, 58 insertions(+), 7 deletions(-)

diff --git a/etc/NEWS b/etc/NEWS
index c9da296..051adf3 100644
--- a/etc/NEWS
+++ b/etc/NEWS
@@ -1187,6 +1187,9 @@ project's root directory, respectively.
 So typing 'C-u RET' in the "*xref*" buffer quits its window
 before navigating to the selected location.
 
+*** New option xref-search-program.
+So far Grep and ripgrep are supported.
+
 ** json.el
 
 ---
diff --git a/lisp/progmodes/xref.el b/lisp/progmodes/xref.el
index e1dd6e5..2a47809 100644
--- a/lisp/progmodes/xref.el
+++ b/lisp/progmodes/xref.el
@@ -1262,12 +1262,56 @@ IGNORES is a list of glob patterns for files to ignore."
 (declare-function tramp-tramp-file-p "tramp")
 (declare-function tramp-file-local-name "tramp")
 
+;; TODO: Experiment with 'xargs -P4' (or any other number).
+;; This speeds up either command, even more than rg's '-j4' does.
+;; Ripgrep gets jumbled output, though, even with --line-buffered.
+;; But Grep seems to be stable. Even without --line-buffered.
+(defcustom xref-search-program-alist
+  '((grep
+     .
+     ;; '-s' because 'git ls-files' can output broken symlinks.
+     "xargs -0 grep <C> -snHE -e <R>")
+    (ripgrep
+     .
+     ;; Note: by default, ripgrep's output order is non-deterministic
+     ;; (https://github.com/BurntSushi/ripgrep/issues/152)
+     ;; because it does the search in parallel.  You can use the template
+     ;; without the '| sort ...' part if GNU sort is not available on
+     ;; your system and/or stable ordering is not important to you.
+     ;; Note#2: '!*/' is there to filter out dirs (e.g. submodules).
+     "xargs -0 rg <C> -nH --no-messages -g '!*/' -e <R> | sort -t: -k1 -k2n"
+     ))
+  "Associative list mapping program identifiers to command templates.
+
+Program identifier should be a symbol, named after the search program.
+
+The command template must be a shell command (or usually a
+pipeline) that will search the list of files which will be piped
+from stdin.  The template should have following fields:
+
+  <C> for extra arguments such as -i and --color
+  <R> for the regexp itself (in Extended format)"
+  :type '(repeat
+          (cons (symbol :tag "Program identifier")
+                (string :tag "Command template"))))
+
+(defcustom xref-search-program 'grep
+  "The program to use to search inside files.
+
+This must reference a corresponding entry in `xref-search-program-alist'."
+  :type `(choice
+          (const :tag "Use Grep" grep)
+          (const :tag "Use ripgrep" ripgrep)
+          (symbol :tag "User defined")))
+
 ;;;###autoload
 (defun xref-matches-in-files (regexp files)
   "Find all matches for REGEXP in FILES.
 Return a list of xref values.
 FILES must be a list of absolute file names."
   (cl-assert (consp files))
+  (require 'grep)
+  (defvar grep-highlight-matches)
   (pcase-let*
       ((output (get-buffer-create " *project grep output*"))
        (`(,grep-re ,file-group ,line-group . ,_) (car grep-regexp-alist))
@@ -1277,13 +1321,17 @@ FILES must be a list of absolute file names."
        ;; first file is remote, they all are, and on the same host.
        (dir (file-name-directory (car files)))
        (remote-id (file-remote-p dir))
-       ;; 'git ls-files' can output broken symlinks.
-       (command (format "xargs -0 grep %s -snHE -e %s"
-                        (if (and case-fold-search
-                                 (isearch-no-upper-case-p regexp t))
-                            "-i"
-                          "")
-                        (shell-quote-argument (xref--regexp-to-extended 
regexp)))))
+       ;; The 'auto' default would be fine too, but ripgrep can't handle
+       ;; the options we pass in that case.
+       (grep-highlight-matches nil)
+       (command (grep-expand-template (cdr
+                                       (or
+                                        (assoc
+                                         xref-search-program
+                                         xref-search-program-alist)
+                                        (user-error "Unknown search program 
`%s'"
+                                                    xref-search-program)))
+                                      (xref--regexp-to-extended regexp))))
     (when remote-id
       (require 'tramp)
       (setq files (mapcar