master 5af9918: Allow back-references in syntax-propertize-rules.

[Tassilo Horn]

branch: master
commit 5af991872d5024b69272588772961bafef5a35bb
Author: Tassilo Horn <address@hidden>
Commit: Tassilo Horn <address@hidden>

    Allow back-references in syntax-propertize-rules.
    
    * lisp/emacs-lisp/syntax.el (syntax-propertize--shift-groups-and-backrefs):
    Renamed from syntax-propertize--shift-groups, and also shift
    back-references.
    (syntax-propertize-rules): Adapt docstring and use renamed function.
    * test/lisp/emacs-lisp/syntax-tests.el: New test.
    (syntax-propertize--shift-groups-and-backrefs): New ERT test.
---
 lisp/emacs-lisp/syntax.el            | 35 +++++++++++++------
 test/lisp/emacs-lisp/syntax-tests.el | 67 ++++++++++++++++++++++++++++++++++++
 2 files changed, 92 insertions(+), 10 deletions(-)

diff --git a/lisp/emacs-lisp/syntax.el b/lisp/emacs-lisp/syntax.el
index 46dc8d9..ce495af 100644
--- a/lisp/emacs-lisp/syntax.el
+++ b/lisp/emacs-lisp/syntax.el
@@ -139,14 +139,28 @@ delimiter or an Escaped or Char-quoted character."))
                  (point-max))))
   (cons beg end))
 
-(defun syntax-propertize--shift-groups (re n)
-  (replace-regexp-in-string
-   "\\\\(\\?\\([0-9]+\\):"
-   (lambda (s)
-     (replace-match
-      (number-to-string (+ n (string-to-number (match-string 1 s))))
-      t t s 1))
-   re t t))
+(defun syntax-propertize--shift-groups-and-backrefs (re n)
+  (let ((new-re (replace-regexp-in-string
+                 "\\\\(\\?\\([0-9]+\\):"
+                 (lambda (s)
+                   (replace-match
+                    (number-to-string
+                     (+ n (string-to-number (match-string 1 s))))
+                    t t s 1))
+                 re t t))
+        (pos 0))
+    (while (string-match "\\\\\\([0-9]+\\)" new-re pos)
+      (setq pos (+ 1 (match-beginning 1)))
+      (when (save-match-data
+              ;; With \N, the \ must be in a subregexp context, i.e.,
+              ;; not in a character class or in a \{\} repetition.
+              (subregexp-context-p new-re (match-beginning 0)))
+        (let ((shifted (+ n (string-to-number (match-string 1 new-re)))))
+          (when (> shifted 9)
+            (error "There may be at most nine back-references"))
+          (setq new-re (replace-match (number-to-string shifted)
+                                      t t new-re 1)))))
+    new-re))
 
 (defmacro syntax-propertize-precompile-rules (&rest rules)
   "Return a precompiled form of RULES to pass to `syntax-propertize-rules'.
@@ -190,7 +204,8 @@ for subsequent HIGHLIGHTs.
 Also SYNTAX is free to move point, in which case RULES may not be applied to
 some parts of the text or may be applied several times to other parts.
 
-Note: back-references in REGEXPs do not work."
+Note: There may be at most nine back-references in the REGEXPs of
+all RULES in total."
   (declare (debug (&rest &or symbolp    ;FIXME: edebug this eval step.
                          (form &rest
                                (numberp
@@ -219,7 +234,7 @@ Note: back-references in REGEXPs do not work."
                  ;; tell when *this* match 0 has succeeded.
                  (cl-incf offset)
                  (setq re (concat "\\(" re "\\)")))
-               (setq re (syntax-propertize--shift-groups re offset))
+               (setq re (syntax-propertize--shift-groups-and-backrefs re 
offset))
                (let ((code '())
                      (condition
                       (cond
diff --git a/test/lisp/emacs-lisp/syntax-tests.el 
b/test/lisp/emacs-lisp/syntax-tests.el
new file mode 100644
index 0000000..9d4c411
--- /dev/null
+++ b/test/lisp/emacs-lisp/syntax-tests.el
@@ -0,0 +1,67 @@
+;;; syntax-tests.el --- tests for syntax.el  -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2020 Free Software Foundation, Inc.
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+;;; Code:
+
+(require 'ert)
+(require 'syntax)
+
+(ert-deftest syntax-propertize--shift-groups-and-backrefs ()
+  "Test shifting of numbered groups and back-references in regexps."
+  ;; A numbered group must be shifted.
+  (should
+   (string=
+    (syntax-propertize--shift-groups-and-backrefs
+     "\\(?2:[abc]+\\)foobar" 2)
+    "\\(?4:[abc]+\\)foobar"))
+  ;; A back-reference \1 on a normal sub-regexp context must be
+  ;; shifted.
+  (should
+   (string=
+    (syntax-propertize--shift-groups-and-backrefs "\\(a\\)\\1" 2)
+    "\\(a\\)\\3"))
+  ;; Shifting must not happen if the \1 appears in a character class,
+  ;; or in a \{\} repetition construct (although \1 isn't valid there
+  ;; anyway).
+  (let ((rx-with-class "\\(a\\)[\\1-2]")
+        (rx-with-rep "\\(a\\)\\{1,\\1\\}"))
+    (should
+     (string=
+      (syntax-propertize--shift-groups-and-backrefs rx-with-class 2)
+      rx-with-class))
+    (should
+     (string=
+      (syntax-propertize--shift-groups-and-backrefs rx-with-rep 2)
+      rx-with-rep)))
+  ;; Now numbered groups and back-references in combination.
+  (should
+   (string=
+    (syntax-propertize--shift-groups-and-backrefs
+     "\\(?2:[abc]+\\)foo\\(\\2\\)" 2)
+    "\\(?4:[abc]+\\)foo\\(\\4\\)"))
+  ;; Emacs supports only the back-references \1,...,\9, so when a
+  ;; shift would result in \10 or more, an error must be signalled.
+  (should-error
+   (syntax-propertize--shift-groups-and-backrefs "\\(a\\)\\3" 7)))
+
+;; Local Variables:
+;; no-byte-compile: t
+;; End:
+
+;;; syntax-tests.el ends here.