feature/tree-sitter e0760599b0 3/5: Remove the contextual hack in tree-sitter fontification

[Yuan Fu]

branch: feature/tree-sitter
commit e0760599b045a7ef3828ae4b624246b6beec2e75
Author: Yuan Fu <casouri@gmail.com>
Commit: Yuan Fu <casouri@gmail.com>

    Remove the contextual hack in tree-sitter fontification
    
    We now have a better facility that can replace the contextual
    hack. The C part is in the previous commit, and the Lisp part work
    will be in the next commit.
    
    * doc/lispref/modes.texi (Parser-based Font Lock): Update manual.
    * lisp/progmodes/js.el (js--treesit-font-lock-settings)
    * lisp/progmodes/python.el (python--treesit-settings)
    * lisp/progmodes/ts-mode.el (ts-mode--font-lock-settings): Stop marking
    contextual nodes.
    * lisp/treesit.el (treesit-font-lock-contextual-post-process): Remove
    function.
    (treesit-font-lock-fontify-region): Remove code processing contextual
    nodes.
---
 doc/lispref/modes.texi    | 20 -------------------
 lisp/progmodes/js.el      |  5 +----
 lisp/progmodes/python.el  |  3 +--
 lisp/progmodes/ts-mode.el |  5 +----
 lisp/treesit.el           | 51 -----------------------------------------------
 5 files changed, 3 insertions(+), 81 deletions(-)

diff --git a/doc/lispref/modes.texi b/doc/lispref/modes.texi
index f61848c948..71687b0783 100644
--- a/doc/lispref/modes.texi
+++ b/doc/lispref/modes.texi
@@ -4064,26 +4064,6 @@ priority.  If a capture name is neither a face nor a 
function, it is
 ignored.
 @end defun
 
-Contextual entities, like multi-line strings, or @code{/* */} style
-comments, need special care, because change in these entities might
-cause change in a large portion of the buffer.  For example, inserting
-the closing comment delimiter @code{*/} will change all the text
-between it and the opening delimiter to comment face.  Such entities
-should be captured in a special name @code{contextual}, so Emacs can
-correctly update their fontification.  Here is an example for
-comments:
-
-@example
-@group
-(treesit-font-lock-rules
- :language 'javascript
- :feature 'comment
- :override t
- '((comment) @@font-lock-comment-face)
-   (comment) @@contextual))
-@end group
-@end example
-
 @defvar treesit-font-lock-feature-list
 This is a list of lists of feature symbols.  Each element of the list
 is a list that represents a decoration level.
diff --git a/lisp/progmodes/js.el b/lisp/progmodes/js.el
index ac2b6a27b7..fd3737f8b6 100644
--- a/lisp/progmodes/js.el
+++ b/lisp/progmodes/js.el
@@ -3455,8 +3455,7 @@ This function is intended for use in 
`after-change-functions'."
    :language 'javascript
    :override t
    :feature 'comment
-   `((comment) @font-lock-comment-face
-     (comment) @contextual)
+   `((comment) @font-lock-comment-face)
 
    :language 'javascript
    :override t
@@ -3478,9 +3477,7 @@ This function is intended for use in 
`after-change-functions'."
    :feature 'string
    `((regex pattern: (regex_pattern)) @font-lock-string-face
      (string) @font-lock-string-face
-     (string) @contextual
      (template_string) @js--fontify-template-string
-     (template_string) @contextual
      (template_substitution ["${" "}"] @font-lock-builtin-face))
 
    :language 'javascript
diff --git a/lisp/progmodes/python.el b/lisp/progmodes/python.el
index a5d02d0fcb..7919484e09 100644
--- a/lisp/progmodes/python.el
+++ b/lisp/progmodes/python.el
@@ -1047,8 +1047,7 @@ be fontified."
    :language 'python
    :override t
    ;; TODO Document on why we do this.
-   '((string :anchor "\"" @python--treesit-fontify-string)
-     (string) @contextual)
+   '((string :anchor "\"" @python--treesit-fontify-string))
 
    :feature 'string-interpolation
    :language 'python
diff --git a/lisp/progmodes/ts-mode.el b/lisp/progmodes/ts-mode.el
index c5a6f5fc60..c826302c7a 100644
--- a/lisp/progmodes/ts-mode.el
+++ b/lisp/progmodes/ts-mode.el
@@ -104,8 +104,7 @@
    :language 'tsx
    :override t
    :feature 'comment
-   `((comment) @font-lock-comment-face
-     (comment) @contextual)
+   `((comment) @font-lock-comment-face)
 
    :language 'tsx
    :override t
@@ -127,9 +126,7 @@
    :feature 'string
    `((regex pattern: (regex_pattern)) @font-lock-string-face
      (string) @font-lock-string-face
-     (string) @contextual
      (template_string) @js--fontify-template-string
-     (template_string) @contextual
      (template_substitution ["${" "}"] @font-lock-builtin-face))
 
    :language 'tsx
diff --git a/lisp/treesit.el b/lisp/treesit.el
index 1070b66f70..31a31dda41 100644
--- a/lisp/treesit.el
+++ b/lisp/treesit.el
@@ -777,53 +777,6 @@ instead."
         (remove-text-properties start end '(rear-nonsticky nil))
       (put-text-property start end 'rear-nonsticky new-prop))))
 
-;; This post-processing tries to deal with the following scenario:
-;; User inserts "/*" in a buffer under C mode, then goes down the
-;; buffer and inserts "*/".  Before the user inserts "*/", tree-sitter
-;; cannot construct a comment node and the parse tree is incomplete,
-;; and we can't fontify the comment.  But once the user inserts the
-;; "*/", the parse-tree is complete and we want to refontify the whole
-;; comment, and possibly text after comment (the "/*" could damage the
-;; parse tree enough that makes tree-sitter unable to produce
-;; reasonable information for text after it).
-;;
-;; So we set jit-lock-context-unfontify-pos to comment start, and
-;; jit-lock-context will refontify text after that position in a
-;; timer.  Refontifying those text will end up calling this function
-;; again, and we don't want to fall into infinite recursion.  So we
-;; mark the end of the comment with a text property, to be able to
-;; distinguish between initial and follow-up invocation of this
-;; function.
-(defun treesit-font-lock-contextual-post-process
-    (node start end &optional verbose)
-  "Post-process contextual syntax NODE for fontification between START and END.
-NODE is a comment or string node, START and END specify the region
-being fontified.
-
-If VERBOSE is non-nil, print debugging information."
-  (let* ((node-start (treesit-node-start node))
-         (node-end (treesit-node-end node))
-         (node-end-1 (max (point-min) (1- node-end)))
-         (prop-sym 'treesit-context-refontify-in-progress))
-    (when verbose
-      (message "Contextual: region: %s-%s, node: %s-%s"
-               start end node-start node-end))
-    (when (and (< node-start start) (<= node-end end))
-      (if (get-text-property node-end-1 prop-sym)
-          ;; We are called from a refontification by jit-lock-context,
-          ;; caused by a previous call to this function.
-          (progn (when verbose
-                   (message "Contextual: in progress"))
-                 (remove-text-properties
-                  node-end-1 node-end `(,prop-sym nil))
-                 (treesit--set-nonsticky node-end-1 node-end prop-sym t))
-        ;; We are called from a normal fontification.
-        (when verbose
-          (message "Contextual: initial"))
-        (setq jit-lock-context-unfontify-pos node-start)
-        (put-text-property node-end-1 node-end prop-sym t)
-        (treesit--set-nonsticky node-end-1 node-end prop-sym)))))
-
 ;; Some details worth explaining:
 ;;
 ;; 1. When we apply face to a node, we clip the face into the
@@ -870,10 +823,6 @@ If LOUDLY is non-nil, display some debugging information."
                 (when (and (< start node-end)
                            (< node-start end))
                   (cond
-                   ((eq face 'contextual)
-                    (treesit-font-lock-contextual-post-process
-                     node start end
-                     (or loudly treesit--font-lock-verbose)))
                    ((facep face)
                     (treesit-fontify-with-override
                      (max node-start start) (min node-end end)