[elpa] externals/parser-generator 096436cccd 15/19: Added unit test for state-based lexer

[Christian Johansson]

branch: externals/parser-generator
commit 096436cccd98c098bfb92272900518007b980117
Author: Christian Johansson <christian@cvj.se>
Commit: Christian Johansson <christian@cvj.se>

    Added unit test for state-based lexer
---
 TODO.md                                    |   1 -
 parser-generator-lex-analyzer.el           |   9 ++-
 test/parser-generator-lex-analyzer-test.el | 124 ++++++++++++++++++++++++++++-
 3 files changed, 130 insertions(+), 4 deletions(-)

diff --git a/TODO.md b/TODO.md
index bc6e33d65e..b72f9c8b94 100644
--- a/TODO.md
+++ b/TODO.md
@@ -15,7 +15,6 @@ Functions (with validations) to set global variables:
 * Change lex-function return value count
 * Verify that parser-generator-lex-analyzer--index is used in exported 
lex-analyzers
 * Verify that parser-generator-lex-analyzer--state-init is used in exported 
lex-analyzers
-* Use buffer when lexing in case more tokens are return than what are needed
 * Use buffer when lexing in exported parsers as well
 
 ## LR-Parser
diff --git a/parser-generator-lex-analyzer.el b/parser-generator-lex-analyzer.el
index f3d4c323bc..52a2e841a9 100644
--- a/parser-generator-lex-analyzer.el
+++ b/parser-generator-lex-analyzer.el
@@ -132,7 +132,9 @@
              (token
               (nth 0 result-list))
              (new-index
-              (nth 2 result-list)))
+              (nth 2 result-list))
+             (new-state
+              (nth 3 result-list)))
         (push
          token
          look-ahead)
@@ -141,7 +143,10 @@
          (1+ look-ahead-length))
         (setq
          index
-         new-index)))
+         new-index)
+        (setq
+         state
+         new-state)))
 
     (nreverse look-ahead)))
 
diff --git a/test/parser-generator-lex-analyzer-test.el 
b/test/parser-generator-lex-analyzer-test.el
index b14d3734b7..eac8a1eb69 100644
--- a/test/parser-generator-lex-analyzer-test.el
+++ b/test/parser-generator-lex-analyzer-test.el
@@ -88,7 +88,58 @@
 
   (parser-generator-lex-analyzer--reset)
   (should-error
-    (parser-generator-lex-analyzer--peek-next-look-ahead))
+   (parser-generator-lex-analyzer--peek-next-look-ahead))
+
+  (setq parser-generator--look-ahead-number 6)
+  (setq
+   parser-generator-lex-analyzer--function
+   (lambda (index state)
+     (let* ((string '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5) ("e" 5 . 
6)))
+            (string-length (length string))
+            (max-index index)
+            (tokens)
+            (new-index)
+            (new-state)
+            (next-token))
+       (while (and
+               (< (1- index) string-length)
+               (< (1- index) max-index))
+         (setq next-token (nth (1- index) string))
+         (cond
+          ((equal state nil)
+           (if (string= (car next-token) "c")
+               (progn
+                 (push next-token tokens)
+                 (setq index (1+ index))
+                 (setq next-token (nth (1- index) string))
+                 (push next-token tokens)
+                 (setq index (1+ index))
+                 (setq new-state (list 'epislon)))
+             (if (string= (car next-token) "e")
+                 (error "Invalid token: %s" next-token)
+               (setq next-token (nth (1- index) string))
+               (when (string= (car next-token) "d")
+                 (error "Invalid token: %s" next-token))
+               (setq new-index (cdr (cdr (nth (1- index) string))))
+               (push next-token tokens)
+               (setq index (1+ index)))))
+          ((equal state (list 'epislon))
+           (setq next-token (nth (1- index) string))
+           (when (string= (car next-token) "d")
+             (error "Invalid token: %s" next-token))
+           (setq new-index (cdr (cdr (nth (1- index) string))))
+           (push next-token tokens)
+           (setq index (1+ index)))
+          (t
+           (error "Invalid state: %s" state))))
+       (list (nreverse tokens) nil nil new-state))))
+  (parser-generator-lex-analyzer--reset)
+  (should
+   (equal
+    '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5) ("e" 5 . 6) ($))
+    (parser-generator-lex-analyzer--peek-next-look-ahead)))
+
+  (message "Passed peek for state-based lexer")
 
   (message "Ended tests for 
(parser-generator-lex-analyzer--peek-next-look-ahead)"))
 
@@ -138,6 +189,77 @@
     '(($))
     (parser-generator-lex-analyzer--pop-token)))
 
+  (setq parser-generator--look-ahead-number 1)
+  (setq
+   parser-generator-lex-analyzer--function
+   (lambda (index state)
+     (let* ((string '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5) ("e" 5 . 
6)))
+            (string-length (length string))
+            (max-index index)
+            (tokens)
+            (new-index)
+            (new-state)
+            (next-token))
+       (while (and
+               (< (1- index) string-length)
+               (< (1- index) max-index))
+         (setq next-token (nth (1- index) string))
+         (cond
+          ((equal state nil)
+           (if (string= (car next-token) "c")
+               (progn
+                 (push next-token tokens)
+                 (setq index (1+ index))
+                 (setq next-token (nth (1- index) string))
+                 (push next-token tokens)
+                 (setq index (1+ index))
+                 (setq new-state (list 'epislon)))
+             (if (string= (car next-token) "e")
+                 (error "Invalid token: %s" next-token)
+               (setq next-token (nth (1- index) string))
+               (when (string= (car next-token) "d")
+                 (error "Invalid token: %s" next-token))
+               (setq new-index (cdr (cdr (nth (1- index) string))))
+               (push next-token tokens)
+               (setq index (1+ index)))))
+          ((equal state (list 'epislon))
+           (setq next-token (nth (1- index) string))
+           (when (string= (car next-token) "d")
+             (error "Invalid token: %s" next-token))
+           (setq new-index (cdr (cdr (nth (1- index) string))))
+           (push next-token tokens)
+           (setq index (1+ index)))
+          (t
+           (error "Invalid state: %s" state))))
+       (list (nreverse tokens) nil nil new-state))))
+  (parser-generator-lex-analyzer--reset)
+  (should
+   (equal
+    '(("a" 1 . 2))
+    (parser-generator-lex-analyzer--pop-token)))
+  (should
+   (equal
+    '(("b" 2 . 3))
+    (parser-generator-lex-analyzer--pop-token)))
+  (should
+   (equal
+    '(("c" 3 . 4))
+    (parser-generator-lex-analyzer--pop-token)))
+  (should
+   (equal
+    '(("d" 4 . 5))
+    (parser-generator-lex-analyzer--pop-token)))
+  (should
+   (equal
+    '(("e" 5 . 6))
+    (parser-generator-lex-analyzer--pop-token)))
+  (should
+   (equal
+    '(($))
+    (parser-generator-lex-analyzer--pop-token)))
+
+  (message "Passed pop for state-based lexer")
+
   (message "Ended tests for (parser-generator-lex-analyzer--pop-token)"))
 
 (defun parser-generator-lex-analyzer-test ()