From: Mark Wooding <mdw@distorted.org.uk>
Date: Thu, 26 May 2016 08:26:09 +0000 (+0100)
Subject: src/lexer-impl.lisp: Don't always skip a token.
X-Git-Url: https://git.distorted.org.uk/~mdw/sod/commitdiff_plain/450a4be6a6d832ce1e54169d9cc7740f5a04dc89?ds=inline

src/lexer-impl.lisp: Don't always skip a token.

It's usually, or at least often, the wrong thing to do.  Consider, for
example, how to recover from trying to parse

	( list[x] )

when presented with

	( )

Instead, add a `:force-progress' argument to the `error' parser.  The
default is /not/ to consume a token; I think this is right.
---

diff --git a/doc/parsing.tex b/doc/parsing.tex
index d0671a2..439d6d9 100644
--- a/doc/parsing.tex
+++ b/doc/parsing.tex
@@ -792,7 +792,8 @@ file-location protocols.
 \end{describe}
 
 \begin{describe}{parseform}
-    {error (@[[ :ignore-unconsumed @<flag> @]])                 \\ \ind
+    {error (@[[ :ignore-unconsumed @<flag> @!
+                :force-process @<flag> @]])                     \\ \ind
       @<sub-parser> @<recover-parser>}
 \end{describe}
 
diff --git a/src/lexer-impl.lisp b/src/lexer-impl.lisp
index 821849b..f00994a 100644
--- a/src/lexer-impl.lisp
+++ b/src/lexer-impl.lisp
@@ -59,7 +59,8 @@
       (return (values token-types nil consumedp)))
     (scanner-step scanner)))
 
-(defun parse-error-recover (scanner parser recover &key ignore-unconsumed)
+(defun parse-error-recover (scanner parser recover
+			    &key ignore-unconsumed force-progress)
   "This is the implementation of the `error' parser."
   (multiple-value-bind (result win consumedp) (funcall parser)
     (cond ((or win
@@ -84,8 +85,8 @@
 	   ;; current token.  Finally, if we are at EOF then our best bet is
 	   ;; simply to propagate the current failure back to the caller, but
 	   ;; we handled that case above.
-	   (syntax-error scanner result :continuep t)
-	   (unless consumedp (scanner-step scanner))
+	   (syntax-error scanner result)
+	   (when (and force-progress (not consumedp)) (scanner-step scanner))
 	   (funcall recover)))))
 
 ;;;--------------------------------------------------------------------------
diff --git a/src/lexer-proto.lisp b/src/lexer-proto.lisp
index c6d6f28..349e2a0 100644
--- a/src/lexer-proto.lisp
+++ b/src/lexer-proto.lisp
@@ -116,7 +116,7 @@
 
 (export 'error)
 (defparse error (:context (context token-scanner-context)
-		 (&key ignore-unconsumed)
+		 (&key ignore-unconsumed force-progress)
 		 sub &optional (recover t))
   "Try to parse SUB; if it fails then report an error, and parse RECOVER.
 
@@ -135,7 +135,8 @@
   `(parse-error-recover ,(parser-scanner context)
 			(parser () ,sub)
 			(parser () ,recover)
-			:ignore-unconsumed ,ignore-unconsumed))
+			:ignore-unconsumed ,ignore-unconsumed
+			:force-progress ,force-progress))
 
 ;;;--------------------------------------------------------------------------
 ;;; Lexical analysis utilities.