3 ;;; Scanner protocol definitions.
5 ;;; (c) 2009 Straylight/Edgeware
8 ;;;----- Licensing notice ---------------------------------------------------
10 ;;; This file is part of the Sensble Object Design, an object system for C.
12 ;;; SOD is free software; you can redistribute it and/or modify
13 ;;; it under the terms of the GNU General Public License as published by
14 ;;; the Free Software Foundation; either version 2 of the License, or
15 ;;; (at your option) any later version.
17 ;;; SOD is distributed in the hope that it will be useful,
18 ;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
19 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 ;;; GNU General Public License for more details.
22 ;;; You should have received a copy of the GNU General Public License
23 ;;; along with SOD; if not, write to the Free Software Foundation,
24 ;;; Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
26 (cl:in-package #:sod-parser)
28 ;;;--------------------------------------------------------------------------
29 ;;; Scanner context protocol.
31 (export 'parser-scanner)
32 (defgeneric parser-scanner (context)
34 "Return the symbol naming the CONTEXT's run-time scanner."))
36 (export 'scanner-context)
37 (defclass scanner-context ()
38 ((scanner :initarg :scanner :type symbol :reader parser-scanner))
40 "Base class for scanner contexts.
42 A scanner is simply an object maintaining the run-time state of a parsing
43 operation, in the same way as a parser context maintains the compile-time
44 state. So the scanner context is a compile-time context which expands to
45 calls to use the run-time scanner. See?
47 This class provides common compile-time behaviour for `parser-at-eof-p'
48 and friends by invoking corresponding methods on the scanner object at
51 ;;;--------------------------------------------------------------------------
52 ;;; Basic scanner protocol.
54 (export 'scanner-at-eof-p)
55 (defgeneric scanner-at-eof-p (scanner)
57 "Answer whether the SCANNER is at end-of-file.
59 It is an error to query the current item when at end-of-file."))
61 (export 'scanner-step)
62 (defgeneric scanner-step (scanner)
64 "Advance the SCANNER to the next item.
66 The precise nature of the items isn't known at this level, so a protocol
67 for accessing them is left for later."))
69 ;;;--------------------------------------------------------------------------
70 ;;; Scanner place-capture protocol.
72 (export 'scanner-capture-place)
73 (defgeneric scanner-capture-place (scanner)
75 "Capture the SCANNER's current place and return it.")
77 (error "Scanner ~S doesn't support rewinding." scanner)))
79 (export 'scanner-restore-place)
80 (defgeneric scanner-restore-place (scanner place)
82 "`Rewind' the SCANNER to the captured PLACE.
84 The place was previously captured by `scanner-capture-place'."))
86 (export 'scanner-release-place)
87 (defgeneric scanner-release-place (scanner place)
89 "Release a PLACE captured from the SCANNER.
91 The place was previously captured by `scanner-capture-place'.")
92 (:method (scanner place) nil))
94 (export 'with-scanner-place)
95 (defmacro with-scanner-place ((place scanner) &body body)
96 "Evaluate BODY with PLACE bound to the captured current place.
98 Automatically releases the place when the BODY finishes. Note that
99 if you wanted to circumvent the cleanup then you should have used
100 `with-parser-place', which does all of this in the meta-level."
102 `(let ((,place (scanner-capture-place ,scanner)))
103 (unwind-protect (progn ,@body)
104 (scanner-release-place ,scanner ,place)))))
106 ;;;--------------------------------------------------------------------------
107 ;;; Character scanner protocol.
109 (export 'character-scanner)
110 (defclass character-scanner ()
112 (:documentation "Base class for character scanners."))
114 (export 'character-scanner-context)
115 (defclass character-scanner-context
116 (scanner-context character-parser-context)
119 "A context for a richer character-oriented scanner."))
121 (export 'scanner-current-char)
122 (defgeneric scanner-current-char (scanner)
124 "Returns the SCANNER's current character.
126 You advance to the next one using `scanner-step'."))
128 (export 'scanner-unread)
129 (defgeneric scanner-unread (scanner char)
131 "Rewind SCANNER by one character, specifically CHAR.
133 CHAR must be the character most recently stepped over by `scanner-step' --
134 it is an error to unread before the first call to `scanner-step'. It is
135 also an error to unread after encountering end-of-file."))
137 (export 'scanner-interval)
138 (defgeneric scanner-interval (scanner place-a &optional place-b)
140 "Return the characters from PLACE-A up to (but not including) PLACE-B.
142 The characters are returned as a string. If PLACE-B is omitted, return
143 the characters up to (but not including) the current position. It is an
144 error if PLACE-B precedes PLACE-A or they are from different scanners."))
146 (export '(scanner-filename scanner-line scanner-column))
147 (defgeneric scanner-filename (scanner)
148 (:documentation "Return the filename backing the SCANNER.")
149 (:method (scanner) nil))
150 (defgeneric scanner-line (scanner)
151 (:documentation "Return the SCANNER's current line number.")
152 (:method (scanner) nil))
153 (defgeneric scanner-column (scanner)
154 (:documentation "Return the SCANNER's current column number.")
155 (:method (scanner) nil))
157 (defun scanner-file-location (scanner)
158 "Capture the current location of the SCANNER.
160 This uses the generic functions `scanner-filename', `scanner-line' and
161 `scanner-column' to compute its result. There are default methods on
162 these functions which make up dummy results.
164 There is a method for `file-location' defined on `character-scanner' which
165 simply calls this function; but since some scanners are structure-objects
166 rather than standard-objects they can't include `character-scanner' as a
168 (make-file-location (scanner-filename scanner)
169 (scanner-line scanner)
170 (scanner-column scanner)))
172 ;;;--------------------------------------------------------------------------
173 ;;; Token scanner protocol.
177 (export '(token-scanner-place token-scanner-place-p))
178 (defstruct token-scanner-place
179 "A link in the chain of lookahead tokens; capturable as a place.
181 If the scanner's place is captured, it starts to maintain a list of
182 lookahead tokens. The list contains internal links -- it works out
183 slightly easier that way. This is basically a simpler version of the
184 charbuf scanner (q.v.); most significantly, the chain links here do double
185 duty as place markers.
187 The details of this structure are not a defined part of the token scanner
190 (scanner nil :type token-scanner :read-only t)
191 (next nil :type (or token-scanner-place null))
192 (type nil :read-only t)
193 (value nil :read-only t)
194 (line 1 :type (or fixnum null) :read-only t)
195 (column 0 :type (or fixnum null) :read-only t))
197 ;; The token scanner base class and parser context.
199 (export '(token-scanner token-type token-value))
200 (defclass token-scanner ()
201 ((type :reader token-type)
202 (value :reader token-value)
203 (captures :initform 0 :type fixnum)
204 (tail :initform nil :type (or token-scanner-place null))
205 (filename :initarg :filename :type string :reader scanner-filename)
206 (line :initarg :line :initform 1 :type fixnum :accessor scanner-line)
207 (column :initarg :column :initform 0
208 :type fixnum :accessor scanner-column))
210 "A rewindable scanner for tokenizing.
212 The scanner should be used via the parser protocol; see also the token
213 scanner protocol, which explains the model.
215 Subclasses must provide the detailed scanning behaviour -- most notably
216 the `scanner-token' generic function -- and also implement a method on
217 `file-location' to return the location. The `scanner-token' method should
218 also update the `line' and `column' slots to track the position in the
219 underlying source, if appropriate. This class will handle the remaining
220 details, such as dealing correctly with rewinding."))
222 (export 'token-scanner-context)
223 (defclass token-scanner-context (scanner-context token-parser-context)
226 "A parser context for a richer token-based scanners."))
230 (export 'scanner-token)
231 (defgeneric scanner-token (scanner)
233 "Internal protocol: read the next token from the SCANNER.
235 This function is called by `scanner-step' to actually read the next token
236 if necessary. It should return two values: the token's `type' and its
239 ;;;--------------------------------------------------------------------------
240 ;;; Character scanner streams.
242 ;;; This seems like an abstraction inversion, but it's important if we're to
243 ;;; `read' from a character scanner.
245 (export 'character-scanner-stream)
246 (defclass character-scanner-stream (fundamental-character-input-stream)
247 ((scanner :initarg :scanner))
249 "A stream which reads from a character scanner.
251 The SCANNER must implement the character scanner protcol, including
252 `scanner-current-char', `scanner-step', and `scanner-unread'; it is not
253 necessary that the scanner implement the place-capture protocol.
255 The stream can be made more efficient by implementing
256 `stream-read-sequence' and `stream-read-line' in a scanner-specific
259 (export 'make-scanner-stream)
260 (defgeneric make-scanner-stream (scanner)
262 "Return a stream which reads from the SCANNER.
264 The default method simply constructs a `character-scanner-stream'
265 instance. Subclasses of `character-scanner' can override this method in
266 order to return instances of more efficient stream subclasses.")
267 (:method ((scanner character-scanner))
268 (make-instance 'character-scanner-stream :scanner scanner)))
270 ;;;----- That's all, folks --------------------------------------------------