| 1 | ;;; -*-lisp-*- |
| 2 | ;;; |
| 3 | ;;; Scanner protocol definitions. |
| 4 | ;;; |
| 5 | ;;; (c) 2009 Straylight/Edgeware |
| 6 | ;;; |
| 7 | |
| 8 | ;;;----- Licensing notice --------------------------------------------------- |
| 9 | ;;; |
| 10 | ;;; This file is part of the Sensible Object Design, an object system for C. |
| 11 | ;;; |
| 12 | ;;; SOD is free software; you can redistribute it and/or modify |
| 13 | ;;; it under the terms of the GNU General Public License as published by |
| 14 | ;;; the Free Software Foundation; either version 2 of the License, or |
| 15 | ;;; (at your option) any later version. |
| 16 | ;;; |
| 17 | ;;; SOD is distributed in the hope that it will be useful, |
| 18 | ;;; but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 19 | ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 20 | ;;; GNU General Public License for more details. |
| 21 | ;;; |
| 22 | ;;; You should have received a copy of the GNU General Public License |
| 23 | ;;; along with SOD; if not, write to the Free Software Foundation, |
| 24 | ;;; Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. |
| 25 | |
| 26 | (cl:in-package #:sod-parser) |
| 27 | |
| 28 | ;;;-------------------------------------------------------------------------- |
| 29 | ;;; Scanner context protocol. |
| 30 | |
| 31 | (export 'parser-scanner) |
| 32 | (defgeneric parser-scanner (context) |
| 33 | (:documentation |
| 34 | "Return the symbol naming the CONTEXT's run-time scanner.")) |
| 35 | |
| 36 | (export 'scanner-context) |
| 37 | (defclass scanner-context () |
| 38 | ((scanner :initarg :scanner :type symbol :reader parser-scanner)) |
| 39 | (:documentation |
| 40 | "Base class for scanner contexts. |
| 41 | |
| 42 | A scanner is simply an object maintaining the run-time state of a parsing |
| 43 | operation, in the same way as a parser context maintains the compile-time |
| 44 | state. So the scanner context is a compile-time context which expands to |
| 45 | calls to use the run-time scanner. See? |
| 46 | |
| 47 | This class provides common compile-time behaviour for `parser-at-eof-p' |
| 48 | and friends by invoking corresponding methods on the scanner object at |
| 49 | run-time.")) |
| 50 | |
| 51 | ;;;-------------------------------------------------------------------------- |
| 52 | ;;; Basic scanner protocol. |
| 53 | |
| 54 | (export 'scanner-at-eof-p) |
| 55 | (defgeneric scanner-at-eof-p (scanner) |
| 56 | (:documentation |
| 57 | "Answer whether the SCANNER is at end-of-file. |
| 58 | |
| 59 | It is an error to query the current item when at end-of-file.")) |
| 60 | |
| 61 | (export 'scanner-step) |
| 62 | (defgeneric scanner-step (scanner) |
| 63 | (:documentation |
| 64 | "Advance the SCANNER to the next item. |
| 65 | |
| 66 | The precise nature of the items isn't known at this level, so a protocol |
| 67 | for accessing them is left for later.")) |
| 68 | |
| 69 | ;;;-------------------------------------------------------------------------- |
| 70 | ;;; Scanner place-capture protocol. |
| 71 | |
| 72 | (export 'scanner-capture-place) |
| 73 | (defgeneric scanner-capture-place (scanner) |
| 74 | (:documentation |
| 75 | "Capture the SCANNER's current place and return it.") |
| 76 | (:method (scanner) |
| 77 | (error "Scanner ~S doesn't support rewinding." scanner))) |
| 78 | |
| 79 | (export 'scanner-restore-place) |
| 80 | (defgeneric scanner-restore-place (scanner place) |
| 81 | (:documentation |
| 82 | "`Rewind' the SCANNER to the captured PLACE. |
| 83 | |
| 84 | The place was previously captured by `scanner-capture-place'.")) |
| 85 | |
| 86 | (export 'scanner-release-place) |
| 87 | (defgeneric scanner-release-place (scanner place) |
| 88 | (:documentation |
| 89 | "Release a PLACE captured from the SCANNER. |
| 90 | |
| 91 | The place was previously captured by `scanner-capture-place'.") |
| 92 | (:method (scanner place) nil)) |
| 93 | |
| 94 | (export 'with-scanner-place) |
| 95 | (defmacro with-scanner-place ((place scanner) &body body) |
| 96 | "Evaluate BODY with PLACE bound to the captured current place. |
| 97 | |
| 98 | Automatically releases the place when the BODY finishes. Note that |
| 99 | if you wanted to circumvent the cleanup then you should have used |
| 100 | `with-parser-place', which does all of this in the meta-level." |
| 101 | (once-only (scanner) |
| 102 | (multiple-value-bind (docs decls body) (parse-body body :docp nil) |
| 103 | (declare (ignore docs)) |
| 104 | `(let ((,place (scanner-capture-place ,scanner))) |
| 105 | ,@decls |
| 106 | (unwind-protect (progn ,@body) |
| 107 | (scanner-release-place ,scanner ,place)))))) |
| 108 | |
| 109 | ;;;-------------------------------------------------------------------------- |
| 110 | ;;; Character scanner protocol. |
| 111 | |
| 112 | (export 'character-scanner) |
| 113 | (defclass character-scanner () |
| 114 | () |
| 115 | (:documentation "Base class for character scanners.")) |
| 116 | |
| 117 | (export 'character-scanner-context) |
| 118 | (defclass character-scanner-context |
| 119 | (scanner-context character-parser-context) |
| 120 | () |
| 121 | (:documentation |
| 122 | "A context for a richer character-oriented scanner.")) |
| 123 | |
| 124 | (export 'scanner-current-char) |
| 125 | (defgeneric scanner-current-char (scanner) |
| 126 | (:documentation |
| 127 | "Returns the SCANNER's current character. |
| 128 | |
| 129 | You advance to the next one using `scanner-step'.")) |
| 130 | |
| 131 | (export 'scanner-unread) |
| 132 | (defgeneric scanner-unread (scanner char) |
| 133 | (:documentation |
| 134 | "Rewind SCANNER by one character, specifically CHAR. |
| 135 | |
| 136 | CHAR must be the character most recently stepped over by `scanner-step' -- |
| 137 | it is an error to unread before the first call to `scanner-step'. It is |
| 138 | also an error to unread after encountering end-of-file.")) |
| 139 | |
| 140 | (export 'scanner-interval) |
| 141 | (defgeneric scanner-interval (scanner place-a &optional place-b) |
| 142 | (:documentation |
| 143 | "Return the characters from PLACE-A up to (but not including) PLACE-B. |
| 144 | |
| 145 | The characters are returned as a string. If PLACE-B is omitted, return |
| 146 | the characters up to (but not including) the current position. It is an |
| 147 | error if PLACE-B precedes PLACE-A or they are from different scanners.")) |
| 148 | |
| 149 | (export '(scanner-filename scanner-line scanner-column)) |
| 150 | (defgeneric scanner-filename (scanner) |
| 151 | (:documentation "Return the filename backing the SCANNER.") |
| 152 | (:method (scanner) nil)) |
| 153 | (defgeneric scanner-line (scanner) |
| 154 | (:documentation "Return the SCANNER's current line number.") |
| 155 | (:method (scanner) nil)) |
| 156 | (defgeneric scanner-column (scanner) |
| 157 | (:documentation "Return the SCANNER's current column number.") |
| 158 | (:method (scanner) nil)) |
| 159 | |
| 160 | (defun scanner-file-location (scanner) |
| 161 | "Capture the current location of the SCANNER. |
| 162 | |
| 163 | This uses the generic functions `scanner-filename', `scanner-line' and |
| 164 | `scanner-column' to compute its result. There are default methods on |
| 165 | these functions which make up dummy results. |
| 166 | |
| 167 | There is a method for `file-location' defined on `character-scanner' which |
| 168 | simply calls this function; but since some scanners are structure-objects |
| 169 | rather than standard-objects they can't include `character-scanner' as a |
| 170 | superclass." |
| 171 | (make-file-location (scanner-filename scanner) |
| 172 | (scanner-line scanner) |
| 173 | (scanner-column scanner))) |
| 174 | |
| 175 | ;;;-------------------------------------------------------------------------- |
| 176 | ;;; Token scanner protocol. |
| 177 | |
| 178 | ;; The token scanner base class and parser context. |
| 179 | |
| 180 | (export '(token-scanner token-type token-value)) |
| 181 | (defclass token-scanner () |
| 182 | ((%type :reader token-type) |
| 183 | (value :reader token-value) |
| 184 | (captures :initform 0 :type fixnum) |
| 185 | (tail :initform nil :type (or token-scanner-place null)) |
| 186 | (filename :initarg :filename :type string :reader scanner-filename) |
| 187 | (line :initarg :line :initform 1 :type fixnum :accessor scanner-line) |
| 188 | (column :initarg :column :initform 0 |
| 189 | :type fixnum :accessor scanner-column)) |
| 190 | (:documentation |
| 191 | "A rewindable scanner for tokenizing. |
| 192 | |
| 193 | The scanner should be used via the parser protocol; see also the token |
| 194 | scanner protocol, which explains the model. |
| 195 | |
| 196 | Subclasses must provide the detailed scanning behaviour -- most notably |
| 197 | the `scanner-token' generic function -- and also implement a method on |
| 198 | `file-location' to return the location. The `scanner-token' method should |
| 199 | also update the `line' and `column' slots to track the position in the |
| 200 | underlying source, if appropriate. This class will handle the remaining |
| 201 | details, such as dealing correctly with rewinding.")) |
| 202 | |
| 203 | (export 'token-scanner-context) |
| 204 | (defclass token-scanner-context (scanner-context token-parser-context) |
| 205 | () |
| 206 | (:documentation |
| 207 | "A parser context for a richer token-based scanners.")) |
| 208 | |
| 209 | ;; A place marker. |
| 210 | |
| 211 | (export '(token-scanner-place token-scanner-place-p)) |
| 212 | (defstruct (token-scanner-place |
| 213 | (:constructor make-token-scanner-place |
| 214 | (&key scanner next type value line column |
| 215 | &aux (%type type)))) |
| 216 | "A link in the chain of lookahead tokens; capturable as a place. |
| 217 | |
| 218 | If the scanner's place is captured, it starts to maintain a list of |
| 219 | lookahead tokens. The list contains internal links -- it works out |
| 220 | slightly easier that way. This is basically a simpler version of the |
| 221 | charbuf scanner (q.v.); most significantly, the chain links here do double |
| 222 | duty as place markers. |
| 223 | |
| 224 | The details of this structure are not a defined part of the token scanner |
| 225 | protocol." |
| 226 | |
| 227 | (scanner nil :type token-scanner :read-only t) |
| 228 | (next nil :type (or token-scanner-place null)) |
| 229 | (%type nil :read-only t) |
| 230 | (value nil :read-only t) |
| 231 | (line 1 :type (or fixnum null) :read-only t) |
| 232 | (column 0 :type (or fixnum null) :read-only t)) |
| 233 | (define-access-wrapper token-scanner-place-type token-scanner-place-%type |
| 234 | :read-only t) |
| 235 | |
| 236 | ;; Protocol. |
| 237 | |
| 238 | (export 'scanner-token) |
| 239 | (defgeneric scanner-token (scanner) |
| 240 | (:documentation |
| 241 | "Internal protocol: read the next token from the SCANNER. |
| 242 | |
| 243 | This function is called by `scanner-step' to actually read the next token |
| 244 | if necessary. It should return two values: the token's `type' and its |
| 245 | `value'.")) |
| 246 | |
| 247 | ;;;-------------------------------------------------------------------------- |
| 248 | ;;; Character scanner streams. |
| 249 | ;;; |
| 250 | ;;; This seems like an abstraction inversion, but it's important if we're to |
| 251 | ;;; `read' from a character scanner. |
| 252 | |
| 253 | (export 'character-scanner-stream) |
| 254 | (defclass character-scanner-stream (fundamental-character-input-stream) |
| 255 | ((scanner :initarg :scanner)) |
| 256 | (:documentation |
| 257 | "A stream which reads from a character scanner. |
| 258 | |
| 259 | The SCANNER must implement the character scanner protcol, including |
| 260 | `scanner-current-char', `scanner-step', and `scanner-unread'; it is not |
| 261 | necessary that the scanner implement the place-capture protocol. |
| 262 | |
| 263 | The stream can be made more efficient by implementing |
| 264 | `stream-read-sequence' and `stream-read-line' in a scanner-specific |
| 265 | manner.")) |
| 266 | |
| 267 | (export 'make-scanner-stream) |
| 268 | (defgeneric make-scanner-stream (scanner) |
| 269 | (:documentation |
| 270 | "Return a stream which reads from the SCANNER. |
| 271 | |
| 272 | The default method simply constructs a `character-scanner-stream' |
| 273 | instance. Subclasses of `character-scanner' can override this method in |
| 274 | order to return instances of more efficient stream subclasses.") |
| 275 | (:method ((scanner character-scanner)) |
| 276 | (make-instance 'character-scanner-stream :scanner scanner))) |
| 277 | |
| 278 | ;;;----- That's all, folks -------------------------------------------------- |