Today's wip.

[sod] / src / lexer-impl.lisp

;;; -*-lisp-*-
;;;
;;; Implementation of lexical analysis protocol.
;;;
;;; (c) 2009 Straylight/Edgeware
;;;

;;;----- Licensing notice ---------------------------------------------------
;;;
;;; This file is part of the Sensble Object Design, an object system for C.
;;;
;;; SOD is free software; you can redistribute it and/or modify
;;; it under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 2 of the License, or
;;; (at your option) any later version.
;;;
;;; SOD is distributed in the hope that it will be useful,
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with SOD; if not, write to the Free Software Foundation,
;;; Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

(cl:in-package #:sod)

;;;--------------------------------------------------------------------------
;;; Class implementation.

(defmethod shared-initialize :after
    ((scanner sod-token-scanner) slot-names &key)
  (default-slot (scanner 'sod-parser::filename slot-names)
    (scanner-filename (token-scanner-char-scanner scanner))))

(defmethod make-scanner-stream ((scanner sod-token-scanner))
  (make-scanner-stream (token-scanner-char-scanner scanner)))

;;;--------------------------------------------------------------------------
;;; Indicators and error messages.

(defvar *indicator-map* (make-hash-table)
  "Hash table mapping indicator objects to human-readable descriptions.")

(defun show-char (stream char &optional colonp atsignp)
  "Format CHAR to STREAM in a readable way.

   Usable in `format''s ~/.../ command."
  (declare (ignore colonp atsignp))
  (cond ((null char) (write-string "<eof>" stream))
        ((and (graphic-char-p char) (char/= char #\space))
         (format stream "`~C'" char))
        (t (format stream "<~(~:C~)>" char))))

(defun skip-until (scanner token-types &key keep-end)
  "This is the implementation of the `skip-until' parser."
  (do ((consumedp nil t))
      ((member (token-type scanner) token-types)
       (unless keep-end (scanner-step scanner))
       (values nil t (or keep-end consumedp)))
    (when (scanner-at-eof-p scanner)
      (return (values token-types nil consumedp)))
    (scanner-step scanner)))

(defun parse-error-recover (scanner parser recover)
  "This is the implementation of the `error' parser."
  (multiple-value-bind (result win consumedp) (funcall parser)
    (cond ((or win (and (not consumedp) (scanner-at-eof-p scanner)))
           ;; If we succeeded then there's nothing for us to do here.  On the
           ;; other hand, if we failed, didn't consume any tokens, and we're
           ;; at end-of-file, then there's not much hope of making onward
           ;; progress, so in this case we propagate the failure rather than
           ;; trying to recover.  And we assume that the continuation will
           ;; somehow arrange to report the problem, and avoid inundating the
           ;; user with error reports.
           (values result win consumedp))
          (t
           ;; Now we have to do some kind of sensible error recovery.  The
           ;; important thing to do here is to make sure that we make some
           ;; progress.  If we consumed any tokens then we're fine, and we'll
           ;; just try the provided recovery strategy.  Otherwise, if we're
           ;; not at EOF, then we can ensure progress by discarding the
           ;; current token.  Finally, if we are at EOF then our best bet is
           ;; simply to propagate the current failure back to the caller, but
           ;; we handled that case above.
           (syntax-error scanner result :continuep t)
           (unless consumedp (scanner-step scanner))
           (funcall recover)))))

;;;--------------------------------------------------------------------------
;;; Token scanning.

(defmethod scanner-token ((scanner sod-token-scanner))
  (with-slots (char-scanner line column) scanner
    (with-parser-context (character-scanner-context :scanner char-scanner)

      (flet ((scan-digits (&key (radix 10) (min 1) (init 0))
               ;; Scan and return a sequence of digits.
               (parse (many (acc init (+ (* acc radix) it) :min min)
                        (label (list :digit radix)
                               (filter (lambda (ch)
                                         (digit-char-p ch radix))))))))

        ;; Skip initial junk, and remember the place.
        (loop
          (setf (scanner-line scanner) (scanner-line char-scanner)
                (scanner-column scanner) (scanner-column char-scanner))
          (cond-parse (:consumedp cp :expected exp)
            ((satisfies whitespace-char-p) (parse :whitespace))
            ((scan-comment char-scanner))
            (t (if cp (lexer-error char-scanner exp cp) (return)))))

        ;; Now parse something.
        (cond-parse (:consumedp cp :expected exp)

          ;; Alphanumerics mean we read an identifier.
          ((or #\_ (satisfies alpha-char-p))
           (values :id (with-output-to-string (out)
                         (write-char it out)
                         (parse (many (nil nil (write-char it out))
                                  (or #\_ (satisfies alphanumericp)))))))

          ;; Quotes introduce a literal.
          ((seq ((quote (or #\" #\'))
                 (contents (many (out (make-string-output-stream)
                                      (progn (write-char it out) out)
                                      :final (get-output-stream-string out))
                             (or (and #\\ :any) (not quote))))
                 (nil (char quote)))
             (ecase quote
               (#\" contents)
               (#\' (case (length contents)
                      (1 (char contents 0))
                      (0 (cerror* "Empty character literal") #\?)
                      (t (cerror* "Too many characters in literal")
                         (char contents 0))))))
           (values (etypecase it
                     (character :char)
                     (string :string))
                   it))

          ;; Zero introduces a chosen-radix integer.
          ((and #\0
                (or (and (or #\b #\B) (scan-digits :radix 2))
                    (and (or #\o #\O) (scan-digits :radix 8))
                    (and (or #\x #\X) (scan-digits :radix 16))
                    (scan-digits :radix 8 :min 0)))
           (values :int it))

          ;; Any other digit forces radix-10.
          ((seq ((d (filter digit-char-p))
                 (i (scan-digits :radix 10 :min 0 :init d)))
             i)
           (values :int it))

          ;; Some special punctuation sequences are single tokens.
          ("..." (values :ellipsis nil))

          ;; Any other character is punctuation.
          (:any (values it nil))

          ;; End of file means precisely that.
          (:eof (values :eof nil))

          ;; Report errors and try again.  Because we must have consumed some
          ;; input in order to get here (we've matched both :any and :eof) we
          ;; must make progress on every call.
          (t
           (assert cp)
           (lexer-error char-scanner exp cp)
           (scanner-token scanner)))))))

;;;----- That's all, folks --------------------------------------------------