(cl:in-package #:sod)

(defun list-tokens (scanner)
  (let ((toke (make-instance 'sod-token-scanner :char-scanner scanner)))
    (loop collect (list (token-type toke) (token-value toke))
	  until (scanner-at-eof-p toke)
	  do (scanner-step toke))))