(load-file "util.ath")

;(load-file "elms-defs.ath")

(load-file "ed.ath")

(structure Token
  LPAREN
  RPAREN
  LBRACK
  RBRACK 
  LSQB
  RSQB
  DOT
  BBDOT
  STAR
  PLUS-STAR
  EXP-OP
  IN
  SUBSET
  UNION
  INTER
  PROD 
  NOT
  AND
  OR
  IF
  IFF 
  ALL
  SOME
  SEP 
  EQ
  NEQ
  COMMA
  (ID Ide))

(define (legal-id-char? c)
  (& (printable? c) (~ (member? c "()[]{}=!><+*^|&~.,"))))

(define (get-id str res)
  (match str
    ([] [(rev res) []])
    ((list-of c rest) (check ((legal-id-char? c) (get-id rest (add c res)))
			     (else [(rev res) str])))))
(define (get-kwd str)
  (match str
    ((split "~" str') [NOT str'])
    ((split "||" str') [OR str'])
    ((split "&" str') [AND str'])
    ((split "!=" str') [NEQ str'])
    ((split "<==>" str') [IFF str'])
    ((split "==>" str') [IF str'])
    ((split "=" str') [EQ str'])
    ((split "ALL" str') [ALL str'])
    ((split "EXISTS" str') [SOME str'])
    ((split "subset" str') [SUBSET str'])
    ((split "in" str') (check ((|| (null? str) (~ (printable? (head str')))) [IN str'])
			      (else ['none str])))
    ((split "X" str') [PROD str'])
    ((split "|" str') [SEP str'])
    ((split "*" str') [STAR str'])
    ((split "," str') [COMMA str'])
    ((split "^" str') [EXP-OP str'])
    ((split "+" str') [PLUS-STAR str'])
    ((split "." str') [DOT str'])
    ((split "o" str') [BBDOT str'])
    ((split "union" str') [UNION str'])
    ((split "intersection" str') [INTER str'])
    ((split "{" str') [LBRACK str'])
    ((split "}" str') [RBRACK str'])
    ((split "(" str') [LPAREN str'])
    ((split ")" str') [RPAREN str'])
    ((split "[" str') [LSQB str'])
    ((split "]" str') [RSQB str'])
    (_ ['none str])))

(define (get-tokens str tokens)
  (match (skip-until str printable?)
    ([] (rev tokens))
    (str' (match (get-kwd str')
            (['none _] (match (get-id str' [])
                         ([id rest] (get-tokens rest (add (ID (string->id id)) tokens)))))
            ([kwd rest] (get-tokens rest (add kwd tokens)))))))


(define (tokenize str)
  (get-tokens str []))

(define tok tokenize)