lexProgram :: Parser Char [Token]
lexProgram = some lexToken <* many (satisfy isSpace) <* eof
- >>= \ts->pure (map (\t->(0, 0, t)) ts)
+ >>= \ts->pure $ (map (\t->(0, 0, t)) ts) ++ [(0, 0, EndOfFileToken)]
lexToken :: Parser Char TokenValue
lexToken =
//Comments
- (list (fromString "//") >>| until top (item '\n') >>| lexToken) <|>
- (list (fromString "/*") >>| until top (list (fromString "*/")) >>| lexToken) <|>
+ (list (fromString "//") >>| lexUntilNL >>| lexToken) <|>
+ (list (fromString "/*") >>| lexUntilCommentClose >>| lexToken) <|>
//Keyword tokens
(lexKw "var" VarToken) <|>
(lexKw "Void" VoidToken) <|>
//Number tokens
(liftM (NumberToken o toInt o toString) $ some $ satisfy isDigit) <|>
//Ident tokens
- (liftM IdentToken $ some $ satisfy isIdentChar) <|>
+ (liftM (IdentToken o toString) $ some $ satisfy isIdentChar) <|>
(satisfy isSpace >>| lexToken)
-// (eof >>| pure EndOfFileToken)
where
+ lexUntilNL = top until (eof <|> (item '\n' >>| pure Void))
+ lexUntilCommentClose = top until list (fromString "*/")
isIdentChar c = isAlphanum c || c == '_'
lexOp s tv = list (fromString s) >>| pure tv
lexKw kw tv = lexOp kw tv <* check (not o isIdentChar) >>| pure tv