From: pimjager <pim@pimjager.nl>
Date: Thu, 11 Feb 2016 21:43:21 +0000 (+0100)
Subject: Implemented lexer function
X-Git-Url: https://git.martlubbers.net/?a=commitdiff_plain;h=82d6648e3777f5bb74fa1825f18cf41922f3b782;p=cc1516.git

Implemented lexer function
---

diff --git a/lex.icl b/lex.icl
index d46aaa8..cbdcd65 100644
--- a/lex.icl
+++ b/lex.icl
@@ -11,6 +11,7 @@ import Data.Maybe
 import Control.Applicative
 import Control.Monad
 import Control.Monad.State
+import Data.Functor
 from Data.Func import $
 
 // Misschien moeten we hier continuation style van maken
@@ -18,14 +19,16 @@ instance toString lexerOutput where
 	toString l = "dit is een lexer output, danwel error\n"
 
 lexer :: [Char] -> LexerOutput [Token]
-lexer _ = Left "Not Implemented"
+lexer i = let (out, rest) = runParser (many anyT) i in case rest of
+            []  = out
+            _   = Left "Unexpected input after last token"
 
 //lexer functions
 identT  = alpha >>= \a -> many (char '_' <|> alphaNum) >>= \as -> return $ IdentToken [a:as]
 numberT = optional (char '-') >>= \sign -> (some digit) >>= \n -> case sign of
             Nothing -> return $ NumberToken $ 5  //fromString n
             _       -> return $ NumberToken $ -5 //(fromString n) * -1
-charLT  = liftM CharToken item
+charLT  = CharToken <$> (char '\'' *> item <* char '\'')
 char2T  = item >>= \c1 -> case c1 of
             ':' = char ':' >>| return DoubleColonToken
             '<' = char '=' >>| return LesserEqToken
@@ -34,6 +37,7 @@ char2T  = item >>= \c1 -> case c1 of
             '&' = char '&' >>| return AmpersandsToken
             '|' = char '|' >>| return PipesToken
             '-' = char '>' >>| return ArrowToken
+            _   = zero
 char1T  = item >>= \c1 -> findT c1 charTokenMap
 varT    = string (fromString "var") >>| return VarToken
 voidT   = string (fromString "Void") >>| return VoidToken
@@ -43,6 +47,9 @@ elseT   = string (fromString "else") >>| return ElseToken
 whileT  = string (fromString "while") >>| return WhileToken
 trueT   = string (fromString "True") >>| return TrueToken
 falseT  = string (fromString "False") >>| return FalseToken
+//note, for the anyToken parser the order matters!
+anyT    = char2T <|> char1T <|> varT <|> voidT <|> returnT <|> ifT <|> elseT <|> whileT <|>
+          trueT <|> falseT <|> numberT <|> identT <|> charLT
 
 Start :: *World -> *World
 Start w