From: Mart Lubbers Date: Wed, 24 Feb 2016 21:29:43 +0000 (+0100) Subject: lexer with line numbers X-Git-Url: https://git.martlubbers.net/?a=commitdiff_plain;h=345b80b49413bb1b52735fb155faded41456a6b7;p=cc1516.git lexer with line numbers --- diff --git a/lex.dcl b/lex.dcl index 7e41bc3..e141b94 100644 --- a/lex.dcl +++ b/lex.dcl @@ -2,7 +2,8 @@ definition module lex import Data.Either -:: Token +:: Token :== (Int, TokenValue) +:: TokenValue //Value tokens = IdentToken [Char] // Identifier | NumberToken [Char]// Integer diff --git a/lex.icl b/lex.icl index e2d679a..82e6139 100644 --- a/lex.icl +++ b/lex.icl @@ -8,7 +8,7 @@ import StdBool import StdList import StdChar -SingleCharTokens :: Map Char Token +SingleCharTokens :: Map Char TokenValue SingleCharTokens = fromList [ ('(', BraceOpenToken), (')', BraceCloseToken), ('{', CBraceOpenToken), ('}', CBraceCloseToken), ('[', SquareOpenToken), (']', SquareCloseToken), @@ -22,47 +22,57 @@ EscapeMap = fromList [('a', toChar 7), ('b', '\b'), ('f', '\f'), ('n', '\n'), ('r', '\r'), ('t', '\t'), ('v', '\v')] lexer :: [Char] -> LexerOutput -lexer [] = Right [] -lexer x = case lex x of - (Right t, rest) = lexer rest >>= \ts.Right [t:ts] - (Left e, _) = Left e +lexer x = lexer` x 1 + where + lexer` :: [Char] Int -> LexerOutput + lexer` [] i = Right [] + lexer` x i = case lex x i of + (Right t, rest, i) = lexer` rest i >>= \ts.Right [t:ts] + (Left e, _, _) = Left e -lex :: [Char] -> (Either String Token, [Char]) -lex [] = (Right EndOfFileToken, []) -lex ['/':'/':xs] = lex $ dropWhile ((<>) '\n') xs -lex ['/':'*':xs] = - lex $ tl $ map snd $ dropWhile ((<>) ('*', '/')) $ zip2 [zero:xs] xs -lex ['v':'a':'r':xs] = (Right VarToken, xs) -lex ['V':'o':'i':'d':xs] = (Right VoidToken, xs) -lex ['r':'e':'t':'u':'r':'n':xs] = (Right ReturnToken, xs) -lex ['i':'f':xs] = (Right IfToken, xs) -lex ['e':'l':'s':'e':xs] = (Right ElseToken, xs) -lex ['w':'h':'i':'l':'e':xs] = (Right WhileToken, xs) -lex ['T':'r':'u':'e':xs] = (Right TrueToken, xs) -lex ['F':'a':'l':'s':'e':xs] = (Right FalseToken, xs) -lex ['I':'n':'t':xs] = (Right IntTypeToken, xs) -lex ['C':'h':'a':'r':xs] = (Right CharTypeToken, xs) -lex ['B':'o':'o':'l':xs] = (Right BoolTypeToken, xs) -lex [':':':':xs] = (Right DoubleColonToken, xs) -lex ['!':'=':xs] = (Right NotEqualToken, xs) -lex ['<':'=':xs] = (Right LesserEqToken, xs) -lex ['>':'=':xs] = (Right GreaterEqToken, xs) -lex ['=':'=':xs] = (Right EqualsToken, xs) -lex ['&':'&':xs] = (Right AmpersandsToken, xs) -lex ['|':'|':xs] = (Right PipesToken, xs) -lex ['-':'>':xs] = (Right ArrowToken, xs) -lex ['\'':x:'\'':xs] = (Right (CharToken x), xs) +ret:: TokenValue [Char] Int -> (Either String Token, [Char], Int) +ret tv rest i = (Right (i, tv), rest, i) + +err :: String Int -> (Either String Token, [Char], Int) +err s i = (Left ("Line " +++ toString i +++ ": " +++ s), [], i) + +lex :: [Char] -> (Int -> (Either String Token, [Char], Int)) +lex [] = ret EndOfFileToken [] +lex ['/':'/':xs] = lex (dropWhile ((<>) '\n') xs) +lex ['/':'*':xs] = \i.let (c, r) = span ((<>) ('*','/')) $ zip2 [' ':xs] xs in + lex (tl $ map snd $ tl $ r) (i + length [x\\(x, _)<-c | x == '\n']) +lex ['v':'a':'r':xs] = ret VarToken xs +lex ['V':'o':'i':'d':xs] = ret VoidToken xs +lex ['r':'e':'t':'u':'r':'n':xs] = ret ReturnToken xs +lex ['i':'f':xs] = ret IfToken xs +lex ['e':'l':'s':'e':xs] = ret ElseToken xs +lex ['w':'h':'i':'l':'e':xs] = ret WhileToken xs +lex ['T':'r':'u':'e':xs] = ret TrueToken xs +lex ['F':'a':'l':'s':'e':xs] = ret FalseToken xs +lex ['I':'n':'t':xs] = ret IntTypeToken xs +lex ['C':'h':'a':'r':xs] = ret CharTypeToken xs +lex ['B':'o':'o':'l':xs] = ret BoolTypeToken xs +lex [':':':':xs] = ret DoubleColonToken xs +lex ['!':'=':xs] = ret NotEqualToken xs +lex ['<':'=':xs] = ret LesserEqToken xs +lex ['>':'=':xs] = ret GreaterEqToken xs +lex ['=':'=':xs] = ret EqualsToken xs +lex ['&':'&':xs] = ret AmpersandsToken xs +lex ['|':'|':xs] = ret PipesToken xs +lex ['-':'>':xs] = ret ArrowToken xs +lex ['\'':x:'\'':xs] = ret (CharToken x) xs lex ['\'':'\\':x:'\'':xs] = case get x EscapeMap of - Just t = (Right (CharToken t), xs) - _ = (Left ("Unknown escape: \\" +++ toString x), []) + Just t = ret (CharToken t) xs + _ = err ("Unknown escape: \\" +++ toString x) lex ['-':t=:[x:xs]] -| isDigit x = let (i, r) = span isDigit t in (Right (NumberToken ['-':i]), r) -| otherwise = (Right DashToken, xs) +| isDigit x = let (n, r) = span isDigit t in ret (NumberToken ['-':n]) r +| otherwise = ret DashToken xs lex t=:[x:xs] = case get x SingleCharTokens of - (Just tok) = (Right tok, xs) + (Just tok) = ret tok xs Nothing + | x == '\n' = \i.lex xs (i+1) | isSpace x = lex xs - | isDigit x = let (i, r) = span isDigit t in (Right (NumberToken i), r) - | isAlpha x = let (i, r) = span isIdent t in (Right (IdentToken i), r) + | isDigit x = let (v, r) = span isDigit t in ret (NumberToken v) r + | isAlpha x = let (v, r) = span isIdent t in ret (IdentToken v) r with isIdent c = isAlphanum c || c == '_' - | otherwise = (Left ("Unexpected character: " +++ toString x), []) + | otherwise = err ("Unexpected character: " +++ toString x) diff --git a/main.icl b/main.icl index ced5b0a..67ab090 100644 --- a/main.icl +++ b/main.icl @@ -6,12 +6,20 @@ import StdBool import lex import parse +//Start :: *World -> (ParserOutput, *World) +Start w +# (out, w) = stdio w +# (toparse, out) = readEntireFile out +# (_, w) = fclose out w += (lexer toparse, w) +/* Start :: *World -> (ParserOutput, *World) Start w # (out, w) = stdio w # (toparse, out) = readEntireFile out # (_, w) = fclose out w = (parse (lexer toparse), w) +*/ readEntireFile :: *File -> *([Char], *File) readEntireFile f diff --git a/main.prj b/main.prj index 5f00e02..5ccec5d 100644 --- a/main.prj +++ b/main.prj @@ -31,6 +31,7 @@ Global Link LinkMethod: Static GenerateRelocations: False + GenerateSymbolTable: False GenerateLinkMap: False LinkResources: False ResourceSource: @@ -407,7 +408,7 @@ OtherModules Fusion: False Module Name: Control.Applicative - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False @@ -421,7 +422,7 @@ OtherModules Fusion: False Module Name: Control.Monad - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False @@ -435,7 +436,7 @@ OtherModules Fusion: False Module Name: Data.Either - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False @@ -449,7 +450,7 @@ OtherModules Fusion: False Module Name: Data.Foldable - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False @@ -463,7 +464,7 @@ OtherModules Fusion: False Module Name: Data.Func - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False @@ -477,7 +478,7 @@ OtherModules Fusion: False Module Name: Data.Functor - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False @@ -491,7 +492,7 @@ OtherModules Fusion: False Module Name: Data.List - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False @@ -505,7 +506,7 @@ OtherModules Fusion: False Module Name: Data.Map - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False @@ -519,7 +520,7 @@ OtherModules Fusion: False Module Name: Data.Maybe - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False @@ -533,7 +534,7 @@ OtherModules Fusion: False Module Name: Data.Monoid - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False @@ -547,7 +548,7 @@ OtherModules Fusion: False Module Name: Data.Set - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False @@ -561,7 +562,7 @@ OtherModules Fusion: False Module Name: Data.Traversable - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False @@ -575,7 +576,7 @@ OtherModules Fusion: False Module Name: Data.Void - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False @@ -589,7 +590,7 @@ OtherModules Fusion: False Module Name: System.IO - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False @@ -603,7 +604,7 @@ OtherModules Fusion: False Module Name: Text - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False @@ -617,7 +618,7 @@ OtherModules Fusion: False Module Name: Text.JSON - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False @@ -631,7 +632,7 @@ OtherModules Fusion: False Module Name: Text.PPrint - Dir: {Application}/lib/iTasks-SDK/Dependencies/Platform/OS-Independent + Dir: {Application}/lib/iTasks-SDK/Dependencies/clean-platform/src/libraries/OS-Independent Compiler NeverMemoryProfile: False NeverTimeProfile: False