Commit 2bc6a0ff authored by Jan Rasmus Tikovsky 's avatar Jan Rasmus Tikovsky
Browse files

Tokenstream now provides span information instead of positions

parent 56b1be01
......@@ -15,9 +15,9 @@ module CompilerEnv where
import qualified Data.Map as Map (Map, keys, toList)
import Curry.Base.Ident (ModuleIdent)
import Curry.Base.Position (Position)
import Curry.Base.Ident (ModuleIdent)
import Curry.Base.Pretty
import Curry.Base.Span (Span)
import Curry.Syntax
import Base.TopEnv (allLocalBindings)
......@@ -37,7 +37,7 @@ data CompilerEnv = CompilerEnv
{ moduleIdent :: ModuleIdent -- ^ identifier of the module
, filePath :: FilePath -- ^ 'FilePath' of compilation target
, extensions :: [KnownExtension] -- ^ enabled language extensions
, tokens :: [(Position, Token)] -- ^ token list of module
, tokens :: [(Span, Token)] -- ^ token list of module
, interfaceEnv :: InterfaceEnv -- ^ declarations of imported interfaces
, aliasEnv :: AliasEnv -- ^ aliases for imported modules
, tyConsEnv :: TCEnv -- ^ type constructors
......
......@@ -37,6 +37,7 @@ import Curry.Base.Ident
import Curry.Base.Monad
import Curry.Base.Position
import Curry.Base.Pretty
import Curry.Base.Span
import Curry.ExtendedFlat.InterfaceEquivalence (eqInterface)
import Curry.Files.Filenames
import Curry.Files.PathUtils
......@@ -125,7 +126,7 @@ loadModule opts m fn = do
return (cEnv { filePath = fn, tokens = toks }, mdl)
parseModule :: Options -> ModuleIdent -> FilePath
-> CYIO ([(Position, CS.Token)], CS.Module)
-> CYIO ([(Span, CS.Token)], CS.Module)
parseModule opts m fn = do
mbSrc <- liftIO $ readModule fn
case mbSrc of
......@@ -135,10 +136,10 @@ parseModule opts m fn = do
prepd <- preprocess (optPrepOpts opts) fn ul
-- We ignore the warnings issued by the lexer because
-- they will be issued a second time during parsing.
posToks <- liftCYM $ silent $ CS.lexSource fn prepd
ast <- liftCYM $ CS.parseModule fn prepd
checked <- checkModuleHeader opts m fn ast
return (posToks, checked)
spanToks <- liftCYM $ silent $ CS.lexSource fn prepd
ast <- liftCYM $ CS.parseModule fn prepd
checked <- checkModuleHeader opts m fn ast
return (spanToks, checked)
preprocess :: PrepOpts -> FilePath -> String -> CYIO String
preprocess opts fn src
......@@ -278,7 +279,7 @@ writeParsed opts (env, mdl) = when srcTarget $ liftIO $
writeHtml :: Options -> CompEnv CS.Module -> CYIO ()
writeHtml opts (env, mdl) = when htmlTarget $
source2html opts (moduleIdent env) (tokens env) mdl
source2html opts (moduleIdent env) (map (\(sp, tok) -> (span2Pos sp, tok)) (tokens env)) mdl
where htmlTarget = Html `elem` optTargetTypes opts
writeInterface :: Options -> CompilerEnv -> CS.Interface -> CYIO ()
......
{- |
Module : $Header$
Description : Generating List of Tokens and Positions
Description : Generating List of Tokens and Spans
Copyright : (c) 2015 - 2016, Katharina Rahf
2015 - 2016, Björn Peemöller
2015 - 2016, Jan Tikovsky
This module defines a function for writing the list of tokens
and positions of a Curry source module into a separate file.
and spans of a Curry source module into a separate file.
-}
module TokenStream (showTokenStream) where
......@@ -14,18 +14,27 @@ module TokenStream (showTokenStream) where
import Data.List (intercalate)
import Curry.Base.Position (Position (..))
import Curry.Base.Span (Span (..))
import Curry.Syntax (Token (..), Category (..), Attributes (..))
-- |Show a list of 'Position' and 'Token' tuples.
-- |Show a list of 'Span' and 'Token' tuples.
-- The list is split into one tuple on each line to increase readability.
showTokenStream :: [(Position, Token)] -> String
showTokenStream :: [(Span, Token)] -> String
showTokenStream [] = "[]\n"
showTokenStream ts = "[ " ++ intercalate "\n, " (map showPT ts) ++ "\n]\n"
where showPT (p, t) = "(" ++ showPosition p ++ ", " ++ showToken t ++ ")"
showTokenStream ts = "[ " ++ intercalate "\n, " (map showST filteredTs) ++ "\n]\n"
where filteredTs = filter (not . isVirtual) ts
showST (sp, t) = "(" ++ showSpan sp ++ ", " ++ showToken t ++ ")"
isVirtual :: (Span, Token) -> Bool
isVirtual (_, Token cat _) = cat `elem` [EOF, VRightBrace, VSemicolon]
-- show 'span' as "((startLine, startColumn), (endLine, endColumn))"
showSpan :: Span -> String
showSpan sp = "(" ++ showPos (start sp) ++ ", " ++ showPos (end sp) ++ ")"
-- show 'Position' as "(line, column)"
showPosition :: Position -> String
showPosition p = "(" ++ show (line p) ++ ", " ++ show (column p) ++ ")"
showPos :: Position -> String
showPos p = "(" ++ show (line p) ++ ", " ++ show (column p) ++ ")"
-- |Show tokens and their value if needed
showToken :: Token -> String
......@@ -86,8 +95,6 @@ showToken (Token Bar _) = "Bar"
showToken (Token LeftArrow _) = "LeftArrow"
showToken (Token RightArrow _) = "RightArrow"
showToken (Token Tilde _) = "Tilde"
showToken (Token Bind _) = "Bind"
showToken (Token Select _) = "Select"
-- special identifiers
showToken (Token Id_as _) = "Id_as"
showToken (Token Id_ccall _) = "Id_ccall"
......
......@@ -5,7 +5,14 @@
--- Bernd Brassel (sequence, mapM)
--- @version April 2005
------------------------------------------------------------------------------
{- nested comment in one line -}
{- This
is
a
nested
commend
-}
module Maybe(
isJust, isNothing,
fromJust, fromMaybe, listToMaybe, maybeToList,
......@@ -41,7 +48,7 @@ maybeToList (Just a) = [a]
listToMaybe :: [a] -> Maybe a
listToMaybe [] = Nothing
listToMaybe (a:_) = Just a
catMaybes :: [Maybe a] -> [a]
catMaybes ms = [ m | (Just m) <- ms ]
......
stringGap :: String
stringGap = "Dies \
\ ist ein \
\ String gap."
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment