[DevState] Changed Parsing
This commit is contained in:
Sven Heidemann
@@ -1,5 +1,6 @@
|
||||
from Interpreter.Validator import Validator
|
||||
from Interpreter.Lexer import Lexer
|
||||
# from Interpreter.Parser_Old import Parser
|
||||
from Interpreter.Parser import Parser
|
||||
from Interpreter.Repo import Repo
|
||||
from Interpreter.Utils import Utils
|
||||
|
||||
@@ -2,9 +2,8 @@ from typing import List
|
||||
|
||||
from Interpreter.Repo import Repo
|
||||
from Interpreter.Utils import Utils
|
||||
from Models.Interpreter.Token import Token
|
||||
from Models.Interpreter.TokenTypes import TokenTypes
|
||||
from Models.Interpreter.UnresolvedTokenTypes import UnresolvedTokenTypes
|
||||
from Models.Token.Token import Token
|
||||
from Models.Token.TokenTypes import TokenTypes, UnresolvedTokenTypes
|
||||
|
||||
|
||||
class Lexer:
|
||||
@@ -19,7 +18,7 @@ class Lexer:
|
||||
"""
|
||||
Creates token object
|
||||
:param value:
|
||||
:param token_type:
|
||||
:param input_token_type:
|
||||
:return:
|
||||
"""
|
||||
token_type: TokenTypes = TokenTypes.Empty
|
||||
@@ -29,7 +28,7 @@ class Lexer:
|
||||
if value in self.__repo.keywords:
|
||||
token_type = TokenTypes.Keyword
|
||||
|
||||
elif value in self.__repo.types:
|
||||
elif value in self.__repo.datatypes:
|
||||
token_type = TokenTypes.Type
|
||||
|
||||
elif value in self.__repo.bool_values:
|
||||
|
||||
@@ -1,20 +1,8 @@
|
||||
from typing import List, Optional
|
||||
|
||||
from termcolor import colored
|
||||
|
||||
from Interpreter.Repo import Repo
|
||||
from Interpreter.Utils import Utils
|
||||
from Models.AbstractSyntaxTree.Class import Class
|
||||
from Models.AbstractSyntaxTree.Function import Function
|
||||
from Models.AbstractSyntaxTree.Library import Library
|
||||
from Models.AbstractSyntaxTree.Variable import Variable
|
||||
from Models.Interpreter.Error import Error, ErrorCode
|
||||
from Models.Interpreter.ExpressionCharacters import ExpressionCharacters
|
||||
from Models.Interpreter.FormatCharacters import FormatCharacters
|
||||
from Models.Interpreter.Keywords import Keywords
|
||||
from Models.Interpreter.Token import Token
|
||||
from Models.Interpreter.TokenTypes import TokenTypes
|
||||
from Models.Interpreter.Types import Types
|
||||
from Models.Token.Token import Token
|
||||
|
||||
|
||||
class Parser:
|
||||
@@ -23,295 +11,16 @@ class Parser:
|
||||
self.__repo = repo
|
||||
self.__utils = utils
|
||||
|
||||
# library status
|
||||
self.__is_creating_library = False
|
||||
self.__is_saving_library = False
|
||||
self.__library: Optional[Library] = None
|
||||
self.__next_is_public = False
|
||||
|
||||
# class status
|
||||
self.__is_creating_class = False
|
||||
self.__is_saving_class = False
|
||||
self.__class: Optional[Class] = None
|
||||
self.__expected_tokens: List[Token] = []
|
||||
|
||||
# function status
|
||||
self.__is_creating_function = False
|
||||
self.__is_creating_function_parameters = False
|
||||
self.__is_saving_function = False
|
||||
self.__function: Optional[Function] = None
|
||||
|
||||
# variable status
|
||||
self.__is_creating_variable = False
|
||||
self.__is_saving_variable = False
|
||||
self.__variable: Optional[Variable] = None
|
||||
|
||||
# type status
|
||||
self.__is_creating_type = False
|
||||
self.__type: Optional[Types] = None
|
||||
|
||||
# common
|
||||
self.__next_is_public = True
|
||||
self.__ast: Optional[None] = None
|
||||
|
||||
def parse(self, tokens: List[Token]) -> None:
|
||||
"""
|
||||
Creates abstract syntax tree from token list
|
||||
:param tokens:
|
||||
:return:
|
||||
"""
|
||||
# output
|
||||
if len(tokens) == 0:
|
||||
return None
|
||||
if len(tokens) > 0:
|
||||
toks = []
|
||||
for tok in tokens:
|
||||
toks.append({tok.type.name, tok.value})
|
||||
|
||||
output_tokens = []
|
||||
for t in tokens:
|
||||
output_tokens.append({t.value: t.type})
|
||||
# print(output_tokens)
|
||||
|
||||
for i in range(0, len(tokens)):
|
||||
token = tokens[i]
|
||||
print(colored(f'>> {self.__utils.line_number}: {token.type} {token.value}', 'blue'))
|
||||
|
||||
""" error status checks """
|
||||
# check if var name is valid
|
||||
if i-1 >= 0 and self.__is_creating_variable and tokens[i-1].value == Keywords.Variable:
|
||||
if token.type != TokenTypes.Name:
|
||||
self.__utils.error(Error(ErrorCode.Unexpected, f'{token.type.name}: {token.value}'))
|
||||
|
||||
""" normal status checks """
|
||||
# token is member of variable value
|
||||
if self.__is_saving_variable and token.value != FormatCharacters.chars.value[FormatCharacters.Semicolon.value] and self.__variable is not None:
|
||||
print(6, 'save variable value')
|
||||
self.__variable.value.append(token)
|
||||
|
||||
# token type keyword
|
||||
if token.type == TokenTypes.Keyword:
|
||||
self.__check_token_keyword(tokens, i, token)
|
||||
|
||||
# token type format character
|
||||
elif token.type == TokenTypes.Format_Character:
|
||||
self.__check_token_format_character(token)
|
||||
|
||||
# token type expression character
|
||||
elif token.type == TokenTypes.Expression_Character:
|
||||
self.__check_token_expression_character(token)
|
||||
|
||||
# token type name
|
||||
elif token.type == TokenTypes.Name:
|
||||
self.__check_token_name(token)
|
||||
|
||||
# token type type
|
||||
elif token.type == TokenTypes.Type:
|
||||
self.__check_token_type(token)
|
||||
|
||||
""" validation function """
|
||||
def __check_token_keyword(self, tokens: List[Token], i: int, token: Token) -> None:
|
||||
if token.value == Keywords.Library:
|
||||
# create library
|
||||
self.__is_creating_library = True
|
||||
self.__library = Library()
|
||||
self.__set_is_public(self.__library)
|
||||
print(1, 'create lib')
|
||||
|
||||
elif token.value == Keywords.Class:
|
||||
# create class
|
||||
self.__is_creating_class = True
|
||||
self.__class = Class()
|
||||
self.__set_is_public(self.__class)
|
||||
print(2, 'create class')
|
||||
|
||||
elif token.value == Keywords.Function:
|
||||
# create function
|
||||
self.__is_creating_function = True
|
||||
self.__function = Function()
|
||||
self.__set_is_public(self.__function)
|
||||
print(3, 'create function')
|
||||
|
||||
elif token.value == Keywords.Variable:
|
||||
self.__is_creating_variable = True
|
||||
self.__variable = Variable()
|
||||
print(6, 'create variable')
|
||||
|
||||
elif token.value == Keywords.Public:
|
||||
if self.__is_saving_function:
|
||||
self.__utils.error(Error(ErrorCode.Unexpected, f'{token.type.name}: {token.value}'))
|
||||
return
|
||||
|
||||
self.__next_is_public = True
|
||||
print(0, 'Next is public')
|
||||
|
||||
elif self.__next_is_public:
|
||||
if i - 1 > 0:
|
||||
error_msg = f'{token.type.name}: {tokens[i - 1].value}'
|
||||
else:
|
||||
error_msg = 'keyword: public'
|
||||
|
||||
self.__utils.error(Error(ErrorCode.Unexpected, error_msg))
|
||||
|
||||
def __check_token_format_character(self, token: Token) -> None:
|
||||
if token.value == FormatCharacters.chars.value[FormatCharacters.Left_Brace.value]:
|
||||
|
||||
if self.__is_creating_library:
|
||||
# set status to save vars and funcs to library
|
||||
self.__is_creating_library = False
|
||||
self.__is_saving_library = True
|
||||
print(1, 'status save lib')
|
||||
|
||||
elif self.__is_creating_class:
|
||||
# set status to save vars and funcs to class
|
||||
self.__is_creating_class = False
|
||||
self.__is_saving_class = True
|
||||
print(2, 'status save class')
|
||||
|
||||
elif self.__is_creating_function:
|
||||
# set status to save vars and commands to function
|
||||
self.__is_creating_function = False
|
||||
self.__is_saving_function = True
|
||||
print(3, 'status save function')
|
||||
|
||||
# check for type end
|
||||
if self.__is_creating_type:
|
||||
self.__check_for_type_end()
|
||||
|
||||
elif token.value == FormatCharacters.chars.value[FormatCharacters.Right_Brace.value]:
|
||||
if self.__is_saving_function:
|
||||
# save function to class
|
||||
self.__is_saving_function = False
|
||||
|
||||
if self.__class is not None:
|
||||
self.__class.functions.append(self.__function)
|
||||
print(3, f'saved function {self.__function.name}')
|
||||
else:
|
||||
self.__utils.error(Error(ErrorCode.Unexpected, f'function: {self.__function.name}'))
|
||||
|
||||
self.__function = None
|
||||
|
||||
elif self.__is_saving_class:
|
||||
# save class to library
|
||||
self.__is_saving_class = False
|
||||
|
||||
if self.__library is not None:
|
||||
self.__library.classes.append(self.__class)
|
||||
print(2, f'saved class {self.__class.name}')
|
||||
else:
|
||||
self.__utils.error(Error(ErrorCode.Unexpected, f'class: {self.__class.name}'))
|
||||
|
||||
self.__class = None
|
||||
|
||||
elif self.__is_saving_library:
|
||||
# save library to ast
|
||||
self.__is_saving_library = False
|
||||
|
||||
self.__repo.AST.libs.append(self.__library)
|
||||
print(1, f'saved lib {self.__library.name}')
|
||||
self.__library = None
|
||||
|
||||
elif token.value == FormatCharacters.chars.value[FormatCharacters.Left_Parenthesis.value]:
|
||||
if self.__is_creating_function:
|
||||
self.__is_creating_function_parameters = True
|
||||
print(4, 'create function parameters')
|
||||
|
||||
elif token.value == FormatCharacters.chars.value[FormatCharacters.Right_Parenthesis.value]:
|
||||
if self.__is_creating_function:
|
||||
self.__is_creating_function_parameters = False
|
||||
self.__type = None
|
||||
print(4, 'save function parameters')
|
||||
|
||||
# check for type end
|
||||
if self.__is_creating_type:
|
||||
self.__check_for_type_end()
|
||||
|
||||
elif token.value == FormatCharacters.chars.value[FormatCharacters.Colon.value]:
|
||||
self.__check_create_type(token)
|
||||
|
||||
elif token.value == FormatCharacters.chars.value[FormatCharacters.Semicolon.value]:
|
||||
if self.__is_creating_type:
|
||||
self.__check_for_type_end()
|
||||
|
||||
if self.__is_saving_variable:
|
||||
self.__is_saving_variable = False
|
||||
self.__add_variable()
|
||||
self.__variable = None
|
||||
print(6, 'status saved variable')
|
||||
|
||||
elif self.__is_creating_variable:
|
||||
self.__is_creating_variable = False
|
||||
self.__is_saving_variable = True
|
||||
self.__add_variable()
|
||||
self.__variable = None
|
||||
print(6, 'status saved variable')
|
||||
|
||||
def __check_token_expression_character(self, token: Token) -> None:
|
||||
if token.value == ExpressionCharacters.chars.value[ExpressionCharacters.Equal.value]:
|
||||
# check for type end
|
||||
if self.__is_creating_type:
|
||||
self.__is_creating_variable = False
|
||||
self.__is_saving_variable = True
|
||||
self.__check_for_type_end()
|
||||
|
||||
def __check_token_name(self, token: Token) -> None:
|
||||
if self.__is_creating_type and self.__type is not None:
|
||||
self.__type = token.value
|
||||
print(5, 'save type')
|
||||
|
||||
elif self.__is_creating_variable and self.__variable is not None:
|
||||
self.__variable.name = token.value
|
||||
print(6, 'save variable name')
|
||||
|
||||
elif self.__is_creating_library:
|
||||
# save library name
|
||||
self.__library.name = token.value
|
||||
print(1, 'lib name')
|
||||
|
||||
elif self.__is_creating_class:
|
||||
# save class name
|
||||
self.__class.name = token.value
|
||||
print(2, 'class name')
|
||||
|
||||
elif self.__is_creating_function and not self.__is_creating_function_parameters:
|
||||
# save function name
|
||||
self.__function.name = token.value
|
||||
print(3, 'function name')
|
||||
|
||||
def __check_token_type(self, token: Token) -> None:
|
||||
if self.__is_creating_function and self.__function is not None and not self.__is_creating_function_parameters:
|
||||
self.__function.return_type = Types(Types.strings.value.index(token.value))
|
||||
print(3, 'save function type')
|
||||
|
||||
elif self.__is_creating_type:
|
||||
self.__type = Types(Types.strings.value.index(token.value))
|
||||
print(5, 'save type')
|
||||
|
||||
""" util functions """
|
||||
def __set_is_public(self, ast_element):
|
||||
if ast_element is not None and self.__next_is_public:
|
||||
ast_element.is_public = True
|
||||
self.__next_is_public = False
|
||||
|
||||
def __check_for_type_end(self):
|
||||
# end status creating type
|
||||
self.__is_creating_type = False
|
||||
print(5, 'status saved type')
|
||||
|
||||
def __check_create_type(self, token: Token):
|
||||
if self.__is_creating_function or self.__is_creating_function_parameters or self.__is_creating_variable:
|
||||
self.__is_creating_type = True
|
||||
print(5, 'create type')
|
||||
else:
|
||||
self.__utils.error(Error(ErrorCode.Unexpected, f'{token.type.name}: {token.value}'))
|
||||
|
||||
def __add_variable(self):
|
||||
if len(self.__variable.value) == 0:
|
||||
self.__variable.value.append(Token(TokenTypes.Empty, Types.strings.value[Types.Empty.value]))
|
||||
|
||||
if self.__type is not None:
|
||||
self.__variable.type = self.__type
|
||||
else:
|
||||
self.__utils.error(Error(ErrorCode.Expected, f'type'))
|
||||
|
||||
if self.__is_saving_function and self.__function is not None:
|
||||
self.__function.variables.append(self.__variable)
|
||||
|
||||
elif self.__is_saving_class and self.__class is not None:
|
||||
self.__class.variables.append(self.__variable)
|
||||
|
||||
else:
|
||||
self.__utils.error(Error(ErrorCode.Unexpected, f'variable: {self.__variable.name}'))
|
||||
print(toks)
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
from Models.AbstractSyntaxTree.RuntimeAbstractSyntaxTree import RuntimeAbstractSyntaxTree
|
||||
from Models.Interpreter.Booleans import Booleans
|
||||
from Models.Interpreter.FormatCharacters import FormatCharacters
|
||||
from Models.Interpreter.Keywords import Keywords
|
||||
from Models.Interpreter.Types import Types
|
||||
from Models.Interpreter.Datatypes import Datatypes
|
||||
from Models.Token.TokenValueTypes import FormatCharacters, Booleans, Keywords, ExpressionCharacters
|
||||
|
||||
|
||||
class Repo:
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.debug = True
|
||||
self.line_number: int = 0
|
||||
|
||||
# interpreter
|
||||
self.keywords = [
|
||||
@@ -29,7 +27,7 @@ class Repo:
|
||||
Keywords.If,
|
||||
Keywords.ElseIf,
|
||||
Keywords.Else,
|
||||
Keywords.Pass,
|
||||
Keywords.Continue,
|
||||
Keywords.If,
|
||||
Keywords.Return,
|
||||
# loops
|
||||
@@ -38,15 +36,14 @@ class Repo:
|
||||
# access
|
||||
Keywords.Public
|
||||
]
|
||||
self.types = [
|
||||
Types.strings.value[Types.Any.value],
|
||||
Types.strings.value[Types.Number.value],
|
||||
Types.strings.value[Types.String.value],
|
||||
Types.strings.value[Types.Bool.value],
|
||||
Types.strings.value[Types.List.value],
|
||||
Types.strings.value[Types.Dict.value],
|
||||
Types.strings.value[Types.Empty.value],
|
||||
Types.strings.value[Types.Void.value]
|
||||
self.datatypes = [
|
||||
Datatypes.strings.value[Datatypes.Empty.value],
|
||||
Datatypes.strings.value[Datatypes.Any.value],
|
||||
Datatypes.strings.value[Datatypes.Number.value],
|
||||
Datatypes.strings.value[Datatypes.String.value],
|
||||
Datatypes.strings.value[Datatypes.Bool.value],
|
||||
Datatypes.strings.value[Datatypes.List.value],
|
||||
Datatypes.strings.value[Datatypes.Dict.value]
|
||||
]
|
||||
self.format_chars = [
|
||||
FormatCharacters.chars.value[FormatCharacters.Left_Brace.value],
|
||||
@@ -60,10 +57,16 @@ class Repo:
|
||||
FormatCharacters.chars.value[FormatCharacters.Comma.value],
|
||||
FormatCharacters.chars.value[FormatCharacters.Point.value]
|
||||
]
|
||||
self.expr_chars = ['+', '-', '*', '/', '=', '^']
|
||||
self.expr_chars = [
|
||||
ExpressionCharacters.chars.value[ExpressionCharacters.Plus.value],
|
||||
ExpressionCharacters.chars.value[ExpressionCharacters.Minus.value],
|
||||
ExpressionCharacters.chars.value[ExpressionCharacters.Asterisk.value],
|
||||
ExpressionCharacters.chars.value[ExpressionCharacters.Slash.value],
|
||||
ExpressionCharacters.chars.value[ExpressionCharacters.Equal.value],
|
||||
ExpressionCharacters.chars.value[ExpressionCharacters.Caret.value]
|
||||
]
|
||||
self.bool_expr_chars = ['<', '>', '!', '!=', '==', '>=', '<=', '&&', '||']
|
||||
self.bool_values = [Booleans.Right, Booleans.Wrong]
|
||||
|
||||
# runtime
|
||||
self.is_error = None
|
||||
self.AST: RuntimeAbstractSyntaxTree = RuntimeAbstractSyntaxTree()
|
||||
|
||||
@@ -8,7 +8,6 @@ class Utils:
|
||||
|
||||
def __init__(self, repo: Repo) -> None:
|
||||
self.__repo = repo
|
||||
self.line_number: int = 0
|
||||
|
||||
def input(self, prefix: str) -> str:
|
||||
return input(prefix)
|
||||
@@ -18,5 +17,5 @@ class Utils:
|
||||
|
||||
def error(self, error: Error) -> None:
|
||||
self.__repo.is_error = error
|
||||
print(colored(f'Error in line {self.line_number}\n{self.__repo.is_error.msg}', 'red'))
|
||||
print(colored(f'Error in line {self.__repo.line_number}\n{self.__repo.is_error.msg}', 'red'))
|
||||
# exit()
|
||||
|
||||
Reference in New Issue
Block a user