[DevState] Changed Parsing

This commit is contained in:
Sven Heidemann 2020-09-23 06:48:38 +02:00
parent dfc4b44b25
commit 7058927970
29 changed files with 224 additions and 541 deletions

View File

@ -1,7 +1,40 @@
lib main {
LibraryDefinition:
IsPublic: <True, False>
Name: <NAME>
<public> class test {
ClassDefinition:
IsPublic: <True, False>
Name: <NAME>
Instructions: <LIST OF SUB AST>
FunctionDefinition:
IsPublic: <True, False>
Name: <NAME>
Arguments: <LIST OF VariableDefinition>
ReturnType: <Bool, String, Num, Empty, Any>
Instructions: <LIST OF SUB AST>
VariableDefinition:
IsPublic: <True, False>
Name: <NAME>
Type: <Bool, String, Num, Empty, Any>
Value: <LIST OF SUB AST>
IsAbleToBeEmpty: <True, False>
FunctionCall:
Function: <NAME, ID>
Arguments: <LIST OF SUB AST>
Assignment:
Target: <NAME, ID>
Value: <LIST OF SUB AST>
Operation:
Type: <+, -, *, />
Arguments: <LIST OF SUB AST>
Value:
Value: <1-9, '', true/false>
Type: <Bool, String, Number>
<public> func test(): void {
<public> var: bool <= test <<Bool, String, Num, Any, Instance>> >;

View File

@ -3,4 +3,4 @@ lib:
vars
funcs:
vars
commands
instructions

View File

@ -10,7 +10,7 @@ public lib Main {
var test2: Program2 = empty;
var test3: Program2 = Test();
func Main(): void {
func Main(): empty {
#func Main {
var hallo: any;
#output('Hello');

View File

@ -1,5 +1,6 @@
from Interpreter.Validator import Validator
from Interpreter.Lexer import Lexer
# from Interpreter.Parser_Old import Parser
from Interpreter.Parser import Parser
from Interpreter.Repo import Repo
from Interpreter.Utils import Utils

View File

@ -2,9 +2,8 @@ from typing import List
from Interpreter.Repo import Repo
from Interpreter.Utils import Utils
from Models.Interpreter.Token import Token
from Models.Interpreter.TokenTypes import TokenTypes
from Models.Interpreter.UnresolvedTokenTypes import UnresolvedTokenTypes
from Models.Token.Token import Token
from Models.Token.TokenTypes import TokenTypes, UnresolvedTokenTypes
class Lexer:
@ -19,7 +18,7 @@ class Lexer:
"""
Creates token object
:param value:
:param token_type:
:param input_token_type:
:return:
"""
token_type: TokenTypes = TokenTypes.Empty
@ -29,7 +28,7 @@ class Lexer:
if value in self.__repo.keywords:
token_type = TokenTypes.Keyword
elif value in self.__repo.types:
elif value in self.__repo.datatypes:
token_type = TokenTypes.Type
elif value in self.__repo.bool_values:

View File

@ -1,20 +1,8 @@
from typing import List, Optional
from termcolor import colored
from Interpreter.Repo import Repo
from Interpreter.Utils import Utils
from Models.AbstractSyntaxTree.Class import Class
from Models.AbstractSyntaxTree.Function import Function
from Models.AbstractSyntaxTree.Library import Library
from Models.AbstractSyntaxTree.Variable import Variable
from Models.Interpreter.Error import Error, ErrorCode
from Models.Interpreter.ExpressionCharacters import ExpressionCharacters
from Models.Interpreter.FormatCharacters import FormatCharacters
from Models.Interpreter.Keywords import Keywords
from Models.Interpreter.Token import Token
from Models.Interpreter.TokenTypes import TokenTypes
from Models.Interpreter.Types import Types
from Models.Token.Token import Token
class Parser:
@ -23,295 +11,16 @@ class Parser:
self.__repo = repo
self.__utils = utils
# library status
self.__is_creating_library = False
self.__is_saving_library = False
self.__library: Optional[Library] = None
self.__next_is_public = False
# class status
self.__is_creating_class = False
self.__is_saving_class = False
self.__class: Optional[Class] = None
self.__expected_tokens: List[Token] = []
# function status
self.__is_creating_function = False
self.__is_creating_function_parameters = False
self.__is_saving_function = False
self.__function: Optional[Function] = None
# variable status
self.__is_creating_variable = False
self.__is_saving_variable = False
self.__variable: Optional[Variable] = None
# type status
self.__is_creating_type = False
self.__type: Optional[Types] = None
# common
self.__next_is_public = True
self.__ast: Optional[None] = None
def parse(self, tokens: List[Token]) -> None:
"""
Creates abstract syntax tree from token list
:param tokens:
:return:
"""
# output
if len(tokens) == 0:
return None
if len(tokens) > 0:
toks = []
for tok in tokens:
toks.append({tok.type.name, tok.value})
output_tokens = []
for t in tokens:
output_tokens.append({t.value: t.type})
# print(output_tokens)
for i in range(0, len(tokens)):
token = tokens[i]
print(colored(f'>> {self.__utils.line_number}: {token.type} {token.value}', 'blue'))
""" error status checks """
# check if var name is valid
if i-1 >= 0 and self.__is_creating_variable and tokens[i-1].value == Keywords.Variable:
if token.type != TokenTypes.Name:
self.__utils.error(Error(ErrorCode.Unexpected, f'{token.type.name}: {token.value}'))
""" normal status checks """
# token is member of variable value
if self.__is_saving_variable and token.value != FormatCharacters.chars.value[FormatCharacters.Semicolon.value] and self.__variable is not None:
print(6, 'save variable value')
self.__variable.value.append(token)
# token type keyword
if token.type == TokenTypes.Keyword:
self.__check_token_keyword(tokens, i, token)
# token type format character
elif token.type == TokenTypes.Format_Character:
self.__check_token_format_character(token)
# token type expression character
elif token.type == TokenTypes.Expression_Character:
self.__check_token_expression_character(token)
# token type name
elif token.type == TokenTypes.Name:
self.__check_token_name(token)
# token type type
elif token.type == TokenTypes.Type:
self.__check_token_type(token)
""" validation function """
def __check_token_keyword(self, tokens: List[Token], i: int, token: Token) -> None:
if token.value == Keywords.Library:
# create library
self.__is_creating_library = True
self.__library = Library()
self.__set_is_public(self.__library)
print(1, 'create lib')
elif token.value == Keywords.Class:
# create class
self.__is_creating_class = True
self.__class = Class()
self.__set_is_public(self.__class)
print(2, 'create class')
elif token.value == Keywords.Function:
# create function
self.__is_creating_function = True
self.__function = Function()
self.__set_is_public(self.__function)
print(3, 'create function')
elif token.value == Keywords.Variable:
self.__is_creating_variable = True
self.__variable = Variable()
print(6, 'create variable')
elif token.value == Keywords.Public:
if self.__is_saving_function:
self.__utils.error(Error(ErrorCode.Unexpected, f'{token.type.name}: {token.value}'))
return
self.__next_is_public = True
print(0, 'Next is public')
elif self.__next_is_public:
if i - 1 > 0:
error_msg = f'{token.type.name}: {tokens[i - 1].value}'
else:
error_msg = 'keyword: public'
self.__utils.error(Error(ErrorCode.Unexpected, error_msg))
def __check_token_format_character(self, token: Token) -> None:
if token.value == FormatCharacters.chars.value[FormatCharacters.Left_Brace.value]:
if self.__is_creating_library:
# set status to save vars and funcs to library
self.__is_creating_library = False
self.__is_saving_library = True
print(1, 'status save lib')
elif self.__is_creating_class:
# set status to save vars and funcs to class
self.__is_creating_class = False
self.__is_saving_class = True
print(2, 'status save class')
elif self.__is_creating_function:
# set status to save vars and commands to function
self.__is_creating_function = False
self.__is_saving_function = True
print(3, 'status save function')
# check for type end
if self.__is_creating_type:
self.__check_for_type_end()
elif token.value == FormatCharacters.chars.value[FormatCharacters.Right_Brace.value]:
if self.__is_saving_function:
# save function to class
self.__is_saving_function = False
if self.__class is not None:
self.__class.functions.append(self.__function)
print(3, f'saved function {self.__function.name}')
else:
self.__utils.error(Error(ErrorCode.Unexpected, f'function: {self.__function.name}'))
self.__function = None
elif self.__is_saving_class:
# save class to library
self.__is_saving_class = False
if self.__library is not None:
self.__library.classes.append(self.__class)
print(2, f'saved class {self.__class.name}')
else:
self.__utils.error(Error(ErrorCode.Unexpected, f'class: {self.__class.name}'))
self.__class = None
elif self.__is_saving_library:
# save library to ast
self.__is_saving_library = False
self.__repo.AST.libs.append(self.__library)
print(1, f'saved lib {self.__library.name}')
self.__library = None
elif token.value == FormatCharacters.chars.value[FormatCharacters.Left_Parenthesis.value]:
if self.__is_creating_function:
self.__is_creating_function_parameters = True
print(4, 'create function parameters')
elif token.value == FormatCharacters.chars.value[FormatCharacters.Right_Parenthesis.value]:
if self.__is_creating_function:
self.__is_creating_function_parameters = False
self.__type = None
print(4, 'save function parameters')
# check for type end
if self.__is_creating_type:
self.__check_for_type_end()
elif token.value == FormatCharacters.chars.value[FormatCharacters.Colon.value]:
self.__check_create_type(token)
elif token.value == FormatCharacters.chars.value[FormatCharacters.Semicolon.value]:
if self.__is_creating_type:
self.__check_for_type_end()
if self.__is_saving_variable:
self.__is_saving_variable = False
self.__add_variable()
self.__variable = None
print(6, 'status saved variable')
elif self.__is_creating_variable:
self.__is_creating_variable = False
self.__is_saving_variable = True
self.__add_variable()
self.__variable = None
print(6, 'status saved variable')
def __check_token_expression_character(self, token: Token) -> None:
if token.value == ExpressionCharacters.chars.value[ExpressionCharacters.Equal.value]:
# check for type end
if self.__is_creating_type:
self.__is_creating_variable = False
self.__is_saving_variable = True
self.__check_for_type_end()
def __check_token_name(self, token: Token) -> None:
if self.__is_creating_type and self.__type is not None:
self.__type = token.value
print(5, 'save type')
elif self.__is_creating_variable and self.__variable is not None:
self.__variable.name = token.value
print(6, 'save variable name')
elif self.__is_creating_library:
# save library name
self.__library.name = token.value
print(1, 'lib name')
elif self.__is_creating_class:
# save class name
self.__class.name = token.value
print(2, 'class name')
elif self.__is_creating_function and not self.__is_creating_function_parameters:
# save function name
self.__function.name = token.value
print(3, 'function name')
def __check_token_type(self, token: Token) -> None:
if self.__is_creating_function and self.__function is not None and not self.__is_creating_function_parameters:
self.__function.return_type = Types(Types.strings.value.index(token.value))
print(3, 'save function type')
elif self.__is_creating_type:
self.__type = Types(Types.strings.value.index(token.value))
print(5, 'save type')
""" util functions """
def __set_is_public(self, ast_element):
if ast_element is not None and self.__next_is_public:
ast_element.is_public = True
self.__next_is_public = False
def __check_for_type_end(self):
# end status creating type
self.__is_creating_type = False
print(5, 'status saved type')
def __check_create_type(self, token: Token):
if self.__is_creating_function or self.__is_creating_function_parameters or self.__is_creating_variable:
self.__is_creating_type = True
print(5, 'create type')
else:
self.__utils.error(Error(ErrorCode.Unexpected, f'{token.type.name}: {token.value}'))
def __add_variable(self):
if len(self.__variable.value) == 0:
self.__variable.value.append(Token(TokenTypes.Empty, Types.strings.value[Types.Empty.value]))
if self.__type is not None:
self.__variable.type = self.__type
else:
self.__utils.error(Error(ErrorCode.Expected, f'type'))
if self.__is_saving_function and self.__function is not None:
self.__function.variables.append(self.__variable)
elif self.__is_saving_class and self.__class is not None:
self.__class.variables.append(self.__variable)
else:
self.__utils.error(Error(ErrorCode.Unexpected, f'variable: {self.__variable.name}'))
print(toks)

View File

@ -1,14 +1,12 @@
from Models.AbstractSyntaxTree.RuntimeAbstractSyntaxTree import RuntimeAbstractSyntaxTree
from Models.Interpreter.Booleans import Booleans
from Models.Interpreter.FormatCharacters import FormatCharacters
from Models.Interpreter.Keywords import Keywords
from Models.Interpreter.Types import Types
from Models.Interpreter.Datatypes import Datatypes
from Models.Token.TokenValueTypes import FormatCharacters, Booleans, Keywords, ExpressionCharacters
class Repo:
def __init__(self) -> None:
self.debug = True
self.line_number: int = 0
# interpreter
self.keywords = [
@ -29,7 +27,7 @@ class Repo:
Keywords.If,
Keywords.ElseIf,
Keywords.Else,
Keywords.Pass,
Keywords.Continue,
Keywords.If,
Keywords.Return,
# loops
@ -38,15 +36,14 @@ class Repo:
# access
Keywords.Public
]
self.types = [
Types.strings.value[Types.Any.value],
Types.strings.value[Types.Number.value],
Types.strings.value[Types.String.value],
Types.strings.value[Types.Bool.value],
Types.strings.value[Types.List.value],
Types.strings.value[Types.Dict.value],
Types.strings.value[Types.Empty.value],
Types.strings.value[Types.Void.value]
self.datatypes = [
Datatypes.strings.value[Datatypes.Empty.value],
Datatypes.strings.value[Datatypes.Any.value],
Datatypes.strings.value[Datatypes.Number.value],
Datatypes.strings.value[Datatypes.String.value],
Datatypes.strings.value[Datatypes.Bool.value],
Datatypes.strings.value[Datatypes.List.value],
Datatypes.strings.value[Datatypes.Dict.value]
]
self.format_chars = [
FormatCharacters.chars.value[FormatCharacters.Left_Brace.value],
@ -60,10 +57,16 @@ class Repo:
FormatCharacters.chars.value[FormatCharacters.Comma.value],
FormatCharacters.chars.value[FormatCharacters.Point.value]
]
self.expr_chars = ['+', '-', '*', '/', '=', '^']
self.expr_chars = [
ExpressionCharacters.chars.value[ExpressionCharacters.Plus.value],
ExpressionCharacters.chars.value[ExpressionCharacters.Minus.value],
ExpressionCharacters.chars.value[ExpressionCharacters.Asterisk.value],
ExpressionCharacters.chars.value[ExpressionCharacters.Slash.value],
ExpressionCharacters.chars.value[ExpressionCharacters.Equal.value],
ExpressionCharacters.chars.value[ExpressionCharacters.Caret.value]
]
self.bool_expr_chars = ['<', '>', '!', '!=', '==', '>=', '<=', '&&', '||']
self.bool_values = [Booleans.Right, Booleans.Wrong]
# runtime
self.is_error = None
self.AST: RuntimeAbstractSyntaxTree = RuntimeAbstractSyntaxTree()

View File

@ -8,7 +8,6 @@ class Utils:
def __init__(self, repo: Repo) -> None:
self.__repo = repo
self.line_number: int = 0
def input(self, prefix: str) -> str:
return input(prefix)
@ -18,5 +17,5 @@ class Utils:
def error(self, error: Error) -> None:
self.__repo.is_error = error
print(colored(f'Error in line {self.line_number}\n{self.__repo.is_error.msg}', 'red'))
print(colored(f'Error in line {self.__repo.line_number}\n{self.__repo.is_error.msg}', 'red'))
# exit()

View File

@ -1,8 +1,7 @@
import os
import sys
from pprint import pprint
from Models.Interpreter.Error import Error, ErrorCode
from Models.Interpreter.Error import Error, ErrorCodes
from ServiceInitializer import ServiceInitializer
@ -21,8 +20,8 @@ class Main:
"""
i = 0
while self.__repo.is_error is not None:
self.__utils.line_number = i + 1
self.__interpreter.interpret(self.__utils.line_number, input('> '))
self.__repo.line_number = i + 1
self.__interpreter.interpret(self.__repo.line_number, input('> '))
i += 1
def files(self, file: str) -> None:
@ -34,12 +33,10 @@ class Main:
if os.path.isfile(file):
f = open(file, 'r', encoding='utf-8').readlines()
for i in range(0, len(f)):
self.__utils.line_number = i + 1
self.__repo.line_number = i + 1
self.__interpreter.interpret(f[i])
print(self.__repo.AST)
else:
self.__utils.is_error(Error(ErrorCode.FileNotFound))
self.__utils.is_error(Error(ErrorCodes.FileNotFound))
if __name__ == '__main__':

View File

@ -1,9 +1,30 @@
from typing import List
from Models.Interpreter.Token import Token
from Models.Interpreter.Datatypes import Datatypes
from Models.Token.Token import Token
from Models.Token.TokenTypes import TokenTypes
from Models.Token.TokenValueTypes import ExpressionCharacters
class AbstractSyntaxTree:
class ValueNode:
def __init__(self, value: Token = 'empty', datatype: Token = Token(TokenTypes.Type, Datatypes.strings.value[Datatypes.Empty.value])):
self.value = value
self.type = datatype
class BinaryOperationNode:
def __init__(self, left: Token, op_token: Token, right: Token):
self.left = left
self.op_token = op_token
self.right = right
self.operation_chars = [
ExpressionCharacters.chars.value[ExpressionCharacters.Plus.value],
ExpressionCharacters.chars.value[ExpressionCharacters.Minus.value],
ExpressionCharacters.chars.value[ExpressionCharacters.Asterisk.value],
ExpressionCharacters.chars.value[ExpressionCharacters.Slash.value],
ExpressionCharacters.chars.value[ExpressionCharacters.Caret.value]
]
def eval(self):
if self.op_token.value not in self.operation_chars:
return eval(f'{self.left.value} {self.op_token.value} {self.right.value}')
def __init__(self):
self.tokens: List[Token] = []

View File

@ -1,13 +0,0 @@
from typing import List
from Models.AbstractSyntaxTree.Function import Function
from Models.AbstractSyntaxTree.Variable import Variable
class Class:
def __init__(self):
self.name: str = ''
self.variables: List[Variable] = []
self.functions: List[Function] = []
self.is_public = False

View File

@ -1,14 +0,0 @@
from typing import List
from Models.AbstractSyntaxTree.Variable import Variable
from Models.Interpreter.Types import Types
class Function:
def __init__(self) -> None:
self.name: str = ''
self.variables: List[Variable] = []
self.instructions: [] = []
self.return_type: Types = Types.Any
self.is_public = False

View File

@ -1,9 +0,0 @@
from Models.AbstractSyntaxTree.AbstractSyntaxTree import AbstractSyntaxTree
from Models.AbstractSyntaxTree.InstructionTypes import InstructionTypes
class Instruction:
def __init__(self):
self.type: InstructionTypes = InstructionTypes.Unknown
self.ast: AbstractSyntaxTree = AbstractSyntaxTree()

View File

@ -1,10 +0,0 @@
from enum import Enum
class InstructionTypes(Enum):
Unknown = 0
Expression = 1
Bool_Expression = 2
Assignment = 3
Call = 4

View File

@ -1,11 +0,0 @@
from typing import List
from Models.AbstractSyntaxTree.Class import Class
class Library:
def __init__(self):
self.name: str = ''
self.classes: List[Class] = []
self.is_public = False

View File

@ -1,9 +0,0 @@
from typing import List
from Models.AbstractSyntaxTree.Library import Library
class RuntimeAbstractSyntaxTree:
def __init__(self):
self.libs: List[Library] = []

View File

@ -1,12 +0,0 @@
from typing import List
from Models.Interpreter.Token import Token
class Variable:
def __init__(self):
self.name: str = ''
self.type: str = ''
self.value: List[Token] = []
self.is_public = False

View File

@ -1,4 +0,0 @@
class Booleans:
Right = 'true'
Wrong = 'false'

View File

@ -0,0 +1,22 @@
from enum import Enum
class Datatypes(Enum):
Empty = 0
Any = 1
Number = 2
String = 3
Bool = 4
List = 5
Dict = 6
strings = [
'empty',
'any',
'number',
'string',
'bool',
'list',
'dict'
]

View File

@ -1,7 +1,7 @@
from enum import Enum
class ErrorCode(Enum):
class ErrorCodes(Enum):
StartFailed = 'Start failed'
FileNotFound = 'File not found'
@ -21,7 +21,7 @@ class ErrorCode(Enum):
class Error:
def __init__(self, code: ErrorCode, msg: str = ''):
def __init__(self, code: ErrorCodes, msg: str = '') -> None:
self.code = code
self.msg = code.value.format(msg)

View File

@ -1,19 +0,0 @@
from enum import Enum
class ExpressionCharacters(Enum):
Plus = 0
Minus = 1
Asterisk = 2
Slash = 3
Equal = 4
Caret = 5
chars = [
'+',
'-',
'*',
'/',
'=',
'^'
]

View File

@ -1,27 +0,0 @@
from enum import Enum
class FormatCharacters(Enum):
Left_Brace = 0
Right_Brace = 1
Left_Parenthesis = 2
Right_Parenthesis = 3
Left_Bracket = 4
Right_Bracket = 5
Semicolon = 6
Colon = 7
Comma = 8
Point = 9
chars = [
'{',
'}',
'(',
')',
'[',
']',
';',
':',
',',
'.'
]

View File

@ -1,30 +0,0 @@
class Keywords:
# define keywords
Library: str = 'lib'
Class: str = 'class'
Function: str = 'func'
Variable: str = 'var'
Use: str = 'use'
From: str = 'from'
# builtin functions
Output: str = 'output'
Input: str = 'input'
Length: str = 'length'
Range: str = 'range'
Exit: str = 'exit'
# normal keywords
If: str = 'if'
ElseIf: str = 'elseif'
Else: str = 'else'
Pass: str = 'pass'
In: str = 'in'
Return: str = 'return'
# loops
While: str = 'while'
For: str = 'for'
# access
Public: str = 'public'

View File

@ -1,24 +0,0 @@
from enum import Enum
class Types(Enum):
Any = 0
Number = 1
String = 2
Bool = 3
List = 4
Dict = 5
Empty = 6
Void = 7
strings = [
'any',
'number',
'string',
'bool',
'list',
'dict',
'empty',
'void'
]

View File

@ -1,11 +0,0 @@
from enum import Enum
class UnresolvedTokenTypes(Enum):
Empty = 0
Word = 1
Number = 2
String = 3
Expression_Character = 4
Bool_Expression_Character = 5
Format_Character = 6

View File

@ -1,8 +1,8 @@
from Models.Interpreter import TokenTypes
from Models.Token import TokenTypes
class Token:
def __init__(self, token_type: TokenTypes, value: str) -> None:
self.type: TokenTypes = token_type
self.value = value
self.value: str = value

View File

@ -12,3 +12,13 @@ class TokenTypes(Enum):
Expression_Character = 7
Bool_Expression_Character = 8
Format_Character = 9
class UnresolvedTokenTypes(Enum):
Empty = 0
Word = 1
Number = 2
String = 3
Expression_Character = 4
Bool_Expression_Character = 5
Format_Character = 6

View File

@ -0,0 +1,82 @@
from enum import Enum
class Keywords:
# define keywords
Library: str = 'lib'
Class: str = 'class'
Function: str = 'func'
Variable: str = 'var'
Use: str = 'use'
From: str = 'from'
# builtin functions
Output: str = 'output'
Input: str = 'input'
Length: str = 'length'
Range: str = 'range'
Exit: str = 'exit'
# normal keywords
If: str = 'if'
ElseIf: str = 'elseif'
Else: str = 'else'
Continue: str = 'continue'
In: str = 'in'
Return: str = 'return'
# loops
While: str = 'while'
For: str = 'for'
# access
Public: str = 'public'
class Booleans:
Right = 'true'
Wrong = 'false'
class ExpressionCharacters(Enum):
Plus = 0
Minus = 1
Asterisk = 2
Slash = 3
Equal = 4
Caret = 5
chars = [
'+',
'-',
'*',
'/',
'=',
'^'
]
class FormatCharacters(Enum):
Left_Brace = 0
Right_Brace = 1
Left_Parenthesis = 2
Right_Parenthesis = 3
Left_Bracket = 4
Right_Bracket = 5
Semicolon = 6
Colon = 7
Comma = 8
Point = 9
chars = [
'{',
'}',
'(',
')',
'[',
']',
';',
':',
',',
'.'
]

View File