From 4251e63cc9ebcdde0e3b9e9e44f04275cf80cd85 Mon Sep 17 00:00:00 2001 From: Bram Dingelstad Date: Mon, 6 Dec 2021 17:42:27 +0100 Subject: [PATCH] capitalised some files --- TestScene.tscn | 2 +- addons/Wol/core/Constants.gd | 214 ++++++ addons/Wol/core/Library.gd | 25 + addons/Wol/core/Program.gd | 146 ++++ addons/Wol/core/Value.gd | 137 ++++ addons/Wol/core/compiler/Compiler.gd | 461 +++++++++++ addons/Wol/core/compiler/Lexer.gd | 485 ++++++++++++ addons/Wol/core/compiler/Parser.gd | 1065 ++++++++++++++++++++++++++ dialogue.yarn => dialogue.wol | 56 +- project.godot | 2 +- 10 files changed, 2563 insertions(+), 30 deletions(-) create mode 100644 addons/Wol/core/Constants.gd create mode 100644 addons/Wol/core/Library.gd create mode 100644 addons/Wol/core/Program.gd create mode 100644 addons/Wol/core/Value.gd create mode 100644 addons/Wol/core/compiler/Compiler.gd create mode 100644 addons/Wol/core/compiler/Lexer.gd create mode 100644 addons/Wol/core/compiler/Parser.gd rename dialogue.yarn => dialogue.wol (99%) diff --git a/TestScene.tscn b/TestScene.tscn index 9e03b5f..f32bb44 100644 --- a/TestScene.tscn +++ b/TestScene.tscn @@ -14,7 +14,7 @@ __meta__ = { [node name="Wol" type="Node" parent="Dialogue"] script = ExtResource( 1 ) -path = "res://dialogue.yarn" +path = "res://dialogue.wol" auto_start = true variable_storage = { } diff --git a/addons/Wol/core/Constants.gd b/addons/Wol/core/Constants.gd new file mode 100644 index 0000000..c700d85 --- /dev/null +++ b/addons/Wol/core/Constants.gd @@ -0,0 +1,214 @@ +extends Object + +enum ExecutionState { + Stopped, + Running, + WaitingForOption, + Suspended +} + +enum HandlerState { + PauseExecution, + ContinueExecution +} + +#Compile Status return +enum CompileStatus { + Succeeded, SucceededUntaggedStrings, +} + +enum ByteCode { + # opA = string: label name + Label, + # opA = string: label name + JumpTo, + # peek string from stack and jump to that label + Jump, + # opA = int: string number + RunLine, + # opA = string: command text + RunCommand, + # opA = int: string number for option to add + AddOption, + # present the current list of options, then clear the list; most recently + # selected option will be on the top of the stack + ShowOptions, + # opA = int: string number in table; push string to stack + PushString, + # opA = float: number to push to stack + PushNumber, + # opA = int (0 or 1): bool to push to stack + PushBool, + # pushes a null value onto the stack + PushNull, + # opA = string: label name if top of stack is not null, zero or false, jumps + # to that label + JumpIfFalse, + # discard top of stack + Pop, + # opA = string; looks up function, pops as many arguments as needed, result is + # pushed to stack + CallFunc, + # opA = name of variable to get value of and push to stack + PushVariable, + # opA = name of variable to store top of stack in + StoreVariable, + # stops execution + Stop, + # run the node whose name is at the top of the stack + RunNode +} + +enum TokenType { + + #0 Special tokens + Whitespace, Indent, Dedent, EndOfLine, EndOfInput, + + #5 Numbers. Everybody loves a number + Number, + + #6 Strings. Everybody also loves a string + Str, + + #7 '#' + TagMarker, + + #8 Command syntax ('<>') + BeginCommand, EndCommand, + + ExpressionFunctionStart, # { + ExpressionFunctionEnd, # } + + FormatFunctionStart, # [ + FormatFunctionEnd, # ] + + #10 Variables ('$foo') + Variable, + + #11 Shortcut syntax ('->') + ShortcutOption, + + #12 Option syntax ('[[Let's go here|Destination]]') + OptionStart, # [[ + OptionDelimit, # | + OptionEnd, # ]] + + #15 Command types (specially recognised command word) + IfToken, ElseIf, ElseToken, EndIf, Set, + + #20 Boolean values + TrueToken, FalseToken, + + #22 The null value + NullToken, + + #23 Parentheses + LeftParen, RightParen, + + #25 Parameter delimiters + Comma, + + #26 Operators + EqualTo, # ==, eq, is + GreaterThan, # >, gt + GreaterThanOrEqualTo, # >=, gte + LessThan, # <, lt + LessThanOrEqualTo, # <=, lte + NotEqualTo, # !=, neq + + #32 Logical operators + Or, # ||, or + And, # &&, and + Xor, # ^, xor + Not, # !, not + + # this guy's special because '=' can mean either 'equal to' + #36 or 'becomes' depending on context + EqualToOrAssign, # =, to + + #37 + UnaryMinus, # -; this is differentiated from Minus + # when parsing expressions + + #38 + Add, # + + Minus, # - + Multiply, # * + Divide, # / + Modulo, # % + + #43 + AddAssign, # += + MinusAssign, # -= + MultiplyAssign, # *= + DivideAssign, # /= + + Comment, # a run of text that we ignore + + Identifier, # a single word (used for functions) + + Text # a run of text until we hit other syntax +} + +enum ExpressionType { + Value, + FunctionCall +} + +enum StatementTypes { + CustomCommand, + ShortcutOptionGroup, + Block, + IfStatement, + OptionStatement, + AssignmentStatement, + Line, + InlineExpression +} + +enum ValueType { + Number, + Str, + Boolean, + Variable, + Nullean +} + +static func token_type_name(value): + for key in TokenType.keys(): + if TokenType[key] == value: + return key + return 'NOTVALID' + +static func merge_dir(target, patch): + for key in patch: + target[key] = patch[key] + +static func bytecode_name(bytecode): + return [ + 'Label', + 'JumpTo', + 'Jump', + 'RunLine', + 'RunCommand', + 'AddOption', + 'ShowOptions', + 'PushString', + 'PushNumber', + 'PushBool', + 'PushNull', + 'JumpIfFalse', + 'Pop', + 'CallFunc', + 'PushVariable', + 'StoreVariable', + 'Stop', + 'RunNode' + ][bytecode] + +static func token_name(type): + for key in TokenType.keys(): + if TokenType[key] == type: + return key + return '' + diff --git a/addons/Wol/core/Library.gd b/addons/Wol/core/Library.gd new file mode 100644 index 0000000..a0ff7c5 --- /dev/null +++ b/addons/Wol/core/Library.gd @@ -0,0 +1,25 @@ +extends Object + +const FunctionInfo = preload('res://addons/Wol/core/FunctionInfo.gd') +const Constants = preload('res://addons/Wol/core/Constants.gd') + +var functions = {} +var virtual_machine + +func get_function(name): + if functions.has(name): + return functions[name] + else : + printerr('Invalid Function: %s'% name) + return + +func import_library(other): + Constants.merge_dir(functions, other.functions) + other.virtual_machine = virtual_machine + +func register_function(name, parameter_count, function, returns_value): + var functionInfo = FunctionInfo.new(name, parameter_count, function, returns_value) + functions[name] = functionInfo + +func deregister_function(name): + functions.erase(name) diff --git a/addons/Wol/core/Program.gd b/addons/Wol/core/Program.gd new file mode 100644 index 0000000..4b9d282 --- /dev/null +++ b/addons/Wol/core/Program.gd @@ -0,0 +1,146 @@ +extends Object + +const Constants = preload('res://addons/Wol/core/Constants.gd') + +var name = '' +var filename = '' +var strings = {} +var nodes = {} + +class Line: + var text = '' + var node_name = '' + var line_number = -1 + var file_name = '' + var implicit = false + var substitutions = [] + var meta = [] + + func _init(_text, _node_name, _line_number, _file_name, _implicit, _meta): + text = _text + node_name = _node_name + file_name = _file_name + implicit = _implicit + meta = _meta + + func clone(): + return get_script().new(text, node_name, line_number, file_name, implicit, meta) + + func _to_string(): + return '%s:%d: "%s"' % [file_name.get_file(), line_number, text] + +class Option: + var line + var id = -1 + var destination = '' + + func _init(_line, _id, _destination): + line = _line + id = _id + destination = _destination + + func clone(): + return get_script().new(self) + +class Command: + var command = '' + + func _init(_command): + command = _command + +class WolNode: + var name = '' + var instructions = [] + var labels = {} + var tags = [] + var source_id = '' + + func _init(other = null): + if other != null and other.get_script() == self.get_script(): + name = other.name + instructions += other.instructions + for key in other.labels.keys(): + labels[key] = other.labels[key] + tags += other.tags + source_id = other.source_id + + func equals(other): + if other.get_script() != get_script(): + return false + if other.name != name: + return false + if other.instructions != instructions: + return false + if other.labels != labels: + return false + if other.source_id != source_id: + return false + return true + + func _to_string(): + return "WolNode[%s:%s]" % [name, source_id] + +# TODO: Make this make sense +class Operand: + enum ValueType { + None, + StringValue, + BooleanValue, + FloatValue + } + + var value + var type + + func _init(_value): + if typeof(_value) == TYPE_OBJECT and _value.get_script() == get_script(): + set_value(_value.value) + else: + set_value(_value) + + func set_value(_value): + match typeof(_value): + TYPE_REAL,TYPE_INT: + set_number(_value) + TYPE_BOOL: + set_boolean(_value) + TYPE_STRING: + set_string(_value) + + func set_boolean(_value): + value = _value + type = ValueType.BooleanValue + return self + + func set_string(_value): + value = _value + type = ValueType.StringValue + return self + + func set_number(_value): + value = _value + type = ValueType.FloatValue + return self + + func clear_value(): + type = ValueType.None + value = null + + func clone(): + return get_script().new(self) + + func _to_string(): + return "Operand[%s:%s]" % [type, value] + +class Instruction: + var operation = -1 + var operands = [] + + func _init(other = null): + if other != null and other.get_script() == self.get_script(): + self.operation = other.operation + self.operands += other.operands + + func _to_string(): + return Constants.bytecode_name(operation) + ':' + operands as String + diff --git a/addons/Wol/core/Value.gd b/addons/Wol/core/Value.gd new file mode 100644 index 0000000..161df5a --- /dev/null +++ b/addons/Wol/core/Value.gd @@ -0,0 +1,137 @@ +extends Object + +const Constants = preload('res://addons/Wol/core/Constants.gd') + +const NANI = 'NaN' + +var type = Constants.ValueType.Nullean +var number = 0 +var string = '' +var variable = '' +var boolean = false + +func _init(value = NANI): + if typeof(value) == TYPE_OBJECT and value.get_script() == get_script(): + if value.type == Constants.ValueType.Variable: + type = value.type + variable = value.variable + else: + set_value(value) + +func value(): + match type: + Constants.ValueType.Number: + return number + Constants.ValueType.Str: + return string + Constants.ValueType.Boolean: + return boolean + Constants.ValueType.Variable: + return variable + return null + +func as_bool(): + match type: + Constants.ValueType.Number: + return number != 0 + Constants.ValueType.Str: + return !string.empty() + Constants.ValueType.Boolean: + return boolean + return false + +func as_string(): + return '%s' % value() + +func as_number(): + match type: + Constants.ValueType.Number: + return number + Constants.ValueType.Str: + return float(string) + Constants.ValueType.Boolean: + return 0.0 if !boolean else 1.0 + return .0 + +func set_value(value): + if value == null or (typeof(value) == TYPE_STRING and value == NANI): + type = Constants.ValueType.Nullean + return + + match typeof(value): + TYPE_INT, TYPE_REAL: + type = Constants.ValueType.Number + number = value + TYPE_STRING: + type = Constants.ValueType.Str + string = value + TYPE_BOOL: + type = Constants.ValueType.Boolean + boolean = value + +func add(other): + if type == Constants.ValueType.Str or other.type == Constants.ValueType.Str: + return get_script().new('%s%s' % [value(), other.value()]) + if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number: + return get_script().new(number + other.number) + return null + +func equals(other): + if other.get_script() != get_script(): + return false + if other.value() != value(): + return false + # TODO: Add more equality cases + return true + +func sub(other): + if type == Constants.ValueType.Str or other.type == Constants.ValueType.Str: + return get_script().new(str(value()).replace(str(other.value()),'')) + if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number: + return get_script().new(number - other.number) + return null + +func mult(other): + if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number: + return get_script().new(number * other.number) + return null + +func div(other): + if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number: + return get_script().new(number / other.number) + return null + +func mod(other): + if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number: + return get_script().new(number % other.number) + return null + +func negative(): + if type == Constants.ValueType.Number: + return get_script().new(-number) + return null + +func greater(other): + if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number: + return number > other.number + return false + +func less(other): + if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number: + return number < other.number + return false + +func geq(other): + if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number: + return number > other.number or equals(other) + return false + +func leq(other): + if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number: + return number < other.number or equals(other) + return false + +func _to_string(): + return 'value(type[%s]: %s)' % [type,value()] + + diff --git a/addons/Wol/core/compiler/Compiler.gd b/addons/Wol/core/compiler/Compiler.gd new file mode 100644 index 0000000..617042d --- /dev/null +++ b/addons/Wol/core/compiler/Compiler.gd @@ -0,0 +1,461 @@ +extends Object + +signal error(message, line_number, column) + +const Constants = preload('res://addons/Wol/core/Constants.gd') +const Lexer = preload('res://addons/Wol/core/compiler/Lexer.gd') +const Program = preload('res://addons/Wol/core/Program.gd') +const Parser = preload('res://addons/Wol/core/compiler/Parser.gd') + +const INVALID_TITLE = '[\\[<>\\]{}\\|:\\s#\\$]' + +var source = '' +var filename = '' + +var current_node +var has_implicit_string_tags = false +var soft_assert = false + +var string_count = 0 +var string_table = {} +var label_count = 0 + +func _init(_filename, _source = null, _soft_assert = false): + filename = _filename + soft_assert = _soft_assert + + if not _filename and _source: + self.source = _source + else: + var file = File.new() + file.open(_filename, File.READ) + self.source = file.get_as_text() + file.close() + + var source_lines = source.split('\n') + for i in range(source_lines.size()): + source_lines[i] = source_lines[i].strip_edges(false, true) + + source = source_lines.join('\n') + +func get_headers(offset = 0): + var header_property = RegEx.new() + var header_sep = RegEx.new() + + header_sep.compile('---(\r\n|\r|\n)') + header_property.compile('(?.*): *(?.*)') + + self.assert(header_sep.search(source), 'No headers found!') + + var title = '' + var position = Vector2.ZERO + + var source_lines = source.split('\n') + var line_number = offset + while line_number < source_lines.size(): + var line = source_lines[line_number] + line_number += 1 + + if not line.empty(): + var result = header_property.search(line) + + if result != null: + var field = result.get_string('field') + var value = result.get_string('value') + + if field == 'title': + var regex = RegEx.new() + regex.compile(INVALID_TITLE) + self.assert(not regex.search(value), 'Invalid characters in title "%s", correct to "%s"' % [value, regex.sub(value, '', true)]) + + title = value + + if field == 'position': + var regex = RegEx.new() + regex.compile('^position:.*,.*\\d$') + self.assert(regex.search(line), 'Couldn\'t parse position property in the headers, got "%s" instead in node "%s"' % [value, title]) + + position = Vector2(int(value.split(',')[0].strip_edges()), int(value.split(',')[1].strip_edges())) + + # TODO: Implement color and tags + + if line == '---': + break + + return { + 'title': title, + 'position': position + } + +func get_body(offset = 0): + var body_lines = [] + + var source_lines = source.split('\n') + var recording = false + var line_number = offset + + while line_number < source_lines.size() and source_lines[line_number] != '===': + if recording: + body_lines.append(source_lines[line_number]) + + recording = recording or source_lines[line_number] == '---' + line_number += 1 + + line_number += 1 + + return PoolStringArray(body_lines).join('\n') + +func get_nodes(): + var nodes = [] + var line_number = 0 + var source_lines = source.split('\n') + while line_number < source_lines.size(): + var headers = get_headers(line_number) + var body = get_body(line_number) + headers.body = body + + nodes.append(headers) + + # Add +2 to the final line to skip the === from that node + line_number = Array(source_lines).find_last(body.split('\n')[-1]) + 2 + + while line_number < source_lines.size() and source_lines[line_number].empty(): + line_number += 1 + + return nodes + +func assert(statement, message, line_number = -1, column = -1, _absolute_line_number = -1): + if not soft_assert: + assert(statement, message + ('; on line %d column %d' % [line_number, column])) + elif not statement: + emit_signal('error', message, line_number, column) + +func compile(): + var parsed_nodes = [] + for node in get_nodes(): + var lexer = Lexer.new(self, filename, node.title, node.body) + var tokens = lexer.tokenize() + + # In case of lexer error + if not tokens: + return + + var parser = Parser.new(self, node.title, tokens) + var parser_node = parser.parse_node() + + parser_node.name = node.title + parsed_nodes.append(parser_node) + + var program = Program.new() + program.filename = filename + + for node in parsed_nodes: + compile_node(program, node) + + for key in string_table: + program.strings[key] = string_table[key] + + return program + +func compile_node(program, parsed_node): + self.assert(not program.nodes.has(parsed_node.name), 'Duplicate node in program: %s' % parsed_node.name) + + var node_compiled = Program.WolNode.new() + + node_compiled.name = parsed_node.name + node_compiled.tags = parsed_node.tags + + if parsed_node.source != null and not parsed_node.source.empty(): + node_compiled.source_id = register_string( + parsed_node.source, + parsed_node.name, + 'line:' + parsed_node.name, + 0, + [] + ) + else: + var start_label = register_label() + emit(Constants.ByteCode.Label, node_compiled, [Program.Operand.new(start_label)]) + + for statement in parsed_node.statements: + generate_statement(node_compiled, statement) + + var dangling_options = false + for instruction in node_compiled.instructions: + if instruction.operation == Constants.ByteCode.AddOption: + dangling_options = true + if instruction.operation == Constants.ByteCode.ShowOptions: + dangling_options = false + + if dangling_options: + emit(Constants.ByteCode.ShowOptions, node_compiled) + emit(Constants.ByteCode.RunNode, node_compiled) + else: + emit(Constants.ByteCode.Stop, node_compiled) + + program.nodes[node_compiled.name] = node_compiled + +func register_string(text, node_name, id = '', line_number = -1, tags = []): + var line_id_used = '' + var implicit = false + + if id.empty(): + line_id_used = '%s-%s-%d' % [filename, node_name, string_count] + string_count += 1 + + #use this when we generate implicit tags + #they are not saved and are generated + #aka dummy tags that change on each compilation + has_implicit_string_tags = true + + implicit = true + else : + line_id_used = id + implicit = false + + var string_info = Program.Line.new(text, node_name, line_number, filename, implicit, tags) + string_table[line_id_used] = string_info + + return line_id_used + +func register_label(comment = ''): + label_count += 1 + return 'Label%s%s' % [label_count, comment] + +func emit(bytecode, node = current_node, operands = []): + var instruction = Program.Instruction.new(null) + instruction.operation = bytecode + instruction.operands = operands + + if node == null: + printerr('Trying to emit to null node with byte_code: %s' % bytecode) + return + + node.instructions.append(instruction) + + if bytecode == Constants.ByteCode.Label: + node.labels[instruction.operands[0].value] = node.instructions.size() - 1 + +func generate_statement(node, statement): + match statement.type: + Constants.StatementTypes.CustomCommand: + generate_custom_command(node, statement.custom_command) + + Constants.StatementTypes.ShortcutOptionGroup: + generate_shortcut_group(node, statement.shortcut_option_group) + + Constants.StatementTypes.Block: + generate_block(node, statement.block.statements) + + Constants.StatementTypes.IfStatement: + generate_if(node, statement.if_statement) + + Constants.StatementTypes.OptionStatement: + generate_option(node, statement.option_statement) + + Constants.StatementTypes.AssignmentStatement: + generate_assignment(node, statement.assignment) + + Constants.StatementTypes.Line: + generate_line(node, statement) + _: + self.assert(false, statement.line_number, 'Illegal statement type [%s]. Could not generate code.' % statement.type) + +func generate_custom_command(node, command): + # TODO: See if the first tree of this statement is being used + if command.expression != null: + generate_expression(node, command.expression) + else: + var command_string = command.client_command + if command_string == 'stop': + emit(Constants.ByteCode.Stop, node) + else : + emit(Constants.ByteCode.RunCommand, node, [Program.Operand.new(command_string)]) + +func generate_line(node, statement): + # TODO: Implement proper line numbers (global and local) + var line = statement.line + var expression_count = line.substitutions.size() + + while not line.substitutions.empty(): + var inline_expression = line.substitutions.pop_back() + generate_expression(node, inline_expression.expression) + + var num = register_string(line.line_text, node.name, line.line_id, statement.line_number, line.tags); + emit(Constants.ByteCode.RunLine, node,[Program.Operand.new(num), Program.Operand.new(expression_count)]) + +func generate_shortcut_group(node, shortcut_group): + var end = register_label('group_end') + var labels = [] + var option_count = 0 + + for option in shortcut_group.options: + var endof_clause = '' + var op_destination = register_label('option_%s' % [option_count + 1]) + + labels.append(op_destination) + + if option.condition != null: + endof_clause = register_label('conditional_%s' % option_count) + generate_expression(node, option.condition) + emit(Constants.ByteCode.JumpIfFalse, node, [Program.Operand.new(endof_clause)]) + + var label_line_id = '' #TODO: Add tag support + var label_string_id = register_string( + option.label, + node.name, + label_line_id, + option.line_number, + [] + ) + + emit(Constants.ByteCode.AddOption, node, [Program.Operand.new(label_string_id), Program.Operand.new(op_destination)]) + + if option.condition != null: + emit(Constants.ByteCode.Label, node, [Program.Operand.new(endof_clause)]) + emit(Constants.ByteCode.Pop, node) + + option_count += 1 + + emit(Constants.ByteCode.ShowOptions, node) + emit(Constants.ByteCode.Jump, node) + + option_count = 0 + + for option in shortcut_group.options: + emit(Constants.ByteCode.Label, node, [Program.Operand.new(labels[option_count])]) + + if option.node != null: + generate_block(node, option.node.statements) + emit(Constants.ByteCode.JumpTo, node, [Program.Operand.new(end)]) + option_count += 1 + + emit(Constants.ByteCode.Label, node, [Program.Operand.new(end)]) + emit(Constants.ByteCode.Pop, node) + +func generate_block(node, statements = []): + if not statements.empty(): + for statement in statements: + generate_statement(node, statement) + + +func generate_if(node, if_statement): + var endif = register_label('endif') + + for clause in if_statement.clauses: + var end_clause = register_label('skip_clause') + + if clause.expression != null: + generate_expression(node, clause.expression) + emit(Constants.ByteCode.JumpIfFalse, node, [Program.Operand.new(end_clause)]) + + generate_block(node, clause.statements) + emit(Constants.ByteCode.JumpTo, node, [Program.Operand.new(endif)]) + + if clause.expression != null: + emit(Constants.ByteCode.Label, node, [Program.Operand.new(end_clause)]) + + if clause.expression != null: + emit(Constants.ByteCode.Pop) + + emit(Constants.ByteCode.Label, node, [Program.Operand.new(endif)]) + +func generate_option(node, option): + var destination = option.destination + + if option.label == null or option.label.empty(): + emit(Constants.ByteCode.RunNode, node, [Program.Operand.new(destination)]) + else : + var line_id = '' #TODO: ADD TAG SUPPORT + var string_id = register_string(option.label, node.name, line_id, option.line_number, []) + + emit(Constants.ByteCode.AddOption, node, [Program.Operand.new(string_id), Program.Operand.new(destination)]) + +func generate_assignment(node, assignment): + if assignment.operation == Constants.TokenType.EqualToOrAssign: + generate_expression(node, assignment.value) + else : + emit(Constants.ByteCode.PushVariable, node, [assignment.destination]) + generate_expression(node, assignment.value) + + match assignment.operation: + Constants.TokenType.AddAssign: + emit( + Constants.ByteCode.CallFunc, + node, + [Program.Operand.new(Constants.token_type_name(Constants.TokenType.Add))] + ) + Constants.TokenType.MinusAssign: + emit( + Constants.ByteCode.CallFunc, + node, + [Program.Operand.new(Constants.token_type_name(Constants.TokenType.Minus))] + ) + Constants.TokenType.MultiplyAssign: + emit( + Constants.ByteCode.CallFunc, + node, + [Program.Operand.new(Constants.token_type_name(Constants.TokenType.MultiplyAssign))] + ) + Constants.TokenType.DivideAssign: + emit( + Constants.ByteCode.CallFunc, + node, + [Program.Operand.new(Constants.token_type_name(Constants.TokenType.DivideAssign))] + ) + _: + printerr('Unable to generate assignment') + + emit(Constants.ByteCode.StoreVariable, node, [Program.Operand.new(assignment.destination)]) + emit(Constants.ByteCode.Pop, node) + +func generate_expression(node, expression): + match expression.type: + Constants.ExpressionType.Value: + generate_value(node, expression.value) + Constants.ExpressionType.FunctionCall: + for parameter in expression.parameters: + generate_expression(node, parameter) + + emit(Constants.ByteCode.PushNumber, node, [Program.Operand.new(expression.parameters.size())]) + emit(Constants.ByteCode.CallFunc, node, [Program.Operand.new(expression.function)]) + _: + printerr('No expression.') + +func generate_value(node, value): + match value.value.type: + Constants.ValueType.Number: + emit( + Constants.ByteCode.PushNumber, + node, + [Program.Operand.new(value.value.as_number())] + ) + Constants.ValueType.Str: + var id = register_string( + value.value.as_string(), + node.name, + '', + value.line_number, + [] + ) + emit( + Constants.ByteCode.PushString, + node, + [Program.Operand.new(id)] + ) + Constants.ValueType.Boolean: + emit( + Constants.ByteCode.PushBool, + node, + [Program.Operand.new(value.value.as_bool())] + ) + Constants.ValueType.Variable: + emit( + Constants.ByteCode.PushVariable, + node, + [Program.Operand.new(value.value.variable)] + ) + Constants.ValueType.Nullean: + emit(Constants.ByteCode.PushNull, node) + _: + printerr('Unrecognized valuenode type: %s' % value.value.type) diff --git a/addons/Wol/core/compiler/Lexer.gd b/addons/Wol/core/compiler/Lexer.gd new file mode 100644 index 0000000..b651b99 --- /dev/null +++ b/addons/Wol/core/compiler/Lexer.gd @@ -0,0 +1,485 @@ +extends Object + +const Constants = preload('res://addons/Wol/core/Constants.gd') + +const LINE_COMMENT = '//' +const FORWARD_SLASH = '/' +const LINE_SEPARATOR = '\n' + +const BASE = 'base' +const DASH = '-' +const COMMAND = 'command' +const LINK = 'link' +const SHORTCUT = 'shortcut' +const TAG = 'tag' +const EXPRESSION = 'expression' +const ASSIGNMENT = 'assignment' +const OPTION = 'option' +const OR = 'or' +const DESTINATION = 'destination' +const INLINE = 'inline' +const FORMAT_FUNCTION = 'format' + +var WHITESPACE = '\\s*' + +var compiler +var filename = '' +var title = '' +var text = '' + +var states = {} +var default_state +var current_state + +var indent_stack = [] +var should_track_indent = false + +func _init(_compiler, _filename, _title, _text): + create_states() + + compiler = _compiler + filename = _filename + title = _title + text = _text + +func create_states(): + var patterns = {} + patterns[Constants.TokenType.Text] = ['.*', 'any text'] + + patterns[Constants.TokenType.Number] = ['\\-?[0-9]+(\\.[0-9+])?', 'any number'] + patterns[Constants.TokenType.Str] = ['\"([^\"\\\\]*(?:\\.[^\"\\\\]*)*)\"', 'any text'] + patterns[Constants.TokenType.TagMarker] = ['\\#', 'a tag #'] + patterns[Constants.TokenType.LeftParen] = ['\\(', 'left parenthesis ('] + patterns[Constants.TokenType.RightParen] = ['\\)', 'right parenthesis )'] + patterns[Constants.TokenType.EqualTo] = ['(==|is(?!\\w)|eq(?!\\w))', '"=", "is" or "eq"'] + patterns[Constants.TokenType.EqualToOrAssign] = ['(=|to(?!\\w))', '"=" or "to"'] + patterns[Constants.TokenType.NotEqualTo] = ['(\\!=|neq(?!\\w))', '"!=" or "neq"'] + patterns[Constants.TokenType.GreaterThanOrEqualTo] = ['(\\>=|gte(?!\\w))', '">=" or "gte"'] + patterns[Constants.TokenType.GreaterThan] = ['(\\>|gt(?!\\w))', '">" or "gt"'] + patterns[Constants.TokenType.LessThanOrEqualTo] = ['(\\<=|lte(?!\\w))', '"<=" or "lte"'] + patterns[Constants.TokenType.LessThan] = ['(\\<|lt(?!\\w))', '"<" or "lt"'] + patterns[Constants.TokenType.AddAssign] = ['\\+=', '"+="'] + patterns[Constants.TokenType.MinusAssign] = ['\\-=', '"-="'] + patterns[Constants.TokenType.MultiplyAssign] = ['\\*=', '"*="'] + patterns[Constants.TokenType.DivideAssign] = ['\\/=', '"/="'] + patterns[Constants.TokenType.Add] = ['\\+', '"+"'] + patterns[Constants.TokenType.Minus] = ['\\-', '"-"'] + patterns[Constants.TokenType.Multiply] = ['\\*', '"*"'] + patterns[Constants.TokenType.Divide] = ['\\/', '"/"'] + patterns[Constants.TokenType.Modulo] = ['\\%', '"%"'] + patterns[Constants.TokenType.And] = ['(\\&\\&|and(?!\\w))', '"&&" or "and"'] + patterns[Constants.TokenType.Or] = ['(\\|\\||or(?!\\w))', '"||" or "or"'] + patterns[Constants.TokenType.Xor] = ['(\\^|xor(?!\\w))', '"^" or "xor"'] + patterns[Constants.TokenType.Not] = ['(\\!|not(?!\\w))', '"!" or "not"'] + patterns[Constants.TokenType.Variable] = ['\\$([A-Za-z0-9_\\.])+', 'any variable'] + patterns[Constants.TokenType.Comma] = ['\\,', '","'] + patterns[Constants.TokenType.TrueToken] = ['true(?!\\w)', '"true"'] + patterns[Constants.TokenType.FalseToken] = ['false(?!\\w)', '"false"'] + patterns[Constants.TokenType.NullToken] = ['null(?!\\w)', '"null"'] + patterns[Constants.TokenType.BeginCommand] = ['\\<\\<', 'beginning of a command "<<"'] + patterns[Constants.TokenType.EndCommand] = ['\\>\\>', 'ending of a command ">>"'] + patterns[Constants.TokenType.OptionStart] = ['\\[\\[', 'start of an option "[["'] + patterns[Constants.TokenType.OptionEnd] = ['\\]\\]', 'end of an option "]]"'] + patterns[Constants.TokenType.OptionDelimit] = ['\\|', 'middle of an option "|"'] + patterns[Constants.TokenType.Identifier] = ['[a-zA-Z0-9_:\\.]+', 'any reference to another node'] + patterns[Constants.TokenType.IfToken] = ['if(?!\\w)', '"if"'] + patterns[Constants.TokenType.ElseToken] = ['else(?!\\w)', '"else"'] + patterns[Constants.TokenType.ElseIf] = ['elseif(?!\\w)', '"elseif"'] + patterns[Constants.TokenType.EndIf] = ['endif(?!\\w)', '"endif"'] + patterns[Constants.TokenType.Set] = ['set(?!\\w)', '"set"'] + patterns[Constants.TokenType.ShortcutOption] = ['\\-\\>\\s*', '"->"'] + patterns[Constants.TokenType.ExpressionFunctionStart] = ['\\{', '"{"'] + patterns[Constants.TokenType.ExpressionFunctionEnd] = ['\\}', '"}"'] + patterns[Constants.TokenType.FormatFunctionStart] = ['(? previous_indentation: + indent_stack.push_front([indentation, true]) + + var indent = Token.new( + Constants.TokenType.Indent, + current_state, + filename, + line_number, + previous_indentation + ) + indent.value = '%*s' % [indentation - previous_indentation, ''] + + should_track_indent = false + token_stack.push_front(indent) + + elif indentation < previous_indentation: + while indentation < indent_stack.front()[0]: + var top = indent_stack.pop_front()[1] + if top: + var deindent = Token.new(Constants.TokenType.Dedent, current_state, line_number, 0) + token_stack.push_front(deindent) + + var column = indentation + var whitespace = RegEx.new() + whitespace.compile(WHITESPACE) + + while column < fresh_line.length(): + if fresh_line.substr(column).begins_with(LINE_COMMENT): + break + + var matched = false + + for rule in current_state.rules: + var found = rule.regex.search(fresh_line, column) + + if !found: + continue + + var token_text = '' + + # NOTE: If this is text then we back up to the most recent delimiting token + # and treat everything from there as text. + if rule.token_type == Constants.TokenType.Text: + + var start_index = indentation + + if token_stack.size() > 0 : + while token_stack.front().type == Constants.TokenType.Identifier: + token_stack.pop_front() + + var start_delimit_token = token_stack.front() + start_index = start_delimit_token.column + + if start_delimit_token.type == Constants.TokenType.Indent: + start_index += start_delimit_token.value.length() + if start_delimit_token.type == Constants.TokenType.Dedent: + start_index = indentation + + column = start_index + var end_index = found.get_start() + found.get_string().length() + + token_text = fresh_line.substr(start_index, end_index - start_index) + else: + token_text = found.get_string() + + column += token_text.length() + + if rule.token_type == Constants.TokenType.Str: + token_text = token_text.substr(1, token_text.length() - 2) + token_text = token_text.replace('\\\\', '\\') + token_text = token_text.replace('\\\'','\'') + + var token = Token.new( + rule.token_type, + current_state, + filename, + line_number, + column, + token_text + ) + token.delimits_text = rule.delimits_text + + token_stack.push_front(token) + + if rule.enter_state != null and rule.enter_state.length() > 0: + if not states.has(rule.enter_state): + printerr('State[%s] not known - line(%s) col(%s)' % [rule.enter_state, line_number, column]) + return [] + + enter_state(states[rule.enter_state]) + + if should_track_indent: + if indent_stack.front()[0] < indentation: + indent_stack.append([indentation, false]) + + matched = true + break + + if not matched: + var rules = [] + for rule in current_state.rules: + rules.append('"%s" (%s)' % [Constants.token_type_name(rule.token_type), rule.human_readable_identifier]) + + var error_data = [ + PoolStringArray(rules).join(', ') if rules.size() == 1 else PoolStringArray(rules.slice(0, rules.size() - 2)).join(', ') + ' or %s' % rules[-1], + filename, + title, + line_number, + column + ] + compiler.assert(false, 'Expected %s in file %s in node "%s" on line #%d (column #%d)' % error_data, line_number, column) + return + + var last_whitespace = whitespace.search(line, column) + if last_whitespace: + column += last_whitespace.get_string().length() + + token_stack.invert() + + return token_stack + +func line_indentation(line): + var indent_regex = RegEx.new() + indent_regex.compile('^(\\s*)') + + var found = indent_regex.search(line) + + if !found or found.get_string().length() <= 0: + return 0 + + return found.get_string().length() + +func enter_state(state): + current_state = state; + if current_state.track_indent: + should_track_indent = true + +class Token: + var type = -1 + var value = '' + + var filename = '' + var line_number = -1 + var column = -1 + var text = '' + + var delimits_text = false + var parameter_count = -1 + var lexer_state = '' + + func _init(_type, _state, _filename, _line_number = -1, _column = -1, _value = ''): + type = _type + lexer_state = _state.name + filename = _filename + line_number = _line_number + column = _column + value = _value + + func _to_string(): + return '%s (%s) at %s:%s (state: %s)' % [Constants.token_type_name(type), value, line_number, column, lexer_state] + +class LexerState: + var name = '' + var patterns = {} + var rules = [] + var track_indent = false + + func _init(_patterns): + patterns = _patterns + + func add_transition(type, state = '', delimit_text = false): + var pattern = '\\G%s' % patterns[type][0] + var rule = Rule.new(type, pattern, patterns[type][1], state, delimit_text) + rules.append(rule) + return rule + + func add_text_rule(type, state = ''): + if contains_text_rule() : + printerr('State already contains Text rule') + return null + + var delimiters:Array = [] + for rule in rules: + if rule.delimits_text: + delimiters.append('%s' % rule.regex.get_pattern().substr(2)) + + var pattern = '\\G((?!%s).)*' % [PoolStringArray(delimiters).join('|')] + var rule = add_transition(type, state) + rule.regex = RegEx.new() + rule.regex.compile(pattern) + rule.is_text_rule = true + return rule + + func contains_text_rule(): + for rule in rules: + if rule.is_text_rule: + return true + return false + +class Rule: + var regex + + var enter_state = '' + var token_type = -1 + var is_text_rule = false + var delimits_text = false + var human_readable_identifier = '' + + func _init(_type, _regex, _human_readable_identifier, _enter_state, _delimits_text): + token_type = _type + + regex = RegEx.new() + regex.compile(_regex) + + human_readable_identifier = _human_readable_identifier + enter_state = _enter_state + delimits_text = _delimits_text + + func _to_string(): + return '[Rule : %s (%s) - %s]' % [Constants.token_type_name(token_type), human_readable_identifier, regex] diff --git a/addons/Wol/core/compiler/Parser.gd b/addons/Wol/core/compiler/Parser.gd new file mode 100644 index 0000000..b4385eb --- /dev/null +++ b/addons/Wol/core/compiler/Parser.gd @@ -0,0 +1,1065 @@ +extends Object + +# warnings-disable + +const Constants = preload('res://addons/Wol/core/Constants.gd') +const Lexer = preload('res://addons/Wol/core/compiler/Lexer.gd') +const Value = preload('res://addons/Wol/core/Value.gd') + +var compiler +var title = '' +var tokens = [] + +func _init(_compiler, _title, _tokens): + compiler = _compiler + title = _title + tokens = _tokens + +enum Associativity { + Left, + Right, + None +} + +func parse_node(): + return WolNode.new('Start', null, self) + +func next_symbol_is(valid_types): + return tokens.front().type in valid_types + +# NOTE: 0 look ahead for `<<` and `else` +func next_symbols_are(valid_types): + var temporary = [] + tokens + for type in valid_types: + if temporary.pop_front().type != type: + return false + return true + +func expect_symbol(token_types = []): + var token = tokens.pop_front() as Lexer.Token + + if token_types.size() == 0: + if token.type == Constants.TokenType.EndOfInput: + compiler.assert(false, 'Unexpected end of input') + return token + + for type in token_types: + if token.type == type: + return token + + var token_names = [] + for type in token_types: + token_names.append(Constants.token_type_name(type)) + + var error_guess = '\n' + + if Constants.token_type_name(token.type) == 'Identifier' \ + and Constants.token_type_name(token_types[0]) == 'OptionEnd': + error_guess += 'Does the node your refer to have a space in it?' + else: + error_guess = '' + + var error_data = [ + PoolStringArray(token_names).join(', '), + Constants.token_type_name(token.type), + error_guess + ] + compiler.assert(false, 'Expected token "%s" but got "%s"%s' % error_data, token.line_number, token.column) + return token + +static func tab(indent_level, input, newline = true): + return '%*s| %s%s' % [indent_level * 2, '', input, '' if not newline else '\n'] + +class ParseNode: + var name = '' + + var parent + var line_number = -1 + var tags = [] + + func _init(_parent, _parser): + parent = _parent + + var tokens = _parser.tokens as Array + if tokens.size() > 0: + line_number = tokens.front().line_number + + tags = [] + + func tree_string(_indent_level): + return 'Not_implemented' + + func tags_to_string(_indent_level): + return 'TAGSNOTIMPLEMENTED' + + func get_node_parent(): + var node = self + while node != null: + if node is ParseNode: + return node as WolNode + node = node.parent + return null + + func tab(indent_level, input, newline = true): + return '%*s| %s%s' % [ indent_level * 2, '', input, '' if !newline else '\n'] + +class WolNode extends ParseNode: + var source = '' + + var editor_node_tags = [] + var statements = [] + var parser + + func _init(_name, parent, _parser).(parent, _parser): + name = _name + parser = _parser + while parser.tokens.size() > 0 \ + and not parser.next_symbol_is([Constants.TokenType.Dedent, Constants.TokenType.EndOfInput]): + + parser.compiler.assert( + not parser.next_symbol_is([Constants.TokenType.Indent]), + 'Found a stray indentation!', + parser.tokens.front().line_number, + parser.tokens.front().column + ) + + statements.append(Statement.new(self, parser)) + + func tree_string(indent_level): + var info = [] + for statement in statements: + info.append(statement.tree_string(indent_level + 1)) + + return PoolStringArray(info).join('') + +class Statement extends ParseNode: + var Type = Constants.StatementTypes + + var type = -1 + var block + var if_statement + var option_statement + var assignment + var shortcut_option_group + var custom_command + var line + + func _init(parent, parser).(parent, parser): + if Block.can_parse(parser): + block = Block.new(self, parser) + type = Type.Block + + elif IfStatement.can_parse(parser): + if_statement = IfStatement.new(self, parser) + type = Type.IfStatement + + elif OptionStatement.can_parse(parser): + option_statement = OptionStatement.new(self, parser) + type = Type.OptionStatement + + elif Assignment.can_parse(parser): + assignment = Assignment.new(self, parser) + type = Type.AssignmentStatement + + elif ShortcutOptionGroup.can_parse(parser): + shortcut_option_group = ShortcutOptionGroup.new(self, parser) + type = Type.ShortcutOptionGroup + + elif CustomCommand.can_parse(parser): + custom_command = CustomCommand.new(self, parser) + type = Type.CustomCommand + + elif parser.next_symbol_is([Constants.TokenType.Text]): + line = LineNode.new(self, parser) + type = Type.Line + + else: + parser.compiler.assert(false, 'Expected a statement but got %s instead. (probably an imbalanced if statement)' % parser.tokens.front()._to_string()) + + var tags = [] + + while parser.next_symbol_is([Constants.TokenType.TagMarker]): + parser.expect_symbol([Constants.TokenType.TagMarker]) + var tag = parser.expect_symbol([Constants.TokenType.Identifier]).value + tags.append(tag) + + if tags.size() > 0: + self.tags = tags + + func tree_string(indent_level): + var info = [] + + match type: + Type.Block: + info.append(block.tree_string(indent_level)) + Type.IfStatement: + info.append(if_statement.tree_string(indent_level)) + Type.AssignmentStatement: + info.append(assignment.tree_string(indent_level)) + Type.OptionStatement: + info.append(option_statement.tree_string(indent_level)) + Type.ShortcutOptionGroup: + info.append(shortcut_option_group.tree_string(indent_level)) + Type.CustomCommand: + info.append(custom_command.tree_string(indent_level)) + Type.Line: + info.append(line.tree_string(indent_level)) + _: + printerr('Cannot print statement') + + return PoolStringArray(info).join('') + +class InlineExpression extends ParseNode: + var expression + + func _init(parent, parser).(parent, parser): + parser.expect_symbol([Constants.TokenType.ExpressionFunctionStart]) + expression = ExpressionNode.parse(self, parser) + parser.expect_symbol([Constants.TokenType.ExpressionFunctionEnd]) + + static func can_parse(parser): + return parser.next_symbol_is([Constants.TokenType.ExpressionFunctionStart]) + + func tree_string(_indent_level): + return "InlineExpression:" + +# Returns a format_text string as [ name "{0}" key1="value1" key2="value2" ] +class FormatFunctionNode extends ParseNode: + var format_text = '' + var expression_value + + func _init(parent:ParseNode, parser, expressionCount:int).(parent, parser): + format_text="[" + parser.expect_symbol([Constants.TokenType.FormatFunctionStart]) + + # FIXME: Add exit condition in case of failure + while not parser.next_symbol_is([Constants.TokenType.FormatFunctionEnd]): + if parser.next_symbol_is([Constants.TokenType.Text]): + format_text += parser.expect_symbol().value + + if InlineExpression.can_parse(parser): + expression_value = InlineExpression.new(self, parser) + format_text +=" \"{%d}\" " % expressionCount + parser.expect_symbol() + format_text+="]" + + static func can_parse(parser): + return parser.next_symbol_is([Constants.TokenType.FormatFunctionStart]) + + # TODO: Make format prettier and add more information + func tree_string(_indent_level): + return "FormatFunction" + +class LineNode extends ParseNode: + var line_text = '' + # FIXME: Right now we are putting the formatfunctions and inline expressions in the same + # list but if at some point we want to strongly type our sub list we need to make a new + # parse node that can have either an InlineExpression or a FunctionFormat + # .. This is a consideration for Godot4.x + var substitutions = [] + var line_id = '' + var line_tags = [] + + # NOTE: If format function an inline functions are both present + # returns a line in the format "Some text {0} and some other {1}[format "{2}" key="value" key="value"]" + + func _init(parent, parser).(parent, parser): + while parser.next_symbol_is( + [ + Constants.TokenType.FormatFunctionStart, + Constants.TokenType.ExpressionFunctionStart, + Constants.TokenType.Text, + Constants.TokenType.TagMarker + ] + ): + + if FormatFunctionNode.can_parse(parser): + var format_function = FormatFunctionNode.new(self, parser, substitutions.size()) + if format_function.expression_value != null: + substitutions.append(format_function.expression_value) + + line_text += format_function.format_text + + elif InlineExpression.can_parse(parser): + var inline_expression = InlineExpression.new(self, parser) + line_text += '{%d}' % substitutions.size() + substitutions.append(inline_expression) + + elif parser.next_symbols_are([Constants.TokenType.TagMarker, Constants.TokenType.Identifier]): + parser.expect_symbol() + var tag_token = parser.expect_symbol([ Constants.TokenType.Identifier ]) + if tag_token.value.begins_with("line:"): + if line_id.empty(): + line_id = tag_token.value + else: + printerr("Too many line_tags @[%s:%d]" % [parser.currentNodeName, tag_token.line_number]) + return + else: + tags.append(tag_token.value) + + else: + var token = parser.expect_symbol() + if token.line_number == line_number and token.type != Constants.TokenType.BeginCommand: + line_text += token.value + else: + parser.tokens.push_front(token) + break + + + func tree_string(indent_level): + return tab(indent_level, 'Line: (%s)[%d]' % [line_text, substitutions.size()]) + + +class CustomCommand extends ParseNode: + + enum Type { + Expression, + ClientCommand + } + + var type = -1 + var expression + var client_command + + func _init(parent, parser).(parent, parser): + parser.expect_symbol([Constants.TokenType.BeginCommand]) + + var command_tokens = [] + command_tokens.append(parser.expect_symbol()) + + # FIXME: add exit condition + while not parser.next_symbol_is([Constants.TokenType.EndCommand]): + command_tokens.append(parser.expect_symbol()) + + parser.expect_symbol([Constants.TokenType.EndCommand]) + + #if first token is identifier and second is leftt parenthesis + #evaluate as function + if command_tokens.size() > 1 \ + and command_tokens[0].type == Constants.TokenType.Identifier \ + and command_tokens[1].type == Constants.TokenType.LeftParen: + + var p = get_script().new(command_tokens, parser.library) + expression = ExpressionNode.parse(self, p) + type = Type.Expression + + else: + # otherwise evaluate command + type = Type.ClientCommand + client_command = command_tokens[0].value + + func tree_string(indent_level): + match type: + Type.Expression: + return tab(indent_level, 'Expression: %s' % expression.tree_string(indent_level+1)) + Type.ClientCommand: + return tab(indent_level, 'Command: %s' % client_command) + return '' + + static func can_parse(parser): + return (parser.next_symbols_are([Constants.TokenType.BeginCommand, Constants.TokenType.Text]) + or parser.next_symbols_are([Constants.TokenType.BeginCommand, Constants.TokenType.Identifier])) + +class ShortcutOptionGroup extends ParseNode: + var options = [] + + func _init(parent, parser).(parent, parser): + + # parse options until there is no more + # expect one otherwise invalid + + var index = 0 + while parser.next_symbol_is([Constants.TokenType.ShortcutOption]): + options.append(ShortCutOption.new(index, self, parser)) + index += 1 + + func tree_string(indent_level): + var info = [] + + info.append(tab(indent_level, 'Shortcut Option Group{')) + + for option in options: + info.append(option.tree_string(indent_level+1)) + + info.append(tab(indent_level, '}')) + + return PoolStringArray(info).join('') + + static func can_parse(parser): + return parser.next_symbol_is([Constants.TokenType.ShortcutOption]) + +class ShortCutOption extends ParseNode: + var label = '' + var condition + var node + + func _init(index, parent, parser).(parent, parser): + parser.expect_symbol([Constants.TokenType.ShortcutOption]) + label = parser.expect_symbol([Constants.TokenType.Text]).value + + # FIXME: Parse the conditional << if $x >> when it exists + var tags = [] + while parser.next_symbols_are([Constants.TokenType.BeginCommand, Constants.TokenType.IfToken]) \ + or parser.next_symbol_is([Constants.TokenType.TagMarker]): + + if parser.next_symbols_are([Constants.TokenType.BeginCommand, Constants.TokenType.IfToken]): + parser.expect_symbol([Constants.TokenType.BeginCommand]) + parser.expect_symbol([Constants.TokenType.IfToken]) + condition = ExpressionNode.parse(self, parser) + parser.expect_symbol([Constants.TokenType.EndCommand]) + + elif parser.next_symbol_is([Constants.TokenType.TagMarker]): + parser.expect_symbol([Constants.TokenType.TagMarker]) + var tag = parser.expect_symbol([Constants.TokenType.Identifier]).value + tags.append(tag) + + + self.tags = tags + # parse remaining statements + + if parser.next_symbol_is([Constants.TokenType.Indent]): + parser.expect_symbol([Constants.TokenType.Indent]) + node = WolNode.new('%s.%s' % [parent.name, index], self, parser) + parser.expect_symbol([Constants.TokenType.Dedent]) + + + func tree_string(indent_level): + var info = [] + + info.append(tab(indent_level, 'Option \'%s\'' % label)) + + if condition != null: + info.append(tab(indent_level + 1, '(when:')) + info.append(condition.tree_string(indent_level + 2)) + info.append(tab(indent_level + 1, '),')) + + if node != null: + info.append(tab(indent_level, '{')) + info.append(node.tree_string(indent_level + 1)) + info.append(tab(indent_level, '}')) + + return PoolStringArray(info).join('') + +#Blocks are groups of statements with the same indent level +class Block extends ParseNode: + + var statements = [] + + func _init(parent, parser).(parent, parser): + #read indent + parser.expect_symbol([Constants.TokenType.Indent]) + + #keep reading statements until we hit a dedent + # FIXME: find exit condition + while not parser.next_symbol_is([Constants.TokenType.Dedent]): + #parse all statements including nested blocks + statements.append(Statement.new(self, parser)) + + #clean up dedent + parser.expect_symbol([Constants.TokenType.Dedent]) + + + func tree_string(indent_level): + var info = [] + + info.append(tab(indent_level, 'Block {')) + + for statement in statements: + info.append(statement.tree_string(indent_level + 1)) + + info.append(tab(indent_level, '}')) + + return PoolStringArray(info).join('') + + static func can_parse(parser): + return parser.next_symbol_is([Constants.TokenType.Indent]) + +# NOTE: Option Statements are links to other nodes +class OptionStatement extends ParseNode: + var destination = '' + var label = '' + + func _init(parent, parser).(parent, parser): + var strings = [] + + # NOTE: parse [[LABEL + parser.expect_symbol([Constants.TokenType.OptionStart]) + strings.append(parser.expect_symbol([Constants.TokenType.Text]).value) + + # NOTE: if there is a | get the next string + if parser.next_symbol_is([Constants.TokenType.OptionDelimit]): + parser.expect_symbol([Constants.TokenType.OptionDelimit]) + var t = parser.expect_symbol([Constants.TokenType.Text, Constants.TokenType.Identifier]) + + strings.append(t.value as String) + + label = strings[0] if strings.size() > 1 else '' + destination = strings[1] if strings.size() > 1 else strings[0] + + parser.expect_symbol([Constants.TokenType.OptionEnd]) + + func tree_string(indent_level): + if label != null: + return tab(indent_level, 'Option: %s -> %s' % [label, destination]) + else: + return tab(indent_level, 'Option: -> %s' % destination) + + static func can_parse(parser): + return parser.next_symbol_is([Constants.TokenType.OptionStart]) + +class IfStatement extends ParseNode: + var clauses = []# + + func _init(parent, parser).(parent, parser): + + #<> + var prime = Clause.new() + + parser.expect_symbol([Constants.TokenType.BeginCommand]) + parser.expect_symbol([Constants.TokenType.IfToken]) + prime.expression = ExpressionNode.parse(self, parser) + parser.expect_symbol([Constants.TokenType.EndCommand]) + + #read statements until 'endif' or 'else' or 'else if' + var statements = []#statement + while not parser.next_symbols_are([Constants.TokenType.BeginCommand, Constants.TokenType.EndIf]) \ + and not parser.next_symbols_are([Constants.TokenType.BeginCommand, Constants.TokenType.ElseToken]) \ + and not parser.next_symbols_are([Constants.TokenType.BeginCommand, Constants.TokenType.ElseIf]): + + statements.append(Statement.new(self, parser)) + + #ignore dedent + while parser.next_symbol_is([Constants.TokenType.Dedent]): + parser.expect_symbol([Constants.TokenType.Dedent]) + + prime.statements = statements + clauses.append(prime) + + #handle all else if + while parser.next_symbols_are([Constants.TokenType.BeginCommand, Constants.TokenType.ElseIf]): + var clause_elif = Clause.new() + + #parse condition syntax + parser.expect_symbol([Constants.TokenType.BeginCommand]) + parser.expect_symbol([Constants.TokenType.ElseIf]) + clause_elif.expression = ExpressionNode.parse(self, parser) + parser.expect_symbol([Constants.TokenType.EndCommand]) + + + var elif_statements = []#statement + while not parser.next_symbols_are([Constants.TokenType.BeginCommand, Constants.TokenType.EndIf]) \ + and not parser.next_symbols_are([Constants.TokenType.BeginCommand, Constants.TokenType.ElseToken]) \ + and not parser.next_symbols_are([Constants.TokenType.BeginCommand, Constants.TokenType.ElseIf]): + + elif_statements.append(Statement.new(self, parser)) + + #ignore dedent + while parser.next_symbol_is([Constants.TokenType.Dedent]): + parser.expect_symbol([Constants.TokenType.Dedent]) + + clause_elif.statements = statements + clauses.append(clause_elif) + + #handle else if exists + if (parser.next_symbols_are([Constants.TokenType.BeginCommand, + Constants.TokenType.ElseToken, Constants.TokenType.EndCommand])): + + #expect no expression - just <> + parser.expect_symbol([Constants.TokenType.BeginCommand]) + parser.expect_symbol([Constants.TokenType.ElseToken]) + parser.expect_symbol([Constants.TokenType.EndCommand]) + + #parse until hit endif + var clause_else = Clause.new() + var el_statements = []#statement + while not parser.next_symbols_are([Constants.TokenType.BeginCommand, Constants.TokenType.EndIf]): + el_statements.append(Statement.new(self, parser)) + + clause_else.statements = el_statements + clauses.append(clause_else) + + #ignore dedent + while parser.next_symbol_is([Constants.TokenType.Dedent]): + parser.expect_symbol([Constants.TokenType.Dedent]) + + #finish + parser.expect_symbol([Constants.TokenType.BeginCommand]) + parser.expect_symbol([Constants.TokenType.EndIf]) + parser.expect_symbol([Constants.TokenType.EndCommand]) + + + func tree_string(indent_level): + var info = [] + var first = true + + for clause in clauses: + if first: + info.append(tab(indent_level, 'if:')) + elif clause.expression!=null: + info.append(tab(indent_level, 'Else If')) + else: + info.append(tab(indent_level, 'Else:')) + + info.append(clause.tree_string(indent_level +1)) + + return info.join('') + + static func can_parse(parser): + return parser.next_symbols_are([Constants.TokenType.BeginCommand, Constants.TokenType.IfToken]) + pass + +class ValueNode extends ParseNode: + var value + + func _init(parent, parser, token = null).(parent, parser): + + var t = token + if t == null : + parser.expect_symbol([Constants.TokenType.Number, + Constants.TokenType.Variable, Constants.TokenType.Str]) + use_token(t) + + #store value depending on type + func use_token(token): + match token.type: + Constants.TokenType.Number: + value = Value.new(float(token.value)) + Constants.TokenType.Str: + value = Value.new(token.value) + Constants.TokenType.FalseToken: + value = Value.new(false) + Constants.TokenType.TrueToken: + value = Value.new(true) + Constants.TokenType.Variable: + value = Value.new(null) + value.type = Constants.ValueType.Variable + value.variable = token.value + Constants.TokenType.NullToken: + value = Value.new(null) + _: + printerr('%s, Invalid token type' % token.name) + + func tree_string(indent_level): + return tab(indent_level, '%s' % value.value()) + +class ExpressionNode extends ParseNode: + var type + var value + var function + var parameters = [] + + func _init(parent, parser, _value, _function = '', _parameters = []).(parent, parser): + if _value != null: + type = Constants.ExpressionType.Value + value = _value + + else: + type = Constants.ExpressionType.FunctionCall + function = _function + parameters = _parameters + + func tree_string(indent_level): + var info = [] + match type: + Constants.ExpressionType.Value: + return value.tree_string(indent_level) + + Constants.ExpressionType.FunctionCall: + info.append(tab(indent_level, 'Func[%s - parameters(%s)]:{'%[function, parameters.size()])) + for param in parameters: + info.append(param.tree_string(indent_level+1)) + info.append(tab(indent_level, '}')) + + return info.join('') + + # Using Djikstra's shunting-yard algorithm to convert stream of expresions into postfix notation, + # & then build a tree of expressions + + # TODO: Rework expression parsing + static func parse(parent, parser): + var rpn = [] + var op_stack = [] + + #track parameters + var func_stack = [] + + var valid_types = [ + Constants.TokenType.Number, + Constants.TokenType.Variable, + Constants.TokenType.Str, + Constants.TokenType.LeftParen, + Constants.TokenType.RightParen, + Constants.TokenType.Identifier, + Constants.TokenType.Comma, + Constants.TokenType.TrueToken, + Constants.TokenType.FalseToken, + Constants.TokenType.NullToken + ] + valid_types += Operator.op_types() + valid_types.invert() + + var last + + print(parser.tokens.slice(0, 6)) + #read expression content + while parser.tokens.size() > 0 and parser.next_symbol_is(valid_types): + print(parser.tokens.front()) + var next = parser.expect_symbol(valid_types) + + if next.type in [ + Constants.TokenType.Variable, + Constants.TokenType.Number, + Constants.TokenType.Str, + Constants.TokenType.FalseToken, + Constants.TokenType.TrueToken, + Constants.TokenType.NullToken + ]: + + # Output primitives + print('adding value "%s" in expression' % next) + print(op_stack.size()) + if func_stack.size() != 0: + op_stack.append(next) + else: + rpn.append(next) + elif next.type == Constants.TokenType.Identifier: + print('adding function') + op_stack.push_back(next) + func_stack.push_back(next) + + #next token is parent - left + next = parser.expect_symbol([Constants.TokenType.LeftParen]) + op_stack.push_back(next) + + elif next.type == Constants.TokenType.Comma: + #resolve sub expression before moving on + while op_stack.back().type != Constants.TokenType.LeftParen: + var p = op_stack.pop_back() + if p == null: + printerr('unbalanced parenthesis %s' % next.name) + break + rpn.append(p) + + + #next token in op_stack left paren + # next parser token not allowed to be right paren or comma + if parser.next_symbol_is([Constants.TokenType.RightParen, + Constants.TokenType.Comma]): + printerr('Expected Expression : %s' % parser.tokens.front().name) + + #find the closest function on stack + #increment parameters + func_stack.back().parameter_count += 1 + + elif Operator.is_op(next.type): + #this is an operator + + #if this is a minus, we need to determine if it is a + #unary minus or a binary minus. + #unary minus looks like this : -1 + #binary minus looks like this 2 - 3 + #thins get complex when we say stuff like: 1 + -1 + #but its easier when we realize that a minus + #is only unary when the last token was a left paren, + #an operator, or its the first token. + + if next.type == Constants.TokenType.Minus: + if last == null \ + or last.type == Constants.TokenType.LeftParen \ + or Operator.is_op(last.type): + #unary minus + next.type = Constants.TokenType.UnaryMinus + + #cannot assign inside expression + # x = a is the same as x == a + if next.type == Constants.TokenType.EqualToOrAssign: + next.type = Constants.TokenType.EqualTo + + #operator precedence + while ExpressionNode.is_apply_precedence(next.type, op_stack, parser): + var op = op_stack.pop_back() + rpn.append(op) + + op_stack.push_back(next) + + elif next.type == Constants.TokenType.LeftParen: + #entered parenthesis sub expression + op_stack.push_back(next) + elif next.type == Constants.TokenType.RightParen: + #leaving sub expression + # resolve order of operations + var parameters = [] + while op_stack.back().type != Constants.TokenType.LeftParen: + parameters.append(op_stack.pop_back()) + + parser.compiler.assert( + op_stack.back() != null, + 'Unbalanced parenthasis #RightParen. Parser.ExpressionNode' + ) + + + rpn.append_array(parameters) + op_stack.pop_back() + # FIXME: Something is going on with parameter counting, fixed for now + # but needs a bigger rework + if op_stack.back().type == Constants.TokenType.Identifier: + #function call + #last token == left paren this == no parameters + #else + #we have more than 1 param + # if last.type != Constants.TokenType.LeftParen: + # func_stack.back().parameter_count += 1 + func_stack.back().parameter_count = parameters.size() + + rpn.append(op_stack.pop_back()) + func_stack.pop_back() + + #record last token used + last = next + + #no more tokens : pop operators to output + while op_stack.size() > 0: + rpn.append(op_stack.pop_back()) + + #if rpn is empty then this is not expression + if rpn.size() == 0: + printerr('Error parsing expression: Expression not found!') + + #build expression tree + var first = rpn.front() + var eval_stack = []#ExpressionNode + + print(rpn) + + while rpn.size() > 0: + var next = rpn.pop_front() + if Operator.is_op(next.type): + #operation + var info = Operator.op_info(next.type) + + if eval_stack.size() < info.arguments: + printerr( + 'Error parsing : Not enough arguments for %s [ got %s expected - was %s]' \ + % [ + Constants.token_type_name(next.type), + eval_stack.size(), + info.arguments + ] + ) + + var function_parameters = [] + for _i in range(info.arguments): + function_parameters.append(eval_stack.pop_back()) + + function_parameters.invert() + + var function_name = get_func_name(next.type) + var expression = ExpressionNode.new(parent, parser, null, function_name, function_parameters) + + eval_stack.append(expression) + + # A function call + elif next.type == Constants.TokenType.Identifier: + var function_name = next.value + prints(function_name, next.parameter_count) + + var function_parameters = [] + for _i in range(next.parameter_count): + function_parameters.append(eval_stack.pop_back()) + + function_parameters.invert() + + var expression = ExpressionNode.new(parent, parser, null, function_name, function_parameters) + + eval_stack.append(expression) + + # A raw value + else: + var raw_value = ValueNode.new(parent, parser, next) + var expression = ExpressionNode.new(parent, parser, raw_value) + eval_stack.append(expression) + + + # NOTE: We should have a single root expression left + # if more then we failed + parser.compiler.assert( + eval_stack.size() == 1, + '[%s] Error parsing expression (stack did not reduce correctly)' % first, + first.line_number, + first.column + ) + + return eval_stack.pop_back() + + static func get_func_name(_type): + var string = '' + + for key in Constants.TokenType.keys(): + if Constants.TokenType[key] == _type: + return key + return string + + static func is_apply_precedence(_type, operator_stack, parser): + if operator_stack.size() == 0: + return false + + if not Operator.is_op(_type): + parser.compiler.assert(false, 'Unable to parse expression!') + return false + + var second = operator_stack.back().type + + if not Operator.is_op(second): + return false + + var first_info = Operator.op_info(_type) + var second_info = Operator.op_info(second) + + return \ + (first_info.associativity == Associativity.Left \ + and first_info.precedence <= second_info.precedence) \ + or \ + (first_info.associativity == Associativity.Right \ + and first_info.precedence < second_info.precedence) + +class Assignment extends ParseNode: + var destination + var value + var operation + + func _init(parent, parser).(parent, parser): + parser.expect_symbol([Constants.TokenType.BeginCommand]) + parser.expect_symbol([Constants.TokenType.Set]) + destination = parser.expect_symbol([Constants.TokenType.Variable]).value + operation = parser.expect_symbol(Assignment.valid_ops()).type + value = ExpressionNode.parse(self, parser) + parser.expect_symbol([Constants.TokenType.EndCommand]) + + func tree_string(indent_level): + var info = [] + info.append(tab(indent_level, 'set:')) + info.append(tab(indent_level + 1, destination)) + info.append(tab(indent_level + 1, Constants.token_type_name(operation))) + info.append(value.tree_string(indent_level + 1)) + return PoolStringArray(info).join('') + + + static func can_parse(parser): + return parser.next_symbols_are([ + Constants.TokenType.BeginCommand, + Constants.TokenType.Set + ]) + + static func valid_ops(): + return [ + Constants.TokenType.EqualToOrAssign, + Constants.TokenType.AddAssign, + Constants.TokenType.MinusAssign, + Constants.TokenType.DivideAssign, + Constants.TokenType.MultiplyAssign + ] + +class Operator extends ParseNode: + var op_type + + func _init(parent, parser, _op_type = null).(parent, parser): + if _op_type == null : + op_type = parser.expect_symbol(Operator.op_types()).type + else: + op_type = _op_type + + func tree_string(indent_level): + var info = [] + info.append(tab(indent_level, op_type)) + return info.join('') + + static func op_info(op): + if not Operator.is_op(op) : + printerr('%s is not a valid operator' % op.name) + + #determine associativity and operands + # each operand has + var TokenType = Constants.TokenType + + match op: + TokenType.Not, TokenType.UnaryMinus: + return OperatorInfo.new(Associativity.Right, 30, 1) + TokenType.Multiply, TokenType.Divide, TokenType.Modulo: + return OperatorInfo.new(Associativity.Left, 20, 2) + TokenType.Add, TokenType.Minus: + return OperatorInfo.new(Associativity.Left, 15, 2) + TokenType.GreaterThan, TokenType.LessThan, TokenType.GreaterThanOrEqualTo, TokenType.LessThanOrEqualTo: + return OperatorInfo.new(Associativity.Left, 10, 2) + TokenType.EqualTo, TokenType.EqualToOrAssign, TokenType.NotEqualTo: + return OperatorInfo.new(Associativity.Left, 5, 2) + TokenType.And: + return OperatorInfo.new(Associativity.Left, 4, 2) + TokenType.Or: + return OperatorInfo.new(Associativity.Left, 3, 2) + TokenType.Xor: + return OperatorInfo.new(Associativity.Left, 2, 2) + _: + printerr('Unknown operator: %s' % op.name) + return null + + static func is_op(type): + return type in op_types() + + static func op_types(): + return [ + Constants.TokenType.Not, + Constants.TokenType.UnaryMinus, + + Constants.TokenType.Add, + Constants.TokenType.Minus, + Constants.TokenType.Divide, + Constants.TokenType.Multiply, + Constants.TokenType.Modulo, + + Constants.TokenType.EqualToOrAssign, + Constants.TokenType.EqualTo, + Constants.TokenType.GreaterThan, + Constants.TokenType.GreaterThanOrEqualTo, + Constants.TokenType.LessThan, + Constants.TokenType.LessThanOrEqualTo, + Constants.TokenType.NotEqualTo, + + Constants.TokenType.And, + Constants.TokenType.Or, + + Constants.TokenType.Xor + ] + +class OperatorInfo: + var associativity + var precedence = -1 + var arguments = -1 + + func _init(_associativity, _precedence, _arguments): + associativity = _associativity + precedence = _precedence + arguments = _arguments + +class Clause: + var expression + var statements = [] #Statement + + func _init(_expression = null, _statements = []): + expression = _expression + statements = _statements + + func tree_string(indent_level): + var info = [] + + if expression != null: + info.append(expression.tree_string(indent_level)) + info.append(tab(indent_level, '{')) + + for statement in statements: + info.append(statement.tree_string(indent_level + 1)) + + info.append(tab(indent_level, '}')) + return PoolStringArray(info).join('') + + func tab(indent_level, input, newline = true): + return '%*s| %s%s' % [indent_level * 2, '', input, '' if !newline else '\n'] diff --git a/dialogue.yarn b/dialogue.wol similarity index 99% rename from dialogue.yarn rename to dialogue.wol index 4faa15e..869e4b6 100644 --- a/dialogue.yarn +++ b/dialogue.wol @@ -1,31 +1,3 @@ -title: Start -tags: -colorID: -position: 0, 0 ---- -<> -<> - -// remove "to" to trigger error -<> -<> - -// Implement inline expressions -<> -Narrator: You, {$direction} way! -<> -Narrator: Do you know you've been here {visit_count()} times? -You: Did you know one + one equals {$one + $one}? -Narrator: You wanna go somewhere? - --> Go to the store - [[TheStore]] --> How much did I visit the store? - Narrator: You've been to the store { visit_count("TheStore") } times. - [[Start]] --> Lets stay here and talk - [[Talk]] -=== title: TheStore tags: colorID: @@ -68,3 +40,31 @@ Narrator: Do you want to continue talking? [[Start]] -> No === +title: Start +tags: +colorID: +position: 0, 0 +--- +<> +<> + +// remove "to" to trigger error +<> +<> + +// Implement inline expressions +<> +Narrator: You, {$direction} way! +<> +Narrator: Do you know you've been here {visit_count()} times? +You: Did you know one + one equals {$one + $one}? +Narrator: You wanna go somewhere? + +-> Go to the store + [[TheStore]] +-> How much did I visit the store? + Narrator: You've been to the store { visit_count("TheStore") } times. + [[Start]] +-> Lets stay here and talk + [[Talk]] +=== \ No newline at end of file diff --git a/project.godot b/project.godot index e6a9e01..e94e828 100644 --- a/project.godot +++ b/project.godot @@ -11,7 +11,7 @@ config_version=4 [application] config/name="Wol" -run/main_scene="res://Dialogue.tscn" +run/main_scene="res://TestScene.tscn" config/icon="res://icon.png" [debug]