Removed more ghost files (on mac os)
This commit is contained in:
parent
369b01cd47
commit
ca63c6810d
|
|
@ -1,461 +0,0 @@
|
|||
extends Object
|
||||
|
||||
signal error(message, line_number, column)
|
||||
|
||||
const Constants = preload('res://addons/Wol/core/Constants.gd')
|
||||
const Lexer = preload('res://addons/Wol/core/compiler/Lexer.gd')
|
||||
const Program = preload('res://addons/Wol/core/Program.gd')
|
||||
const Parser = preload('res://addons/Wol/core/compiler/Parser.gd')
|
||||
|
||||
const INVALID_TITLE = '[\\[<>\\]{}\\|:\\s#\\$]'
|
||||
|
||||
var source = ''
|
||||
var filename = ''
|
||||
|
||||
var current_node
|
||||
var has_implicit_string_tags = false
|
||||
var soft_assert = false
|
||||
|
||||
var string_count = 0
|
||||
var string_table = {}
|
||||
var label_count = 0
|
||||
|
||||
func _init(_filename, _source = null, _soft_assert = false):
|
||||
filename = _filename
|
||||
soft_assert = _soft_assert
|
||||
|
||||
if not _filename and _source:
|
||||
self.source = _source
|
||||
else:
|
||||
var file = File.new()
|
||||
file.open(_filename, File.READ)
|
||||
self.source = file.get_as_text()
|
||||
file.close()
|
||||
|
||||
var source_lines = source.split('\n')
|
||||
for i in range(source_lines.size()):
|
||||
source_lines[i] = source_lines[i].strip_edges(false, true)
|
||||
|
||||
source = source_lines.join('\n')
|
||||
|
||||
func get_headers(offset = 0):
|
||||
var header_property = RegEx.new()
|
||||
var header_sep = RegEx.new()
|
||||
|
||||
header_sep.compile('---(\r\n|\r|\n)')
|
||||
header_property.compile('(?<field>.*): *(?<value>.*)')
|
||||
|
||||
self.assert(header_sep.search(source), 'No headers found!')
|
||||
|
||||
var title = ''
|
||||
var position = Vector2.ZERO
|
||||
|
||||
var source_lines = source.split('\n')
|
||||
var line_number = offset
|
||||
while line_number < source_lines.size():
|
||||
var line = source_lines[line_number]
|
||||
line_number += 1
|
||||
|
||||
if not line.empty():
|
||||
var result = header_property.search(line)
|
||||
|
||||
if result != null:
|
||||
var field = result.get_string('field')
|
||||
var value = result.get_string('value')
|
||||
|
||||
if field == 'title':
|
||||
var regex = RegEx.new()
|
||||
regex.compile(INVALID_TITLE)
|
||||
self.assert(not regex.search(value), 'Invalid characters in title "%s", correct to "%s"' % [value, regex.sub(value, '', true)])
|
||||
|
||||
title = value
|
||||
|
||||
if field == 'position':
|
||||
var regex = RegEx.new()
|
||||
regex.compile('^position:.*,.*\\d$')
|
||||
self.assert(regex.search(line), 'Couldn\'t parse position property in the headers, got "%s" instead in node "%s"' % [value, title])
|
||||
|
||||
position = Vector2(int(value.split(',')[0].strip_edges()), int(value.split(',')[1].strip_edges()))
|
||||
|
||||
# TODO: Implement color and tags
|
||||
|
||||
if line == '---':
|
||||
break
|
||||
|
||||
return {
|
||||
'title': title,
|
||||
'position': position
|
||||
}
|
||||
|
||||
func get_body(offset = 0):
|
||||
var body_lines = []
|
||||
|
||||
var source_lines = source.split('\n')
|
||||
var recording = false
|
||||
var line_number = offset
|
||||
|
||||
while line_number < source_lines.size() and source_lines[line_number] != '===':
|
||||
if recording:
|
||||
body_lines.append(source_lines[line_number])
|
||||
|
||||
recording = recording or source_lines[line_number] == '---'
|
||||
line_number += 1
|
||||
|
||||
line_number += 1
|
||||
|
||||
return PoolStringArray(body_lines).join('\n')
|
||||
|
||||
func get_nodes():
|
||||
var nodes = []
|
||||
var line_number = 0
|
||||
var source_lines = source.split('\n')
|
||||
while line_number < source_lines.size():
|
||||
var headers = get_headers(line_number)
|
||||
var body = get_body(line_number)
|
||||
headers.body = body
|
||||
|
||||
nodes.append(headers)
|
||||
|
||||
# Add +2 to the final line to skip the === from that node
|
||||
line_number = Array(source_lines).find_last(body.split('\n')[-1]) + 2
|
||||
|
||||
while line_number < source_lines.size() and source_lines[line_number].empty():
|
||||
line_number += 1
|
||||
|
||||
return nodes
|
||||
|
||||
func assert(statement, message, line_number = -1, column = -1, _absolute_line_number = -1):
|
||||
if not soft_assert:
|
||||
assert(statement, message + ('; on line %d column %d' % [line_number, column]))
|
||||
elif not statement:
|
||||
emit_signal('error', message, line_number, column)
|
||||
|
||||
func compile():
|
||||
var parsed_nodes = []
|
||||
for node in get_nodes():
|
||||
var lexer = Lexer.new(self, filename, node.title, node.body)
|
||||
var tokens = lexer.tokenize()
|
||||
|
||||
# In case of lexer error
|
||||
if not tokens:
|
||||
return
|
||||
|
||||
var parser = Parser.new(self, node.title, tokens)
|
||||
var parser_node = parser.parse_node()
|
||||
|
||||
parser_node.name = node.title
|
||||
parsed_nodes.append(parser_node)
|
||||
|
||||
var program = Program.new()
|
||||
program.filename = filename
|
||||
|
||||
for node in parsed_nodes:
|
||||
compile_node(program, node)
|
||||
|
||||
for key in string_table:
|
||||
program.strings[key] = string_table[key]
|
||||
|
||||
return program
|
||||
|
||||
func compile_node(program, parsed_node):
|
||||
self.assert(not program.nodes.has(parsed_node.name), 'Duplicate node in program: %s' % parsed_node.name)
|
||||
|
||||
var node_compiled = Program.WolNode.new()
|
||||
|
||||
node_compiled.name = parsed_node.name
|
||||
node_compiled.tags = parsed_node.tags
|
||||
|
||||
if parsed_node.source != null and not parsed_node.source.empty():
|
||||
node_compiled.source_id = register_string(
|
||||
parsed_node.source,
|
||||
parsed_node.name,
|
||||
'line:' + parsed_node.name,
|
||||
0,
|
||||
[]
|
||||
)
|
||||
else:
|
||||
var start_label = register_label()
|
||||
emit(Constants.ByteCode.Label, node_compiled, [Program.Operand.new(start_label)])
|
||||
|
||||
for statement in parsed_node.statements:
|
||||
generate_statement(node_compiled, statement)
|
||||
|
||||
var dangling_options = false
|
||||
for instruction in node_compiled.instructions:
|
||||
if instruction.operation == Constants.ByteCode.AddOption:
|
||||
dangling_options = true
|
||||
if instruction.operation == Constants.ByteCode.ShowOptions:
|
||||
dangling_options = false
|
||||
|
||||
if dangling_options:
|
||||
emit(Constants.ByteCode.ShowOptions, node_compiled)
|
||||
emit(Constants.ByteCode.RunNode, node_compiled)
|
||||
else:
|
||||
emit(Constants.ByteCode.Stop, node_compiled)
|
||||
|
||||
program.nodes[node_compiled.name] = node_compiled
|
||||
|
||||
func register_string(text, node_name, id = '', line_number = -1, tags = []):
|
||||
var line_id_used = ''
|
||||
var implicit = false
|
||||
|
||||
if id.empty():
|
||||
line_id_used = '%s-%s-%d' % [filename, node_name, string_count]
|
||||
string_count += 1
|
||||
|
||||
#use this when we generate implicit tags
|
||||
#they are not saved and are generated
|
||||
#aka dummy tags that change on each compilation
|
||||
has_implicit_string_tags = true
|
||||
|
||||
implicit = true
|
||||
else :
|
||||
line_id_used = id
|
||||
implicit = false
|
||||
|
||||
var string_info = Program.Line.new(text, node_name, line_number, filename, implicit, tags)
|
||||
string_table[line_id_used] = string_info
|
||||
|
||||
return line_id_used
|
||||
|
||||
func register_label(comment = ''):
|
||||
label_count += 1
|
||||
return 'Label%s%s' % [label_count, comment]
|
||||
|
||||
func emit(bytecode, node = current_node, operands = []):
|
||||
var instruction = Program.Instruction.new(null)
|
||||
instruction.operation = bytecode
|
||||
instruction.operands = operands
|
||||
|
||||
if node == null:
|
||||
printerr('Trying to emit to null node with byte_code: %s' % bytecode)
|
||||
return
|
||||
|
||||
node.instructions.append(instruction)
|
||||
|
||||
if bytecode == Constants.ByteCode.Label:
|
||||
node.labels[instruction.operands[0].value] = node.instructions.size() - 1
|
||||
|
||||
func generate_statement(node, statement):
|
||||
match statement.type:
|
||||
Constants.StatementTypes.CustomCommand:
|
||||
generate_custom_command(node, statement.custom_command)
|
||||
|
||||
Constants.StatementTypes.ShortcutOptionGroup:
|
||||
generate_shortcut_group(node, statement.shortcut_option_group)
|
||||
|
||||
Constants.StatementTypes.Block:
|
||||
generate_block(node, statement.block.statements)
|
||||
|
||||
Constants.StatementTypes.IfStatement:
|
||||
generate_if(node, statement.if_statement)
|
||||
|
||||
Constants.StatementTypes.OptionStatement:
|
||||
generate_option(node, statement.option_statement)
|
||||
|
||||
Constants.StatementTypes.AssignmentStatement:
|
||||
generate_assignment(node, statement.assignment)
|
||||
|
||||
Constants.StatementTypes.Line:
|
||||
generate_line(node, statement)
|
||||
_:
|
||||
self.assert(false, statement.line_number, 'Illegal statement type [%s]. Could not generate code.' % statement.type)
|
||||
|
||||
func generate_custom_command(node, command):
|
||||
# TODO: See if the first tree of this statement is being used
|
||||
if command.expression != null:
|
||||
generate_expression(node, command.expression)
|
||||
else:
|
||||
var command_string = command.client_command
|
||||
if command_string == 'stop':
|
||||
emit(Constants.ByteCode.Stop, node)
|
||||
else :
|
||||
emit(Constants.ByteCode.RunCommand, node, [Program.Operand.new(command_string)])
|
||||
|
||||
func generate_line(node, statement):
|
||||
# TODO: Implement proper line numbers (global and local)
|
||||
var line = statement.line
|
||||
var expression_count = line.substitutions.size()
|
||||
|
||||
while not line.substitutions.empty():
|
||||
var inline_expression = line.substitutions.pop_back()
|
||||
generate_expression(node, inline_expression.expression)
|
||||
|
||||
var num = register_string(line.line_text, node.name, line.line_id, statement.line_number, line.tags);
|
||||
emit(Constants.ByteCode.RunLine, node,[Program.Operand.new(num), Program.Operand.new(expression_count)])
|
||||
|
||||
func generate_shortcut_group(node, shortcut_group):
|
||||
var end = register_label('group_end')
|
||||
var labels = []
|
||||
var option_count = 0
|
||||
|
||||
for option in shortcut_group.options:
|
||||
var endof_clause = ''
|
||||
var op_destination = register_label('option_%s' % [option_count + 1])
|
||||
|
||||
labels.append(op_destination)
|
||||
|
||||
if option.condition != null:
|
||||
endof_clause = register_label('conditional_%s' % option_count)
|
||||
generate_expression(node, option.condition)
|
||||
emit(Constants.ByteCode.JumpIfFalse, node, [Program.Operand.new(endof_clause)])
|
||||
|
||||
var label_line_id = '' #TODO: Add tag support
|
||||
var label_string_id = register_string(
|
||||
option.label,
|
||||
node.name,
|
||||
label_line_id,
|
||||
option.line_number,
|
||||
[]
|
||||
)
|
||||
|
||||
emit(Constants.ByteCode.AddOption, node, [Program.Operand.new(label_string_id), Program.Operand.new(op_destination)])
|
||||
|
||||
if option.condition != null:
|
||||
emit(Constants.ByteCode.Label, node, [Program.Operand.new(endof_clause)])
|
||||
emit(Constants.ByteCode.Pop, node)
|
||||
|
||||
option_count += 1
|
||||
|
||||
emit(Constants.ByteCode.ShowOptions, node)
|
||||
emit(Constants.ByteCode.Jump, node)
|
||||
|
||||
option_count = 0
|
||||
|
||||
for option in shortcut_group.options:
|
||||
emit(Constants.ByteCode.Label, node, [Program.Operand.new(labels[option_count])])
|
||||
|
||||
if option.node != null:
|
||||
generate_block(node, option.node.statements)
|
||||
emit(Constants.ByteCode.JumpTo, node, [Program.Operand.new(end)])
|
||||
option_count += 1
|
||||
|
||||
emit(Constants.ByteCode.Label, node, [Program.Operand.new(end)])
|
||||
emit(Constants.ByteCode.Pop, node)
|
||||
|
||||
func generate_block(node, statements = []):
|
||||
if not statements.empty():
|
||||
for statement in statements:
|
||||
generate_statement(node, statement)
|
||||
|
||||
|
||||
func generate_if(node, if_statement):
|
||||
var endif = register_label('endif')
|
||||
|
||||
for clause in if_statement.clauses:
|
||||
var end_clause = register_label('skip_clause')
|
||||
|
||||
if clause.expression != null:
|
||||
generate_expression(node, clause.expression)
|
||||
emit(Constants.ByteCode.JumpIfFalse, node, [Program.Operand.new(end_clause)])
|
||||
|
||||
generate_block(node, clause.statements)
|
||||
emit(Constants.ByteCode.JumpTo, node, [Program.Operand.new(endif)])
|
||||
|
||||
if clause.expression != null:
|
||||
emit(Constants.ByteCode.Label, node, [Program.Operand.new(end_clause)])
|
||||
|
||||
if clause.expression != null:
|
||||
emit(Constants.ByteCode.Pop)
|
||||
|
||||
emit(Constants.ByteCode.Label, node, [Program.Operand.new(endif)])
|
||||
|
||||
func generate_option(node, option):
|
||||
var destination = option.destination
|
||||
|
||||
if option.label == null or option.label.empty():
|
||||
emit(Constants.ByteCode.RunNode, node, [Program.Operand.new(destination)])
|
||||
else :
|
||||
var line_id = '' #TODO: ADD TAG SUPPORT
|
||||
var string_id = register_string(option.label, node.name, line_id, option.line_number, [])
|
||||
|
||||
emit(Constants.ByteCode.AddOption, node, [Program.Operand.new(string_id), Program.Operand.new(destination)])
|
||||
|
||||
func generate_assignment(node, assignment):
|
||||
if assignment.operation == Constants.TokenType.EqualToOrAssign:
|
||||
generate_expression(node, assignment.value)
|
||||
else :
|
||||
emit(Constants.ByteCode.PushVariable, node, [assignment.destination])
|
||||
generate_expression(node, assignment.value)
|
||||
|
||||
match assignment.operation:
|
||||
Constants.TokenType.AddAssign:
|
||||
emit(
|
||||
Constants.ByteCode.CallFunc,
|
||||
node,
|
||||
[Program.Operand.new(Constants.token_type_name(Constants.TokenType.Add))]
|
||||
)
|
||||
Constants.TokenType.MinusAssign:
|
||||
emit(
|
||||
Constants.ByteCode.CallFunc,
|
||||
node,
|
||||
[Program.Operand.new(Constants.token_type_name(Constants.TokenType.Minus))]
|
||||
)
|
||||
Constants.TokenType.MultiplyAssign:
|
||||
emit(
|
||||
Constants.ByteCode.CallFunc,
|
||||
node,
|
||||
[Program.Operand.new(Constants.token_type_name(Constants.TokenType.MultiplyAssign))]
|
||||
)
|
||||
Constants.TokenType.DivideAssign:
|
||||
emit(
|
||||
Constants.ByteCode.CallFunc,
|
||||
node,
|
||||
[Program.Operand.new(Constants.token_type_name(Constants.TokenType.DivideAssign))]
|
||||
)
|
||||
_:
|
||||
printerr('Unable to generate assignment')
|
||||
|
||||
emit(Constants.ByteCode.StoreVariable, node, [Program.Operand.new(assignment.destination)])
|
||||
emit(Constants.ByteCode.Pop, node)
|
||||
|
||||
func generate_expression(node, expression):
|
||||
match expression.type:
|
||||
Constants.ExpressionType.Value:
|
||||
generate_value(node, expression.value)
|
||||
Constants.ExpressionType.FunctionCall:
|
||||
for parameter in expression.parameters:
|
||||
generate_expression(node, parameter)
|
||||
|
||||
emit(Constants.ByteCode.PushNumber, node, [Program.Operand.new(expression.parameters.size())])
|
||||
emit(Constants.ByteCode.CallFunc, node, [Program.Operand.new(expression.function)])
|
||||
_:
|
||||
printerr('No expression.')
|
||||
|
||||
func generate_value(node, value):
|
||||
match value.value.type:
|
||||
Constants.ValueType.Number:
|
||||
emit(
|
||||
Constants.ByteCode.PushNumber,
|
||||
node,
|
||||
[Program.Operand.new(value.value.as_number())]
|
||||
)
|
||||
Constants.ValueType.Str:
|
||||
var id = register_string(
|
||||
value.value.as_string(),
|
||||
node.name,
|
||||
'',
|
||||
value.line_number,
|
||||
[]
|
||||
)
|
||||
emit(
|
||||
Constants.ByteCode.PushString,
|
||||
node,
|
||||
[Program.Operand.new(id)]
|
||||
)
|
||||
Constants.ValueType.Boolean:
|
||||
emit(
|
||||
Constants.ByteCode.PushBool,
|
||||
node,
|
||||
[Program.Operand.new(value.value.as_bool())]
|
||||
)
|
||||
Constants.ValueType.Variable:
|
||||
emit(
|
||||
Constants.ByteCode.PushVariable,
|
||||
node,
|
||||
[Program.Operand.new(value.value.variable)]
|
||||
)
|
||||
Constants.ValueType.Nullean:
|
||||
emit(Constants.ByteCode.PushNull, node)
|
||||
_:
|
||||
printerr('Unrecognized valuenode type: %s' % value.value.type)
|
||||
|
|
@ -1,485 +0,0 @@
|
|||
extends Object
|
||||
|
||||
const Constants = preload('res://addons/Wol/core/Constants.gd')
|
||||
|
||||
const LINE_COMMENT = '//'
|
||||
const FORWARD_SLASH = '/'
|
||||
const LINE_SEPARATOR = '\n'
|
||||
|
||||
const BASE = 'base'
|
||||
const DASH = '-'
|
||||
const COMMAND = 'command'
|
||||
const LINK = 'link'
|
||||
const SHORTCUT = 'shortcut'
|
||||
const TAG = 'tag'
|
||||
const EXPRESSION = 'expression'
|
||||
const ASSIGNMENT = 'assignment'
|
||||
const OPTION = 'option'
|
||||
const OR = 'or'
|
||||
const DESTINATION = 'destination'
|
||||
const INLINE = 'inline'
|
||||
const FORMAT_FUNCTION = 'format'
|
||||
|
||||
var WHITESPACE = '\\s*'
|
||||
|
||||
var compiler
|
||||
var filename = ''
|
||||
var title = ''
|
||||
var text = ''
|
||||
|
||||
var states = {}
|
||||
var default_state
|
||||
var current_state
|
||||
|
||||
var indent_stack = []
|
||||
var should_track_indent = false
|
||||
|
||||
func _init(_compiler, _filename, _title, _text):
|
||||
create_states()
|
||||
|
||||
compiler = _compiler
|
||||
filename = _filename
|
||||
title = _title
|
||||
text = _text
|
||||
|
||||
func create_states():
|
||||
var patterns = {}
|
||||
patterns[Constants.TokenType.Text] = ['.*', 'any text']
|
||||
|
||||
patterns[Constants.TokenType.Number] = ['\\-?[0-9]+(\\.[0-9+])?', 'any number']
|
||||
patterns[Constants.TokenType.Str] = ['\"([^\"\\\\]*(?:\\.[^\"\\\\]*)*)\"', 'any text']
|
||||
patterns[Constants.TokenType.TagMarker] = ['\\#', 'a tag #']
|
||||
patterns[Constants.TokenType.LeftParen] = ['\\(', 'left parenthesis (']
|
||||
patterns[Constants.TokenType.RightParen] = ['\\)', 'right parenthesis )']
|
||||
patterns[Constants.TokenType.EqualTo] = ['(==|is(?!\\w)|eq(?!\\w))', '"=", "is" or "eq"']
|
||||
patterns[Constants.TokenType.EqualToOrAssign] = ['(=|to(?!\\w))', '"=" or "to"']
|
||||
patterns[Constants.TokenType.NotEqualTo] = ['(\\!=|neq(?!\\w))', '"!=" or "neq"']
|
||||
patterns[Constants.TokenType.GreaterThanOrEqualTo] = ['(\\>=|gte(?!\\w))', '">=" or "gte"']
|
||||
patterns[Constants.TokenType.GreaterThan] = ['(\\>|gt(?!\\w))', '">" or "gt"']
|
||||
patterns[Constants.TokenType.LessThanOrEqualTo] = ['(\\<=|lte(?!\\w))', '"<=" or "lte"']
|
||||
patterns[Constants.TokenType.LessThan] = ['(\\<|lt(?!\\w))', '"<" or "lt"']
|
||||
patterns[Constants.TokenType.AddAssign] = ['\\+=', '"+="']
|
||||
patterns[Constants.TokenType.MinusAssign] = ['\\-=', '"-="']
|
||||
patterns[Constants.TokenType.MultiplyAssign] = ['\\*=', '"*="']
|
||||
patterns[Constants.TokenType.DivideAssign] = ['\\/=', '"/="']
|
||||
patterns[Constants.TokenType.Add] = ['\\+', '"+"']
|
||||
patterns[Constants.TokenType.Minus] = ['\\-', '"-"']
|
||||
patterns[Constants.TokenType.Multiply] = ['\\*', '"*"']
|
||||
patterns[Constants.TokenType.Divide] = ['\\/', '"/"']
|
||||
patterns[Constants.TokenType.Modulo] = ['\\%', '"%"']
|
||||
patterns[Constants.TokenType.And] = ['(\\&\\&|and(?!\\w))', '"&&" or "and"']
|
||||
patterns[Constants.TokenType.Or] = ['(\\|\\||or(?!\\w))', '"||" or "or"']
|
||||
patterns[Constants.TokenType.Xor] = ['(\\^|xor(?!\\w))', '"^" or "xor"']
|
||||
patterns[Constants.TokenType.Not] = ['(\\!|not(?!\\w))', '"!" or "not"']
|
||||
patterns[Constants.TokenType.Variable] = ['\\$([A-Za-z0-9_\\.])+', 'any variable']
|
||||
patterns[Constants.TokenType.Comma] = ['\\,', '","']
|
||||
patterns[Constants.TokenType.TrueToken] = ['true(?!\\w)', '"true"']
|
||||
patterns[Constants.TokenType.FalseToken] = ['false(?!\\w)', '"false"']
|
||||
patterns[Constants.TokenType.NullToken] = ['null(?!\\w)', '"null"']
|
||||
patterns[Constants.TokenType.BeginCommand] = ['\\<\\<', 'beginning of a command "<<"']
|
||||
patterns[Constants.TokenType.EndCommand] = ['\\>\\>', 'ending of a command ">>"']
|
||||
patterns[Constants.TokenType.OptionStart] = ['\\[\\[', 'start of an option "[["']
|
||||
patterns[Constants.TokenType.OptionEnd] = ['\\]\\]', 'end of an option "]]"']
|
||||
patterns[Constants.TokenType.OptionDelimit] = ['\\|', 'middle of an option "|"']
|
||||
patterns[Constants.TokenType.Identifier] = ['[a-zA-Z0-9_:\\.]+', 'any reference to another node']
|
||||
patterns[Constants.TokenType.IfToken] = ['if(?!\\w)', '"if"']
|
||||
patterns[Constants.TokenType.ElseToken] = ['else(?!\\w)', '"else"']
|
||||
patterns[Constants.TokenType.ElseIf] = ['elseif(?!\\w)', '"elseif"']
|
||||
patterns[Constants.TokenType.EndIf] = ['endif(?!\\w)', '"endif"']
|
||||
patterns[Constants.TokenType.Set] = ['set(?!\\w)', '"set"']
|
||||
patterns[Constants.TokenType.ShortcutOption] = ['\\-\\>\\s*', '"->"']
|
||||
patterns[Constants.TokenType.ExpressionFunctionStart] = ['\\{', '"{"']
|
||||
patterns[Constants.TokenType.ExpressionFunctionEnd] = ['\\}', '"}"']
|
||||
patterns[Constants.TokenType.FormatFunctionStart] = ['(?<!\\[)\\[(?!\\[)', '"["']
|
||||
patterns[Constants.TokenType.FormatFunctionEnd] = ['\\]', '"]"']
|
||||
|
||||
var shortcut_option = SHORTCUT + DASH + OPTION
|
||||
var shortcut_option_tag = shortcut_option + DASH + TAG
|
||||
var command_or_expression = COMMAND + DASH + OR + DASH + EXPRESSION
|
||||
var link_destination = LINK + DASH + DESTINATION
|
||||
var format_expression = FORMAT_FUNCTION + DASH + EXPRESSION
|
||||
var inline_expression = INLINE + DASH + EXPRESSION
|
||||
var link_inline_expression = LINK + DASH + INLINE + DASH + EXPRESSION
|
||||
var link_format_expression = LINK + DASH + FORMAT_FUNCTION + DASH + EXPRESSION
|
||||
|
||||
states = {}
|
||||
|
||||
states[BASE] = LexerState.new(patterns)
|
||||
states[BASE].add_transition(Constants.TokenType.BeginCommand, COMMAND, true)
|
||||
states[BASE].add_transition(Constants.TokenType.ExpressionFunctionStart, inline_expression, true)
|
||||
states[BASE].add_transition(Constants.TokenType.FormatFunctionStart, FORMAT_FUNCTION, true)
|
||||
states[BASE].add_transition(Constants.TokenType.OptionStart, LINK, true)
|
||||
states[BASE].add_transition(Constants.TokenType.ShortcutOption, shortcut_option)
|
||||
states[BASE].add_transition(Constants.TokenType.TagMarker, TAG, true)
|
||||
states[BASE].add_text_rule(Constants.TokenType.Text)
|
||||
|
||||
#FIXME - Tags are not being proccessed properly this way. We must look for the format #{identifier}:{value}
|
||||
# Possible solution is to add more transitions
|
||||
states[TAG] = LexerState.new(patterns)
|
||||
states[TAG].add_transition(Constants.TokenType.Identifier, BASE)
|
||||
|
||||
states[shortcut_option] = LexerState.new(patterns)
|
||||
states[shortcut_option].track_indent = true
|
||||
states[shortcut_option].add_transition(Constants.TokenType.BeginCommand, EXPRESSION, true)
|
||||
states[shortcut_option].add_transition(Constants.TokenType.ExpressionFunctionStart, inline_expression, true)
|
||||
states[shortcut_option].add_transition(Constants.TokenType.TagMarker, shortcut_option_tag, true)
|
||||
states[shortcut_option].add_text_rule(Constants.TokenType.Text, BASE)
|
||||
|
||||
states[shortcut_option_tag] = LexerState.new(patterns)
|
||||
states[shortcut_option_tag].add_transition(Constants.TokenType.Identifier, shortcut_option)
|
||||
|
||||
states[COMMAND] = LexerState.new(patterns)
|
||||
states[COMMAND].add_transition(Constants.TokenType.IfToken, EXPRESSION)
|
||||
states[COMMAND].add_transition(Constants.TokenType.ElseToken)
|
||||
states[COMMAND].add_transition(Constants.TokenType.ElseIf, EXPRESSION)
|
||||
states[COMMAND].add_transition(Constants.TokenType.EndIf)
|
||||
states[COMMAND].add_transition(Constants.TokenType.Set, ASSIGNMENT)
|
||||
states[COMMAND].add_transition(Constants.TokenType.EndCommand, BASE, true)
|
||||
states[COMMAND].add_transition(Constants.TokenType.Identifier, command_or_expression)
|
||||
states[COMMAND].add_text_rule(Constants.TokenType.Text)
|
||||
|
||||
states[command_or_expression] = LexerState.new(patterns)
|
||||
states[command_or_expression].add_transition(Constants.TokenType.LeftParen, EXPRESSION)
|
||||
states[command_or_expression].add_transition(Constants.TokenType.EndCommand, BASE, true)
|
||||
states[command_or_expression].add_text_rule(Constants.TokenType.Text)
|
||||
|
||||
states[ASSIGNMENT] = LexerState.new(patterns)
|
||||
states[ASSIGNMENT].add_transition(Constants.TokenType.Variable)
|
||||
states[ASSIGNMENT].add_transition(Constants.TokenType.EqualToOrAssign, EXPRESSION)
|
||||
states[ASSIGNMENT].add_transition(Constants.TokenType.AddAssign, EXPRESSION)
|
||||
states[ASSIGNMENT].add_transition(Constants.TokenType.MinusAssign, EXPRESSION)
|
||||
states[ASSIGNMENT].add_transition(Constants.TokenType.MultiplyAssign, EXPRESSION)
|
||||
states[ASSIGNMENT].add_transition(Constants.TokenType.DivideAssign, EXPRESSION)
|
||||
|
||||
states[FORMAT_FUNCTION] = LexerState.new(patterns)
|
||||
states[FORMAT_FUNCTION].add_transition(Constants.TokenType.FormatFunctionEnd, BASE, true)
|
||||
states[FORMAT_FUNCTION].add_transition(Constants.TokenType.ExpressionFunctionStart, format_expression, true)
|
||||
states[FORMAT_FUNCTION].add_text_rule(Constants.TokenType.Text)
|
||||
|
||||
|
||||
states[format_expression] = LexerState.new(patterns)
|
||||
states[format_expression].add_transition(Constants.TokenType.ExpressionFunctionEnd, FORMAT_FUNCTION)
|
||||
form_expression_state(states[format_expression])
|
||||
|
||||
states[inline_expression] = LexerState.new(patterns)
|
||||
states[inline_expression].add_transition(Constants.TokenType.ExpressionFunctionEnd, BASE)
|
||||
form_expression_state(states[inline_expression])
|
||||
|
||||
states[EXPRESSION] = LexerState.new(patterns)
|
||||
states[EXPRESSION].add_transition(Constants.TokenType.EndCommand, BASE)
|
||||
# states[EXPRESSION].add_transition(Constants.TokenType.FormatFunctionEnd, BASE)
|
||||
form_expression_state(states[EXPRESSION])
|
||||
|
||||
states[LINK] = LexerState.new(patterns)
|
||||
states[LINK].add_transition(Constants.TokenType.OptionEnd, BASE, true)
|
||||
states[LINK].add_transition(Constants.TokenType.ExpressionFunctionStart, link_inline_expression, true)
|
||||
states[LINK].add_transition(Constants.TokenType.FormatFunctionStart, link_format_expression, true)
|
||||
states[LINK].add_transition(Constants.TokenType.FormatFunctionEnd, LINK, true)
|
||||
states[LINK].add_transition(Constants.TokenType.OptionDelimit, link_destination, true)
|
||||
states[LINK].add_text_rule(Constants.TokenType.Text)
|
||||
|
||||
states[link_format_expression] = LexerState.new(patterns)
|
||||
states[link_format_expression].add_transition(Constants.TokenType.FormatFunctionEnd, LINK, true)
|
||||
states[link_format_expression].add_transition(Constants.TokenType.ExpressionFunctionStart, link_inline_expression, true)
|
||||
states[link_format_expression].add_text_rule(Constants.TokenType.Text)
|
||||
|
||||
states[link_inline_expression] = LexerState.new(patterns)
|
||||
states[link_inline_expression].add_transition(Constants.TokenType.ExpressionFunctionEnd, LINK)
|
||||
form_expression_state(states[link_inline_expression])
|
||||
|
||||
states[link_destination] = LexerState.new(patterns)
|
||||
states[link_destination].add_transition(Constants.TokenType.Identifier)
|
||||
states[link_destination].add_transition(Constants.TokenType.OptionEnd, BASE)
|
||||
|
||||
default_state = states[BASE]
|
||||
|
||||
for key in states.keys():
|
||||
states[key].name = key
|
||||
|
||||
func form_expression_state(expression_state):
|
||||
expression_state.add_transition(Constants.TokenType.Number)
|
||||
expression_state.add_transition(Constants.TokenType.Str)
|
||||
expression_state.add_transition(Constants.TokenType.LeftParen)
|
||||
expression_state.add_transition(Constants.TokenType.RightParen)
|
||||
expression_state.add_transition(Constants.TokenType.EqualTo)
|
||||
expression_state.add_transition(Constants.TokenType.EqualToOrAssign)
|
||||
expression_state.add_transition(Constants.TokenType.NotEqualTo)
|
||||
expression_state.add_transition(Constants.TokenType.GreaterThanOrEqualTo)
|
||||
expression_state.add_transition(Constants.TokenType.GreaterThan)
|
||||
expression_state.add_transition(Constants.TokenType.LessThanOrEqualTo)
|
||||
expression_state.add_transition(Constants.TokenType.LessThan)
|
||||
expression_state.add_transition(Constants.TokenType.Add)
|
||||
expression_state.add_transition(Constants.TokenType.Minus)
|
||||
expression_state.add_transition(Constants.TokenType.Multiply)
|
||||
expression_state.add_transition(Constants.TokenType.Divide)
|
||||
expression_state.add_transition(Constants.TokenType.Modulo)
|
||||
expression_state.add_transition(Constants.TokenType.And)
|
||||
expression_state.add_transition(Constants.TokenType.Or)
|
||||
expression_state.add_transition(Constants.TokenType.Xor)
|
||||
expression_state.add_transition(Constants.TokenType.Not)
|
||||
expression_state.add_transition(Constants.TokenType.Variable)
|
||||
expression_state.add_transition(Constants.TokenType.Comma)
|
||||
expression_state.add_transition(Constants.TokenType.TrueToken)
|
||||
expression_state.add_transition(Constants.TokenType.FalseToken)
|
||||
expression_state.add_transition(Constants.TokenType.NullToken)
|
||||
expression_state.add_transition(Constants.TokenType.Identifier)
|
||||
|
||||
func tokenize():
|
||||
var tokens = []
|
||||
|
||||
indent_stack.clear()
|
||||
indent_stack.push_front([0, false])
|
||||
should_track_indent = false
|
||||
current_state = default_state
|
||||
|
||||
var lines = text.split(LINE_SEPARATOR)
|
||||
var line_number = 1
|
||||
|
||||
lines.append('')
|
||||
|
||||
for line in lines:
|
||||
var line_tokens = tokenize_line(line, line_number)
|
||||
if line_tokens == null:
|
||||
return
|
||||
|
||||
tokens.append_array(line_tokens)
|
||||
line_number += 1
|
||||
|
||||
var end_of_input = Token.new(
|
||||
Constants.TokenType.EndOfInput,
|
||||
current_state,
|
||||
line_number,
|
||||
0
|
||||
)
|
||||
tokens.append(end_of_input)
|
||||
|
||||
return tokens
|
||||
|
||||
func tokenize_line(line, line_number):
|
||||
var token_stack = []
|
||||
|
||||
var fresh_line = line.replace('\t', ' ').replace('\r', '')
|
||||
|
||||
var indentation = line_indentation(line)
|
||||
var previous_indentation = indent_stack.front()[0]
|
||||
|
||||
if should_track_indent and indentation > previous_indentation:
|
||||
indent_stack.push_front([indentation, true])
|
||||
|
||||
var indent = Token.new(
|
||||
Constants.TokenType.Indent,
|
||||
current_state,
|
||||
filename,
|
||||
line_number,
|
||||
previous_indentation
|
||||
)
|
||||
indent.value = '%*s' % [indentation - previous_indentation, '']
|
||||
|
||||
should_track_indent = false
|
||||
token_stack.push_front(indent)
|
||||
|
||||
elif indentation < previous_indentation:
|
||||
while indentation < indent_stack.front()[0]:
|
||||
var top = indent_stack.pop_front()[1]
|
||||
if top:
|
||||
var deindent = Token.new(Constants.TokenType.Dedent, current_state, line_number, 0)
|
||||
token_stack.push_front(deindent)
|
||||
|
||||
var column = indentation
|
||||
var whitespace = RegEx.new()
|
||||
whitespace.compile(WHITESPACE)
|
||||
|
||||
while column < fresh_line.length():
|
||||
if fresh_line.substr(column).begins_with(LINE_COMMENT):
|
||||
break
|
||||
|
||||
var matched = false
|
||||
|
||||
for rule in current_state.rules:
|
||||
var found = rule.regex.search(fresh_line, column)
|
||||
|
||||
if !found:
|
||||
continue
|
||||
|
||||
var token_text = ''
|
||||
|
||||
# NOTE: If this is text then we back up to the most recent delimiting token
|
||||
# and treat everything from there as text.
|
||||
if rule.token_type == Constants.TokenType.Text:
|
||||
|
||||
var start_index = indentation
|
||||
|
||||
if token_stack.size() > 0 :
|
||||
while token_stack.front().type == Constants.TokenType.Identifier:
|
||||
token_stack.pop_front()
|
||||
|
||||
var start_delimit_token = token_stack.front()
|
||||
start_index = start_delimit_token.column
|
||||
|
||||
if start_delimit_token.type == Constants.TokenType.Indent:
|
||||
start_index += start_delimit_token.value.length()
|
||||
if start_delimit_token.type == Constants.TokenType.Dedent:
|
||||
start_index = indentation
|
||||
|
||||
column = start_index
|
||||
var end_index = found.get_start() + found.get_string().length()
|
||||
|
||||
token_text = fresh_line.substr(start_index, end_index - start_index)
|
||||
else:
|
||||
token_text = found.get_string()
|
||||
|
||||
column += token_text.length()
|
||||
|
||||
if rule.token_type == Constants.TokenType.Str:
|
||||
token_text = token_text.substr(1, token_text.length() - 2)
|
||||
token_text = token_text.replace('\\\\', '\\')
|
||||
token_text = token_text.replace('\\\'','\'')
|
||||
|
||||
var token = Token.new(
|
||||
rule.token_type,
|
||||
current_state,
|
||||
filename,
|
||||
line_number,
|
||||
column,
|
||||
token_text
|
||||
)
|
||||
token.delimits_text = rule.delimits_text
|
||||
|
||||
token_stack.push_front(token)
|
||||
|
||||
if rule.enter_state != null and rule.enter_state.length() > 0:
|
||||
if not states.has(rule.enter_state):
|
||||
printerr('State[%s] not known - line(%s) col(%s)' % [rule.enter_state, line_number, column])
|
||||
return []
|
||||
|
||||
enter_state(states[rule.enter_state])
|
||||
|
||||
if should_track_indent:
|
||||
if indent_stack.front()[0] < indentation:
|
||||
indent_stack.append([indentation, false])
|
||||
|
||||
matched = true
|
||||
break
|
||||
|
||||
if not matched:
|
||||
var rules = []
|
||||
for rule in current_state.rules:
|
||||
rules.append('"%s" (%s)' % [Constants.token_type_name(rule.token_type), rule.human_readable_identifier])
|
||||
|
||||
var error_data = [
|
||||
PoolStringArray(rules).join(', ') if rules.size() == 1 else PoolStringArray(rules.slice(0, rules.size() - 2)).join(', ') + ' or %s' % rules[-1],
|
||||
filename,
|
||||
title,
|
||||
line_number,
|
||||
column
|
||||
]
|
||||
compiler.assert(false, 'Expected %s in file %s in node "%s" on line #%d (column #%d)' % error_data, line_number, column)
|
||||
return
|
||||
|
||||
var last_whitespace = whitespace.search(line, column)
|
||||
if last_whitespace:
|
||||
column += last_whitespace.get_string().length()
|
||||
|
||||
token_stack.invert()
|
||||
|
||||
return token_stack
|
||||
|
||||
func line_indentation(line):
|
||||
var indent_regex = RegEx.new()
|
||||
indent_regex.compile('^(\\s*)')
|
||||
|
||||
var found = indent_regex.search(line)
|
||||
|
||||
if !found or found.get_string().length() <= 0:
|
||||
return 0
|
||||
|
||||
return found.get_string().length()
|
||||
|
||||
func enter_state(state):
|
||||
current_state = state;
|
||||
if current_state.track_indent:
|
||||
should_track_indent = true
|
||||
|
||||
class Token:
|
||||
var type = -1
|
||||
var value = ''
|
||||
|
||||
var filename = ''
|
||||
var line_number = -1
|
||||
var column = -1
|
||||
var text = ''
|
||||
|
||||
var delimits_text = false
|
||||
var parameter_count = -1
|
||||
var lexer_state = ''
|
||||
|
||||
func _init(_type, _state, _filename, _line_number = -1, _column = -1, _value = ''):
|
||||
type = _type
|
||||
lexer_state = _state.name
|
||||
filename = _filename
|
||||
line_number = _line_number
|
||||
column = _column
|
||||
value = _value
|
||||
|
||||
func _to_string():
|
||||
return '%s (%s) at %s:%s (state: %s)' % [Constants.token_type_name(type), value, line_number, column, lexer_state]
|
||||
|
||||
class LexerState:
|
||||
var name = ''
|
||||
var patterns = {}
|
||||
var rules = []
|
||||
var track_indent = false
|
||||
|
||||
func _init(_patterns):
|
||||
patterns = _patterns
|
||||
|
||||
func add_transition(type, state = '', delimit_text = false):
|
||||
var pattern = '\\G%s' % patterns[type][0]
|
||||
var rule = Rule.new(type, pattern, patterns[type][1], state, delimit_text)
|
||||
rules.append(rule)
|
||||
return rule
|
||||
|
||||
func add_text_rule(type, state = ''):
|
||||
if contains_text_rule() :
|
||||
printerr('State already contains Text rule')
|
||||
return null
|
||||
|
||||
var delimiters:Array = []
|
||||
for rule in rules:
|
||||
if rule.delimits_text:
|
||||
delimiters.append('%s' % rule.regex.get_pattern().substr(2))
|
||||
|
||||
var pattern = '\\G((?!%s).)*' % [PoolStringArray(delimiters).join('|')]
|
||||
var rule = add_transition(type, state)
|
||||
rule.regex = RegEx.new()
|
||||
rule.regex.compile(pattern)
|
||||
rule.is_text_rule = true
|
||||
return rule
|
||||
|
||||
func contains_text_rule():
|
||||
for rule in rules:
|
||||
if rule.is_text_rule:
|
||||
return true
|
||||
return false
|
||||
|
||||
class Rule:
|
||||
var regex
|
||||
|
||||
var enter_state = ''
|
||||
var token_type = -1
|
||||
var is_text_rule = false
|
||||
var delimits_text = false
|
||||
var human_readable_identifier = ''
|
||||
|
||||
func _init(_type, _regex, _human_readable_identifier, _enter_state, _delimits_text):
|
||||
token_type = _type
|
||||
|
||||
regex = RegEx.new()
|
||||
regex.compile(_regex)
|
||||
|
||||
human_readable_identifier = _human_readable_identifier
|
||||
enter_state = _enter_state
|
||||
delimits_text = _delimits_text
|
||||
|
||||
func _to_string():
|
||||
return '[Rule : %s (%s) - %s]' % [Constants.token_type_name(token_type), human_readable_identifier, regex]
|
||||
File diff suppressed because it is too large
Load diff
Reference in a new issue