capitalised some files

This commit is contained in:
Bram Dingelstad 2021-12-06 17:42:27 +01:00
parent 0be6309941
commit 4251e63cc9
10 changed files with 2563 additions and 30 deletions

View file

@ -14,7 +14,7 @@ __meta__ = {
[node name="Wol" type="Node" parent="Dialogue"] [node name="Wol" type="Node" parent="Dialogue"]
script = ExtResource( 1 ) script = ExtResource( 1 )
path = "res://dialogue.yarn" path = "res://dialogue.wol"
auto_start = true auto_start = true
variable_storage = { variable_storage = {
} }

View file

@ -0,0 +1,214 @@
extends Object
enum ExecutionState {
Stopped,
Running,
WaitingForOption,
Suspended
}
enum HandlerState {
PauseExecution,
ContinueExecution
}
#Compile Status return
enum CompileStatus {
Succeeded, SucceededUntaggedStrings,
}
enum ByteCode {
# opA = string: label name
Label,
# opA = string: label name
JumpTo,
# peek string from stack and jump to that label
Jump,
# opA = int: string number
RunLine,
# opA = string: command text
RunCommand,
# opA = int: string number for option to add
AddOption,
# present the current list of options, then clear the list; most recently
# selected option will be on the top of the stack
ShowOptions,
# opA = int: string number in table; push string to stack
PushString,
# opA = float: number to push to stack
PushNumber,
# opA = int (0 or 1): bool to push to stack
PushBool,
# pushes a null value onto the stack
PushNull,
# opA = string: label name if top of stack is not null, zero or false, jumps
# to that label
JumpIfFalse,
# discard top of stack
Pop,
# opA = string; looks up function, pops as many arguments as needed, result is
# pushed to stack
CallFunc,
# opA = name of variable to get value of and push to stack
PushVariable,
# opA = name of variable to store top of stack in
StoreVariable,
# stops execution
Stop,
# run the node whose name is at the top of the stack
RunNode
}
enum TokenType {
#0 Special tokens
Whitespace, Indent, Dedent, EndOfLine, EndOfInput,
#5 Numbers. Everybody loves a number
Number,
#6 Strings. Everybody also loves a string
Str,
#7 '#'
TagMarker,
#8 Command syntax ('<<foo>>')
BeginCommand, EndCommand,
ExpressionFunctionStart, # {
ExpressionFunctionEnd, # }
FormatFunctionStart, # [
FormatFunctionEnd, # ]
#10 Variables ('$foo')
Variable,
#11 Shortcut syntax ('->')
ShortcutOption,
#12 Option syntax ('[[Let's go here|Destination]]')
OptionStart, # [[
OptionDelimit, # |
OptionEnd, # ]]
#15 Command types (specially recognised command word)
IfToken, ElseIf, ElseToken, EndIf, Set,
#20 Boolean values
TrueToken, FalseToken,
#22 The null value
NullToken,
#23 Parentheses
LeftParen, RightParen,
#25 Parameter delimiters
Comma,
#26 Operators
EqualTo, # ==, eq, is
GreaterThan, # >, gt
GreaterThanOrEqualTo, # >=, gte
LessThan, # <, lt
LessThanOrEqualTo, # <=, lte
NotEqualTo, # !=, neq
#32 Logical operators
Or, # ||, or
And, # &&, and
Xor, # ^, xor
Not, # !, not
# this guy's special because '=' can mean either 'equal to'
#36 or 'becomes' depending on context
EqualToOrAssign, # =, to
#37
UnaryMinus, # -; this is differentiated from Minus
# when parsing expressions
#38
Add, # +
Minus, # -
Multiply, # *
Divide, # /
Modulo, # %
#43
AddAssign, # +=
MinusAssign, # -=
MultiplyAssign, # *=
DivideAssign, # /=
Comment, # a run of text that we ignore
Identifier, # a single word (used for functions)
Text # a run of text until we hit other syntax
}
enum ExpressionType {
Value,
FunctionCall
}
enum StatementTypes {
CustomCommand,
ShortcutOptionGroup,
Block,
IfStatement,
OptionStatement,
AssignmentStatement,
Line,
InlineExpression
}
enum ValueType {
Number,
Str,
Boolean,
Variable,
Nullean
}
static func token_type_name(value):
for key in TokenType.keys():
if TokenType[key] == value:
return key
return 'NOTVALID'
static func merge_dir(target, patch):
for key in patch:
target[key] = patch[key]
static func bytecode_name(bytecode):
return [
'Label',
'JumpTo',
'Jump',
'RunLine',
'RunCommand',
'AddOption',
'ShowOptions',
'PushString',
'PushNumber',
'PushBool',
'PushNull',
'JumpIfFalse',
'Pop',
'CallFunc',
'PushVariable',
'StoreVariable',
'Stop',
'RunNode'
][bytecode]
static func token_name(type):
for key in TokenType.keys():
if TokenType[key] == type:
return key
return ''

View file

@ -0,0 +1,25 @@
extends Object
const FunctionInfo = preload('res://addons/Wol/core/FunctionInfo.gd')
const Constants = preload('res://addons/Wol/core/Constants.gd')
var functions = {}
var virtual_machine
func get_function(name):
if functions.has(name):
return functions[name]
else :
printerr('Invalid Function: %s'% name)
return
func import_library(other):
Constants.merge_dir(functions, other.functions)
other.virtual_machine = virtual_machine
func register_function(name, parameter_count, function, returns_value):
var functionInfo = FunctionInfo.new(name, parameter_count, function, returns_value)
functions[name] = functionInfo
func deregister_function(name):
functions.erase(name)

146
addons/Wol/core/Program.gd Normal file
View file

@ -0,0 +1,146 @@
extends Object
const Constants = preload('res://addons/Wol/core/Constants.gd')
var name = ''
var filename = ''
var strings = {}
var nodes = {}
class Line:
var text = ''
var node_name = ''
var line_number = -1
var file_name = ''
var implicit = false
var substitutions = []
var meta = []
func _init(_text, _node_name, _line_number, _file_name, _implicit, _meta):
text = _text
node_name = _node_name
file_name = _file_name
implicit = _implicit
meta = _meta
func clone():
return get_script().new(text, node_name, line_number, file_name, implicit, meta)
func _to_string():
return '%s:%d: "%s"' % [file_name.get_file(), line_number, text]
class Option:
var line
var id = -1
var destination = ''
func _init(_line, _id, _destination):
line = _line
id = _id
destination = _destination
func clone():
return get_script().new(self)
class Command:
var command = ''
func _init(_command):
command = _command
class WolNode:
var name = ''
var instructions = []
var labels = {}
var tags = []
var source_id = ''
func _init(other = null):
if other != null and other.get_script() == self.get_script():
name = other.name
instructions += other.instructions
for key in other.labels.keys():
labels[key] = other.labels[key]
tags += other.tags
source_id = other.source_id
func equals(other):
if other.get_script() != get_script():
return false
if other.name != name:
return false
if other.instructions != instructions:
return false
if other.labels != labels:
return false
if other.source_id != source_id:
return false
return true
func _to_string():
return "WolNode[%s:%s]" % [name, source_id]
# TODO: Make this make sense
class Operand:
enum ValueType {
None,
StringValue,
BooleanValue,
FloatValue
}
var value
var type
func _init(_value):
if typeof(_value) == TYPE_OBJECT and _value.get_script() == get_script():
set_value(_value.value)
else:
set_value(_value)
func set_value(_value):
match typeof(_value):
TYPE_REAL,TYPE_INT:
set_number(_value)
TYPE_BOOL:
set_boolean(_value)
TYPE_STRING:
set_string(_value)
func set_boolean(_value):
value = _value
type = ValueType.BooleanValue
return self
func set_string(_value):
value = _value
type = ValueType.StringValue
return self
func set_number(_value):
value = _value
type = ValueType.FloatValue
return self
func clear_value():
type = ValueType.None
value = null
func clone():
return get_script().new(self)
func _to_string():
return "Operand[%s:%s]" % [type, value]
class Instruction:
var operation = -1
var operands = []
func _init(other = null):
if other != null and other.get_script() == self.get_script():
self.operation = other.operation
self.operands += other.operands
func _to_string():
return Constants.bytecode_name(operation) + ':' + operands as String

137
addons/Wol/core/Value.gd Normal file
View file

@ -0,0 +1,137 @@
extends Object
const Constants = preload('res://addons/Wol/core/Constants.gd')
const NANI = 'NaN'
var type = Constants.ValueType.Nullean
var number = 0
var string = ''
var variable = ''
var boolean = false
func _init(value = NANI):
if typeof(value) == TYPE_OBJECT and value.get_script() == get_script():
if value.type == Constants.ValueType.Variable:
type = value.type
variable = value.variable
else:
set_value(value)
func value():
match type:
Constants.ValueType.Number:
return number
Constants.ValueType.Str:
return string
Constants.ValueType.Boolean:
return boolean
Constants.ValueType.Variable:
return variable
return null
func as_bool():
match type:
Constants.ValueType.Number:
return number != 0
Constants.ValueType.Str:
return !string.empty()
Constants.ValueType.Boolean:
return boolean
return false
func as_string():
return '%s' % value()
func as_number():
match type:
Constants.ValueType.Number:
return number
Constants.ValueType.Str:
return float(string)
Constants.ValueType.Boolean:
return 0.0 if !boolean else 1.0
return .0
func set_value(value):
if value == null or (typeof(value) == TYPE_STRING and value == NANI):
type = Constants.ValueType.Nullean
return
match typeof(value):
TYPE_INT, TYPE_REAL:
type = Constants.ValueType.Number
number = value
TYPE_STRING:
type = Constants.ValueType.Str
string = value
TYPE_BOOL:
type = Constants.ValueType.Boolean
boolean = value
func add(other):
if type == Constants.ValueType.Str or other.type == Constants.ValueType.Str:
return get_script().new('%s%s' % [value(), other.value()])
if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number:
return get_script().new(number + other.number)
return null
func equals(other):
if other.get_script() != get_script():
return false
if other.value() != value():
return false
# TODO: Add more equality cases
return true
func sub(other):
if type == Constants.ValueType.Str or other.type == Constants.ValueType.Str:
return get_script().new(str(value()).replace(str(other.value()),''))
if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number:
return get_script().new(number - other.number)
return null
func mult(other):
if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number:
return get_script().new(number * other.number)
return null
func div(other):
if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number:
return get_script().new(number / other.number)
return null
func mod(other):
if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number:
return get_script().new(number % other.number)
return null
func negative():
if type == Constants.ValueType.Number:
return get_script().new(-number)
return null
func greater(other):
if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number:
return number > other.number
return false
func less(other):
if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number:
return number < other.number
return false
func geq(other):
if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number:
return number > other.number or equals(other)
return false
func leq(other):
if type == Constants.ValueType.Number and other.type == Constants.ValueType.Number:
return number < other.number or equals(other)
return false
func _to_string():
return 'value(type[%s]: %s)' % [type,value()]

View file

@ -0,0 +1,461 @@
extends Object
signal error(message, line_number, column)
const Constants = preload('res://addons/Wol/core/Constants.gd')
const Lexer = preload('res://addons/Wol/core/compiler/Lexer.gd')
const Program = preload('res://addons/Wol/core/Program.gd')
const Parser = preload('res://addons/Wol/core/compiler/Parser.gd')
const INVALID_TITLE = '[\\[<>\\]{}\\|:\\s#\\$]'
var source = ''
var filename = ''
var current_node
var has_implicit_string_tags = false
var soft_assert = false
var string_count = 0
var string_table = {}
var label_count = 0
func _init(_filename, _source = null, _soft_assert = false):
filename = _filename
soft_assert = _soft_assert
if not _filename and _source:
self.source = _source
else:
var file = File.new()
file.open(_filename, File.READ)
self.source = file.get_as_text()
file.close()
var source_lines = source.split('\n')
for i in range(source_lines.size()):
source_lines[i] = source_lines[i].strip_edges(false, true)
source = source_lines.join('\n')
func get_headers(offset = 0):
var header_property = RegEx.new()
var header_sep = RegEx.new()
header_sep.compile('---(\r\n|\r|\n)')
header_property.compile('(?<field>.*): *(?<value>.*)')
self.assert(header_sep.search(source), 'No headers found!')
var title = ''
var position = Vector2.ZERO
var source_lines = source.split('\n')
var line_number = offset
while line_number < source_lines.size():
var line = source_lines[line_number]
line_number += 1
if not line.empty():
var result = header_property.search(line)
if result != null:
var field = result.get_string('field')
var value = result.get_string('value')
if field == 'title':
var regex = RegEx.new()
regex.compile(INVALID_TITLE)
self.assert(not regex.search(value), 'Invalid characters in title "%s", correct to "%s"' % [value, regex.sub(value, '', true)])
title = value
if field == 'position':
var regex = RegEx.new()
regex.compile('^position:.*,.*\\d$')
self.assert(regex.search(line), 'Couldn\'t parse position property in the headers, got "%s" instead in node "%s"' % [value, title])
position = Vector2(int(value.split(',')[0].strip_edges()), int(value.split(',')[1].strip_edges()))
# TODO: Implement color and tags
if line == '---':
break
return {
'title': title,
'position': position
}
func get_body(offset = 0):
var body_lines = []
var source_lines = source.split('\n')
var recording = false
var line_number = offset
while line_number < source_lines.size() and source_lines[line_number] != '===':
if recording:
body_lines.append(source_lines[line_number])
recording = recording or source_lines[line_number] == '---'
line_number += 1
line_number += 1
return PoolStringArray(body_lines).join('\n')
func get_nodes():
var nodes = []
var line_number = 0
var source_lines = source.split('\n')
while line_number < source_lines.size():
var headers = get_headers(line_number)
var body = get_body(line_number)
headers.body = body
nodes.append(headers)
# Add +2 to the final line to skip the === from that node
line_number = Array(source_lines).find_last(body.split('\n')[-1]) + 2
while line_number < source_lines.size() and source_lines[line_number].empty():
line_number += 1
return nodes
func assert(statement, message, line_number = -1, column = -1, _absolute_line_number = -1):
if not soft_assert:
assert(statement, message + ('; on line %d column %d' % [line_number, column]))
elif not statement:
emit_signal('error', message, line_number, column)
func compile():
var parsed_nodes = []
for node in get_nodes():
var lexer = Lexer.new(self, filename, node.title, node.body)
var tokens = lexer.tokenize()
# In case of lexer error
if not tokens:
return
var parser = Parser.new(self, node.title, tokens)
var parser_node = parser.parse_node()
parser_node.name = node.title
parsed_nodes.append(parser_node)
var program = Program.new()
program.filename = filename
for node in parsed_nodes:
compile_node(program, node)
for key in string_table:
program.strings[key] = string_table[key]
return program
func compile_node(program, parsed_node):
self.assert(not program.nodes.has(parsed_node.name), 'Duplicate node in program: %s' % parsed_node.name)
var node_compiled = Program.WolNode.new()
node_compiled.name = parsed_node.name
node_compiled.tags = parsed_node.tags
if parsed_node.source != null and not parsed_node.source.empty():
node_compiled.source_id = register_string(
parsed_node.source,
parsed_node.name,
'line:' + parsed_node.name,
0,
[]
)
else:
var start_label = register_label()
emit(Constants.ByteCode.Label, node_compiled, [Program.Operand.new(start_label)])
for statement in parsed_node.statements:
generate_statement(node_compiled, statement)
var dangling_options = false
for instruction in node_compiled.instructions:
if instruction.operation == Constants.ByteCode.AddOption:
dangling_options = true
if instruction.operation == Constants.ByteCode.ShowOptions:
dangling_options = false
if dangling_options:
emit(Constants.ByteCode.ShowOptions, node_compiled)
emit(Constants.ByteCode.RunNode, node_compiled)
else:
emit(Constants.ByteCode.Stop, node_compiled)
program.nodes[node_compiled.name] = node_compiled
func register_string(text, node_name, id = '', line_number = -1, tags = []):
var line_id_used = ''
var implicit = false
if id.empty():
line_id_used = '%s-%s-%d' % [filename, node_name, string_count]
string_count += 1
#use this when we generate implicit tags
#they are not saved and are generated
#aka dummy tags that change on each compilation
has_implicit_string_tags = true
implicit = true
else :
line_id_used = id
implicit = false
var string_info = Program.Line.new(text, node_name, line_number, filename, implicit, tags)
string_table[line_id_used] = string_info
return line_id_used
func register_label(comment = ''):
label_count += 1
return 'Label%s%s' % [label_count, comment]
func emit(bytecode, node = current_node, operands = []):
var instruction = Program.Instruction.new(null)
instruction.operation = bytecode
instruction.operands = operands
if node == null:
printerr('Trying to emit to null node with byte_code: %s' % bytecode)
return
node.instructions.append(instruction)
if bytecode == Constants.ByteCode.Label:
node.labels[instruction.operands[0].value] = node.instructions.size() - 1
func generate_statement(node, statement):
match statement.type:
Constants.StatementTypes.CustomCommand:
generate_custom_command(node, statement.custom_command)
Constants.StatementTypes.ShortcutOptionGroup:
generate_shortcut_group(node, statement.shortcut_option_group)
Constants.StatementTypes.Block:
generate_block(node, statement.block.statements)
Constants.StatementTypes.IfStatement:
generate_if(node, statement.if_statement)
Constants.StatementTypes.OptionStatement:
generate_option(node, statement.option_statement)
Constants.StatementTypes.AssignmentStatement:
generate_assignment(node, statement.assignment)
Constants.StatementTypes.Line:
generate_line(node, statement)
_:
self.assert(false, statement.line_number, 'Illegal statement type [%s]. Could not generate code.' % statement.type)
func generate_custom_command(node, command):
# TODO: See if the first tree of this statement is being used
if command.expression != null:
generate_expression(node, command.expression)
else:
var command_string = command.client_command
if command_string == 'stop':
emit(Constants.ByteCode.Stop, node)
else :
emit(Constants.ByteCode.RunCommand, node, [Program.Operand.new(command_string)])
func generate_line(node, statement):
# TODO: Implement proper line numbers (global and local)
var line = statement.line
var expression_count = line.substitutions.size()
while not line.substitutions.empty():
var inline_expression = line.substitutions.pop_back()
generate_expression(node, inline_expression.expression)
var num = register_string(line.line_text, node.name, line.line_id, statement.line_number, line.tags);
emit(Constants.ByteCode.RunLine, node,[Program.Operand.new(num), Program.Operand.new(expression_count)])
func generate_shortcut_group(node, shortcut_group):
var end = register_label('group_end')
var labels = []
var option_count = 0
for option in shortcut_group.options:
var endof_clause = ''
var op_destination = register_label('option_%s' % [option_count + 1])
labels.append(op_destination)
if option.condition != null:
endof_clause = register_label('conditional_%s' % option_count)
generate_expression(node, option.condition)
emit(Constants.ByteCode.JumpIfFalse, node, [Program.Operand.new(endof_clause)])
var label_line_id = '' #TODO: Add tag support
var label_string_id = register_string(
option.label,
node.name,
label_line_id,
option.line_number,
[]
)
emit(Constants.ByteCode.AddOption, node, [Program.Operand.new(label_string_id), Program.Operand.new(op_destination)])
if option.condition != null:
emit(Constants.ByteCode.Label, node, [Program.Operand.new(endof_clause)])
emit(Constants.ByteCode.Pop, node)
option_count += 1
emit(Constants.ByteCode.ShowOptions, node)
emit(Constants.ByteCode.Jump, node)
option_count = 0
for option in shortcut_group.options:
emit(Constants.ByteCode.Label, node, [Program.Operand.new(labels[option_count])])
if option.node != null:
generate_block(node, option.node.statements)
emit(Constants.ByteCode.JumpTo, node, [Program.Operand.new(end)])
option_count += 1
emit(Constants.ByteCode.Label, node, [Program.Operand.new(end)])
emit(Constants.ByteCode.Pop, node)
func generate_block(node, statements = []):
if not statements.empty():
for statement in statements:
generate_statement(node, statement)
func generate_if(node, if_statement):
var endif = register_label('endif')
for clause in if_statement.clauses:
var end_clause = register_label('skip_clause')
if clause.expression != null:
generate_expression(node, clause.expression)
emit(Constants.ByteCode.JumpIfFalse, node, [Program.Operand.new(end_clause)])
generate_block(node, clause.statements)
emit(Constants.ByteCode.JumpTo, node, [Program.Operand.new(endif)])
if clause.expression != null:
emit(Constants.ByteCode.Label, node, [Program.Operand.new(end_clause)])
if clause.expression != null:
emit(Constants.ByteCode.Pop)
emit(Constants.ByteCode.Label, node, [Program.Operand.new(endif)])
func generate_option(node, option):
var destination = option.destination
if option.label == null or option.label.empty():
emit(Constants.ByteCode.RunNode, node, [Program.Operand.new(destination)])
else :
var line_id = '' #TODO: ADD TAG SUPPORT
var string_id = register_string(option.label, node.name, line_id, option.line_number, [])
emit(Constants.ByteCode.AddOption, node, [Program.Operand.new(string_id), Program.Operand.new(destination)])
func generate_assignment(node, assignment):
if assignment.operation == Constants.TokenType.EqualToOrAssign:
generate_expression(node, assignment.value)
else :
emit(Constants.ByteCode.PushVariable, node, [assignment.destination])
generate_expression(node, assignment.value)
match assignment.operation:
Constants.TokenType.AddAssign:
emit(
Constants.ByteCode.CallFunc,
node,
[Program.Operand.new(Constants.token_type_name(Constants.TokenType.Add))]
)
Constants.TokenType.MinusAssign:
emit(
Constants.ByteCode.CallFunc,
node,
[Program.Operand.new(Constants.token_type_name(Constants.TokenType.Minus))]
)
Constants.TokenType.MultiplyAssign:
emit(
Constants.ByteCode.CallFunc,
node,
[Program.Operand.new(Constants.token_type_name(Constants.TokenType.MultiplyAssign))]
)
Constants.TokenType.DivideAssign:
emit(
Constants.ByteCode.CallFunc,
node,
[Program.Operand.new(Constants.token_type_name(Constants.TokenType.DivideAssign))]
)
_:
printerr('Unable to generate assignment')
emit(Constants.ByteCode.StoreVariable, node, [Program.Operand.new(assignment.destination)])
emit(Constants.ByteCode.Pop, node)
func generate_expression(node, expression):
match expression.type:
Constants.ExpressionType.Value:
generate_value(node, expression.value)
Constants.ExpressionType.FunctionCall:
for parameter in expression.parameters:
generate_expression(node, parameter)
emit(Constants.ByteCode.PushNumber, node, [Program.Operand.new(expression.parameters.size())])
emit(Constants.ByteCode.CallFunc, node, [Program.Operand.new(expression.function)])
_:
printerr('No expression.')
func generate_value(node, value):
match value.value.type:
Constants.ValueType.Number:
emit(
Constants.ByteCode.PushNumber,
node,
[Program.Operand.new(value.value.as_number())]
)
Constants.ValueType.Str:
var id = register_string(
value.value.as_string(),
node.name,
'',
value.line_number,
[]
)
emit(
Constants.ByteCode.PushString,
node,
[Program.Operand.new(id)]
)
Constants.ValueType.Boolean:
emit(
Constants.ByteCode.PushBool,
node,
[Program.Operand.new(value.value.as_bool())]
)
Constants.ValueType.Variable:
emit(
Constants.ByteCode.PushVariable,
node,
[Program.Operand.new(value.value.variable)]
)
Constants.ValueType.Nullean:
emit(Constants.ByteCode.PushNull, node)
_:
printerr('Unrecognized valuenode type: %s' % value.value.type)

View file

@ -0,0 +1,485 @@
extends Object
const Constants = preload('res://addons/Wol/core/Constants.gd')
const LINE_COMMENT = '//'
const FORWARD_SLASH = '/'
const LINE_SEPARATOR = '\n'
const BASE = 'base'
const DASH = '-'
const COMMAND = 'command'
const LINK = 'link'
const SHORTCUT = 'shortcut'
const TAG = 'tag'
const EXPRESSION = 'expression'
const ASSIGNMENT = 'assignment'
const OPTION = 'option'
const OR = 'or'
const DESTINATION = 'destination'
const INLINE = 'inline'
const FORMAT_FUNCTION = 'format'
var WHITESPACE = '\\s*'
var compiler
var filename = ''
var title = ''
var text = ''
var states = {}
var default_state
var current_state
var indent_stack = []
var should_track_indent = false
func _init(_compiler, _filename, _title, _text):
create_states()
compiler = _compiler
filename = _filename
title = _title
text = _text
func create_states():
var patterns = {}
patterns[Constants.TokenType.Text] = ['.*', 'any text']
patterns[Constants.TokenType.Number] = ['\\-?[0-9]+(\\.[0-9+])?', 'any number']
patterns[Constants.TokenType.Str] = ['\"([^\"\\\\]*(?:\\.[^\"\\\\]*)*)\"', 'any text']
patterns[Constants.TokenType.TagMarker] = ['\\#', 'a tag #']
patterns[Constants.TokenType.LeftParen] = ['\\(', 'left parenthesis (']
patterns[Constants.TokenType.RightParen] = ['\\)', 'right parenthesis )']
patterns[Constants.TokenType.EqualTo] = ['(==|is(?!\\w)|eq(?!\\w))', '"=", "is" or "eq"']
patterns[Constants.TokenType.EqualToOrAssign] = ['(=|to(?!\\w))', '"=" or "to"']
patterns[Constants.TokenType.NotEqualTo] = ['(\\!=|neq(?!\\w))', '"!=" or "neq"']
patterns[Constants.TokenType.GreaterThanOrEqualTo] = ['(\\>=|gte(?!\\w))', '">=" or "gte"']
patterns[Constants.TokenType.GreaterThan] = ['(\\>|gt(?!\\w))', '">" or "gt"']
patterns[Constants.TokenType.LessThanOrEqualTo] = ['(\\<=|lte(?!\\w))', '"<=" or "lte"']
patterns[Constants.TokenType.LessThan] = ['(\\<|lt(?!\\w))', '"<" or "lt"']
patterns[Constants.TokenType.AddAssign] = ['\\+=', '"+="']
patterns[Constants.TokenType.MinusAssign] = ['\\-=', '"-="']
patterns[Constants.TokenType.MultiplyAssign] = ['\\*=', '"*="']
patterns[Constants.TokenType.DivideAssign] = ['\\/=', '"/="']
patterns[Constants.TokenType.Add] = ['\\+', '"+"']
patterns[Constants.TokenType.Minus] = ['\\-', '"-"']
patterns[Constants.TokenType.Multiply] = ['\\*', '"*"']
patterns[Constants.TokenType.Divide] = ['\\/', '"/"']
patterns[Constants.TokenType.Modulo] = ['\\%', '"%"']
patterns[Constants.TokenType.And] = ['(\\&\\&|and(?!\\w))', '"&&" or "and"']
patterns[Constants.TokenType.Or] = ['(\\|\\||or(?!\\w))', '"||" or "or"']
patterns[Constants.TokenType.Xor] = ['(\\^|xor(?!\\w))', '"^" or "xor"']
patterns[Constants.TokenType.Not] = ['(\\!|not(?!\\w))', '"!" or "not"']
patterns[Constants.TokenType.Variable] = ['\\$([A-Za-z0-9_\\.])+', 'any variable']
patterns[Constants.TokenType.Comma] = ['\\,', '","']
patterns[Constants.TokenType.TrueToken] = ['true(?!\\w)', '"true"']
patterns[Constants.TokenType.FalseToken] = ['false(?!\\w)', '"false"']
patterns[Constants.TokenType.NullToken] = ['null(?!\\w)', '"null"']
patterns[Constants.TokenType.BeginCommand] = ['\\<\\<', 'beginning of a command "<<"']
patterns[Constants.TokenType.EndCommand] = ['\\>\\>', 'ending of a command ">>"']
patterns[Constants.TokenType.OptionStart] = ['\\[\\[', 'start of an option "[["']
patterns[Constants.TokenType.OptionEnd] = ['\\]\\]', 'end of an option "]]"']
patterns[Constants.TokenType.OptionDelimit] = ['\\|', 'middle of an option "|"']
patterns[Constants.TokenType.Identifier] = ['[a-zA-Z0-9_:\\.]+', 'any reference to another node']
patterns[Constants.TokenType.IfToken] = ['if(?!\\w)', '"if"']
patterns[Constants.TokenType.ElseToken] = ['else(?!\\w)', '"else"']
patterns[Constants.TokenType.ElseIf] = ['elseif(?!\\w)', '"elseif"']
patterns[Constants.TokenType.EndIf] = ['endif(?!\\w)', '"endif"']
patterns[Constants.TokenType.Set] = ['set(?!\\w)', '"set"']
patterns[Constants.TokenType.ShortcutOption] = ['\\-\\>\\s*', '"->"']
patterns[Constants.TokenType.ExpressionFunctionStart] = ['\\{', '"{"']
patterns[Constants.TokenType.ExpressionFunctionEnd] = ['\\}', '"}"']
patterns[Constants.TokenType.FormatFunctionStart] = ['(?<!\\[)\\[(?!\\[)', '"["']
patterns[Constants.TokenType.FormatFunctionEnd] = ['\\]', '"]"']
var shortcut_option = SHORTCUT + DASH + OPTION
var shortcut_option_tag = shortcut_option + DASH + TAG
var command_or_expression = COMMAND + DASH + OR + DASH + EXPRESSION
var link_destination = LINK + DASH + DESTINATION
var format_expression = FORMAT_FUNCTION + DASH + EXPRESSION
var inline_expression = INLINE + DASH + EXPRESSION
var link_inline_expression = LINK + DASH + INLINE + DASH + EXPRESSION
var link_format_expression = LINK + DASH + FORMAT_FUNCTION + DASH + EXPRESSION
states = {}
states[BASE] = LexerState.new(patterns)
states[BASE].add_transition(Constants.TokenType.BeginCommand, COMMAND, true)
states[BASE].add_transition(Constants.TokenType.ExpressionFunctionStart, inline_expression, true)
states[BASE].add_transition(Constants.TokenType.FormatFunctionStart, FORMAT_FUNCTION, true)
states[BASE].add_transition(Constants.TokenType.OptionStart, LINK, true)
states[BASE].add_transition(Constants.TokenType.ShortcutOption, shortcut_option)
states[BASE].add_transition(Constants.TokenType.TagMarker, TAG, true)
states[BASE].add_text_rule(Constants.TokenType.Text)
#FIXME - Tags are not being proccessed properly this way. We must look for the format #{identifier}:{value}
# Possible solution is to add more transitions
states[TAG] = LexerState.new(patterns)
states[TAG].add_transition(Constants.TokenType.Identifier, BASE)
states[shortcut_option] = LexerState.new(patterns)
states[shortcut_option].track_indent = true
states[shortcut_option].add_transition(Constants.TokenType.BeginCommand, EXPRESSION, true)
states[shortcut_option].add_transition(Constants.TokenType.ExpressionFunctionStart, inline_expression, true)
states[shortcut_option].add_transition(Constants.TokenType.TagMarker, shortcut_option_tag, true)
states[shortcut_option].add_text_rule(Constants.TokenType.Text, BASE)
states[shortcut_option_tag] = LexerState.new(patterns)
states[shortcut_option_tag].add_transition(Constants.TokenType.Identifier, shortcut_option)
states[COMMAND] = LexerState.new(patterns)
states[COMMAND].add_transition(Constants.TokenType.IfToken, EXPRESSION)
states[COMMAND].add_transition(Constants.TokenType.ElseToken)
states[COMMAND].add_transition(Constants.TokenType.ElseIf, EXPRESSION)
states[COMMAND].add_transition(Constants.TokenType.EndIf)
states[COMMAND].add_transition(Constants.TokenType.Set, ASSIGNMENT)
states[COMMAND].add_transition(Constants.TokenType.EndCommand, BASE, true)
states[COMMAND].add_transition(Constants.TokenType.Identifier, command_or_expression)
states[COMMAND].add_text_rule(Constants.TokenType.Text)
states[command_or_expression] = LexerState.new(patterns)
states[command_or_expression].add_transition(Constants.TokenType.LeftParen, EXPRESSION)
states[command_or_expression].add_transition(Constants.TokenType.EndCommand, BASE, true)
states[command_or_expression].add_text_rule(Constants.TokenType.Text)
states[ASSIGNMENT] = LexerState.new(patterns)
states[ASSIGNMENT].add_transition(Constants.TokenType.Variable)
states[ASSIGNMENT].add_transition(Constants.TokenType.EqualToOrAssign, EXPRESSION)
states[ASSIGNMENT].add_transition(Constants.TokenType.AddAssign, EXPRESSION)
states[ASSIGNMENT].add_transition(Constants.TokenType.MinusAssign, EXPRESSION)
states[ASSIGNMENT].add_transition(Constants.TokenType.MultiplyAssign, EXPRESSION)
states[ASSIGNMENT].add_transition(Constants.TokenType.DivideAssign, EXPRESSION)
states[FORMAT_FUNCTION] = LexerState.new(patterns)
states[FORMAT_FUNCTION].add_transition(Constants.TokenType.FormatFunctionEnd, BASE, true)
states[FORMAT_FUNCTION].add_transition(Constants.TokenType.ExpressionFunctionStart, format_expression, true)
states[FORMAT_FUNCTION].add_text_rule(Constants.TokenType.Text)
states[format_expression] = LexerState.new(patterns)
states[format_expression].add_transition(Constants.TokenType.ExpressionFunctionEnd, FORMAT_FUNCTION)
form_expression_state(states[format_expression])
states[inline_expression] = LexerState.new(patterns)
states[inline_expression].add_transition(Constants.TokenType.ExpressionFunctionEnd, BASE)
form_expression_state(states[inline_expression])
states[EXPRESSION] = LexerState.new(patterns)
states[EXPRESSION].add_transition(Constants.TokenType.EndCommand, BASE)
# states[EXPRESSION].add_transition(Constants.TokenType.FormatFunctionEnd, BASE)
form_expression_state(states[EXPRESSION])
states[LINK] = LexerState.new(patterns)
states[LINK].add_transition(Constants.TokenType.OptionEnd, BASE, true)
states[LINK].add_transition(Constants.TokenType.ExpressionFunctionStart, link_inline_expression, true)
states[LINK].add_transition(Constants.TokenType.FormatFunctionStart, link_format_expression, true)
states[LINK].add_transition(Constants.TokenType.FormatFunctionEnd, LINK, true)
states[LINK].add_transition(Constants.TokenType.OptionDelimit, link_destination, true)
states[LINK].add_text_rule(Constants.TokenType.Text)
states[link_format_expression] = LexerState.new(patterns)
states[link_format_expression].add_transition(Constants.TokenType.FormatFunctionEnd, LINK, true)
states[link_format_expression].add_transition(Constants.TokenType.ExpressionFunctionStart, link_inline_expression, true)
states[link_format_expression].add_text_rule(Constants.TokenType.Text)
states[link_inline_expression] = LexerState.new(patterns)
states[link_inline_expression].add_transition(Constants.TokenType.ExpressionFunctionEnd, LINK)
form_expression_state(states[link_inline_expression])
states[link_destination] = LexerState.new(patterns)
states[link_destination].add_transition(Constants.TokenType.Identifier)
states[link_destination].add_transition(Constants.TokenType.OptionEnd, BASE)
default_state = states[BASE]
for key in states.keys():
states[key].name = key
func form_expression_state(expression_state):
expression_state.add_transition(Constants.TokenType.Number)
expression_state.add_transition(Constants.TokenType.Str)
expression_state.add_transition(Constants.TokenType.LeftParen)
expression_state.add_transition(Constants.TokenType.RightParen)
expression_state.add_transition(Constants.TokenType.EqualTo)
expression_state.add_transition(Constants.TokenType.EqualToOrAssign)
expression_state.add_transition(Constants.TokenType.NotEqualTo)
expression_state.add_transition(Constants.TokenType.GreaterThanOrEqualTo)
expression_state.add_transition(Constants.TokenType.GreaterThan)
expression_state.add_transition(Constants.TokenType.LessThanOrEqualTo)
expression_state.add_transition(Constants.TokenType.LessThan)
expression_state.add_transition(Constants.TokenType.Add)
expression_state.add_transition(Constants.TokenType.Minus)
expression_state.add_transition(Constants.TokenType.Multiply)
expression_state.add_transition(Constants.TokenType.Divide)
expression_state.add_transition(Constants.TokenType.Modulo)
expression_state.add_transition(Constants.TokenType.And)
expression_state.add_transition(Constants.TokenType.Or)
expression_state.add_transition(Constants.TokenType.Xor)
expression_state.add_transition(Constants.TokenType.Not)
expression_state.add_transition(Constants.TokenType.Variable)
expression_state.add_transition(Constants.TokenType.Comma)
expression_state.add_transition(Constants.TokenType.TrueToken)
expression_state.add_transition(Constants.TokenType.FalseToken)
expression_state.add_transition(Constants.TokenType.NullToken)
expression_state.add_transition(Constants.TokenType.Identifier)
func tokenize():
var tokens = []
indent_stack.clear()
indent_stack.push_front([0, false])
should_track_indent = false
current_state = default_state
var lines = text.split(LINE_SEPARATOR)
var line_number = 1
lines.append('')
for line in lines:
var line_tokens = tokenize_line(line, line_number)
if line_tokens == null:
return
tokens.append_array(line_tokens)
line_number += 1
var end_of_input = Token.new(
Constants.TokenType.EndOfInput,
current_state,
line_number,
0
)
tokens.append(end_of_input)
return tokens
func tokenize_line(line, line_number):
var token_stack = []
var fresh_line = line.replace('\t', ' ').replace('\r', '')
var indentation = line_indentation(line)
var previous_indentation = indent_stack.front()[0]
if should_track_indent and indentation > previous_indentation:
indent_stack.push_front([indentation, true])
var indent = Token.new(
Constants.TokenType.Indent,
current_state,
filename,
line_number,
previous_indentation
)
indent.value = '%*s' % [indentation - previous_indentation, '']
should_track_indent = false
token_stack.push_front(indent)
elif indentation < previous_indentation:
while indentation < indent_stack.front()[0]:
var top = indent_stack.pop_front()[1]
if top:
var deindent = Token.new(Constants.TokenType.Dedent, current_state, line_number, 0)
token_stack.push_front(deindent)
var column = indentation
var whitespace = RegEx.new()
whitespace.compile(WHITESPACE)
while column < fresh_line.length():
if fresh_line.substr(column).begins_with(LINE_COMMENT):
break
var matched = false
for rule in current_state.rules:
var found = rule.regex.search(fresh_line, column)
if !found:
continue
var token_text = ''
# NOTE: If this is text then we back up to the most recent delimiting token
# and treat everything from there as text.
if rule.token_type == Constants.TokenType.Text:
var start_index = indentation
if token_stack.size() > 0 :
while token_stack.front().type == Constants.TokenType.Identifier:
token_stack.pop_front()
var start_delimit_token = token_stack.front()
start_index = start_delimit_token.column
if start_delimit_token.type == Constants.TokenType.Indent:
start_index += start_delimit_token.value.length()
if start_delimit_token.type == Constants.TokenType.Dedent:
start_index = indentation
column = start_index
var end_index = found.get_start() + found.get_string().length()
token_text = fresh_line.substr(start_index, end_index - start_index)
else:
token_text = found.get_string()
column += token_text.length()
if rule.token_type == Constants.TokenType.Str:
token_text = token_text.substr(1, token_text.length() - 2)
token_text = token_text.replace('\\\\', '\\')
token_text = token_text.replace('\\\'','\'')
var token = Token.new(
rule.token_type,
current_state,
filename,
line_number,
column,
token_text
)
token.delimits_text = rule.delimits_text
token_stack.push_front(token)
if rule.enter_state != null and rule.enter_state.length() > 0:
if not states.has(rule.enter_state):
printerr('State[%s] not known - line(%s) col(%s)' % [rule.enter_state, line_number, column])
return []
enter_state(states[rule.enter_state])
if should_track_indent:
if indent_stack.front()[0] < indentation:
indent_stack.append([indentation, false])
matched = true
break
if not matched:
var rules = []
for rule in current_state.rules:
rules.append('"%s" (%s)' % [Constants.token_type_name(rule.token_type), rule.human_readable_identifier])
var error_data = [
PoolStringArray(rules).join(', ') if rules.size() == 1 else PoolStringArray(rules.slice(0, rules.size() - 2)).join(', ') + ' or %s' % rules[-1],
filename,
title,
line_number,
column
]
compiler.assert(false, 'Expected %s in file %s in node "%s" on line #%d (column #%d)' % error_data, line_number, column)
return
var last_whitespace = whitespace.search(line, column)
if last_whitespace:
column += last_whitespace.get_string().length()
token_stack.invert()
return token_stack
func line_indentation(line):
var indent_regex = RegEx.new()
indent_regex.compile('^(\\s*)')
var found = indent_regex.search(line)
if !found or found.get_string().length() <= 0:
return 0
return found.get_string().length()
func enter_state(state):
current_state = state;
if current_state.track_indent:
should_track_indent = true
class Token:
var type = -1
var value = ''
var filename = ''
var line_number = -1
var column = -1
var text = ''
var delimits_text = false
var parameter_count = -1
var lexer_state = ''
func _init(_type, _state, _filename, _line_number = -1, _column = -1, _value = ''):
type = _type
lexer_state = _state.name
filename = _filename
line_number = _line_number
column = _column
value = _value
func _to_string():
return '%s (%s) at %s:%s (state: %s)' % [Constants.token_type_name(type), value, line_number, column, lexer_state]
class LexerState:
var name = ''
var patterns = {}
var rules = []
var track_indent = false
func _init(_patterns):
patterns = _patterns
func add_transition(type, state = '', delimit_text = false):
var pattern = '\\G%s' % patterns[type][0]
var rule = Rule.new(type, pattern, patterns[type][1], state, delimit_text)
rules.append(rule)
return rule
func add_text_rule(type, state = ''):
if contains_text_rule() :
printerr('State already contains Text rule')
return null
var delimiters:Array = []
for rule in rules:
if rule.delimits_text:
delimiters.append('%s' % rule.regex.get_pattern().substr(2))
var pattern = '\\G((?!%s).)*' % [PoolStringArray(delimiters).join('|')]
var rule = add_transition(type, state)
rule.regex = RegEx.new()
rule.regex.compile(pattern)
rule.is_text_rule = true
return rule
func contains_text_rule():
for rule in rules:
if rule.is_text_rule:
return true
return false
class Rule:
var regex
var enter_state = ''
var token_type = -1
var is_text_rule = false
var delimits_text = false
var human_readable_identifier = ''
func _init(_type, _regex, _human_readable_identifier, _enter_state, _delimits_text):
token_type = _type
regex = RegEx.new()
regex.compile(_regex)
human_readable_identifier = _human_readable_identifier
enter_state = _enter_state
delimits_text = _delimits_text
func _to_string():
return '[Rule : %s (%s) - %s]' % [Constants.token_type_name(token_type), human_readable_identifier, regex]

File diff suppressed because it is too large Load diff

View file

@ -1,31 +1,3 @@
title: Start
tags:
colorID:
position: 0, 0
---
<<a_custom_command>>
<<command_with multiple arguments>>
// remove "to" to trigger error
<<set $direction to "that">>
<<set $one to 1>>
// Implement inline expressions
<<if visit_count() == 1>>
Narrator: You, {$direction} way!
<<endif>>
Narrator: Do you know you've been here {visit_count()} times?
You: Did you know one + one equals {$one + $one}?
Narrator: You wanna go somewhere?
-> Go to the store
[[TheStore]]
-> How much did I visit the store?
Narrator: You've been to the store { visit_count("TheStore") } times.
[[Start]]
-> Lets stay here and talk
[[Talk]]
===
title: TheStore title: TheStore
tags: tags:
colorID: colorID:
@ -68,3 +40,31 @@ Narrator: Do you want to continue talking?
[[Start]] [[Start]]
-> No -> No
=== ===
title: Start
tags:
colorID:
position: 0, 0
---
<<a_custom_command>>
<<command_with multiple arguments>>
// remove "to" to trigger error
<<set $direction to "that">>
<<set $one to 1>>
// Implement inline expressions
<<if visit_count() == 1>>
Narrator: You, {$direction} way!
<<endif>>
Narrator: Do you know you've been here {visit_count()} times?
You: Did you know one + one equals {$one + $one}?
Narrator: You wanna go somewhere?
-> Go to the store
[[TheStore]]
-> How much did I visit the store?
Narrator: You've been to the store { visit_count("TheStore") } times.
[[Start]]
-> Lets stay here and talk
[[Talk]]
===

View file

@ -11,7 +11,7 @@ config_version=4
[application] [application]
config/name="Wol" config/name="Wol"
run/main_scene="res://Dialogue.tscn" run/main_scene="res://TestScene.tscn"
config/icon="res://icon.png" config/icon="res://icon.png"
[debug] [debug]