mirrors/Terra
1
0
Fork 0
mirror of https://github.com/PolyhedralDev/Terra.git synced 2026-04-04 06:46:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix various tokenizer issues

Browse Source
This commit is contained in:
dfsek
2020-12-22 02:58:42 -07:00
parent 88da796923
commit 72d4370878
4 changed files with 26 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common/src/main/java/com/dfsek/terra/api/structures/parser/Parser.java
View File

@@ -165,7 +165,7 @@ public class Parser {
private Returnable<?> parseGroup(List<Token> tokens, Map<String, Variable<?>> variableMap) throws ParseException {
ParserUtil.checkType(tokens.remove(0), Token.Type.GROUP_BEGIN);
Returnable<?> expression = parseExpression(tokens, true, variableMap);
Returnable<?> expression = parseExpression(tokens, true, variableMap); // Parse inside of group as a separate expression
ParserUtil.checkType(tokens.remove(0), Token.Type.GROUP_END);
return expression;
}

33
common/src/main/java/com/dfsek/terra/api/structures/tokenizer/Tokenizer.java
View File

@@ -11,7 +11,7 @@ import java.util.Set;
public class Tokenizer {
private final Lookahead reader;
private final Set<Character> syntaxSignificant = Sets.newHashSet(';', '(', ')', '"', ',', '\\', '=', '{', '}', '+', '-', '*', '/', '>', '<', '!'); // Reserved chars
public static final Set<Character> syntaxSignificant = Sets.newHashSet(';', '(', ')', '"', ',', '\\', '=', '{', '}', '+', '-', '*', '/', '>', '<', '!'); // Reserved chars
private final Set<String> keywords = Sets.newHashSet("if", "while", "num", "bool", "str");
@@ -36,11 +36,6 @@ public class Tokenizer {
if(reader.matches("/*", true)) skipTo("*/"); // Skip multi line comment
if(reader.matches("true", true))
return new Token("true", Token.Type.BOOLEAN, new Position(reader.getLine(), reader.getIndex()));
if(reader.matches("false", true))
return new Token("false", Token.Type.BOOLEAN, new Position(reader.getLine(), reader.getIndex()));
if(reader.matches("==", true))
return new Token("==", Token.Type.EQUALS_OPERATOR, new Position(reader.getLine(), reader.getIndex()));
@@ -60,13 +55,6 @@ public class Tokenizer {
if(reader.matches("&&", true))
return new Token("&&", Token.Type.BOOLEAN_AND, new Position(reader.getLine(), reader.getIndex()));
if(reader.matches("num ", true))
return new Token("num ", Token.Type.NUMBER_VARIABLE, new Position(reader.getLine(), reader.getIndex()));
if(reader.matches("str ", true))
return new Token("str ", Token.Type.STRING_VARIABLE, new Position(reader.getLine(), reader.getIndex()));
if(reader.matches("bool ", true))
return new Token("bool ", Token.Type.BOOLEAN_VARIABLE, new Position(reader.getLine(), reader.getIndex()));
if(isNumberStart()) {
StringBuilder num = new StringBuilder();
@@ -129,7 +117,24 @@ public class Tokenizer {
String tokenString = token.toString();
return new Token(tokenString, keywords.contains(tokenString) ? Token.Type.KEYWORD : Token.Type.IDENTIFIER, new Position(reader.getLine(), reader.getIndex()));
if(tokenString.equals("true"))
return new Token(tokenString, Token.Type.BOOLEAN, new Position(reader.getLine(), reader.getIndex()));
if(tokenString.equals("false"))
return new Token(tokenString, Token.Type.BOOLEAN, new Position(reader.getLine(), reader.getIndex()));
if(tokenString.equals("num"))
return new Token(tokenString, Token.Type.NUMBER_VARIABLE, new Position(reader.getLine(), reader.getIndex()));
if(tokenString.equals("str"))
return new Token(tokenString, Token.Type.STRING_VARIABLE, new Position(reader.getLine(), reader.getIndex()));
if(tokenString.equals("bool"))
return new Token(tokenString, Token.Type.BOOLEAN_VARIABLE, new Position(reader.getLine(), reader.getIndex()));
if(tokenString.equals("if"))
return new Token(tokenString, Token.Type.KEYWORD, new Position(reader.getLine(), reader.getIndex()));
if(tokenString.equals("while"))
return new Token(tokenString, Token.Type.KEYWORD, new Position(reader.getLine(), reader.getIndex()));
return new Token(tokenString, Token.Type.IDENTIFIER, new Position(reader.getLine(), reader.getIndex()));
}
private boolean isNumberLike() {

2
common/src/test/java/structure/ParserTest.java
View File

@@ -51,6 +51,8 @@ public class ParserTest {
System.out.println("Took " + (double) t / 1000000);
block.apply(null);
block.apply(null);
}
private static class Test1 implements Function<Void> {

4
common/src/test/resources/test.tesf
View File

@@ -5,6 +5,10 @@ num testVar = 3.4;
bool boolean = true;
str stringVar = "hello!";
bool iftest = false;
bool truetest = false;
num iterator = 0;
while(iterator < 5) {