mirrors/Terra
1
0
Fork 0
mirror of https://github.com/PolyhedralDev/Terra.git synced 2026-04-09 09:16:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

add escape character to parser string literals

Browse Source
This commit is contained in:
dfsek
2020-12-19 21:03:14 -07:00
parent f970838ecf
commit 474962db39
9 changed files with 69 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
common/src/main/java/com/dfsek/terra/api/structures/parser/Function.java
View File

@@ -3,10 +3,10 @@ package com.dfsek.terra.api.structures.parser;
import com.dfsek.terra.api.math.vector.Location;
import com.dfsek.terra.api.platform.world.Chunk;
public interface Function {
void apply(Location location);
public interface Function<T> {
T apply(Location location);
void apply(Location location, Chunk chunk);
T apply(Location location, Chunk chunk);
String name();
}

46
common/src/main/java/com/dfsek/terra/api/structures/parser/Parser.java
View File

@@ -15,21 +15,21 @@ import java.util.stream.Collectors;
public class Parser {
private final String data;
private final Map<String, FunctionBuilder<? extends Function>> functions = new HashMap<>();
private final Map<String, FunctionBuilder<? extends Function<?>>> functions = new HashMap<>();
Set<Token.Type> allowedArguments = Sets.newHashSet(Token.Type.STRING, Token.Type.NUMBER, Token.Type.IDENTIFIER);
public Parser(String data) {
this.data = data;
}
public Parser addFunction(String name, FunctionBuilder<? extends Function> functionBuilder) {
public Parser addFunction(String name, FunctionBuilder<? extends Function<?>> functionBuilder) {
functions.put(name, functionBuilder);
return this;
}
public List<Function> parse() throws ParseException {
public List<Function<?>> parse() throws ParseException {
Tokenizer tokenizer = new Tokenizer(data);
List<Function> builtFunctions = new GlueList<>();
List<Function<?>> builtFunctions = new GlueList<>();
List<Token> functionBuilder = new GlueList<>();
Token token = null;
while(tokenizer.hasNext()) {
@@ -46,32 +46,15 @@ public class Parser {
checkType(functionBuilder.remove(0), Token.Type.BODY_BEGIN); // Second is body begin
boolean expectingSeparator = false;
List<Token> args = new GlueList<>();
while(!functionBuilder.get(0).getType().equals(Token.Type.BODY_END)) {
Token current = functionBuilder.remove(0);
if(expectingSeparator) {
checkType(current, Token.Type.SEPARATOR);
expectingSeparator = false;
} else {
if(!allowedArguments.contains(current.getType()))
throw new ParseException("Token type " + current.getType() + " not allowed in arguments: " + current.getStart());
args.add(current);
expectingSeparator = true;
}
}
List<Token> args = getArgs(functionBuilder); // Extract arguments, consume the rest.
functionBuilder.remove(0); // Remove body end
for(Token t : args) System.out.println("TOKEN: " + t);
checkType(functionBuilder.remove(0), Token.Type.STATEMENT_END);
List<String> arg = args.stream().map(Token::getContent).collect(Collectors.toList());
for(String s : arg) System.out.println("ARG: " + s);
FunctionBuilder<?> builder = functions.get(identifier.getContent());
if(arg.size() != builder.getArguments().size())
throw new ParseException("Expected " + builder.getArguments().size() + " arguments, found " + arg.size() + ": " + identifier.getStart());
@@ -89,6 +72,25 @@ public class Parser {
return builtFunctions;
}
private List<Token> getArgs(List<Token> functionBuilder) throws ParseException {
List<Token> args = new GlueList<>();
boolean expectingSeparator = false;
while(!functionBuilder.get(0).getType().equals(Token.Type.BODY_END)) {
Token current = functionBuilder.remove(0);
if(expectingSeparator) {
checkType(current, Token.Type.SEPARATOR);
expectingSeparator = false;
} else {
if(!allowedArguments.contains(current.getType()))
throw new ParseException("Token type " + current.getType() + " not allowed in arguments: " + current.getStart());
args.add(current);
expectingSeparator = true;
}
}
return args;
}
private void checkType(Token token, Token.Type expected) throws ParseException {
if(!token.getType().equals(expected))
throw new ParseException("Expected " + expected + " but found " + token.getType() + ": " + token.getStart());

4
common/src/main/java/com/dfsek/terra/api/structures/parser/lang/Block.java Normal file
View File

@@ -0,0 +1,4 @@
package com.dfsek.terra.api.structures.parser.lang;
public class Block {
}

4
common/src/main/java/com/dfsek/terra/api/structures/parser/lang/Expression.java Normal file
View File

@@ -0,0 +1,4 @@
package com.dfsek.terra.api.structures.parser.lang;
public class Expression {
}

4
common/src/main/java/com/dfsek/terra/api/structures/parser/lang/Item.java Normal file
View File

@@ -0,0 +1,4 @@
package com.dfsek.terra.api.structures.parser.lang;
public interface Item {
}

2
common/src/main/java/com/dfsek/terra/api/structures/tokenizer/Token.java
View File

@@ -29,6 +29,6 @@ public class Token {
}
public enum Type {
IDENTIFIER, NUMBER, STRING, BOOLEAN, BODY_BEGIN, BODY_END, STATEMENT_END, SEPARATOR
IDENTIFIER, NUMBER, STRING, BOOLEAN, BODY_BEGIN, BODY_END, STATEMENT_END, SEPARATOR, BLOCK_BEGIN, BLOCK_END
}
}

15
common/src/main/java/com/dfsek/terra/api/structures/tokenizer/Tokenizer.java
View File

@@ -11,7 +11,8 @@ import java.util.Set;
public class Tokenizer {
private final Lookahead reader;
private final Set<Character> syntaxSignificant = Sets.newHashSet(';', '(', ')', '"', '[', ']', ',');
private final Set<Character> syntaxSignificant = Sets.newHashSet(';', '(', ')', '"', ',', '\\', // Currently used chars
'{', '}'); // Reserved chars
public Tokenizer(String data) {
@@ -42,7 +43,13 @@ public class Tokenizer {
if(reader.current().is('"')) {
reader.consume(); // Consume first quote
StringBuilder string = new StringBuilder();
while(!reader.current().is('"')) {
boolean ignoreNext = false;
while((!reader.current().is('"')) || ignoreNext) {
if(reader.current().is('\\') && !ignoreNext) {
ignoreNext = true;
reader.consume();
continue;
} else ignoreNext = false;
if(reader.current().isEOF())
throw new FormatException("No end of string literal found. " + reader.getLine() + ":" + reader.getIndex());
string.append(reader.consume());
@@ -59,6 +66,10 @@ public class Tokenizer {
return new Token(reader.consume().toString(), Token.Type.STATEMENT_END, new Position(reader.getLine(), reader.getIndex()));
if(reader.current().is(','))
return new Token(reader.consume().toString(), Token.Type.SEPARATOR, new Position(reader.getLine(), reader.getIndex()));
if(reader.current().is('{'))
return new Token(reader.consume().toString(), Token.Type.BLOCK_BEGIN, new Position(reader.getLine(), reader.getIndex()));
if(reader.current().is('}'))
return new Token(reader.consume().toString(), Token.Type.BLOCK_END, new Position(reader.getLine(), reader.getIndex()));
StringBuilder token = new StringBuilder();
while(!reader.current().isEOF() && !isSyntaxSignificant(reader.current().getCharacter())) {

24
common/src/test/java/structure/ParserTest.java
View File

@@ -31,14 +31,15 @@ public class ParserTest {
}
});
List<Function> functions = parser.parse();
long l = System.nanoTime();
List<Function<?>> functions = parser.parse();
long t = System.nanoTime() - l;
System.out.println("Took " + (double) t / 1000000);
for(Function f : functions) {
System.out.println(f);
}
for(Function<?> f : functions) System.out.println(f);
}
private static class Test1 implements Function {
private static class Test1 implements Function<Void> {
private final String a;
private final double b;
@@ -56,18 +57,23 @@ public class ParserTest {
}
@Override
public void apply(Location location) {
public Void apply(Location location) {
return null;
}
@Override
public void apply(Location location, Chunk chunk) {
public Void apply(Location location, Chunk chunk) {
return null;
}
@Override
public String name() {
return null;
}
@Override
public String toString() {
return "string: " + a + ", double: " + b;
}
}
}

4
common/src/test/resources/test.tesf
View File

@@ -1,4 +1,2 @@
test("hello", 1);
test("ghgjhgjhgj", 3.4);
test("ghgj{}()\"\\hgjhgj", 3.4);