From 474962db399b4bef76086a88b4b9d3c742dfa6a7 Mon Sep 17 00:00:00 2001 From: dfsek Date: Sat, 19 Dec 2020 21:03:14 -0700 Subject: [PATCH] add escape character to parser string literals --- .../terra/api/structures/parser/Function.java | 6 +-- .../terra/api/structures/parser/Parser.java | 46 ++++++++++--------- .../api/structures/parser/lang/Block.java | 4 ++ .../structures/parser/lang/Expression.java | 4 ++ .../api/structures/parser/lang/Item.java | 4 ++ .../terra/api/structures/tokenizer/Token.java | 2 +- .../api/structures/tokenizer/Tokenizer.java | 15 +++++- .../src/test/java/structure/ParserTest.java | 24 ++++++---- common/src/test/resources/test.tesf | 4 +- 9 files changed, 69 insertions(+), 40 deletions(-) create mode 100644 common/src/main/java/com/dfsek/terra/api/structures/parser/lang/Block.java create mode 100644 common/src/main/java/com/dfsek/terra/api/structures/parser/lang/Expression.java create mode 100644 common/src/main/java/com/dfsek/terra/api/structures/parser/lang/Item.java diff --git a/common/src/main/java/com/dfsek/terra/api/structures/parser/Function.java b/common/src/main/java/com/dfsek/terra/api/structures/parser/Function.java index ec4393f48..d60f60cc8 100644 --- a/common/src/main/java/com/dfsek/terra/api/structures/parser/Function.java +++ b/common/src/main/java/com/dfsek/terra/api/structures/parser/Function.java @@ -3,10 +3,10 @@ package com.dfsek.terra.api.structures.parser; import com.dfsek.terra.api.math.vector.Location; import com.dfsek.terra.api.platform.world.Chunk; -public interface Function { - void apply(Location location); +public interface Function { + T apply(Location location); - void apply(Location location, Chunk chunk); + T apply(Location location, Chunk chunk); String name(); } diff --git a/common/src/main/java/com/dfsek/terra/api/structures/parser/Parser.java b/common/src/main/java/com/dfsek/terra/api/structures/parser/Parser.java index 399212b99..7a991d226 100644 --- a/common/src/main/java/com/dfsek/terra/api/structures/parser/Parser.java +++ b/common/src/main/java/com/dfsek/terra/api/structures/parser/Parser.java @@ -15,21 +15,21 @@ import java.util.stream.Collectors; public class Parser { private final String data; - private final Map> functions = new HashMap<>(); + private final Map>> functions = new HashMap<>(); Set allowedArguments = Sets.newHashSet(Token.Type.STRING, Token.Type.NUMBER, Token.Type.IDENTIFIER); public Parser(String data) { this.data = data; } - public Parser addFunction(String name, FunctionBuilder functionBuilder) { + public Parser addFunction(String name, FunctionBuilder> functionBuilder) { functions.put(name, functionBuilder); return this; } - public List parse() throws ParseException { + public List> parse() throws ParseException { Tokenizer tokenizer = new Tokenizer(data); - List builtFunctions = new GlueList<>(); + List> builtFunctions = new GlueList<>(); List functionBuilder = new GlueList<>(); Token token = null; while(tokenizer.hasNext()) { @@ -46,32 +46,15 @@ public class Parser { checkType(functionBuilder.remove(0), Token.Type.BODY_BEGIN); // Second is body begin - boolean expectingSeparator = false; - List args = new GlueList<>(); - - while(!functionBuilder.get(0).getType().equals(Token.Type.BODY_END)) { - Token current = functionBuilder.remove(0); - if(expectingSeparator) { - checkType(current, Token.Type.SEPARATOR); - expectingSeparator = false; - } else { - if(!allowedArguments.contains(current.getType())) - throw new ParseException("Token type " + current.getType() + " not allowed in arguments: " + current.getStart()); - args.add(current); - expectingSeparator = true; - } - } + List args = getArgs(functionBuilder); // Extract arguments, consume the rest. functionBuilder.remove(0); // Remove body end - for(Token t : args) System.out.println("TOKEN: " + t); - checkType(functionBuilder.remove(0), Token.Type.STATEMENT_END); List arg = args.stream().map(Token::getContent).collect(Collectors.toList()); - for(String s : arg) System.out.println("ARG: " + s); FunctionBuilder builder = functions.get(identifier.getContent()); if(arg.size() != builder.getArguments().size()) throw new ParseException("Expected " + builder.getArguments().size() + " arguments, found " + arg.size() + ": " + identifier.getStart()); @@ -89,6 +72,25 @@ public class Parser { return builtFunctions; } + private List getArgs(List functionBuilder) throws ParseException { + List args = new GlueList<>(); + boolean expectingSeparator = false; + + while(!functionBuilder.get(0).getType().equals(Token.Type.BODY_END)) { + Token current = functionBuilder.remove(0); + if(expectingSeparator) { + checkType(current, Token.Type.SEPARATOR); + expectingSeparator = false; + } else { + if(!allowedArguments.contains(current.getType())) + throw new ParseException("Token type " + current.getType() + " not allowed in arguments: " + current.getStart()); + args.add(current); + expectingSeparator = true; + } + } + return args; + } + private void checkType(Token token, Token.Type expected) throws ParseException { if(!token.getType().equals(expected)) throw new ParseException("Expected " + expected + " but found " + token.getType() + ": " + token.getStart()); diff --git a/common/src/main/java/com/dfsek/terra/api/structures/parser/lang/Block.java b/common/src/main/java/com/dfsek/terra/api/structures/parser/lang/Block.java new file mode 100644 index 000000000..385264f2b --- /dev/null +++ b/common/src/main/java/com/dfsek/terra/api/structures/parser/lang/Block.java @@ -0,0 +1,4 @@ +package com.dfsek.terra.api.structures.parser.lang; + +public class Block { +} diff --git a/common/src/main/java/com/dfsek/terra/api/structures/parser/lang/Expression.java b/common/src/main/java/com/dfsek/terra/api/structures/parser/lang/Expression.java new file mode 100644 index 000000000..b2c539e3e --- /dev/null +++ b/common/src/main/java/com/dfsek/terra/api/structures/parser/lang/Expression.java @@ -0,0 +1,4 @@ +package com.dfsek.terra.api.structures.parser.lang; + +public class Expression { +} diff --git a/common/src/main/java/com/dfsek/terra/api/structures/parser/lang/Item.java b/common/src/main/java/com/dfsek/terra/api/structures/parser/lang/Item.java new file mode 100644 index 000000000..0940ba797 --- /dev/null +++ b/common/src/main/java/com/dfsek/terra/api/structures/parser/lang/Item.java @@ -0,0 +1,4 @@ +package com.dfsek.terra.api.structures.parser.lang; + +public interface Item { +} diff --git a/common/src/main/java/com/dfsek/terra/api/structures/tokenizer/Token.java b/common/src/main/java/com/dfsek/terra/api/structures/tokenizer/Token.java index e7502195a..dd6d88373 100644 --- a/common/src/main/java/com/dfsek/terra/api/structures/tokenizer/Token.java +++ b/common/src/main/java/com/dfsek/terra/api/structures/tokenizer/Token.java @@ -29,6 +29,6 @@ public class Token { } public enum Type { - IDENTIFIER, NUMBER, STRING, BOOLEAN, BODY_BEGIN, BODY_END, STATEMENT_END, SEPARATOR + IDENTIFIER, NUMBER, STRING, BOOLEAN, BODY_BEGIN, BODY_END, STATEMENT_END, SEPARATOR, BLOCK_BEGIN, BLOCK_END } } diff --git a/common/src/main/java/com/dfsek/terra/api/structures/tokenizer/Tokenizer.java b/common/src/main/java/com/dfsek/terra/api/structures/tokenizer/Tokenizer.java index f2bb0da99..9eb73ffc3 100644 --- a/common/src/main/java/com/dfsek/terra/api/structures/tokenizer/Tokenizer.java +++ b/common/src/main/java/com/dfsek/terra/api/structures/tokenizer/Tokenizer.java @@ -11,7 +11,8 @@ import java.util.Set; public class Tokenizer { private final Lookahead reader; - private final Set syntaxSignificant = Sets.newHashSet(';', '(', ')', '"', '[', ']', ','); + private final Set syntaxSignificant = Sets.newHashSet(';', '(', ')', '"', ',', '\\', // Currently used chars + '{', '}'); // Reserved chars public Tokenizer(String data) { @@ -42,7 +43,13 @@ public class Tokenizer { if(reader.current().is('"')) { reader.consume(); // Consume first quote StringBuilder string = new StringBuilder(); - while(!reader.current().is('"')) { + boolean ignoreNext = false; + while((!reader.current().is('"')) || ignoreNext) { + if(reader.current().is('\\') && !ignoreNext) { + ignoreNext = true; + reader.consume(); + continue; + } else ignoreNext = false; if(reader.current().isEOF()) throw new FormatException("No end of string literal found. " + reader.getLine() + ":" + reader.getIndex()); string.append(reader.consume()); @@ -59,6 +66,10 @@ public class Tokenizer { return new Token(reader.consume().toString(), Token.Type.STATEMENT_END, new Position(reader.getLine(), reader.getIndex())); if(reader.current().is(',')) return new Token(reader.consume().toString(), Token.Type.SEPARATOR, new Position(reader.getLine(), reader.getIndex())); + if(reader.current().is('{')) + return new Token(reader.consume().toString(), Token.Type.BLOCK_BEGIN, new Position(reader.getLine(), reader.getIndex())); + if(reader.current().is('}')) + return new Token(reader.consume().toString(), Token.Type.BLOCK_END, new Position(reader.getLine(), reader.getIndex())); StringBuilder token = new StringBuilder(); while(!reader.current().isEOF() && !isSyntaxSignificant(reader.current().getCharacter())) { diff --git a/common/src/test/java/structure/ParserTest.java b/common/src/test/java/structure/ParserTest.java index dbc98b64c..24d57a8b5 100644 --- a/common/src/test/java/structure/ParserTest.java +++ b/common/src/test/java/structure/ParserTest.java @@ -31,14 +31,15 @@ public class ParserTest { } }); - List functions = parser.parse(); + long l = System.nanoTime(); + List> functions = parser.parse(); + long t = System.nanoTime() - l; + System.out.println("Took " + (double) t / 1000000); - for(Function f : functions) { - System.out.println(f); - } + for(Function f : functions) System.out.println(f); } - private static class Test1 implements Function { + private static class Test1 implements Function { private final String a; private final double b; @@ -56,18 +57,23 @@ public class ParserTest { } @Override - public void apply(Location location) { - + public Void apply(Location location) { + return null; } @Override - public void apply(Location location, Chunk chunk) { - + public Void apply(Location location, Chunk chunk) { + return null; } @Override public String name() { return null; } + + @Override + public String toString() { + return "string: " + a + ", double: " + b; + } } } diff --git a/common/src/test/resources/test.tesf b/common/src/test/resources/test.tesf index f1cc70dca..7197182b4 100644 --- a/common/src/test/resources/test.tesf +++ b/common/src/test/resources/test.tesf @@ -1,4 +1,2 @@ test("hello", 1); - -test("ghgjhgjhgj", 3.4); - +test("ghgj{}()\"\\hgjhgj", 3.4);