diff --git a/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Lookahead.java b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Lookahead.java index 6e88fd046..5e41bfd37 100644 --- a/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Lookahead.java +++ b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Lookahead.java @@ -6,6 +6,9 @@ import java.io.IOException; import java.io.Reader; import java.util.List; +/** + * Stream-like data structure that allows viewing future elements without consuming current. + */ public class Lookahead { private final List buffer = new GlueList<>(); private final Reader input; @@ -17,10 +20,20 @@ public class Lookahead { this.input = r; } + /** + * Get the current character without consuming it. + * + * @return + */ public Char current() { return next(0); } + /** + * Consume and return one character. + * + * @return Character that was consumed. + */ public Char consume() { Char consumed = current(); consume(1); @@ -28,7 +41,7 @@ public class Lookahead { } /** - * Fetch and consume the next character. + * Fetch the next character. * * @return Next character */ @@ -69,6 +82,11 @@ public class Lookahead { } else return buffer.get(ahead); } + /** + * Consume an amount of characters + * + * @param amount Number of characters to consume + */ public void consume(int amount) { if(amount < 0) throw new IllegalArgumentException(); while(amount-- > 0) { diff --git a/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Position.java b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Position.java new file mode 100644 index 000000000..2d1f70257 --- /dev/null +++ b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Position.java @@ -0,0 +1,11 @@ +package com.dfsek.terra.structure.v2.tokenizer; + +public class Position { + private final int line; + private final int index; + + public Position(int line, int index) { + this.line = line; + this.index = index; + } +} diff --git a/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Token.java b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Token.java new file mode 100644 index 000000000..5fca5ed00 --- /dev/null +++ b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Token.java @@ -0,0 +1,4 @@ +package com.dfsek.terra.structure.v2.tokenizer; + +public class Token { +} diff --git a/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Tokenizer.java b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Tokenizer.java index 95c9df378..ca9061cf3 100644 --- a/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Tokenizer.java +++ b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Tokenizer.java @@ -7,5 +7,6 @@ public class Tokenizer { public Tokenizer(String data) { reader = new Lookahead(new StringReader(data)); + } } diff --git a/src/main/java/com/dfsek/terra/structure/v2/tokenizer/exceptions/TokenizerException.java b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/exceptions/TokenizerException.java new file mode 100644 index 000000000..e3f0e9c0b --- /dev/null +++ b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/exceptions/TokenizerException.java @@ -0,0 +1,4 @@ +package com.dfsek.terra.structure.v2.tokenizer.exceptions; + +public abstract class TokenizerException extends Exception { +}