Tokenizer stuff

src/main/java/com/dfsek/terra/structure/v2/tokenizer/Lookahead.java

@@ -6,6 +6,9 @@ import java.io.IOException;
 import java.io.Reader;
 import java.util.List;
 
+/**
+ * Stream-like data structure that allows viewing future elements without consuming current.
+ */
 public class Lookahead {
     private final List<Char> buffer = new GlueList<>();
     private final Reader input;
@@ -17,10 +20,20 @@ public class Lookahead {
         this.input = r;
     }
 
+    /**
+     * Get the current character without consuming it.
+     *
+     * @return
+     */
     public Char current() {
         return next(0);
     }
 
+    /**
+     * Consume and return one character.
+     *
+     * @return Character that was consumed.
+     */
     public Char consume() {
         Char consumed = current();
         consume(1);
@@ -28,7 +41,7 @@ public class Lookahead {
     }
 
     /**
-     * Fetch and consume the next character.
+     * Fetch the next character.
      *
      * @return Next character
      */
@@ -69,6 +82,11 @@ public class Lookahead {
         } else return buffer.get(ahead);
     }
 
+    /**
+     * Consume an amount of characters
+     *
+     * @param amount Number of characters to consume
+     */
     public void consume(int amount) {
         if(amount < 0) throw new IllegalArgumentException();
         while(amount-- > 0) {

src/main/java/com/dfsek/terra/structure/v2/tokenizer/Position.java

@@ -0,0 +1,11 @@
+package com.dfsek.terra.structure.v2.tokenizer;
+
+public class Position {
+    private final int line;
+    private final int index;
+
+    public Position(int line, int index) {
+        this.line = line;
+        this.index = index;
+    }
+}

src/main/java/com/dfsek/terra/structure/v2/tokenizer/Token.java

@@ -0,0 +1,4 @@
+package com.dfsek.terra.structure.v2.tokenizer;
+
+public class Token {
+}

src/main/java/com/dfsek/terra/structure/v2/tokenizer/Tokenizer.java

@@ -7,5 +7,6 @@ public class Tokenizer {
 
     public Tokenizer(String data) {
         reader = new Lookahead(new StringReader(data));
+
     }
 }

src/main/java/com/dfsek/terra/structure/v2/tokenizer/exceptions/TokenizerException.java

@@ -0,0 +1,4 @@
+package com.dfsek.terra.structure.v2.tokenizer.exceptions;
+
+public abstract class TokenizerException extends Exception {
+}