working tokenizer

common/src/test/java/structure/LookaheadTest.java

@@ -1,6 +1,6 @@
 package structure;
 
-import com.dfsek.terra.structure.v2.tokenizer.Lookahead;
+import com.dfsek.terra.api.structures.tokenizer.Lookahead;
 import org.junit.jupiter.api.Test;
 
 import java.io.StringReader;

common/src/test/java/structure/TokenizerTest.java

@@ -0,0 +1,23 @@
+package structure;
+
+import com.dfsek.terra.api.structures.tokenizer.Token;
+import com.dfsek.terra.api.structures.tokenizer.Tokenizer;
+import com.dfsek.terra.api.structures.tokenizer.exceptions.TokenizerException;
+import org.apache.commons.io.IOUtils;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+
+public class TokenizerTest {
+    @Test
+    public void tokens() throws IOException, TokenizerException {
+        Tokenizer tokenizer = new Tokenizer(IOUtils.toString(getClass().getResourceAsStream("/test.tesf")));
+
+        for(int i = 0; i < 100; i++) {
+            Token t = tokenizer.fetch();
+            if(t == null) break;
+            System.out.println(t);
+
+        }
+    }
+}

common/src/test/resources/test.tesf

@@ -0,0 +1,18 @@
+FUNCTION(1, "hello");
+
+;
+)
+
+FUNCTION2(1.2, "he423llo");
+
+// comment
+
+FUNCTION3(3.4, "35234523452345");
+
+/*
+block comment
+fsfsdf
+gsdfgsdfg
+*/
+
+FUNCTION(1, "hello");