From 8f58ba17a8f1e487f0a0bccabbd6704eadcdf7f7 Mon Sep 17 00:00:00 2001 From: dfsek Date: Thu, 10 Dec 2020 01:26:39 -0700 Subject: [PATCH] Begin work on tokenizer --- .../dfsek/terra/structure/v2/Function.java | 5 ++ .../com/dfsek/terra/structure/v2/Parser.java | 4 + .../terra/structure/v2/tokenizer/Char.java | 35 ++++++++ .../structure/v2/tokenizer/Lookahead.java | 83 +++++++++++++++++++ .../structure/v2/tokenizer/Tokenizer.java | 11 +++ .../terra/structure/v2/tokenizer/Tokens.java | 5 ++ src/test/java/structure/LookaheadTest.java | 20 +++++ 7 files changed, 163 insertions(+) create mode 100644 src/main/java/com/dfsek/terra/structure/v2/Function.java create mode 100644 src/main/java/com/dfsek/terra/structure/v2/Parser.java create mode 100644 src/main/java/com/dfsek/terra/structure/v2/tokenizer/Char.java create mode 100644 src/main/java/com/dfsek/terra/structure/v2/tokenizer/Lookahead.java create mode 100644 src/main/java/com/dfsek/terra/structure/v2/tokenizer/Tokenizer.java create mode 100644 src/main/java/com/dfsek/terra/structure/v2/tokenizer/Tokens.java create mode 100644 src/test/java/structure/LookaheadTest.java diff --git a/src/main/java/com/dfsek/terra/structure/v2/Function.java b/src/main/java/com/dfsek/terra/structure/v2/Function.java new file mode 100644 index 000000000..71c008edc --- /dev/null +++ b/src/main/java/com/dfsek/terra/structure/v2/Function.java @@ -0,0 +1,5 @@ +package com.dfsek.terra.structure.v2; + +public interface Function { + void apply(); +} diff --git a/src/main/java/com/dfsek/terra/structure/v2/Parser.java b/src/main/java/com/dfsek/terra/structure/v2/Parser.java new file mode 100644 index 000000000..82ef17732 --- /dev/null +++ b/src/main/java/com/dfsek/terra/structure/v2/Parser.java @@ -0,0 +1,4 @@ +package com.dfsek.terra.structure.v2; + +public class Parser { +} diff --git a/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Char.java b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Char.java new file mode 100644 index 000000000..82b8c68a2 --- /dev/null +++ b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Char.java @@ -0,0 +1,35 @@ +package com.dfsek.terra.structure.v2.tokenizer; + +public class Char { + private final char character; + private final int index; + private final int line; + + + public Char(char character, int index, int line) { + this.character = character; + this.index = index; + this.line = line; + } + + public char getCharacter() { + return character; + } + + public int getIndex() { + return index; + } + + public int getLine() { + return line; + } + + public boolean is(char... tests) { + for(char test : tests) { + if(test == character && test != '\0') { + return true; + } + } + return false; + } +} diff --git a/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Lookahead.java b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Lookahead.java new file mode 100644 index 000000000..6e88fd046 --- /dev/null +++ b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Lookahead.java @@ -0,0 +1,83 @@ +package com.dfsek.terra.structure.v2.tokenizer; + +import org.polydev.gaea.util.GlueList; + +import java.io.IOException; +import java.io.Reader; +import java.util.List; + +public class Lookahead { + private final List buffer = new GlueList<>(); + private final Reader input; + private int index = 0; + private int line = 0; + private boolean end = false; + + public Lookahead(Reader r) { + this.input = r; + } + + public Char current() { + return next(0); + } + + public Char consume() { + Char consumed = current(); + consume(1); + return consumed; + } + + /** + * Fetch and consume the next character. + * + * @return Next character + */ + private Char fetch() { + try { + int c = input.read(); + if(c == -1) return null; + if(c == '\n') { + line++; + index = 0; + } + index++; + return new Char((char) c, line, index); + } catch(IOException e) { + e.printStackTrace(); + return null; + } + } + + /** + * Fetch a future character without consuming it. + * + * @param ahead Distance ahead to peek + * @return Character + */ + public Char next(int ahead) { + if(ahead < 0) throw new IllegalArgumentException(); + + while(buffer.size() <= ahead && !end) { + Char item = fetch(); + if(item != null) { + buffer.add(item); + } else end = true; + } + + if(ahead >= buffer.size()) { + return null; + } else return buffer.get(ahead); + } + + public void consume(int amount) { + if(amount < 0) throw new IllegalArgumentException(); + while(amount-- > 0) { + if(!buffer.isEmpty()) buffer.remove(0); // Remove top item from buffer. + else { + if(end) return; + Char item = fetch(); + if(item == null) end = true; + } + } + } +} diff --git a/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Tokenizer.java b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Tokenizer.java new file mode 100644 index 000000000..95c9df378 --- /dev/null +++ b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Tokenizer.java @@ -0,0 +1,11 @@ +package com.dfsek.terra.structure.v2.tokenizer; + +import java.io.StringReader; + +public class Tokenizer { + private final Lookahead reader; + + public Tokenizer(String data) { + reader = new Lookahead(new StringReader(data)); + } +} diff --git a/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Tokens.java b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Tokens.java new file mode 100644 index 000000000..92b4e8566 --- /dev/null +++ b/src/main/java/com/dfsek/terra/structure/v2/tokenizer/Tokens.java @@ -0,0 +1,5 @@ +package com.dfsek.terra.structure.v2.tokenizer; + +public enum Tokens { + FUNCTION, SEPARATOR, ARGUMENT +} diff --git a/src/test/java/structure/LookaheadTest.java b/src/test/java/structure/LookaheadTest.java new file mode 100644 index 000000000..a3c84d493 --- /dev/null +++ b/src/test/java/structure/LookaheadTest.java @@ -0,0 +1,20 @@ +package structure; + +import com.dfsek.terra.structure.v2.tokenizer.Lookahead; +import org.junit.jupiter.api.Test; + +import java.io.StringReader; + +public class LookaheadTest { + @Test + public void lookahead() { + Lookahead lookahead = new Lookahead(new StringReader("Test string...")); + + for(int i = 0; lookahead.next(i) != null; i++) { + System.out.print(lookahead.next(i).getCharacter()); + } + while(lookahead.next(0) != null) { + System.out.print(lookahead.consume().getCharacter()); + } + } +}