mirrors/Terra
1
0
Fork 0
mirror of https://github.com/PolyhedralDev/Terra.git synced 2026-04-04 14:56:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

working tokenizer

Browse Source
This commit is contained in:
dfsek
2020-12-19 01:50:56 -07:00
parent 1ce884d1c7
commit 29e2746e72
29 changed files with 402 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common/src/test/java/structure/LookaheadTest.java
View File

@@ -1,6 +1,6 @@
package structure;
import com.dfsek.terra.structure.v2.tokenizer.Lookahead;
import com.dfsek.terra.api.structures.tokenizer.Lookahead;
import org.junit.jupiter.api.Test;
import java.io.StringReader;

23
common/src/test/java/structure/TokenizerTest.java Normal file
View File

@@ -0,0 +1,23 @@
package structure;
import com.dfsek.terra.api.structures.tokenizer.Token;
import com.dfsek.terra.api.structures.tokenizer.Tokenizer;
import com.dfsek.terra.api.structures.tokenizer.exceptions.TokenizerException;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Test;
import java.io.IOException;
public class TokenizerTest {
@Test
public void tokens() throws IOException, TokenizerException {
Tokenizer tokenizer = new Tokenizer(IOUtils.toString(getClass().getResourceAsStream("/test.tesf")));
for(int i = 0; i < 100; i++) {
Token t = tokenizer.fetch();
if(t == null) break;
System.out.println(t);
}
}
}