feat: tokenizer scaffolding

2026-05-01 17:48:57 +00:00 · 2026-04-16 17:20:19 +01:00
parent 4fd1edaaf7
commit a54831a979
2 changed files with 42 additions and 0 deletions
@@ -0,0 +1,17 @@
+package tokenizer
+
+type TokenType int
+
+const (
+	KEYWORD TokenType = iota
+	SYMBOL
+	INT_CONST
+	STR_CONST
+	IDENTIFIER
+)
+
+type Token struct {
+	Value string
+	Type TokenType
+}
+