wip: extracting tokens

internal/tokenizer.go

@@ -1,17 +0,0 @@
-package compiler
-
-type TokenType int
-
-const (
-	KEYWORD TokenType = iota
-	SYMBOL
-	INT_CONST
-	STR_CONST
-	IDENTIFIER
-)
-
-type Token struct {
-	Value string
-	Type TokenType
-}
-

internal/tokenizer/tokenizer.go

@@ -0,0 +1,84 @@
+package tokenizer
+
+import (
+	"fmt"
+	"regexp"
+)
+
+type TokenType int
+
+const (
+	KEYWORD TokenType = iota
+	SYMBOL
+	INT_CONST
+	STR_CONST
+	IDENTIFIER
+)
+
+var KEYWORDS = [...]string{
+	"class",
+	"constructor",
+	"function",
+	"method",
+	"field",
+	"static",
+	"var",
+	"int",
+	"char",
+	"boolean",
+	"void",
+	"true",
+	"false",
+	"null",
+	"this",
+	"let",
+	"do",
+	"if",
+	"else",
+	"while",
+	"return",
+}
+
+var SYMBOLS = [...]string{
+	"{",
+	"}",
+	"(",
+	")",
+	"[",
+	"]",
+	".",
+	",",
+	";",
+	"+",
+	"-",
+	"*",
+	"/",
+	"&",
+	"|",
+	"<",
+	">",
+	"=",
+	"~",
+}
+
+type Token struct {
+	Value string
+	Type  TokenType
+}
+
+func ExtractTokens(tokens *[]Token, source []byte) error {
+	i := 0
+
+	for i < len(source) {
+		t := string(source[i])
+
+		if match, _ := regexp.MatchString("^[[:space:]]$", t); match {
+			i++
+		} else {
+			fmt.Println(t)
+			i++
+		}
+	}
+
+	return nil
+}

main.go

@@ -6,16 +6,25 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+
+	"github.com/hazemKrimi/jack-compiler/internal/tokenizer"
 )
 
 func process(inputPath string) error {
-	outputPath := strings.Replace(inputPath, ".jack", ".xml", 1)
 	source, err := os.ReadFile(inputPath)
 
 	if err != nil {
 		return err
 	}
 
+	tokens := make([]tokenizer.Token, 0, 1000)
+
+	if err := tokenizer.ExtractTokens(&tokens, source); err != nil {
+		return err
+	}
+
+	outputPath := strings.Replace(inputPath, ".jack", ".xml", 1)
+
 	if err := os.WriteFile(outputPath, source, 0644); err != nil {
 		return err
 	}