diff --git a/internal/tokenizer.go b/internal/tokenizer.go deleted file mode 100644 index 6bc80c9..0000000 --- a/internal/tokenizer.go +++ /dev/null @@ -1,17 +0,0 @@ -package compiler - -type TokenType int - -const ( - KEYWORD TokenType = iota - SYMBOL - INT_CONST - STR_CONST - IDENTIFIER -) - -type Token struct { - Value string - Type TokenType -} - diff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.go new file mode 100644 index 0000000..49c3f4e --- /dev/null +++ b/internal/tokenizer/tokenizer.go @@ -0,0 +1,84 @@ +package tokenizer + +import ( + "fmt" + "regexp" +) + +type TokenType int + +const ( + KEYWORD TokenType = iota + SYMBOL + INT_CONST + STR_CONST + IDENTIFIER +) + +var KEYWORDS = [...]string{ + "class", + "constructor", + "function", + "method", + "field", + "static", + "var", + "int", + "char", + "boolean", + "void", + "true", + "false", + "null", + "this", + "let", + "do", + "if", + "else", + "while", + "return", +} + +var SYMBOLS = [...]string{ + "{", + "}", + "(", + ")", + "[", + "]", + ".", + ",", + ";", + "+", + "-", + "*", + "/", + "&", + "|", + "<", + ">", + "=", + "~", +} + +type Token struct { + Value string + Type TokenType +} + +func ExtractTokens(tokens *[]Token, source []byte) error { + i := 0 + + for i < len(source) { + t := string(source[i]) + + if match, _ := regexp.MatchString("^[[:space:]]$", t); match { + i++ + } else { + fmt.Println(t) + i++ + } + } + + return nil +} diff --git a/main.go b/main.go index 40fbd77..246e870 100644 --- a/main.go +++ b/main.go @@ -6,16 +6,25 @@ import ( "path/filepath" "regexp" "strings" + + "github.com/hazemKrimi/jack-compiler/internal/tokenizer" ) func process(inputPath string) error { - outputPath := strings.Replace(inputPath, ".jack", ".xml", 1) source, err := os.ReadFile(inputPath) if err != nil { return err } + tokens := make([]tokenizer.Token, 0, 1000) + + if err := tokenizer.ExtractTokens(&tokens, source); err != nil { + return err + } + + outputPath := strings.Replace(inputPath, ".jack", ".xml", 1) + if err := os.WriteFile(outputPath, source, 0644); err != nil { return err }