lexer: Split operation tokens into their own token

The +, -, *, and / tokens used to be TOKEN_OP, but the TOKEN_OP has been
removed and a token for each operation has been introduced. Python's
token names were followed: https://docs.python.org/3/library/token.html

Signed-off-by: Carlos Maniero <carlosmaniero@gmail.com>
Reviewed-by: Johnny Richard <johnny@johnnyrichar.com>

1 files changed, 36 insertions, 5 deletions

diff --git a/src/lexer.c b/src/lexer.c
index bbf29fc..e5f232b 100644
--- a/src/lexer.c
+++ b/src/lexer.c
@@ -151,9 +151,32 @@ lexer_next_token(lexer_t *lexer, token_t *token)
       return;
     }
 
-    if (lexer_current_char(lexer) == '+' || lexer_current_char(lexer) == '-' || lexer_current_char(lexer) == '*' ||
-        lexer_current_char(lexer) == '/' || lexer_current_char(lexer) == '=') {
-      lexer_define_literal_token_props(lexer, token, TOKEN_OP);
+    if (lexer_current_char(lexer) == '+') {
+      lexer_define_literal_token_props(lexer, token, TOKEN_PLUS);
+      lexer_drop_char(lexer);
+      return;
+    }
+
+    if (lexer_current_char(lexer) == '-') {
+      lexer_define_literal_token_props(lexer, token, TOKEN_MINUS);
+      lexer_drop_char(lexer);
+      return;
+    }
+
+    if (lexer_current_char(lexer) == '*') {
+      lexer_define_literal_token_props(lexer, token, TOKEN_STAR);
+      lexer_drop_char(lexer);
+      return;
+    }
+
+    if (lexer_current_char(lexer) == '/') {
+      lexer_define_literal_token_props(lexer, token, TOKEN_SLASH);
+      lexer_drop_char(lexer);
+      return;
+    }
+
+    if (lexer_current_char(lexer) == '=') {
+      lexer_define_literal_token_props(lexer, token, TOKEN_EQUAL);
       lexer_drop_char(lexer);
       return;
     }
@@ -248,8 +271,16 @@ token_kind_to_str(token_kind_t kind)
       return "}";
     case TOKEN_NUMBER:
       return "TOKEN_NUMBER";
-    case TOKEN_OP:
-      return "TOKEN_OP";
+    case TOKEN_PLUS:
+      return "+";
+    case TOKEN_MINUS:
+      return "-";
+    case TOKEN_STAR:
+      return "*";
+    case TOKEN_SLASH:
+      return "/";
+    case TOKEN_EQUAL:
+      return "=";
     case TOKEN_EOF:
       return "TOKEN_EOF";
     case TOKEN_UNKNOWN: