lexer: Add tokenizer for OP and UNKNOWN tokens

We want to tokenizer arithmetic expressions.

We are handling exceptional cases with UNKNOWN token.

Co-authored-by: Carlos Maniero <carlosmaniero@gmail.com>
Signed-off-by: Johnny Richard <johnny@johnnyrichard.com>

1 files changed, 18 insertions, 3 deletions

diff --git a/src/lexer.c b/src/lexer.c
index bc40efc..6c38f6e 100644
--- a/src/lexer.c
+++ b/src/lexer.c
@@ -18,6 +18,7 @@
 #include <assert.h>
 #include <ctype.h>
 #include <errno.h>
+#include <stdbool.h>
 #include <stdio.h>
 #include <string.h>
 #include "lexer.h"
@@ -39,7 +40,7 @@ static void
 lexer_define_literal_token_props(lexer_t *lexer, token_t *token, token_kind_t kind)
 {
     token->kind = kind;
-    token->value = string_view_new(lexer->src, 1);
+    token->value = string_view_new(lexer->src + lexer->cur, 1);
     token->filepath = lexer->filepath;
     token->row = lexer->row;
     token->col = lexer->cur - lexer->bol;
@@ -140,9 +141,19 @@ lexer_next_token(lexer_t *lexer, token_t *token)
       lexer_drop_char(lexer);
       return;
     }
+
+    if (lexer_current_char(lexer) == '+'
+        || lexer_current_char(lexer) == '-'
+        || lexer_current_char(lexer) == '*'
+        || lexer_current_char(lexer) == '/') {
+      lexer_define_literal_token_props(lexer, token, TOKEN_OP);
+      lexer_drop_char(lexer);
+      return;
+    }
   }
 
-  token->kind = TOKEN_EOF;
+  lexer_define_literal_token_props(lexer, token, TOKEN_UNKNOWN);
+  lexer_drop_char(lexer);
   return;
 }
 
@@ -224,10 +235,14 @@ token_kind_to_str(token_kind_t kind)
       return "}";
     case TOKEN_NUMBER:
       return "TOKEN_NUMBER";
+    case TOKEN_OP:
+      return "TOKEN_OP";
     case TOKEN_EOF:
       return "TOKEN_EOF";
+    case TOKEN_UNKNOWN:
+      return "TOKEN_UNKNOWN";
     default:
-      return "UNKNOW_TOKEN";
+      assert(false && "unreachable");
   }
 }