From 8b9899bcf338d4aca95419dfd1006f7f822a16c9 Mon Sep 17 00:00:00 2001
From: Johnny Richard <johnny@johnnyrichard.com>
Date: Tue, 18 Apr 2023 18:33:36 +0200
Subject: lexer: Add tokenizer for OP and UNKNOWN tokens

We want to tokenizer arithmetic expressions.

We are handling exceptional cases with UNKNOWN token.

Co-authored-by: Carlos Maniero <carlosmaniero@gmail.com>
Signed-off-by: Johnny Richard <johnny@johnnyrichard.com>
---
 src/lexer.c | 21 ++++++++++++++++++---
 src/lexer.h |  4 +++-
 2 files changed, 21 insertions(+), 4 deletions(-)

(limited to 'src')

diff --git a/src/lexer.c b/src/lexer.c
index bc40efc..6c38f6e 100644
--- a/src/lexer.c
+++ b/src/lexer.c
@@ -18,6 +18,7 @@
 #include <assert.h>
 #include <ctype.h>
 #include <errno.h>
+#include <stdbool.h>
 #include <stdio.h>
 #include <string.h>
 #include "lexer.h"
@@ -39,7 +40,7 @@ static void
 lexer_define_literal_token_props(lexer_t *lexer, token_t *token, token_kind_t kind)
 {
     token->kind = kind;
-    token->value = string_view_new(lexer->src, 1);
+    token->value = string_view_new(lexer->src + lexer->cur, 1);
     token->filepath = lexer->filepath;
     token->row = lexer->row;
     token->col = lexer->cur - lexer->bol;
@@ -140,9 +141,19 @@ lexer_next_token(lexer_t *lexer, token_t *token)
       lexer_drop_char(lexer);
       return;
     }
+
+    if (lexer_current_char(lexer) == '+'
+        || lexer_current_char(lexer) == '-'
+        || lexer_current_char(lexer) == '*'
+        || lexer_current_char(lexer) == '/') {
+      lexer_define_literal_token_props(lexer, token, TOKEN_OP);
+      lexer_drop_char(lexer);
+      return;
+    }
   }
 
-  token->kind = TOKEN_EOF;
+  lexer_define_literal_token_props(lexer, token, TOKEN_UNKNOWN);
+  lexer_drop_char(lexer);
   return;
 }
 
@@ -224,10 +235,14 @@ token_kind_to_str(token_kind_t kind)
       return "}";
     case TOKEN_NUMBER:
       return "TOKEN_NUMBER";
+    case TOKEN_OP:
+      return "TOKEN_OP";
     case TOKEN_EOF:
       return "TOKEN_EOF";
+    case TOKEN_UNKNOWN:
+      return "TOKEN_UNKNOWN";
     default:
-      return "UNKNOW_TOKEN";
+      assert(false && "unreachable");
   }
 }
 
diff --git a/src/lexer.h b/src/lexer.h
index a091fb7..ae8ab60 100644
--- a/src/lexer.h
+++ b/src/lexer.h
@@ -31,7 +31,9 @@ typedef enum {
     TOKEN_OCURLY,
     TOKEN_CCURLY,
     TOKEN_NUMBER,
-    TOKEN_EOF
+    TOKEN_OP,
+    TOKEN_EOF,
+    TOKEN_UNKNOWN
 } token_kind_t;
 
 typedef struct token_t {
-- 
cgit v1.2.3