From 8b9899bcf338d4aca95419dfd1006f7f822a16c9 Mon Sep 17 00:00:00 2001
From: Johnny Richard <johnny@johnnyrichard.com>
Date: Tue, 18 Apr 2023 18:33:36 +0200
Subject: lexer: Add tokenizer for OP and UNKNOWN tokens

We want to tokenizer arithmetic expressions.

We are handling exceptional cases with UNKNOWN token.

Co-authored-by: Carlos Maniero <carlosmaniero@gmail.com>
Signed-off-by: Johnny Richard <johnny@johnnyrichard.com>
---
 src/lexer.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'src/lexer.h')

diff --git a/src/lexer.h b/src/lexer.h
index a091fb7..ae8ab60 100644
--- a/src/lexer.h
+++ b/src/lexer.h
@@ -31,7 +31,9 @@ typedef enum {
     TOKEN_OCURLY,
     TOKEN_CCURLY,
     TOKEN_NUMBER,
-    TOKEN_EOF
+    TOKEN_OP,
+    TOKEN_EOF,
+    TOKEN_UNKNOWN
 } token_kind_t;
 
 typedef struct token_t {
-- 
cgit v1.2.3