lexer: Peek next token

The only way to get the next token was by consuming it. So then, our
parser starts to become hard to understand, once sometimes we just
want to take a look on the next token to understand what should be the
next kind of expression.

This commit introduces a new function that will help us to improve our
parser implementation.

Signed-off-by: Johnny Richard <johnny@johnnyrichard.com>
Reviewed-by: Carlos Maniero <carlos@maniero.me>

3 files changed, 39 insertions, 3 deletions

diff --git a/src/lexer.c b/src/lexer.c
index 3730620..f937170 100644
--- a/src/lexer.c
+++ b/src/lexer.c
@@ -218,6 +218,20 @@ lexer_load_file_contents(lexer_t *lexer)
 }
 
 void
+lexer_peek_next_token(lexer_t *lexer, token_t *token)
+{
+  uint32_t cur = lexer->cur;
+  uint32_t row = lexer->row;
+  uint32_t bol = lexer->bol;
+
+  lexer_next_token(lexer, token);
+
+  lexer->cur = cur;
+  lexer->row = row;
+  lexer->bol = bol;
+}
+
+void
 lexer_step_back_to(lexer_t *lexer, token_t *token)
 {
   lexer->cur = token->bol + token->col;
diff --git a/src/lexer.h b/src/lexer.h
index d4e84e1..0b9f2ad 100644
--- a/src/lexer.h
+++ b/src/lexer.h
@@ -85,6 +85,9 @@ lexer_drop_char(lexer_t *lexer);
 void
 lexer_step_back_to(lexer_t *lexer, token_t *token);
 
+void
+lexer_peek_next_token(lexer_t *lexer, token_t *token);
+
 char *
 token_kind_to_str(token_kind_t kind);
 
diff --git a/test/lexer_test.c b/test/lexer_test.c
index bbb6784..5326f5c 100644
--- a/test/lexer_test.c
+++ b/test/lexer_test.c
@@ -19,14 +19,14 @@
 #include "munit.h"
 
 void
-make_lexer_from_static_src(lexer_t *lexer, char *src, int srclen)
+make_lexer_from_static_src(lexer_t *lexer, char *src)
 {
   lexer->srclen = 0;
   lexer->cur = 0;
   lexer->row = 0;
   lexer->bol = 0;
   lexer->src = src;
-  lexer->srclen = srclen;
+  lexer->srclen = strlen(src);
 }
 
 void
@@ -34,7 +34,7 @@ assert_token_at(char *source, int token_index, token_kind_t expected_kind, char
 {
   lexer_t lexer;
   token_t token;
-  make_lexer_from_static_src(&lexer, source, strlen(source));
+  make_lexer_from_static_src(&lexer, source);
 
   while (token_index > 0) {
     lexer_next_token(&lexer, &token);
@@ -86,10 +86,29 @@ test_tokenize_unknown(const MunitParameter params[], void *user_data_or_fixture)
   return MUNIT_OK;
 }
 
+static MunitResult
+test_peek_next_token(const MunitParameter params[], void *user_data_or_fixture)
+{
+  lexer_t lexer;
+  make_lexer_from_static_src(&lexer, "()");
+
+  token_t peek_token;
+  lexer_peek_next_token(&lexer, &peek_token);
+
+  token_t next_token;
+  lexer_next_token(&lexer, &next_token);
+
+  assert_int(peek_token.kind, ==, next_token.kind);
+  assert_char((char)peek_token.value.str[0], ==, (char)next_token.value.str[0]);
+
+  return MUNIT_OK;
+}
+
 static MunitTest tests[] = {
   { "/test_tokenize_digit", test_tokenize_number, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL },
   { "/test_tokenize_op", test_tokenize_op, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL },
   { "/test_tokenize_unknown", test_tokenize_unknown, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL },
+  { "/test_peek_next_token", test_peek_next_token, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL },
   { NULL, NULL, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL }
 };