lexer: Peek next token

The only way to get the next token was by consuming it. So then, our
parser starts to become hard to understand, once sometimes we just
want to take a look on the next token to understand what should be the
next kind of expression.

This commit introduces a new function that will help us to improve our
parser implementation.

Signed-off-by: Johnny Richard <johnny@johnnyrichard.com>
Reviewed-by: Carlos Maniero <carlos@maniero.me>

1 files changed, 22 insertions, 3 deletions

diff --git a/test/lexer_test.c b/test/lexer_test.c
index bbb6784..5326f5c 100644
--- a/test/lexer_test.c
+++ b/test/lexer_test.c
@@ -19,14 +19,14 @@
 #include "munit.h"
 
 void
-make_lexer_from_static_src(lexer_t *lexer, char *src, int srclen)
+make_lexer_from_static_src(lexer_t *lexer, char *src)
 {
   lexer->srclen = 0;
   lexer->cur = 0;
   lexer->row = 0;
   lexer->bol = 0;
   lexer->src = src;
-  lexer->srclen = srclen;
+  lexer->srclen = strlen(src);
 }
 
 void
@@ -34,7 +34,7 @@ assert_token_at(char *source, int token_index, token_kind_t expected_kind, char
 {
   lexer_t lexer;
   token_t token;
-  make_lexer_from_static_src(&lexer, source, strlen(source));
+  make_lexer_from_static_src(&lexer, source);
 
   while (token_index > 0) {
     lexer_next_token(&lexer, &token);
@@ -86,10 +86,29 @@ test_tokenize_unknown(const MunitParameter params[], void *user_data_or_fixture)
   return MUNIT_OK;
 }
 
+static MunitResult
+test_peek_next_token(const MunitParameter params[], void *user_data_or_fixture)
+{
+  lexer_t lexer;
+  make_lexer_from_static_src(&lexer, "()");
+
+  token_t peek_token;
+  lexer_peek_next_token(&lexer, &peek_token);
+
+  token_t next_token;
+  lexer_next_token(&lexer, &next_token);
+
+  assert_int(peek_token.kind, ==, next_token.kind);
+  assert_char((char)peek_token.value.str[0], ==, (char)next_token.value.str[0]);
+
+  return MUNIT_OK;
+}
+
 static MunitTest tests[] = {
   { "/test_tokenize_digit", test_tokenize_number, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL },
   { "/test_tokenize_op", test_tokenize_op, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL },
   { "/test_tokenize_unknown", test_tokenize_unknown, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL },
+  { "/test_peek_next_token", test_peek_next_token, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL },
   { NULL, NULL, NULL, NULL, MUNIT_TEST_OPTION_NONE, NULL }
 };