add comment supress options for lexer

include/cherry/read.h

@@ -25,15 +25,19 @@
 
 struct CyArray;
 
+#define CY_DEFAULT 0
+#define CY_SUPRESS_COMMENTS 1
+
 struct org_cherry_context {
   const char* filename;
   const byte_t* begin;  
   const byte_t* src;
   struct CryArray* buffer;
+  cy_flags_t flags;
 };
 
 
-struct org_cherry_context*   org_cherry_context(const byte_t* source, const char* filename);
+struct org_cherry_context*   org_cherry_context(const byte_t* source, const char* filename, cy_flags_t flags);
 struct org_cherry_context*   org_cherry_context_repl(const byte_t* source);
 void                         org_cherry_context_free(struct org_cherry_context* context);
 

include/cherry/standard.h

@@ -35,4 +35,5 @@ typedef double      cy_float_t;
 typedef char        cy_boolean_t;
 typedef uint32_t    cy_unicode_t;
 
+typedef uint32_t    cy_flags_t;
 

source/read.c

@@ -27,13 +27,14 @@
 
 
 struct org_cherry_context*   
-org_cherry_context(const byte_t* source, const char* filename)
+org_cherry_context(const byte_t* source, const char* filename, cy_flags_t flags)
 {
 	struct org_cherry_context* context = malloc(sizeof(struct org_cherry_context));
 	context->filename = filename;
 	context->begin = source;
 	context->src = (byte_t*) source;
 	context->buffer = cry_array_new(sizeof(byte_t));
+	context->flags = flags;
 
 	return context;
 }
@@ -42,7 +43,7 @@ org_cherry_context(const byte_t* source, const char* filename)
 struct org_cherry_context*   
 org_cherry_context_repl(const byte_t* source)
 {
-	return org_cherry_context(source, 0);
+	return org_cherry_context(source, 0, CY_SUPRESS_COMMENTS);
 }
 
 
@@ -514,6 +515,8 @@ lex_comment(struct org_cherry_context* context)
 
 	cry_array_append(buffer, "\0", 1);
 
+	context->src = p;
+
 	return TOK_COMMENT;
 }
 
@@ -566,9 +569,9 @@ org_cherry_lex(struct org_cherry_context* context)
 {
 	assert(context != 0);
 
-	const byte_t* p = context->src;
-
 	while(TRUE) {
+		const byte_t* p = context->src;
+
 		cry_array_clear(context->buffer);
 
 		unicode_t ch = org_cherry_utf8_get(p);
@@ -595,6 +598,10 @@ org_cherry_lex(struct org_cherry_context* context)
 			case '\'':
 				LEX_RETURN(TOK_QUOTE);
 			case ';':
+				if(context->flags & CY_SUPRESS_COMMENTS) {
+					lex_comment(context);
+					continue;
+				}
 				return lex_comment(context);
 	    	case '0': case '1': case '2': case '3': case '4':
 			case '5': case '6': case '7': case '8': case '9':
@@ -655,6 +662,12 @@ org_cherry_token_length(struct org_cherry_context* context)
 }
 
 
+static struct org_cherry_value*
+org_cherry_read_pair(struct org_cherry_context* context)
+{
+
+}
+
 struct org_cherry_value*
 org_cherry_read(struct org_cherry_context* context)
 {
@@ -685,6 +698,7 @@ org_cherry_read(struct org_cherry_context* context)
             case TOK_SYMBOL:
                 return org_cherry_symbol_from_string(org_cherry_token_string(context));
             case TOK_ROUNDLEFTBRACE:
+				return org_cherry_read_pair(context);
 
 			default:
 				org_cherry_error(context, "bad input with token %s", 

test/lex.c

@@ -93,7 +93,7 @@ static void test_lex_string(const_pointer data)
 static void test_lex_comment(const_pointer data)
 {
 	byte_t* comment = "; comment\n";
-	struct org_cherry_context* c = org_cherry_context_repl(comment);
+	struct org_cherry_context* c = org_cherry_context(comment, 0, CY_DEFAULT);
 
 	assert(org_cherry_lex(c) == TOK_COMMENT);
 	assert(strcmp(org_cherry_token_string(c), "; comment") == 0);