Commit a26613c2 authored by Chris Müller's avatar Chris Müller
Browse files

add position marking and rewinding to lexer

parent c41c7d31
......@@ -25,16 +25,10 @@
struct CyArray;
struct org_cherry_file {
struct org_cherry_context {
const char* filename;
const byte_t* begin;
};
struct org_cherry_context {
struct org_cherry_file* file;
byte_t* src;
const byte_t* src;
struct CryArray* buffer;
};
......@@ -70,7 +64,10 @@ enum org_cherry_tok {
const byte_t* org_cherry_tok_to_string(enum org_cherry_tok token);
enum org_cherry_tok org_cherry_lex(struct org_cherry_context* context);
byte_t* org_cherry_token_string(struct org_cherry_context* context);
const byte_t* org_cherry_pos(struct org_cherry_context* context);
void org_cherry_rewind(struct org_cherry_context* context, const byte_t* pos);
const byte_t* org_cherry_token_string(struct org_cherry_context* context);
size_t org_cherry_token_length(struct org_cherry_context* context);
struct org_cherry_value* org_cherry_read(struct org_cherry_context* context);
......@@ -30,13 +30,10 @@ struct org_cherry_context*
org_cherry_context(const byte_t* source, const char* filename)
{
struct org_cherry_context* context = malloc(sizeof(struct org_cherry_context));
struct org_cherry_file* file = malloc(sizeof(struct org_cherry_file));
file->filename = filename;
file->begin = source;
context->filename = filename;
context->begin = source;
context->src = (byte_t*) source;
context->buffer = cry_array_new(sizeof(byte_t));
context->file = file;
return context;
}
......@@ -56,7 +53,6 @@ org_cherry_context_free(struct org_cherry_context* context)
cry_array_free(context->buffer);
free(context->file);
free(context);
}
......@@ -66,8 +62,8 @@ org_cherry_error(struct org_cherry_context* context, const char* format, ...)
va_list args;
va_start(args, format);
if(context->file->filename != 0)
fprintf(stderr, "ERROR %s", context->file->filename);
if(context->filename != 0)
fprintf(stderr, "ERROR %s", context->filename);
else
fprintf(stderr, "ERROR (console)");
......@@ -140,7 +136,7 @@ static enum org_cherry_tok
lex_float(struct org_cherry_context* context)
{
struct CryArray* buffer = context->buffer;
byte_t* p = context->src;
const byte_t* p = context->src;
unicode_t ch = org_cherry_utf8_get(p);
enum FloatState state = FP_START;
......@@ -232,7 +228,7 @@ static enum org_cherry_tok
lex_number(struct org_cherry_context* context)
{
struct CryArray* buffer = context->buffer;
byte_t* p = context->src;
const byte_t* p = context->src;
unicode_t ch = org_cherry_utf8_get(p);
enum org_cherry_tok token = TOK_DEC;
......@@ -506,7 +502,7 @@ static enum org_cherry_tok
lex_comment(struct org_cherry_context* context)
{
struct CryArray* buffer = context->buffer;
byte_t* p = context->src;
const byte_t* p = context->src;
unicode_t ch = org_cherry_utf8_get(p);
while(ch != '\0' && ch != '\r' && ch != '\n') {
......@@ -535,7 +531,7 @@ static enum org_cherry_tok
lex_symbol(struct org_cherry_context* context)
{
struct CryArray* buffer = context->buffer;
byte_t* p = context->src;
const byte_t* p = context->src;
unicode_t ch = org_cherry_utf8_get(p);
while(is_symbol_character(ch)) {
......@@ -570,7 +566,7 @@ org_cherry_lex(struct org_cherry_context* context)
{
assert(context != 0);
byte_t* p = context->src;
const byte_t* p = context->src;
while(TRUE) {
cry_array_clear(context->buffer);
......@@ -624,12 +620,29 @@ org_cherry_lex(struct org_cherry_context* context)
return TOK_EOF;
}
byte_t*
const byte_t*
org_cherry_pos(struct org_cherry_context* context)
{
assert(context != 0);
return context->src;
}
void
org_cherry_rewind(struct org_cherry_context* context, const byte_t* pos)
{
assert(context != 0);
assert(context->begin >= pos && pos >= context->src);
context->src = pos;
}
const byte_t*
org_cherry_token_string(struct org_cherry_context* context)
{
assert(context->buffer != 0);
return (byte_t*) cry_array_get(context->buffer, 0);
return (const byte_t*) cry_array_get(context->buffer, 0);
}
......@@ -667,7 +680,11 @@ org_cherry_read(struct org_cherry_context* context)
return org_cherry_float_from_string(org_cherry_token_string(context));
case TOK_STRING:
return org_cherry_string_from_string(org_cherry_token_string(context));
case TOK_CHAR:
return org_cherry_char_from_string(org_cherry_token_string(context));
case TOK_SYMBOL:
return org_cherry_symbol_from_string(org_cherry_token_string(context));
case TOK_ROUNDLEFTBRACE:
default:
org_cherry_error(context, "bad input with token %s",
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment