Todays Work :)

.gitignore

@@ -1,5 +1,6 @@
 *.swp
 *.d
 *.o
+*.I
 tags
 TAGS

first.c

@@ -27,6 +27,9 @@ bool run_function(struct str str);
 bool tkn_parse_pp_directive(struct tokenizer *tkn, struct token *out_tk,
                             enum err *out_err);
 
+bool tkn_parse_decl(struct tokenizer *tkn, struct token *out_tk,
+                    enum err *out_err);
+
 enum c_token_type {
     _TT = TK_LAST,
     TK_PP_DEF,
@@ -85,9 +88,11 @@ main(int argc, char *argv[])
             goto error_exit;
         }
         switch ( tk.type ) {
-        case TK_POUND:
-            if ( ! tkn_parse_pp_directive(&tkn, NULL, &err) ) goto error_exit;
-            break;
+        case TK_POUND: {
+            struct token pp_tk = {0};
+            if ( ! tkn_parse_pp_directive(&tkn, &pp_tk, &err) ) goto error_exit;
+            } break;
+        case TK_ID: if ( ! tkn_parse_decl(&tkn, NULL, &err) ) goto error_exit;
         case TK_NL: break;
         default: fprintf(stderr, "%s ERROR: Invalid Token `%s`\n",
                          tokenizer_token_loc_temp(&tkn, &tk, TLF_VIM, NULL),
@@ -98,20 +103,6 @@ main(int argc, char *argv[])
         tk = tokenizer_next_token(&tkn, &err);
     } while ( tk.type != TK_EOF );
 
-/*
-    if ( ! tkn_expect(&tkn, TK_POUND) ) return 1;
-
-    if ( ! tkn_expect_id(&tkn, "run") ) return 1;
-
-    if ( ! tkn_expect(&tkn, TK_L_CUR_BRACES) ) return 1;
-
-    if ( ! tkn_parse_function(&tkn) ) return 1;
-
-    if ( ! tkn_expect(&tkn, TK_R_CUR_BRACES) ) return 1;
-
-    if ( ! tkn_expect(&tkn, TK_EOF) ) return 1;
-*/
-
     printf("%s\n", err_to_name[err]);
 
 error_exit:
@@ -251,6 +242,7 @@ tkn_parse_pp_directive(struct tokenizer *tkn, struct token *out_tk,
         case TK_NUM_LIT:
             tk_ws = (struct token_wstr *) &tk;
             break;
+        case TK_NL: goto define_wout_value;
         default:
             fprintf(stderr,
                     "%s Got wrong token, expected:"
@@ -268,6 +260,7 @@ tkn_parse_pp_directive(struct tokenizer *tkn, struct token *out_tk,
         tk_def.loc_end = tk_ws->loc_end;
 
         if ( ! tkn_expect(tkn, TK_NL, NULL, perr) ) return false;
+define_wout_value:
 
         LIB_SET_IF_NOT_NULL(out_tk, *(struct token *) &tk_def);
         return true;
@@ -277,7 +270,30 @@ tkn_parse_pp_directive(struct tokenizer *tkn, struct token *out_tk,
         struct token_pp_inc tk_inc = {0};
         tk_inc.type = TK_PP_INC;
 
-        if ( ! tkn_expect(tkn, TK_L_ANG_BRACKET, NULL, perr) ) return false;
+        tk = tokenizer_next_token(tkn, &err);
+        if ( err != ERR_OK ) {
+            fprintf(stderr, "Failed to get next token: %s\n",
+                    err_to_name[err]);
+            return false;
+        }
+
+        if ( tk.type == TK_STR_LIT ) {
+            tk_ws = (struct token_wstr *) &tk;
+            tk_inc.loc_start = tk_ws->loc_start;
+            tk_inc.loc_end = tk_ws->loc_end;
+
+            memcpy(buf, tk_ws->string.data, tk_ws->string.size);
+            tk_inc.string.data = buf;
+            tk_inc.string.size = tk_ws->string.size;
+            goto include_str_lit;
+        } else if ( tk.type != TK_L_ANG_BRACKET) {
+            fprintf(stderr,
+                    "%s Got wrong token, expected:"
+                    " TK_ID/TK_STR_LIT/TK_NUM_LIT, got: %s\n",
+                    tokenizer_token_loc_temp(tkn, &tk, TLF_VIM, NULL),
+                    token_to_cstr(tk.type));
+            return false;
+        }
 
         if ( ! tkn_expect(tkn, TK_ID, &tk, perr) ) return false;
         tk_ws = (struct token_wstr *) &tk;
@@ -285,17 +301,18 @@ tkn_parse_pp_directive(struct tokenizer *tkn, struct token *out_tk,
         tk_inc.loc_start = tk_ws->loc_start;
         tk_inc.loc_end = tk_ws->loc_end;
 
+        memcpy(buf, tk_ws->string.data, tk_ws->string.size);
+
         tk_inc.string.data = buf;
         tk_inc.string.size = tk_ws->string.size;
 
-        memcpy(buf, tk_ws->string.data, tk_ws->string.size);
-
         if ( ! tkn_expect(tkn, TK_DOT, NULL, perr) ) return false;
         if ( ! tkn_expect_id(tkn, "h", NULL, perr) ) return false;
 
         if ( ! tkn_expect(tkn, TK_R_ANG_BRACKET, NULL, perr) ) return false;
         if ( ! tkn_expect(tkn, TK_NL, NULL, perr) ) return false;
 
+include_str_lit:
         LIB_SET_IF_NOT_NULL(out_tk, *(struct token *) &tk_inc);
         return true;
     }
@@ -312,11 +329,75 @@ tkn_parse_pp_directive(struct tokenizer *tkn, struct token *out_tk,
         return true;
     }
 
-    /* Format src.c:line:col: msg */
+    *perr = -1;
     fprintf(stderr, "%s ERROR: Invalid Pre-Compiler directive `%.*s`\n",
             tokenizer_token_loc_temp(tkn, &tk, TLF_VIM, NULL),
             (int) tk_ws->string.size, tk_ws->string.data);
+    return false;
+}
+
+bool
+tkn_parse_decl(struct tokenizer *tkn, struct token *out_tk, enum err *out_err)
+{
+    /* static char buf[1024] = {0}; */
+    /* static char buf2[1024] = {0}; */
+    struct token tk = {0};
+    struct token_wstr *tk_ws = NULL;
+    struct token_wstr tk_type = {0};
+    enum err err = ERR_OK;
+    enum err *perr = &err;
+
+    LIB_ARG_IF_NOT_NULL_MUST_BE(out_err, ERR_OK, false);
+    LIB_ARG_MUST_NOT_BE_NULL(tkn, out_err, false);
+
+    if ( out_err != NULL ) {
+        perr = out_err;
+    }
+
+    tk_type = *(struct token_wstr *) &tkn->last;
+
+    if ( ! tkn_expect(tkn, TK_ID, &tk, perr) ) return false;
+
+    switch ( tokenizer_next_token_type(tkn, perr) ) {
+    case TK_L_BRACES: {
+        TODO("Implement function declaration");
+        } break;
+    case TK_INVALID:
+        fprintf(stderr, "Failed to get next token: %s\n", err_to_name[*perr]);
+        return false;
+    default:
+        fprintf(stderr,
+                "%s Got wrong token, expected: TK_L_BRACES, got: %s\n",
+                tokenizer_token_loc_temp(tkn, &tk, TLF_VIM, NULL),
+                token_to_cstr(tk.type));
+    }
+
 
+
+/*
+    tk = tokenizer_next_token(tkn, perr);
+    if ( *perr != ERR_OK ) {
+        fprintf(stderr, "Failed to get next token: %s\n", err_to_name[*perr]);
+        return false;
+    }
+
+    if ( tk.type != TK_ID ) {
+        fprintf(stderr, "%s Got wrong token, expected: TK_ID, got: %s\n",
+                tokenizer_token_loc_temp(tkn, &tk, TLF_VIM, NULL),
+                token_to_cstr(tk.type));
+        return false;
+    }
+    */
+
+
+    UNUSED(tk_type);
+    UNUSED(out_tk);
+
+    *perr = ERR_OK;
+    tk_ws = (struct token_wstr *) &tkn->last;
+    fprintf(stderr, "%s ERROR: Invalid declaration id `%.*s`\n",
+            tokenizer_token_loc_temp(tkn, &tkn->last, TLF_VIM, NULL),
+            (int) tk_ws->string.size, tk_ws->string.data);
     return false;
 }
 

src/lib.h

@@ -114,9 +114,16 @@ void free(void *ptr);
 void *memcpy(void *dest, const void *src, u64 n);
 void *memset(void *s, int c, u64 n);
 char *getenv(const char *name);
+void abort(void);
 
 /* ----------------------------- START LIB DEF ----------------------------- */
 
+# define TODO(msg) \
+    do { \
+        fprintf(stderr, "%s:%d "msg"\n", __FILE__, __LINE__); \
+        abort(); \
+    } while(0);
+
 # define LIB_COALESCE2(arg1, arg2) ((arg1 != Null) ? arg1 : arg2)
 
 # define LIB_SET_IF_NULL(var, val) \
@@ -588,17 +595,6 @@ struct tokenizer_options {
     bool (*is_str_lit)(struct tokenizer *tkn, struct str str);
 };
 
-struct tokenizer {
-    u64 i;
-
-    struct tokenizer_options opts;
-
-    struct path src;
-    struct str code;
-
-    void *edata; /* This is not used by the lib */
-};
-
 enum tokenizer_loc_format {
     TLF_VIM = 0
 };
@@ -724,12 +720,27 @@ struct token_wstr {
     struct str string;
 };
 
+struct tokenizer {
+    u64 i;
+
+    struct tokenizer_options opts;
+
+    struct token last;
+
+    struct path src;
+    struct str code;
+
+    void *edata; /* This is not used by the lib */
+};
+
 
 struct tokenizer tokenizer_create(struct str code, struct path src,
                                   struct tokenizer_options *opts,
                                   enum err *out_err);
 
 struct token tokenizer_next_token(struct tokenizer *tkn, enum err *out_err);
+enum token_type tokenizer_next_token_type(struct tokenizer *tkn,
+                                          enum err *out_err);
 bool tokenizer_is_next(struct tokenizer *tkn, enum token_type type,
                        struct token *out_tk, enum err *out_err);
 bool tokenizer_is_next_id(struct tokenizer *tkn, struct str string,
@@ -805,10 +816,7 @@ tokenizer_next_token(struct tokenizer *tkn, enum err *out_err)
     buf.size = 0;
 
     if ( tkn->i >= tkn->code.size ) {
-        tk.type = TK_EOF;
-        tk.loc_start = tkn->i;
-        tk.loc_end   = tkn->i;
-        return tk;
+        goto eof_tk;
     }
 
     c = tkn->code.data[tkn->i++];
@@ -821,7 +829,7 @@ tokenizer_next_token(struct tokenizer *tkn, enum err *out_err)
         struct token_wstr tk_ws = {0};
 
         tk_ws.type = TK_ID;
-        tk_ws.loc_start = tkn->i;
+        tk_ws.loc_start = tkn->i-1;
 
         do {
             buf.data[buf.size++] = c;
@@ -835,16 +843,17 @@ tokenizer_next_token(struct tokenizer *tkn, enum err *out_err)
         if ( tkn->opts.is_num_lit(tkn, tk_ws.string) == true ) {
             tk_ws.type = TK_NUM_LIT;
         }
-        tk_ws.loc_end = tkn->i;
+        tk_ws.loc_end = tkn->i-1;
 
-        return *(struct token *)&tk_ws;
+        tk = *(struct token *)&tk_ws;
+        goto exit;
     }
 
     if ( tkn->opts.is_digit(tkn, c) == true ) {
         struct token_wstr tk_ws = {0};
 
         tk_ws.type = TK_NUM_LIT;
-        tk_ws.loc_start = tkn->i;
+        tk_ws.loc_start = tkn->i-1;
 
         do {
             buf.data[buf.size++] = c;
@@ -854,16 +863,17 @@ tokenizer_next_token(struct tokenizer *tkn, enum err *out_err)
 
         tk_ws.string.data = buf.data;
         tk_ws.string.size = buf.size;
-        tk_ws.loc_end = tkn->i;
+        tk_ws.loc_end = tkn->i-1;
 
-        return *(struct token *)&tk_ws;
+        tk = *(struct token *)&tk_ws;
+        goto exit;
     }
 
     if ( tkn->opts.is_str_lit_start(tkn, c) == true ) {
         struct token_wstr tk_ws = {0};
 
         tk_ws.type = TK_STR_LIT;
-        tk_ws.loc_start = tkn->i;
+        tk_ws.loc_start = tkn->i-1;
 
         do {
             buf.data[buf.size++] = c;
@@ -873,25 +883,51 @@ tokenizer_next_token(struct tokenizer *tkn, enum err *out_err)
 
         tk_ws.string.data = buf.data;
         tk_ws.string.size = buf.size - 1;
-        tk_ws.loc_end = tkn->i;
+        tk_ws.loc_end = tkn->i-1;
 
-        return *(struct token *)&tk_ws;
+        tk = *(struct token *)&tk_ws;
+        goto exit;
     }
 
+    tk.type = c;
+
     if ( tkn->i >= tkn->code.size ) {
+eof_tk:
         tk.type = TK_EOF;
-        tk.loc_start = tkn->i;
-        tk.loc_end   = tkn->i;
-        return tk;
     }
 
-    tk.type = c;
-    tk.loc_start = tkn->i;
-    tk.loc_end   = tkn->i;
+    tk.loc_start = tkn->i-1;
+    tk.loc_end   = tkn->i-1;
 
+exit:
+    tkn->last = tk;
     return tk;
 }
 
+enum token_type
+tokenizer_next_token_type(struct tokenizer *tkn, enum err *out_err)
+{
+    struct token tk = {0};
+    enum err err = ERR_OK;
+    enum err *perr = &err;
+
+    LIB_ARG_IF_NOT_NULL_MUST_BE(out_err, ERR_OK, TK_INVALID);
+
+    LIB_ARG_MUST_NOT_BE_NULL(tkn, out_err, TK_INVALID);
+
+    if ( out_err != NULL ) {
+        perr = out_err;
+    }
+
+    tk = tokenizer_next_token(tkn, perr);
+    if ( *perr != ERR_OK ) {
+        return TK_INVALID;
+    }
+
+    LIB_SET_IF_NOT_NULL(out_err, ERR_OK);
+    return tk.type;
+}
+
 bool
 tokenizer_is_next(struct tokenizer *tkn, enum token_type type,
                   struct token *out_tk, enum err *out_err)
@@ -984,7 +1020,6 @@ tokenizer_token_loc_temp(const struct tokenizer *tkn, const struct token *tk,
     switch ( format ) {
     case TLF_VIM: {
         u64 col = 0;
-        /* Format src.c:line:col: msg */
 
         memcpy(buf.data, tkn->src.data, tkn->src.size);
         buf.size += tkn->src.size;
@@ -1003,7 +1038,7 @@ tokenizer_token_loc_temp(const struct tokenizer *tkn, const struct token *tk,
             buf.size += line_str.size;
 
             /* Col values wrong */
-            while ( tkn->code.data[i] != '\n' ) {
+            while ( tkn->code.data[i] != '\n' && i > 0 ) {
                 --i;
             }
 

src/str.h

@@ -24,6 +24,9 @@ struct str str_rstrip(struct str str);
 struct str str_lstrip(struct str str);
 struct str str_strip(struct str str);
 
+size_t str_lindex(struct str str, char c);
+size_t str_rindex(struct str str, char c);
+
 struct str_tokenizer str_tokenize(struct str str, char c);
 struct str str_tokenizer_next(struct str_tokenizer *st);
 
@@ -35,7 +38,7 @@ bool str_eq_cstr(struct str str, const char *cstr, size_t cstr_size);
 
 #include <ctype.h>
 
-struct str 
+struct str
 str_from_cstr(const char *cstr, size_t cstr_size)
 {
 	struct str str;
@@ -53,7 +56,7 @@ str_rstrip(struct str str)
 	return str;
 }
 
-struct str 
+struct str
 str_lstrip(struct str str)
 {
 	while ( isspace(*str.data) ) {
@@ -63,13 +66,40 @@ str_lstrip(struct str str)
 	return str;
 }
 
-struct str 
+struct str
 str_strip(struct str str)
 {
 	return str_lstrip(str_rstrip(str));
 }
 
-struct str_tokenizer 
+size_t
+str_lindex(struct str str, char c)
+{
+	size_t i = 0;
+	for ( ; i < str.size; ++i ) {
+		if ( str.data[i] == c ) {
+			return i;
+		}
+	}
+	return (size_t) -1;
+}
+
+size_t
+str_rindex(struct str str, char c)
+{
+	size_t i = str.size - 1;
+	for ( ; i > 0; --i ) {
+		if ( str.data[i] == c ) {
+			return i;
+		}
+	}
+	if ( str.data[i] == c ) {
+		return i;
+	}
+	return (size_t) -1;
+}
+
+struct str_tokenizer
 str_tokenize(struct str str, char c)
 {
 	struct str_tokenizer st = {0};
@@ -78,7 +108,7 @@ str_tokenize(struct str str, char c)
 	return st;
 }
 
-struct str 
+struct str
 str_tokenizer_next(struct str_tokenizer *st)
 {
 	struct str str;
@@ -99,7 +129,7 @@ str_tokenizer_next(struct str_tokenizer *st)
 	str.data += st->cur;
 	str.size = 0;
 
-	while ( str.data[str.size] != st->c 
+	while ( str.data[str.size] != st->c
 		&& st->cur < st->str.size ) {
 		++str.size;
 		++st->cur;
@@ -128,7 +158,7 @@ ret_err:
 	return str;
 }
 
-struct str 
+struct str
 str_slice(struct str str, size_t from, size_t to)
 {
 	if ( from > str.size ) {
@@ -152,7 +182,7 @@ ret_err:
 	return str;
 }
 
-bool 
+bool
 str_eq_cstr(struct str str, const char *cstr, size_t cstr_size)
 {
 	size_t i = 0;