annotate lwcc/token.c @ 295:4b17780f2777 ccdev

Checkpoint lwcc development Changed tactics with the preprocessor. Instead of getting clever and trying to do things the "fast" way, instead, just tokenize the whole input and process it that way. Also, set up so the preprocessor and compiler can be integrated instead of having to have a specifically correct output for the preprocessed file. Also removed the subdirectories in the lwcc directory. It made things more complicated than they needed to be.
author William Astle <lost@l-w.ca>
date Thu, 12 Sep 2013 22:06:26 -0600
parents
children 856caf91ffaa
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
295
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
1 /*
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
2 lwcc/token.c
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
3
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
4 Copyright © 2013 William Astle
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
5
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
6 This file is part of LWTOOLS.
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
7
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
8 LWTOOLS is free software: you can redistribute it and/or modify it under the
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
9 terms of the GNU General Public License as published by the Free Software
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
10 Foundation, either version 3 of the License, or (at your option) any later
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
11 version.
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
12
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
13 This program is distributed in the hope that it will be useful, but WITHOUT
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
16 more details.
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
17
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
18 You should have received a copy of the GNU General Public License along with
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
19 this program. If not, see <http://www.gnu.org/licenses/>.
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
20 */
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
21
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
22 #include <stdlib.h>
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
23
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
24 #include <lw_alloc.h>
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
25 #include <lw_string.h>
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
26
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
27 #include "token.h"
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
28
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
29 struct token *token_create(int ttype, char *strval, int row, int col, const char *fn)
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
30 {
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
31 struct token *t;
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
32
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
33 t = lw_alloc(sizeof(struct token));
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
34 t -> ttype = ttype;
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
35 if (strval)
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
36 t -> strval = lw_strdup(strval);
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
37 else
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
38 strval = NULL;
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
39 t -> lineno = row;
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
40 t -> column = col;
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
41 t -> fn = fn;
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
42 t -> next = NULL;
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
43 t -> prev = NULL;
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
44 return t;
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
45 }
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
46
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
47 void token_free(struct token *t)
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
48 {
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
49 lw_free(t -> strval);
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
50 lw_free(t);
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
51 }
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
52
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
53 struct token *token_dup(struct token *t)
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
54 {
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
55 struct token *t2;
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
56
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
57 t2 = lw_alloc(sizeof(struct token));
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
58 (*t2) = (*t);
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
59 t2 -> next = NULL;
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
60 t2 -> prev = NULL;
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
61 if (t -> strval)
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
62 t2 -> strval = lw_strdup(t -> strval);
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
63 return t2;
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
64 }
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
65
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
66 static struct { int ttype; char *tstr; } tok_strs[] =
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
67 {
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
68 { TOK_WSPACE, " " },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
69 { TOK_EOL, "\n" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
70 { TOK_DIV, "/" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
71 { TOK_ADD, "+" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
72 { TOK_SUB, "-" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
73 { TOK_OPAREN, "(" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
74 { TOK_CPAREN, ")" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
75 { TOK_NE, "!=" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
76 { TOK_EQ, "==" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
77 { TOK_LE, "<=" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
78 { TOK_LT, "<" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
79 { TOK_GE, ">=" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
80 { TOK_GT, ">" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
81 { TOK_BAND, "&&" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
82 { TOK_BOR, "||" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
83 { TOK_BNOT, "!" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
84 { TOK_MOD, "%"},
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
85 { TOK_COMMA, "," },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
86 { TOK_ELLIPSIS, "..." },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
87 { TOK_QMARK, "?" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
88 { TOK_COLON, ":" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
89 { TOK_OBRACE, "{" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
90 { TOK_CBRACE, "}" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
91 { TOK_OSQUARE, "[" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
92 { TOK_CSQUARE, "]" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
93 { TOK_COM, "~" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
94 { TOK_EOS, ";" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
95 { TOK_HASH, "#" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
96 { TOK_DBLHASH, "##" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
97 { TOK_XOR, "^" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
98 { TOK_XORASS, "^=" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
99 { TOK_STAR, "*" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
100 { TOK_MULASS, "*=" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
101 { TOK_DIVASS, "/=" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
102 { TOK_ASS, "=" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
103 { TOK_MODASS, "%=" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
104 { TOK_SUBASS, "-=" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
105 { TOK_DBLSUB, "--" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
106 { TOK_ADDASS, "+=" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
107 { TOK_DBLADD, "++" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
108 { TOK_BWAND, "&" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
109 { TOK_BWANDASS, "&=" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
110 { TOK_BWOR, "|" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
111 { TOK_BWORASS, "|=" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
112 { TOK_LSH, "<<" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
113 { TOK_LSHASS, "<<=" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
114 { TOK_RSH, ">>" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
115 { TOK_RSHASS, ">>=" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
116 { TOK_DOT, "." },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
117 { TOK_ARROW, "->" },
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
118 { TOK_NONE, "" }
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
119 };
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
120
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
121 void token_print(struct token *t, FILE *f)
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
122 {
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
123 int i;
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
124 for (i = 0; tok_strs[i].ttype != TOK_NONE; i++)
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
125 {
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
126 if (tok_strs[i].ttype == t -> ttype)
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
127 {
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
128 fprintf(f, "%s", tok_strs[i].tstr);
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
129 break;
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
130 }
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
131 }
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
132 if (t -> strval)
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
133 fprintf(f, "%s", t -> strval);
4b17780f2777 Checkpoint lwcc development
William Astle <lost@l-w.ca>
parents:
diff changeset
134 }