-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtokenizer.c
More file actions
84 lines (74 loc) · 1.79 KB
/
tokenizer.c
File metadata and controls
84 lines (74 loc) · 1.79 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
#include <string.h>
#include <ctype.h>
#include "hbasic.h"
#include "tokenizer.h"
#include "util.h"
char *currline;
byte currpos;
byte endpos;
byte dblQuoteMatch;
void tokenize(char *line) {
currline = line;
currpos = 0;
endpos = strlen(line);
dblQuoteMatch = FALSE;
}
void nextToken(struct Token *token) {
if (currpos >= endpos) {
token->type = TYPE_EOL;
token->value[0] = '\0';
return;
}
byte startpos = currpos;
token->type = TYPE_UNDEFINED;
byte insideString = FALSE;
for (int i = currpos; i < endpos; i++) {
char c = currline[i];
if (c == '"' && dblQuoteMatch) {
dblQuoteMatch = !dblQuoteMatch;
token->type = TYPE_DBL_QUOTE;
break;
} else if (c == '"') {
token->type = TYPE_DBL_QUOTE;
dblQuoteMatch = !dblQuoteMatch;
break;
} else if (dblQuoteMatch) {
currpos++;
} else if (c == '\n' || c == '\r') {
token->type = TYPE_EOL;
currpos++;
break;
} else if (isspace(c)) {
while (isspace(c)) {
c = currline[++currpos];
}
break;
} else {
currpos++;
}
/*
if (insideString && c != '"') {
currpos++;
break;
} else if (insideString) {
currpos++;
} else if (isspace(c)) { // whitespace so done with currtoken
break;
} else if (c == '"' && !insideString && currpos == startpos) {
token->type = TYPE_STRING;
insideString = TRUE;
currpos++;
} else if (c == '"' && !insideString) {
currpos++;
insideString = TRUE;
}*/
}
for (int i = 0; i < currpos - startpos; i++) {
token->value[i] = currline[i + startpos];
}
token->value[currpos - startpos] = '\0';
rtrimwhitespace(token->value);
if (!strcmp("QUIT", token->value) || !strcmp("EXIT", token->value)) {
token->type = TYPE_QUIT;
}
}