-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathScanner.cpp
More file actions
97 lines (67 loc) · 2.76 KB
/
Scanner.cpp
File metadata and controls
97 lines (67 loc) · 2.76 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
//
// Created by LE, Duc Anh on 8/3/15.
//
#include "Scanner.h"
namespace CLDEParser {
SPtrTokenizerVector &Scanner::Tokenizers() {
return _tokenizers;
}
SPtrTokenVector Scanner::Scan(const std::string &string) const {
SPtrTokenizerList matchedTokenizers;
SPtrTokenVector tokens;
std::string::const_iterator cIter = string.cbegin();
std::string::const_iterator cEndr = string.cend();
while (cIter != string.end()) {
matchedTokenizers.clear();
for (auto &tokenizer: _tokenizers) {
tokenizer->Reset();
if (tokenizer->BeginWithCharacter(*cIter)) { matchedTokenizers.push_back(tokenizer); }
}
if (matchedTokenizers.size() == 0) {
std::string desc;
desc.reserve(Common::BufferSize::EIGHTY);
auto cTokenEnd = tokens.cend();
auto cTokenIter = tokens.cbegin();
if (tokens.size() > _exceptionTokenNo) {
cTokenIter = cTokenIter + (tokens.size() - _exceptionTokenNo);
}
for (; cTokenIter != cTokenEnd; ++cTokenIter) {
desc += (*cTokenIter)->CopyToString();
}
desc += *cIter;
throw Exceptions::ScannerException{
(int) Exceptions::ScannerException::ScannerExceptionCode::TokenUnmatchable, desc, (*cIter)};
}
tokens.push_back(ProcessAndMoveNext(cIter, cEndr, matchedTokenizers));
}
return tokens;
}
SPtrToken Scanner::ProcessAndMoveNext(std::string::const_iterator &cIter,
std::string::const_iterator &cEnd,
SPtrTokenizerList &matchedTokenizers) const {
std::vector<SPtrTokenizerList::const_iterator> unmatched;
SPtrTokenizer sptrTokenizer;
while (cIter != cEnd) {
unmatched.clear();
// Validate character with tokenizers
for (SPtrTokenizerList::const_iterator iter = matchedTokenizers.cbegin();
iter != matchedTokenizers.cend(); ++iter) {
auto &tokenizer = *iter;
if (!tokenizer->Validate(*cIter)) {
unmatched.push_back(iter);
}
}
// Remove unmatched tokenizers
for (auto &iter : unmatched) {
matchedTokenizers.erase(iter);
}
// Break if there is no tokenizer left
if (matchedTokenizers.size() == 0) {
break;
}
sptrTokenizer = *(matchedTokenizers.cbegin());
++cIter;
}
return sptrTokenizer->CreateSPtrToken();
}
}