/*

* Copyright (C) 1999-2000 Harri Porten ([email protected])

* Copyright (C) 2006, 2007, 2008, 2009, 2011, 2012, 2013 Apple Inc. All Rights Reserved.

* Copyright (C) 2007 Cameron Zwarich ([email protected])

* Copyright (C) 2010 Zoltan Herczeg ([email protected])

* Copyright (C) 2012 Mathias Bynens ([email protected])

*

* This library is free software; you can redistribute it and/or

* modify it under the terms of the GNU Library General Public

* License as published by the Free Software Foundation; either

* version 2 of the License, or (at your option) any later version.

*

* This library is distributed in the hope that it will be useful,

* but WITHOUT ANY WARRANTY; without even the implied warranty of

* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

* Library General Public License for more details.

*

* You should have received a copy of the GNU Library General Public License

* along with this library; see the file COPYING.LIB. If not, write to

* the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,

* Boston, MA 02110-1301, USA.

*

*/

#include "config.h"

#include "Lexer.h"

#include "JSFunctionInlines.h"

#include "BuiltinNames.h"

#include "JSGlobalObjectFunctions.h"

#include "Identifier.h"

#include "NodeInfo.h"

#include "Nodes.h"

#include "JSCInlines.h"

#include <wtf/dtoa.h>

#include <ctype.h>

#include <limits.h>

#include <string.h>

#include <wtf/Assertions.h>

#include "KeywordLookup.h"

#include "Lexer.lut.h"

#include "Parser.h"

namespace JSC {

Keywords::Keywords(VM& vm)

: m_vm(vm)

, m_keywordTable(JSC::mainTable)

{

}

enum CharacterType {

// Types for the main switch

// The first three types are fixed, and also used for identifying

// ASCII alpha and alphanumeric characters (see isIdentStart and isIdentPart).

CharacterIdentifierStart,

CharacterZero,

CharacterNumber,

CharacterInvalid,

CharacterLineTerminator,

CharacterExclamationMark,

CharacterOpenParen,

CharacterCloseParen,

CharacterOpenBracket,

CharacterCloseBracket,

CharacterComma,

CharacterColon,

CharacterQuestion,

CharacterTilde,

CharacterQuote,

CharacterDot,

CharacterSlash,

CharacterBackSlash,

CharacterSemicolon,

CharacterOpenBrace,

CharacterCloseBrace,

CharacterAdd,

CharacterSub,

CharacterMultiply,

CharacterModulo,

CharacterAnd,

CharacterXor,

CharacterOr,

CharacterLess,

CharacterGreater,

CharacterEqual,

// Other types (only one so far)

CharacterWhiteSpace,

CharacterPrivateIdentifierStart

};

// 256 Latin-1 codes

static const unsigned short typesOfLatin1Characters[256] = {

/* 0 - Null */ CharacterInvalid,

/* 1 - Start of Heading */ CharacterInvalid,

/* 2 - Start of Text */ CharacterInvalid,

/* 3 - End of Text */ CharacterInvalid,

/* 4 - End of Transm. */ CharacterInvalid,

/* 5 - Enquiry */ CharacterInvalid,

/* 6 - Acknowledgment */ CharacterInvalid,

/* 7 - Bell */ CharacterInvalid,

/* 8 - Back Space */ CharacterInvalid,

/* 9 - Horizontal Tab */ CharacterWhiteSpace,

/* 10 - Line Feed */ CharacterLineTerminator,

/* 11 - Vertical Tab */ CharacterWhiteSpace,

/* 12 - Form Feed */ CharacterWhiteSpace,

/* 13 - Carriage Return */ CharacterLineTerminator,

/* 14 - Shift Out */ CharacterInvalid,

/* 15 - Shift In */ CharacterInvalid,

/* 16 - Data Line Escape */ CharacterInvalid,

/* 17 - Device Control 1 */ CharacterInvalid,

/* 18 - Device Control 2 */ CharacterInvalid,

/* 19 - Device Control 3 */ CharacterInvalid,

/* 20 - Device Control 4 */ CharacterInvalid,

/* 21 - Negative Ack. */ CharacterInvalid,

/* 22 - Synchronous Idle */ CharacterInvalid,

/* 23 - End of Transmit */ CharacterInvalid,

/* 24 - Cancel */ CharacterInvalid,

/* 25 - End of Medium */ CharacterInvalid,

/* 26 - Substitute */ CharacterInvalid,

/* 27 - Escape */ CharacterInvalid,

/* 28 - File Separator */ CharacterInvalid,

/* 29 - Group Separator */ CharacterInvalid,

/* 30 - Record Separator */ CharacterInvalid,

/* 31 - Unit Separator */ CharacterInvalid,

/* 32 - Space */ CharacterWhiteSpace,

/* 33 - ! */ CharacterExclamationMark,

/* 34 - " */ CharacterQuote,

/* 35 - # */ CharacterInvalid,

/* 36 - $ */ CharacterIdentifierStart,

/* 37 - % */ CharacterModulo,

/* 38 - & */ CharacterAnd,

/* 39 - ' */ CharacterQuote,

/* 40 - ( */ CharacterOpenParen,

/* 41 - ) */ CharacterCloseParen,

/* 42 - * */ CharacterMultiply,

/* 43 - + */ CharacterAdd,

/* 44 - , */ CharacterComma,

/* 45 - - */ CharacterSub,

/* 46 - . */ CharacterDot,

/* 47 - / */ CharacterSlash,

/* 48 - 0 */ CharacterZero,

/* 49 - 1 */ CharacterNumber,

/* 50 - 2 */ CharacterNumber,

/* 51 - 3 */ CharacterNumber,

/* 52 - 4 */ CharacterNumber,

/* 53 - 5 */ CharacterNumber,

/* 54 - 6 */ CharacterNumber,

/* 55 - 7 */ CharacterNumber,

/* 56 - 8 */ CharacterNumber,

/* 57 - 9 */ CharacterNumber,

/* 58 - : */ CharacterColon,

/* 59 - ; */ CharacterSemicolon,

/* 60 - < */ CharacterLess,

/* 61 - = */ CharacterEqual,

/* 62 - > */ CharacterGreater,

/* 63 - ? */ CharacterQuestion,

/* 64 - @ */ CharacterPrivateIdentifierStart,

/* 65 - A */ CharacterIdentifierStart,

/* 66 - B */ CharacterIdentifierStart,

/* 67 - C */ CharacterIdentifierStart,

/* 68 - D */ CharacterIdentifierStart,

/* 69 - E */ CharacterIdentifierStart,

/* 70 - F */ CharacterIdentifierStart,

/* 71 - G */ CharacterIdentifierStart,

/* 72 - H */ CharacterIdentifierStart,

/* 73 - I */ CharacterIdentifierStart,

/* 74 - J */ CharacterIdentifierStart,

/* 75 - K */ CharacterIdentifierStart,

/* 76 - L */ CharacterIdentifierStart,

/* 77 - M */ CharacterIdentifierStart,

/* 78 - N */ CharacterIdentifierStart,

/* 79 - O */ CharacterIdentifierStart,

/* 80 - P */ CharacterIdentifierStart,

/* 81 - Q */ CharacterIdentifierStart,

/* 82 - R */ CharacterIdentifierStart,

/* 83 - S */ CharacterIdentifierStart,

/* 84 - T */ CharacterIdentifierStart,

/* 85 - U */ CharacterIdentifierStart,

/* 86 - V */ CharacterIdentifierStart,

/* 87 - W */ CharacterIdentifierStart,

/* 88 - X */ CharacterIdentifierStart,

/* 89 - Y */ CharacterIdentifierStart,

/* 90 - Z */ CharacterIdentifierStart,

/* 91 - [ */ CharacterOpenBracket,

/* 92 - \ */ CharacterBackSlash,

/* 93 - ] */ CharacterCloseBracket,

/* 94 - ^ */ CharacterXor,

/* 95 - _ */ CharacterIdentifierStart,

/* 96 - ` */ CharacterInvalid,

/* 97 - a */ CharacterIdentifierStart,

/* 98 - b */ CharacterIdentifierStart,

/* 99 - c */ CharacterIdentifierStart,

/* 100 - d */ CharacterIdentifierStart,

/* 101 - e */ CharacterIdentifierStart,

/* 102 - f */ CharacterIdentifierStart,

/* 103 - g */ CharacterIdentifierStart,

/* 104 - h */ CharacterIdentifierStart,

/* 105 - i */ CharacterIdentifierStart,

/* 106 - j */ CharacterIdentifierStart,

/* 107 - k */ CharacterIdentifierStart,

/* 108 - l */ CharacterIdentifierStart,

/* 109 - m */ CharacterIdentifierStart,

/* 110 - n */ CharacterIdentifierStart,

/* 111 - o */ CharacterIdentifierStart,

/* 112 - p */ CharacterIdentifierStart,

/* 113 - q */ CharacterIdentifierStart,

/* 114 - r */ CharacterIdentifierStart,

/* 115 - s */ CharacterIdentifierStart,

/* 116 - t */ CharacterIdentifierStart,

/* 117 - u */ CharacterIdentifierStart,

/* 118 - v */ CharacterIdentifierStart,

/* 119 - w */ CharacterIdentifierStart,

/* 120 - x */ CharacterIdentifierStart,

/* 121 - y */ CharacterIdentifierStart,

/* 122 - z */ CharacterIdentifierStart,

/* 123 - { */ CharacterOpenBrace,

/* 124 - | */ CharacterOr,

/* 125 - } */ CharacterCloseBrace,

/* 126 - ~ */ CharacterTilde,

/* 127 - Delete */ CharacterInvalid,

/* 128 - Cc category */ CharacterInvalid,

/* 129 - Cc category */ CharacterInvalid,

/* 130 - Cc category */ CharacterInvalid,

/* 131 - Cc category */ CharacterInvalid,

/* 132 - Cc category */ CharacterInvalid,

/* 133 - Cc category */ CharacterInvalid,

/* 134 - Cc category */ CharacterInvalid,

/* 135 - Cc category */ CharacterInvalid,

/* 136 - Cc category */ CharacterInvalid,

/* 137 - Cc category */ CharacterInvalid,

/* 138 - Cc category */ CharacterInvalid,

/* 139 - Cc category */ CharacterInvalid,

/* 140 - Cc category */ CharacterInvalid,

/* 141 - Cc category */ CharacterInvalid,

/* 142 - Cc category */ CharacterInvalid,

/* 143 - Cc category */ CharacterInvalid,

/* 144 - Cc category */ CharacterInvalid,

/* 145 - Cc category */ CharacterInvalid,

/* 146 - Cc category */ CharacterInvalid,

/* 147 - Cc category */ CharacterInvalid,

/* 148 - Cc category */ CharacterInvalid,

/* 149 - Cc category */ CharacterInvalid,

/* 150 - Cc category */ CharacterInvalid,

/* 151 - Cc category */ CharacterInvalid,

/* 152 - Cc category */ CharacterInvalid,

/* 153 - Cc category */ CharacterInvalid,

/* 154 - Cc category */ CharacterInvalid,

/* 155 - Cc category */ CharacterInvalid,

/* 156 - Cc category */ CharacterInvalid,

/* 157 - Cc category */ CharacterInvalid,

/* 158 - Cc category */ CharacterInvalid,

/* 159 - Cc category */ CharacterInvalid,

/* 160 - Zs category (nbsp) */ CharacterWhiteSpace,

/* 161 - Po category */ CharacterInvalid,

/* 162 - Sc category */ CharacterInvalid,

/* 163 - Sc category */ CharacterInvalid,

/* 164 - Sc category */ CharacterInvalid,

/* 165 - Sc category */ CharacterInvalid,

/* 166 - So category */ CharacterInvalid,

/* 167 - So category */ CharacterInvalid,

/* 168 - Sk category */ CharacterInvalid,

/* 169 - So category */ CharacterInvalid,

/* 170 - Ll category */ CharacterIdentifierStart,

/* 171 - Pi category */ CharacterInvalid,

/* 172 - Sm category */ CharacterInvalid,

/* 173 - Cf category */ CharacterInvalid,

/* 174 - So category */ CharacterInvalid,

/* 175 - Sk category */ CharacterInvalid,

/* 176 - So category */ CharacterInvalid,

/* 177 - Sm category */ CharacterInvalid,

/* 178 - No category */ CharacterInvalid,

/* 179 - No category */ CharacterInvalid,

/* 180 - Sk category */ CharacterInvalid,

/* 181 - Ll category */ CharacterIdentifierStart,

/* 182 - So category */ CharacterInvalid,

/* 183 - Po category */ CharacterInvalid,

/* 184 - Sk category */ CharacterInvalid,

/* 185 - No category */ CharacterInvalid,

/* 186 - Ll category */ CharacterIdentifierStart,

/* 187 - Pf category */ CharacterInvalid,

/* 188 - No category */ CharacterInvalid,

/* 189 - No category */ CharacterInvalid,

/* 190 - No category */ CharacterInvalid,

/* 191 - Po category */ CharacterInvalid,

/* 192 - Lu category */ CharacterIdentifierStart,

/* 193 - Lu category */ CharacterIdentifierStart,

/* 194 - Lu category */ CharacterIdentifierStart,

/* 195 - Lu category */ CharacterIdentifierStart,

/* 196 - Lu category */ CharacterIdentifierStart,

/* 197 - Lu category */ CharacterIdentifierStart,

/* 198 - Lu category */ CharacterIdentifierStart,

/* 199 - Lu category */ CharacterIdentifierStart,

/* 200 - Lu category */ CharacterIdentifierStart,

/* 201 - Lu category */ CharacterIdentifierStart,

/* 202 - Lu category */ CharacterIdentifierStart,

/* 203 - Lu category */ CharacterIdentifierStart,

/* 204 - Lu category */ CharacterIdentifierStart,

/* 205 - Lu category */ CharacterIdentifierStart,

/* 206 - Lu category */ CharacterIdentifierStart,

/* 207 - Lu category */ CharacterIdentifierStart,

/* 208 - Lu category */ CharacterIdentifierStart,

/* 209 - Lu category */ CharacterIdentifierStart,

/* 210 - Lu category */ CharacterIdentifierStart,

/* 211 - Lu category */ CharacterIdentifierStart,

/* 212 - Lu category */ CharacterIdentifierStart,

/* 213 - Lu category */ CharacterIdentifierStart,

/* 214 - Lu category */ CharacterIdentifierStart,

/* 215 - Sm category */ CharacterInvalid,

/* 216 - Lu category */ CharacterIdentifierStart,

/* 217 - Lu category */ CharacterIdentifierStart,

/* 218 - Lu category */ CharacterIdentifierStart,

/* 219 - Lu category */ CharacterIdentifierStart,

/* 220 - Lu category */ CharacterIdentifierStart,

/* 221 - Lu category */ CharacterIdentifierStart,

/* 222 - Lu category */ CharacterIdentifierStart,

/* 223 - Ll category */ CharacterIdentifierStart,

/* 224 - Ll category */ CharacterIdentifierStart,

/* 225 - Ll category */ CharacterIdentifierStart,

/* 226 - Ll category */ CharacterIdentifierStart,

/* 227 - Ll category */ CharacterIdentifierStart,

/* 228 - Ll category */ CharacterIdentifierStart,

/* 229 - Ll category */ CharacterIdentifierStart,

/* 230 - Ll category */ CharacterIdentifierStart,

/* 231 - Ll category */ CharacterIdentifierStart,

/* 232 - Ll category */ CharacterIdentifierStart,

/* 233 - Ll category */ CharacterIdentifierStart,

/* 234 - Ll category */ CharacterIdentifierStart,

/* 235 - Ll category */ CharacterIdentifierStart,

/* 236 - Ll category */ CharacterIdentifierStart,

/* 237 - Ll category */ CharacterIdentifierStart,

/* 238 - Ll category */ CharacterIdentifierStart,

/* 239 - Ll category */ CharacterIdentifierStart,

/* 240 - Ll category */ CharacterIdentifierStart,

/* 241 - Ll category */ CharacterIdentifierStart,

/* 242 - Ll category */ CharacterIdentifierStart,

/* 243 - Ll category */ CharacterIdentifierStart,

/* 244 - Ll category */ CharacterIdentifierStart,

/* 245 - Ll category */ CharacterIdentifierStart,

/* 246 - Ll category */ CharacterIdentifierStart,

/* 247 - Sm category */ CharacterInvalid,

/* 248 - Ll category */ CharacterIdentifierStart,

/* 249 - Ll category */ CharacterIdentifierStart,

/* 250 - Ll category */ CharacterIdentifierStart,

/* 251 - Ll category */ CharacterIdentifierStart,

/* 252 - Ll category */ CharacterIdentifierStart,

/* 253 - Ll category */ CharacterIdentifierStart,

/* 254 - Ll category */ CharacterIdentifierStart,

/* 255 - Ll category */ CharacterIdentifierStart

};

// This table provides the character that results from \X where X is the index in the table beginning

// with SPACE. A table value of 0 means that more processing needs to be done.

static const LChar singleCharacterEscapeValuesForASCII[128] = {

/* 0 - Null */ 0,

/* 1 - Start of Heading */ 0,

/* 2 - Start of Text */ 0,

/* 3 - End of Text */ 0,

/* 4 - End of Transm. */ 0,

/* 5 - Enquiry */ 0,

/* 6 - Acknowledgment */ 0,

/* 7 - Bell */ 0,

/* 8 - Back Space */ 0,

/* 9 - Horizontal Tab */ 0,

/* 10 - Line Feed */ 0,

/* 11 - Vertical Tab */ 0,

/* 12 - Form Feed */ 0,

/* 13 - Carriage Return */ 0,

/* 14 - Shift Out */ 0,

/* 15 - Shift In */ 0,

/* 16 - Data Line Escape */ 0,

/* 17 - Device Control 1 */ 0,

/* 18 - Device Control 2 */ 0,

/* 19 - Device Control 3 */ 0,

/* 20 - Device Control 4 */ 0,

/* 21 - Negative Ack. */ 0,

/* 22 - Synchronous Idle */ 0,

/* 23 - End of Transmit */ 0,

/* 24 - Cancel */ 0,

/* 25 - End of Medium */ 0,

/* 26 - Substitute */ 0,

/* 27 - Escape */ 0,

/* 28 - File Separator */ 0,

/* 29 - Group Separator */ 0,

/* 30 - Record Separator */ 0,

/* 31 - Unit Separator */ 0,

/* 32 - Space */ ' ',

/* 33 - ! */ '!',

/* 34 - " */ '"',

/* 35 - # */ '#',

/* 36 - $ */ '$',

/* 37 - % */ '%',

/* 38 - & */ '&',

/* 39 - ' */ '\'',

/* 40 - ( */ '(',

/* 41 - ) */ ')',

/* 42 - * */ '*',

/* 43 - + */ '+',

/* 44 - , */ ',',

/* 45 - - */ '-',

/* 46 - . */ '.',

/* 47 - / */ '/',

/* 48 - 0 */ 0,

/* 49 - 1 */ 0,

/* 50 - 2 */ 0,

/* 51 - 3 */ 0,

/* 52 - 4 */ 0,

/* 53 - 5 */ 0,

/* 54 - 6 */ 0,

/* 55 - 7 */ 0,

/* 56 - 8 */ 0,

/* 57 - 9 */ 0,

/* 58 - : */ ':',

/* 59 - ; */ ';',

/* 60 - < */ '<',

/* 61 - = */ '=',

/* 62 - > */ '>',

/* 63 - ? */ '?',

/* 64 - @ */ '@',

/* 65 - A */ 'A',

/* 66 - B */ 'B',

/* 67 - C */ 'C',

/* 68 - D */ 'D',

/* 69 - E */ 'E',

/* 70 - F */ 'F',

/* 71 - G */ 'G',

/* 72 - H */ 'H',

/* 73 - I */ 'I',

/* 74 - J */ 'J',

/* 75 - K */ 'K',

/* 76 - L */ 'L',

/* 77 - M */ 'M',

/* 78 - N */ 'N',

/* 79 - O */ 'O',

/* 80 - P */ 'P',

/* 81 - Q */ 'Q',

/* 82 - R */ 'R',

/* 83 - S */ 'S',

/* 84 - T */ 'T',

/* 85 - U */ 'U',

/* 86 - V */ 'V',

/* 87 - W */ 'W',

/* 88 - X */ 'X',

/* 89 - Y */ 'Y',

/* 90 - Z */ 'Z',

/* 91 - [ */ '[',

/* 92 - \ */ '\\',

/* 93 - ] */ ']',

/* 94 - ^ */ '^',

/* 95 - _ */ '_',

/* 96 - ` */ '`',

/* 97 - a */ 'a',

/* 98 - b */ 0x08,

/* 99 - c */ 'c',

/* 100 - d */ 'd',

/* 101 - e */ 'e',

/* 102 - f */ 0x0C,

/* 103 - g */ 'g',

/* 104 - h */ 'h',

/* 105 - i */ 'i',

/* 106 - j */ 'j',

/* 107 - k */ 'k',

/* 108 - l */ 'l',

/* 109 - m */ 'm',

/* 110 - n */ 0x0A,

/* 111 - o */ 'o',

/* 112 - p */ 'p',

/* 113 - q */ 'q',

/* 114 - r */ 0x0D,

/* 115 - s */ 's',

/* 116 - t */ 0x09,

/* 117 - u */ 0,

/* 118 - v */ 0x0B,

/* 119 - w */ 'w',

/* 120 - x */ 0,

/* 121 - y */ 'y',

/* 122 - z */ 'z',

/* 123 - { */ '{',

/* 124 - | */ '|',

/* 125 - } */ '}',

/* 126 - ~ */ '~',

/* 127 - Delete */ 0

};

template <typename T>

Lexer<T>::Lexer(VM* vm, JSParserStrictness strictness)

: m_isReparsing(false)

, m_vm(vm)

, m_parsingBuiltinFunction(strictness == JSParseBuiltin)

{

}

template <typename T>

Lexer<T>::~Lexer()

{

}

template <typename T>

String Lexer<T>::invalidCharacterMessage() const

{

switch (m_current) {

case 0:

return "Invalid character: '\\0'";

case 10:

return "Invalid character: '\\n'";

case 11:

return "Invalid character: '\\v'";

case 13:

return "Invalid character: '\\r'";

case 35:

return "Invalid character: '#'";

case 64:

return "Invalid character: '@'";

case 96:

return "Invalid character: '`'";

default:

return String::format("Invalid character '\\u%04u'", static_cast<unsigned>(m_current)).impl();

}

}

template <typename T>

ALWAYS_INLINE const T* Lexer<T>::currentSourcePtr() const

{

ASSERT(m_code <= m_codeEnd);

return m_code;

}

template <typename T>

void Lexer<T>::setCode(const SourceCode& source, ParserArena* arena)

{

m_arena = &arena->identifierArena();

m_lineNumber = source.firstLine();

m_lastToken = -1;

const String& sourceString = source.provider()->source();

if (!sourceString.isNull())

setCodeStart(sourceString.impl());

else

m_codeStart = 0;

m_source = &source;

m_sourceOffset = source.startOffset();

m_codeStartPlusOffset = m_codeStart + source.startOffset();

m_code = m_codeStartPlusOffset;

m_codeEnd = m_codeStart + source.endOffset();

m_error = false;

m_atLineStart = true;

m_lineStart = m_code;

m_lexErrorMessage = String();

m_buffer8.reserveInitialCapacity(initialReadBufferCapacity);

m_buffer16.reserveInitialCapacity((m_codeEnd - m_code) / 2);

if (LIKELY(m_code < m_codeEnd))

m_current = *m_code;

else

m_current = 0;

ASSERT(currentOffset() == source.startOffset());

}

template <typename T>

template <int shiftAmount> ALWAYS_INLINE void Lexer<T>::internalShift()

{

m_code += shiftAmount;

ASSERT(currentOffset() >= currentLineStartOffset());

m_current = *m_code;

}

template <typename T>

ALWAYS_INLINE void Lexer<T>::shift()

{

// At one point timing showed that setting m_current to 0 unconditionally was faster than an if-else sequence.

m_current = 0;

++m_code;

if (LIKELY(m_code < m_codeEnd))

m_current = *m_code;

}

template <typename T>

ALWAYS_INLINE bool Lexer<T>::atEnd() const

{

ASSERT(!m_current || m_code < m_codeEnd);

return UNLIKELY(UNLIKELY(!m_current) && m_code == m_codeEnd);

}

template <typename T>

ALWAYS_INLINE T Lexer<T>::peek(int offset) const

{

ASSERT(offset > 0 && offset < 5);

const T* code = m_code + offset;

return (code < m_codeEnd) ? *code : 0;

}

template <typename T>

typename Lexer<T>::UnicodeHexValue Lexer<T>::parseFourDigitUnicodeHex()

{

T char1 = peek(1);

T char2 = peek(2);

T char3 = peek(3);

if (UNLIKELY(!isASCIIHexDigit(m_current) || !isASCIIHexDigit(char1) || !isASCIIHexDigit(char2) || !isASCIIHexDigit(char3)))

return UnicodeHexValue((m_code + 4) >= m_codeEnd ? UnicodeHexValue::IncompleteHex : UnicodeHexValue::InvalidHex);

int result = convertUnicode(m_current, char1, char2, char3);

shift();

shift();

shift();

shift();

return UnicodeHexValue(result);

}

template <typename T>

void Lexer<T>::shiftLineTerminator()

{

ASSERT(isLineTerminator(m_current));

m_positionBeforeLastNewline = currentPosition();

T prev = m_current;

shift();

// Allow both CRLF and LFCR.

if (prev + m_current == '\n' + '\r')

shift();

++m_lineNumber;

}

template <typename T>

ALWAYS_INLINE bool Lexer<T>::lastTokenWasRestrKeyword() const

{

return m_lastToken == CONTINUE || m_lastToken == BREAK || m_lastToken == RETURN || m_lastToken == THROW;

}

static NEVER_INLINE bool isNonLatin1IdentStart(UChar c)

{

return U_GET_GC_MASK(c) & U_GC_L_MASK;

}

static ALWAYS_INLINE bool isLatin1(LChar)

{

return true;

}

static ALWAYS_INLINE bool isLatin1(UChar c)

{

return c < 256;

}

static inline bool isIdentStart(LChar c)

{

return typesOfLatin1Characters[c] == CharacterIdentifierStart;

}

static inline bool isIdentStart(UChar c)

{

return isLatin1(c) ? isIdentStart(static_cast<LChar>(c)) : isNonLatin1IdentStart(c);

}

static NEVER_INLINE bool isNonLatin1IdentPart(int c)

{

return (U_GET_GC_MASK(c) & (U_GC_L_MASK | U_GC_MN_MASK | U_GC_MC_MASK | U_GC_ND_MASK | U_GC_PC_MASK)) || c == 0x200C || c == 0x200D;

}

static ALWAYS_INLINE bool isIdentPart(LChar c)

{

// Character types are divided into two groups depending on whether they can be part of an

// identifier or not. Those whose type value is less or equal than CharacterNumber can be

// part of an identifier. (See the CharacterType definition for more details.)

return typesOfLatin1Characters[c] <= CharacterNumber;

}

static ALWAYS_INLINE bool isIdentPart(UChar c)

{

return isLatin1(c) ? isIdentPart(static_cast<LChar>(c)) : isNonLatin1IdentPart(c);

}

static inline LChar singleEscape(int c)

{

if (c < 128) {

ASSERT(static_cast<size_t>(c) < ARRAY_SIZE(singleCharacterEscapeValuesForASCII));

return singleCharacterEscapeValuesForASCII[c];

}

return 0;

}

template <typename T>

inline void Lexer<T>::record8(int c)

{

ASSERT(c >= 0);

ASSERT(c <= 0xFF);

m_buffer8.append(static_cast<LChar>(c));

}

template <typename T>

inline void assertCharIsIn8BitRange(T c)

{

UNUSED_PARAM(c);

ASSERT(c >= 0);

ASSERT(c <= 0xFF);

}

template <>

inline void assertCharIsIn8BitRange(UChar c)

{

UNUSED_PARAM(c);

ASSERT(c <= 0xFF);

}

template <>

inline void assertCharIsIn8BitRange(LChar)

{

}

template <typename T>

inline void Lexer<T>::append8(const T* p, size_t length)

{

size_t currentSize = m_buffer8.size();

m_buffer8.grow(currentSize + length);

LChar* rawBuffer = m_buffer8.data() + currentSize;

for (size_t i = 0; i < length; i++) {

T c = p[i];

assertCharIsIn8BitRange(c);

rawBuffer[i] = c;

}

}

template <typename T>

inline void Lexer<T>::append16(const LChar* p, size_t length)

{

size_t currentSize = m_buffer16.size();

m_buffer16.grow(currentSize + length);

UChar* rawBuffer = m_buffer16.data() + currentSize;

for (size_t i = 0; i < length; i++)

rawBuffer[i] = p[i];

}

template <typename T>

inline void Lexer<T>::record16(T c)

{

m_buffer16.append(c);

}

template <typename T>

inline void Lexer<T>::record16(int c)

{

ASSERT(c >= 0);

ASSERT(c <= static_cast<int>(USHRT_MAX));

m_buffer16.append(static_cast<UChar>(c));

}

#if !ASSERT_DISABLED

bool isSafeBuiltinIdentifier(VM& vm, const Identifier* ident)

{

if (!ident)

return true;

/* Just block any use of suspicious identifiers. This is intended to

* be used as a safety net while implementing builtins.

*/

if (*ident == vm.propertyNames->builtinNames().callPublicName())

return false;

if (*ident == vm.propertyNames->builtinNames().applyPublicName())

return false;

if (*ident == vm.propertyNames->eval)

return false;

if (*ident == vm.propertyNames->Function)

return false;

return true;

}

#endif

template <>

template <bool shouldCreateIdentifier> ALWAYS_INLINE JSTokenType Lexer<LChar>::parseIdentifier(JSTokenData* tokenData, unsigned lexerFlags, bool strictMode)

{

const ptrdiff_t remaining = m_codeEnd - m_code;

if ((remaining >= maxTokenLength) && !(lexerFlags & LexerFlagsIgnoreReservedWords)) {

JSTokenType keyword = parseKeyword<shouldCreateIdentifier>(tokenData);

if (keyword != IDENT) {

ASSERT((!shouldCreateIdentifier) || tokenData->ident);

return keyword == RESERVED_IF_STRICT && !strictMode ? IDENT : keyword;

}

}

bool isPrivateName = m_current == '@' && m_parsingBuiltinFunction;

if (isPrivateName)

shift();

const LChar* identifierStart = currentSourcePtr();

unsigned identifierLineStart = currentLineStartOffset();

while (isIdentPart(m_current))

shift();

if (UNLIKELY(m_current == '\\')) {

setOffsetFromSourcePtr(identifierStart, identifierLineStart);

return parseIdentifierSlowCase<shouldCreateIdentifier>(tokenData, lexerFlags, strictMode);

}

const Identifier* ident = 0;

if (shouldCreateIdentifier || m_parsingBuiltinFunction) {

int identifierLength = currentSourcePtr() - identifierStart;

ident = makeIdentifier(identifierStart, identifierLength);

if (m_parsingBuiltinFunction) {

if (!isSafeBuiltinIdentifier(*m_vm, ident) && !isPrivateName) {

m_lexErrorMessage = makeString("The use of '", ident->string(), "' is disallowed in builtin functions.");

return ERRORTOK;

}

if (isPrivateName)

ident = m_vm->propertyNames->getPrivateName(*ident);

else if (*ident == m_vm->propertyNames->undefinedKeyword)

tokenData->ident = &m_vm->propertyNames->undefinedPrivateName;

if (!ident)

return INVALID_PRIVATE_NAME_ERRORTOK;

}

tokenData->ident = ident;

} else

tokenData->ident = 0;

if (UNLIKELY((remaining < maxTokenLength) && !(lexerFlags & LexerFlagsIgnoreReservedWords)) && !isPrivateName) {

ASSERT(shouldCreateIdentifier);

if (remaining < maxTokenLength) {

const HashTableValue* entry = m_vm->keywords->getKeyword(*ident);

ASSERT((remaining < maxTokenLength) || !entry);

if (!entry)

return IDENT;

JSTokenType token = static_cast<JSTokenType>(entry->lexerValue());

return (token != RESERVED_IF_STRICT) || strictMode ? token : IDENT;

}

return IDENT;

}

return IDENT;

}

template <>

template <bool shouldCreateIdentifier> ALWAYS_INLINE JSTokenType Lexer<UChar>::parseIdentifier(JSTokenData* tokenData, unsigned lexerFlags, bool strictMode)

{

const ptrdiff_t remaining = m_codeEnd - m_code;

if ((remaining >= maxTokenLength) && !(lexerFlags & LexerFlagsIgnoreReservedWords)) {

JSTokenType keyword = parseKeyword<shouldCreateIdentifier>(tokenData);

if (keyword != IDENT) {

ASSERT((!shouldCreateIdentifier) || tokenData->ident);

return keyword == RESERVED_IF_STRICT && !strictMode ? IDENT : keyword;

}

}

bool isPrivateName = m_current == '@' && m_parsingBuiltinFunction;

if (isPrivateName)

shift();

const UChar* identifierStart = currentSourcePtr();

int identifierLineStart = currentLineStartOffset();

UChar orAllChars = 0;

while (isIdentPart(m_current)) {

orAllChars |= m_current;

shift();

}

if (UNLIKELY(m_current == '\\')) {

ASSERT(!isPrivateName);

setOffsetFromSourcePtr(identifierStart, identifierLineStart);

return parseIdentifierSlowCase<shouldCreateIdentifier>(tokenData, lexerFlags, strictMode);

}

bool isAll8Bit = false;

if (!(orAllChars & ~0xff))

isAll8Bit = true;

const Identifier* ident = 0;

if (shouldCreateIdentifier || m_parsingBuiltinFunction) {

int identifierLength = currentSourcePtr() - identifierStart;

if (isAll8Bit)

ident = makeIdentifierLCharFromUChar(identifierStart, identifierLength);

else

ident = makeIdentifier(identifierStart, identifierLength);

if (m_parsingBuiltinFunction) {

if (!isSafeBuiltinIdentifier(*m_vm, ident) && !isPrivateName) {

m_lexErrorMessage = makeString("The use of '", ident->string(), "' is disallowed in builtin functions.");

return ERRORTOK;

}

if (isPrivateName)

ident = m_vm->propertyNames->getPrivateName(*ident);

else if (*ident == m_vm->propertyNames->undefinedKeyword)

tokenData->ident = &m_vm->propertyNames->undefinedPrivateName;

if (!ident)

return INVALID_PRIVATE_NAME_ERRORTOK;

}

tokenData->ident = ident;

} else

tokenData->ident = 0;

if (UNLIKELY((remaining < maxTokenLength) && !(lexerFlags & LexerFlagsIgnoreReservedWords)) && !isPrivateName) {

ASSERT(shouldCreateIdentifier);

if (remaining < maxTokenLength) {

const HashTableValue* entry = m_vm->keywords->getKeyword(*ident);

ASSERT((remaining < maxTokenLength) || !entry);

if (!entry)

return IDENT;

JSTokenType token = static_cast<JSTokenType>(entry->lexerValue());

return (token != RESERVED_IF_STRICT) || strictMode ? token : IDENT;

}

return IDENT;

}

return IDENT;

}

template <typename T>

template <bool shouldCreateIdentifier> JSTokenType Lexer<T>::parseIdentifierSlowCase(JSTokenData* tokenData, unsigned lexerFlags, bool strictMode)

{

const ptrdiff_t remaining = m_codeEnd - m_code;

const T* identifierStart = currentSourcePtr();

bool bufferRequired = false;

while (true) {

if (LIKELY(isIdentPart(m_current))) {

shift();

continue;

}

if (LIKELY(m_current != '\\'))

break;

// \uXXXX unicode characters.

bufferRequired = true;

if (identifierStart != currentSourcePtr())

m_buffer16.append(identifierStart, currentSourcePtr() - identifierStart);

shift();

if (UNLIKELY(m_current != 'u'))

return atEnd() ? UNTERMINATED_IDENTIFIER_ESCAPE_ERRORTOK : INVALID_IDENTIFIER_ESCAPE_ERRORTOK;

shift();

UnicodeHexValue character = parseFourDigitUnicodeHex();

if (UNLIKELY(!character.isValid()))

return character.valueType() == UnicodeHexValue::IncompleteHex ? UNTERMINATED_IDENTIFIER_UNICODE_ESCAPE_ERRORTOK : INVALID_IDENTIFIER_UNICODE_ESCAPE_ERRORTOK;

UChar ucharacter = static_cast<UChar>(character.value());

if (UNLIKELY(m_buffer16.size() ? !isIdentPart(ucharacter) : !isIdentStart(ucharacter)))

return INVALID_IDENTIFIER_UNICODE_ESCAPE_ERRORTOK;

if (shouldCreateIdentifier)

record16(ucharacter);

identifierStart = currentSourcePtr();

}

int identifierLength;

const Identifier* ident = 0;

if (shouldCreateIdentifier) {

if (!bufferRequired) {

identifierLength = currentSourcePtr() - identifierStart;

ident = makeIdentifier(identifierStart, identifierLength);

} else {

if (identifierStart != currentSourcePtr())

m_buffer16.append(identifierStart, currentSourcePtr() - identifierStart);

ident = makeIdentifier(m_buffer16.data(), m_buffer16.size());

}

tokenData->ident = ident;

} else

tokenData->ident = 0;

if (LIKELY(!bufferRequired && !(lexerFlags & LexerFlagsIgnoreReservedWords))) {

ASSERT(shouldCreateIdentifier);

// Keywords must not be recognized if there was an \uXXXX in the identifier.

if (remaining < maxTokenLength) {

const HashTableValue* entry = m_vm->keywords->getKeyword(*ident);

ASSERT((remaining < maxTokenLength) || !entry);

if (!entry)

return IDENT;

JSTokenType token = static_cast<JSTokenType>(entry->lexerValue());

return (token != RESERVED_IF_STRICT) || strictMode ? token : IDENT;

}

return IDENT;

}

m_buffer16.resize(0);

return IDENT;

}

static ALWAYS_INLINE bool characterRequiresParseStringSlowCase(LChar character)

{

return character < 0xE;

}

static ALWAYS_INLINE bool characterRequiresParseStringSlowCase(UChar character)

{

return character < 0xE || character > 0xFF;

}

template <typename T>