/*

* Copyright (C) 2011 Apple Inc. All rights reserved.

*

* Redistribution and use in source and binary forms, with or without

* modification, are permitted provided that the following conditions

* are met:

* 1. Redistributions of source code must retain the above copyright

* notice, this list of conditions and the following disclaimer.

* 2. Redistributions in binary form must reproduce the above copyright

* notice, this list of conditions and the following disclaimer in the

* documentation and/or other materials provided with the distribution.

*

* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY

* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR

* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,

* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,

* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR

* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY

* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

*/

#ifndef DFGSpeculativeJIT_h

#define DFGSpeculativeJIT_h

#if ENABLE(DFG_JIT)

#include "DFGAbstractState.h"

#include "DFGGenerationInfo.h"

#include "DFGJITCompiler.h"

#include "DFGOSRExit.h"

#include "DFGOperations.h"

#include "MarkedAllocator.h"

#include "ValueRecovery.h"

namespace JSC { namespace DFG {

class JSValueOperand;

class SpeculativeJIT;

class SpeculateIntegerOperand;

class SpeculateStrictInt32Operand;

class SpeculateDoubleOperand;

class SpeculateCellOperand;

class SpeculateBooleanOperand;

enum ValueSourceKind {

SourceNotSet,

ValueInRegisterFile,

Int32InRegisterFile,

CellInRegisterFile,

BooleanInRegisterFile,

DoubleInRegisterFile,

HaveNode

};

class ValueSource {

public:

ValueSource()

: m_nodeIndex(nodeIndexFromKind(SourceNotSet))

{

}

explicit ValueSource(ValueSourceKind valueSourceKind)

: m_nodeIndex(nodeIndexFromKind(valueSourceKind))

{

ASSERT(kind() != SourceNotSet);

ASSERT(kind() != HaveNode);

}

explicit ValueSource(NodeIndex nodeIndex)

: m_nodeIndex(nodeIndex)

{

ASSERT(kind() == HaveNode);

}

static ValueSource forPrediction(PredictedType prediction)

{

if (isInt32Prediction(prediction))

return ValueSource(Int32InRegisterFile);

if (isArrayPrediction(prediction) || isByteArrayPrediction(prediction))

return ValueSource(CellInRegisterFile);

if (isBooleanPrediction(prediction))

return ValueSource(BooleanInRegisterFile);

return ValueSource(ValueInRegisterFile);

}

bool isSet() const

{

return kindFromNodeIndex(m_nodeIndex) != SourceNotSet;

}

ValueSourceKind kind() const

{

return kindFromNodeIndex(m_nodeIndex);

}

NodeIndex nodeIndex() const

{

ASSERT(kind() == HaveNode);

return m_nodeIndex;

}

#ifndef NDEBUG

void dump(FILE* out) const;

#endif

private:

static NodeIndex nodeIndexFromKind(ValueSourceKind kind)

{

ASSERT(kind >= SourceNotSet && kind < HaveNode);

return NoNode - kind;

}

static ValueSourceKind kindFromNodeIndex(NodeIndex nodeIndex)

{

unsigned kind = static_cast<unsigned>(NoNode - nodeIndex);

if (kind >= static_cast<unsigned>(HaveNode))

return HaveNode;

return static_cast<ValueSourceKind>(kind);

}

NodeIndex m_nodeIndex;

};

// === SpeculativeJIT ===

//

// The SpeculativeJIT is used to generate a fast, but potentially

// incomplete code path for the dataflow. When code generating

// we may make assumptions about operand types, dynamically check,

// and bail-out to an alternate code path if these checks fail.

// Importantly, the speculative code path cannot be reentered once

// a speculative check has failed. This allows the SpeculativeJIT

// to propagate type information (including information that has

// only speculatively been asserted) through the dataflow.

class SpeculativeJIT {

friend struct OSRExit;

private:

typedef JITCompiler::TrustedImm32 TrustedImm32;

typedef JITCompiler::Imm32 Imm32;

typedef JITCompiler::TrustedImmPtr TrustedImmPtr;

typedef JITCompiler::ImmPtr ImmPtr;

// These constants are used to set priorities for spill order for

// the register allocator.

#if USE(JSVALUE64)

enum SpillOrder {

SpillOrderConstant = 1, // no spill, and cheap fill

SpillOrderSpilled = 2, // no spill

SpillOrderJS = 4, // needs spill

SpillOrderCell = 4, // needs spill

SpillOrderStorage = 4, // needs spill

SpillOrderInteger = 5, // needs spill and box

SpillOrderBoolean = 5, // needs spill and box

SpillOrderDouble = 6, // needs spill and convert

};

#elif USE(JSVALUE32_64)

enum SpillOrder {

SpillOrderConstant = 1, // no spill, and cheap fill

SpillOrderSpilled = 2, // no spill

SpillOrderJS = 4, // needs spill

SpillOrderStorage = 4, // needs spill

SpillOrderDouble = 4, // needs spill

SpillOrderInteger = 5, // needs spill and box

SpillOrderCell = 5, // needs spill and box

SpillOrderBoolean = 5, // needs spill and box

};

#endif

enum UseChildrenMode { CallUseChildren, UseChildrenCalledExplicitly };

public:

SpeculativeJIT(JITCompiler&);

bool compile();

void createOSREntries();

void linkOSREntries(LinkBuffer&);

Node& at(NodeIndex nodeIndex)

{

return m_jit.graph()[nodeIndex];

}

Node& at(NodeUse nodeUse)

{

return at(nodeUse.index());

}

GPRReg fillInteger(NodeIndex, DataFormat& returnFormat);

FPRReg fillDouble(NodeIndex);

#if USE(JSVALUE64)

GPRReg fillJSValue(NodeIndex);

#elif USE(JSVALUE32_64)

bool fillJSValue(NodeIndex, GPRReg&, GPRReg&, FPRReg&);

#endif

GPRReg fillStorage(NodeIndex);

// lock and unlock GPR & FPR registers.

void lock(GPRReg reg)

{

m_gprs.lock(reg);

}

void lock(FPRReg reg)

{

m_fprs.lock(reg);

}

void unlock(GPRReg reg)

{

m_gprs.unlock(reg);

}

void unlock(FPRReg reg)

{

m_fprs.unlock(reg);

}

// Used to check whether a child node is on its last use,

// and its machine registers may be reused.

bool canReuse(NodeIndex nodeIndex)

{

VirtualRegister virtualRegister = at(nodeIndex).virtualRegister();

GenerationInfo& info = m_generationInfo[virtualRegister];

return info.canReuse();

}

bool canReuse(NodeUse nodeUse)

{

return canReuse(nodeUse.index());

}

GPRReg reuse(GPRReg reg)

{

m_gprs.lock(reg);

return reg;

}

FPRReg reuse(FPRReg reg)

{

m_fprs.lock(reg);

return reg;

}

// Allocate a gpr/fpr.

GPRReg allocate()

{

VirtualRegister spillMe;

GPRReg gpr = m_gprs.allocate(spillMe);

if (spillMe != InvalidVirtualRegister) {

#if USE(JSVALUE32_64)

GenerationInfo& info = m_generationInfo[spillMe];

ASSERT(info.registerFormat() != DataFormatJSDouble);

if ((info.registerFormat() & DataFormatJS))

m_gprs.release(info.tagGPR() == gpr ? info.payloadGPR() : info.tagGPR());

#endif

spill(spillMe);

}

return gpr;

}

GPRReg allocate(GPRReg specific)

{

VirtualRegister spillMe = m_gprs.allocateSpecific(specific);

if (spillMe != InvalidVirtualRegister) {

#if USE(JSVALUE32_64)

GenerationInfo& info = m_generationInfo[spillMe];

ASSERT(info.registerFormat() != DataFormatJSDouble);

if ((info.registerFormat() & DataFormatJS))

m_gprs.release(info.tagGPR() == specific ? info.payloadGPR() : info.tagGPR());

#endif

spill(spillMe);

}

return specific;

}

GPRReg tryAllocate()

{

return m_gprs.tryAllocate();

}

FPRReg fprAllocate()

{

VirtualRegister spillMe;

FPRReg fpr = m_fprs.allocate(spillMe);

if (spillMe != InvalidVirtualRegister)

spill(spillMe);

return fpr;

}

// Check whether a VirtualRegsiter is currently in a machine register.

// We use this when filling operands to fill those that are already in

// machine registers first (by locking VirtualRegsiters that are already

// in machine register before filling those that are not we attempt to

// avoid spilling values we will need immediately).

bool isFilled(NodeIndex nodeIndex)

{

VirtualRegister virtualRegister = at(nodeIndex).virtualRegister();

GenerationInfo& info = m_generationInfo[virtualRegister];

return info.registerFormat() != DataFormatNone;

}

bool isFilledDouble(NodeIndex nodeIndex)

{

VirtualRegister virtualRegister = at(nodeIndex).virtualRegister();

GenerationInfo& info = m_generationInfo[virtualRegister];

return info.registerFormat() == DataFormatDouble;

}

// Called on an operand once it has been consumed by a parent node.

void use(NodeIndex nodeIndex)

{

VirtualRegister virtualRegister = at(nodeIndex).virtualRegister();

GenerationInfo& info = m_generationInfo[virtualRegister];

// use() returns true when the value becomes dead, and any

// associated resources may be freed.

if (!info.use())

return;

// Release the associated machine registers.

DataFormat registerFormat = info.registerFormat();

#if USE(JSVALUE64)

if (registerFormat == DataFormatDouble)

m_fprs.release(info.fpr());

else if (registerFormat != DataFormatNone)

m_gprs.release(info.gpr());

#elif USE(JSVALUE32_64)

if (registerFormat == DataFormatDouble || registerFormat == DataFormatJSDouble)

m_fprs.release(info.fpr());

else if (registerFormat & DataFormatJS) {

m_gprs.release(info.tagGPR());

m_gprs.release(info.payloadGPR());

} else if (registerFormat != DataFormatNone)

m_gprs.release(info.gpr());

#endif

}

void use(NodeUse nodeUse)

{

use(nodeUse.index());

}

static void markCellCard(MacroAssembler&, GPRReg ownerGPR, GPRReg scratchGPR1, GPRReg scratchGPR2);

static void writeBarrier(MacroAssembler&, GPRReg ownerGPR, GPRReg scratchGPR1, GPRReg scratchGPR2, WriteBarrierUseKind);

void writeBarrier(GPRReg ownerGPR, GPRReg valueGPR, NodeUse valueUse, WriteBarrierUseKind, GPRReg scratchGPR1 = InvalidGPRReg, GPRReg scratchGPR2 = InvalidGPRReg);

void writeBarrier(GPRReg ownerGPR, JSCell* value, WriteBarrierUseKind, GPRReg scratchGPR1 = InvalidGPRReg, GPRReg scratchGPR2 = InvalidGPRReg);

void writeBarrier(JSCell* owner, GPRReg valueGPR, NodeUse valueUse, WriteBarrierUseKind, GPRReg scratchGPR1 = InvalidGPRReg);

static GPRReg selectScratchGPR(GPRReg preserve1 = InvalidGPRReg, GPRReg preserve2 = InvalidGPRReg, GPRReg preserve3 = InvalidGPRReg, GPRReg preserve4 = InvalidGPRReg)

{

if (preserve1 != GPRInfo::regT0 && preserve2 != GPRInfo::regT0 && preserve3 != GPRInfo::regT0 && preserve4 != GPRInfo::regT0)

return GPRInfo::regT0;

if (preserve1 != GPRInfo::regT1 && preserve2 != GPRInfo::regT1 && preserve3 != GPRInfo::regT1 && preserve4 != GPRInfo::regT1)

return GPRInfo::regT1;

if (preserve1 != GPRInfo::regT2 && preserve2 != GPRInfo::regT2 && preserve3 != GPRInfo::regT2 && preserve4 != GPRInfo::regT2)

return GPRInfo::regT2;

if (preserve1 != GPRInfo::regT3 && preserve2 != GPRInfo::regT3 && preserve3 != GPRInfo::regT3 && preserve4 != GPRInfo::regT3)

return GPRInfo::regT3;

return GPRInfo::regT4;

}

// Called by the speculative operand types, below, to fill operand to

// machine registers, implicitly generating speculation checks as needed.

GPRReg fillSpeculateInt(NodeIndex, DataFormat& returnFormat);

GPRReg fillSpeculateIntStrict(NodeIndex);

FPRReg fillSpeculateDouble(NodeIndex);

GPRReg fillSpeculateCell(NodeIndex);

GPRReg fillSpeculateBoolean(NodeIndex);

private:

void compile(Node&);

void compileMovHint(Node&);

void compile(BasicBlock&);

void checkArgumentTypes();

void clearGenerationInfo();

// These methods are used when generating 'unexpected'

// calls out from JIT code to C++ helper routines -

// they spill all live values to the appropriate

// slots in the RegisterFile without changing any state

// in the GenerationInfo.

void silentSpillGPR(VirtualRegister spillMe, GPRReg source)

{

GenerationInfo& info = m_generationInfo[spillMe];

ASSERT(info.registerFormat() != DataFormatNone);

ASSERT(info.registerFormat() != DataFormatDouble);

if (!info.needsSpill())

return;

DataFormat registerFormat = info.registerFormat();

#if USE(JSVALUE64)

ASSERT(info.gpr() == source);

if (registerFormat == DataFormatInteger)

m_jit.store32(source, JITCompiler::addressFor(spillMe));

else {

ASSERT(registerFormat & DataFormatJS || registerFormat == DataFormatCell || registerFormat == DataFormatStorage);

m_jit.storePtr(source, JITCompiler::addressFor(spillMe));

}

#elif USE(JSVALUE32_64)

if (registerFormat & DataFormatJS) {

ASSERT(info.tagGPR() == source || info.payloadGPR() == source);

m_jit.store32(source, source == info.tagGPR() ? JITCompiler::tagFor(spillMe) : JITCompiler::payloadFor(spillMe));

} else {

ASSERT(info.gpr() == source);

m_jit.store32(source, JITCompiler::payloadFor(spillMe));

}

#endif

}

void silentSpillFPR(VirtualRegister spillMe, FPRReg source)

{

GenerationInfo& info = m_generationInfo[spillMe];

ASSERT(info.registerFormat() == DataFormatDouble);

if (!info.needsSpill()) {

// it's either a constant or it's already been spilled

ASSERT(at(info.nodeIndex()).hasConstant() || info.spillFormat() != DataFormatNone);

return;

}

// it's neither a constant nor has it been spilled.

ASSERT(!at(info.nodeIndex()).hasConstant());

ASSERT(info.spillFormat() == DataFormatNone);

ASSERT(info.fpr() == source);

m_jit.storeDouble(source, JITCompiler::addressFor(spillMe));

}

void silentFillGPR(VirtualRegister spillMe, GPRReg target)

{

GenerationInfo& info = m_generationInfo[spillMe];

NodeIndex nodeIndex = info.nodeIndex();

Node& node = at(nodeIndex);

ASSERT(info.registerFormat() != DataFormatNone);

ASSERT(info.registerFormat() != DataFormatDouble);

DataFormat registerFormat = info.registerFormat();

if (registerFormat == DataFormatInteger) {

ASSERT(info.gpr() == target);

ASSERT(isJSInteger(info.registerFormat()));

if (node.hasConstant()) {

ASSERT(isInt32Constant(nodeIndex));

m_jit.move(Imm32(valueOfInt32Constant(nodeIndex)), target);

} else

m_jit.load32(JITCompiler::payloadFor(spillMe), target);

return;

}

if (registerFormat == DataFormatBoolean) {

#if USE(JSVALUE64)

ASSERT_NOT_REACHED();

#elif USE(JSVALUE32_64)

ASSERT(info.gpr() == target);

if (node.hasConstant()) {

ASSERT(isBooleanConstant(nodeIndex));

m_jit.move(Imm32(valueOfBooleanConstant(nodeIndex)), target);

} else

m_jit.load32(JITCompiler::payloadFor(spillMe), target);

#endif

return;

}

if (registerFormat == DataFormatCell) {

ASSERT(info.gpr() == target);

if (node.hasConstant()) {

JSValue value = valueOfJSConstant(nodeIndex);

ASSERT(value.isCell());

m_jit.move(ImmPtr(value.asCell()), target);

} else

m_jit.loadPtr(JITCompiler::payloadFor(spillMe), target);

return;

}

if (registerFormat == DataFormatStorage) {

ASSERT(info.gpr() == target);

m_jit.loadPtr(JITCompiler::addressFor(spillMe), target);

return;

}

ASSERT(registerFormat & DataFormatJS);

#if USE(JSVALUE64)

ASSERT(info.gpr() == target);

if (node.hasConstant())

m_jit.move(valueOfJSConstantAsImmPtr(nodeIndex), target);

else if (info.spillFormat() == DataFormatInteger) {

ASSERT(registerFormat == DataFormatJSInteger);

m_jit.load32(JITCompiler::payloadFor(spillMe), target);

m_jit.orPtr(GPRInfo::tagTypeNumberRegister, target);

} else if (info.spillFormat() == DataFormatDouble) {

ASSERT(registerFormat == DataFormatJSDouble);

m_jit.loadPtr(JITCompiler::addressFor(spillMe), target);

m_jit.subPtr(GPRInfo::tagTypeNumberRegister, target);

} else

m_jit.loadPtr(JITCompiler::addressFor(spillMe), target);

#else

ASSERT(info.tagGPR() == target || info.payloadGPR() == target);

if (node.hasConstant()) {

JSValue v = valueOfJSConstant(nodeIndex);

m_jit.move(info.tagGPR() == target ? Imm32(v.tag()) : Imm32(v.payload()), target);

} else if (info.payloadGPR() == target)

m_jit.load32(JITCompiler::payloadFor(spillMe), target);

else { // Fill the Tag

switch (info.spillFormat()) {

case DataFormatInteger:

ASSERT(registerFormat == DataFormatJSInteger);

m_jit.move(TrustedImm32(JSValue::Int32Tag), target);

break;

case DataFormatCell:

ASSERT(registerFormat == DataFormatJSCell);

m_jit.move(TrustedImm32(JSValue::CellTag), target);

break;

case DataFormatBoolean:

ASSERT(registerFormat == DataFormatJSBoolean);

m_jit.move(TrustedImm32(JSValue::BooleanTag), target);

break;

default:

m_jit.load32(JITCompiler::tagFor(spillMe), target);

break;

}

}

#endif

}

void silentFillFPR(VirtualRegister spillMe, GPRReg canTrample, FPRReg target)

{

GenerationInfo& info = m_generationInfo[spillMe];

ASSERT(info.fpr() == target);

NodeIndex nodeIndex = info.nodeIndex();

Node& node = at(nodeIndex);

#if USE(JSVALUE64)

ASSERT(info.registerFormat() == DataFormatDouble);

if (node.hasConstant()) {

ASSERT(isNumberConstant(nodeIndex));

m_jit.move(ImmPtr(bitwise_cast<void*>(valueOfNumberConstant(nodeIndex))), canTrample);

m_jit.movePtrToDouble(canTrample, target);

return;

}

if (info.spillFormat() != DataFormatNone && info.spillFormat() != DataFormatDouble) {

// it was already spilled previously and not as a double, which means we need unboxing.

ASSERT(info.spillFormat() & DataFormatJS);

m_jit.loadPtr(JITCompiler::addressFor(spillMe), canTrample);

unboxDouble(canTrample, target);

return;

}

m_jit.loadDouble(JITCompiler::addressFor(spillMe), target);

#elif USE(JSVALUE32_64)

UNUSED_PARAM(canTrample);

ASSERT(info.registerFormat() == DataFormatDouble || info.registerFormat() == DataFormatJSDouble);

if (node.hasConstant()) {

ASSERT(isNumberConstant(nodeIndex));

m_jit.loadDouble(addressOfDoubleConstant(nodeIndex), target);

} else

m_jit.loadDouble(JITCompiler::addressFor(spillMe), target);

#endif

}

void silentSpillAllRegisters(GPRReg exclude, GPRReg exclude2 = InvalidGPRReg)

{

for (gpr_iterator iter = m_gprs.begin(); iter != m_gprs.end(); ++iter) {

GPRReg gpr = iter.regID();

if (iter.name() != InvalidVirtualRegister && gpr != exclude && gpr != exclude2)

silentSpillGPR(iter.name(), gpr);

}

for (fpr_iterator iter = m_fprs.begin(); iter != m_fprs.end(); ++iter) {

if (iter.name() != InvalidVirtualRegister)

silentSpillFPR(iter.name(), iter.regID());

}

}

void silentSpillAllRegisters(FPRReg exclude)

{

for (gpr_iterator iter = m_gprs.begin(); iter != m_gprs.end(); ++iter) {

if (iter.name() != InvalidVirtualRegister)

silentSpillGPR(iter.name(), iter.regID());

}

for (fpr_iterator iter = m_fprs.begin(); iter != m_fprs.end(); ++iter) {

FPRReg fpr = iter.regID();

if (iter.name() != InvalidVirtualRegister && fpr != exclude)

silentSpillFPR(iter.name(), fpr);

}

}

void silentFillAllRegisters(GPRReg exclude, GPRReg exclude2 = InvalidGPRReg)

{

GPRReg canTrample = GPRInfo::regT0;

if (exclude == GPRInfo::regT0)

canTrample = GPRInfo::regT1;

for (fpr_iterator iter = m_fprs.begin(); iter != m_fprs.end(); ++iter) {

if (iter.name() != InvalidVirtualRegister)

silentFillFPR(iter.name(), canTrample, iter.regID());

}

for (gpr_iterator iter = m_gprs.begin(); iter != m_gprs.end(); ++iter) {

GPRReg gpr = iter.regID();

if (iter.name() != InvalidVirtualRegister && gpr != exclude && gpr != exclude2)

silentFillGPR(iter.name(), gpr);

}

}

void silentFillAllRegisters(FPRReg exclude)

{

GPRReg canTrample = GPRInfo::regT0;

for (fpr_iterator iter = m_fprs.begin(); iter != m_fprs.end(); ++iter) {

FPRReg fpr = iter.regID();

if (iter.name() != InvalidVirtualRegister && fpr != exclude)

silentFillFPR(iter.name(), canTrample, fpr);

}

for (gpr_iterator iter = m_gprs.begin(); iter != m_gprs.end(); ++iter) {

if (iter.name() != InvalidVirtualRegister)

silentFillGPR(iter.name(), iter.regID());

}

}

// These methods convert between doubles, and doubles boxed and JSValues.

#if USE(JSVALUE64)

GPRReg boxDouble(FPRReg fpr, GPRReg gpr)

{

return m_jit.boxDouble(fpr, gpr);

}

FPRReg unboxDouble(GPRReg gpr, FPRReg fpr)

{

return m_jit.unboxDouble(gpr, fpr);

}

GPRReg boxDouble(FPRReg fpr)

{

return boxDouble(fpr, allocate());

}

#elif USE(JSVALUE32_64)

void boxDouble(FPRReg fpr, GPRReg tagGPR, GPRReg payloadGPR)

{

m_jit.boxDouble(fpr, tagGPR, payloadGPR);

}

void unboxDouble(GPRReg tagGPR, GPRReg payloadGPR, FPRReg fpr, FPRReg scratchFPR)

{

m_jit.unboxDouble(tagGPR, payloadGPR, fpr, scratchFPR);

}

#endif

// Spill a VirtualRegister to the RegisterFile.

void spill(VirtualRegister spillMe)

{

GenerationInfo& info = m_generationInfo[spillMe];

#if USE(JSVALUE32_64)

if (info.registerFormat() == DataFormatNone) // it has been spilled. JS values which have two GPRs can reach here

return;

#endif

// Check the GenerationInfo to see if this value need writing

// to the RegisterFile - if not, mark it as spilled & return.

if (!info.needsSpill()) {

info.setSpilled();

return;

}

DataFormat spillFormat = info.registerFormat();

switch (spillFormat) {

case DataFormatStorage: {

// This is special, since it's not a JS value - as in it's not visible to JS

// code.

m_jit.storePtr(info.gpr(), JITCompiler::addressFor(spillMe));

info.spill(DataFormatStorage);

return;

}

case DataFormatInteger: {

m_jit.store32(info.gpr(), JITCompiler::payloadFor(spillMe));

info.spill(DataFormatInteger);

return;

}

#if USE(JSVALUE64)

case DataFormatDouble: {

m_jit.storeDouble(info.fpr(), JITCompiler::addressFor(spillMe));

info.spill(DataFormatDouble);

return;

}

default:

// The following code handles JSValues, int32s, and cells.

ASSERT(spillFormat == DataFormatCell || spillFormat & DataFormatJS);

GPRReg reg = info.gpr();

// We need to box int32 and cell values ...

// but on JSVALUE64 boxing a cell is a no-op!

if (spillFormat == DataFormatInteger)

m_jit.orPtr(GPRInfo::tagTypeNumberRegister, reg);

// Spill the value, and record it as spilled in its boxed form.

m_jit.storePtr(reg, JITCompiler::addressFor(spillMe));

info.spill((DataFormat)(spillFormat | DataFormatJS));

return;

#elif USE(JSVALUE32_64)

case DataFormatCell:

case DataFormatBoolean: {

m_jit.store32(info.gpr(), JITCompiler::payloadFor(spillMe));

info.spill(spillFormat);

return;

}

case DataFormatDouble:

case DataFormatJSDouble: {

// On JSVALUE32_64 boxing a double is a no-op.

m_jit.storeDouble(info.fpr(), JITCompiler::addressFor(spillMe));

info.spill(DataFormatJSDouble);

return;

}

default:

// The following code handles JSValues.

ASSERT(spillFormat & DataFormatJS);

m_jit.store32(info.tagGPR(), JITCompiler::tagFor(spillMe));

m_jit.store32(info.payloadGPR(), JITCompiler::payloadFor(spillMe));

info.spill(spillFormat);

return;

#endif

}

}

bool isStrictInt32(NodeIndex);

bool isKnownInteger(NodeIndex);

bool isKnownNumeric(NodeIndex);

bool isKnownCell(NodeIndex);

bool isKnownNotInteger(NodeIndex);

bool isKnownNotNumber(NodeIndex);

bool isKnownBoolean(NodeIndex);

bool isKnownNotBoolean(NodeIndex);

bool isKnownNotCell(NodeIndex);

// Checks/accessors for constant values.

bool isConstant(NodeIndex nodeIndex) { return m_jit.isConstant(nodeIndex); }

bool isJSConstant(NodeIndex nodeIndex) { return m_jit.isJSConstant(nodeIndex); }

bool isInt32Constant(NodeIndex nodeIndex) { return m_jit.isInt32Constant(nodeIndex); }

bool isDoubleConstant(NodeIndex nodeIndex) { return m_jit.isDoubleConstant(nodeIndex); }

bool isNumberConstant(NodeIndex nodeIndex) { return m_jit.isNumberConstant(nodeIndex); }

bool isBooleanConstant(NodeIndex nodeIndex) { return m_jit.isBooleanConstant(nodeIndex); }

bool isFunctionConstant(NodeIndex nodeIndex) { return m_jit.isFunctionConstant(nodeIndex); }

int32_t valueOfInt32Constant(NodeIndex nodeIndex) { return m_jit.valueOfInt32Constant(nodeIndex); }

double valueOfNumberConstant(NodeIndex nodeIndex) { return m_jit.valueOfNumberConstant(nodeIndex); }

int32_t valueOfNumberConstantAsInt32(NodeIndex nodeIndex)

{

if (isInt32Constant(nodeIndex))

return valueOfInt32Constant(nodeIndex);

return JSC::toInt32(valueOfNumberConstant(nodeIndex));

}

#if USE(JSVALUE32_64)

void* addressOfDoubleConstant(NodeIndex nodeIndex) { return m_jit.addressOfDoubleConstant(nodeIndex); }

#endif

JSValue valueOfJSConstant(NodeIndex nodeIndex) { return m_jit.valueOfJSConstant(nodeIndex); }

bool valueOfBooleanConstant(NodeIndex nodeIndex) { return m_jit.valueOfBooleanConstant(nodeIndex); }

JSFunction* valueOfFunctionConstant(NodeIndex nodeIndex) { return m_jit.valueOfFunctionConstant(nodeIndex); }

bool isNullConstant(NodeIndex nodeIndex)

{

if (!isConstant(nodeIndex))

return false;

return valueOfJSConstant(nodeIndex).isNull();

}

Identifier* identifier(unsigned index)

{

return &m_jit.codeBlock()->identifier(index);

}

// Spill all VirtualRegisters back to the RegisterFile.

void flushRegisters()

{

for (gpr_iterator iter = m_gprs.begin(); iter != m_gprs.end(); ++iter) {

if (iter.name() != InvalidVirtualRegister) {

spill(iter.name());

iter.release();

}

}

for (fpr_iterator iter = m_fprs.begin(); iter != m_fprs.end(); ++iter) {

if (iter.name() != InvalidVirtualRegister) {

spill(iter.name());

iter.release();

}

}

}

#ifndef NDEBUG

// Used to ASSERT flushRegisters() has been called prior to

// calling out from JIT code to a C helper function.

bool isFlushed()

{

for (gpr_iterator iter = m_gprs.begin(); iter != m_gprs.end(); ++iter) {

if (iter.name() != InvalidVirtualRegister)

return false;

}

for (fpr_iterator iter = m_fprs.begin(); iter != m_fprs.end(); ++iter) {

if (iter.name() != InvalidVirtualRegister)

return false;

}

return true;

}

#endif

#if USE(JSVALUE64)

MacroAssembler::ImmPtr valueOfJSConstantAsImmPtr(NodeIndex nodeIndex)

{

return MacroAssembler::ImmPtr(JSValue::encode(valueOfJSConstant(nodeIndex)));

}

#endif

// Helper functions to enable code sharing in implementations of bit/shift ops.

void bitOp(NodeType op, int32_t imm, GPRReg op1, GPRReg result)

{

switch (op) {

case BitAnd:

m_jit.and32(Imm32(imm), op1, result);

break;

case BitOr:

m_jit.or32(Imm32(imm), op1, result);

break;

case BitXor:

m_jit.xor32(Imm32(imm), op1, result);

break;

default:

ASSERT_NOT_REACHED();

}

}

void bitOp(NodeType op, GPRReg op1, GPRReg op2, GPRReg result)

{

switch (op) {

case BitAnd:

m_jit.and32(op1, op2, result);

break;

case BitOr:

m_jit.or32(op1, op2, result);

break;

case BitXor:

m_jit.xor32(op1, op2, result);

break;

default:

ASSERT_NOT_REACHED();

}

}

void shiftOp(NodeType op, GPRReg op1, int32_t shiftAmount, GPRReg result)

{

switch (op) {

case BitRShift:

m_jit.rshift32(op1, Imm32(shiftAmount), result);

break;

case BitLShift:

m_jit.lshift32(op1, Imm32(shiftAmount), result);

break;

case BitURShift:

m_jit.urshift32(op1, Imm32(shiftAmount), result);

break;

default:

ASSERT_NOT_REACHED();

}

}

void shiftOp(NodeType op, GPRReg op1, GPRReg shiftAmount, GPRReg result)

{

switch (op) {

case BitRShift:

m_jit.rshift32(op1, shiftAmount, result);

break;

case BitLShift:

m_jit.lshift32(op1, shiftAmount, result);

break;

case BitURShift:

m_jit.urshift32(op1, shiftAmount, result);

break;

default:

ASSERT_NOT_REACHED();

}

}

// Returns the node index of the branch node if peephole is okay, NoNode otherwise.

NodeIndex detectPeepHoleBranch()

{

NodeIndex lastNodeIndex = m_jit.graph().m_blocks[m_block]->end - 1;

// Check that no intervening nodes will be generated.

for (NodeIndex index = m_compileIndex + 1; index < lastNodeIndex; ++index) {

if (at(index).shouldGenerate())

return NoNode;

}

// Check if the lastNode is a branch on this node.

Node& lastNode = at(lastNodeIndex);

return lastNode.op == Branch && lastNode.child1().index() == m_compileIndex ? lastNodeIndex : NoNode;

}

void nonSpeculativeValueToNumber(Node&);

void nonSpeculativeValueToInt32(Node&);

void nonSpeculativeUInt32ToNumber(Node&);

enum SpillRegistersMode { NeedToSpill, DontSpill };

#if USE(JSVALUE64)

JITCompiler::Call cachedGetById(CodeOrigin, GPRReg baseGPR, GPRReg resultGPR, GPRReg scratchGPR, unsigned identifierNumber, JITCompiler::Jump slowPathTarget = JITCompiler::Jump(), SpillRegistersMode = NeedToSpill);

void cachedPutById(CodeOrigin, GPRReg base, GPRReg value, NodeUse valueUse, GPRReg scratchGPR, unsigned identifierNumber, PutKind, JITCompiler::Jump slowPathTarget = JITCompiler::Jump());

#elif USE(JSVALUE32_64)

JITCompiler::Call cachedGetById(CodeOrigin, GPRReg baseTagGPROrNone, GPRReg basePayloadGPR, GPRReg resultTagGPR, GPRReg resultPayloadGPR, GPRReg scratchGPR, unsigned identifierNumber, JITCompiler::Jump slowPathTarget = JITCompiler::Jump(), SpillRegistersMode = NeedToSpill);

void cachedPutById(CodeOrigin, GPRReg basePayloadGPR, GPRReg valueTagGPR, GPRReg valuePayloadGPR, NodeUse valueUse, GPRReg scratchGPR, unsigned identifierNumber, PutKind, JITCompiler::Jump slowPathTarget = JITCompiler::Jump());

#endif

void nonSpeculativeNonPeepholeCompareNull(NodeUse operand, bool invert = false);

void nonSpeculativePeepholeBranchNull(NodeUse operand, NodeIndex branchNodeIndex, bool invert = false);

bool nonSpeculativeCompareNull(Node&, NodeUse operand, bool invert = false);

void nonSpeculativePeepholeBranch(Node&, NodeIndex branchNodeIndex, MacroAssembler::RelationalCondition, S_DFGOperation_EJJ helperFunction);

void nonSpeculativeNonPeepholeCompare(Node&, MacroAssembler::RelationalCondition, S_DFGOperation_EJJ helperFunction);

bool nonSpeculativeCompare(Node&, MacroAssembler::RelationalCondition, S_DFGOperation_EJJ helperFunction);

void nonSpeculativePeepholeStrictEq(Node&, NodeIndex branchNodeIndex, bool invert = false);

void nonSpeculativeNonPeepholeStrictEq(Node&, bool invert = false);

bool nonSpeculativeStrictEq(Node&, bool invert = false);

void compileInstanceOfForObject(Node&, GPRReg valueReg, GPRReg prototypeReg, GPRReg scratchAndResultReg);

void compileInstanceOf(Node&);

// Access to our fixed callee CallFrame.

MacroAssembler::Address callFrameSlot(int slot)

{

return MacroAssembler::Address(GPRInfo::callFrameRegister, (m_jit.codeBlock()->m_numCalleeRegisters + slot) * static_cast<int>(sizeof(Register)));

}

// Access to our fixed callee CallFrame.

MacroAssembler::Address argumentSlot(int argument)

{

return MacroAssembler::Address(GPRInfo::callFrameRegister, (m_jit.codeBlock()->m_numCalleeRegisters + argumentToOperand(argument)) * static_cast<int>(sizeof(Register)));

}

MacroAssembler::Address callFrameTagSlot(int slot)

{

return MacroAssembler::Address(GPRInfo::callFrameRegister, (m_jit.codeBlock()->m_numCalleeRegisters + slot) * static_cast<int>(sizeof(Register)) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag));

}

MacroAssembler::Address callFramePayloadSlot(int slot)

{

return MacroAssembler::Address(GPRInfo::callFrameRegister, (m_jit.codeBlock()->m_numCalleeRegisters + slot) * static_cast<int>(sizeof(Register)) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));

}

MacroAssembler::Address argumentTagSlot(int argument)

{

return MacroAssembler::Address(GPRInfo::callFrameRegister, (m_jit.codeBlock()->m_numCalleeRegisters + argumentToOperand(argument)) * static_cast<int>(sizeof(Register)) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag));

}

MacroAssembler::Address argumentPayloadSlot(int argument)

{

return MacroAssembler::Address(GPRInfo::callFrameRegister, (m_jit.codeBlock()->m_numCalleeRegisters + argumentToOperand(argument)) * static_cast<int>(sizeof(Register)) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));

}

void emitCall(Node&);

// Called once a node has completed code generation but prior to setting

// its result, to free up its children. (This must happen prior to setting

// the nodes result, since the node may have the same VirtualRegister as

// a child, and as such will use the same GeneratioInfo).

void useChildren(Node&);

// These method called to initialize the the GenerationInfo

// to describe the result of an operation.

void integerResult(GPRReg reg, NodeIndex nodeIndex, DataFormat format = DataFormatInteger, UseChildrenMode mode = CallUseChildren)

{

Node& node = at(nodeIndex);

if (mode == CallUseChildren)

useChildren(node);

VirtualRegister virtualRegister = node.virtualRegister();

GenerationInfo& info = m_generationInfo[virtualRegister];

if (format == DataFormatInteger) {

m_jit.jitAssertIsInt32(reg);

m_gprs.retain(reg, virtualRegister, SpillOrderInteger);

info.initInteger(nodeIndex, node.refCount(), reg);

} else {

#if USE(JSVALUE64)

ASSERT(format == DataFormatJSInteger);

m_jit.jitAssertIsJSInt32(reg);

m_gprs.retain(reg, virtualRegister, SpillOrderJS);

info.initJSValue(nodeIndex, node.refCount(), reg, format);

#elif USE(JSVALUE32_64)

ASSERT_NOT_REACHED();

#endif

}

}

void integerResult(GPRReg reg, NodeIndex nodeIndex, UseChildrenMode mode)

{

integerResult(reg, nodeIndex, DataFormatInteger, mode);

}

void noResult(NodeIndex nodeIndex, UseChildrenMode mode = CallUseChildren)

{

if (mode == UseChildrenCalledExplicitly)

return;

Node& node = at(nodeIndex);

useChildren(node);

}

void cellResult(GPRReg reg, NodeIndex nodeIndex, UseChildrenMode mode = CallUseChildren)

{