std::vector<AttentionTreeEncDec::Data*>& trainData_,

std::vector<AttentionTreeEncDec::Data*>& devData_,

const int inputDim, const int hiddenDim, const Real scale,

const bool useBlackOut_,

const int blackOutSampleNum, const Real blackOutAlpha,

AttentionTreeEncDec::AttentionType attenType_,

AttentionTreeEncDec::ParsedTree parsedTree_,

AttentionTreeEncDec::InitDec initDec_,

const bool reversed_, const bool biasOne_, const Real clipThreshold_,

const int beamSize_, const int maxGeneNum_,

const int miniBatchSize_, const int threadNum_,

const Real learningRate_, const bool learningRateSchedule_,

const int vocaThreshold_,

const bool inputFeeding_,

const std::string& saveDirName_):

attenType(attenType_), parsedTree(parsedTree_), initDec(initDec_), useBlackOut(useBlackOut_),

reversed(reversed_), biasOne(biasOne_), clipThreshold(clipThreshold_),

sourceVoc(sourceVoc_), targetVoc(targetVoc_),

trainData(trainData_), devData(devData_),

beamSize(beamSize_), maxGeneNum(maxGeneNum_),

miniBatchSize(miniBatchSize_), threadNum(threadNum_),

learningRate(learningRate_), learningRateSchedule(learningRateSchedule_),

vocaThreshold(vocaThreshold_),

inputFeeding(inputFeeding_),

saveDirName(saveDirName_)

{

this->enc = LSTM(inputDim, hiddenDim); // Encoder; set dimension

this->enc.init(this->rnd, scale);

this->dec = LSTM(inputDim, hiddenDim, hiddenDim); // Decoder; set dimension

this->dec.init(this->rnd, scale);

this->encTree = TreeLSTM(inputDim, hiddenDim); // Encoder Tree; set dimension

this->encTree.init(this->rnd, scale);

this->initDecTree = TreeLSTM(inputDim, hiddenDim); // Decoder's Initial Tree; set dimension

this->initDecTree.init(this->rnd, scale);

if (this->biasOne) { // LSTMs' biases set to 1

this->enc.bf.fill(1.0);

this->dec.bf.fill(1.0);

this->encTree.bfl.fill(1.0);

this->encTree.bfr.fill(1.0);

this->initDecTree.bfl.fill(1.0);

this->initDecTree.bfr.fill(1.0);

}

//

this->sourceEmbed = MatD(inputDim, this->sourceVoc.tokenList.size());

this->targetEmbed = MatD(inputDim, this->targetVoc.tokenList.size());

this->rnd.uniform(this->sourceEmbed, scale);

this->rnd.uniform(this->targetEmbed, scale);

this->zeros = VecD::Zero(hiddenDim); // Zero vector

// initialize W

this->Wst = MatD(hiddenDim, hiddenDim); // s_tilde's weight for decoder s_t

this->Wct = MatD(hiddenDim, hiddenDim); // .. for Tree and Sequnece

this->bs = VecD::Zero(hiddenDim); // .. for bias

this->rnd.uniform(this->Wst, scale);

this->rnd.uniform(this->Wct, scale);

// attentionType == GENERAL

this->WgeneralTree = MatD(hiddenDim, hiddenDim); // attenType == GENERAL; for Tree and Sequence

this->rnd.uniform(this->WgeneralTree, scale);

// attentional weight set to a uniform distribution

// beacuase of score(i, j) = exp(0) = 1

// initDec == CONCAT

this->WcellSeq = MatD(hiddenDim, hiddenDim); // Initial Decoder LSTM (cell memory); c = tanh(W0*c_seq + W1*c_tree)

this->WcellTree = MatD(hiddenDim, hiddenDim);

this->WhSeq = MatD(hiddenDim, hiddenDim); // ................... (hidden state); h = tanh(W2*h_seq + W3*h_tree)

this->WhTree = MatD(hiddenDim, hiddenDim);

this->rnd.uniform(this->WcellSeq, scale);

this->rnd.uniform(this->WcellTree, scale);

this->rnd.uniform(this->WhSeq, scale);

this->rnd.uniform(this->WhTree, scale);

// for Tree Encoder

this->initTreeEncState =

new AttentionTreeEncDec::StateNode(-1, new TreeLSTM::State, 0, 0); // tokenIndex=-1; (root or node);

this->initTreeEncState->state->h = this->zeros;

this->initTreeEncState->state->c = this->zeros;

this->initTreeEncState->state->delc = this->zeros;

if (!this->useBlackOut) {

this->softmax = SoftMax(hiddenDim, this->targetVoc.tokenList.size());

}

else {

VecD freq = VecD(this->targetVoc.tokenList.size());

for (int i = 0; i < (int)this->targetVoc.tokenList.size(); ++i) {

freq.coeffRef(i, 0) = this->targetVoc.tokenList[i]->count;

}

this->blackOut = BlackOut(hiddenDim, this->targetVoc.tokenList.size(), blackOutSampleNum);

this->blackOut.initSampling(freq, blackOutAlpha);

}

std::vector<LSTM::State*>& encState) { // Encoder for sequence

this->enc.forward(this->sourceEmbed.col(src[0]), encState[1]);

encState[1]->delc = this->zeros; // (!) Initialize here for backward

encState[1]->delh = this->zeros;

for (int i = 1; i < (int)src.size(); ++i) {

this->enc.forward(this->sourceEmbed.col(src[i]), encState[i], encState[i+1]);

encState[i+1]->delc = this->zeros; // (!) Initialize here for backward

encState[i+1]->delh = this->zeros;

}

node->state->delh = this->zeros;

node->state->delc = this->zeros;

// leaf

if (node->tokenIndex >= 0) {

return;

}

// non-leaf

else {

this->encodeTree(node->left);

this->encodeTree(node->right);

this->encTree.forward((TreeLSTM::State*)node->state, node->left->state, node->right->state);

}

bool operator()(const AttentionTreeEncDec::DecCandidate left, const AttentionTreeEncDec::DecCandidate right) {

return left.score > right.score;

}

this->encode(src, state->encState); // encoder

if (state->encTreeNodeVec.size() > 0) {

for (int i = 0; i < (int)state->encTreeLeafVec.size(); ++i) {

state->encTreeLeafVec[i]->state->c = state->encState[i+1]->c; // ``encState`` is 1-origin; encState[0] == h0

state->encTreeLeafVec[i]->state->h = state->encState[i+1]->h;

}

this->encodeTree(state->encTreeState); // encoder for TREE

//this->showTreeNode(state->encTreeState);

} else {}

VecD& s_tilde, VecD& c0, VecD& s0, const int i) {

if (i == 0) { // initialize decoder's initial state

if (initDec == AttentionTreeEncDec::TREELSTM) { // Left: Sequence, Right: Tree (Phrase node)

// copy ``c`` and ``h``

this->initDecTree.forward((TreeLSTM::State*)state->decState[i],

state->encState.back(), state->encTreeState->state);

} else if (initDec == AttentionTreeEncDec::CONCAT) { // inspired by (Bahdanau et al., 2015)

c0 = this->WcellSeq*state->encState.back()->c

+ this->WcellTree*state->encTreeState->state->c;

s0 = this->WhSeq*state->encState.back()->h

+ this->WhTree*state->encTreeState->state->h;

ActFunc::tanh(c0);

ActFunc::tanh(s0);

state->decState[i]->c = c0; // tanh(W[cell_Seq; cell_Tree])

state->decState[i]->h = s0; // tanh(W[h_Seq; h_Tree])

} else {}

}

else { // i >= 1

if (this->inputFeeding) {

// input-feeding approach [Luong et al., EMNLP2015]

this->dec.forward(this->targetEmbed.col(data->tgt[i-1]), s_tilde,

state->decState[i-1], state->decState[i]); // (xt, at (use previous ``s_tilde``, prev, cur)

} else {

this->dec.forward(this->targetEmbed.col(data->tgt[i-1]),

state->decState[i-1], state->decState[i]); // (xt, prev, cur)

}

}

VecD& contextTree, VecD& alphaTree, VecD& s_tilde, const int i) {

/* Attention */

s_tilde = this->bs;

s_tilde.noalias() += this->Wst*decState[i]->h; // Set ``s_tilde``

// sequence and Tree (phrase node)

contextTree = this->zeros;

this->calculateAlpha(state, decState[i], alphaTree);

for (int j = 1; j < (int)state->encState.size(); ++j) {

contextTree += alphaTree.coeff(j-1, 0)*state->encState[j]->h;

}

for (int j = 0; j < (int)state->encTreeNodeVec.size(); ++j) {

contextTree += alphaTree.coeff(j+(int)state->encState.size()-1, 0)*state->encTreeNodeVec[j]->state->h;

}

s_tilde += this->Wct*contextTree;

ActFunc::tanh(s_tilde); // s~tanh(W_c[ht])

VecD& s_tilde, const std::vector<int>& tgt,

VecD& c0, VecD& s0, const int i) { // For translate()

if (i == 0) { // initialize decoder's initial state

if (initDec == AttentionTreeEncDec::TREELSTM) { // Left: Sequence, Right: Tree (Phrase node)

// copy ``c`` and ``h``

this->initDecTree.forward((TreeLSTM::State*)decState[i],

state->encState.back(), state->encTreeState->state);

} else if (initDec == AttentionTreeEncDec::CONCAT) { // inspired by (Bahdanau et al., 2015)

c0 = this->WcellSeq*state->encState.back()->c

+ this->WcellTree*state->encTreeState->state->c;

s0 = this->WhSeq*state->encState.back()->h

+ this->WhTree*state->encTreeState->state->h;

ActFunc::tanh(c0);

ActFunc::tanh(s0);

decState[i]->c = c0; // tanh(W[cell_Seq; cell_Tree])

decState[i]->h = s0; // tanh(W[h_Seq; h_Tree])

} else {}

}

else { // i >= 1

if (this->inputFeeding) {

// input-feeding approach [Luong et al., EMNLP2015]

this->dec.forward(this->targetEmbed.col(tgt[i-1]), s_tilde,

decState[i-1], decState[i]); // (xt, at (use previous ``s_tilde``, prev, cur)

} else {

this->dec.forward(this->targetEmbed.col(tgt[i-1]),

decState[i-1], decState[i]); // (xt, prev, cur)

}

}

std::ifstream ifs("stat.txt");

std::vector<std::string> res;

VecD prob;

int len = 0;

for (std::string line; std::getline(ifs, line); ++len){

Utils::split(line, res);

prob = VecD(res.size());

for (int i = 0; i < (int)res.size(); ++i){

prob.coeffRef(i, 0) = atof(res[i].c_str());

}

if (prob.sum() == 0.0){

continue;

}

for (int i = 0; i < prob.rows(); ++i){

if (prob.coeff(i, 0) == 0.0){

continue;

}

stat[len][i] = prob.coeff(i, 0);

}

}

AttentionTreeEncDec::State* state,

const int beamSize, const int maxLength, const int showNum) {

const Real minScore = -1.0e+05;

MatD score(this->targetEmbed.cols(), beamSize);

VecD targetDist;

std::vector<int> tgt;

std::vector<LSTM::State*> stateList;

std::vector<AttentionTreeEncDec::DecCandidate> candidate(beamSize), candidateTmp(beamSize);

VecD c0, s0; // cell memory and hidden state for initial Decorder (initDec == CONCAT)

const int alphaSize = state->encState.size()-1+state->encTreeNodeVec.size();

VecD alphaTree = VecD(alphaSize); // Vector for attentional weight

VecD contextTree; // C_Tree = Σ (alpha * hidden state)

this->encoder(src, state);

for (int i = 0; i < maxLength; ++i) {

for (int j = 0; j < beamSize; ++j) {

if (candidate[j].stop) {

score.col(j).fill(candidate[j].score);

continue;

}

if (i == 0) {

candidate[j].decState.push_back(new TreeLSTM::State);

} else {

candidate[j].decState.push_back(new LSTM::State);

}

stateList.push_back(candidate[j].decState.back()); // ``stateList`` holds a list of the added LSTM units

this->candidateDecoder(state, candidate[j].decState, candidate[j].s_tilde, candidate[j].tgt, c0, s0, i);

this->decoderAttention(state, candidate[j].decState, contextTree, alphaTree, candidate[j].s_tilde, i);

// candidate[j].showAlphaTree.row(i) = alphaTree.transpose();

if (!this->useBlackOut) {

this->softmax.calcDist(candidate[j].s_tilde, targetDist);

}

else {

this->blackOut.calcDist(candidate[j].s_tilde, targetDist);

}

score.col(j).array() = candidate[j].score + targetDist.array().log();

}

for (int j = 0, row, col; j < beamSize; ++j) {

score.maxCoeff(&row, &col); // Greedy;

candidateTmp[j] = candidate[col];

candidateTmp[j].score = score.coeff(row, col);

if (candidateTmp[j].stop) { // if "EOS" comes up...

score.col(col).fill(minScore);

continue;

}

candidateTmp[j].tgt.push_back(row);

if (row == this->targetVoc.eosIndex) {

candidateTmp[j].stop = true;

}

if (i == 0) {

score.row(row).fill(minScore);

}

else {

score.coeffRef(row, col) = minScore;

}

}

candidate = candidateTmp;

std::sort(candidate.begin(), candidate.end(), sort_pred());

if (candidate[0].tgt.back() == this->targetVoc.eosIndex) {

break;

}

}

for (auto it = src.begin(); it != src.end(); ++it) {

std::cout << this->sourceVoc.tokenList[*it]->str << " ";

}

std::cout << std::endl;

for (int i = 0; i < showNum; ++i) {

std::cout << i+1 << " (" << candidate[i].score << "): ";

for (auto it = candidate[i].tgt.begin(); it != candidate[i].tgt.end(); ++it) {

std::cout << this->targetVoc.tokenList[*it]->str << " ";

}

std::cout << std::endl;

}

for (auto it = src.begin(); it != src.end(); ++it) {

std::cout << this->sourceVoc.tokenList[*it]->str << " ";

}

std::cout << std::endl;

for (auto it = tgt.begin(); it != tgt.end(); ++it) {

std::cout << this->targetVoc.tokenList[*it]->str << " ";

}

std::cout << std::endl;

this->deleteStateList(stateList);

this->clearState(state, srcParsedFlag);

std::vector<int>& trans, AttentionTreeEncDec::State* state,

const int beamSize, const int maxLength) {

const Real minScore = -1.0e+05;

MatD score(this->targetEmbed.cols(), beamSize);

VecD targetDist;

std::vector<int> tgt;

std::vector<LSTM::State*> stateList;

std::vector<AttentionTreeEncDec::DecCandidate> candidate(beamSize), candidateTmp(beamSize);

VecD c0, s0; // cell memory and hidden state for initial Decorder (initDec == CONCAT)

const int alphaSize = state->encState.size()-1+state->encTreeNodeVec.size();

VecD alphaTree = VecD(alphaSize); // x for attentional weight

VecD contextTree; // C_Tree = Σ (alpha * hidden state)

this->encoder(src, state);

for (int i = 0; i < maxLength; ++i) {

for (int j = 0; j < beamSize; ++j) {

if (candidate[j].stop) {

score.col(j).fill(candidate[j].score);

continue;

}

if (i == 0) {

candidate[j].decState.push_back(new TreeLSTM::State);

} else {

candidate[j].decState.push_back(new LSTM::State);

}

stateList.push_back(candidate[j].decState.back()); // ``stateList`` holds a list of the added LSTM units

this->candidateDecoder(state, candidate[j].decState, candidate[j].s_tilde, candidate[j].tgt, c0, s0, i);

this->decoderAttention(state, candidate[j].decState, contextTree, alphaTree, candidate[j].s_tilde, i);

// candidate[j].showAlphaTree.row(i) = alphaTree.transpose();

if (!this->useBlackOut) {

this->softmax.calcDist(candidate[j].s_tilde, targetDist);

}

else {

this->blackOut.calcDist(candidate[j].s_tilde, targetDist);

}

score.col(j).array() = candidate[j].score + targetDist.array().log();

}

for (int j = 0, row, col; j < beamSize; ++j) {

score.maxCoeff(&row, &col); // Greedy;

candidateTmp[j] = candidate[col];

candidateTmp[j].score = score.coeff(row, col);

if (candidateTmp[j].stop) { // if "EOS" comes up...

score.col(col).fill(minScore);

continue;

}

candidateTmp[j].tgt.push_back(row);

if (row == this->targetVoc.eosIndex) {

candidateTmp[j].stop = true;

}

if (i == 0) {

score.row(row).fill(minScore);

} else {

score.coeffRef(row, col) = minScore;

}

}

candidate = candidateTmp;

std::sort(candidate.begin(), candidate.end(), sort_pred());

if (candidate[0].tgt.back() == this->targetVoc.eosIndex) {

break;

}

}

this->makeTrans(candidate[0].tgt, trans);

this->deleteStateList(stateList);

this->clearState(state, srcParsedFlag);

std::vector<int>& trans, AttentionTreeEncDec::State* state,

const std::unordered_map<int, std::unordered_map<int, Real> >& stat,

const int beamSize, const int maxLength) {

const Real minScore = -1.0e+05;

const int srcLen = src.size()-1;

MatD score(this->targetEmbed.cols(), beamSize);

VecD targetDist;

std::vector<int> tgt;

std::vector<LSTM::State*> stateList;

std::vector<AttentionTreeEncDec::DecCandidate> candidate(beamSize), candidateTmp(beamSize);

VecD c0, s0; // cell memory and hidden state for initial Decorder (initDec == CONCAT)

const int alphaSize = state->encState.size()-1+state->encTreeNodeVec.size();

VecD alphaTree = VecD(alphaSize); // x for attentional weight

VecD contextTree; // C_Tree = Σ (alpha * hidden state)

for (int j=0; j < beamSize; ++j) {

candidate[j].showAlphaTree = MatD::Zero(maxLength, alphaSize);

}

/* Encoder starts */

this->encoder(src, state);

for (int i = 0; i < maxLength; ++i) {

for (int j = 0; j < beamSize; ++j) {

if (candidate[j].stop) {

score.col(j).fill(candidate[j].score);

continue;

} else{}

if (i == 0) {

candidate[j].decState.push_back(new TreeLSTM::State);

} else {

candidate[j].decState.push_back(new LSTM::State);

}

stateList.push_back(candidate[j].decState.back()); // ``stateList`` holds a list of the added LSTM units

this->candidateDecoder(state, candidate[j].decState, candidate[j].s_tilde, candidate[j].tgt, c0, s0, i);

this->decoderAttention(state, candidate[j].decState, contextTree, alphaTree, candidate[j].s_tilde, i);

// candidate[j].showAlphaTree.row(i) = alphaTree.transpose();

if (!this->useBlackOut) {

this->softmax.calcDist(candidate[j].s_tilde, targetDist);

}

else {

this->blackOut.calcDist(candidate[j].s_tilde, targetDist);

}

score.col(j).array() = candidate[j].score + targetDist.array().log();

}

for (int j = 0, row, col; j < beamSize; ++j) {

score.maxCoeff(&row, &col); // Greedy;

candidateTmp[j] = candidate[col];

candidateTmp[j].score = score.coeff(row, col);

if (candidateTmp[j].stop) { // if "EOS" comes up...

score.col(col).fill(minScore);

continue;

}

if (row == this->targetVoc.eosIndex) {

if (stat.count(srcLen)){

const int tgtLen = candidateTmp[j].tgt.size();

if (stat.at(srcLen).count(tgtLen)){

candidateTmp[j].score += log(stat.at(srcLen).at(tgtLen));

}

else {

candidateTmp[j].score = minScore;

score.coeffRef(row, col) = minScore;

--j;

continue;

}

}

candidateTmp[j].stop = true;

}

candidateTmp[j].tgt.push_back(row);

if (i == 0) {

score.row(row).fill(minScore);

}

else {

score.coeffRef(row, col) = minScore;

}

}

candidate = candidateTmp;

std::sort(candidate.begin(), candidate.end(), sort_pred());

if (candidate[0].tgt.back() == this->targetVoc.eosIndex) {

break;

}

}

this->makeTrans(candidate[0].tgt, trans);

this->deleteStateList(stateList);

this->clearState(state, srcParsedFlag);

AttentionTreeEncDec::State* state,

const std::vector<int>& src, const std::vector<int>& tgt){

std::string fileName = "pathToSaveOfAlphaTree"; // TODO: Modify the path

std::ofstream outputFile;

outputFile.open(fileName, std::ios::app);

outputFile << std::endl;

outputFile << "Src: ";

for (int i = 0; i < src.size(); ++i){

outputFile << this->sourceVoc.tokenList[src[i]]->str << " ";

}

outputFile << std::endl;

outputFile << "Tgt: ";

for (int i = 0; i < tgt.size(); ++i){

outputFile << this->targetVoc.tokenList[tgt[i]]->str << " ";

}

outputFile << std::endl;

for (int i = 0; i < (int)tgt.size(); ++i) {

outputFile << "***" << this->targetVoc.tokenList[tgt[i]]->str << ": " << std::endl;

print(this->targetVoc.tokenList[tgt[i]]->str);

for (int j = 0, row, col; j < 5; ++j) { // Show Top5

showAlphaTree.row(i).maxCoeff(&row, &col);

outputFile << " (" << showAlphaTree.coeff(i, col) << ") ";

print(showAlphaTree.coeff(i, col));

if (col < (int)state->encState.size()-1) {

outputFile << this->sourceVoc.tokenList[src[col]]->str;

print(this->sourceVoc.tokenList[src[col]]->str);

}

else{

showTreeNode(state->encTreeNodeVec[col+1-(int)state->encState.size()], outputFile);

}

showAlphaTree.coeffRef(i, col) = -1.0;

outputFile << std::endl;

}

outputFile << std::endl << std::flush;

if (this->targetVoc.tokenList[tgt[i]]->str == "*EOS*") break;

}

outputFile << "##################################" <<std::endl;

outputFile.close();

AttentionTreeEncDec::State* state,

std::vector<BlackOut::State*>& blackOutState) {

VecD targetDist;

VecD c0, s0; // cell memory and hidden state for initial Decorder (initDec == CONCAT)

VecD contextTree; // C_Tree = Σ (alpha * hidden state)

const int alphaSize = state->encState.size()-1+state->encTreeNodeVec.size();

VecD alphaTree = VecD(alphaSize); // Vector for attentional weight

// MatD showAlphaTree = MatD::Zero(data->tgt.size(), state->encState.size()-1+state->encTreeNodeVec.size());

VecD s_tilde; // Decoder; s~

VecD del_stilde;

VecD del_contextTree;

VecD del_alphaTree = VecD::Zero(alphaSize);

VecD del_alignScore; // delta for alignment score

VecD del_c0, del_s0;

loss = 0.0;

this->encoder(data->src, state);

for (int i = 0; i < (int)data->tgt.size(); ++i) {

this->decoder(data, state, s_tilde, c0, s0, i);

this->decoderAttention(state, state->decState, contextTree, alphaTree, s_tilde, i);

//showAlphaTree.push_back(MatD::Zero(data->tgt.size(),

// state->encState.size()-1+state->encTreeNodeVec.size()));

//showAlphaTree.back().row(i) = alphaTree[i].transpose();

if (!this->useBlackOut) {

this->softmax.calcDist(s_tilde, targetDist);

loss += this->softmax.calcLoss(targetDist, data->tgt[i]);

this->softmax.backward(s_tilde, targetDist, data->tgt[i], del_stilde, grad.softmaxGrad);

}

else {

this->blackOut.sampling(data->tgt[i], grad.blackOutState);

// *(blackOutState[i]) = grad.blackOutState; // for gradient checker

this->blackOut.calcSampledDist(s_tilde, targetDist, grad.blackOutState);

loss += this->blackOut.calcSampledLoss(targetDist);

this->blackOut.backward(s_tilde, targetDist, grad.blackOutState, del_stilde, grad.blackOutGrad);

}

/* Attention's Backpropagation */

// del_stilde

del_stilde.array() *= ActFunc::tanhPrime(s_tilde).array();

state->decState[i]->delh = this->Wst.transpose()*del_stilde;

grad.Wst += del_stilde*state->decState[i]->h.transpose();

grad.bs += del_stilde;

grad.Wct += del_stilde*contextTree.transpose();

del_contextTree = this->Wct.transpose()*del_stilde;

// del_contextTree

for (int j = 0; j < (int)data->src.size(); ++j) { // Seq

state->encState[j+1]->delh += alphaTree.coeff(j, 0) * del_contextTree;

del_alphaTree.coeffRef(j, 0) = del_contextTree.dot(state->encState[j+1]->h);

}

for (int j = 0; j < (int)state->encTreeNodeVec.size(); ++j) { // Tree (Phrase node)

state->encTreeNodeVec[j]->state->delh += alphaTree.coeff(j+(int)data->src.size(), 0) * del_contextTree;

del_alphaTree.coeffRef(j+(int)data->src.size(), 0) = del_contextTree.dot(state->encTreeNodeVec[j]->state->h);

}

del_alignScore = alphaTree.array()*(del_alphaTree.array()-alphaTree.dot(del_alphaTree)); // X.array() - scalar; np.array() -= 1

if (this->attenType == AttentionTreeEncDec::DOT) { // h^T*s

for (int j = 0; j < (int)data->src.size(); ++j) { // Seq

state->encState[j+1]->delh += del_alignScore.coeff(j, 0)*state->decState[i]->h;

state->decState[i]->delh += del_alignScore.coeff(j, 0)*state->encState[j+1]->h;

}

for (int j = 0; j < (int)state->encTreeNodeVec.size(); ++j) { // Tree (phrase node)

state->encTreeNodeVec[j]->state->delh +=

del_alignScore.coeff(j+(int)data->src.size(), 0)*state->decState[i]->h;

state->decState[i]->delh += del_alignScore.coeff(j+(int)data->src.size(), 0)*state->encTreeNodeVec[j]->state->h;

}

} else if (this->attenType == AttentionTreeEncDec::GENERAL) { // s^T*W*h

for (int j = 0; j < (int)data->src.size(); ++j) {

state->encState[j+1]->delh += (this->WgeneralTree.transpose()*state->decState[i]->h)*del_alignScore.coeff(j, 0);

state->decState[i]->delh += (this->WgeneralTree*state->encState[j+1]->h)*del_alignScore.coeff(j, 0);

grad.WgeneralTree += del_alignScore.coeff(j, 0)*state->decState[i]->h*state->encState[j+1]->h.transpose();

}

for (int j = 0; j < (int)state->encTreeNodeVec.size(); ++j) {

state->encTreeNodeVec[j]->state->delh +=

(this->WgeneralTree.transpose()*state->decState[i]->h)*del_alignScore.coeff(j+(int)data->src.size(), 0);

state->decState[i]->delh +=

(this->WgeneralTree*state->encTreeNodeVec[j]->state->h)*del_alignScore.coeff(j+(int)data->src.size(), 0);

grad.WgeneralTree +=

del_alignScore.coeff(j+(int)data->src.size(), 0)*state->decState[i]->h*state->encTreeNodeVec[j]->state->h.transpose();

}

} else {}

}

state->decState.back()->delc = this->zeros;

for (int i = data->tgt.size()-1; i >= 1; --i) {

state->decState[i-1]->delc = this->zeros;

this->dec.backward(state->decState[i-1], state->decState[i],

grad.lstmTgtGrad, this->targetEmbed.col(data->tgt[i-1]));

if (grad.targetEmbed.count(data->tgt[i-1])) {

grad.targetEmbed.at(data->tgt[i-1]) += state->decState[i]->delx;

}

else {

grad.targetEmbed[data->tgt[i-1]] = state->decState[i]->delx;

}

}

// Decoder -> Encoder

if (initDec == AttentionTreeEncDec::TREELSTM) {

this->initDecTree.backward((TreeLSTM::State*)state->decState[0], state->encState.back(), state->encTreeState->state,

grad.treeLstmInitDecGrad);

} else if (initDec == AttentionTreeEncDec::CONCAT) {

del_c0 = state->decState[0]->delc;

del_s0 = state->decState[0]->delh;

del_c0.array() *= ActFunc::tanhPrime(c0).array();

del_s0.array() *= ActFunc::tanhPrime(s0).array();

state->encState.back()->delc += this->WcellSeq.transpose()*del_c0;

grad.WcellSeq += del_c0*state->encState.back()->c.transpose();

state->encTreeState->state->delc += this->WcellTree.transpose()*del_c0;

grad.WcellTree += del_c0*state->encTreeState->state->c.transpose();

state->encState.back()->delh += this->WhSeq.transpose()*del_s0;

grad.WhSeq += del_s0*state->encState.back()->h.transpose();

state->encTreeState->state->delh += this->WhTree.transpose()*del_s0;

grad.WhTree += del_s0*state->encTreeState->state->h.transpose();

} else {}

// backprop through structure

if (data->srcParsedFlag) {

state->leafCounter = 0;

this->backpropThroughStructure(state->encTreeState,

state->encTreeState->left, state->encTreeState->right,

grad, state);

} else {}

// Encoder Backpropagation after adding leaves' gradients (this is already done in ``backpropThroughStructure``)

for (int i = data->src.size(); i >= 1; --i) {

if (i == 1) {

this->enc.backward(state->encState[i], grad.lstmSrcGrad,

this->sourceEmbed.col(data->src[i-1]));

} else {

this->enc.backward(state->encState[i-1], state->encState[i], grad.lstmSrcGrad,

this->sourceEmbed.col(data->src[i-1]));

}

if (grad.sourceEmbed.count(data->src[i-1])) {

grad.sourceEmbed.at(data->src[i-1]) += state->encState[i]->delx;

}

else {

grad.sourceEmbed[data->src[i-1]] = state->encState[i]->delx;

}

}

AttentionTreeEncDec::State* state,

std::vector<BlackOut::State*>& blackOutState,

std::vector<VecD>& s_tilde, std::vector<VecD>& del_stilde,

std::vector<VecD>& contextTreeList) {

VecD targetDist;

VecD c0, s0; // cell memory and hidden state for initial Decorder (initDec == CONCAT)

const int alphaSize = state->encState.size()-1+state->encTreeNodeVec.size();

MatD alphaTree = MatD(alphaSize, data->tgt.size());

// std::vector<VecD> showAlphaTree;

VecD del_contextTree;

VecD del_alphaTree = VecD::Zero(alphaSize);

VecD del_alignScore; // delta for alignment score

VecD del_c0, del_s0;

loss = 0.0;

this->encoder(data->src, state);

for (int i = 0; i < (int)data->tgt.size(); ++i) {

if (i == 0) {

if (initDec == AttentionTreeEncDec::TREELSTM) { // Left: Sequence, Right: Tree (Phrase node)

// copy ``c`` and ``h`` and set them into ``initialState``

this->initDecTree.forward((TreeLSTM::State*)state->decState[i],

state->encState.back(), state->encTreeState->state);

} else if (initDec == AttentionTreeEncDec::CONCAT) { //inspired

c0 = this->WcellSeq*state->encState.back()->c

+ this->WcellTree*state->encTreeState->state->c;

s0 = this->WhSeq*state->encState.back()->h

+ this->WhTree*state->encTreeState->state->h;

ActFunc::tanh(c0);

ActFunc::tanh(s0);

state->decState[i]->c = c0; // tanh(W[cell_Seq; cell_Tree])

state->decState[i]->h = s0; // tanh(W[h_Seq; h_Tree])

} else {}

} else { // i >= 1

// input-feeding approach [Luong et al., EMNLP2015]

this->dec.forward(this->targetEmbed.col(data->tgt[i-1]), s_tilde[i-1],

state->decState[i-1], state->decState[i]); // (xt, at, prev, cur)

}

/* Attention */

s_tilde[i] = this->bs;

s_tilde[i].noalias() += this->Wst*state->decState[i]->h;

// sequence and Tree (phrase node)

contextTreeList[i] = this->zeros; // initialize

// Matrix for attentional weight

//showAlphaTree.push_back(MatD::Zero(data->tgt.size(),

// state->encState.size()-1+state->encTreeNodeVec.size()));

this->calculateAlpha(state, state->decState[i], alphaTree, i); // (!) Insert the i-th alphaTree

//showAlphaTree.back().row(i) = alphaTree[i].transpose();

for (int j = 1; j < (int)state->encState.size(); ++j) {

contextTreeList[i].noalias() += alphaTree.coeff(j-1, i)*state->encState[j]->h;

}

for (int j = 0; j < (int)state->encTreeNodeVec.size(); ++j) {

contextTreeList[i].noalias() += alphaTree.coeff(j+(int)state->encState.size()-1, i)*state->encTreeNodeVec[j]->state->h;

}

s_tilde[i].noalias() += this->Wct*contextTreeList[i];

ActFunc::tanh(s_tilde[i]); // s~tanh(W_c[ht; c_Tree])

if (!this->useBlackOut) {

this->softmax.calcDist(s_tilde[i], targetDist);

loss += this->softmax.calcLoss(targetDist, data->tgt[i]);

this->softmax.backward(s_tilde[i], targetDist, data->tgt[i], del_stilde[i], grad.softmaxGrad);

}

else {

this->blackOut.sampling(data->tgt[i], grad.blackOutState);

// *(blackOutState[i]) = grad.blackOutState; // for gradient checker

this->blackOut.calcSampledDist(s_tilde[i], targetDist, grad.blackOutState);

loss += this->blackOut.calcSampledLoss(targetDist);

this->blackOut.backward(s_tilde[i], targetDist, grad.blackOutState, del_stilde[i], grad.blackOutGrad);

}

}

state->decState.back()->delc = this->zeros;

state->decState.back()->delh = this->zeros;

for (int i = (int)data->tgt.size()-1; i >= 0; --i) {

/* Attention's Backpropagation */

// del_stilde

if (i < (int)data->tgt.size()-1) {

del_stilde[i] += state->decState[i+1]->dela; // add gradients to the previous del_stilde

// by input-feeding [Luong et al., EMNLP2015]

} else {}

del_stilde[i].array() *= ActFunc::tanhPrime(s_tilde[i]).array();

state->decState[i]->delh.noalias() += this->Wst.transpose()*del_stilde[i];

grad.Wst.noalias() += del_stilde[i]*state->decState[i]->h.transpose();

grad.bs += del_stilde[i];

grad.Wct.noalias() += del_stilde[i]*contextTreeList[i].transpose();

del_contextTree.noalias() = this->Wct.transpose()*del_stilde[i];

// del_contextTree

for (int j = 0; j < (int)data->src.size(); ++j) { // Seq

state->encState[j+1]->delh.noalias() += alphaTree.coeff(j, i) * del_contextTree;

del_alphaTree.coeffRef(j, 0) = del_contextTree.dot(state->encState[j+1]->h);

}

for (int j = 0; j < (int)state->encTreeNodeVec.size(); ++j) { // Tree (Phrase node)

state->encTreeNodeVec[j]->state->delh.noalias() += alphaTree.coeff(j+(int)data->src.size(), i) * del_contextTree;

del_alphaTree.coeffRef(j+(int)data->src.size(), 0) = del_contextTree.dot(state->encTreeNodeVec[j]->state->h);

}

del_alignScore = alphaTree.col(i).array()*(del_alphaTree.array()-alphaTree.col(i).dot(del_alphaTree)); // X.array() - scalar; np.array() -= 1

if (this->attenType == AttentionTreeEncDec::DOT) { // h^T*s

for (int j = 0; j < (int)data->src.size(); ++j) {

state->encState[j+1]->delh.noalias() += del_alignScore.coeff(j, 0)*state->decState[i]->h;

state->decState[i]->delh.noalias() += del_alignScore.coeff(j, 0)*state->encState[j+1]->h;

}

for (int j = 0; j < (int)state->encTreeNodeVec.size(); ++j) {

state->encTreeNodeVec[j]->state->delh.noalias() +=

del_alignScore.coeff(j+(int)data->src.size(), 0)*state->decState[i]->h;

state->decState[i]->delh.noalias() += del_alignScore.coeff(j+(int)data->src.size(), 0)*state->encTreeNodeVec[j]->state->h;

}

} else if (this->attenType == AttentionTreeEncDec::GENERAL) { // s^T*W*h

for (int j = 0; j < (int)data->src.size(); ++j) {

state->encState[j+1]->delh += (this->WgeneralTree.transpose()*state->decState[i]->h)*del_alignScore.coeff(j, 0);

state->decState[i]->delh += (this->WgeneralTree*state->encState[j+1]->h)*del_alignScore.coeff(j, 0);

grad.WgeneralTree += del_alignScore.coeff(j, 0)*state->decState[i]->h*state->encState[j+1]->h.transpose();

}

for (int j = 0; j < (int)state->encTreeNodeVec.size(); ++j) {

state->encTreeNodeVec[j]->state->delh +=

(this->WgeneralTree.transpose()*state->decState[i]->h)*del_alignScore.coeff(j+(int)data->src.size(), 0);

state->decState[i]->delh +=

(this->WgeneralTree*state->encTreeNodeVec[j]->state->h)*del_alignScore.coeff(j+(int)data->src.size(), 0);

grad.WgeneralTree +=

del_alignScore.coeff(j+(int)data->src.size(), 0)*state->decState[i]->h*state->encTreeNodeVec[j]->state->h.transpose();

}

} else {}

if (i > 0) {

state->decState[i-1]->delc = this->zeros;

state->decState[i-1]->delh = this->zeros;

this->dec.backward(state->decState[i-1], state->decState[i],

grad.lstmTgtGrad, this->targetEmbed.col(data->tgt[i-1]), s_tilde[i-1]);

if (grad.targetEmbed.count(data->tgt[i-1])) {

grad.targetEmbed.at(data->tgt[i-1]) += state->decState[i]->delx;

}

else {

grad.targetEmbed[data->tgt[i-1]] = state->decState[i]->delx;

}

} else {}

}

// Decoder -> Encoder

if (initDec == AttentionTreeEncDec::TREELSTM) {

this->initDecTree.backward((TreeLSTM::State*)state->decState[0], state->encState.back(), state->encTreeState->state,

grad.treeLstmInitDecGrad);

} else if (initDec == AttentionTreeEncDec::CONCAT) {

del_c0 = state->decState[0]->delc;

del_s0 = state->decState[0]->delh;

del_c0.array() *= ActFunc::tanhPrime(c0).array();

del_s0.array() *= ActFunc::tanhPrime(s0).array();

state->encState.back()->delc += this->WcellSeq.transpose()*del_c0;

grad.WcellSeq += del_c0*state->encState.back()->c.transpose();

state->encTreeState->state->delc += this->WcellTree.transpose()*del_c0;

grad.WcellTree += del_c0*state->encTreeState->state->c.transpose();

state->encState.back()->delh += this->WhSeq.transpose()*del_s0;

grad.WhSeq += del_s0*state->encState.back()->h.transpose();

state->encTreeState->state->delh += this->WhTree.transpose()*del_s0;

grad.WhTree += del_s0*state->encTreeState->state->h.transpose();

} else {}

//backprop through structure

if (data->srcParsedFlag) {

state->leafCounter = 0;

this->backpropThroughStructure(state->encTreeState,

state->encTreeState->left, state->encTreeState->right,

grad, state);

} else {}

// Encoder Backpropagation while adding leaves' gradients (this is already done in ``backpropThroughStructure``)

for (int i = data->src.size(); i >= 1; --i) {

if (i == 1) {

this->enc.backward(state->encState[i], grad.lstmSrcGrad,

this->sourceEmbed.col(data->src[i-1]));

} else {

this->enc.backward(state->encState[i-1], state->encState[i], grad.lstmSrcGrad,

this->sourceEmbed.col(data->src[i-1]));

}

if (grad.sourceEmbed.count(data->src[i-1])) {

grad.sourceEmbed.at(data->src[i-1]) += state->encState[i]->delx;

}

else {

grad.sourceEmbed[data->src[i-1]] = state->encState[i]->delx;

}

}

const LSTM::State* decState, VecD& alphaTree) { // calculate attentional weight;

/* ``decState`` is a encNodeVec itsself and ``encTreeNodeVec`` for TreeLSTM units */

const int encStateSize = state->encState.size();

const int encTreeSize = state->encTreeNodeVec.size();

if (this->attenType == AttentionTreeEncDec::DOT) { //inner product; h^T*s

// Sequnce (leaf)

for (int i = 1; i < encStateSize; ++i) { // encState is 1-origin (encState[0] == h0)

alphaTree.coeffRef(i-1, 0) = state->encState[i]->h.dot(decState->h); // coeffRef: insertion

}

// Tree (phrase node)

for (int i = 0; i < encTreeSize; ++i) { // encState[0]=h0

alphaTree.coeffRef(i+encStateSize-1, 0) =

state->encTreeNodeVec[i]->state->h.dot(decState->h); // coeffRef: insertion

}

}

else if (this->attenType == AttentionTreeEncDec::GENERAL) { // s^T*W*h

for (int i = 1; i < (int)state->encState.size(); ++i) {

alphaTree.coeffRef(i-1, 0) = decState->h.dot(this->WgeneralTree * state->encState[i]->h);

}

for (int i = 0; i < (int)state->encTreeNodeVec.size(); ++i) {

alphaTree.coeffRef(i+(int)state->encState.size()-1, 0) =

decState->h.dot(this->WgeneralTree * state->encTreeNodeVec[i]->state->h);

}

} else {}

// softmax of ``alphaTree``

alphaTree.array() -= alphaTree.maxCoeff(); // stable softmax

alphaTree = alphaTree.array().exp(); // exp() operation for all elements; np.exp(alphaTree)

alphaTree /= alphaTree.array().sum(); // alphaTree.sum()

const LSTM::State* decState, MatD& alphaTree, const int colNum) { // calculate attentional weight;

/* ``decState`` is a encNodeVec itsself and ``encTreeNodeVec`` for TreeLSTM units */

const int encStateSize = state->encState.size();

const int encTreeSize = state->encTreeNodeVec.size();

if (this->attenType == AttentionTreeEncDec::DOT) { // inner product: h^T*s

// Sequnce (leaf)

for (int i = 1; i < encStateSize; ++i) { // encState is 1-origin (encState[0] == h0)

alphaTree.coeffRef(i-1, colNum) = state->encState[i]->h.dot(decState->h); // coeffRef: insertion

}

// Tree (phrase node)

for (int i = 0; i < encTreeSize; ++i) { // encState[0]=h0

alphaTree.coeffRef(i+encStateSize-1, colNum) =

state->encTreeNodeVec[i]->state->h.dot(decState->h); // coeffRef: insertion

}

}

else if (this->attenType == AttentionTreeEncDec::GENERAL) { // s^T*W*h

for (int i = 1; i < (int)state->encState.size(); ++i) {

alphaTree.coeffRef(i-1, colNum) = decState->h.dot(this->WgeneralTree * state->encState[i]->h);

}

for (int i = 0; i < (int)state->encTreeNodeVec.size(); ++i) {

alphaTree.coeffRef(i+(int)state->encState.size()-1, colNum) =

decState->h.dot(this->WgeneralTree * state->encTreeNodeVec[i]->state->h);

}

} else {}

// softmax of ``alphaTree``

alphaTree.col(colNum).array() -= alphaTree.col(colNum).maxCoeff(); // stable softmax

alphaTree.col(colNum) = alphaTree.col(colNum).array().exp(); // exp() operation for all elements; np.exp(alphaTree)

alphaTree.col(colNum) /= alphaTree.col(colNum).array().sum(); // alphaTree.sum()

this->Wst -= learningRate * grad.Wst;

this->bs -= learningRate * grad.bs;

this->Wct -= learningRate * grad.Wct;

this->WgeneralTree -= learningRate * grad.WgeneralTree;

this->WcellSeq -= learningRate * grad.WcellSeq;

this->WcellTree -= learningRate * grad.WcellTree;

this->WhSeq -= learningRate * grad.WhSeq;

this->WhTree -= learningRate * grad.WcellTree;

///* TODO: Freezing embed

for (auto it = grad.sourceEmbed.begin(); it != grad.sourceEmbed.end(); ++it) {

this->sourceEmbed.col(it->first) -= learningRate * it->second;

}

for (auto it = grad.targetEmbed.begin(); it != grad.targetEmbed.end(); ++it) {

this->targetEmbed.col(it->first) -= learningRate * it->second;

}

AttentionTreeEncDec::StateNode* left,

AttentionTreeEncDec::StateNode* right,