#include "common.h"

#include "evserver.h"

char* version = "7.51";

char sourceInput[200];

FILE* userInitFile;

int externalTagger = 0;

static bool argumentsSeen = false;

char* configFile = "cs_init.txt"; // can set config params

char language[40]; // indicate current language used

char livedata[500]; // where is the livedata folder

char languageFolder[500]; // where is the livedata language folder

char systemFolder[500]; // where is the livedata system folder

bool noboot = false;

static char* erasename = "csuser_erase";

bool build0Requested = false;

bool build1Requested = false;

bool servertrace = false;

bool pendingRestart = false;

bool pendingUserReset = false;

bool rebooting = false;

bool assignedLogin = false;

char apikey[100];

DEBUGAPI debugInput = NULL;

DEBUGAPI debugOutput = NULL;

#define MAX_RETRIES 20

clock_t startTimeInfo; // start time of current volley

char revertBuffer[INPUT_BUFFER_SIZE]; // copy of user input so can :revert if desired

int argc;

char ** argv;

char postProcessing = 0; // copy of output generated during MAIN control. Postprocessing can prepend to it

unsigned int tokenCount; // for document performc

bool callback = false; // when input is callback,alarm,loopback, dont want to enable :retry for that...

int timerLimit = 0; // limit time per volley

int timerCheckRate = 0; // how often to check calls for time

clock_t volleyStartTime = 0;

int timerCheckInstance = 0;

static char* privateParams = NULL;

static char treetaggerParams[200];

static char encryptParams[300];

static char decryptParams[300];

char hostname[100];

bool nosuchbotrestart = false; // restart if no such bot

char users[100];

char logs[100];

char topic[100];

char tmp[100];

char buildfiles[100];

char* derivationSentence[MAX_SENTENCE_LENGTH];

int derivationLength;

char authorizations[200]; // for allowing debug commands

char ourMainInputBuffer[INPUT_BUFFER_SIZE]; // user input buffer - ptr to primaryInputBuffer

char* mainInputBuffer; // user input buffer - ptr to primaryInputBuffer

char* ourMainOutputBuffer; // main user output buffer

char* mainOutputBuffer; //

char* readBuffer; // main scratch reading buffer (files, etc)

bool readingDocument = false;

bool redo = false; // retry backwards any amount

bool oobExists = false;

bool docstats = false;

unsigned int docSentenceCount = 0;

char rawSentenceCopy[INPUT_BUFFER_SIZE]; // current raw sentence

char *evsrv_arg = NULL;

unsigned short int derivationIndex[256];

bool overrideAuthorization = false;

clock_t startSystem; // time chatscript started

unsigned int choiceCount = 0;

int always = 1; // just something to stop the complaint about a loop based on a constant

// support for callbacks

unsigned long callBackTime = 0; // one-shot pending time - forces callback with "[callback]" when output was [callback=xxx] when no user submitted yet

unsigned long callBackDelay = 0; // one-shot pending time - forces callback with "[callback]" when output was [callback=xxx] when no user submitted yet

unsigned long loopBackTime = 0; // ongoing callback forces callback with "[loopback]" when output was "[loopback=xxx]"

unsigned long loopBackDelay = 0; // ongoing callback forces callback with "[loopback]" when output was "[loopback=xxx]"

unsigned long alarmTime = 0;

unsigned long alarmDelay = 0; // one-shot pending time - forces callback with "[alarm]" when output was [callback=xxx] when no user submitted yet

unsigned int outputLength = 100000; // max output before breaking.

char* extraTopicData = 0; // supplemental topic set by :extratopic

// server data

#ifdef DISCARDSERVER

bool server = false;

#else

std::string interfaceKind = "0.0.0.0";

#ifdef WIN32

bool server = false; // default is standalone on Windows

#elif IOS

bool server = true; // default is server on LINUX

#else

bool server = true; // default is server on LINUX

#endif

#endif

unsigned int port = 1024; // server port

bool commandLineCompile = false;

PrepareMode tmpPrepareMode = NO_MODE; // controls what processing is done in preparation NO_MODE, POS_MODE, PREPARE_MODE

PrepareMode prepareMode = NO_MODE; // controls what processing is done in preparation NO_MODE, POS_MODE, PREPARE_MODE

bool noReact = false;

// :source:document data

bool documentMode = false; // read input as a document not as chat

FILE* sourceFile = NULL; // file to use for :source

EchoSource echoSource = NO_SOURCE_ECHO; // for :source, echo that input to nowhere, user, or log

unsigned long sourceStart = 0; // beginning time of source file

unsigned int sourceTokens = 0;

unsigned int sourceLines = 0;

// status of user input

unsigned int volleyCount = 0; // which user volley is this

bool moreToComeQuestion = false; // is there a ? in later sentences

bool moreToCome = false; // are there more sentences pending

uint64 tokenControl = 0; // results from tokenization and prepare processing

unsigned int responseControl = ALL_RESPONSES; // std output behaviors

char* nextInput; // ptr to rest of users input after current sentence

static char oldInputBuffer[INPUT_BUFFER_SIZE]; // copy of the sentence we are processing

char currentInput[INPUT_BUFFER_SIZE]; // the sentence we are processing BUG why both

// general display and flow controls

bool quitting = false; // intending to exit chatbot

int systemReset = 0; // intending to reload system - 1 (mild reset) 2 (full user reset)

bool autonumber = false; // display number of volley to user

bool showWhy = false; // show user rules generating his output

bool showTopic = false; // show resulting topic on output

bool showTopics = false; // show all relevant topics

bool showInput = false; // Log internal input additions

bool showReject = false; // Log internal repeat rejections additions

bool all = false; // generate all possible answers to input

int regression = NO_REGRESSION; // regression testing in progress

unsigned int trace = 0; // current tracing flags

bool debugEntry = false;

char bootcmd[200]; // current boot tracing flags

unsigned int timing = 0; // current timing flags

bool shortPos = false; // display pos results as you go

int inputRetryRejoinderTopic = NO_REJOINDER;

int inputRetryRejoinderRuleID = NO_REJOINDER;

int sentenceRetryRejoinderTopic = NO_REJOINDER;

int sentenceRetryRejoinderRuleID = NO_REJOINDER;

char oktest[MAX_WORD_SIZE]; // auto response test

char activeTopic[200];

char respondLevel = 0; // rejoinder level of a random choice block

static bool oobPossible = false; // 1st thing could be oob

int inputCounter = 0; // protecting ^input from cycling

int totalCounter = 0; // protecting ^input from cycling

static char userPrefix[MAX_WORD_SIZE]; // label prefix for user input

static char botPrefix[MAX_WORD_SIZE]; // label prefix for bot output

bool unusedRejoinder; // inputRejoinder has been executed, blocking further calls to ^Rejoinder

char inputCopy[INPUT_BUFFER_SIZE]; // the original input we were given, we will work on this

// outputs generated

RESPONSE responseData[MAX_RESPONSE_SENTENCES+1];

unsigned char responseOrder[MAX_RESPONSE_SENTENCES+1];

int responseIndex;

int inputSentenceCount; // which sentence of volley is this

static void HandleBoot(WORDP boot,bool reboot);

///////////////////////////////////////////////

/// SYSTEM STARTUP AND SHUTDOWN

///////////////////////////////////////////////

void InitStandalone()

{

startSystem = clock() / CLOCKS_PER_SEC;

*currentFilename = 0; // no files are open (if logging bugs)

tokenControl = 0;

*computerID = 0; // default bot

#ifndef DISCARDTESTING

InitDebugger();

#endif

}

static void SetBotVariable(char* word)

{

char* eq = strchr(word, '=');

if (eq)

{

*eq = 0;

ReturnToAfterLayer(currentBeforeLayer-1, true);

*word = USERVAR_PREFIX;

SetUserVariable(word, eq + 1);

if (server) Log(SERVERLOG, (char*)"botvariable: %s = %s\r\n", word, eq + 1);

else printf((char*)"botvariable: %s = %s\r\n", word, eq + 1);

NoteBotVariables(); // these go into level 1

LockLayer(false);

}

}

static void HandleBoot(WORDP boot, bool reboot)

{

if (reboot) rebooting = true;

if (boot && !noboot) // run script on startup of system. data it generates will also be layer 1 data

{

int oldtrace = trace;

if (*bootcmd)

{

char* at = bootcmd;

if (*bootcmd == '"')

{

++at;

if (at[strlen(at) - 1] == '"') at[strlen(at) - 1] = 0;

}

DoCommand(at, NULL, false);

}

if (!reboot) UnlockLayer(LAYER_BOOT); // unlock it to add stuff

FACT* F = factFree;

Callback(boot, (char*)"()", true, true); // do before world is locked

*ourMainOutputBuffer = 0; // remove any boot message

myBot = 0; // restore fact owner to generic all

if (!rebooting)

{

while (++F <= factFree)

{

if (F->flags & FACTTRANSIENT) F->flags |= FACTDEAD;

else F->flags |= FACTBUILD1; // convert these to level 1

}

NoteBotVariables(); // convert user variables read into bot variables in boot layer

LockLayer(false); // rewrite level 2 start data with augmented from script data

}

else

{

ReturnToAfterLayer(2, false); // dict/fact/strings reverted and any extra topic loaded info (but CSBoot process NOT lost)

}

trace = (modifiedTrace) ? modifiedTraceVal : oldtrace;

stackFree = stackStart; // drop any possible stack used

}

rebooting = false;

}

int CountWordsInBuckets(int& unused, unsigned int* depthcount,int limit)

{

memset(depthcount, 0, sizeof(int)*1000);

unused = 0;

int words = 0;

for (int i = 0; i <= maxHashBuckets; ++i)

{

unsigned int n = 0;

if (!hashbuckets[i]) ++unused;

else

{

WORDP D = Index2Word(hashbuckets[i]);

while (D != dictionaryBase)

{

++n;

D = Index2Word(GETNEXTNODE(D));

}

}

words += n;

if (n < limit) depthcount[n]++;

else

{

int xx = 0;

}

}

return words;

}

void CreateSystem()

{

overrideAuthorization = false; // always start safe

loading = true;

char* os;

mystart("createsystem");

#ifdef WIN32

os = "Windows";

#elif IOS

os = "IOS";

#elif __MACH__

os = "MACH";

#elif FREEBSD

os = "FreeBSD";

#else

os = "LINUX";

#endif

if (!buffers) // restart asking for new memory allocations

{

maxBufferSize = (maxBufferSize + 63);

maxBufferSize &= 0xffffffC0; // force 64 bit align

unsigned int total = maxBufferLimit * maxBufferSize;

buffers = (char*) malloc(total); // have it around already for messages

if (!buffers)

{

printf((char*)"%s",(char*)"cannot allocate buffer space");

exit(1);

}

bufferIndex = 0;

readBuffer = AllocateBuffer(); // technically if variables are huge, readuserdata would overflow this

baseBufferIndex = bufferIndex;

}

char data[MAX_WORD_SIZE];

char* kind;

int pid = 0;

#ifdef EVSERVER

kind = "EVSERVER";

pid = getpid();

#elif DEBUG

kind = "Debug";

#else

kind = "Release";

#endif

sprintf(data,(char*)"ChatScript %s Version %s pid: %d %ld bit %s compiled %s",kind,version,pid,(long int)(sizeof(char*) * 8),os,compileDate);

strcat(data,(char*)" host=");

strcat(data,hostname);

if (server) Log(SERVERLOG,(char*)"Server %s\r\n",data);

strcat(data,(char*)"\r\n");

printf((char*)"%s",data);

int oldtrace = trace; // in case trace turned on by default

trace = 0;

*oktest = 0;

sprintf(data,(char*)"Params: dict:%ld fact:%ld text:%ldkb hash:%ld \r\n",(long int)maxDictEntries,(long int)maxFacts,(long int)(maxHeapBytes/1000),(long int)maxHashBuckets);

if (server) Log(SERVERLOG,(char*)"%s",data);

else printf((char*)"%s",data);

sprintf(data,(char*)" buffer:%dx%dkb cache:%dx%dkb userfacts:%d outputlimit:%d loglimit:%d\r\n",(int)maxBufferLimit,(int)(maxBufferSize/1000),(int)userCacheCount,(int)(userCacheSize/1000),(int)userFactCount,

outputsize,logsize);

if (server) Log(SERVERLOG,(char*)"%s",data);

else printf((char*)"%s",data);

InitScriptSystem();

InitVariableSystem();

ReloadSystem(); // builds base facts and dictionary (from wordnet)

LoadTopicSystem(); // dictionary reverts to wordnet zone

*currentFilename = 0;

computerID[0] = 0;

if (!assignedLogin) loginName[0] = loginID[0] = 0;

*botPrefix = *userPrefix = 0;

char word[MAX_WORD_SIZE];

char buffer[MAX_WORD_SIZE];

FILE* in = FopenReadOnly(configFile);

if (in)

{

while (ReadALine(buffer, in, MAX_WORD_SIZE) >= 0)

{

ReadCompiledWord(buffer, word);

if (*word == 'V') SetBotVariable(word); // these are level 1 values

}

fclose(in);

}

for (int i = 1; i < argc; ++i)

{

strcpy(word,argv[i]);

if (*word == '"' && word[1] == 'V')

{

memmove(word,word+1,strlen(word));

size_t len = strlen(word);

if (word[len-1] == '"') word[len-1] = 0;

}

if (*word == 'V') SetBotVariable(word); // predefined bot variable in level 1

}

#ifndef DISCARDTESTING

if (debugEntry) Debugger("");

#endif

kernelVariableThreadList = botVariableThreadList;

botVariableThreadList = 0;

UnlockLayer(LAYER_BOOT); // unlock it to add stuff

HandleBoot(FindWord((char*)"^csboot"),false);// run script on startup of system. data it generates will also be layer 1 data

LockLayer(true);

InitSpellCheck(); // after boot vocabulary added

unsigned int factUsedMemKB = ( factFree-factBase) * sizeof(FACT) / 1000;

unsigned int factFreeMemKB = ( factEnd-factFree) * sizeof(FACT) / 1000;

unsigned int dictUsedMemKB = ( dictionaryFree-dictionaryBase) * sizeof(WORDENTRY) / 1000;

// dictfree shares text space

unsigned int textUsedMemKB = ( heapBase-heapFree) / 1000;

unsigned int textFreeMemKB = ( heapFree- heapEnd) / 1000;

unsigned int bufferMemKB = (maxBufferLimit * maxBufferSize) / 1000;

unsigned int used = factUsedMemKB + dictUsedMemKB + textUsedMemKB + bufferMemKB;

used += (userTopicStoreSize + userTableSize) /1000;

unsigned int free = factFreeMemKB + textFreeMemKB;

char route[MAX_WORD_SIZE];

unsigned int bytes = (tagRuleCount * MAX_TAG_FIELDS * sizeof(uint64)) / 1000;

used += bytes;

char buf2[MAX_WORD_SIZE];

char buf1[MAX_WORD_SIZE];

char buf[MAX_WORD_SIZE];

strcpy(buf,StdIntOutput(factFree-factBase));

strcpy(buf2,StdIntOutput(textFreeMemKB));

unsigned int depthcount[1000];

int unused;

int words = CountWordsInBuckets(unused, depthcount,1000);

char depthmsg[1000];

int x = 1000;

while (!depthcount[--x]); // find first used slot

char* at = depthmsg;

*at = 0;

for (int i = 0; i <= x; ++i)

{

sprintf(at, "%d.", depthcount[i]);

at += strlen(at);

}

sprintf(data, (char*)"Used %dMB: dict %s (%dkb) hashdepths %s words %d unusedbuckets %d fact %s (%dkb) heap %dkb\r\n",

used / 1000,

StdIntOutput(dictionaryFree - dictionaryBase - 1),

dictUsedMemKB, depthmsg,words,unused,

buf,

factUsedMemKB,

textUsedMemKB);

if (server) Log(SERVERLOG,(char*)"%s",data);

else printf((char*)"%s",data);

sprintf(data,(char*)" buffer (%dkb) cache (%dkb)\r\n",

bufferMemKB,

(userTopicStoreSize + userTableSize)/1000);

if(server) Log(SERVERLOG,(char*)"%s",data);

else printf((char*)"%s",data);

strcpy(buf,StdIntOutput(factEnd-factFree)); // unused facts

strcpy(buf1,StdIntOutput(textFreeMemKB)); // unused text

strcpy(buf2,StdIntOutput(free/1000));

sprintf(data,(char*)"Free %sMB: dict %s fact %s stack/heap %sKB \r\n\r\n",

buf2,StdIntOutput(((unsigned int)maxDictEntries)-(dictionaryFree-dictionaryBase)),buf,buf1);

if (server) Log(SERVERLOG,(char*)"%s",data);

else printf((char*)"%s",data);

trace = oldtrace;

#ifdef DISCARDSERVER

printf((char*)" Server disabled.\r\n");

#endif

#ifdef DISCARDSCRIPTCOMPILER

if(server) Log(SERVERLOG,(char*)" Script compiler disabled.\r\n");

else printf((char*)" Script compiler disabled.\r\n");

#endif

#ifdef DISCARDTESTING

if(server) Log(SERVERLOG,(char*)" Testing disabled.\r\n");

else printf((char*)" Testing disabled.\r\n");

#else

*callerIP = 0;

if (!assignedLogin) *loginID = 0;

if (server && VerifyAuthorization(FopenReadOnly((char*)"authorizedIP.txt"))) // authorizedIP

{

Log(SERVERLOG,(char*)" *** Server WIDE OPEN to :command use.\r\n");

}

#endif

#ifdef DISCARDJSONOPEN

printf((char*)"%s",(char*)" JSONOpen access disabled.\r\n");

#endif

#ifdef TREETAGGER

printf((char*)" TreeTagger access enabled: %s\r\n",language);

#endif

#ifndef DISCARDMONGO

if (*mongodbparams) sprintf(route," Mongo enabled. FileSystem routed to %s\r\n",mongodbparams);

else sprintf(route," Mongo enabled.\r\n");

if (server) Log(SERVERLOG,route);

else printf(route);

#endif

#ifndef DISCARDPOSTGRES

if (*postgresparams) sprintf(route, " Postgres enabled. FileSystem routed to %s\r\n", postgresparams);

else sprintf(route, " Postgres enabled.\r\n");

if (server) Log(SERVERLOG, route);

else printf(route);

#endif

printf((char*)"%s",(char*)"\r\n");

loading = false;

}

void ReloadSystem()

{// reset the basic system through wordnet but before topics

InitFacts(); // malloc space

InitStackHeap(); // malloc space

InitDictionary(); // malloc space

InitCache(); // malloc space

// make sets for the part of speech data

LoadDictionary();

InitFunctionSystem();

#ifndef DISCARDTESTING

InitCommandSystem();

#endif

ExtendDictionary(); // store labels of concepts onto variables.

DefineSystemVariables();

ClearUserVariables();

if (!ReadBinaryFacts(FopenStaticReadOnly(UseDictionaryFile((char*)"facts.bin")))) // DICT

{

AcquirePosMeanings(true);

WORDP safeDict = dictionaryFree;

ReadFacts(UseDictionaryFile((char*)"facts.txt"),NULL,0);

if ( safeDict != dictionaryFree) myexit((char*)"dict changed on read of facts");

WriteBinaryFacts(FopenBinaryWrite(UseDictionaryFile((char*)"facts.bin")),factBase);

}

else AcquirePosMeanings(false);

char name[MAX_WORD_SIZE];

sprintf(name,(char*)"%s/%s/systemfacts.txt",livedata,language);

ReadFacts(name,NULL,0); // part of wordnet, not level 0 build

ReadLiveData(); // considered part of basic system before a build

InitTextUtilities1(); // also part of basic system before a build

WordnetLockDictionary();

}

static void ProcessArgument(char* arg)

{

if (!argumentsSeen)

{

char path[MAX_WORD_SIZE];

*path = 0;

GetCurrentDir(path, MAX_WORD_SIZE);

printf("CommandLine: %s\r\n",path);

argumentsSeen = true;

}

printf(" %s\r\n",arg);

if (!stricmp(arg,(char*)"trace")) trace = (unsigned int) -1;

else if (!strnicmp(arg,(char*)"language=",9))

{

strcpy(language,arg+9);

MakeUpperCase(language);

}

else if (!strnicmp(arg, "erasename=", 10)) erasename = arg + 10;

else if (!stricmp(arg,"userencrypt")) userEncrypt = true;

else if (!stricmp(arg,"ltmencrypt")) ltmEncrypt = true;

else if (!stricmp(arg,"noboot")) noboot = true;

else if (!stricmp(arg, "servertrace")) servertrace = true;

else if (!strnicmp(arg,(char*)"apikey=",7)) strcpy(apikey,arg+7);

else if (!strnicmp(arg,(char*)"logsize=",8)) logsize = atoi(arg+8); // bytes avail for log buffer

else if (!strnicmp(arg, (char*)"loglimit=", 9)) loglimit = atoi(arg + 9); // max mb of log file before change files

else if (!strnicmp(arg,(char*)"outputsize=",11)) outputsize = atoi(arg+11); // bytes avail for log buffer

else if (!stricmp(arg, (char*)"time")) timing = (unsigned int)-1 ^ TIME_ALWAYS;

else if (!strnicmp(arg,(char*)"bootcmd=",8)) strcpy(bootcmd,arg+8);

else if (!strnicmp(arg,(char*)"dir=",4))

{

#ifdef WIN32

if (!SetCurrentDirectory(arg+4)) printf((char*)"unable to change to %s\r\n",arg+4);

#else

chdir(arg+5);

#endif

}

else if (!strnicmp(arg,(char*)"source=",7))

{

if (arg[7] == '"')

{

strcpy(sourceInput,arg+8);

size_t len = strlen(sourceInput);

if (sourceInput[len-1] == '"') sourceInput[len-1] = 0;

}

else strcpy(sourceInput,arg+7);

}

else if (!strnicmp(arg,(char*)"login=",6))

{

assignedLogin = true;

strcpy(loginID,arg+6);

}

else if (!strnicmp(arg,(char*)"output=",7)) outputLength = atoi(arg+7);

else if (!strnicmp(arg,(char*)"save=",5))

{

volleyLimit = atoi(arg+5);

if (volleyLimit > 255) volleyLimit = 255; // cant store higher

}

// memory sizings

else if (!strnicmp(arg,(char*)"hash=",5))

{

maxHashBuckets = atoi(arg+5); // size of hash

setMaxHashBuckets = true;

}

else if (!strnicmp(arg,(char*)"dict=",5)) maxDictEntries = atoi(arg+5); // how many dict words allowed

else if (!strnicmp(arg,(char*)"fact=",5)) maxFacts = atoi(arg+5); // fact entries

else if (!strnicmp(arg,(char*)"text=",5)) maxHeapBytes = atoi(arg+5) * 1000; // stack and heap bytes in pages

else if (!strnicmp(arg,(char*)"cache=",6)) // value of 10x0 means never save user data

{

userCacheSize = atoi(arg+6) * 1000;

char* number = strchr(arg+6,'x');

if (number) userCacheCount = atoi(number+1);

}

else if (!strnicmp(arg,(char*)"userfacts=",10)) userFactCount = atoi(arg+10); // how many user facts allowed

else if (!stricmp(arg,(char*)"redo")) redo = true; // enable redo

else if (!strnicmp(arg,(char*)"authorize=",10)) strcpy(authorizations,arg+10); // whitelist debug commands

else if (!stricmp(arg,(char*)"nodebug")) *authorizations = '1';

else if (!strnicmp(arg,(char*)"users=",6 )) strcpy(users,arg+6);

else if (!strnicmp(arg,(char*)"logs=",5 )) strcpy(logs,arg+5);

else if (!strnicmp(arg,(char*)"topic=",6 )) strcpy(topic,arg+6);

else if (!strnicmp(arg,(char*)"tmp=",4 )) strcpy(tmp,arg+4);

else if (!strnicmp(arg, (char*)"buildfiles=", 11)) strcpy(buildfiles, arg + 11);

else if (!strnicmp(arg,(char*)"private=",8)) privateParams = arg+8;

else if (!stricmp(arg,(char*)"treetagger")) strcpy(treetaggerParams,"1");

else if (!strnicmp(arg,(char*)"encrypt=",8)) strcpy(encryptParams,arg+8);

else if (!strnicmp(arg,(char*)"decrypt=",8)) strcpy(decryptParams,arg+8);

else if (!strnicmp(arg,(char*)"livedata=",9) )

{

strcpy(livedata,arg+9);

sprintf(systemFolder,(char*)"%s/SYSTEM",arg+9);

sprintf(languageFolder,(char*)"%s/%s",arg+9,language);

}

else if (!strnicmp(arg,(char*)"nosuchbotrestart=",17) )

{

if (!stricmp(arg+17,"true")) nosuchbotrestart = true;

else nosuchbotrestart = false;

}

else if (!strnicmp(arg,(char*)"system=",7) ) strcpy(systemFolder,arg+7);

else if (!strnicmp(arg,(char*)"english=",8) ) strcpy(languageFolder,arg+8);

#ifndef DISCARDPOSTGRES

else if (!strnicmp(arg,(char*)"pguser=",7) ) strcpy(postgresparams, arg+7);

#endif

#ifndef DISCARDMONGO

else if (!strnicmp(arg,(char*)"mongo=",6) ) strcpy(mongodbparams,arg+6);

#endif

#ifndef DISCARDCLIENT

else if (!strnicmp(arg,(char*)"client=",7)) // client=1.2.3.4:1024 or client=localhost:1024

{

server = false;

char buffer[MAX_WORD_SIZE];

strcpy(serverIP,arg+7);

char* portVal = strchr(serverIP,':');

if ( portVal)

{

*portVal = 0;

port = atoi(portVal+1);

}

if (!*loginID)

{

printf((char*)"%s",(char*)"\r\nEnter client user name: ");

ReadALine(buffer,stdin);

printf((char*)"%s",(char*)"\r\n");

Client(buffer);

}

else Client(loginID);

myexit((char*)"client ended");

}

#endif

else if (!stricmp(arg,(char*)"userlog")) userLog = true;

else if (!stricmp(arg,(char*)"nouserlog")) userLog = false;

else if (!strnicmp(arg, (char*)"hidefromlog=", 12))

{

char* at = arg + 12;

if (*at == '"') ++at;

strcpy(hide, at);

size_t len = strlen(hide);

if (hide[len - 1] == '"') hide[len - 1] = 0;

}

#ifndef DISCARDSERVER

else if (!stricmp(arg,(char*)"serverretry")) serverRetryOK = true;

else if (!stricmp(arg,(char*)"local")) server = false; // local standalone

else if (!stricmp(arg,(char*)"noserverlog")) serverLog = false;

else if (!stricmp(arg,(char*)"serverlog")) serverLog = true;

else if (!stricmp(arg,(char*)"noserverprelog")) serverPreLog = false;

else if (!stricmp(arg,(char*)"serverctrlz")) serverctrlz = 1;

else if (!strnicmp(arg,(char*)"port=",5)) // be a server

{

port = atoi(arg+5); // accept a port=

sprintf(serverLogfileName,(char*)"%s/serverlog%d.txt",logs,port);

server = true;

}

#ifdef EVSERVER

else if (!strnicmp(arg, "fork=", 5))

{

static char forkCount[10];

sprintf(forkCount,(char*)"evsrv:fork=%d",atoi(arg+5));

evsrv_arg = forkCount;

}

#endif

else if (!strnicmp(arg,(char*)"interface=",10)) interfaceKind = string(arg+10); // specify interface

#endif

}

void ProcessArguments(int argc, char* argv[])

{

for (int i = 1; i < argc; ++i) ProcessArgument(argv[i]);

}

static void ReadConfig()

{

char buffer[MAX_WORD_SIZE];

FILE* in = FopenReadOnly(configFile);

if (!in) return;

while (ReadALine(buffer,in,MAX_WORD_SIZE) >= 0) ProcessArgument(SkipWhitespace(buffer));

fclose(in);

}

unsigned int InitSystem(int argcx, char * argvx[],char* unchangedPath, char* readablePath, char* writeablePath, USERFILESYSTEM* userfiles, DEBUGAPI infn, DEBUGAPI outfn)

{ // this work mostly only happens on first startup, not on a restart

strcpy(hostname,(char*)"local");

*sourceInput = 0;

*buildfiles = 0;

*apikey = 0;

*bootcmd = 0;

#ifndef DISCARDMONGO

*mongodbparams = 0;

#endif

#ifndef DISCARDPOSTGRES

*postgresparams = 0;

#endif

#ifdef TREETAGGER

*treetaggerParams = 0;

#endif

*authorizations = 0;

debugInput = infn;

debugOutput = outfn;

nameDepth[0] = "";

argc = argcx;

argv = argvx;

InitFileSystem(unchangedPath,readablePath,writeablePath);

if (userfiles) memcpy((void*)&userFileSystem,userfiles,sizeof(userFileSystem));

for (unsigned int i = 0; i <= MAX_WILDCARDS; ++i)

{

*wildcardOriginalText[i] = *wildcardCanonicalText[i] = 0;

wildcardPosition[i] = 0;

}

strcpy(users,(char*)"USERS");

strcpy(logs,(char*)"LOGS");

strcpy(topic,(char*)"TOPIC");

strcpy(tmp,(char*)"TMP");

strcpy(language,(char*)"ENGLISH");

*loginID = 0;

for (int i = 1; i < argc; ++i) // essentials

{

if (!strnicmp(argv[i],(char*)"buffer=",7)) // number of large buffers available 8x80000

{

maxBufferLimit = atoi(argv[i]+7);

char* size = strchr(argv[i]+7,'x');

if (size) maxBufferSize = atoi(size+1) *1000;

if (maxBufferSize < OUTPUT_BUFFER_SIZE)

{

printf((char*)"Buffer cannot be less than OUTPUT_BUFFER_SIZE of %d\r\n",OUTPUT_BUFFER_SIZE);

myexit((char*)"buffer size less than output buffer size");

}

}

if (!strnicmp(argv[i],(char*)"config=",7)) configFile = argv[i]+7;

}

currentRuleOutputBase = currentOutputBase = ourMainOutputBuffer = (char*)malloc(outputsize);

currentOutputLimit = outputsize;

// need buffers for things that run ahead like servers and such.

maxBufferSize = (maxBufferSize + 63);

maxBufferSize &= 0xffffffc0; // force 64 bit align on size

unsigned int total = maxBufferLimit * maxBufferSize;

buffers = (char*) malloc(total); // have it around already for messages

if (!buffers)

{

printf((char*)"%s",(char*)"cannot allocate buffer space");

return 1;

}

bufferIndex = 0;

readBuffer = AllocateBuffer();

baseBufferIndex = bufferIndex;

quitting = false;

echo = true;

ReadConfig();

ProcessArguments(argc,argv);

MakeDirectory(tmp);

if (argumentsSeen) printf("\r\n");

argumentsSeen = false;

InitTextUtilities();

sprintf(logFilename,(char*)"%s/log%d.txt",logs,port); // DEFAULT LOG

sprintf(serverLogfileName,(char*)"%s/serverlog%d.txt",logs,port); // DEFAULT LOG

strcpy(livedata,(char*)"LIVEDATA"); // default directory for dynamic stuff

strcpy(systemFolder,(char*)"LIVEDATA/SYSTEM"); // default directory for dynamic stuff

sprintf(languageFolder,"LIVEDATA/%s",language); // default directory for dynamic stuff

if (redo) autonumber = true;

// defaults where not specified

if (userLog == LOGGING_NOT_SET) userLog = 1; // default ON for user if unspecified

if (serverLog == LOGGING_NOT_SET) serverLog = 1; // default ON for server if unspecified

int oldserverlog = serverLog;

serverLog = true;

#ifndef DISCARDSERVER

if (server)

{

#ifndef EVSERVER

GrabPort();

#else

printf("evcalled pid: %d\r\n",getpid());

if (evsrv_init(interfaceKind, port, evsrv_arg) < 0) exit(4); // additional params will apply to each child and they will load data each

#endif

#ifdef WIN32

PrepareServer();

#endif

}

#endif

if (volleyLimit == -1) volleyLimit = DEFAULT_VOLLEY_LIMIT;

for (int i = 1; i < argc; ++i)

{

if (!strnicmp(argv[i], (char*)"build0=", 7)) build0Requested = true;

if (!strnicmp(argv[i], (char*)"build1=", 7)) build1Requested = true;

}

CreateSystem();

for (int i = 1; i < argc; ++i)

{

#ifndef DISCARDSCRIPTCOMPILER

if (!strnicmp(argv[i],(char*)"build0=",7))

{

sprintf(logFilename,(char*)"%s/build0_log.txt",users);

FILE* in = FopenUTF8Write(logFilename);

FClose(in);

commandLineCompile = true;

int result = ReadTopicFiles(argv[i]+7,BUILD0,NO_SPELL);

myexit((char*)"build0 complete",result);

}

if (!strnicmp(argv[i],(char*)"build1=",7))

{

sprintf(logFilename,(char*)"%s/build1_log.txt",users);

FILE* in = FopenUTF8Write(logFilename);

FClose(in);

commandLineCompile = true;

int result = ReadTopicFiles(argv[i]+7,BUILD1,NO_SPELL);

myexit((char*)"build1 complete",result);

}

#endif

if (!strnicmp(argv[i],(char*)"timer=",6))

{

char* x = strchr(argv[i]+6,'x'); // 15000x10

if (x)

{

*x = 0;

timerCheckRate = atoi(x+1);

}

timerLimit = atoi(argv[i]+6);

}

#ifndef DISCARDTESTING

if (!stricmp(argv[i],(char*)"debug")) debugEntry = true;

else if (!strnicmp(argv[i],(char*)"debug=",6))

{

char* ptr = SkipWhitespace(argv[i]+6);

commandLineCompile = true;

if (*ptr == ':') DoCommand(ptr,mainOutputBuffer);

myexit((char*)"test complete");

}

#endif

if (!stricmp(argv[i],(char*)"trace")) trace = (unsigned int) -1; // make trace work on login

if (!stricmp(argv[i], (char*)"time")) timing = (unsigned int)-1 ^ TIME_ALWAYS; // make timing work on login

}

if (server)

{

struct tm ptm;

#ifndef EVSERVER

Log(SERVERLOG, "\r\n\r\n======== Began server %s compiled %s on host %s port %d at %s serverlog:%d userlog: %d\r\n",version,compileDate,hostname,port,GetTimeInfo(&ptm,true),oldserverlog,userLog);

printf((char*)"\r\n\r\n======== Began server %s compiled %s on host %s port %d at %s serverlog:%d userlog: %d\r\n",version,compileDate,hostname,port,GetTimeInfo(&ptm,true),oldserverlog,userLog);

#else

Log(SERVERLOG, "\r\n\r\n======== Began EV server pid %d %s compiled %s on host %s port %d at %s serverlog:%d userlog: %d\r\n",getpid(),version,compileDate,hostname,port,GetTimeInfo(&ptm,true),oldserverlog,userLog);

printf((char*)"\r\n\r\n======== Began EV server pid %d %s compiled %s on host %s port %d at %s serverlog:%d userlog: %d\r\n",getpid(),version,compileDate,hostname,port,GetTimeInfo(&ptm,true),oldserverlog,userLog);

#endif

}

serverLog = oldserverlog;

echo = false;

InitStandalone();

#ifndef DISCARDPOSTGRES

#ifndef EVSERVER

if (*postgresparams) PGUserFilesCode(); // unforked process can hook directly. Forked must hook AFTER forking

#endif

#endif

#ifndef DISCARDMONGO

if (*mongodbparams) MongoSystemInit(mongodbparams);

#endif

#ifdef TREETAGGER

InitTreeTagger(treetaggerParams);

#endif

#ifdef PRIVATE_CODE

PrivateInit(privateParams);

#endif

EncryptInit(encryptParams);

DecryptInit(decryptParams);

ResetEncryptTags();

return 0;

}

void PartiallyCloseSystem() // server data (queues etc) remain available

{

WORDP shutdown = FindWord((char*)"^csshutdown");

if (shutdown) Callback(shutdown,(char*)"()",false,true);

#ifndef DISCARDJAVASCRIPT

DeleteTransientJavaScript(); // unload context if there

DeletePermanentJavaScript(); // javascript permanent system

#endif

FreeAllUserCaches(); // user system

CloseDictionary(); // dictionary system

CloseFacts(); // fact system

CloseBuffers(); // memory system

#ifndef DISCARDPOSTGRES

PostgresShutDown();

#endif

#ifndef DISCARDMONGO

MongoSystemRestart();

#endif

#ifdef PRIVATE_CODE

PrivateRestart(); // must come AFTER any mongo/postgress init (to allow encrypt/decrypt override)

#endif

}

void CloseSystem()

{

PartiallyCloseSystem();

// server remains up on a restart

#ifndef DISCARDSERVER

CloseServer();

#endif

// user file rerouting stays up on a restart

#ifndef DISCARDPOSTGRES

PGUserFilesCloseCode();

#endif

#ifndef DISCARDMONGO

MongoSystemShutdown();

#endif

#ifdef PRIVATE_CODE

PrivateShutdown(); // must come last after any mongo/postgress

#endif

#ifndef DISCARDJSONOPEN

CurlShutdown();

#endif

if (logmainbuffer) free(logmainbuffer);

logmainbuffer = NULL;

if (ourMainOutputBuffer) free(ourMainOutputBuffer);

ourMainOutputBuffer = NULL;

}

////////////////////////////////////////////////////////

/// INPUT PROCESSING

////////////////////////////////////////////////////////

#ifdef LINUX

#define LINUXORMAC 1

#endif

#ifdef __MACH__

#define LINUXORMAC 1

#endif

#ifdef LINUXORMAC

#include <fcntl.h>

int kbhit(void)

{

struct termios oldt, newt;

int ch;

int oldf;

tcgetattr(STDIN_FILENO, &oldt);

newt = oldt;

newt.c_lflag &= ~(ICANON | ECHO);

tcsetattr(STDIN_FILENO, TCSANOW, &newt);

oldf = fcntl(STDIN_FILENO, F_GETFL, 0);

fcntl(STDIN_FILENO, F_SETFL, oldf | O_NONBLOCK);

ch = getchar();

tcsetattr(STDIN_FILENO, TCSANOW, &oldt);

fcntl(STDIN_FILENO, F_SETFL, oldf);

if(ch != EOF)

{

ungetc(ch, stdin);

return 1;

}

return 0;

}

#endif

int FindOOBEnd(int start)

{

if (*wordStarts[start] == OOB_START)

{

while (++start < wordCount && *wordStarts[start] != OOB_END); // find a close

return (*wordStarts[start] == OOB_END) ? (start + 1) : 1;

}

else return 1; // none

}

void ProcessOOB(char* output)

{

char* ptr = SkipWhitespace(output);

if (*ptr == OOB_START) // out-of-band data?

{

char* at = ptr;

bool quote = false;

int bracket = 1;

while (*++at) // do full balancing

{

if (*at == '"' && *(at-1) != '\\') quote = !quote;

else if (!quote)