#include "common.h"

//------------------------

// ALWAYS AVAILABLE

//------------------------

#define PATTERNDEPTH 1000

bool nowarndupconcept = false;

unsigned int supplementalColumn = 0;

static HEAPREF undefinedCallThreadList = NULL;

static HEAPREF undefinedConceptThreadList = NULL;

unsigned int currentBuild = 0;

static bool nospellcheck = false;

static bool noPatternOptimization = true;

static unsigned int conceptID = 0; // name of concept set

char* patternStarter = NULL;

char* patternEnder = NULL;

const char* linestartpoint = NULL;

static bool isConcept = false;

static WORDP currentFunctionDefinition; // current macro defining or executing

HEAPREF deadfactsList = 0;

HEAPREF languageadjustedfactsList = 0;

static unsigned int complexity = 0;

static bool livecall = false;

bool echorulepattern = false;

static unsigned int priorLine = 0;

static char* currentTopicBots = NULL;

bool autoset = false;

static char macroName[MAX_WORD_SIZE];

static uint64 macroid;

char* dataBase = NULL;

static char* dataChunk = NULL;

static char* outputStart = NULL;

static char* lineStart = NULL;

static bool globalBotScope = false;

static HEAPREF beenHereThreadList = NULL;

char* newScriptBuffer = NULL;

char* oldScriptBuffer = NULL;

static char display[MAX_DISPLAY][100];

static int displayIndex = 0;

static char* incomingPtrSys = 0; // cache AFTER token find ptr when peeking.

static char lookaheadSys[MAX_WORD_SIZE]; // cache token found when peeking

static unsigned int hasWarnings; // number of warnings generated

unsigned int hasErrors;

uint64 grade = 0; // vocabulary warning

char* lastDeprecation = 0;

CompileStatus compiling = NOT_COMPILING; // script compiler in progress

bool patternContext = false; // current compiling a pattern

unsigned int buildId; // current build

static int callingSystem = 0;

static bool chunking = false;

static unsigned int substitutes;

static unsigned int cases;

static unsigned int badword;

static unsigned int functionCall;

static bool isDescribe = false;

char* tableinput = NULL;

static char functionArguments[MAX_ARG_LIMIT + 1][500];

static int functionArgumentCount = 0;

char warnings[MAX_WARNINGS][MAX_WORD_SIZE];

unsigned int warnIndex = 0;

static char baseName[SMALL_WORD_SIZE];

char errors[MAX_ERRORS][MAX_WORD_SIZE];

unsigned int errorIndex = 0;

char scopeBotName[MAX_WORD_SIZE];

static bool renameInProgress = false;

static bool endtopicSeen = false; // needed when ending a plan

static char* nextToken; // current lookahead token

unsigned int buildID = 0;

static char* topicFiles[] = // files created by a topic refresh from scratch

{

(char*)"describe", // document variables functions concepts topics etc

(char*)"facts", // hold facts

(char*)"allfacts", // hold all binary fast facts

(char*)"allwords", // hold all binary fast words

(char*)"keywords", // holds topic and concepts keywords

(char*)"macros", // holds macro definitions

(char*)"map", // where things are defined (will also do map.json)

(char*)"script", // hold topic definitions

(char*)"plans", // hold plan definitions

(char*)"dict", // dictionary changes

(char*)"canon", // private canonical values

0

};

static void WriteKey(char* word);

static FILE* mapFile = NULL; // for IDE

static FILE* mapFileJson = NULL; // easier to parse

static char* ReadMacro(char* ptr, FILE* in, char* kind, unsigned int build,char* data);

static unsigned int mapTopicFileCount = 0;

static unsigned int mapItemCount = 0;

static unsigned int mapRuleCount = 0;

void EraseTopicBin(unsigned int build, char* name)

{

int i = -1;

int result;

while (topicFiles[++i])

{

char file[SMALL_WORD_SIZE];

sprintf(file, (char*)"%s/BUILD%s/%s%s.bin", topicfolder, name, topicFiles[i], name); // new style

result = remove(file);

}

}

void InitScriptSystem()

{

compiling = NOT_COMPILING;

oldScriptBuffer = NULL;

newScriptBuffer = NULL;

mapFile = NULL;

mapFileJson = NULL;

outputStart = NULL;

}

void AddWarning(char* buffer)

{

char c = buffer[MAX_WORD_SIZE - 300];

size_t len = strlen(buffer);

if (len > (MAX_WORD_SIZE - 300)) buffer[MAX_WORD_SIZE - 300] = 0;

bool chop = false;

if (buffer[len - 2] == '\r') buffer[len - 2] = 0;

sprintf(warnings[warnIndex], (char*)"%s ending line %u col %u", buffer, currentFileLine, currentLineColumn);

char* ptr = warnings[warnIndex] + strlen(warnings[warnIndex]);

if (*currentFilename) sprintf(ptr, " of %s", currentFilename);

warnIndex++;

buffer[MAX_WORD_SIZE - 300] = c;

strcat(buffer, "\r\n");

if (strstr(warnings[warnIndex-1],(char*)"is not a known word")) {++badword;}

else if (strstr(warnings[warnIndex-1],(char*)" changes ")) {++substitutes;}

else if (strstr(warnings[warnIndex-1],(char*)"is unknown as a word")) {++badword;}

else if (strstr(warnings[warnIndex-1],(char*)"in opposite case")){++cases;}

else if (strstr(warnings[warnIndex - 1], (char*)"multiple spellings")) { ++cases; }

else if (strstr(warnings[warnIndex-1],(char*)"a function call")){++functionCall;}

if (warnIndex >= MAX_WARNINGS) --warnIndex;

}

bool StartScriptCompiler(bool normal)

{

#ifndef DISCARDSCRIPTCOMPILER

if (nextToken && normal) return false; // we are already in a build

if (oldScriptBuffer) return false; // already running one

conceptID = 0;

beenHereThreadList = NULL;

livecall = !normal;

oldScriptBuffer = newScriptBuffer; // so we can nest calls to script compiler

warnIndex = errorIndex = 0;

newScriptBuffer = AllocateStack(NULL, maxBufferSize);

nextToken = AllocateStack(NULL, maxBufferSize); // able to swallow big token

#endif

return true;

}

void EndScriptCompiler()

{

#ifndef DISCARDSCRIPTCOMPILER

if (newScriptBuffer)

{

ReleaseStack(newScriptBuffer);

newScriptBuffer = oldScriptBuffer;

oldScriptBuffer = NULL;

nextToken = NULL;

}

#endif

}

void ScriptError()

{

#ifndef DISCARDSCRIPTCOMPILER

callingSystem = 0;

chunking = false;

outputStart = NULL;

renameInProgress = false;

if (compiling || csapicall == TEST_OUTPUT || csapicall == TEST_PATTERN)

{

++hasErrors;

patternContext = false;

if (*scopeBotName) Log(USERLOG,"*** Error- line %d col %d of %s bot:%s : ", currentFileLine, currentLineColumn + supplementalColumn, currentFilename, scopeBotName);

else Log(USERLOG,"*** Error- line %d col %d of %s: ", currentFileLine, currentLineColumn + supplementalColumn, currentFilename);

}

#endif

}

#ifndef DISCARDSCRIPTCOMPILER

void ScriptWarn()

{

if (compiling)

{

++hasWarnings;

if (*currentFilename)

{

if (*scopeBotName) Log(USERLOG,"*** Warning- line %d col %d of %s bot:%s : ", currentFileLine, currentLineColumn + supplementalColumn,currentFilename, scopeBotName);

else Log(USERLOG,"*** Warning- line %d col %d of %s: ", currentFileLine, currentLineColumn+supplementalColumn,currentFilename);

}

else Log(USERLOG,"*** Warning- ");

}

}

static void AddBeenHere(WORDP D)

{

D->internalBits |= BEEN_HERE;

beenHereThreadList = AllocateHeapval(HV1_WORDP,beenHereThreadList, (uint64)D);// save name

}

void UnbindBeenHere()

{

while (beenHereThreadList)

{

uint64 D;

beenHereThreadList = UnpackHeapval(beenHereThreadList, D,discard);

((WORDP)D)->internalBits &= -1 ^ BEEN_HERE;

}

}

#endif

void AddError(char* buffer)

{

char seen[MAX_WORD_SIZE];

*seen = 0;

char* at = seen;

if (patternStarter && patternEnder && csapicall != COMPILE_PATTERN && csapicall != COMPILE_OUTPUT)

{

strcpy(at, "--> ");

at += 4;

size_t len = patternEnder - patternStarter;

*patternEnder = 0;

if (len < 100) strcpy(at, patternStarter);

else

{

strncpy(at, patternStarter,50);

at += 50;

sprintf(at, "%s"," ... ");

sprintf(at + 5, "%s", patternEnder - 50);

}

strcat(at, " <--");

}

char message[MAX_WORD_SIZE];

if (*buffer == '\r') ++buffer;

if (*buffer == '\n') ++buffer;

size_t len = strlen(buffer);

char* ptr = buffer + ( len > MAX_WORD_SIZE - 300 ? MAX_WORD_SIZE - 300 : len) - 1;

while (ptr > buffer && (*ptr == '\n' || *ptr == '\r')) --ptr;

char c = *(++ptr);

*ptr = 0;

if (!*currentFilename) // dse compilepattern

{

sprintf(message, "%s ended %s line %u col %u ", buffer, seen, currentFileLine, currentLineColumn+supplementalColumn);

strcat(message, "\r\n");

}

else sprintf(message, "%s ended line %u col %u of %s %s\r\n", buffer, currentFileLine, currentLineColumn+supplementalColumn, currentFilename, scopeBotName);

*ptr = c;

sprintf(errors[errorIndex++], (char*)"%s\r\n", message);

if (errorIndex >= MAX_ERRORS) --errorIndex;

}

static char* FindComparison(char* word)

{

if (!*word || !word[1] || !word[2] || *word == '"') return NULL; // if token is short, we cannot do the below word+1 scans

if (*word == '.') return NULL; // .<_3 is not a comparison

if (*word == '\\') return NULL; // escaped is not a comparison

if (*word == '!' && word[1] == '?' && word[2] == '$') return NULL;

if (*word == '_' && word[1] == '?' && word[2] == '$') return NULL;

if (*word == '?' && word[1] == '$') return NULL;

char* at = strchr(word + 1, '!');

if (at && *word == '!') at = NULL; // ignore !!

if (!at)

{

at = strchr(word + 1, '<');

if (at && at[1] == '<') return NULL; // << is not a comparison

}

if (!at)

{

at = strchr(word + 1, '>');

if (at && at[1] == '>') return NULL; // >> is not a comparison

}

if (!at)

{

at = strchr(word + 1, '&');

if (at && (at[1] == '_' || at[1] == ' ' || !(IsDigit(at[1]) || at[1] == USERVAR_PREFIX || at[1] == '#'))) at = 0; // ignore & as part of a name, e.g. AT&T

}

if (!at) at = strchr(word + 1, '=');

if (!at) at = strchr(word + 1, '?'); // member of set

if (!at)

{

at = strchr(word + 1, '!'); // negation

if (at && (at[1] == '=' || at[1] == '?'));

else at = NULL;

}

return at;

}

static void InsureAppropriateCase(char* word)

{

char c;

char* at = FindComparison(word);

// force to lower case various standard things

// topcs/sets/classes/user vars/ functions and function vars are always lower case

if (at) // a comparison has 2 sides

{

c = *at;

*at = 0;

InsureAppropriateCase(word);

if (at[1] == '=' || at[1] == '?') InsureAppropriateCase(at + 2); // == or >= or such

else InsureAppropriateCase(at + 1);

*at = c;

}

else if (*word == '_' || *word == '\'') InsureAppropriateCase(word + 1);

else if (*word == USERVAR_PREFIX)

{

char* dot = strchr(word, '.');

if (dot) *dot = 0;

MakeLowerCase(word);

if (dot) *dot = '.';

}

else if ((*word == '^' && word[1] != '"') || *word == '~' || *word == SYSVAR_PREFIX || *word == '|') MakeLowerCase(word);

else if (*word == '@' && IsDigit(word[1])) MakeLowerCase(word); // potential factref like @2subject

}

static int GetFunctionArgument(char* arg) // get index of argument (1-based) if it is value, else -1

{

for (int i = 1; i <= functionArgumentCount; ++i)

{

if (!stricmp(arg, functionArguments[i])) return i;

}

return -1; // failed

}

static void FindDeprecated(char* ptr, char* value, char* message)

{

char* comment = strstr(ptr, (char*)"# ");

char* at = ptr;

size_t len = strlen(value);

while (at)

{

at = strstr(at, value);

if (!at) break;

if (*(at - 1) == USERVAR_PREFIX) // $$xxx should be ignored

{

at += 2;

continue;

}

if (comment && at > comment) return; // inside a comment

char word[MAX_WORD_SIZE];

ReadCompiledWord(at, word);

if (!stricmp(value, word))

{

lastDeprecation = at;

BADSCRIPT(message);

}

at += len;

}

}

static void AddDisplay(char* word)

{

MakeLowerCase(word);

for (int i = 0; i < displayIndex; ++i)

{

if (!strcmp(word, display[i])) return; // no duplicates needed

}

strcpy(display[displayIndex], word);

if (++displayIndex >= MAX_DISPLAY) BADSCRIPT("Display argument limited to %d: %s\r\n", MAX_DISPLAY, word)

}

static char* ReadDisplay(FILE* in, char* ptr)

{

char word[SMALL_WORD_SIZE];

ptr = ReadNextSystemToken(in, ptr, word, false);

while (1)

{

ptr = ReadNextSystemToken(in, ptr, word, false);

if (*word == ')') break;

if (*word != USERVAR_PREFIX)

BADSCRIPT("Display argument must be uservar of $$ $ or $_: %s\r\n", word)

if (strchr(word, '.'))

BADSCRIPT("Display argument cannot be dot-selected %s\r\n", word)

AddDisplay(word); // explicit display

}

return ptr;

}

char* ReadSystemToken(char* ptr, char* word, bool separateUnderscore) // how we tokenize system stuff (rules and topic system) words -remaps & to AND

{

*word = 0;

if (!ptr) return 0;

char tmp[MAX_WORD_SIZE];

char* start = word;

*start = 0;

ptr = SkipWhitespace(ptr);

while (compiling == PIECE_COMPILE && *ptr == '\\' && ptr[1] == 'n') // api calls

{

ptr = SkipWhitespace(ptr + 2);

currentFileLine += 1;

currentLineColumn = 0;

}

FindDeprecated(ptr, (char*)"$login", (char*)"Deprecated $login needs to be $cs_login\r\n");

FindDeprecated(ptr, (char*)"$userfactlimit", (char*)"Deprecated $userfactlimit needs to be $cs_userfactlimit\r\n");

FindDeprecated(ptr, (char*)"$crashmsg", (char*)"Deprecated $crashmsg needs to be $cs_crashmsg\r\n");

FindDeprecated(ptr, (char*)"$randindex", (char*)"Deprecated $randindex needs to be $cs_randindex\r\n");

FindDeprecated(ptr, (char*)"$wildcardseparator", (char*)"Deprecated $wildcardseparator needs to be $cs_wildcardseparator\r\n");

FindDeprecated(ptr, (char*)"$abstract", (char*)"Deprecated $abstract needs to be $cs_abstract\r\n");

FindDeprecated(ptr, (char*)"$prepass", (char*)"Deprecated $prepass needs to be $cs_prepass\r\n");

FindDeprecated(ptr, (char*)"$control_main", (char*)"Deprecated $control_main needs to be $cs_control_main\r\n");

FindDeprecated(ptr, (char*)"$control_pre", (char*)"Deprecated $control_pre needs to be $cs_control_pre\r\n");

FindDeprecated(ptr, (char*)"$control_post", (char*)"Deprecated $control_post needs to be $cs_control_post\r\n");

#ifdef INFORMATION

/***

A token is nominally a contiguous collection of characters broken off by tab or space(since return and newline are stripped off).

Tokens to include whitespace are encased in doublequotes.

Characters with reserved status automatically also break into individual tokens and to include them you must put \ before them.These include :

[]() {} always and separate into individual tokens except for _(_[_{

< > and << >> are reserved, but only when at start or end of token.Allowed comparisons embedded.As is <= and >=

Tokens ending with ' or 's break off(possessive) in patterns.

Tokens starting with prefix characters ' or ! or _ keep together, except per reserved tokens. '$junk is one token.

Variables ending with punctuation separate the punctuation.$hello.is two tokens as is _0.

Reserved characters in a composite token with _ before or after are kept.E.g.This_(_story_is_)_done

You can include a reserved tokens by putting \ in front of them.

Some tokens revise their start, like the pattern tokens representing comparison.They do this in the script compiler.

***/

#endif

// strings

if (*ptr == '\'' && ptr[1] == '"')

{

*word++ = *ptr++;

}

if (*ptr == '"' || (*ptr == '^' && ptr[1] == '"') || (*ptr == '^' && ptr[1] == '\'') || (*ptr == '\\' && ptr[1] == '"')) // doublequote maybe with functional heading

{

// simple \"

if (*ptr == '\\' && (!ptr[2] || ptr[2] == ' ' || ptr[2] == '\t' || ptr[2] == ENDUNIT)) // legal

{

*word = '\\';

word[1] = '"';

word[2] = 0;

return ptr + 2;

}

bool backslash = false;

bool noblank = true;

bool functionString = false;

if (*ptr == '^')

{

*word++ = *ptr++; // ^"script" swallows ^

noblank = false; // allowed blanks at start or rear

functionString = true;

}

else if (*ptr == '\\') // \"string is this"

{

backslash = true;

++ptr;

}

char* end = ReadQuote(ptr,word,backslash,noblank,MAX_WORD_SIZE); // swallow ending marker and points past

if (end)

{

if (*word == '"' && word[1] != FUNCTIONSTRING && !functionString) return end; // all legal within

// NOW WE SEE A FUNCTION STRING

// when seeing ^, see if it remaps as a function argument

// check for internal ^ also...

char* hat = word - 1;

if ((*word == '"' || *word == '\'') && functionString) hat = word; // came before

else if (*word == '"' && word[1] == FUNCTIONSTRING) hat = word + 1;

else if ((word[1] == '"' || word[1] == '\'') && *word == FUNCTIONSTRING) hat = word;

// locate any local variable references in active strings

char* at = word;

while ((at = strchr(at,USERVAR_PREFIX)))

{

if (at[1] == LOCALVAR_PREFIX)

{

char* start = at;

while (++at)

{

if (!IsAlphaUTF8OrDigit(*at) && *at != '_' && *at != '-')

{

char c = *at;

*at = 0;

AddDisplay(start);

*at = c;

break;

}

}

}

else ++at;

}

while ((hat = strchr(hat + 1,'^'))) // find a hat within

{

if (IsDigit(hat[1])) continue; // normal internal

if (*(hat - 1) == '\\') continue; // escaped

char* atx = hat;

while (*++atx && (IsAlphaUTF8OrDigit(*atx) || *atx == '_')) { ; }

char c = *atx;

*atx = 0;

int index = GetFunctionArgument(hat);

WORDP D = FindWord(hat); // in case its a function name

*atx = c;

if (index >= 0) // was a function argument

{

strcpy(tmp,atx); // protect chunk

sprintf(hat,(char*)"^%d%s",index,tmp);

}

else if (D && D->internalBits & FUNCTION_NAME) { ; }

else if (!renameInProgress && !(hat[1] == USERVAR_PREFIX || hat[1] == MATCHVAR_PREFIX))

{

*atx = 0;

WARNSCRIPT((char*)"%s is not a recognized function argument. Is it intended to be?",hat)

*atx = c;

}

}

hat = word - 1;

while ((hat = strchr(hat + 1,'_'))) // rename _var?

{

if (*(hat - 1) == '$') continue;

if (IsAlphaUTF8OrDigit(*(hat - 1)) || *(hat - 1) == '_' || *(hat - 1) == '-') continue; // not a starter

if (IsDigit(hat[1])) continue; // normal _ var

if (*(hat - 1) == '\\' || *(hat - 1) == '"') continue; // escaped or quoted

char* atx = hat;

while (*++atx && (IsAlphaUTF8OrDigit(*atx))) { ; } // find end

WORDP D = FindWord(hat,atx - hat,LOWERCASE_LOOKUP);

if (D && D->internalBits & RENAMED) // remap matchvar inside string

{

strcpy(tmp,atx); // protect chunk

sprintf(hat + 1,(char*)"%u%s",(unsigned int)D->properties,tmp);

}

}

hat = word - 1;

while ((hat = strchr(hat + 1,'@'))) // rename @set?

{

if (IsAlphaUTF8OrDigit(*(hat - 1))) continue; // not a starter

if (IsDigit(hat[1]) || hat[1] == '_') continue; // normal @ var or @_marker

if (*(hat - 1) == '\\') continue; // escaped

char* atx = GetSetEnd(hat);

WORDP D = FindWord(hat,atx - hat,LOWERCASE_LOOKUP);

if (D && D->internalBits & RENAMED) // rename @set inside string

{

strcpy(tmp,atx); // protect chunk

sprintf(hat + 1,(char*)"%u%s",(unsigned int)D->properties,tmp);

}

else if (!renameInProgress) // can do anything safely in a simple quoted string

{

char c = 0;

if (at && *at)

{

c = *at;

*at = 0;

}

WARNSCRIPT((char*)"%s is not a recognized @rename. Is it intended to be?\r\n",hat)

if (c) *at = c;

}

}

hat = word - 1;

if (strstr(readBuffer, "rename:")) // accept rename of existing constant twice in a row

hat = " ";

while ((hat = strchr(hat + 1,'#'))) // rename #constant or ##constant

{

if (*(hat - 1) == '\\') continue; // escaped

if (IsAlphaUTF8OrDigit(*(hat - 1)) || IsDigit(hat[1]) || *(hat - 1) == '&') continue; // not a starter, maybe #533; constant stuff

char* at = hat;

if (at[1] == '#') ++at; // user constant

while (*++at && (IsAlphaUTF8OrDigit(*at) || *at == '_')) { ; } // find end

strcpy(tmp,at); // protect chunk

*at = 0;

uint64 n;

if (hat[1] == '#' && IsAlphaUTF8(hat[2])) // user constant

{

WORDP D = FindWord(hat,at - hat,LOWERCASE_LOOKUP);

if (D && D->internalBits & RENAMED) // remap #constant inside string

{

n = D->properties;

if (D->internalBits & CONSTANT_IS_NEGATIVE)

{

int64 x = (int64)n;

x = -x;

sprintf(hat,(char*)"%lld%s",(long long int) x,tmp);

}

else

{

sprintf(hat,(char*)"%lld%s",(long long int) n,tmp);

}

}

}

else // system constant

{

n = FindPropertyValueByName(hat + 1);

if (!n) n = FindSystemValueByName(hat + 1);

if (!n) n = FindParseValueByName(hat + 1);

if (!n) n = FindMiscValueByName(hat + 1);

if (n)

{

sprintf(hat,(char*)"%lld%s",(long long int) n,tmp);

}

}

if (!*hat)

{

*hat = '#';

BADSCRIPT((char*)"Bad # constant %s\r\n",hat)

}

}

return end; // if we did swallow a string

}

if (*ptr == '\\') // was this \"xxx with NO closing

{

memmove(word + 1,word,strlen(word) + 1);

*word = '\\';

}

else

{

word = start;

if (*start == '^') --ptr;

}

}

// reading concept, careful handling for emoticons emoji

if (isConcept)

{

if (!*ptr) return ptr;

while (*ptr != ' ' && *ptr) *word++ = *ptr++;

*word = 0;

if (*(word - 1) == ')') // closing ) for concept, not part of emoticon?

{

if (( *start == '~' || IsAlphaUTF8(*start)) && IsAlphaUTF8(start[1]) && IsAlphaUTF8(start[2])) // clear word

{

--ptr;

--word;

}

}

*word = 0;

if (start) return ptr; // test always true, but makes debug visible on start

}

// the normal composite token

bool quote = false;

char* xxorig = ptr;

bool var = (*ptr == '$');

int brackets = 0;

char quotechar = '"';

bool activestring = false;

while (*ptr)

{

if (*ptr == ENDUNIT) break;

if (patternContext && quote) {} // allow stuff in comparison quote

else if (*ptr == ' ' || (*ptr == '\t' && convertTabs) || (*ptr == '\\' && ptr[1] == 'n' && compiling == PIECE_COMPILE)) break; // legal

if (patternContext && activestring == false && *ptr == '^' && (ptr[1] == '"' || ptr[1] == '\''))

{

quotechar = ptr[1];

activestring = true;

}

if (patternContext && *ptr == quotechar)

quote = !quote;

char c = *ptr++;

if (c == '\t' && !convertTabs && word != start)

{

--ptr;

break; // end word with tab

}

// not for output, but for patterns, track line numbers

if (csapicall == COMPILE_PATTERN && c == '\\' && (*ptr == 'n' || *ptr == '\t' || *ptr == '\r')) // break off manual new line

{

--ptr;

break;

}

*word++ = c;

*word = 0;

if (*start == '\t' && !convertTabs )

break; // return tab as unique word

if ((word - start) > (MAX_WORD_SIZE - 2)) break; // avoid overflow

if (c == '\\') *word++ = *ptr++; //escaped

// want to leave array json notation alone but react to [...] touching a variable - $var]

else if (var && c == '[') // ANY variable should be separated by space from a [ if not json array

{

++brackets; // this MUST then be a json array and brackets will balance

if (brackets > 1) BADSCRIPT("$var MUST be separated from [ unless you intend json array reference\r\n")

}

else if (var && c == ']')

{

if (--brackets < 0) // if brackets is set, we must be in json array

{

--ptr;

--word;

break;

}

}

else if (GetNestingData(c) && !quote) // break off nesting attached to a started token unless its an escaped token

{

size_t len = word - start;

if (len == 1) break; // automatically token by itself

if (len == 2)

{

if ((*start == '_' || *start == '!') && (c == '[' || c == '(' || c == '{')) break; // one token as _( or !(

if (*start == '\\') break; // one token escaped

}

// split off into two tokens

--ptr;

--word;

break;

}

}

*word = 0;

word = start;

size_t len = strlen(word);

if (len == 0) return ptr;

if (patternContext && word[len - 1] == '"' && word[len - 2] != '\\')

{

char* quote = strchr(word, '"');

if (quote == word + len - 1)

BADSCRIPT("Tailing quote without start: %s\r\n", word)

}

if (*word == '#' && !strstr(readBuffer,"rename:")) // is this a constant from dictionary.h? or user constant

{

uint64 n;

if (word[1] == '#' && IsAlphaUTF8(word[2])) // user constant

{

WORDP D = FindWord(word,0,LOWERCASE_LOOKUP);

if (D && D->internalBits & RENAMED) // remap #constant

{

n = D->properties;

if (D->internalBits & CONSTANT_IS_NEGATIVE)

{

int64 x = (int64)n;

x = -x;

sprintf(word,(char*)"%lld",(long long int) x);

}

else

{

sprintf(word,(char*)"%lld",(long long int) n);

}

}

else if (renameInProgress) { ; } // leave token alone, defining

else BADSCRIPT((char*)"Bad user constant %s\r\n",word)

}

else // system constant

{

n = FindPropertyValueByName(word + 1);

if (!n) n = FindSystemValueByName(word + 1);

if (!n) n = FindParseValueByName(word + 1);

if (!n) n = FindMiscValueByName(word + 1);

if (n)

{

sprintf(word,(char*)"%lld",(long long int) n);

}

else if (!IsDigit(word[1]) && word[1] != '!') //treat rest as a comment line (except if has number after it, which is user text OR internal arg reference for function

{

if (IsAlphaUTF8(word[1]))

BADSCRIPT((char*)"Bad numeric # constant %s\r\n",word)

*ptr = 0;

*word = 0;

}

}

}

if (*word == '_' && (IsAlphaUTF8(word[1]))) // is this a rename _

{

WORDP D = FindWord(word);

if (D && D->internalBits & RENAMED) sprintf(word + 1,(char*)"%u",(unsigned int)D->properties); // remap match var convert to number

// patterns can underscore ANYTING

}

if (*word == '\'' && word[1] == '_' && (IsAlphaUTF8(word[2]))) // is this a rename _ with '

{

WORDP D = FindWord(word + 1);

if (D && D->internalBits & RENAMED) sprintf(word + 2,(char*)"%u",(unsigned int)D->properties); // remap match var convert to number

else if (!renameInProgress && !patternContext) // patterns can underscore ANYTING

WARNSCRIPT((char*)"%s is not a recognized _rename. Should it be?\r\n",word + 1)

}

if (*word == '@' && IsAlphaUTF8(word[1])) // is this a rename @

{

char* at = GetSetEnd(word);

WORDP D = FindWord(word,at - word);

if (D && D->internalBits & RENAMED) // remap @set in string

{

strcpy(tmp,at); // protect chunk

sprintf(word + 1,(char*)"%u%s",(unsigned int)D->properties,tmp);

}

else if (!renameInProgress) WARNSCRIPT((char*)"%s is not a recognized @rename. Is it intended to be?\r\n",word)

}

if (*word == '@' && word[1] == '_' && IsAlphaUTF8(word[2])) // is this a rename @_0+

{

size_t lenx = strlen(word);

WORDP D = FindWord(word + 1, lenx - 1); // @_data marker

char c = 0;

if (!D)

{

c = word[lenx - 1];

word[lenx - 1] = 0;

D = FindWord(word + 1, lenx - 2);

word[lenx - 1] = c;

}

if (D && D->internalBits & RENAMED)

{

if (c) sprintf(word + 2,(char*)"%u%c",(unsigned int)D->properties,c); // remap @set in string

else sprintf(word + 2, (char*)"%u", (unsigned int)D->properties); // remap @set in string

}

else if (!renameInProgress) WARNSCRIPT((char*)"%s is not a recognized @rename. Is it intended to be?\r\n",word)

}

// some tokens require special splitting

// break off starting << from <<hello

if (*word == '<' && word[1] != '=')

{

if (len == 3 && *word == word[1] && word[2] == '=') { ; }

else if (word[1] == '<')

{

if (word[2]) // not assign operator

{

ptr -= strlen(word) - 2; // safe

word[2] = 0;

len -= 2;

}

}

}

// break off ending >> from hello>>

if (len > 2 && word[len - 1] == '>')

{

if (len == 3 && *word == word[1] && word[2] == '=') { ; }

else if (word[len - 2] == '>')

{

ptr -= 2;

word[len - 2] = 0;

len -= 2;

}

}

// break off punctuation from variable end

if (len > 2 && ((*word == USERVAR_PREFIX && !IsDigit(word[1])) || *word == '^' || (*word == '@' && IsDigit(word[1])) || *word == SYSVAR_PREFIX || (*word == MATCHVAR_PREFIX && IsDigit(word[1])) || (*word == '\'' && word[1] == '_'))) // not currency

{

if (word[len - 2] == '\\') {} // not escaped ending punctuationes

else if (!patternContext || word[len - 1] != '?') // BUT NOT $$xxx? in pattern context

{

while (IsRealPunctuation(word[len - 1])) // one would be enough, but $hello... needs to be addressed

{

--len;

--ptr;

}

word[len] = 0;

}

}

// break off opening < in pattern

if (patternContext && *word == '<' && word[1] != '<')

{

ptr -= len - 1;

len = 1;

word[1] = 0;

}

// break off closing > in pattern unless escaped or notted

if (len == 2 && (*word == '!' || *word == '\\')) { ; }

else if (patternContext && len > 1 && word[len - 1] == '>' && word[len - 2] != '>' && word[len - 2] != '_' && word[len - 2] != '!')

{

ptr -= 1;

--len;

word[len] = 0;

}

// find internal comparison op if any

char* at = (patternContext) ? FindComparison(word) : 0;

if (at && *word == '*' && !IsDigit(word[1]))

{

if (compiling) BADSCRIPT((char*)"TOKENS-1 Cannot do comparison on variable gap %s . Memorize and compare against _# instead later.\r\n",word)

}

if (at && *at == '!' && at[1] == '$') { ; } // allow !$xxx

else if (at) // revise comparison operators

{

if (*at == '!') ++at;

++at;

if (*at == '^' && at[1]) // remap function arg on right side.

{

int index = GetFunctionArgument(at);

if (index >= 0) sprintf(at,(char*)"^%d",index);

}

if (*at == '_' && IsAlphaUTF8(word[1])) // remap rename matchvar arg on right side.

{

WORDP D = FindWord(at);

if (D && D->internalBits & RENAMED) sprintf(at,(char*)"_%u",(unsigned int)D->properties);

}

if (*at == '@' && IsAlphaUTF8(word[1])) // remap @set arg on right side.

{

char* at1 = GetSetEnd(at);

WORDP D = FindWord(at,at1 - at);

if (D && D->internalBits & RENAMED) // remap @set on right side

{

strcpy(tmp,at1); // protect chunk

sprintf(at + 1,(char*)"%u%s",(unsigned int)D->properties,tmp);

}

}

// check for remap on LHS

if (*word == '^')

{

char c = *--at;

*at = 0;

int index = GetFunctionArgument(word);

*at = c;

if (index >= 0)

{

sprintf(tmp,(char*)"^%d%s",index,at);

strcpy(word,tmp);

}

}

// check for rename on LHS

if (*word == '_' && IsAlphaUTF8(word[1]))

{

char* atx = word;

while (IsAlphaUTF8OrDigit(*++atx)) { ; }

WORDP D = FindWord(word,atx - word);

if (D && D->internalBits & RENAMED) // remap match var

{

sprintf(tmp,(char*)"%u%s",(unsigned int)D->properties,atx);

strcpy(word + 1,tmp);

}

}

// check for rename on LHS

if (*word == '@' && IsAlphaUTF8(word[1]))

{

char* atx = GetSetEnd(word);

WORDP D = FindWord(word,atx - word);

if (D && D->internalBits & RENAMED) // remap @set in string

{

strcpy(tmp,atx); // protect chunk

sprintf(word + 1,(char*)"%u%s",(unsigned int)D->properties,tmp);

}

}

}

// when seeing ^, see if it remaps as a function argument

// check for internal ^ also...

char* hat = word - 1;

if (*word == '$' && word[1] == '^' && strstr(word, ":=")) { return ptr; } // fn define

else while ((hat = strchr(hat + 1,'^'))) // find a hat within

{

char* at = hat;

while (*++at && (IsAlphaUTF8(*at) || *at == '_' || IsDigit(*at))) { ; }

char c = *at;

*at = 0; // terminate it so internal ^ is recognized uniquely

strcpy(tmp,hat);

*at = c;

while (*tmp)

{

int index = GetFunctionArgument(tmp);

if (index >= 0)

{

char remainder[MAX_WORD_SIZE];

strcpy(remainder,at); // protect chunk AFTER this

sprintf(hat,(char*)"^%d%s",index,remainder);

break;

}

else tmp[0] = 0; // just abort it for now shrink it smaller, to handle @9subject kinds of behaviors

}

}

// same for quoted function arg

if (*word == '\'' && word[1] == '^' && word[2])

{

int index = GetFunctionArgument(word + 1);

if (index >= 0) sprintf(word,(char*)"'^%d",index);

}

// break apart math on variables eg $value+2 as a service to the user

if ((*word == '%' || *word == '$') && word[1]) // cannot use _ here as that will break memorization pattern tokens

{

char* atx = word + 1;

if (atx[1] == '$' || atx[1] == '_') ++atx; // skip over 2ndary marker

--atx;

while (LegalVarChar(*++atx) || (*atx == '\\' && atx[1] == '$')); // find end of initial word - allowing \ for json $

if (*word == '$' && (*atx == '.' || *atx == '[' || *atx == ']') && (LegalVarChar(atx[1]) || atx[1] == '$' || atx[1] == '[' || atx[1] == ']' || (atx[1] == '\\' && atx[2] == '$')))// allow $x.y as a complete name

{

while (LegalVarChar(*++atx) || *atx == '.' || *atx == '$' || (*atx == '[' || *atx == ']' || (*atx == '\\' && atx[1] == '$'))); // find end of field name sequence

if (*(atx - 1) == '.') --atx; // tailing period cannot be part of it

}

if (*atx && IsPunctuation(*atx) & ARITHMETICS && *atx != '=')

{

if (*atx == '.' && atx[1] == '_' && IsDigit(atx[2])) {} // json field reference indirection

else if (*atx == '.' && atx[1] == '\'' && atx[2] == '_' && IsDigit(atx[3])) {} // json field reference indirection

// - is legal in a var or word token

else if (*atx != '-' || (!IsAlphaUTF8OrDigit(atx[1]) && atx[1] != '_'))

{

ptr -= strlen(atx);

*atx = 0;