Skip to content

Commit e1192f5

Browse files
nicolew02CatherineHigg
authored and
rossedens
committed
Gw imprv (#207)
* Gateway-Adding error message to asyncexecjpts to distinguish between inactive and non-existent servers * Adding error message to .gw.getserverids for sync queries * Adding distinct to list of servertypes in getserverids * Modifying asyncexecjpts to call getserverids * Reducing code duplication in getserverids * Adding check for a null server * Adding missing comma in activeservermsg * Checks for correct attribute parameter types * Added indexing, removed accidental whitespace, unnecessary brackets and 'h' * Added more indentation * Atoms and lists accounted for with abs in correctAttType, combined 99/11h type checks, corrected servertype names * Corrected indentation * Check for just sym list type after dictionary type check * Added abs in symbol list check to protect from atoms * Added missing w to end of check * Updating gateway section of Processes.md * Updated error handling Updated error handling with recently added error messages * Adding unit tests for getserverids in gateway * added a client calls example section * Removed unnecessary client call examples * Moving gwtests.q to /code/gateway/gatewaylib.q. Removing functions from gateway.q. Adding line to config * Updated client call examples and added q) prompts Added date col check before running queries across RDB and HDB * removed call to view .rdb namespace * Edited rdb and hdb client call examples * Removed where clause of rdb query don't need a where clause for RDB data in the RDB and HDB client call * Edited RDB hloc function call * Removed .gw.synccallsallowed:0b and added hloc func * /home/nwatterson/gw_imprv/deploy/code/gateway * Updating gwtests.csv. Fixing formatting * Removed brackets and allowing async calls paragraph * Removing line from gwtests.csv * Update Processes.md * Updated client call examples description Co-authored-by: CatherineHigg <[email protected]>
1 parent d242d0f commit e1192f5

5 files changed

Lines changed: 214 additions & 102 deletions

File tree

‎code/gateway/gatewaylib.q‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
//functionality loaded in by gateway
2+
//functions include: getserverids, getserveridstype, getserverscross, buildcross
3+
4+
\d .gw
5+
6+
getserverids:{[att]
7+
if[99h<>type att;
8+
// its a list of servertypes e.g. `rdb`hdb
9+
// check if user attributes are a symbol list
10+
if[not 11h=abs type att;
11+
'" Servertype should be given as either a dictionary(type 99h) or a symbol list (11h)"
12+
];
13+
servertype:distinct att,();
14+
//list of active servers
15+
activeservers:exec distinct servertype from .gw.servers where active;
16+
//list of all servers
17+
allservers:exec distinct servertype from .gw.servers;
18+
activeserversmsg:". Available servers include: ",", " sv string activeservers;
19+
//check if a null argument is passed
20+
if[any null att;'"A null server cannot be passed as an argument",activeserversmsg];
21+
//if any requested servers are missing then:
22+
//if requested server does not exist, return error with list of available servers
23+
//if requested server exists but is currently inactive, return error with list of available servers
24+
if[count servertype except activeservers;
25+
'"the following ",$[max not servertype in allservers;
26+
"are not valid servers: ",", " sv string servertype except allservers;
27+
"requested servers are currently inactive: ",", " sv string servertype except activeservers
28+
],activeserversmsg;
29+
];
30+
:(exec serverid by servertype from .gw.servers where active)[servertype];
31+
];
32+
33+
// its a dictionary of attributes
34+
// check if attribute types are correct types
35+
correctatttypes:`date`servertype`tables!14 11 11h;
36+
w:correctatttypes[key att]=abs type each att;
37+
if[not all w key att;
38+
'"Wrong function parameter types provided.", " ",(" "sv string where not w)," parameter(s) need type(s) ",.Q.s1 correctatttypes where not w
39+
];
40+
41+
serverids:$[`servertype in key att;
42+
raze getserveridstype[delete servertype from att] each (),att`servertype;
43+
getserveridstype[att;`all]];
44+
45+
if[all 0=count each serverids;'"no servers match requested attributes"];
46+
:serverids;
47+
}
48+
49+
getserveridstype:{[att;typ]
50+
// default values
51+
besteffort:1b;
52+
attype:`cross;
53+
54+
servers:$[typ=`all;
55+
exec serverid!attributes from .gw.servers where active;
56+
exec serverid!attributes from .gw.servers where active,servertype=typ];
57+
58+
if[`besteffort in key att;
59+
if[-1h=type att`besteffort;besteffort:att`besteffort];
60+
att:delete besteffort from att;
61+
];
62+
if[`attributetype in key att;
63+
if[-11h=type att`attributetype;attype:att`attributetype];
64+
att:delete attributetype from att;
65+
];
66+
67+
res:$[attype=`independent;getserversindependent[att;servers;besteffort];
68+
getserverscross[att;servers;besteffort]];
69+
70+
serverids:first value flip $[99h=type res; key res; res];
71+
if[all 0=count each serverids;'"no servers match ",string[typ]," requested attributes"];
72+
:serverids;
73+
}
74+
75+
/- build a cross product from a nested dictionary
76+
buildcross:{(cross/){flip (enlist y)#x}[x] each key x}
77+
78+
/- given a dictionary of requirements and a list of attribute dictionaries
79+
/- work out which servers we need to hit to satisfy each requirement
80+
/- we want to satisfy the cross product of requirements - so each attribute has to be available with each other attribute
81+
/- e.g. each symbol has to be availble within each specified date
82+
getserverscross:{[req;att;besteffort]
83+
84+
if[0=count req; :([]serverid:enlist key att)];
85+
86+
s:getserversinitial[req;att];
87+
88+
/- build the cross product of requirements
89+
reqcross:buildcross[req];
90+
91+
/- calculate the cross product of data contributed by each source
92+
/- and drop it from the list of stuff that is required
93+
util:flip `remaining`found!flip ({[x;y;z] (y[0] except found; y[0] inter found:$[0=count y[0];y[0];buildcross x@'where each z])}[req]\)[(reqcross;());value s];
94+
95+
/- check if everything is done
96+
if[(count last util`remaining) and not besteffort;
97+
'"getserverscross: cannot satisfy query as the cross product of all attributes can't be matched"];
98+
/- remove any rows which don't add value
99+
s:1!(0!s) w:where not 0=count each util`found;
100+
/- return the parameters which should be queried for
101+
(key s)!distinct each' flip each util[w]`found
102+
}

‎code/processes/gateway.q‎

Lines changed: 8 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -358,83 +358,6 @@ getserversindependent:{[req;att;besteffort]
358358
/- if you want overlaps, remove the &filter w from the end of this bit of code
359359
(key s)!{(key x)!(value x)@'where each y key x}[req]each value s&filter w}
360360

361-
/- build a cross product from a nested dictionary
362-
buildcross:{(cross/){flip (enlist y)#x}[x] each key x}
363-
364-
/- given a dictionary of requirements and a list of attribute dictionaries
365-
/- work out which servers we need to hit to satisfy each requirement
366-
/- we want to satisfy the cross product of requirements - so each attribute has to be available with each other attribute
367-
/- e.g. each symbol has to be availble within each specified date
368-
getserverscross:{[req;att;besteffort]
369-
370-
if[0=count req; :([]serverid:enlist key att)];
371-
372-
s:getserversinitial[req;att];
373-
374-
/- build the cross product of requirements
375-
reqcross:buildcross[req];
376-
377-
/- calculate the cross product of data contributed by each source
378-
/- and drop it from the list of stuff that is required
379-
util:flip `remaining`found!flip ({[x;y;z] (y[0] except found; y[0] inter found:$[0=count y[0];y[0];buildcross x@'where each z])}[req]\)[(reqcross;());value s];
380-
381-
/- check if everything is done
382-
if[(count last util`remaining) and not besteffort;
383-
'"getserverscross: cannot satisfy query as the cross product of all attributes can't be matched"];
384-
385-
/- remove any rows which don't add value
386-
s:1!(0!s) w:where not 0=count each util`found;
387-
388-
/- return the parameters which should be queried for
389-
(key s)!distinct each' flip each util[w]`found
390-
}
391-
392-
getserverids:{[att]
393-
if[99h<>type att;
394-
// its a list of servertypes e.g. `rdb`hdb
395-
servertype:att,();
396-
missing:servertype except exec distinct servertype from .gw.servers where active;
397-
if[count missing;'"not all of the requested server types are available; missing "," " sv string missing];
398-
:(exec serverid by servertype from .gw.servers where active)[servertype];
399-
];
400-
401-
// its a dictionary of attributes
402-
403-
serverids:$[`servertype in key att;
404-
raze getserveridstype[delete servertype from att] each (),att`servertype;
405-
getserveridstype[att;`all]];
406-
407-
if[all 0=count each serverids;'"no servers match requested attributes"];
408-
:serverids;
409-
}
410-
411-
getserveridstype:{[att;typ]
412-
// default values
413-
besteffort:1b;
414-
attype:`cross;
415-
416-
servers:$[typ=`all;
417-
exec serverid!attributes from .gw.servers where active;
418-
exec serverid!attributes from .gw.servers where active,servertype=typ];
419-
420-
if[`besteffort in key att;
421-
if[-1h=type att`besteffort;besteffort:att`besteffort];
422-
att:delete besteffort from att;
423-
];
424-
if[`attributetype in key att;
425-
if[-11h=type att`attributetype;attype:att`attributetype];
426-
att:delete attributetype from att;
427-
];
428-
429-
res:$[attype=`independent;
430-
getserversindependent[att;servers;besteffort];
431-
getserverscross[att;servers;besteffort]];
432-
433-
serverids:first value flip $[99h=type res; key res; res];
434-
if[all 0=count each serverids;'"no servers match ",string[typ]," requested attributes"];
435-
:serverids;
436-
}
437-
438361
// execute an asynchronous query
439362
asyncexecjpts:{[query;servertype;joinfunction;postback;timeout;sync]
440363
// Check correct function called
@@ -453,28 +376,28 @@ asyncexecjpts:{[query;servertype;joinfunction;postback;timeout;sync]
453376
query:({[u;q]$[`.pm.execas ~ key `.pm.execas;value (`.pm.execas; q; u);`.pm.valp ~ key `.pm.valp; .pm.valp q; value q]}; .z.u; query);
454377
/- if sync calls are allowed disable async calls to avoid query conflicts
455378
$[.gw.synccallsallowed and .z.K<3.6;
456-
errStr:.gw.errorprefix,"only synchronous calls are allowed";
457-
[errStr:"";
379+
errstr:.gw.errorprefix,"only synchronous calls are allowed";
380+
[errstr:"";
458381
if[99h<>type servertype;
459382
// its a list of servertypes e.g. `rdb`hdb
460383
servertype:distinct servertype,();
461-
missing:servertype except exec distinct servertype from .gw.servers where active;
462-
if[count missing; errStr:.gw.errorprefix,"not all of the requested server types are available; missing "," " sv string missing];
384+
errcheck:@[getserverids;servertype;{.gw.errorprefix,x}];
385+
if[10h=type errcheck; errstr:errcheck];
463386
queryattributes:()!();
464387
];
465388
if[99h=type servertype;
466389
// its a dictionary of attributes
467390
queryattributes:servertype;
468391
res:@[getserverids;queryattributes;{.gw.errorprefix,"getserverids: failed with error - ",x}];
469-
if[10h=type res; errStr:res];
470-
if[10h<>type res; if[0=count raze res; errStr:.gw.errorprefix,"no servers match given attributes"]];
392+
if[10h=type res; errstr:res];
393+
if[10h<>type res; if[0=count raze res; errstr:.gw.errorprefix,"no servers match given attributes"]];
471394
servertype:res;
472395
];
473396
]
474397
];
475398
// error has been hit
476-
if[count errStr;
477-
@[neg .z.w;.gw.formatresponse[0b;sync;$[()~postback;errStr;$[-11h=type postback;enlist postback;postback],enlist[query],enlist errStr]];()];
399+
if[count errstr;
400+
@[neg .z.w;.gw.formatresponse[0b;sync;$[()~postback;errstr;$[-11h=type postback;enlist postback;postback],enlist[query],enlist errstr]];()];
478401
:()];
479402

480403
addquerytimeout[query;servertype;queryattributes;joinfunction;postback;timeout;sync];

‎config/settings/gateway.q‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ querykeeptime:0D00:30 // the time to keep queries in the
99
errorprefix:"error: " // the prefix for clients to look for in error strings
1010
clearinactivetime:0D01:00 // the time to keep inactive handle data
1111

12+
\d .proc
13+
loadprocesscode:1b // whether to load the process specific code defined at ${KDBCODE}/{process type}
14+
1215
// Server connection details
1316
\d .servers
1417
CONNECTIONS:`rdb`hdb // list of connections to make at start up

0 commit comments

Comments
 (0)