Skip to content

Commit dedbcb5

Browse files
robertcoendan-shmatthewclark1103
authored
TP schema path removed and sublist check added (#378)
* TP schema path removed and sublist check added * tickerplant corretion + moved file * DA defined utils file and minor error message fix * Removed dependancy on tp * fixed typos and allow for easier integration with bq piece * added forgotten file related to previous commit * fixed typo * added attributes routing for overlapping processes * removed any tp connection * Update dataaccess.q typo Co-authored-by: dan-sh <[email protected]> Co-authored-by: Matthew Clark <[email protected]> Co-authored-by: dan-sh <[email protected]>
1 parent 7d66bbd commit dedbcb5

9 files changed

Lines changed: 50 additions & 19 deletions

File tree

‎README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ Then:
3939
Head to the address it gives you to check if your changes have worked. More information about using mkdocs can be found [here](http://www.mkdocs.org/)
4040

4141
## Release Notes
42+
- **4.1.1, Jun 2021**
43+
* Data Access API bug fixes
44+
* Improved attribute routing surrounding overlapping processes
4245
- **4.1.0, Apr 2021**
4346
* Data Access API Update [Blog post](https://www.aquaq.co.uk/torq-2/data-access-api/) and [Documentation](docs/dataaccess.md)
4447
- **4.0.1, Mar 2021**

‎code/common/checkinputs.q‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,13 @@ checktimeorder:{[dict]
7777
:dict;};
7878

7979
// check parameter is of type symbol
80-
checkinputsym:{[dict;parameter]
80+
checksyminput:{[dict;parameter]
8181
:checktype[-11 11h;dict;parameter];};
8282

83+
// check parameter is of type
84+
checksublist:{[dict;parameter]
85+
:checktype[-5 -6 -7h;dict;parameter];};
86+
8387
// check aggregations are of type dictionary, that the dictionary has symbol keys, that
8488
// the dictionary has symbol values
8589
checkaggregations:{[dict;parameter]

‎code/common/dataaccess.q‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,4 @@ if[`dataaccess in key .proc.params;
5151
.dataaccess.settablepropertiespath[];
5252
// re-initialize on new connections
5353
if[.dataaccess.validtablepropertiespath[];.servers.connectcustom:.dataaccess.connectcustom];
54-
];
55-
56-
57-
\d .schema
58-
schema:.proc.loadf[getenv[`TORQAPPHOME],"/database.q"];
59-
\d .
54+
];

‎code/dataaccess/checkinputs.q‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,7 @@ checktimecolumn:{[dict]
4444
// fill in
4545
filldefaulttimecolumn:{[dict]
4646
if[not `timecolumn in key dict;
47-
:@[dict;`timecolumn;:;.checkinputs.getdefaulttime dict];
48-
defaulttimecolumn:`time^.checkinputs.gettableproperty[dict;`primarytimecolumn];
49-
:@[dict;`timecolumn;:;defaulttimecolumn]];
47+
:@[dict;`timecolumn;:;.checkinputs.getdefaulttime dict]];
5048
:dict;
5149
};
5250

code/common/utils.q renamed to code/dataaccess/dataaccessutils.q

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,16 @@
22

33
//- utils for reading in config
44
readtableproperties:{[tablepropertiepath]
5-
.lg.o[`readtableproperties;"loading table properties"]
5+
.lg.o[`readtableproperties;"loading table properties"];
66
table:`tablename`proctype xkey readcsv[tablepropertiepath;"ssssstsss"]; //read in table from file
77
alltable:?[table;enlist(in;`proctype;enlist`all`);0b;()]; //find any instance of the use "all" or blank for proctype
88
table:table,![alltable;();0b;(enlist`proctype)!enlist(enlist `hdb)],![alltable;();0b;(enlist`proctype)!enlist(enlist `rdb)]; //join rdb and hdb entries for any "all" or blank entries
99
table:![table;enlist(in;`proctype;enlist`all`);0b;`symbol$()]; //remove "all" or blank entries from table
1010
table:?[table;$[.proc.proctype=`gateway;();enlist(=;`proctype;`.proc.proctype)];0b;()];
1111
table:update .eodtime.datatimezone ^ datatimezone, .eodtime.rolltimeoffset ^ rolltimeoffset,.eodtime.rolltimezone^rolltimezone from table;
12-
:update `date ^ partitionfield from table where proctype<>`rdb;
12+
table:update `date ^ partitionfield from table where proctype<>`rdb;
1313
.lg.o[`readtableproperties;"Table properties successfully loaded"];
14+
:table;
1415
};
1516

1617
readcheckinputs:{[checkinputspath] spliltcolumns[readcsv[checkinputspath;"sbs*"];`invalidpairs;`]};
@@ -67,13 +68,13 @@ gettableproperty:extractfromsubdict[;`tableproperties;]; //- extract from `tab
6768

6869
//- get default time from tickerplant or table
6970
getdefaulttime:{[dict]
70-
if[(dict`tablename) in key .schema;
71-
:first ?[meta` sv``schema,dict`tablename;enlist(=;`t;"p");();`c]];
71+
// go to the tableproperties table
72+
if[not ` ~ configure:.checkinputs.tablepropertiesconfig[(dict`tablename),.proc.proctype;`primarytimecolumn];:configure];
7273
timestamp:(exec from meta (dict`tablename) where t in "p")`c;
7374
if[1 < count timestamp; '`$.checkinputs.formatstring["Table has multiple time columns, please select one of the following {} for the parameter timecolumn";timestamp]];
7475
date:(exec from meta (dict`tablename) where t in "d")`c;
7576
if[1 < count date; '`$.checkinputs.formatstring["Table has multiple date columns, please select one of the following {} for the parameter timecolumn";date]];
76-
if[not timestamp = `;:timestamp];
77+
if[not timestamp = `;.checkinputs.tablepropertiesconfig[(dict`tablename),.proc.proctype;`primarytimecolumn]::timestamp;:timestamp];
7778
if[not date = `;:date];
7879
'`$.checkinputs.formatstring["Table:{tablename} does not have a default timecolumn, one must be selected using the time column parameter";dict]
7980
};

‎code/dataaccess/getdata.q‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33

44
getdata:{[inputparams]
5+
if[.proc.proctype in key inputparams;inputparams:inputparams .proc.proctype];
56
requestnumber:.requests.initlogger[inputparams];
67
// [input parameters dict] generic function acting as main access point for data retrieval
78
if[1b~inputparams`getquery;:.dataaccess.buildquery[inputparams]];
@@ -50,6 +51,7 @@ getdata:{[inputparams]
5051
\d .dataaccess
5152

5253
buildquery:{[inputparams]
54+
if[.proc.proctype in key inputparams;inputparams:inputparams .proc.proctype];
5355
inputparams:.dataaccess.checkinputs inputparams;
5456
queryparams:.eqp.extractqueryparams[inputparams;.eqp.queryparams];
5557
if[`procs in key inputparams;:(.proc.proctype,.queryorder.orderquery queryparams)];

‎code/dataaccess/schema.q‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
\d .schema
22

3-
// load in tickerplant schema to determine default time column
4-
schema:.proc.loadf[getenv[`TORQAPPHOME],"/database.q"];
3+
// define the tickerplant to cross check for default timecolumn
4+
tickerplant:`stp1
55
// define allowed operators and operators which can be used with a not statement for filter and freeformwhere parameter
66
allowedops:(<;>;<>;in;within;like;<=;>=;=;~;not);
77
allowednot:(within;like;in);

‎code/gateway/dataaccess.q‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ getdata:{[o]
1717
// Use upserting logic to determine behaviour
1818
options:default,o;
1919
if[`ordering in key o;options[`ordering]: go each options`ordering];
20+
o:adjustqueries[o;partdict o];
2021
// Execute the queries
2122
if[options`getquery;
2223
$[.gw.call .z.w;
@@ -72,7 +73,34 @@ partdict:{[input]
7273
// If the response is a dictionary index into the tablename
7374
procdict:@[procdict;key procdict;{[x;tabname]if[99h=type x;:x[tabname]];:x}[;tabname]];
7475
// returns the dictionary as min date/ max date
75-
:asc @[procdict;key procdict;{:(min x; max x)}]
76+
procdict:asc @[procdict;key procdict;{:(min x; max x)}];
77+
// prevents overlap if more than one process contains a specified date
78+
if[1<count procdict;
79+
procdict:{:$[y~`date$();x;$[within[x 0;(min y;max y)];(1+max[y];x 1);x]]}':[procdict]];
80+
:procdict;
81+
};
82+
83+
// function to adjust the queries being sent to processes to prevent overlap of
84+
// time clause and data being queried on more than one process
85+
adjustqueries:{[options;part]
86+
// if only one process then no need to adjust
87+
if[2>count p:options`procs;:options];
88+
// get the date casting where relevant
89+
st:$[a:-14h~tp:type start:options`starttime;start;`date$start];
90+
et:$[a;options`endtime;`date$options`endtime];
91+
// get the dates that are required by each process
92+
dates:group key[part]where each{within[y;]each value x}[part]'[l:st+til 1+et-st];
93+
dates:l{(min x;max x)}'[dates];
94+
// if start/end time not a date, then adjust dates parameter for the
95+
// correct types
96+
if[not a;
97+
// converts dates dictionary to timestamps/datetimes
98+
dates:$[-15h~tp;{"z"$x};::]{(0D+x 0;x[1]+1D-1)}'[dates];
99+
// convert first and last timestamp to start and end time
100+
dates:@[dates;f;:;(start;dates[f:first key dates;1])];
101+
dates:@[dates;l;:;(dates[l:last key dates;0];options`endtime)]];
102+
// create a dictionary of procs and different queries
103+
:{@[@[x;`starttime;:;y 0];`endtime;:;y 1]}[options]'[dates];
76104
};
77105

78106
// Default dataaccess join allowing for aggregations across processes

‎config/dataaccess/errormessages.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ checkorderingarrangment,"Ordering parameter's values must be paired in the form
2525
checkorderingdirection,"The first item in each of the ordering parameter's pairs must be either `asc or `desc - example: "
2626
orderingvague,"Ordering parameter vague. Ordering by a column that aggregated more than once, as such the aggregation must be specified. The aggregation convention is camel-case, so to order by the aggregation max price, the following would be used: "
2727
orderingnocolumn,"Ordering parameter contains column that is not defined by aggregations, grouping or timebar parameter"
28-
badorder,"Trying to order by column(s): {badorder} that is not defined in the columns argument"
28+
badorder,"Trying to order by column(s): {} that is not defined in the columns argument"
2929
renamekey,"Renamecolumn parameter key must be of type 11h - example:"
3030
renameinput,"Renamecolumn parameter input must be of type 11h - example:"
3131
postback,"Postback argument must be a function that takes in one argument only - the argument can be named anything through function signature but must represent the returned results of all other inputs"

0 commit comments

Comments
 (0)