-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathLed.Tests.fs
More file actions
196 lines (170 loc) · 5.14 KB
/
Led.Tests.fs
File metadata and controls
196 lines (170 loc) · 5.14 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
module OpenDiffix.Core.LedTests
open Xunit
open FsUnit.Xunit
open CommonTypes
let assertRowsDifference rows1 rows2 (diff: List<Row * Row>) =
List.zip rows1 rows2
|> List.filter (fun (left, right) -> Row.equalityComparer.Equals(left, right) |> not)
|> should equal diff
let assertHookDifference csv query (diff: List<Row * Row>) =
let rows (result: QueryEngine.QueryResult) = result.Rows
let withoutHookRows = HookTestHelpers.run [] csv query |> rows
let withHookRows = HookTestHelpers.run [ Led.hook ] csv query |> rows
assertRowsDifference withoutHookRows withHookRows diff
let assertHookFails csv query (errorFragment: string) =
try
HookTestHelpers.run [ Led.hook ] csv query |> ignore
failwith "Expecting query to fail"
with ex ->
let str = ex.Message.ToLower()
if str.Contains(errorFragment.ToLower()) then
()
else
failwith $"Expecting error to contain '%s{errorFragment}'. Got '%s{str}' instead."
let csvWithVictim =
"""
dept,gender,title
math,m,prof
math,m,prof
math,m,prof
math,m,prof
math,f,prof
math,f,prof
math,f,prof
math,f,prof
history,m,prof
history,m,prof
history,m,prof
history,m,prof
history,f,prof
history,f,prof
history,f,prof
history,f,prof
cs,m,prof
cs,m,prof
cs,m,prof
cs,m,prof
cs,f,prof
"""
let csvWithTwoVictims = csvWithVictim + "\n" + "cs,f,prof"
let csvWithThreeCsWomen = csvWithTwoVictims + "\n" + "cs,f,prof"
let csvWithDifferentTitles = csvWithVictim + "\n" + "cs,f,asst"
let csvWithProperStarBucket =
csvWithVictim
+ """
biol,f,asst
chem,m,asst
biol,f,prof
"""
let query =
"""
SELECT dept, gender, title, diffix_count(*, RowIndex), diffix_low_count(RowIndex)
FROM table
GROUP BY 1, 2, 3
"""
[<Fact>]
let ``Merges victim bucket`` () =
assertHookDifference
csvWithVictim
query
[
[| String "cs"; String "m"; String "prof"; Integer 4L; Boolean false |],
[| String "cs"; String "m"; String "prof"; Integer 5L; Boolean false |]
]
[<Fact>]
let ``Merges 2 victims with same title`` () =
assertHookDifference
csvWithTwoVictims
query
[
[| String "cs"; String "m"; String "prof"; Integer 4L; Boolean false |],
[| String "cs"; String "m"; String "prof"; Integer 6L; Boolean false |]
]
[<Fact>]
let ``Does not merge high count buckets`` () =
assertHookDifference csvWithThreeCsWomen query []
[<Fact>]
let ``Does not merge if the victim cannot be singled out`` () =
assertHookDifference csvWithDifferentTitles query []
[<Fact>]
let ``Merges taking precedence before star bucket hook`` () =
// We get a hold of the star bucket results reference via side effects.
let mutable suppressedAnonCount = Null
let pullHookResultsCallback aggCtx starBucket buckets =
suppressedAnonCount <- Bucket.finalizeAggregate 0 aggCtx starBucket
buckets
let starBucketHook = StarBucket.hook pullHookResultsCallback
let rows (result: QueryEngine.QueryResult) = result.Rows
let withoutHookRows = HookTestHelpers.run [] csvWithProperStarBucket query |> rows
let withHookRows =
HookTestHelpers.run [ Led.hook; starBucketHook ] csvWithProperStarBucket query
|> rows
assertRowsDifference
withoutHookRows
withHookRows
[
[| String "cs"; String "m"; String "prof"; Integer 4L; Boolean false |],
[| String "cs"; String "m"; String "prof"; Integer 5L; Boolean false |]
]
// 3 is correct, this is the number of suppressed rows, _we exclude_ the merged row
// from the star bucket.
suppressedAnonCount |> should equal (Integer 3L)
[<Fact>]
let ``Works with count distinct`` () =
assertHookDifference
csvWithVictim
"""
SELECT dept, gender, title, diffix_count(distinct RowIndex, RowIndex), diffix_low_count(RowIndex)
FROM table
GROUP BY 1, 2, 3
"""
[
[| String "cs"; String "m"; String "prof"; Integer 4L; Boolean false |],
[| String "cs"; String "m"; String "prof"; Integer 5L; Boolean false |]
]
[<Fact>]
let ``Works when low count filter is in HAVING`` () =
assertHookDifference
csvWithVictim
"""
SELECT dept, gender, title, diffix_count(*, RowIndex)
FROM table
GROUP BY 1, 2, 3
HAVING NOT diffix_low_count(RowIndex)
"""
[
[| String "cs"; String "m"; String "prof"; Integer 4L |], //
[| String "cs"; String "m"; String "prof"; Integer 5L |]
]
[<Fact>]
let ``Requires low count filter aggregator in scope`` () =
assertHookFails
csvWithVictim
"""
SELECT dept, gender, title, diffix_count(*, RowIndex)
FROM table
GROUP BY 1, 2, 3
"""
"cannot find required DiffixLowCount aggregator"
[<Fact>]
let ``Ignores real count`` () =
assertHookDifference
csvWithVictim
"""
SELECT dept, gender, title, count(*), diffix_count(*, RowIndex), diffix_low_count(RowIndex)
FROM table
GROUP BY 1, 2, 3
"""
[
[| String "cs"; String "m"; String "prof"; Integer 4L; Integer 4L; Boolean false |],
[| String "cs"; String "m"; String "prof"; Integer 4L; Integer 5L; Boolean false |]
]
[<Fact>]
let ``Ignores global aggregation`` () =
assertHookDifference
csvWithVictim
"""
SELECT diffix_count(*, RowIndex)
FROM table
"""
[]