forked from sourcegraph/sourcegraph-public-snapshot
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcleanup_test.go
More file actions
321 lines (283 loc) · 11.3 KB
/
cleanup_test.go
File metadata and controls
321 lines (283 loc) · 11.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
package codegraph
import (
"context"
"fmt"
"sort"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/keegancsmith/sqlf"
"github.com/sourcegraph/log"
"github.com/sourcegraph/log/logtest"
codeintelshared "github.com/sourcegraph/sourcegraph/internal/codeintel/shared"
"github.com/sourcegraph/sourcegraph/internal/database/basestore"
"github.com/sourcegraph/sourcegraph/internal/database/dbtest"
"github.com/sourcegraph/sourcegraph/internal/observation"
)
func TestDeleteLsifDataByUploadIds(t *testing.T) {
logger := logtest.ScopedWith(t, logtest.LoggerOptions{
Level: log.LevelError,
})
codeIntelDB := codeintelshared.NewCodeIntelDB(logger, dbtest.NewDB(t))
store := New(observation.TestContextTB(t), codeIntelDB)
t.Run("scip", func(t *testing.T) {
for i := range 5 {
query := sqlf.Sprintf("INSERT INTO codeintel_scip_metadata (upload_id, text_document_encoding, tooL_name, tool_version, tool_arguments, protocol_version) VALUES (%s, 'utf8', '', '', '{}', 1)", i+1)
if _, err := codeIntelDB.ExecContext(context.Background(), query.Query(sqlf.PostgresBindVar), query.Args()...); err != nil {
t.Fatalf("unexpected error inserting repo: %s", err)
}
}
if err := store.DeleteLsifDataByUploadIds(context.Background(), 2, 4); err != nil {
t.Fatalf("unexpected error clearing bundle data: %s", err)
}
uploadIDs, err := basestore.ScanInts(codeIntelDB.QueryContext(context.Background(), "SELECT upload_id FROM codeintel_scip_metadata"))
if err != nil {
t.Fatalf("Unexpected error querying dump identifiers: %s", err)
}
if diff := cmp.Diff([]int{1, 3, 5}, uploadIDs); diff != "" {
t.Errorf("unexpected dump identifiers (-want +got):\n%s", diff)
}
})
}
func TestDeleteAbandonedSchemaVersionsRecords(t *testing.T) {
logger := logtest.ScopedWith(t, logtest.LoggerOptions{
Level: log.LevelError,
})
codeIntelDB := codeintelshared.NewCodeIntelDB(logger, dbtest.NewDB(t))
store := New(observation.TestContextTB(t), codeIntelDB)
ctx := context.Background()
assertCounts := func(expectedNumSymbols, expectedNumDocuments int) {
numSymbols, _, err := basestore.ScanFirstInt(codeIntelDB.QueryContext(ctx, "SELECT COUNT(*) FROM codeintel_scip_symbols_schema_versions"))
if err != nil {
t.Fatalf("unexpected error fetching count: %s", err)
}
if numSymbols != expectedNumSymbols {
t.Errorf("unexpected number of symbols schema version records. want=%d have=%d", expectedNumSymbols, numSymbols)
}
numDocuments, _, err := basestore.ScanFirstInt(codeIntelDB.QueryContext(ctx, "SELECT COUNT(*) FROM codeintel_scip_document_lookup_schema_versions"))
if err != nil {
t.Fatalf("unexpected error fetching count: %s", err)
}
if numDocuments != expectedNumDocuments {
t.Errorf("unexpected number of documents schema version records. want=%d have=%d", expectedNumDocuments, numDocuments)
}
}
// Insert records backed by a live source
if _, err := codeIntelDB.ExecContext(ctx, `
INSERT INTO codeintel_scip_metadata (upload_id, tool_name, tool_version, tool_arguments, text_document_encoding, protocol_version)
VALUES
(100, '', '', '{}', '', 1),
(102, '', '', '{}', '', 1),
(104, '', '', '{}', '', 1),
(200, '', '', '{}', '', 1),
(202, '', '', '{}', '', 1),
(204, '', '', '{}', '', 1),
(206, '', '', '{}', '', 1);
INSERT INTO codeintel_scip_symbols_schema_versions (upload_id, min_schema_version, max_schema_version) VALUES
(100, 1, 1), -- live
(101, 1, 1), -- abandoned
(102, 1, 1), -- live
(103, 1, 1), -- abandoned
(104, 1, 1); -- live
INSERT INTO codeintel_scip_document_lookup_schema_versions (upload_id, min_schema_version, max_schema_version) VALUES
(200, 1, 1), -- live
(201, 1, 1), -- abandoned
(202, 1, 1), -- live
(203, 1, 1), -- abandoned
(204, 1, 1), -- live
(205, 1, 1), -- abandoned
(206, 1, 1); -- live
`); err != nil {
t.Fatalf("failed to prepare data: %s", err)
}
// Assert test count
assertCounts(5, 7)
// Prune all abandoned records
count, err := store.DeleteAbandonedSchemaVersionsRecords(ctx)
if err != nil {
t.Fatalf("unexpected error deleting abandoned schema version records: %s", err)
}
if expected := 5; count != expected {
t.Errorf("Unexpected count. want=%d have=%d", expected, count)
}
// Assert count of records backed by a metadata record
assertCounts(3, 4)
}
func TestDeleteUnreferencedDocuments(t *testing.T) {
logger := logtest.Scoped(t)
codeIntelDB := codeintelshared.NewCodeIntelDB(logger, dbtest.NewDB(t))
internalStore := basestore.NewWithHandle(codeIntelDB.Handle())
store := New(observation.TestContextTB(t), codeIntelDB)
for i := range 200 {
insertDocumentQuery := sqlf.Sprintf(
`INSERT INTO codeintel_scip_documents(id, schema_version, payload_hash, raw_scip_payload) VALUES (%s, 1, %s, %s)`,
i+1,
fmt.Sprintf("hash-%d", i+1),
fmt.Sprintf("payload-%d", i+1),
)
if err := internalStore.Exec(context.Background(), insertDocumentQuery); err != nil {
t.Fatalf("unexpected error setting up database: %s", err)
}
}
for i := range 200 {
insertDocumentLookupQuery := sqlf.Sprintf(
`INSERT INTO codeintel_scip_document_lookup(upload_id, document_path, document_id) VALUES (%s, %s, %s)`,
42,
fmt.Sprintf("path-%d", i+1),
i+1,
)
if err := internalStore.Exec(context.Background(), insertDocumentLookupQuery); err != nil {
t.Fatalf("unexpected error setting up database: %s", err)
}
if i%3 == 0 {
insertDocumentLookupQuery := sqlf.Sprintf(
`INSERT INTO codeintel_scip_document_lookup(upload_id, document_path, document_id) VALUES (%s, %s, %s)`,
43,
fmt.Sprintf("path-%d", i+1),
i+1,
)
if err := internalStore.Exec(context.Background(), insertDocumentLookupQuery); err != nil {
t.Fatalf("unexpected error setting up database: %s", err)
}
}
}
deleteReferencesQuery := sqlf.Sprintf(`DELETE FROM codeintel_scip_document_lookup WHERE upload_id = 42`)
if err := internalStore.Exec(context.Background(), deleteReferencesQuery); err != nil {
t.Fatalf("unexpected error setting up database: %s", err)
}
// Check too soon
_, count, err := store.DeleteUnreferencedDocuments(context.Background(), 20, time.Minute, time.Now())
if err != nil {
t.Fatalf("unexpected error deleting unreferenced documents: %s", err)
}
if count != 0 {
t.Fatalf("did not expect any expired records, have %d", count)
}
// Consume actual records. We expect 10 batches (200 records deleted / 20 per batch) to be required to
// process this workload.
totalCount := 0
for range 10 {
_, count, err = store.DeleteUnreferencedDocuments(context.Background(), 20, time.Minute, time.Now().Add(time.Minute*5))
if err != nil {
t.Fatalf("unexpected error deleting unreferenced documents: %s", err)
}
totalCount += count
}
if expected := 2 * 200 / 3; totalCount != expected {
t.Fatalf("unexpected number of unreferenced documents deleted. want=%d have=%d", expected, totalCount)
}
// Assert no more records should be available for processing
_, count, err = store.DeleteUnreferencedDocuments(context.Background(), 20, time.Minute, time.Now())
if err != nil {
t.Fatalf("unexpected error deleting unreferenced documents: %s", err)
}
if count != 0 {
t.Fatalf("did not expect any unprocessed records, have %d", count)
}
documentIDsQuery := sqlf.Sprintf(`SELECT id FROM codeintel_scip_documents ORDER BY id`)
ids, err := basestore.ScanInts(internalStore.Query(context.Background(), documentIDsQuery))
if err != nil {
t.Fatalf("unexpected error querying document ids: %s", err)
}
var expectedIDs []int
for i := range 200 {
if i%3 == 0 {
expectedIDs = append(expectedIDs, i+1)
}
}
if diff := cmp.Diff(expectedIDs, ids); diff != "" {
t.Fatalf("unexpected remaining document identifiers (-want +got):\n%s", diff)
}
}
func TestIDsWithMeta(t *testing.T) {
logger := logtest.Scoped(t)
codeIntelDB := codeintelshared.NewCodeIntelDB(logger, dbtest.NewDB(t))
store := New(observation.TestContextTB(t), codeIntelDB)
ctx := context.Background()
if _, err := codeIntelDB.ExecContext(ctx, `
INSERT INTO codeintel_scip_metadata (upload_id, text_document_encoding, tool_name, tool_version, tool_arguments, protocol_version) VALUES (200, 'utf8', '', '', '{}', 1);
INSERT INTO codeintel_scip_metadata (upload_id, text_document_encoding, tool_name, tool_version, tool_arguments, protocol_version) VALUES (202, 'utf8', '', '', '{}', 1);
INSERT INTO codeintel_scip_metadata (upload_id, text_document_encoding, tool_name, tool_version, tool_arguments, protocol_version) VALUES (204, 'utf8', '', '', '{}', 1);
`); err != nil {
t.Fatalf("unexpected error setting up test: %s", err)
}
candidates := []int{
200, // exists
201,
203,
204, // exists
205,
}
ids, err := store.IDsWithMeta(ctx, candidates)
if err != nil {
t.Fatalf("failed to find upload IDs with metadata: %s", err)
}
expectedIDs := []int{
200,
204,
}
sort.Ints(ids)
if diff := cmp.Diff(expectedIDs, ids); diff != "" {
t.Fatalf("unexpected IDs (-want +got):\n%s", diff)
}
}
func TestReconcileCandidates(t *testing.T) {
logger := logtest.Scoped(t)
codeIntelDB := codeintelshared.NewCodeIntelDB(logger, dbtest.NewDB(t))
store := New(observation.TestContextTB(t), codeIntelDB)
ctx := context.Background()
now := time.Unix(1587396557, 0).UTC()
if _, err := codeIntelDB.ExecContext(ctx, `
INSERT INTO codeintel_scip_metadata (upload_id, text_document_encoding, tool_name, tool_version, tool_arguments, protocol_version) VALUES (200, 'utf8', '', '', '{}', 1);
INSERT INTO codeintel_scip_metadata (upload_id, text_document_encoding, tool_name, tool_version, tool_arguments, protocol_version) VALUES (201, 'utf8', '', '', '{}', 1);
INSERT INTO codeintel_scip_metadata (upload_id, text_document_encoding, tool_name, tool_version, tool_arguments, protocol_version) VALUES (202, 'utf8', '', '', '{}', 1);
INSERT INTO codeintel_scip_metadata (upload_id, text_document_encoding, tool_name, tool_version, tool_arguments, protocol_version) VALUES (203, 'utf8', '', '', '{}', 1);
INSERT INTO codeintel_scip_metadata (upload_id, text_document_encoding, tool_name, tool_version, tool_arguments, protocol_version) VALUES (204, 'utf8', '', '', '{}', 1);
INSERT INTO codeintel_scip_metadata (upload_id, text_document_encoding, tool_name, tool_version, tool_arguments, protocol_version) VALUES (205, 'utf8', '', '', '{}', 1);
`); err != nil {
t.Fatalf("unexpected error setting up test: %s", err)
}
// Initial batch of records
ids, err := store.ReconcileCandidatesWithTime(ctx, 4, now)
if err != nil {
t.Fatalf("failed to get candidate IDs for reconciliation: %s", err)
}
expectedIDs := []int{
200,
201,
202,
203,
}
sort.Ints(ids)
if diff := cmp.Diff(expectedIDs, ids); diff != "" {
t.Fatalf("unexpected IDs (-want +got):\n%s", diff)
}
// Wraps around after exhausting first records
ids, err = store.ReconcileCandidatesWithTime(ctx, 4, now.Add(time.Minute*1))
if err != nil {
t.Fatalf("failed to get candidate IDs for reconciliation: %s", err)
}
expectedIDs = []int{
200,
201,
204,
205,
}
sort.Ints(ids)
if diff := cmp.Diff(expectedIDs, ids); diff != "" {
t.Fatalf("unexpected IDs (-want +got):\n%s", diff)
}
// Continues to wrap around
ids, err = store.ReconcileCandidatesWithTime(ctx, 2, now.Add(time.Minute*2))
if err != nil {
t.Fatalf("failed to get candidate IDs for reconciliation: %s", err)
}
expectedIDs = []int{
202,
203,
}
sort.Ints(ids)
if diff := cmp.Diff(expectedIDs, ids); diff != "" {
t.Fatalf("unexpected IDs (-want +got):\n%s", diff)
}
}