-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcopy.yaml
More file actions
56 lines (49 loc) · 1.2 KB
/
copy.yaml
File metadata and controls
56 lines (49 loc) · 1.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
init:
i: 0
gcpCredentials: gcp-e2e
gcpSecrets: ${secrets.$gcpCredentials}
src:
projectID: $gcpSecrets.ProjectID
datasetID: db1
dest:
projectID: $gcpSecrets.ProjectID
datasetID: db1e2e
pipeline:
registerSource:
action: dsunit:register
datastore: ${src.datasetID}
config:
driverName: bigquery
credentials: $gcpCredentials
parameters:
datasetId: $src.datasetID
readTables:
action: dsunit:query
datastore: ${src.datasetID}
SQL: SELECT table_id AS table FROM `${src.projectID}.${src.datasetID}.__TABLES__`
post:
dataset: $Records
copyTables:
loop:
action: print
message: $i/$Len($dataset) -> $dataset[$i].table
copyTable:
action: gcp/bigquery:copy
logging: false
credentials: $gcpCredentials
sourceTable:
projectID: ${src.projectID}
datasetID: ${src.datasetID}
tableID: $dataset[$i].table
destinationTable:
projectID: ${dest.projectID}
datasetID: ${dest.datasetID}
tableID: $dataset[$i].table
inc:
action: nop
init:
_ : $i++
goto:
when: $i < $Len($dataset)
action: goto
task: copyTables