forked from elastic/ml-cpp
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathupload.gradle
More file actions
141 lines (120 loc) · 4.34 KB
/
upload.gradle
File metadata and controls
141 lines (120 loc) · 4.34 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
description = 'Uploads the Machine Learning native binaries to s3'
import org.elastic.gradle.UploadS3Task
import java.util.zip.ZipFile
String versionQualifier = System.getProperty("build.version_qualifier", "")
boolean isSnapshot = "true".equals(System.getProperty("build.snapshot", "true"))
allprojects {
group = 'org.elasticsearch.ml'
version = elasticsearchVersion
if (versionQualifier != "") {
version += '-' + versionQualifier
}
if (isSnapshot) {
version += '-SNAPSHOT'
}
}
String artifactGroupPath = project.group.replaceAll("\\.", "/")
String envMlAwsAccessKey = System.env.ML_AWS_ACCESS_KEY
if (envMlAwsAccessKey != null) {
project.ext.mlAwsAccessKey = envMlAwsAccessKey
} else if (project.hasProperty("ML_AWS_ACCESS_KEY")) {
project.ext.mlAwsAccessKey = ML_AWS_ACCESS_KEY
}
String envMlAwsSecretKey = System.env.ML_AWS_SECRET_KEY
if (envMlAwsSecretKey != null) {
project.ext.mlAwsSecretKey = envMlAwsSecretKey
} else if (project.hasProperty("ML_AWS_SECRET_KEY")) {
project.ext.mlAwsSecretKey = ML_AWS_SECRET_KEY
}
/**
* Gradle 6 gets confused if we try to use its standard dependency management to
* convert artifacts with a classifier to an artifact for the same project
* without a classifier. Therefore this class uses low level functionality to
* get the platform-specific artifacts.
*/
class DownloadPlatformSpecific extends DefaultTask {
@Input
String version = project.version
@Input
String artifactGroupPath = project.group.replaceAll("\\.", "/")
/**
* Base name for the artifacts
*/
@Input
String baseName
/**
* Directory to download platform specific zip files into
*/
@Input
String downloadDirectory
/**
* Directory to extract downloaded zip files into
*/
@OutputDirectory
File extractDirectory
@Input
List<String> platforms = [ 'darwin-x86_64', 'linux-aarch64', 'linux-x86_64', 'windows-x86_64' ]
DownloadPlatformSpecific() {
// Always run this task, in case the platform-specific zips have changed
outputs.upToDateWhen {
return false
}
}
@TaskAction
void combine() {
extractDirectory.deleteDir()
platforms.each {
File zipFile = new File(downloadDirectory, "${baseName}-${version}-${it}.zip")
zipFile.parentFile.mkdirs()
new URL("https://prelert-artifacts.s3.amazonaws.com/maven/${artifactGroupPath}/${baseName}/${version}/${zipFile.name}").withInputStream { i ->
zipFile.withOutputStream { o ->
o << i
}
}
ZipFile zip = new ZipFile(zipFile)
zip.entries().each {
File target = new File(extractDirectory, it.name)
// There can be overlaps between the platform-specific zips, so skip duplicates
if (target.exists() == false) {
if (it.isDirectory()) {
target.mkdirs()
} else {
target.parentFile.mkdirs()
target.withOutputStream { o ->
o << zip.getInputStream(it)
}
target.setLastModified(it.getTime())
}
}
}
zip.close()
}
}
}
task upload(type: UploadS3Task) {
bucket 'prelert-artifacts'
// Only upload the platform-specific artifacts in this task
def zipFileDir = fileTree("${buildDir}/distributions").matching { include "*-aarch64.zip", "*-x86_64.zip" }
for (zipFile in zipFileDir) {
upload zipFile, "maven/${artifactGroupPath}/${artifactName}/${project.version}/${zipFile.name}"
}
description = 'Upload C++ zips to S3 Bucket'
}
task downloadPlatformSpecific(type: DownloadPlatformSpecific) {
baseName = artifactName
downloadDirectory = "${buildDir}/distributions"
extractDirectory = file("${buildDir}/temp")
description = 'Download and extract previously created platform-specific C++ zips'
}
task buildUberZip(type: Zip, dependsOn: downloadPlatformSpecific) {
archiveBaseName = artifactName
archiveVersion = project.version
destinationDirectory = file("${buildDir}/distributions")
from(fileTree(downloadPlatformSpecific.outputs.files.singleFile))
description = 'Create an uber zip from combined platform-specific C++ distributions'
}
task uberUpload(type: UploadS3Task, dependsOn: buildUberZip) {
bucket 'prelert-artifacts'
upload buildUberZip.outputs.files.singleFile, "maven/${artifactGroupPath}/${artifactName}/${project.version}/${buildUberZip.outputs.files.singleFile.name}"
description = 'Upload C++ uber zip to S3 Bucket'
}