-
Notifications
You must be signed in to change notification settings - Fork 20
Expand file tree
/
Copy path.gitlab-ci.yml
More file actions
355 lines (319 loc) · 11.1 KB
/
.gitlab-ci.yml
File metadata and controls
355 lines (319 loc) · 11.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
include:
# This applies global rules to run pipelines for the default branch, tags and all types of merge requests
- template: Workflows/MergeRequest-Pipelines.gitlab-ci.yml
# This includes the GitLab provided dependency scanning jobs
- template: Security/Dependency-Scanning.gitlab-ci.yml
# This includes the GitLab provided SAST jobs.
- template: Security/SAST.gitlab-ci.yml
# This includes the GitLab provided secret detection job.
- template: Security/Secret-Detection.gitlab-ci.yml
stages:
- validate
- build
- test
- publish
- deploy
- visualize
.cache-maven-repo: &cache-maven-repo
key: maven-repo
when: always
policy: pull-push
paths:
- .m2/repository/
default:
image: maven:3.8.5-openjdk-17-slim
cache:
- *cache-maven-repo
artifacts:
# Expire all artifacts already after 36 hours. (Default is around 30 days/4 weeks)
expire_in: 36 hours
variables:
# Arguments passed to maven before CLI arguments, only used from Maven 4 onward
MAVEN_ARGS: '--no-transfer-progress -Dmaven.repo.local=.m2/repository -Dmaven.compiler.useIncrementalCompilation=false -Prelease'
# Jobs are executed in docker, fetch should be available and faster. Could be already configured in the executor.
GIT_STRATEGY: fetch
# Do a shallow clone. 50 is the default value for new repositories from GitLab 12 onwards
GIT_DEPTH: 50
# Print upload and download progress for artifact and cache operations every second
TRANSFER_METER_FREQUENCY: '1s'
# Disable all SAST jobs by default, since we manually enable/configure those
SAST_DISABLED: 'true'
checkstyle:
stage: validate
needs: [ ]
cache:
- <<: *cache-maven-repo
policy: pull
script:
- mvn $MAVEN_ARGS validate
after_script: # Will also execute if the above fails so the errors will be shown in the MergeRequest
- scripts/ci/generate-codequality-report-from-checkstyle.sh ./target/checkstyle-result.xml > checkstyle-report.json
artifacts:
reports:
codequality:
- checkstyle-report.json
checkdbmigrationversions:
image: busybox:latest
stage: validate
needs: [ ]
cache: [ ]
script:
- scripts/ci/check-database-migration-version-numbers.sh
metrics:
image: debian:11-slim
stage: validate
needs: [ ]
cache: [ ]
script:
- printf 'files_total{language="java"} %d\n' "$(find . -type f -name "*.java" | wc -l)" >> metrics.txt
- printf 'code_lines_total{language="java"} %d\n' "$(find . -type f -name "*.java" -print0 | xargs -0 cat | wc -l)" >> metrics.txt
- printf 'todo_occurrences_total %d\n' "$(grep -R -I --exclude-dir={.git,target} TODO . | wc -l)" >> metrics.txt
artifacts:
reports:
metrics: metrics.txt
build:
stage: build
needs: [ ]
cache:
- *cache-maven-repo
# TODO(Alex): Add caching for node/npm/js stuff
#- key: node-stuff
# when: always
# paths:
# - node/
# - node_modules/
script:
- mvn $MAVEN_ARGS -DskipCheckstyle -DskipTests package
- printf 'artifact_size_bytes{artifact="codedefenders.war"} %d\n' "$(stat --printf="%s" target/codedefenders.war)" >> metrics.txt
- rm -r target/codedefenders/WEB-INF/lib
artifacts:
paths:
- target/
reports:
metrics: metrics.txt
###
### Unit and Integration tests
###
unit-tests:
stage: test
# This implies a dependency on the build job (and so downloads its artifacts)
needs:
- build
before_script:
# Update timestamps of build output files, so maven does not think it needs to recompile those
- find ./target | xargs -exec touch -c {} \;
script:
- mvn $MAVEN_ARGS -DskipCheckstyle test
after_script:
- mvn $MAVEN_ARGS jacoco:report
# Print summary line, so GitLab can extract Code Coverage
- cat target/site/jacoco/index.html | grep -o '<tfoot>.*</tfoot>'
# Regex for extracting a coverage percentage, project wide configuration is planned to be deprecated at some point
coverage: '/Total.*?([0-9]{1,3})%/'
artifacts:
paths:
- target/jacoco.exec
reports:
junit:
- target/surefire-reports/TEST-*.xml
# Template for database integration tests, we repeat those for the three supported database versions
.integration-tests_database:
stage: test
needs:
- build
variables:
MYSQL_DATABASE: codedefenders
MYSQL_ROOT_PASSWORD: ''
MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
before_script:
- printf 'url=database:3306/codedefenders\nusername=root\npassword=\n' > src/integration/resources/database.properties
# Update timestamps of build output files, so maven does not think it needs to recompile those
- find ./target | xargs -exec touch -c {} \;
script:
- mvn $MAVEN_ARGS -DskipCheckstyle -DskipUnitTests verify -P it-database-only
artifacts:
paths:
- target/jacoco-it.exec
reports:
junit:
- target/failsafe-reports/TEST-*.xml
integration-tests_database_mariadb-10-5:
extends: .integration-tests_database
services:
- name: mariadb:10.5.10
alias: database
after_script:
- mvn $MAVEN_ARGS jacoco:report-integration
# Print summary line, so GitLab can extract Code Coverage
- cat target/site/jacoco-it/index.html | grep -o '<tfoot>.*</tfoot>'
# Regex for extracting a coverage percentage, project wide configuration is planned to be deprecated at some point
coverage: '/Total.*?([0-9]{1,3})%/'
rules:
- when: manual
allow_failure: true
integration-tests_database_mariadb-10-3:
extends: .integration-tests_database
services:
- name: mariadb:10.3.31
alias: database
rules:
# Exclude job for Draft merge requests
- if: '$CI_MERGE_REQUEST_TITLE !~ /^Draft:/'
when: manual
allow_failure: true
integration-tests_database_mysql-8-0:
extends: .integration-tests_database
services:
- name: mysql:8.0.28
alias: database
rules:
# Exclude job for Draft merge requests
- if: '$CI_MERGE_REQUEST_TITLE !~ /^Draft:/'
when: manual
allow_failure: true
###
### Analysis (Source Code, Dependencies, and Secret detection)
###
spotbugs-sast:
stage: test
needs:
- build
variables:
# We target Java 17 (non default)
SAST_JAVA_VERSION: 17
# We already have a compiled version of the project, no need to recompile it
COMPILE: 'false'
# GitLab Spotbugs uses MAVEN_CLI_OPTS for additional CLI flags
MAVEN_CLI_OPTS: '$MAVEN_ARGS -DskipCheckstyle -DskipTests'
# Needs to be set explicitly even if it's set via $MAVEN_ARGS/$MAVEN_CLI_OPTS above, to prevent a fatal spotbugs
# error: 'lstat /root/.m2/repository: no such file or directory'. There exists an upstream issue for a similar error
# https://gitlab.com/gitlab-org/gitlab/-/issues/334854 but the upstream issue is for maven multi-module projects.
MAVEN_REPO_PATH: '.m2/repository'
before_script:
# Remove unrelated project files
- rm {src/main/resources,target/classes,target/codedefenders/WEB-INF/classes}/data/build.xml {src/test/resources,target/test-classes}/itests/build.xml {src/main/webapp,target/codedefenders}/project-exporter/{build.gradle,gradlew}
artifacts:
reports:
sast: gl-sast-report.json
rules:
# Exclude job for Draft merge requests and trigger it otherwise
- if: '$CI_MERGE_REQUEST_TITLE !~ /^Draft:/'
gemnasium-maven-dependency_scanning:
stage: test
needs: [ ]
variables:
MAVEN_CLI_OPTS: "$MAVEN_ARGS -DskipCheckstyle -DskipTests"
rules:
# Exclude job for Draft merge requests
- if: '$CI_MERGE_REQUEST_TITLE !~ /^Draft:/'
secret_detection:
stage: test
needs: [ ]
cache: [ ]
rules:
# Overwrite rules so job is executed even if $SAST_DISABLED is set
- when: always
semgrep-sast:
stage: test
needs: [ ]
cache: [ ]
rules:
# Overwrite rules so job is executed even if $SAST_DISABLED is set
- when: always
###
### Merge Unit and Integration test coverage and convert it so GitLab can visualize covered/uncovered lines
###
merged-test-coverage:
stage: visualize
needs:
- build
- unit-tests
# - integration-tests_database_mariadb-10-5
script:
- mvn $MAVEN_ARGS -Djacoco.destFile='${project.build.directory}/jacoco-merged.exec' -Djacoco.dataFile='${project.build.directory}/jacoco-merged.exec' jacoco:merge jacoco:report
artifacts:
paths:
- target/site/jacoco/jacoco.xml
reports:
coverage_report:
coverage_format: jacoco
path: target/site/jacoco/jacoco.xml
###
### Publish jobs
###
container-images:
image: quay.io/containers/buildah
stage: publish
tags:
- container
cache: [ ]
needs:
- build
variables:
# Use vfs with buildah. Docker offers overlayfs as a default, but buildah
# cannot stack overlayfs on top of another overlayfs filesystem.
STORAGE_DRIVER: vfs
# You may need this workaround for some errors: https://stackoverflow.com/a/70438141/1233435
BUILDAH_ISOLATION: chroot
before_script:
# Log in to container registries
- export REGISTRY_AUTH_FILE=$HOME/auth.json
- echo "$REGISTRY_QUAY_PASSWORD" | buildah login -u "$REGISTRY_QUAY_USER" --password-stdin quay.io
- echo "$REGISTRY_DOCKER_PASSWORD" | buildah login -u "$REGISTRY_DOCKER_USER" --password-stdin docker.io
- echo "$REGISTRY_GHCR_PASSWORD" | buildah login -u "$REGISTRY_GHCR_USER" --password-stdin ghcr.io
script:
- scripts/ci/build-container-image "localhost/codedefenders:tmp" "production"
- scripts/ci/push-container-images "localhost/codedefenders:tmp" "codedefenders/codedefenders" "quay.io" "docker.io" "ghcr.io"
allow_failure: true # Otherwise pipelines will block on this job
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
when: manual
###
### Deploy jobs
###
.deploy:
image: debian:11-slim
stage: deploy
needs:
- build
cache: [ ]
variables:
# The deploy-jobs do not need the repository in any way
GIT_STRATEGY: none
before_script:
# Taken from: https://docs.gitlab.com/ee/ci/ssh_keys/README.html#ssh-keys-when-using-the-docker-executor
- command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | base64 --decode | ssh-add -
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo "$SSH_HOST_KEY" > ~/.ssh/known_hosts
script:
- scp target/codedefenders.war "[email protected]:/tmp/$DEPLOY_ENV.war"
- ssh [email protected] "chown deploy:tomcat /tmp/$DEPLOY_ENV.war"
- ssh [email protected] "chmod 640 /tmp/$DEPLOY_ENV.war"
- ssh [email protected] "mv /tmp/$DEPLOY_ENV.war /var/lib/tomcat10/webapps/$DEPLOY_ENV.war"
tags:
- deploy
deploy-staging:
extends: .deploy
variables:
DEPLOY_ENV: staging
environment:
name: staging
url: https://codedefenders.fim.uni-passau.de/staging/
resource_group: deploy-staging
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
deploy-production:
extends: .deploy
variables:
DEPLOY_ENV: ROOT
environment:
name: production
url: https://code-defenders.org/
resource_group: deploy-production
allow_failure: true # Otherwise pipelines will block on this job
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
when: manual