diff --git a/hooks/cascading-scans/README.md b/hooks/cascading-scans/README.md index 97165e0fdc..45ed5b4e57 100644 --- a/hooks/cascading-scans/README.md +++ b/hooks/cascading-scans/README.md @@ -162,6 +162,7 @@ zap-http zap-baseline-scan non-invasive medium |-----|------|---------|-------------| | hook.image.repository | string | `"docker.io/securecodebox/hook-cascading-scans"` | Hook image repository | | hook.image.tag | string | defaults to the charts version | The image Tag defaults to the charts version if not defined. | +| hook.priority | int | `0` | Hook priority. Higher priority Hooks are guaranteed to execute before low priority Hooks. | | hook.ttlSecondsAfterFinished | string | `nil` | Seconds after which the kubernetes job for the hook will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ | ## License diff --git a/hooks/cascading-scans/hook/hook.test.js b/hooks/cascading-scans/hook/hook.test.js index 79d838388a..4ed40c2b6b 100644 --- a/hooks/cascading-scans/hook/hook.test.js +++ b/hooks/cascading-scans/hook/hook.test.js @@ -1130,6 +1130,7 @@ test("Templating should apply to environment variables", () => { "value": "foobar.com", }, ], + "hookSelector": Object {}, "initContainers": Array [], "parameters": Array [ "--regular", @@ -1379,6 +1380,7 @@ test("Templating should apply to initContainer environment variables", () => { "spec": Object { "cascades": Object {}, "env": Array [], + "hookSelector": Object {}, "initContainers": Array [ Object { "command": Array [ diff --git a/hooks/cascading-scans/templates/cascading-scans-hook.yaml b/hooks/cascading-scans/templates/cascading-scans-hook.yaml index 87ae80c24b..36f98a826c 100644 --- a/hooks/cascading-scans/templates/cascading-scans-hook.yaml +++ b/hooks/cascading-scans/templates/cascading-scans-hook.yaml @@ -13,6 +13,7 @@ metadata: {{ toYaml . }} {{- end }} spec: + priority: {{ .Values.hook.priority }} type: ReadOnly image: "{{ .Values.hook.image.repository }}:{{ .Values.hook.image.tag | default .Chart.Version }}" imagePullSecrets: diff --git a/hooks/cascading-scans/values.yaml b/hooks/cascading-scans/values.yaml index 50160bdfe3..63e7370d54 100644 --- a/hooks/cascading-scans/values.yaml +++ b/hooks/cascading-scans/values.yaml @@ -17,5 +17,8 @@ hook: # hook.labels -- Add Kubernetes Labels to the hook definition labels: {} + # -- Hook priority. Higher priority Hooks are guaranteed to execute before low priority Hooks. + priority: 0 + # hook.ttlSecondsAfterFinished -- Seconds after which the kubernetes job for the hook will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ ttlSecondsAfterFinished: null diff --git a/hooks/finding-post-processing/README.md b/hooks/finding-post-processing/README.md index 56d496a2bb..65246f0d18 100644 --- a/hooks/finding-post-processing/README.md +++ b/hooks/finding-post-processing/README.md @@ -89,6 +89,7 @@ The `override` field specifies the desired fields and values that need to be upd |-----|------|---------|-------------| | hook.image.repository | string | `"docker.io/securecodebox/hook-finding-post-processing"` | Hook image repository | | hook.image.tag | string | defaults to the charts version | The image Tag defaults to the charts version if not defined. | +| hook.priority | int | `0` | Hook priority. Higher priority Hooks are guaranteed to execute before low priority Hooks. | | hook.ttlSecondsAfterFinished | string | `nil` | Seconds after which the kubernetes job for the hook will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ | | rules | list | `[]` | | diff --git a/hooks/finding-post-processing/templates/finding-post-processing-hook.yaml b/hooks/finding-post-processing/templates/finding-post-processing-hook.yaml index e01efa414f..dae0b91ecb 100644 --- a/hooks/finding-post-processing/templates/finding-post-processing-hook.yaml +++ b/hooks/finding-post-processing/templates/finding-post-processing-hook.yaml @@ -12,6 +12,7 @@ metadata: {{ toYaml . }} {{- end }} spec: + priority: {{ .Values.hook.priority }} type: ReadAndWrite image: "{{ .Values.hook.image.repository }}:{{ .Values.hook.image.tag | default .Chart.Version }}" env: diff --git a/hooks/finding-post-processing/values.yaml b/hooks/finding-post-processing/values.yaml index 4d16d013bf..ba87d2544d 100644 --- a/hooks/finding-post-processing/values.yaml +++ b/hooks/finding-post-processing/values.yaml @@ -33,5 +33,8 @@ hook: # hook.labels -- Add Kubernetes Labels to the hook definition labels: {} + # -- Hook priority. Higher priority Hooks are guaranteed to execute before low priority Hooks. + priority: 0 + # hook.ttlSecondsAfterFinished -- Seconds after which the kubernetes job for the hook will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ ttlSecondsAfterFinished: null diff --git a/hooks/generic-webhook/README.md b/hooks/generic-webhook/README.md index 9efaad1dfb..55e72d4744 100644 --- a/hooks/generic-webhook/README.md +++ b/hooks/generic-webhook/README.md @@ -57,6 +57,7 @@ Kubernetes: `>=v1.11.0-0` |-----|------|---------|-------------| | hook.image.repository | string | `"docker.io/securecodebox/hook-generic-webhook"` | Hook image repository | | hook.image.tag | string | defaults to the charts version | The image Tag defaults to the charts version if not defined. | +| hook.priority | int | `0` | Hook priority. Higher priority Hooks are guaranteed to execute before low priority Hooks. | | hook.ttlSecondsAfterFinished | string | `nil` | Seconds after which the kubernetes job for the hook will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ | | webhookUrl | string | `"http://example.com"` | The URL of your WebHook endpoint | diff --git a/hooks/generic-webhook/templates/webhook-hook.yaml b/hooks/generic-webhook/templates/webhook-hook.yaml index 317b476b2a..2c30a3da84 100644 --- a/hooks/generic-webhook/templates/webhook-hook.yaml +++ b/hooks/generic-webhook/templates/webhook-hook.yaml @@ -12,6 +12,7 @@ metadata: {{ toYaml . }} {{- end }} spec: + priority: {{ .Values.hook.priority }} type: ReadOnly image: "{{ .Values.hook.image.repository }}:{{ .Values.hook.image.tag | default .Chart.Version }}" ttlSecondsAfterFinished: {{ .Values.hook.ttlSecondsAfterFinished }} diff --git a/hooks/generic-webhook/values.yaml b/hooks/generic-webhook/values.yaml index 39da0a067e..cf4794e3a8 100644 --- a/hooks/generic-webhook/values.yaml +++ b/hooks/generic-webhook/values.yaml @@ -20,5 +20,8 @@ hook: # hook.labels -- Add Kubernetes Labels to the hook definition labels: {} + # -- Hook priority. Higher priority Hooks are guaranteed to execute before low priority Hooks. + priority: 0 + # hook.ttlSecondsAfterFinished -- Seconds after which the kubernetes job for the hook will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ ttlSecondsAfterFinished: null diff --git a/hooks/notification/README.md b/hooks/notification/README.md index 6ca700b61f..48c4dd3807 100644 --- a/hooks/notification/README.md +++ b/hooks/notification/README.md @@ -346,6 +346,7 @@ To fill your template with data we provide the following objects. | hook.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images | | hook.image.repository | string | `"docker.io/securecodebox/hook-notification"` | Hook image repository | | hook.image.tag | string | defaults to the charts version | Image tag | +| hook.priority | int | `0` | Hook priority. Higher priority Hooks are guaranteed to execute before low priority Hooks. | | hook.ttlSecondsAfterFinished | string | `nil` | seconds after which the kubernetes job for the hook will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ | | notificationChannels[0].endPoint | string | `"SOME_ENV_KEY"` | | | notificationChannels[0].name | string | `"slack"` | | diff --git a/hooks/notification/templates/notification-hook.yaml b/hooks/notification/templates/notification-hook.yaml index 2961756cf6..5525628b60 100644 --- a/hooks/notification/templates/notification-hook.yaml +++ b/hooks/notification/templates/notification-hook.yaml @@ -12,6 +12,7 @@ metadata: {{ toYaml . }} {{- end }} spec: + priority: {{ .Values.hook.priority }} type: ReadOnly imagePullPolicy: "{{ .Values.hook.image.pullPolicy }}" image: "{{ .Values.hook.image.repository }}:{{ .Values.hook.image.tag | default .Chart.Version }}" diff --git a/hooks/notification/values.yaml b/hooks/notification/values.yaml index 5cf9243c90..87d2d86bb3 100644 --- a/hooks/notification/values.yaml +++ b/hooks/notification/values.yaml @@ -19,6 +19,9 @@ hook: # hook.labels -- Add Kubernetes Labels to the hook definition labels: {} + # -- Hook priority. Higher priority Hooks are guaranteed to execute before low priority Hooks. + priority: 0 + # hook.ttlSecondsAfterFinished -- seconds after which the kubernetes job for the hook will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ ttlSecondsAfterFinished: null diff --git a/hooks/persistence-defectdojo/README.md b/hooks/persistence-defectdojo/README.md index d34fa3d1b1..2e4b71839f 100644 --- a/hooks/persistence-defectdojo/README.md +++ b/hooks/persistence-defectdojo/README.md @@ -229,7 +229,8 @@ spec: | defectdojo.url | string | `"http://defectdojo-django.default.svc"` | Url to the DefectDojo Instance | | hook.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images | | hook.image.repository | string | `"docker.io/securecodebox/hook-persistence-defectdojo"` | Hook image repository | -| hook.image.tag | string | `nil` | Container image tag | +| hook.image.tag | string | defaults to the charts version | Container image tag | +| hook.priority | int | `0` | Hook priority. Higher priority Hooks are guaranteed to execute before low priority Hooks. | ## License [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) diff --git a/hooks/persistence-defectdojo/templates/persistence-provider.yaml b/hooks/persistence-defectdojo/templates/persistence-provider.yaml index 58d374684e..f1641913a0 100644 --- a/hooks/persistence-defectdojo/templates/persistence-provider.yaml +++ b/hooks/persistence-defectdojo/templates/persistence-provider.yaml @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -apiVersion: "execution.securecodebox.io/v1" +apiVersion: execution.securecodebox.io/v1 kind: ScanCompletionHook metadata: name: {{ include "persistence-defectdojo.fullname" . }} @@ -13,6 +13,7 @@ metadata: {{ toYaml . }} {{- end }} spec: + priority: {{ .Values.hook.priority }} {{- if .Values.defectdojo.syncFindingsBack }} type: ReadAndWrite {{- else }} diff --git a/hooks/persistence-defectdojo/values.yaml b/hooks/persistence-defectdojo/values.yaml index 4ec1a461a2..2c06aca8b8 100644 --- a/hooks/persistence-defectdojo/values.yaml +++ b/hooks/persistence-defectdojo/values.yaml @@ -19,6 +19,9 @@ hook: # hook.labels -- Add Kubernetes Labels to the hook definition labels: {} + # -- Hook priority. Higher priority Hooks are guaranteed to execute before low priority Hooks. + priority: 0 + defectdojo: # -- Syncs back (two way sync) all imported findings from DefectDojo to SCB Findings Store. When set to false the hook will only import the findings to DefectDojo (one way sync). syncFindingsBack: true diff --git a/hooks/persistence-elastic/README.md b/hooks/persistence-elastic/README.md index c2094c39f2..4284635768 100644 --- a/hooks/persistence-elastic/README.md +++ b/hooks/persistence-elastic/README.md @@ -82,6 +82,7 @@ the [Luxon documentation](https://moment.github.io/luxon/docs/manual/formatting. | fullnameOverride | string | `""` | | | hook.image.repository | string | `"docker.io/securecodebox/hook-persistence-elastic"` | Hook image repository | | hook.image.tag | string | defaults to the charts version | The image Tag defaults to the charts version if not defined. | +| hook.priority | int | `0` | Hook priority. Higher priority Hooks are guaranteed to execute before low priority Hooks. | | hook.ttlSecondsAfterFinished | string | `nil` | Seconds after which the kubernetes job for the hook will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ | | imagePullSecrets | list | `[]` | | | indexAppendNamespace | bool | `true` | Define if the name of the namespace where this hook is deployed to must be added to the index name. The namespace can be used to separate index by tenants (namespaces). | diff --git a/hooks/persistence-elastic/templates/persistence-provider.yaml b/hooks/persistence-elastic/templates/persistence-provider.yaml index 6a329759c9..124f035b07 100644 --- a/hooks/persistence-elastic/templates/persistence-provider.yaml +++ b/hooks/persistence-elastic/templates/persistence-provider.yaml @@ -13,6 +13,7 @@ metadata: {{ toYaml . }} {{- end }} spec: + priority: {{ .Values.hook.priority }} type: ReadOnly image: "{{ .Values.hook.image.repository }}:{{ .Values.hook.image.tag | default .Chart.Version }}" ttlSecondsAfterFinished: {{ .Values.hook.ttlSecondsAfterFinished }} diff --git a/hooks/persistence-elastic/values.yaml b/hooks/persistence-elastic/values.yaml index ae6c2fa922..038852f5d5 100644 --- a/hooks/persistence-elastic/values.yaml +++ b/hooks/persistence-elastic/values.yaml @@ -89,5 +89,8 @@ hook: # hook.labels -- Add Kubernetes Labels to the hook definition labels: {} + # -- Hook priority. Higher priority Hooks are guaranteed to execute before low priority Hooks. + priority: 0 + # hook.ttlSecondsAfterFinished -- Seconds after which the kubernetes job for the hook will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ ttlSecondsAfterFinished: null diff --git a/hooks/update-field/README.md b/hooks/update-field/README.md index 714d428014..9193b163b0 100644 --- a/hooks/update-field/README.md +++ b/hooks/update-field/README.md @@ -64,6 +64,7 @@ helm upgrade --install ufh secureCodeBox/update-field-hook --set attribute.name= | attribute.value | string | `"my-own-category"` | The value of the attribute you want to add to each finding result | | hook.image.repository | string | `"docker.io/securecodebox/hook-update-field"` | Hook image repository | | hook.image.tag | string | defaults to the charts version | The image Tag defaults to the charts version if not defined. | +| hook.priority | int | `0` | Hook priority. Higher priority Hooks are guaranteed to execute before low priority Hooks. | | hook.ttlSecondsAfterFinished | string | `nil` | Seconds after which the kubernetes job for the hook will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ | ## License diff --git a/hooks/update-field/templates/update-field-hook.yaml b/hooks/update-field/templates/update-field-hook.yaml index 901116780b..fc234cca0f 100644 --- a/hooks/update-field/templates/update-field-hook.yaml +++ b/hooks/update-field/templates/update-field-hook.yaml @@ -12,6 +12,7 @@ metadata: {{ toYaml . }} {{- end }} spec: + priority: {{ .Values.hook.priority }} type: ReadAndWrite image: "{{ .Values.hook.image.repository }}:{{ .Values.hook.image.tag | default .Chart.Version }}" ttlSecondsAfterFinished: {{ .Values.hook.ttlSecondsAfterFinished }} diff --git a/hooks/update-field/values.yaml b/hooks/update-field/values.yaml index d4ad8d3609..9235e35c5a 100644 --- a/hooks/update-field/values.yaml +++ b/hooks/update-field/values.yaml @@ -23,5 +23,8 @@ hook: # hook.labels -- Add Kubernetes Labels to the hook definition labels: {} + # -- Hook priority. Higher priority Hooks are guaranteed to execute before low priority Hooks. + priority: 0 + # hook.ttlSecondsAfterFinished -- Seconds after which the kubernetes job for the hook will be deleted. Requires the Kubernetes TTLAfterFinished controller: https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ ttlSecondsAfterFinished: null diff --git a/operator/apis/execution/v1/scan_types.go b/operator/apis/execution/v1/scan_types.go index 91409b8cdc..07d406077c 100644 --- a/operator/apis/execution/v1/scan_types.go +++ b/operator/apis/execution/v1/scan_types.go @@ -108,6 +108,8 @@ type ScanStatus struct { Findings FindingStats `json:"findings,omitempty"` ReadAndWriteHookStatus []HookStatus `json:"readAndWriteHookStatus,omitempty"` + + OrderedHookStatuses [][]*HookStatus `json:"orderedHookStatuses,omitempty"` } // HookState Describes the State of a Hook on a Scan @@ -125,6 +127,8 @@ type HookStatus struct { HookName string `json:"hookName"` State HookState `json:"state"` JobName string `json:"jobName,omitempty"` + Priority int `json:"priority"` + Type HookType `json:"type"` } // FindingStats contains the general stats about the results of the scan diff --git a/operator/apis/execution/v1/scancompletionhook_types.go b/operator/apis/execution/v1/scancompletionhook_types.go index 2ce5136450..d7beb87ee8 100644 --- a/operator/apis/execution/v1/scancompletionhook_types.go +++ b/operator/apis/execution/v1/scancompletionhook_types.go @@ -30,6 +30,11 @@ type ScanCompletionHookSpec struct { // Defines weather the hook should be able to change the findings or is run in a read only mode. Type HookType `json:"type"` + // Higher priority hooks run before low priority hooks. Within a priority class ReadAndWrite hooks are started before ReadOnly hooks, ReadAndWrite hooks wil be launched in serial, and ReadOnly hooks will be launched in parallel. + // +kubebuilder:default=0 + // +kubebuilder:validation:Optional + Priority int `json:"priority"` + // Image is the container image for the hooks kubernetes job Image string `json:"image,omitempty"` // ImagePullSecrets used to access private hooks images @@ -60,6 +65,7 @@ type ScanCompletionHookStatus struct { // +kubebuilder:object:root=true // +kubebuilder:printcolumn:name="Type",type=string,JSONPath=`.spec.type`,description="ScanCompletionHook Type" +// +kubebuilder:printcolumn:name="Priority",type=string,JSONPath=`.spec.priority`,description="ScanCompletionHook Priority" // +kubebuilder:printcolumn:name="Image",type=string,JSONPath=`.spec.image`,description="ScanCompletionHook Image" // ScanCompletionHook is the Schema for the ScanCompletionHooks API diff --git a/operator/apis/execution/v1/zz_generated.deepcopy.go b/operator/apis/execution/v1/zz_generated.deepcopy.go index 0e791245d3..4653cfa76a 100644 --- a/operator/apis/execution/v1/zz_generated.deepcopy.go +++ b/operator/apis/execution/v1/zz_generated.deepcopy.go @@ -487,6 +487,23 @@ func (in *ScanStatus) DeepCopyInto(out *ScanStatus) { *out = make([]HookStatus, len(*in)) copy(*out, *in) } + if in.OrderedHookStatuses != nil { + in, out := &in.OrderedHookStatuses, &out.OrderedHookStatuses + *out = make([][]*HookStatus, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = make([]*HookStatus, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(HookStatus) + **out = **in + } + } + } + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScanStatus. diff --git a/operator/config/crd/bases/execution.securecodebox.io_scancompletionhooks.yaml b/operator/config/crd/bases/execution.securecodebox.io_scancompletionhooks.yaml index 5758ef4520..2179549b24 100644 --- a/operator/config/crd/bases/execution.securecodebox.io_scancompletionhooks.yaml +++ b/operator/config/crd/bases/execution.securecodebox.io_scancompletionhooks.yaml @@ -24,6 +24,10 @@ spec: jsonPath: .spec.type name: Type type: string + - description: ScanCompletionHook Priority + jsonPath: .spec.priority + name: Priority + type: string - description: ScanCompletionHook Image jsonPath: .spec.image name: Image @@ -174,6 +178,13 @@ spec: type: string type: object type: array + priority: + default: 0 + description: Higher priority hooks run before low priority hooks. + Within a priority class ReadAndWrite hooks are started before ReadOnly + hooks, ReadAndWrite hooks wil be launched in serial, and ReadOnly + hooks will be launched in parallel. + type: integer serviceAccountName: description: ServiceAccountName Name of the serviceAccount Name used. Should only be used if your hook needs specifc RBAC Access. Otherwise diff --git a/operator/config/crd/bases/execution.securecodebox.io_scans.yaml b/operator/config/crd/bases/execution.securecodebox.io_scans.yaml index 3e24c3b2b0..4ae2372436 100644 --- a/operator/config/crd/bases/execution.securecodebox.io_scans.yaml +++ b/operator/config/crd/bases/execution.securecodebox.io_scans.yaml @@ -2906,6 +2906,32 @@ spec: parser & hooks) has been marked as "Done" format: date-time type: string + orderedHookStatuses: + items: + items: + properties: + hookName: + type: string + jobName: + type: string + priority: + type: integer + state: + description: HookState Describes the State of a Hook on a + Scan + type: string + type: + description: HookType Defines weather the hook should be able + to change the findings or is run in a read only mode. + type: string + required: + - hookName + - priority + - state + - type + type: object + type: array + type: array rawResultDownloadLink: description: RawResultDownloadLink link to download the raw result file from. Valid for 7 days @@ -2929,12 +2955,20 @@ spec: type: string jobName: type: string + priority: + type: integer state: description: HookState Describes the State of a Hook on a Scan type: string + type: + description: HookType Defines weather the hook should be able + to change the findings or is run in a read only mode. + type: string required: - hookName + - priority - state + - type type: object type: array state: diff --git a/operator/controllers/execution/scans/hook_reconciler.go b/operator/controllers/execution/scans/hook_reconciler.go index b1f8de9523..e49a36bc39 100644 --- a/operator/controllers/execution/scans/hook_reconciler.go +++ b/operator/controllers/execution/scans/hook_reconciler.go @@ -10,6 +10,7 @@ import ( "k8s.io/apimachinery/pkg/labels" executionv1 "github.com/secureCodeBox/secureCodeBox/operator/apis/execution/v1" + "github.com/secureCodeBox/secureCodeBox/operator/utils" util "github.com/secureCodeBox/secureCodeBox/operator/utils" batch "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -41,27 +42,9 @@ func (r *ScanReconciler) setHookStatus(scan *executionv1.Scan) error { r.Log.Info("Found ScanCompletionHooks", "ScanCompletionHooks", len(scanCompletionHooks.Items)) - readAndWriteHooks := []executionv1.ScanCompletionHook{} - // filter all ReadAndWriteHooks in the scamCompletionHooks list - for _, hook := range scanCompletionHooks.Items { - if hook.Spec.Type == executionv1.ReadAndWrite { - readAndWriteHooks = append(readAndWriteHooks, hook) - } - } - - r.Log.Info("Found ReadAndWriteHooks", "ReadAndWriteHooks", len(readAndWriteHooks)) - - hookStatus := []executionv1.HookStatus{} - - for _, hook := range readAndWriteHooks { - hookStatus = append(hookStatus, executionv1.HookStatus{ - HookName: hook.Name, - State: executionv1.Pending, - }) - } - - scan.Status.State = "ReadAndWriteHookProcessing" - scan.Status.ReadAndWriteHookStatus = hookStatus + orderedHookStatus := util.FromUnorderedList(scanCompletionHooks.Items) + scan.Status.OrderedHookStatuses = orderedHookStatus + scan.Status.State = "HookProcessing" if err := r.Status().Update(ctx, scan); err != nil { r.Log.Error(err, "unable to update Scan status") @@ -71,257 +54,217 @@ func (r *ScanReconciler) setHookStatus(scan *executionv1.Scan) error { return nil } -func (r *ScanReconciler) executeReadAndWriteHooks(scan *executionv1.Scan) error { - // Get the first Hook Status which is not completed. +func (r *ScanReconciler) migrateHookStatus(scan *executionv1.Scan) error { ctx := context.Background() - var nonCompletedHook *executionv1.HookStatus + var scanCompletionHooks executionv1.ScanCompletionHookList + r.Log.Info("Starting hook Status field migrations", "ReadAndWriteHookStatus", scan.Status.ReadAndWriteHookStatus) - for _, hook := range scan.Status.ReadAndWriteHookStatus { - if hook.State != executionv1.Completed { - nonCompletedHook = &hook - break - } + if err := r.List(ctx, &scanCompletionHooks, client.InNamespace(scan.Namespace)); err != nil { + r.Log.V(7).Info("Unable to fetch ScanCompletionHooks") + return err } - // If nil then all hooks are done - if nonCompletedHook == nil { - scan.Status.State = "ReadAndWriteHookCompleted" - if err := r.Status().Update(ctx, scan); err != nil { - r.Log.Error(err, "unable to update Scan status") - return err - } - return nil + // Add new fields to old ReadAndWriteHookStatus object and convert to pointers + strSlice := make([]*executionv1.HookStatus, len(scan.Status.ReadAndWriteHookStatus)) + for i := range scan.Status.ReadAndWriteHookStatus { + strSlice[i] = scan.Status.ReadAndWriteHookStatus[i].DeepCopy() // Keep original ReadAndWriteHookStatus field + strSlice[i].Priority = 0 + strSlice[i].Type = executionv1.ReadAndWrite + r.Log.Info("Converted ReadAndWrite hook Status", "Original", scan.Status.ReadAndWriteHookStatus[i], "New", strSlice[i]) } - switch nonCompletedHook.State { - case executionv1.Pending: - rawFileURL, err := r.PresignedGetURL(scan.UID, scan.Status.RawResultFile, defaultPresignDuration) - if err != nil { - return err - } - findingsFileURL, err := r.PresignedGetURL(scan.UID, "findings.json", defaultPresignDuration) - if err != nil { - return err - } + // Construct new ReadOnly HookStatus for OrderedHookStatuses + var readOnlyHooks []*executionv1.HookStatus + for _, hook := range scanCompletionHooks.Items { + if hook.Spec.Type == executionv1.ReadOnly { + hookStatus := &executionv1.HookStatus{ + HookName: hook.Name, + Priority: 0, + Type: executionv1.ReadOnly, + } - rawFileUploadURL, err := r.PresignedPutURL(scan.UID, scan.Status.RawResultFile, defaultPresignDuration) - if err != nil { - return err - } - findingsUploadURL, err := r.PresignedPutURL(scan.UID, "findings.json", defaultPresignDuration) - if err != nil { - return err - } + if scan.Status.State == "ReadAndWriteHookProcessing" || scan.Status.State == "ReadAndWriteHookCompleted" { + // ReadOnly hooks should not have started yet, so mark them all as pending + hookStatus.State = executionv1.Pending + } else if scan.Status.State == "ReadOnlyHookProcessing" { + // Had already started ReadOnly hooks and should now check status. + // No status for ReadOnly in old CRD, so mark everything as InProgress and let processInProgressHook update it later. + hookStatus.State = executionv1.InProgress + } else if scan.Status.State == "Done" { + // Had completely finished + hookStatus.State = executionv1.Completed + } - var hook executionv1.ScanCompletionHook - if err := r.Get(ctx, types.NamespacedName{Name: nonCompletedHook.HookName, Namespace: scan.Namespace}, &hook); err != nil { - r.Log.Error(err, "Failed to get ReadAndWrite Hook for HookStatus") - return err - } + r.Log.Info("Retrieved new ReadOnly hook Status", "New", hookStatus) - jobs, err := r.getJobsForScan(scan, client.MatchingLabels{ - "securecodebox.io/job-type": "read-and-write-hook", - "securecodebox.io/hook-name": nonCompletedHook.HookName, - }) - if err != nil { - return err - } - if len(jobs.Items) > 0 { - // Job already exists - return nil + readOnlyHooks = append(readOnlyHooks, hookStatus) } + } - jobName, err := r.createJobForHook( - &hook, - scan, - []string{ - rawFileURL, - findingsFileURL, - rawFileUploadURL, - findingsUploadURL, - }, - ) + scan.Status.OrderedHookStatuses = util.OrderHookStatusesInsideAPrioClass(append(readOnlyHooks, strSlice...)) + if scan.Status.State != "Done" { + scan.Status.State = "HookProcessing" + } - // Update the currently executed hook status to "InProgress" - err = r.updateHookStatus(scan, executionv1.HookStatus{ - HookName: nonCompletedHook.HookName, - JobName: jobName, - State: executionv1.InProgress, - }) + if err := r.Status().Update(ctx, scan); err != nil { + r.Log.Error(err, "unable to update Scan status") return err - case executionv1.InProgress: - jobStatus, err := r.checkIfJobIsCompleted(scan, client.MatchingLabels{ - "securecodebox.io/job-type": "read-and-write-hook", - "securecodebox.io/hook-name": nonCompletedHook.HookName, - }) - if err != nil { - r.Log.Error(err, "Failed to check job status for ReadAndWrite Hook") - return err - } - switch jobStatus { - case completed: - // Job is completed => set current Hook to completed - err = r.updateHookStatus(scan, executionv1.HookStatus{ - HookName: nonCompletedHook.HookName, - JobName: nonCompletedHook.JobName, - State: executionv1.Completed, - }) - return err - case incomplete: - // Still waiting for job to finish - return nil - case failed: - for i, hookStatus := range scan.Status.ReadAndWriteHookStatus { - if hookStatus.HookName == nonCompletedHook.HookName { - scan.Status.ReadAndWriteHookStatus[i].State = executionv1.Failed - } else if hookStatus.State == executionv1.Pending { - scan.Status.ReadAndWriteHookStatus[i].State = executionv1.Cancelled - } - } - scan.Status.State = "Errored" - scan.Status.ErrorDescription = fmt.Sprintf("Failed to execute ReadAndWrite Hook '%s' in job '%s'. Check the logs of the hook for more information.", nonCompletedHook.HookName, nonCompletedHook.JobName) - if err := r.Status().Update(ctx, scan); err != nil { - r.Log.Error(err, "unable to update Scan status") - return err - } - } } + r.Log.Info("Finished hook Status field migrations. ReadOnly hook statuses will be updated later.", + "ReadAndWriteHookStatus", scan.Status.ReadAndWriteHookStatus, + "OrderedHookStatuses", scan.Status.OrderedHookStatuses) + return nil } -func containsJobForHook(jobs *batch.JobList, hook executionv1.ScanCompletionHook) bool { - if len(jobs.Items) == 0 { - return false - } +func (r *ScanReconciler) executeHooks(scan *executionv1.Scan) error { + ctx := context.Background() + + err, currentHooks := utils.CurrentHookGroup(scan.Status.OrderedHookStatuses) - for _, job := range jobs.Items { - if job.ObjectMeta.Labels["securecodebox.io/hook-name"] == hook.Name { - return true + if err != nil && scan.Status.State == "Errored" { + r.Log.V(8).Info("Skipping hook execution as it already contains failed hooks.") + return nil + } else if err != nil { + scan.Status.State = "Errored" + scan.Status.ErrorDescription = fmt.Sprintf("Hook execution failed for a unknown hook. Check the scan.status.hookStatus field for more details") + } else if err == nil && currentHooks == nil { + // No hooks left to execute + scan.Status.State = "Done" + } else { + for _, hook := range currentHooks { + err = r.processHook(scan, hook) + + if err != nil { + scan.Status.State = "Errored" + scan.Status.ErrorDescription = fmt.Sprintf("Failed to execute Hook '%s' in job '%s'. Check the logs of the hook for more information.", hook.HookName, hook.JobName) + } } } - return false + if sErr := r.Status().Update(ctx, scan); sErr != nil { + r.Log.Error(sErr, "Unable to update Scan status") + return sErr + } + return err } -func (r *ScanReconciler) startReadOnlyHooks(scan *executionv1.Scan) error { - ctx := context.Background() - - labelSelector, err := r.getLabelSelector(scan) - if err != nil { - return err +func (r *ScanReconciler) processHook(scan *executionv1.Scan, nonCompletedHook *executionv1.HookStatus) error { + var jobType string + if nonCompletedHook.Type == executionv1.ReadOnly { + jobType = "read-only-hook" + } else if nonCompletedHook.Type == executionv1.ReadAndWrite { + jobType = "read-and-write-hook" } - var scanCompletionHooks executionv1.ScanCompletionHookList + r.Log.Info("Processing hook", "hook", nonCompletedHook, "jobType", jobType) - if err := r.List(ctx, &scanCompletionHooks, - client.InNamespace(scan.Namespace), - client.MatchingLabelsSelector{Selector: labelSelector}, - ); err != nil { - r.Log.V(7).Info("Unable to fetch ScanCompletionHooks") - return err + switch nonCompletedHook.State { + case executionv1.Pending: + return r.processPendingHook(scan, nonCompletedHook, jobType) + case executionv1.InProgress: + return r.processInProgressHook(scan, nonCompletedHook, jobType) } + return nil +} - r.Log.Info("Found ScanCompletionHooks", "ScanCompletionHooks", len(scanCompletionHooks.Items)) - - readOnlyHooks := []executionv1.ScanCompletionHook{} - // filter all ReadOnlyHooks in the scamCompletionHooks list - for _, hook := range scanCompletionHooks.Items { - if hook.Spec.Type == executionv1.ReadOnly { - readOnlyHooks = append(readOnlyHooks, hook) - } - } +func (r *ScanReconciler) processPendingHook(scan *executionv1.Scan, status *executionv1.HookStatus, jobType string) error { + ctx := context.Background() - r.Log.Info("Found ReadOnlyHooks", "ReadOnlyHooks", len(readOnlyHooks)) + var hook executionv1.ScanCompletionHook - // If the readOnlyHooks list is empty, nothing more to do - if len(readOnlyHooks) == 0 { - r.Log.Info("Marked scan as done as without running ReadOnly hooks as non were configured", "ScanName", scan.Name) - scan.Status.State = "Done" - var now metav1.Time = metav1.Now() - scan.Status.FinishedAt = &now - if err := r.Status().Update(ctx, scan); err != nil { - r.Log.Error(err, "Unable to update Scan status") - return err - } - return nil + var err error + err = r.Get(ctx, types.NamespacedName{Name: status.HookName, Namespace: scan.Namespace}, &hook) + if err != nil { + r.Log.Error(err, "Failed to get Hook for HookStatus") + return err } - // Get all read-only-hooks for scan to later check that they weren't already created - jobs, err := r.getJobsForScan(scan, client.MatchingLabels{ - "securecodebox.io/job-type": "read-only-hook", + var jobs *batch.JobList + jobs, err = r.getJobsForScan(scan, client.MatchingLabels{ + "securecodebox.io/job-type": jobType, + "securecodebox.io/hook-name": status.HookName, }) if err != nil { return err } + if len(jobs.Items) > 0 { + // job was already started, setting status to correct jobName and state to ensure it's not overwritten with wrong values + status.JobName = jobs.Items[0].Name + status.State = executionv1.InProgress + return nil + } - for _, hook := range readOnlyHooks { - // Check if hook was already executed - if containsJobForHook(jobs, hook) == true { - r.Log.V(4).Info("Skipping creation of job for hook '%s' as it already exists", hook.Name) - // Job was already created - continue - } + var rawFileURL string + rawFileURL, err = r.PresignedGetURL(scan.UID, scan.Status.RawResultFile, defaultPresignDuration) + if err != nil { + return err + } + var findingsFileURL string + findingsFileURL, err = r.PresignedGetURL(scan.UID, "findings.json", defaultPresignDuration) + if err != nil { + return err + } - rawFileURL, err := r.PresignedGetURL(scan.UID, scan.Status.RawResultFile, defaultPresignDuration) + var args = []string{ + rawFileURL, + findingsFileURL, + } + if hook.Spec.Type == executionv1.ReadAndWrite { + var rawFileUploadURL string + rawFileUploadURL, err = r.PresignedPutURL(scan.UID, scan.Status.RawResultFile, defaultPresignDuration) if err != nil { return err } - findingsFileURL, err := r.PresignedGetURL(scan.UID, "findings.json", defaultPresignDuration) + var findingsUploadURL string + findingsUploadURL, err = r.PresignedPutURL(scan.UID, "findings.json", defaultPresignDuration) if err != nil { return err } - - jobName, err := r.createJobForHook( - &hook, - scan, - []string{ - rawFileURL, - findingsFileURL, - }, - ) - if err != nil { - r.Log.Error(err, "Unable to create Job for ReadOnlyHook", "job", jobName) - return err - } + args = append(args, rawFileUploadURL, findingsUploadURL) } - scan.Status.State = "ReadOnlyHookProcessing" - if err := r.Status().Update(ctx, scan); err != nil { - r.Log.Error(err, "Unable to update Scan status") - return err + + var jobName string + jobName, err = r.createJobForHook( + &hook, + scan, + args, + ) + + if err == nil { + // job was already started, setting status to correct jobName and state to ensure it's not overwritten with wrong values + status.JobName = jobName + status.State = executionv1.InProgress + r.Log.Info("Created job for hook", "hook", status) + return nil } - r.Log.Info("Started ReadOnlyHook", "ReadOnlyHookCount", len(readOnlyHooks)) - return nil + + return err } -func (r *ScanReconciler) checkIfReadOnlyHookIsCompleted(scan *executionv1.Scan) error { - ctx := context.Background() - readOnlyHookCompletion, err := r.checkIfJobIsCompleted(scan, client.MatchingLabels{"securecodebox.io/job-type": "read-only-hook"}) +func (r *ScanReconciler) processInProgressHook(scan *executionv1.Scan, status *executionv1.HookStatus, jobType string) error { + jobStatus, err := r.checkIfJobIsCompleted(scan, client.MatchingLabels{ + "securecodebox.io/job-type": jobType, + "securecodebox.io/hook-name": status.HookName, + }) if err != nil { + r.Log.Error(err, "Failed to check job status for Hook") return err } - - if readOnlyHookCompletion == completed { - r.Log.V(7).Info("All ReadOnlyHooks have completed") - scan.Status.State = "Done" - var now metav1.Time = metav1.Now() - scan.Status.FinishedAt = &now - if err := r.Status().Update(ctx, scan); err != nil { - r.Log.Error(err, "Unable to update Scan status") - return err - } - } else if readOnlyHookCompletion == failed { - r.Log.Info("At least one ReadOnlyHook failed") - scan.Status.State = "Errored" - scan.Status.ErrorDescription = "At least one ReadOnlyHook failed, check the hooks kubernetes jobs related to the scan for more details." - if err := r.Status().Update(ctx, scan); err != nil { - r.Log.Error(err, "Unable to update Scan status") - return err + switch jobStatus { + case completed: + // Job is completed => set current Hook to completed + status.State = executionv1.Completed + case incomplete: + // Still waiting for job to finish + case failed: + if status.State == executionv1.Pending { + status.State = executionv1.Cancelled + } else { + status.State = executionv1.Failed } } - - // ReadOnlyHook(s) are still running. At least some of them are. - // Waiting until all are done. return nil } @@ -470,20 +413,6 @@ func (r *ScanReconciler) createJobForHook(hook *executionv1.ScanCompletionHook, return job.Name, nil } -func (r *ScanReconciler) updateHookStatus(scan *executionv1.Scan, hookStatus executionv1.HookStatus) error { - for i, hook := range scan.Status.ReadAndWriteHookStatus { - if hook.HookName == hookStatus.HookName { - scan.Status.ReadAndWriteHookStatus[i] = hookStatus - break - } - } - if err := r.Status().Update(context.Background(), scan); err != nil { - r.Log.Error(err, "unable to update Scan status") - return err - } - return nil -} - func (r *ScanReconciler) getLabelSelector(scan *executionv1.Scan) (labels.Selector, error) { hookSelector := scan.Spec.HookSelector if hookSelector == nil { diff --git a/operator/controllers/execution/scans/scan_controller.go b/operator/controllers/execution/scans/scan_controller.go index 60a4f40967..82d4ccd262 100644 --- a/operator/controllers/execution/scans/scan_controller.go +++ b/operator/controllers/execution/scans/scan_controller.go @@ -81,6 +81,12 @@ func (r *ScanReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. // Handle Finalizer if the scan is getting deleted if !scan.ObjectMeta.DeletionTimestamp.IsZero() { + // Check if this Scan has not yet been converted to new CRD + if scan.Status.OrderedHookStatuses == nil && scan.Status.ReadAndWriteHookStatus != nil && scan.Status.State == "Done" { + if err := r.migrateHookStatus(&scan); err != nil { + return ctrl.Result{}, err + } + } if err := r.handleFinalizer(&scan); err != nil { r.Log.Error(err, "Failed to run Scan Finalizer") return ctrl.Result{}, err @@ -99,12 +105,14 @@ func (r *ScanReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. err = r.checkIfParsingIsCompleted(&scan) case "ParseCompleted": err = r.setHookStatus(&scan) + case "HookProcessing": + err = r.executeHooks(&scan) case "ReadAndWriteHookProcessing": - err = r.executeReadAndWriteHooks(&scan) + fallthrough case "ReadAndWriteHookCompleted": - err = r.startReadOnlyHooks(&scan) + fallthrough case "ReadOnlyHookProcessing": - err = r.checkIfReadOnlyHookIsCompleted(&scan) + err = r.migrateHookStatus(&scan) } if err != nil { return ctrl.Result{}, err diff --git a/operator/crds/execution.securecodebox.io_scancompletionhooks.yaml b/operator/crds/execution.securecodebox.io_scancompletionhooks.yaml index 5758ef4520..2179549b24 100644 --- a/operator/crds/execution.securecodebox.io_scancompletionhooks.yaml +++ b/operator/crds/execution.securecodebox.io_scancompletionhooks.yaml @@ -24,6 +24,10 @@ spec: jsonPath: .spec.type name: Type type: string + - description: ScanCompletionHook Priority + jsonPath: .spec.priority + name: Priority + type: string - description: ScanCompletionHook Image jsonPath: .spec.image name: Image @@ -174,6 +178,13 @@ spec: type: string type: object type: array + priority: + default: 0 + description: Higher priority hooks run before low priority hooks. + Within a priority class ReadAndWrite hooks are started before ReadOnly + hooks, ReadAndWrite hooks wil be launched in serial, and ReadOnly + hooks will be launched in parallel. + type: integer serviceAccountName: description: ServiceAccountName Name of the serviceAccount Name used. Should only be used if your hook needs specifc RBAC Access. Otherwise diff --git a/operator/crds/execution.securecodebox.io_scans.yaml b/operator/crds/execution.securecodebox.io_scans.yaml index 3e24c3b2b0..4ae2372436 100644 --- a/operator/crds/execution.securecodebox.io_scans.yaml +++ b/operator/crds/execution.securecodebox.io_scans.yaml @@ -2906,6 +2906,32 @@ spec: parser & hooks) has been marked as "Done" format: date-time type: string + orderedHookStatuses: + items: + items: + properties: + hookName: + type: string + jobName: + type: string + priority: + type: integer + state: + description: HookState Describes the State of a Hook on a + Scan + type: string + type: + description: HookType Defines weather the hook should be able + to change the findings or is run in a read only mode. + type: string + required: + - hookName + - priority + - state + - type + type: object + type: array + type: array rawResultDownloadLink: description: RawResultDownloadLink link to download the raw result file from. Valid for 7 days @@ -2929,12 +2955,20 @@ spec: type: string jobName: type: string + priority: + type: integer state: description: HookState Describes the State of a Hook on a Scan type: string + type: + description: HookType Defines weather the hook should be able + to change the findings or is run in a read only mode. + type: string required: - hookName + - priority - state + - type type: object type: array state: diff --git a/operator/utils/orderedhookgroups.go b/operator/utils/orderedhookgroups.go new file mode 100644 index 0000000000..f6be0a6df5 --- /dev/null +++ b/operator/utils/orderedhookgroups.go @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: 2021 iteratec GmbH +// +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "fmt" + "sort" + + executionv1 "github.com/secureCodeBox/secureCodeBox/operator/apis/execution/v1" +) + +func CurrentHookGroup(orderedHookGroup [][]*executionv1.HookStatus) (error, []*executionv1.HookStatus) { + for _, group := range orderedHookGroup { + for _, hookStatus := range group { + switch hookStatus.State { + case executionv1.Pending: + return nil, group + case executionv1.InProgress: + return nil, group + case executionv1.Failed: + return fmt.Errorf("Hook %s failed to be executed.", hookStatus.HookName), nil + case executionv1.Cancelled: + return fmt.Errorf("Hook %s was cancelled while it was executed.", hookStatus.HookName), nil + case executionv1.Completed: + // continue to next group + } + } + } + + return nil, nil +} + +func FromUnorderedList(hooks []executionv1.ScanCompletionHook) [][]*executionv1.HookStatus { + // convert ScanCompletionHook objects to HookStatus objects + hookStatuses := mapHookToHookStatus(hooks) + + // Group hookStatuses into a map by their prio class + hooksByPrioClass := map[int][]*executionv1.HookStatus{} + // keep a list of existing classes + prioClasses := []int{} + for _, hookStatus := range hookStatuses { + prio := hookStatus.Priority + + if _, ok := hooksByPrioClass[prio]; ok { + hooksByPrioClass[prio] = append(hooksByPrioClass[prio], hookStatus) + } else { + hooksByPrioClass[prio] = []*executionv1.HookStatus{hookStatus} + prioClasses = append(prioClasses, prio) + } + } + + // sort prio classes in decending order + sort.Slice(prioClasses, func(i, j int) bool { + return prioClasses[i] > prioClasses[j] + }) + + groups := [][]*executionv1.HookStatus{} + for _, prioClass := range prioClasses { + groups = append(groups, OrderHookStatusesInsideAPrioClass(hooksByPrioClass[prioClass])...) + } + + return groups +} + +func mapHookToHookStatus(hooks []executionv1.ScanCompletionHook) []*executionv1.HookStatus { + hookStatuses := []*executionv1.HookStatus{} + + for _, hook := range hooks { + hookStatuses = append(hookStatuses, &executionv1.HookStatus{ + HookName: hook.Name, + State: executionv1.Pending, + Priority: hook.Spec.Priority, + Type: hook.Spec.Type, + }) + } + + return hookStatuses +} + +func OrderHookStatusesInsideAPrioClass(hookStatuses []*executionv1.HookStatus) [][]*executionv1.HookStatus { + groups := [][]*executionv1.HookStatus{} + readOnlyGroups := []*executionv1.HookStatus{} + for _, hookStatus := range hookStatuses { + switch hookStatus.Type { + case executionv1.ReadAndWrite: + groups = append(groups, []*executionv1.HookStatus{ + hookStatus, + }) + case executionv1.ReadOnly: + readOnlyGroups = append(readOnlyGroups, hookStatus) + } + } + + // Append the ReadOnly Hook Group at the end if existent + if len(readOnlyGroups) != 0 { + groups = append(groups, readOnlyGroups) + } + + return groups +} diff --git a/operator/utils/orderedhookgroups_test.go b/operator/utils/orderedhookgroups_test.go new file mode 100644 index 0000000000..acc5bd42ab --- /dev/null +++ b/operator/utils/orderedhookgroups_test.go @@ -0,0 +1,245 @@ +// SPDX-FileCopyrightText: 2021 iteratec GmbH +// +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + executionv1 "github.com/secureCodeBox/secureCodeBox/operator/apis/execution/v1" + corev1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func createHook(name string, hookType executionv1.HookType, prio int) executionv1.ScanCompletionHook { + return executionv1.ScanCompletionHook{ + ObjectMeta: corev1.ObjectMeta{ + Name: name, + Namespace: "default", + }, + Spec: executionv1.ScanCompletionHookSpec{ + Type: hookType, + Priority: prio, + }, + } +} + +var _ = Describe("HookOrderingGroup Creation", func() { + Context("HookOrderingGroup Creation / Sorting (Single Prio)", func() { + It("Should always place ReadAndWrite Hooks into different Groups", func() { + hooks := []executionv1.ScanCompletionHook{ + createHook("rw-1", executionv1.ReadAndWrite, 0), + createHook("rw-2", executionv1.ReadAndWrite, 0), + } + + orderedHookGroups := FromUnorderedList(hooks) + + Expect(orderedHookGroups).To(HaveLen(2), "Should create two groups") + + Expect(orderedHookGroups[0]).To(HaveLen(1), "groups should contain one entry") + Expect(orderedHookGroups[1]).To(HaveLen(1), "groups should contain one entry") + }) + + It("Should place place ReadOnly Hooks into the same groups", func() { + hooks := []executionv1.ScanCompletionHook{ + createHook("ro-1", executionv1.ReadOnly, 0), + createHook("ro-2", executionv1.ReadOnly, 0), + } + + orderedHookGroups := FromUnorderedList(hooks) + + Expect(orderedHookGroups).To(HaveLen(1)) + Expect(orderedHookGroups[0]).To(HaveLen(2)) + }) + + It("Should handle mixed hook types", func() { + hooks := []executionv1.ScanCompletionHook{ + createHook("rw-1", executionv1.ReadAndWrite, 0), + createHook("ro-1", executionv1.ReadOnly, 0), + createHook("rw-2", executionv1.ReadAndWrite, 0), + createHook("ro-2", executionv1.ReadOnly, 0), + } + + orderedHookGroups := FromUnorderedList(hooks) + + Expect(len(orderedHookGroups)).To(Equal(3)) + + Expect(len(orderedHookGroups[0])).To(Equal(1)) + Expect(len(orderedHookGroups[1])).To(Equal(1)) + Expect(len(orderedHookGroups[2])).To(Equal(2)) + }) + }) + + Context("HookOrderingGroup Creation / Sorting (Different Priorities)", func() { + It("Should always place ReadAndWrite Hooks into different Groups", func() { + hooks := []executionv1.ScanCompletionHook{ + createHook("rw-1", executionv1.ReadAndWrite, 0), + createHook("rw-2", executionv1.ReadAndWrite, 1), + } + + orderedHookGroups := FromUnorderedList(hooks) + + Expect(orderedHookGroups).To(HaveLen(2), "Should create two groups") + + Expect(orderedHookGroups).To(Equal([][]*executionv1.HookStatus{ + { + {HookName: "rw-2", State: "Pending", JobName: "", Priority: 1, Type: "ReadAndWrite"}, + }, + { + {HookName: "rw-1", State: "Pending", JobName: "", Priority: 0, Type: "ReadAndWrite"}, + }, + })) + }) + + It("Should order ro hooks properly when they have different priorities", func() { + hooks := []executionv1.ScanCompletionHook{ + createHook("ro-1", executionv1.ReadOnly, 4), + createHook("ro-2", executionv1.ReadOnly, 2), + createHook("ro-3", executionv1.ReadOnly, 3), + createHook("ro-4", executionv1.ReadOnly, 1), + } + + orderedHookGroups := FromUnorderedList(hooks) + + Expect(orderedHookGroups).To(Equal([][]*executionv1.HookStatus{ + { + {HookName: "ro-1", State: "Pending", JobName: "", Priority: 4, Type: "ReadOnly"}, + }, + { + {HookName: "ro-3", State: "Pending", JobName: "", Priority: 3, Type: "ReadOnly"}, + }, + { + {HookName: "ro-2", State: "Pending", JobName: "", Priority: 2, Type: "ReadOnly"}, + }, + { + {HookName: "ro-4", State: "Pending", JobName: "", Priority: 1, Type: "ReadOnly"}, + }, + })) + }) + + It("Should order a mix of ro & rw hooks properly when they have different priorities", func() { + hooks := []executionv1.ScanCompletionHook{ + createHook("ro-1", executionv1.ReadOnly, 4), + createHook("ro-2", executionv1.ReadOnly, 4), + createHook("rw-1", executionv1.ReadAndWrite, 4), + createHook("rw-2", executionv1.ReadAndWrite, 4), + createHook("ro-3", executionv1.ReadOnly, 2), + createHook("rw-3", executionv1.ReadAndWrite, 2), + createHook("ro-4", executionv1.ReadOnly, 3), + createHook("ro-5", executionv1.ReadOnly, 1), + } + + orderedHookGroups := FromUnorderedList(hooks) + + Expect(orderedHookGroups).To(Equal([][]*executionv1.HookStatus{ + { + {HookName: "rw-1", State: "Pending", JobName: "", Priority: 4, Type: "ReadAndWrite"}, + }, + { + {HookName: "rw-2", State: "Pending", JobName: "", Priority: 4, Type: "ReadAndWrite"}, + }, + { + {HookName: "ro-1", State: "Pending", JobName: "", Priority: 4, Type: "ReadOnly"}, + {HookName: "ro-2", State: "Pending", JobName: "", Priority: 4, Type: "ReadOnly"}, + }, + { + {HookName: "ro-4", State: "Pending", JobName: "", Priority: 3, Type: "ReadOnly"}, + }, + { + {HookName: "rw-3", State: "Pending", JobName: "", Priority: 2, Type: "ReadAndWrite"}, + }, + { + {HookName: "ro-3", State: "Pending", JobName: "", Priority: 2, Type: "ReadOnly"}, + }, + { + {HookName: "ro-5", State: "Pending", JobName: "", Priority: 1, Type: "ReadOnly"}, + }, + })) + }) + }) +}) + +var _ = Describe("HookOrderingGroup Retrival", func() { + Context("Current() should return the group of hooks which should be executed at the moment", func() { + It("Should return the first if all hooks are pending", func() { + err, currentHookGroup := CurrentHookGroup([][]*executionv1.HookStatus{ + { + {HookName: "rw-1", State: "Pending", JobName: "", Priority: 4, Type: "ReadAndWrite"}, + }, + { + {HookName: "ro-1", State: "Pending", JobName: "", Priority: 4, Type: "ReadOnly"}, + {HookName: "ro-2", State: "Pending", JobName: "", Priority: 4, Type: "ReadOnly"}, + }, + }) + + Expect(err).To(BeNil()) + Expect(currentHookGroup).To(Equal( + []*executionv1.HookStatus{ + {HookName: "rw-1", State: "Pending", JobName: "", Priority: 4, Type: "ReadAndWrite"}, + }, + )) + }) + + It("Should return the first group if it consists of hooks currently in progress", func() { + err, currentHookGroup := CurrentHookGroup([][]*executionv1.HookStatus{ + { + {HookName: "rw-1", State: "InProgress", JobName: "", Priority: 4, Type: "ReadAndWrite"}, + }, + { + {HookName: "ro-1", State: "Pending", JobName: "", Priority: 4, Type: "ReadOnly"}, + {HookName: "ro-2", State: "Pending", JobName: "", Priority: 4, Type: "ReadOnly"}, + }, + }) + + Expect(err).To(BeNil()) + Expect(currentHookGroup).To(Equal( + []*executionv1.HookStatus{ + {HookName: "rw-1", State: "InProgress", JobName: "", Priority: 4, Type: "ReadAndWrite"}, + }, + )) + }) + + It("Should return the second group if the first group is completed", func() { + err, currentHookGroup := CurrentHookGroup([][]*executionv1.HookStatus{ + { + {HookName: "rw-1", State: "Completed", JobName: "", Priority: 4, Type: "ReadAndWrite"}, + }, + { + {HookName: "ro-1", State: "Pending", JobName: "", Priority: 4, Type: "ReadOnly"}, + {HookName: "ro-2", State: "Pending", JobName: "", Priority: 4, Type: "ReadOnly"}, + }, + }) + + Expect(err).To(BeNil()) + Expect(currentHookGroup).To(Equal( + []*executionv1.HookStatus{ + {HookName: "ro-1", State: "Pending", JobName: "", Priority: 4, Type: "ReadOnly"}, + {HookName: "ro-2", State: "Pending", JobName: "", Priority: 4, Type: "ReadOnly"}, + }, + )) + }) + + It("Should return nil if the first group failed", func() { + err, currentHookGroup := CurrentHookGroup([][]*executionv1.HookStatus{ + { + {HookName: "rw-1", State: "Failed", JobName: "", Priority: 4, Type: "ReadAndWrite"}, + }, + { + {HookName: "ro-1", State: "Pending", JobName: "", Priority: 4, Type: "ReadOnly"}, + {HookName: "ro-2", State: "Pending", JobName: "", Priority: 4, Type: "ReadOnly"}, + }, + }) + + Expect(err).To(MatchError("Hook rw-1 failed to be executed.")) + Expect(currentHookGroup).To(BeNil()) + }) + + It("Should return nil if no hooks are configured", func() { + err, currentHookGroup := CurrentHookGroup([][]*executionv1.HookStatus{}) + + Expect(err).To(BeNil()) + Expect(currentHookGroup).To(BeNil()) + }) + }) +}) diff --git a/operator/utils/suite_test.go b/operator/utils/suite_test.go new file mode 100644 index 0000000000..74d032e36f --- /dev/null +++ b/operator/utils/suite_test.go @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2021 iteratec GmbH +// +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestGinko(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, + "Utils Suite", + ) +}