From 344e87a6eab68e33092d639e2311efd1817a20f2 Mon Sep 17 00:00:00 2001 From: Sean Newman Date: Wed, 8 Sep 2021 23:14:19 -0400 Subject: [PATCH 1/4] Clean up remaining bash templating issues --- dynamicbeat/pkg/assets/assets.go | 26 +++++++++++++++++++ dynamicbeat/pkg/assets/dashboards/all.go | 4 +-- .../pkg/assets/dashboards/team-overview.json | 4 +-- dynamicbeat/pkg/setup/kibana.go | 3 ++- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/dynamicbeat/pkg/assets/assets.go b/dynamicbeat/pkg/assets/assets.go index e7f07885..eca61d82 100644 --- a/dynamicbeat/pkg/assets/assets.go +++ b/dynamicbeat/pkg/assets/assets.go @@ -45,3 +45,29 @@ func ReadTeam(filename string, name string) io.Reader { return bytes.NewReader(buf.Bytes()) } + +func ReadTeamOverview(filename string, name string, checks int) io.Reader { + data, err := f.ReadFile(filename) + if err != nil { + zap.S().Panicf("failed to read embedded asset %s: %s", filename, err) + } + + // Template in the team name and the number of checks per team + vars := struct { + Team string + Checks int + }{name, checks} + tmpl, err := template.New("").Parse(string(data)) + if err != nil { + zap.S().Panicf("failed to read asset %s as template: %s", filename, err) + } + + // Apply the template and write to a byte buffer + var buf bytes.Buffer + err = tmpl.Execute(&buf, vars) + if err != nil { + zap.S().Panicf("failed to template team %s into asset %s: %s", name, filename, err) + } + + return bytes.NewReader(buf.Bytes()) +} diff --git a/dynamicbeat/pkg/assets/dashboards/all.go b/dynamicbeat/pkg/assets/dashboards/all.go index 4972d725..cdfba239 100644 --- a/dynamicbeat/pkg/assets/dashboards/all.go +++ b/dynamicbeat/pkg/assets/dashboards/all.go @@ -10,8 +10,8 @@ func Scoreboard() io.Reader { return assets.Read("dashboards/scoreboard.json") } -func TeamOverview(name string) func() io.Reader { +func TeamOverview(name string, checks int) func() io.Reader { return func() io.Reader { - return assets.ReadTeam("dashboards/team-overview.json", name) + return assets.ReadTeamOverview("dashboards/team-overview.json", name, checks) } } diff --git a/dynamicbeat/pkg/assets/dashboards/team-overview.json b/dynamicbeat/pkg/assets/dashboards/team-overview.json index cf5ea668..bb00190c 100644 --- a/dynamicbeat/pkg/assets/dashboards/team-overview.json +++ b/dynamicbeat/pkg/assets/dashboards/team-overview.json @@ -73,7 +73,7 @@ "version": "Wzg4LDJd", "attributes": { "title": "Check Results Overview - {{.Team}}", - "visState": "{\"title\":\"Check Results Overview - {{.Team}}\",\"type\":\"table\",\"params\":{\"perPage\":${CHECKS},\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"string\"},\"params\":{},\"aggType\":\"top_hits\"},{\"accessor\":2,\"format\":{\"id\":\"boolean\"},\"params\":{},\"aggType\":\"top_hits\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"passed\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Passing\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"name.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":200,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Check Name\"}},{\"id\":\"1\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"message\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Message\"}}]}", + "visState": "{\"title\":\"Check Results Overview - {{.Team}}\",\"type\":\"table\",\"params\":{\"perPage\":{{.Checks}},\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"string\"},\"params\":{},\"aggType\":\"top_hits\"},{\"accessor\":2,\"format\":{\"id\":\"boolean\"},\"params\":{},\"aggType\":\"top_hits\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"passed\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Passing\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"name.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":200,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Check Name\"}},{\"id\":\"1\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"message\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Message\"}}]}", "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", "description": "", "version": 1, @@ -98,7 +98,7 @@ "updated_at": "2020-02-18T11:25:48.753Z", "version": "WzgxLDJd", "attributes": { - "title": "${INDEX}", + "title": "results-{{.Team}}", "timeFieldName": "@timestamp", "fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"check_type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"check_type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"check_type\",\"subType\":\"multi\"},{\"name\":\"group\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"group.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"group\",\"subType\":\"multi\"},{\"name\":\"id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"id\",\"subType\":\"multi\"},{\"name\":\"message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"message\",\"subType\":\"multi\"},{\"name\":\"name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"name\",\"subType\":\"multi\"},{\"name\":\"passed\",\"type\":\"boolean\",\"esTypes\":[\"boolean\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"passed_int\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"score_weight\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"type\",\"subType\":\"multi\"}]" }, diff --git a/dynamicbeat/pkg/setup/kibana.go b/dynamicbeat/pkg/setup/kibana.go index 5f2436cb..68477bd6 100644 --- a/dynamicbeat/pkg/setup/kibana.go +++ b/dynamicbeat/pkg/setup/kibana.go @@ -92,7 +92,8 @@ func Kibana(host string, user string, pass string, verify bool, teams []config.T zap.S().Errorf("failed to add role for %s: %s", team.Name, err) } - err = c.AddDashboard(dashboards.TeamOverview(team.Name)) + // TODO: don't hardcode the number of rows in the table + err = c.AddDashboard(dashboards.TeamOverview(team.Name, 20)) if err != nil { zap.S().Errorf("failed to add team overview dashboard for %s: %s", team.Name, err) } From 4e66113ad5e15c1b115b0951438640fc5dedda0b Mon Sep 17 00:00:00 2001 From: Sean Newman Date: Sun, 12 Sep 2021 22:57:51 -0400 Subject: [PATCH 2/4] Remember to add generic checks --- dynamicbeat/pkg/check/check.go | 21 +++++++++++++++------ dynamicbeat/pkg/setup/checks.go | 3 ++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/dynamicbeat/pkg/check/check.go b/dynamicbeat/pkg/check/check.go index 4e099298..8f496911 100644 --- a/dynamicbeat/pkg/check/check.go +++ b/dynamicbeat/pkg/check/check.go @@ -57,11 +57,11 @@ func (v ValidationError) Error() string { return fmt.Sprintf("Error: check (Type: `%s`, ID: `%s`) is missing value for required field `%s`", v.Type, v.ID, v.Field) } -func (c *Config) Documents() (io.Reader, io.Reader, io.Reader, error) { +func (c *Config) Documents() (io.Reader, io.Reader, io.Reader, io.Reader, error) { def := make(map[string]interface{}) err := json.Unmarshal(c.Definition, &def) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to unmarshal definition for '%s': %s", c.ID, err) + return nil, nil, nil, nil, fmt.Errorf("failed to unmarshal definition for '%s': %s", c.ID, err) } // The check definition document doesn't include the attributes @@ -71,20 +71,29 @@ func (c *Config) Documents() (io.Reader, io.Reader, io.Reader, error) { }{c.Metadata, def} checkDoc, err := json.Marshal(chk) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to marshal definition for '%s': %s", c.ID, err) + return nil, nil, nil, nil, fmt.Errorf("failed to marshal definition for '%s': %s", c.ID, err) + } + + // The generic check definition only includes the metadata + generic := struct { + Metadata + }{c.Metadata} + genericDoc, err := json.Marshal(generic) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("failed to marshal generic definition for '%s': %s", c.ID, err) } admin, err := attributeDoc(c.Attributes.Admin) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to marshal admin attributes for '%s': %s", c.ID, err) + return nil, nil, nil, nil, fmt.Errorf("failed to marshal admin attributes for '%s': %s", c.ID, err) } user, err := attributeDoc(c.Attributes.User) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to marshal user attributes for '%s': %s", c.ID, err) + return nil, nil, nil, nil, fmt.Errorf("failed to marshal user attributes for '%s': %s", c.ID, err) } - return bytes.NewReader(checkDoc), admin, user, nil + return bytes.NewReader(checkDoc), bytes.NewReader(genericDoc), admin, user, nil } func attributeDoc(attributes map[string]string) (io.Reader, error) { diff --git a/dynamicbeat/pkg/setup/checks.go b/dynamicbeat/pkg/setup/checks.go index bcdddfba..6e9ccb49 100644 --- a/dynamicbeat/pkg/setup/checks.go +++ b/dynamicbeat/pkg/setup/checks.go @@ -26,12 +26,13 @@ func Checks(c *esclient.Client, f *checksource.Filesystem) error { } for _, def := range defs { - chk, admin, user, err := def.Documents() + chk, generic, admin, user, err := def.Documents() if err != nil { zap.S().Errorf("skipping check due to error - %s", err) } queueItem(indexer, "checkdef", def.ID, chk) + queueItem(indexer, "checks", def.ID, generic) if admin != nil { queueItem(indexer, fmt.Sprintf("attrib_admin_%s", def.Group), def.ID, admin) } From a89a5dac1dc559f2fe25fc179c8537dba00f5be2 Mon Sep 17 00:00:00 2001 From: Sean Newman Date: Sun, 12 Sep 2021 23:14:37 -0400 Subject: [PATCH 3/4] Update CHANGELOG --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 540e68ee..b8b65635 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,13 @@ Each section organizes entries into the following subsections: [Unreleased] ------------ +### Dynamicbeat + +#### Fixed + +- Replace remaining `${VAR}` template strings with `{{.Var}}` templates in saved objects (#321) +- Ensure generic check definitions are added by `dynamicbeat setup checks` (#321) + [0.8.0] - 2021-08-10 -------------------- From 244dd0676ee3205d1eda1e3e57b6e51ccee4759e Mon Sep 17 00:00:00 2001 From: Sean Newman Date: Sun, 12 Sep 2021 23:25:08 -0400 Subject: [PATCH 4/4] Bump versions --- CHANGELOG.md | 8 +++++++- deployment/medium/ansible/playbook.yml | 2 +- deployment/small/docker/setup.sh | 2 +- kibana-plugin/package.json | 4 ++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8b65635..a057c137 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,11 @@ Each section organizes entries into the following subsections: [Unreleased] ------------ +[0.8.1] - 2021-09-12 +-------------------- + +This release fixes a few bugs in Dynamicbeat related to the new deployment and check-adding processes. + ### Dynamicbeat #### Fixed @@ -385,7 +390,8 @@ This release is in preparation for ISTS 2020. The initial release of Scorestack. -[Unreleased]: https://github.com/scorestack/scorestack/compare/v0.8.0...dev +[Unreleased]: https://github.com/scorestack/scorestack/compare/v0.8.1...dev +[0.8.1]: https://github.com/scorestack/scorestack/compare/v0.8.0...v0.8.1 [0.8.0]: https://github.com/scorestack/scorestack/compare/v0.7.0...v0.8.0 [0.7.0]: https://github.com/scorestack/scorestack/compare/v0.6.2...v0.7.0 [0.6.2]: https://github.com/scorestack/scorestack/compare/v0.6.1...v0.6.2 diff --git a/deployment/medium/ansible/playbook.yml b/deployment/medium/ansible/playbook.yml index a1b6332f..e868f467 100644 --- a/deployment/medium/ansible/playbook.yml +++ b/deployment/medium/ansible/playbook.yml @@ -247,7 +247,7 @@ - Restart Kibana - name: Install plugin - command: /usr/share/kibana/bin/kibana-plugin --allow-root install https://github.com/scorestack/scorestack/releases/download/v0.8.0/kibana-plugin-v0.8.0.zip + command: /usr/share/kibana/bin/kibana-plugin --allow-root install https://github.com/scorestack/scorestack/releases/download/v0.8.1/kibana-plugin-v0.8.1.zip args: creates: /usr/share/kibana/plugins/scorestack notify: diff --git a/deployment/small/docker/setup.sh b/deployment/small/docker/setup.sh index 0c225337..72564b38 100755 --- a/deployment/small/docker/setup.sh +++ b/deployment/small/docker/setup.sh @@ -42,7 +42,7 @@ docker exec ${KIBANA_CONTAINER} bin/kibana-keystore create docker exec ${KIBANA_CONTAINER} /bin/bash -c "bin/kibana-keystore add elasticsearch.password --stdin <<< '${kibana_pass}'" # Install kibana plugin -docker exec ${KIBANA_CONTAINER} /bin/bash -c "bin/kibana-plugin install https://github.com/scorestack/scorestack/releases/download/v0.8.0/kibana-plugin-v0.8.0.zip" +docker exec ${KIBANA_CONTAINER} /bin/bash -c "bin/kibana-plugin install https://github.com/scorestack/scorestack/releases/download/v0.8.1/kibana-plugin-v0.8.1.zip" # Restart kibana to reload credentials from keystore cd config diff --git a/kibana-plugin/package.json b/kibana-plugin/package.json index 5bba2bbb..95b2b75e 100644 --- a/kibana-plugin/package.json +++ b/kibana-plugin/package.json @@ -1,11 +1,11 @@ { "name": "scorestack", - "version": "0.8.0", + "version": "0.8.1", "description": "A Kibana plugin for viewing and modifying Scorestack checks and attributes.", "private": true, "scripts": { "prebuild": "node ../../scripts/build_kibana_platform_plugins --verbose --dist --filter scorestack", - "build": "plugin-helpers build --build-version 0.8.0 --kibana-version 7.9.2", + "build": "plugin-helpers build --build-version 0.8.1 --kibana-version 7.9.2", "kbn": "node ../../scripts/kbn", "lint": "eslint --ignore-path .gitignore --color .", "plugin-helpers": "plugin-helpers",