From 344e87a6eab68e33092d639e2311efd1817a20f2 Mon Sep 17 00:00:00 2001 From: Sean Newman Date: Wed, 8 Sep 2021 23:14:19 -0400 Subject: [PATCH 1/3] Clean up remaining bash templating issues --- dynamicbeat/pkg/assets/assets.go | 26 +++++++++++++++++++ dynamicbeat/pkg/assets/dashboards/all.go | 4 +-- .../pkg/assets/dashboards/team-overview.json | 4 +-- dynamicbeat/pkg/setup/kibana.go | 3 ++- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/dynamicbeat/pkg/assets/assets.go b/dynamicbeat/pkg/assets/assets.go index e7f07885..eca61d82 100644 --- a/dynamicbeat/pkg/assets/assets.go +++ b/dynamicbeat/pkg/assets/assets.go @@ -45,3 +45,29 @@ func ReadTeam(filename string, name string) io.Reader { return bytes.NewReader(buf.Bytes()) } + +func ReadTeamOverview(filename string, name string, checks int) io.Reader { + data, err := f.ReadFile(filename) + if err != nil { + zap.S().Panicf("failed to read embedded asset %s: %s", filename, err) + } + + // Template in the team name and the number of checks per team + vars := struct { + Team string + Checks int + }{name, checks} + tmpl, err := template.New("").Parse(string(data)) + if err != nil { + zap.S().Panicf("failed to read asset %s as template: %s", filename, err) + } + + // Apply the template and write to a byte buffer + var buf bytes.Buffer + err = tmpl.Execute(&buf, vars) + if err != nil { + zap.S().Panicf("failed to template team %s into asset %s: %s", name, filename, err) + } + + return bytes.NewReader(buf.Bytes()) +} diff --git a/dynamicbeat/pkg/assets/dashboards/all.go b/dynamicbeat/pkg/assets/dashboards/all.go index 4972d725..cdfba239 100644 --- a/dynamicbeat/pkg/assets/dashboards/all.go +++ b/dynamicbeat/pkg/assets/dashboards/all.go @@ -10,8 +10,8 @@ func Scoreboard() io.Reader { return assets.Read("dashboards/scoreboard.json") } -func TeamOverview(name string) func() io.Reader { +func TeamOverview(name string, checks int) func() io.Reader { return func() io.Reader { - return assets.ReadTeam("dashboards/team-overview.json", name) + return assets.ReadTeamOverview("dashboards/team-overview.json", name, checks) } } diff --git a/dynamicbeat/pkg/assets/dashboards/team-overview.json b/dynamicbeat/pkg/assets/dashboards/team-overview.json index cf5ea668..bb00190c 100644 --- a/dynamicbeat/pkg/assets/dashboards/team-overview.json +++ b/dynamicbeat/pkg/assets/dashboards/team-overview.json @@ -73,7 +73,7 @@ "version": "Wzg4LDJd", "attributes": { "title": "Check Results Overview - {{.Team}}", - "visState": "{\"title\":\"Check Results Overview - {{.Team}}\",\"type\":\"table\",\"params\":{\"perPage\":${CHECKS},\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"string\"},\"params\":{},\"aggType\":\"top_hits\"},{\"accessor\":2,\"format\":{\"id\":\"boolean\"},\"params\":{},\"aggType\":\"top_hits\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"passed\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Passing\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"name.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":200,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Check Name\"}},{\"id\":\"1\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"message\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Message\"}}]}", + "visState": "{\"title\":\"Check Results Overview - {{.Team}}\",\"type\":\"table\",\"params\":{\"perPage\":{{.Checks}},\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"string\"},\"params\":{},\"aggType\":\"top_hits\"},{\"accessor\":2,\"format\":{\"id\":\"boolean\"},\"params\":{},\"aggType\":\"top_hits\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"passed\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Passing\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"name.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":200,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Check Name\"}},{\"id\":\"1\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"message\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Message\"}}]}", "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", "description": "", "version": 1, @@ -98,7 +98,7 @@ "updated_at": "2020-02-18T11:25:48.753Z", "version": "WzgxLDJd", "attributes": { - "title": "${INDEX}", + "title": "results-{{.Team}}", "timeFieldName": "@timestamp", "fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"check_type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"check_type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"check_type\",\"subType\":\"multi\"},{\"name\":\"group\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"group.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"group\",\"subType\":\"multi\"},{\"name\":\"id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"id\",\"subType\":\"multi\"},{\"name\":\"message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"message\",\"subType\":\"multi\"},{\"name\":\"name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"name\",\"subType\":\"multi\"},{\"name\":\"passed\",\"type\":\"boolean\",\"esTypes\":[\"boolean\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"passed_int\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"score_weight\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"type\",\"subType\":\"multi\"}]" }, diff --git a/dynamicbeat/pkg/setup/kibana.go b/dynamicbeat/pkg/setup/kibana.go index 5f2436cb..68477bd6 100644 --- a/dynamicbeat/pkg/setup/kibana.go +++ b/dynamicbeat/pkg/setup/kibana.go @@ -92,7 +92,8 @@ func Kibana(host string, user string, pass string, verify bool, teams []config.T zap.S().Errorf("failed to add role for %s: %s", team.Name, err) } - err = c.AddDashboard(dashboards.TeamOverview(team.Name)) + // TODO: don't hardcode the number of rows in the table + err = c.AddDashboard(dashboards.TeamOverview(team.Name, 20)) if err != nil { zap.S().Errorf("failed to add team overview dashboard for %s: %s", team.Name, err) } From 4e66113ad5e15c1b115b0951438640fc5dedda0b Mon Sep 17 00:00:00 2001 From: Sean Newman Date: Sun, 12 Sep 2021 22:57:51 -0400 Subject: [PATCH 2/3] Remember to add generic checks --- dynamicbeat/pkg/check/check.go | 21 +++++++++++++++------ dynamicbeat/pkg/setup/checks.go | 3 ++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/dynamicbeat/pkg/check/check.go b/dynamicbeat/pkg/check/check.go index 4e099298..8f496911 100644 --- a/dynamicbeat/pkg/check/check.go +++ b/dynamicbeat/pkg/check/check.go @@ -57,11 +57,11 @@ func (v ValidationError) Error() string { return fmt.Sprintf("Error: check (Type: `%s`, ID: `%s`) is missing value for required field `%s`", v.Type, v.ID, v.Field) } -func (c *Config) Documents() (io.Reader, io.Reader, io.Reader, error) { +func (c *Config) Documents() (io.Reader, io.Reader, io.Reader, io.Reader, error) { def := make(map[string]interface{}) err := json.Unmarshal(c.Definition, &def) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to unmarshal definition for '%s': %s", c.ID, err) + return nil, nil, nil, nil, fmt.Errorf("failed to unmarshal definition for '%s': %s", c.ID, err) } // The check definition document doesn't include the attributes @@ -71,20 +71,29 @@ func (c *Config) Documents() (io.Reader, io.Reader, io.Reader, error) { }{c.Metadata, def} checkDoc, err := json.Marshal(chk) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to marshal definition for '%s': %s", c.ID, err) + return nil, nil, nil, nil, fmt.Errorf("failed to marshal definition for '%s': %s", c.ID, err) + } + + // The generic check definition only includes the metadata + generic := struct { + Metadata + }{c.Metadata} + genericDoc, err := json.Marshal(generic) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("failed to marshal generic definition for '%s': %s", c.ID, err) } admin, err := attributeDoc(c.Attributes.Admin) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to marshal admin attributes for '%s': %s", c.ID, err) + return nil, nil, nil, nil, fmt.Errorf("failed to marshal admin attributes for '%s': %s", c.ID, err) } user, err := attributeDoc(c.Attributes.User) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to marshal user attributes for '%s': %s", c.ID, err) + return nil, nil, nil, nil, fmt.Errorf("failed to marshal user attributes for '%s': %s", c.ID, err) } - return bytes.NewReader(checkDoc), admin, user, nil + return bytes.NewReader(checkDoc), bytes.NewReader(genericDoc), admin, user, nil } func attributeDoc(attributes map[string]string) (io.Reader, error) { diff --git a/dynamicbeat/pkg/setup/checks.go b/dynamicbeat/pkg/setup/checks.go index bcdddfba..6e9ccb49 100644 --- a/dynamicbeat/pkg/setup/checks.go +++ b/dynamicbeat/pkg/setup/checks.go @@ -26,12 +26,13 @@ func Checks(c *esclient.Client, f *checksource.Filesystem) error { } for _, def := range defs { - chk, admin, user, err := def.Documents() + chk, generic, admin, user, err := def.Documents() if err != nil { zap.S().Errorf("skipping check due to error - %s", err) } queueItem(indexer, "checkdef", def.ID, chk) + queueItem(indexer, "checks", def.ID, generic) if admin != nil { queueItem(indexer, fmt.Sprintf("attrib_admin_%s", def.Group), def.ID, admin) } From a89a5dac1dc559f2fe25fc179c8537dba00f5be2 Mon Sep 17 00:00:00 2001 From: Sean Newman Date: Sun, 12 Sep 2021 23:14:37 -0400 Subject: [PATCH 3/3] Update CHANGELOG --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 540e68ee..b8b65635 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,13 @@ Each section organizes entries into the following subsections: [Unreleased] ------------ +### Dynamicbeat + +#### Fixed + +- Replace remaining `${VAR}` template strings with `{{.Var}}` templates in saved objects (#321) +- Ensure generic check definitions are added by `dynamicbeat setup checks` (#321) + [0.8.0] - 2021-08-10 --------------------