Skip to content

Fix Dynamicbeat setup errors #321

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
s-newman merged 3 commits into from
Sep 13, 2021
Merged

Fix Dynamicbeat setup errors #321

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
344e87a
Clean up remaining bash templating issues
s-newman Sep 9, 2021
4e66113
Remember to add generic checks
s-newman Sep 13, 2021
a89a5da
Update CHANGELOG
s-newman Sep 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,13 @@ Each section organizes entries into the following subsections:
[Unreleased]
------------

### Dynamicbeat

#### Fixed

- Replace remaining `${VAR}` template strings with `{{.Var}}` templates in saved objects (#321)
- Ensure generic check definitions are added by `dynamicbeat setup checks` (#321)

[0.8.0] - 2021-08-10
--------------------

Expand Down
26 changes: 26 additions & 0 deletions dynamicbeat/pkg/assets/assets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,29 @@ func ReadTeam(filename string, name string) io.Reader {

return bytes.NewReader(buf.Bytes())
}

func ReadTeamOverview(filename string, name string, checks int) io.Reader {
data, err := f.ReadFile(filename)
if err != nil {
zap.S().Panicf("failed to read embedded asset %s: %s", filename, err)
}

// Template in the team name and the number of checks per team
vars := struct {
Team string
Checks int
}{name, checks}
tmpl, err := template.New("").Parse(string(data))
if err != nil {
zap.S().Panicf("failed to read asset %s as template: %s", filename, err)
}

// Apply the template and write to a byte buffer
var buf bytes.Buffer
err = tmpl.Execute(&buf, vars)
if err != nil {
zap.S().Panicf("failed to template team %s into asset %s: %s", name, filename, err)
}

return bytes.NewReader(buf.Bytes())
}
4 changes: 2 additions & 2 deletions dynamicbeat/pkg/assets/dashboards/all.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ func Scoreboard() io.Reader {
return assets.Read("dashboards/scoreboard.json")
}

func TeamOverview(name string) func() io.Reader {
func TeamOverview(name string, checks int) func() io.Reader {
return func() io.Reader {
return assets.ReadTeam("dashboards/team-overview.json", name)
return assets.ReadTeamOverview("dashboards/team-overview.json", name, checks)
}
}
4 changes: 2 additions & 2 deletions dynamicbeat/pkg/assets/dashboards/team-overview.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@
"version": "Wzg4LDJd",
"attributes": {
"title": "Check Results Overview - {{.Team}}",
"visState": "{\"title\":\"Check Results Overview - {{.Team}}\",\"type\":\"table\",\"params\":{\"perPage\":${CHECKS},\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"string\"},\"params\":{},\"aggType\":\"top_hits\"},{\"accessor\":2,\"format\":{\"id\":\"boolean\"},\"params\":{},\"aggType\":\"top_hits\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"passed\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Passing\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"name.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":200,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Check Name\"}},{\"id\":\"1\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"message\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Message\"}}]}",
"visState": "{\"title\":\"Check Results Overview - {{.Team}}\",\"type\":\"table\",\"params\":{\"perPage\":{{.Checks}},\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"string\"},\"params\":{},\"aggType\":\"top_hits\"},{\"accessor\":2,\"format\":{\"id\":\"boolean\"},\"params\":{},\"aggType\":\"top_hits\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"passed\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Passing\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"name.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":200,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Check Name\"}},{\"id\":\"1\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"message\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Message\"}}]}",
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
"description": "",
"version": 1,
Expand All @@ -98,7 +98,7 @@
"updated_at": "2020-02-18T11:25:48.753Z",
"version": "WzgxLDJd",
"attributes": {
"title": "${INDEX}",
"title": "results-{{.Team}}",
"timeFieldName": "@timestamp",
"fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"check_type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"check_type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"check_type\",\"subType\":\"multi\"},{\"name\":\"group\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"group.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"group\",\"subType\":\"multi\"},{\"name\":\"id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"id\",\"subType\":\"multi\"},{\"name\":\"message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"message\",\"subType\":\"multi\"},{\"name\":\"name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"name\",\"subType\":\"multi\"},{\"name\":\"passed\",\"type\":\"boolean\",\"esTypes\":[\"boolean\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"passed_int\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"score_weight\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"type\",\"subType\":\"multi\"}]"
},
Expand Down
21 changes: 15 additions & 6 deletions dynamicbeat/pkg/check/check.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,11 @@ func (v ValidationError) Error() string {
return fmt.Sprintf("Error: check (Type: `%s`, ID: `%s`) is missing value for required field `%s`", v.Type, v.ID, v.Field)
}

func (c *Config) Documents() (io.Reader, io.Reader, io.Reader, error) {
func (c *Config) Documents() (io.Reader, io.Reader, io.Reader, io.Reader, error) {
def := make(map[string]interface{})
err := json.Unmarshal(c.Definition, &def)
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to unmarshal definition for '%s': %s", c.ID, err)
return nil, nil, nil, nil, fmt.Errorf("failed to unmarshal definition for '%s': %s", c.ID, err)
}

// The check definition document doesn't include the attributes
Expand All @@ -71,20 +71,29 @@ func (c *Config) Documents() (io.Reader, io.Reader, io.Reader, error) {
}{c.Metadata, def}
checkDoc, err := json.Marshal(chk)
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to marshal definition for '%s': %s", c.ID, err)
return nil, nil, nil, nil, fmt.Errorf("failed to marshal definition for '%s': %s", c.ID, err)
}

// The generic check definition only includes the metadata
generic := struct {
Metadata
}{c.Metadata}
genericDoc, err := json.Marshal(generic)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("failed to marshal generic definition for '%s': %s", c.ID, err)
}

admin, err := attributeDoc(c.Attributes.Admin)
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to marshal admin attributes for '%s': %s", c.ID, err)
return nil, nil, nil, nil, fmt.Errorf("failed to marshal admin attributes for '%s': %s", c.ID, err)
}

user, err := attributeDoc(c.Attributes.User)
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to marshal user attributes for '%s': %s", c.ID, err)
return nil, nil, nil, nil, fmt.Errorf("failed to marshal user attributes for '%s': %s", c.ID, err)
}

return bytes.NewReader(checkDoc), admin, user, nil
return bytes.NewReader(checkDoc), bytes.NewReader(genericDoc), admin, user, nil
}

func attributeDoc(attributes map[string]string) (io.Reader, error) {
Expand Down
3 changes: 2 additions & 1 deletion dynamicbeat/pkg/setup/checks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,13 @@ func Checks(c *esclient.Client, f *checksource.Filesystem) error {
}

for _, def := range defs {
chk, admin, user, err := def.Documents()
chk, generic, admin, user, err := def.Documents()
if err != nil {
zap.S().Errorf("skipping check due to error - %s", err)
}

queueItem(indexer, "checkdef", def.ID, chk)
queueItem(indexer, "checks", def.ID, generic)
if admin != nil {
queueItem(indexer, fmt.Sprintf("attrib_admin_%s", def.Group), def.ID, admin)
}
Expand Down
3 changes: 2 additions & 1 deletion dynamicbeat/pkg/setup/kibana.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,8 @@ func Kibana(host string, user string, pass string, verify bool, teams []config.T
zap.S().Errorf("failed to add role for %s: %s", team.Name, err)
}

err = c.AddDashboard(dashboards.TeamOverview(team.Name))
// TODO: don't hardcode the number of rows in the table
err = c.AddDashboard(dashboards.TeamOverview(team.Name, 20))
if err != nil {
zap.S().Errorf("failed to add team overview dashboard for %s: %s", team.Name, err)
}
Expand Down