scorestack · s-newman · Sep 13, 2021 · Sep 9, 2021 · Sep 13, 2021 · Sep 13, 2021
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,6 +23,18 @@ Each section organizes entries into the following subsections:
 [Unreleased]
 ------------
 
+[0.8.1] - 2021-09-12
+--------------------
+
+This release fixes a few bugs in Dynamicbeat related to the new deployment and check-adding processes.
+
+### Dynamicbeat
+
+#### Fixed
+
+- Replace remaining `${VAR}` template strings with `{{.Var}}` templates in saved objects (#321)
+- Ensure generic check definitions are added by `dynamicbeat setup checks` (#321)
+
 [0.8.0] - 2021-08-10
 --------------------
 
@@ -378,7 +390,8 @@ This release is in preparation for ISTS 2020.
 
 The initial release of Scorestack.
 
-[Unreleased]: https://github.com/scorestack/scorestack/compare/v0.8.0...dev
+[Unreleased]: https://github.com/scorestack/scorestack/compare/v0.8.1...dev
+[0.8.1]: https://github.com/scorestack/scorestack/compare/v0.8.0...v0.8.1
 [0.8.0]: https://github.com/scorestack/scorestack/compare/v0.7.0...v0.8.0
 [0.7.0]: https://github.com/scorestack/scorestack/compare/v0.6.2...v0.7.0
 [0.6.2]: https://github.com/scorestack/scorestack/compare/v0.6.1...v0.6.2

diff --git a/deployment/medium/ansible/playbook.yml b/deployment/medium/ansible/playbook.yml
@@ -247,7 +247,7 @@
         - Restart Kibana
 
     - name: Install plugin
-      command: /usr/share/kibana/bin/kibana-plugin --allow-root install https://github.com/scorestack/scorestack/releases/download/v0.8.0/kibana-plugin-v0.8.0.zip
+      command: /usr/share/kibana/bin/kibana-plugin --allow-root install https://github.com/scorestack/scorestack/releases/download/v0.8.1/kibana-plugin-v0.8.1.zip
       args:
         creates: /usr/share/kibana/plugins/scorestack
       notify:

diff --git a/deployment/small/docker/setup.sh b/deployment/small/docker/setup.sh
@@ -42,7 +42,7 @@ docker exec ${KIBANA_CONTAINER} bin/kibana-keystore create
 docker exec ${KIBANA_CONTAINER} /bin/bash -c "bin/kibana-keystore add elasticsearch.password --stdin <<< '${kibana_pass}'"
 
 # Install kibana plugin
-docker exec ${KIBANA_CONTAINER} /bin/bash -c "bin/kibana-plugin install https://github.com/scorestack/scorestack/releases/download/v0.8.0/kibana-plugin-v0.8.0.zip"
+docker exec ${KIBANA_CONTAINER} /bin/bash -c "bin/kibana-plugin install https://github.com/scorestack/scorestack/releases/download/v0.8.1/kibana-plugin-v0.8.1.zip"
 
 # Restart kibana to reload credentials from keystore
 cd config

diff --git a/dynamicbeat/pkg/assets/assets.go b/dynamicbeat/pkg/assets/assets.go
@@ -45,3 +45,29 @@ func ReadTeam(filename string, name string) io.Reader {
 
 	return bytes.NewReader(buf.Bytes())
 }
+
+func ReadTeamOverview(filename string, name string, checks int) io.Reader {
+	data, err := f.ReadFile(filename)
+	if err != nil {
+		zap.S().Panicf("failed to read embedded asset %s: %s", filename, err)
+	}
+
+	// Template in the team name and the number of checks per team
+	vars := struct {
+		Team   string
+		Checks int
+	}{name, checks}
+	tmpl, err := template.New("").Parse(string(data))
+	if err != nil {
+		zap.S().Panicf("failed to read asset %s as template: %s", filename, err)
+	}
+
+	// Apply the template and write to a byte buffer
+	var buf bytes.Buffer
+	err = tmpl.Execute(&buf, vars)
+	if err != nil {
+		zap.S().Panicf("failed to template team %s into asset %s: %s", name, filename, err)
+	}
+
+	return bytes.NewReader(buf.Bytes())
+}
diff --git a/dynamicbeat/pkg/assets/dashboards/all.go b/dynamicbeat/pkg/assets/dashboards/all.go
@@ -10,8 +10,8 @@ func Scoreboard() io.Reader {
 	return assets.Read("dashboards/scoreboard.json")
 }
 
-func TeamOverview(name string) func() io.Reader {
+func TeamOverview(name string, checks int) func() io.Reader {
 	return func() io.Reader {
-		return assets.ReadTeam("dashboards/team-overview.json", name)
+		return assets.ReadTeamOverview("dashboards/team-overview.json", name, checks)
 	}
 }
diff --git a/dynamicbeat/pkg/assets/dashboards/team-overview.json b/dynamicbeat/pkg/assets/dashboards/team-overview.json
@@ -73,7 +73,7 @@
       "version": "Wzg4LDJd",
       "attributes": {
         "title": "Check Results Overview - {{.Team}}",
-        "visState": "{\"title\":\"Check Results Overview - {{.Team}}\",\"type\":\"table\",\"params\":{\"perPage\":${CHECKS},\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"string\"},\"params\":{},\"aggType\":\"top_hits\"},{\"accessor\":2,\"format\":{\"id\":\"boolean\"},\"params\":{},\"aggType\":\"top_hits\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"passed\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Passing\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"name.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":200,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Check Name\"}},{\"id\":\"1\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"message\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Message\"}}]}",
+        "visState": "{\"title\":\"Check Results Overview - {{.Team}}\",\"type\":\"table\",\"params\":{\"perPage\":{{.Checks}},\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"string\"},\"params\":{},\"aggType\":\"top_hits\"},{\"accessor\":2,\"format\":{\"id\":\"boolean\"},\"params\":{},\"aggType\":\"top_hits\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"passed\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Passing\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"name.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":200,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Check Name\"}},{\"id\":\"1\",\"enabled\":true,\"type\":\"top_hits\",\"schema\":\"metric\",\"params\":{\"field\":\"message\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Message\"}}]}",
         "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
         "description": "",
         "version": 1,
@@ -98,7 +98,7 @@
       "updated_at": "2020-02-18T11:25:48.753Z",
       "version": "WzgxLDJd",
       "attributes": {
-        "title": "${INDEX}",
+        "title": "results-{{.Team}}",
         "timeFieldName": "@timestamp",
         "fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"check_type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"check_type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"check_type\",\"subType\":\"multi\"},{\"name\":\"group\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"group.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"group\",\"subType\":\"multi\"},{\"name\":\"id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"id\",\"subType\":\"multi\"},{\"name\":\"message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"message\",\"subType\":\"multi\"},{\"name\":\"name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"name\",\"subType\":\"multi\"},{\"name\":\"passed\",\"type\":\"boolean\",\"esTypes\":[\"boolean\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"passed_int\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"score_weight\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"type\",\"subType\":\"multi\"}]"
       },

diff --git a/dynamicbeat/pkg/check/check.go b/dynamicbeat/pkg/check/check.go
@@ -57,11 +57,11 @@ func (v ValidationError) Error() string {
 	return fmt.Sprintf("Error: check (Type: `%s`, ID: `%s`) is missing value for required field `%s`", v.Type, v.ID, v.Field)
 }
 
-func (c *Config) Documents() (io.Reader, io.Reader, io.Reader, error) {
+func (c *Config) Documents() (io.Reader, io.Reader, io.Reader, io.Reader, error) {
 	def := make(map[string]interface{})
 	err := json.Unmarshal(c.Definition, &def)
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to unmarshal definition for '%s': %s", c.ID, err)
+		return nil, nil, nil, nil, fmt.Errorf("failed to unmarshal definition for '%s': %s", c.ID, err)
 	}
 
 	// The check definition document doesn't include the attributes
@@ -71,20 +71,29 @@ func (c *Config) Documents() (io.Reader, io.Reader, io.Reader, error) {
 	}{c.Metadata, def}
 	checkDoc, err := json.Marshal(chk)
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to marshal definition for '%s': %s", c.ID, err)
+		return nil, nil, nil, nil, fmt.Errorf("failed to marshal definition for '%s': %s", c.ID, err)
+	}
+
+	// The generic check definition only includes the metadata
+	generic := struct {
+		Metadata
+	}{c.Metadata}
+	genericDoc, err := json.Marshal(generic)
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("failed to marshal generic definition for '%s': %s", c.ID, err)
 	}
 
 	admin, err := attributeDoc(c.Attributes.Admin)
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to marshal admin attributes for '%s': %s", c.ID, err)
+		return nil, nil, nil, nil, fmt.Errorf("failed to marshal admin attributes for '%s': %s", c.ID, err)
 	}
 
 	user, err := attributeDoc(c.Attributes.User)
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to marshal user attributes for '%s': %s", c.ID, err)
+		return nil, nil, nil, nil, fmt.Errorf("failed to marshal user attributes for '%s': %s", c.ID, err)
 	}
 
-	return bytes.NewReader(checkDoc), admin, user, nil
+	return bytes.NewReader(checkDoc), bytes.NewReader(genericDoc), admin, user, nil
 }
 
 func attributeDoc(attributes map[string]string) (io.Reader, error) {

diff --git a/dynamicbeat/pkg/setup/checks.go b/dynamicbeat/pkg/setup/checks.go
@@ -26,12 +26,13 @@ func Checks(c *esclient.Client, f *checksource.Filesystem) error {
 	}
 
 	for _, def := range defs {
-		chk, admin, user, err := def.Documents()
+		chk, generic, admin, user, err := def.Documents()
 		if err != nil {
 			zap.S().Errorf("skipping check due to error - %s", err)
 		}
 
 		queueItem(indexer, "checkdef", def.ID, chk)
+		queueItem(indexer, "checks", def.ID, generic)
 		if admin != nil {
 			queueItem(indexer, fmt.Sprintf("attrib_admin_%s", def.Group), def.ID, admin)
 		}

diff --git a/dynamicbeat/pkg/setup/kibana.go b/dynamicbeat/pkg/setup/kibana.go
@@ -92,7 +92,8 @@ func Kibana(host string, user string, pass string, verify bool, teams []config.T
 			zap.S().Errorf("failed to add role for %s: %s", team.Name, err)
 		}
 
-		err = c.AddDashboard(dashboards.TeamOverview(team.Name))
+		// TODO: don't hardcode the number of rows in the table
+		err = c.AddDashboard(dashboards.TeamOverview(team.Name, 20))
 		if err != nil {
 			zap.S().Errorf("failed to add team overview dashboard for %s: %s", team.Name, err)
 		}

diff --git a/kibana-plugin/package.json b/kibana-plugin/package.json
@@ -1,11 +1,11 @@
 {
   "name": "scorestack",
-  "version": "0.8.0",
+  "version": "0.8.1",
   "description": "A Kibana plugin for viewing and modifying Scorestack checks and attributes.",
   "private": true,
   "scripts": {
     "prebuild": "node ../../scripts/build_kibana_platform_plugins --verbose --dist --filter scorestack",
-    "build": "plugin-helpers build --build-version 0.8.0 --kibana-version 7.9.2",
+    "build": "plugin-helpers build --build-version 0.8.1 --kibana-version 7.9.2",
     "kbn": "node ../../scripts/kbn",
     "lint": "eslint --ignore-path .gitignore --color .",
     "plugin-helpers": "plugin-helpers",