add Terraform example for GCP Dataflow to Elasticsearch (elastic#381)

endorama · web-flow · commit 58a1b3e77b13 · 2021-10-15T09:23:34.000-04:00
* add GCP Dataflow to Elasticsearch

* add audit logs example

* add README
diff --git a/GCP Dataflow to Elasticsearch/.gitignore b/GCP Dataflow to Elasticsearch/.gitignore
@@ -0,0 +1,4 @@
+# Ignore any terraform specific folder & hidden files
+.terraform*
+# Ignore terraform state and vars
+terraform.tf*
diff --git a/GCP Dataflow to Elasticsearch/.tool-versions b/GCP Dataflow to Elasticsearch/.tool-versions
@@ -0,0 +1 @@
+terraform 1.0.3
diff --git a/GCP Dataflow to Elasticsearch/README.md b/GCP Dataflow to Elasticsearch/README.md
@@ -0,0 +1,17 @@
+# GCP Dataflow to Elasticsearch
+
+This example allow to setup a GCP Dataflow job to forward GCP Audit logs data to Elasticsearch.
+
+## Setup
+
+Terraform CLI is required for this example.
+
+Install `GCP` integration in your Elasticsearch cluster (example tested with 1.0.1).  
+
+## Versions
+
+Tested with Terraform 1.0.3.
+
+Tested with Elastic stack 7.15.0.
+
+
diff --git a/GCP Dataflow to Elasticsearch/audit.tf b/GCP Dataflow to Elasticsearch/audit.tf
@@ -0,0 +1,43 @@
+resource "google_pubsub_topic" "audit_log_to_es" {
+  name = "audit-logs-to-es"
+}
+
+resource "google_pubsub_subscription" "audit_log_to_es" {
+  name  = "audit-logs-to-es"
+  topic = google_pubsub_topic.audit_log_to_es.name
+}
+
+resource "google_pubsub_topic" "audit_log_to_es_errors" {
+  name = "audit-logs-to-es-errors"
+}
+
+resource "google_logging_project_sink" "audit_log_to_es" {
+  name        = "audit-logs-to-es"
+  description = "Sink Audit logs with sampling for Dataflow Elasticsearch forwarder"
+
+  destination = "pubsub.googleapis.com/${google_pubsub_topic.audit_log_to_es.id}"
+  filter      = "protoPayload.@type=\"type.googleapis.com/google.cloud.audit.AuditLog\" and sample(insertId, ${var.audit_log_sampling})"
+
+  unique_writer_identity = true
+}
+
+resource "google_dataflow_flex_template_job" "forward_audit_logs_to_es" {
+  provider = google-beta
+  name     = "forward-audit-logs-to-es"
+
+  on_delete = var.audit_log_on_job_delete
+
+  container_spec_gcs_path = "gs://dataflow-templates/${var.dataflow_template_version}/flex/PubSub_to_Elasticsearch"
+
+  parameters = {
+    dataset = "audit"
+
+    connectionUrl = var.connection_url
+    apiKey        = var.api_key
+
+    inputSubscription = google_pubsub_subscription.audit_log_to_es.id
+    errorOutputTopic  = google_pubsub_topic.audit_log_to_es_errors.id
+
+    batchSize = var.audit_log_forwarder_batch_size
+  }
+}
diff --git a/GCP Dataflow to Elasticsearch/main.tf b/GCP Dataflow to Elasticsearch/main.tf
@@ -0,0 +1,9 @@
+provider "google" {
+  project = var.project_id
+  region  = var.region
+}
+
+provider "google-beta" {
+  project = var.project_id
+  region  = var.region
+}
diff --git a/GCP Dataflow to Elasticsearch/vars.tf b/GCP Dataflow to Elasticsearch/vars.tf
@@ -0,0 +1,44 @@
+variable "project_id" {
+  type        = string
+  description = "The GCP Project ID."
+}
+
+variable "region" {
+  type        = string
+  description = "GCP region to create resources into."
+}
+
+variable "connection_url" {
+  type        = string
+  description = "The Elasticsearch Cloud ID or connection URL."
+}
+
+variable "api_key" {
+  type        = string
+  description = "The Elasticsearch API key."
+}
+
+variable "dataflow_template_version" {
+  type        = string
+  description = "GCP Dataflow Flex template version. See https://cloud.google.com/dataflow/docs/guides/templates/provided-streaming."
+  default     = "latest"
+}
+
+# Audit logs
+variable "audit_log_sampling" {
+  type        = number
+  description = "Sampling fraction (0 to 1) for Audit logs"
+  default     = 1
+}
+
+variable "audit_log_forwarder_batch_size" {
+  type        = number
+  description = "Batch size for Audit logs fowarder job"
+  default     = 1000
+}
+
+variable "audit_log_on_job_delete" {
+  type        = string
+  description = "Action to perform when Audit logs forwarder job is deleted (drain or cancel)"
+  default     = "drain"
+}
