scorestack · s-newman · Mar 24, 2021 · Mar 23, 2021 · Mar 23, 2021 · Mar 23, 2021
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,6 +23,17 @@ Each section organizes entries into the following subsections:
 [Unreleased]
 ------------
 
+### Dynamicbeat
+
+#### Changed
+
+- Dynamicbeat is now a standalone program that doesn't use libbeat (#302)
+
+#### Removed
+
+- Remove `update_period` setting, update configurations after starting each round (#302)
+- Drop Logstash from architecture (#302)
+
 [0.7.0] - 2020-02-21
 --------------------
 

diff --git a/deployment/medium/ansible/configs/dynamicbeat-pipeline.conf b/deployment/medium/ansible/configs/dynamicbeat-pipeline.conf
diff --git a/deployment/medium/ansible/configs/hosts b/deployment/medium/ansible/configs/hosts
@@ -1,7 +1,6 @@
 127.0.0.1 localhost
 
 {{ nginx_internal_ip }} nginx
-{{ logstash_ip }} logstash
 {{ kibana_ip }} kibana
 {{ elasticsearch1_ip }} elasticsearch1
 {{ elasticsearch2_ip }} elasticsearch2

diff --git a/deployment/medium/ansible/configs/logstash.yml b/deployment/medium/ansible/configs/logstash.yml
diff --git a/deployment/medium/ansible/configs/nginx.conf b/deployment/medium/ansible/configs/nginx.conf
@@ -48,10 +48,3 @@ http {
 
 	include /etc/nginx/conf.d/*.conf;
 }
-
-stream {
-	server {
-		listen 5454;
-		proxy_pass logstash:5454;
-	}
-}
diff --git a/deployment/medium/ansible/playbook.yml b/deployment/medium/ansible/playbook.yml
@@ -10,7 +10,7 @@
         dest: /etc/hosts
 
 - name: Install Java
-  hosts: elasticsearch:logstash
+  hosts: elasticsearch
   become: yes
   tasks:
 
@@ -238,19 +238,7 @@
     - name: Create Dynamicbeat role
       uri:
         url: "https://elasticsearch4:9200/_security/role/dynamicbeat_reader"
-        body: '{"indices":[{"names":["checkdef*","attrib_*"],"privileges":["read"]}]}'
-        body_format: json
-        force_basic_auth: yes
-        method: PUT
-        url_username: elastic
-        url_password: "{{ elastic_password }}"
-        validate_certs: no
-      changed_when: no
-
-    - name: Create Logstash role
-      uri:
-        url: "https://elasticsearch4:9200/_security/role/logstash_writer"
-        body: '{"cluster":["manage_index_templates","monitor","manage_ilm"],"indices":[{"names":["results-*"],"privileges":["write","create","delete","create_index","manage","manage_ilm"]}]}'
+        body: '{"indices":[{"names":["checkdef*","attrib_*"],"privileges":["read"]}, {"names":["results-*"],"privileges":["write","create","create_index"]}]}'
         body_format: json
         force_basic_auth: yes
         method: PUT
@@ -273,80 +261,6 @@
       loop:
         - { username: "root", password: "changeme", full_name: "Extra Superuser", email: "[email protected]", role: "superuser" }
         - { username: "dynamicbeat", password: "changeme", full_name: "Dynamicbeat Definition-Reading User", email: "[email protected]", role: "dynamicbeat_reader" }
-        - { username: "logstash", password: "{{ logstash_password }}", full_name: "Internal Logstash User", email: "[email protected]", role: "logstash_writer" }
-
-###############################################################################
-# LOGSTASH
-###############################################################################
-
-- name: Install Logstash
-  hosts: logstash
-  become: yes
-  tasks:
-
-    - name: Install Logstash package
-      apt:
-        deb: https://artifacts.elastic.co/downloads/logstash/logstash-7.9.2.deb
-
-    - name: Configure Logstash
-      copy:
-        src: configs/logstash.yml
-        dest: /etc/logstash/logstash.yml
-      notify:
-        - Restart Logstash
-
-    - name: Configure Dynamicbeat pipeline
-      template:
-        src: configs/dynamicbeat-pipeline.conf
-        dest: /etc/logstash/conf.d/dynamicbeat-pipeline.conf
-      notify:
-        - Restart Logstash
-
-    - name: Copy root CA certificate
-      copy:
-        src: certificates/ca/ca.crt
-        dest: /etc/logstash/ca.crt
-      notify:
-        - Restart Logstash
-
-    - name: Copy certificate
-      copy:
-        src: certificates/{{ inventory_hostname }}/{{ inventory_hostname }}.crt
-        dest: /etc/logstash/{{ inventory_hostname }}.crt
-      notify:
-        - Restart Logstash
-
-    - name: Copy key
-      copy:
-        src: certificates/{{ inventory_hostname }}/{{ inventory_hostname }}.key
-        dest: /etc/logstash/{{ inventory_hostname }}.key
-        mode: '0600'
-
-    - name: Convert key format
-      command: openssl pkcs8 -topk8 -nocrypt -in /etc/logstash/{{ inventory_hostname }}.key -out /etc/logstash/{{ inventory_hostname }}.key.pkcs8
-      args:
-        creates: /etc/logstash/{{ inventory_hostname }}.key.pkcs8
-      notify:
-        - Restart Logstash
-
-    - name: Set permissions on key
-      file:
-        path: /etc/logstash/{{ inventory_hostname }}.key.pkcs8
-        owner: root
-        group: logstash
-        mode: 0660
-
-    - name: Enable Logstash service
-      systemd:
-        name: logstash
-        enabled: yes
-
-  handlers:
-
-    - name: Restart Logstash
-      systemd:
-        name: logstash
-        state: restarted
 
 ###############################################################################
 # KIBANA

diff --git a/deployment/medium/gcp/certificates.tf b/deployment/medium/gcp/certificates.tf
@@ -235,57 +235,6 @@ resource "null_resource" "elasticsearch4_cert" {
     }
 }
 
-resource "tls_private_key" "logstash_key" {
-    algorithm = "ECDSA"
-    ecdsa_curve = "P256"
-}
-
-resource "null_resource" "logstash_key" {
-    triggers = {
-        key_created = tls_private_key.logstash_key.private_key_pem
-    }
-
-    provisioner "local-exec" {
-        command = "mkdir -p ${var.certificate_destination}/logstash && echo '${tls_private_key.logstash_key.private_key_pem}' > ${var.certificate_destination}/logstash/logstash.key"
-    }
-}
-
-resource "tls_cert_request" "logstash_csr" {
-    key_algorithm = "ECDSA"
-    private_key_pem = tls_private_key.logstash_key.private_key_pem
-
-    subject {
-        common_name = "logstash"
-        organization = "Scorestack"
-    }
-
-    dns_names = ["localhost", "logstash", var.fqdn]
-    ip_addresses = ["127.0.0.1", google_compute_instance.logstash.network_interface.0.network_ip, google_compute_address.nginx.address]
-}
-
-resource "tls_locally_signed_cert" "logstash_cert" {
-    cert_request_pem = tls_cert_request.logstash_csr.cert_request_pem
-    ca_key_algorithm = "ECDSA"
-    ca_private_key_pem = tls_private_key.ca_key.private_key_pem
-    ca_cert_pem = tls_self_signed_cert.ca_cert.cert_pem
-    validity_period_hours = 8760
-
-    allowed_uses = [
-        "server_auth",
-        "client_auth",
-    ]
-}
-
-resource "null_resource" "logstash_cert" {
-    triggers = {
-        cert_created = tls_locally_signed_cert.logstash_cert.cert_pem
-    }
-
-    provisioner "local-exec" {
-        command = "mkdir -p ${var.certificate_destination}/logstash && echo '${tls_locally_signed_cert.logstash_cert.cert_pem}' > ${var.certificate_destination}/logstash/logstash.crt"
-    }
-}
-
 resource "tls_private_key" "kibana_key" {
     algorithm = "ECDSA"
     ecdsa_curve = "P256"

diff --git a/deployment/medium/gcp/instances.tf b/deployment/medium/gcp/instances.tf
@@ -127,31 +127,6 @@ resource "google_compute_instance" "kibana" {
     tags = ["default", "ssh", "kibana"]
 }
 
-resource "google_compute_instance" "logstash" {
-    name = "logstash"
-    description = "A Logstash node."
-
-    boot_disk {
-        initialize_params {
-            image = var.ubuntu
-        }
-    }
-
-    machine_type = "n1-standard-1"
-    zone = var.zone1
-
-    network_interface {
-        network = google_compute_network.internal_network.self_link
-        access_config {}
-    }
-
-    metadata = {
-        ssh-keys = "${var.ssh_user}:${file(var.ssh_pub_key_file)}"
-    }
-
-    tags = ["default", "ssh", "logstash"]
-}
-
 resource "google_compute_instance" "nginx" {
     name = "nginx"
     description = "An Nginx node that also serves as an SSH jump box."
@@ -176,5 +151,5 @@ resource "google_compute_instance" "nginx" {
         ssh-keys = "${var.ssh_user}:${file(var.ssh_pub_key_file)}"
     }
 
-    tags = ["default", "ssh-jump", "proxy", "logstash", "elasticsearch"]
+    tags = ["default", "ssh-jump", "proxy", "elasticsearch"]
 }
diff --git a/deployment/medium/gcp/inventory_template.ini b/deployment/medium/gcp/inventory_template.ini
@@ -1,6 +1,5 @@
 [all]
 nginx ansible_host=${nginx_ip} ansible_ssh_common_args="-o StrictHostKeyChecking=no"
-logstash ansible_host=${logstash_ip}
 kibana ansible_host=${kibana_ip}
 
 [all:vars]
@@ -9,7 +8,6 @@ ansible_ssh_private_key_file=${ssh_priv_key_file}
 bootstrap_password=${bootstrap_password}
 elastic_password=${elastic_password}
 kibana_password=${kibana_password}
-logstash_password=${logstash_password}
 remote_monitoring_user_password=${remote_monitoring_user_password}
 fqdn=${fqdn}
 elasticsearch1_ip=${elasticsearch1_ip}
@@ -18,7 +16,6 @@ elasticsearch3_ip=${elasticsearch3_ip}
 elasticsearch4_ip=${elasticsearch4_ip}
 nginx_ip=${nginx_ip}
 nginx_internal_ip=${nginx_internal_ip}
-logstash_ip=${logstash_ip}
 kibana_ip=${kibana_ip}
 
 # All servers with elasticsearch installed
@@ -42,7 +39,6 @@ elasticsearch4 ansible_host=${elasticsearch4_ip} ram_gb=2
 # Hosts that must be accessed through the ssh jump server
 [jumped]
 kibana
-logstash ram_gb=2
 
 [jumped:children]
 elasticsearch

diff --git a/deployment/medium/gcp/networking.tf b/deployment/medium/gcp/networking.tf
@@ -81,21 +81,6 @@ resource "google_compute_firewall" "kibana" {
     target_tags = ["kibana"]
 }
 
-resource "google_compute_firewall" "logstash" {
-    name = "logstash"
-    description = "Allow traffic to the Dynamicbeat listener on the Logstash server from the Nginx server."
-
-    network = google_compute_network.internal_network.self_link
-
-    allow {
-        protocol = "tcp"
-        ports = ["5454"]
-    }
-
-    source_tags = ["proxy"]
-    target_tags = ["logstash"]
-}
-
 resource "google_compute_firewall" "ssh-jump" {
     name = "ssh-jump"
     description = "Allow SSH traffic to the SSH jump box from the public internet."