Skip to content

Commit cc66c25

Browse files
randomuseridrandomuserid
committed
adding prior ml jobs
adding the v2 ML jobs and a table of contents to the readme file
1 parent 7ef3877 commit cc66c25

33 files changed

+1738
-1
lines changed

‎Machine Learning/Anomaly Detection/README.md‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,16 @@ This directory contains example anomaly detection job configurations.
44

55
TIP: Kibana can also recognize certain types of data and provide specialized
66
wizards for that context. For more details, refer to
7-
[supplied anomaly detection configurations](https://www.elastic.co/guide/en/machine-learning/8.0/ootb-ml-jobs.html).
7+
[supplied anomaly detection configurations](https://www.elastic.co/guide/en/machine-learning/8.0/ootb-ml-jobs.html).
8+
9+
Unsupervised ML Archive - Past Versions of the Anomaly Detection Jobs
10+
11+
These are prior versions of the version 3 ML jobs shipping in Elastic 8.3. They are only needed if running older data sources like Beats or Endpoints in the 7.x version range.
12+
13+
security_linux: version 2 of the Linux anomaly detection jobs, from 2020.
14+
security_windows: version 2 of the Windows anomaly detection jobs, from 2020.
15+
siem_auditbeat: version 1 of the Linux anomaly detection jobs, from 2019.
16+
siem_winlogbeat: version 1 of the Windows anomaly detection jobs, from 2019.
17+
18+
siem_winlogbeat_auth: an anomaly detection job for Windows RDP login events, from 2019.
19+
siem_auditbeat_auth: an anomaly detection job for auth events developed on Linux. The first ML job shipped in the Security solution.

‎Machine Learning/Anomaly Detection/security_linux/logo.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"icon": "logoSecurity"
3+
}

‎Machine Learning/Anomaly Detection/security_linux/manifest.json‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
{
2+
"id": "security_linux",
3+
"title": "Security: Linux",
4+
"description": "Detect suspicious activity using ECS Linux events. Tested with Auditbeat and the Elastic agent.",
5+
"type": "linux data",
6+
"logoFile": "logo.json",
7+
"defaultIndexPattern": "auditbeat-*,logs-endpoint.events.*",
8+
"query": {
9+
"bool": {
10+
"should": [
11+
{
12+
"match": {
13+
"host.os.type": {
14+
"query": "linux",
15+
"operator": "OR"
16+
}
17+
}
18+
},
19+
{
20+
"match": {
21+
"host.os.family": {
22+
"query": "debian",
23+
"operator": "OR"
24+
}
25+
}
26+
},
27+
{
28+
"match": {
29+
"host.os.family": {
30+
"query": "redhat",
31+
"operator": "OR"
32+
}
33+
}
34+
},
35+
{
36+
"match": {
37+
"host.os.family": {
38+
"query": "suse",
39+
"operator": "OR"
40+
}
41+
}
42+
}
43+
]
44+
}
45+
},
46+
"jobs": [
47+
{
48+
"id": "v2_rare_process_by_host_linux_ecs",
49+
"file": "v2_rare_process_by_host_linux_ecs.json"
50+
},
51+
{
52+
"id": "v2_linux_rare_metadata_user",
53+
"file": "v2_linux_rare_metadata_user.json"
54+
},
55+
{
56+
"id": "v2_linux_rare_metadata_process",
57+
"file": "v2_linux_rare_metadata_process.json"
58+
},
59+
{
60+
"id": "v2_linux_anomalous_user_name_ecs",
61+
"file": "v2_linux_anomalous_user_name_ecs.json"
62+
},
63+
{
64+
"id": "v2_linux_anomalous_process_all_hosts_ecs",
65+
"file": "v2_linux_anomalous_process_all_hosts_ecs.json"
66+
},
67+
{
68+
"id": "v2_linux_anomalous_network_port_activity_ecs",
69+
"file": "v2_linux_anomalous_network_port_activity_ecs.json"
70+
}
71+
],
72+
"datafeeds": [
73+
{
74+
"id": "datafeed-v2_rare_process_by_host_linux_ecs",
75+
"file": "datafeed_v2_rare_process_by_host_linux_ecs.json",
76+
"job_id": "v2_rare_process_by_host_linux_ecs"
77+
},
78+
{
79+
"id": "datafeed-v2_linux_rare_metadata_user",
80+
"file": "datafeed_v2_linux_rare_metadata_user.json",
81+
"job_id": "v2_linux_rare_metadata_user"
82+
},
83+
{
84+
"id": "datafeed-v2_linux_rare_metadata_process",
85+
"file": "datafeed_v2_linux_rare_metadata_process.json",
86+
"job_id": "v2_linux_rare_metadata_process"
87+
},
88+
{
89+
"id": "datafeed-v2_linux_anomalous_user_name_ecs",
90+
"file": "datafeed_v2_linux_anomalous_user_name_ecs.json",
91+
"job_id": "v2_linux_anomalous_user_name_ecs"
92+
},
93+
{
94+
"id": "datafeed-v2_linux_anomalous_process_all_hosts_ecs",
95+
"file": "datafeed_v2_linux_anomalous_process_all_hosts_ecs.json",
96+
"job_id": "v2_linux_anomalous_process_all_hosts_ecs"
97+
},
98+
{
99+
"id": "datafeed-v2_linux_anomalous_network_port_activity_ecs",
100+
"file": "datafeed_v2_linux_anomalous_network_port_activity_ecs.json",
101+
"job_id": "v2_linux_anomalous_network_port_activity_ecs"
102+
}
103+
]
104+
}

‎Machine Learning/Anomaly Detection/security_linux/ml/datafeed_v2_linux_anomalous_network_port_activity_ecs.json‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
{
2+
"job_id": "JOB_ID",
3+
"indices": [
4+
"INDEX_PATTERN_NAME"
5+
],
6+
"max_empty_searches": 10,
7+
"query": {
8+
"bool":
9+
{
10+
"filter": [
11+
{"term": {"event.category": "network"}},
12+
{"term": {"event.type": "start"}}
13+
],
14+
"must": [
15+
{
16+
"bool": {
17+
"should": [
18+
{
19+
"match": {
20+
"host.os.type": {
21+
"query": "linux",
22+
"operator": "OR"
23+
}
24+
}
25+
},
26+
{
27+
"match": {
28+
"host.os.family": {
29+
"query": "debian",
30+
"operator": "OR"
31+
}
32+
}
33+
},
34+
{
35+
"match": {
36+
"host.os.family": {
37+
"query": "redhat",
38+
"operator": "OR"
39+
}
40+
}
41+
},
42+
{
43+
"match": {
44+
"host.os.family": {
45+
"query": "suse",
46+
"operator": "OR"
47+
}
48+
}
49+
},
50+
{
51+
"match": {
52+
"host.os.family": {
53+
"query": "ubuntu",
54+
"operator": "OR"
55+
}
56+
}
57+
}
58+
]
59+
}
60+
}
61+
],
62+
"must_not": [
63+
{
64+
"bool": {
65+
"should": [
66+
{"term": {"destination.ip": "127.0.0.1"}},
67+
{"term": {"destination.ip": "::"}},
68+
{"term": {"destination.ip": "::1"}},
69+
{"term": {"user.name":"jenkins"}}
70+
]
71+
}
72+
}
73+
]
74+
}
75+
}
76+
}

‎Machine Learning/Anomaly Detection/security_linux/ml/datafeed_v2_linux_anomalous_process_all_hosts_ecs.json‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
{
2+
"job_id": "JOB_ID",
3+
"indices": [
4+
"INDEX_PATTERN_NAME"
5+
],
6+
"max_empty_searches": 10,
7+
"query": {
8+
"bool": {
9+
"filter": [
10+
{
11+
"term": {
12+
"event.category": "process"
13+
}
14+
},
15+
{
16+
"term": {
17+
"event.type": "start"
18+
}
19+
}
20+
],
21+
"must": [
22+
{
23+
"bool": {
24+
"should": [
25+
{
26+
"match": {
27+
"host.os.type": {
28+
"query": "linux",
29+
"operator": "OR"
30+
}
31+
}
32+
},
33+
{
34+
"match": {
35+
"host.os.family": {
36+
"query": "debian",
37+
"operator": "OR"
38+
}
39+
}
40+
},
41+
{
42+
"match": {
43+
"host.os.family": {
44+
"query": "redhat",
45+
"operator": "OR"
46+
}
47+
}
48+
},
49+
{
50+
"match": {
51+
"host.os.family": {
52+
"query": "suse",
53+
"operator": "OR"
54+
}
55+
}
56+
},
57+
{
58+
"match": {
59+
"host.os.family": {
60+
"query": "ubuntu",
61+
"operator": "OR"
62+
}
63+
}
64+
}
65+
]
66+
}
67+
}
68+
],
69+
"must_not": [
70+
{
71+
"bool": {
72+
"should": [
73+
{
74+
"term": {
75+
"user.name": "jenkins-worker"
76+
}
77+
},
78+
{
79+
"term": {
80+
"user.name": "jenkins-user"
81+
}
82+
},
83+
{
84+
"term": {
85+
"user.name": "jenkins"
86+
}
87+
},
88+
{
89+
"wildcard": {
90+
"process.name": {
91+
"wildcard": "jenkins*"
92+
}
93+
}
94+
}
95+
]
96+
}
97+
}
98+
]
99+
}
100+
}
101+
}

‎Machine Learning/Anomaly Detection/security_linux/ml/datafeed_v2_linux_anomalous_user_name_ecs.json‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
{
2+
"job_id": "JOB_ID",
3+
"indices": [
4+
"INDEX_PATTERN_NAME"
5+
],
6+
"max_empty_searches": 10,
7+
"query": {
8+
"bool": {
9+
"filter": [
10+
{
11+
"term": {
12+
"event.category": "process"
13+
}
14+
},
15+
{
16+
"term": {
17+
"event.type": "start"
18+
}
19+
}
20+
],
21+
"must": [
22+
{
23+
"bool": {
24+
"should": [
25+
{
26+
"match": {
27+
"host.os.type": {
28+
"query": "linux",
29+
"operator": "OR"
30+
}
31+
}
32+
},
33+
{
34+
"match": {
35+
"host.os.family": {
36+
"query": "debian",
37+
"operator": "OR"
38+
}
39+
}
40+
},
41+
{
42+
"match": {
43+
"host.os.family": {
44+
"query": "redhat",
45+
"operator": "OR"
46+
}
47+
}
48+
},
49+
{
50+
"match": {
51+
"host.os.family": {
52+
"query": "suse",
53+
"operator": "OR"
54+
}
55+
}
56+
},
57+
{
58+
"match": {
59+
"host.os.family": {
60+
"query": "ubuntu",
61+
"operator": "OR"
62+
}
63+
}
64+
}
65+
]
66+
}
67+
}
68+
]
69+
}
70+
}
71+
}

0 commit comments

Comments
 (0)