Skip to content

Commit 8bc56c2

Browse files
OrelHaimOrelHaim
authored
Searching indicators by server version in content integrations (demisto#12134)
* search indicators
1 parent b34c478 commit 8bc56c2

14 files changed

Lines changed: 142 additions & 16 deletions

File tree

‎Packs/Base/ReleaseNotes/1_8_10.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
2+
#### Scripts
3+
##### CommonServerPython
4+
- Added an option to search indicators according to server version.
5+
##### GetIndicatorsByQuery
6+
- Fixed an issue where searching more than 10K indicators failed when using ElasticSearch.

‎Packs/Base/Scripts/CommonServerPython/CommonServerPython.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6449,3 +6449,61 @@ def to_display(self):
64496449
'total': len(self.data),
64506450
'data': self.data
64516451
})
6452+
6453+
6454+
class IndicatorsSearcher:
6455+
"""Used in order to search indicators by the paging or serachAfter param
6456+
:type page: ``int``
6457+
:param page: the number of page from which we start search indicators from.
6458+
6459+
:return: No data returned
6460+
:rtype: ``None``
6461+
"""
6462+
def __init__(self, page=0):
6463+
# searchAfter is available in searchIndicators from version 6.1.0
6464+
self._can_use_search_after = is_demisto_version_ge('6.1.0')
6465+
self._search_after_title = 'searchAfter'
6466+
self._search_after_param = None
6467+
self._page = page
6468+
6469+
def search_indicators_by_version(self, from_date=None, query='', size=100, to_date=None, value=''):
6470+
"""There are 2 cases depends on the sever version:
6471+
1. Search indicators using paging, raise the page number in each call.
6472+
2. Search indicators using searchAfter param, update the _search_after_param in each call.
6473+
6474+
:type from_date: ``str``
6475+
:param from_date: the start date to search from.
6476+
6477+
:type query: ``str``
6478+
:param query: indicator search query
6479+
6480+
:type size: ``size``
6481+
:param size: limit the number of returned results.
6482+
6483+
:type to_date: ``str``
6484+
:param to_date: the end date to search until to.
6485+
6486+
:type value: ``str``
6487+
:param value: the indicator value to search.
6488+
6489+
:return: object contains the search results
6490+
:rtype: ``dict``
6491+
"""
6492+
if self._can_use_search_after:
6493+
res = demisto.searchIndicators(fromDate=from_date, toDate=to_date, query=query, size=size, value=value,
6494+
searchAfter=self._search_after_param)
6495+
if self._search_after_title in res and res[self._search_after_title] is not None:
6496+
self._search_after_param = res[self._search_after_title]
6497+
else:
6498+
demisto.log('Elastic search using searchAfter was not found in searchIndicators')
6499+
6500+
else:
6501+
res = demisto.searchIndicators(fromDate=from_date, toDate=to_date, query=query, size=size, page=self._page,
6502+
value=value)
6503+
self._page += 1
6504+
6505+
return res
6506+
6507+
@property
6508+
def page(self):
6509+
return self._page

‎Packs/Base/Scripts/CommonServerPython/CommonServerPython_test.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4063,3 +4063,57 @@ def test_email_indicator_type(self, mocker):
40634063
dbot_score=dbot_score
40644064
)
40654065
assert email_context.to_context()[email_context.CONTEXT_PATH] == {'Address': '[email protected]', 'Domain': 'example.com'}
4066+
4067+
4068+
class TestIndicatorsSearcher:
4069+
def mock_search_after_output(self, fromDate, toDate, query, size, value, searchAfter):
4070+
if not searchAfter:
4071+
searchAfter = 0
4072+
4073+
return {'searchAfter': searchAfter + 1}
4074+
4075+
def test_search_indicators_by_page(self, mocker):
4076+
"""
4077+
Given:
4078+
- Searching indicators couple of times
4079+
- Server version in less than 6.1.0
4080+
When:
4081+
- Mocking search indicators using paging
4082+
Then:
4083+
- The page number is rising
4084+
- The searchAfter param is null
4085+
"""
4086+
from CommonServerPython import IndicatorsSearcher
4087+
mocker.patch.object(demisto, 'searchIndicators', return_value={})
4088+
4089+
search_indicators_obj_paging = IndicatorsSearcher()
4090+
search_indicators_obj_paging._can_use_search_after = False
4091+
4092+
for n in range(5):
4093+
search_indicators_obj_paging.search_indicators_by_version()
4094+
4095+
assert search_indicators_obj_paging._page == 5
4096+
assert not search_indicators_obj_paging._search_after_param
4097+
4098+
def test_search_indicators_by_search_after(self, mocker):
4099+
"""
4100+
Given:
4101+
- Searching indicators couple of times
4102+
- Server version in equal or higher than 6.1.0
4103+
When:
4104+
- Mocking search indicators using the searchAfter parameter
4105+
Then:
4106+
- The search after param is rising
4107+
- The page param is 0
4108+
"""
4109+
from CommonServerPython import IndicatorsSearcher
4110+
mocker.patch.object(demisto, 'searchIndicators', side_effect=self.mock_search_after_output)
4111+
4112+
search_indicators_obj_search_after = IndicatorsSearcher()
4113+
search_indicators_obj_search_after._can_use_search_after = True
4114+
4115+
for n in range(5):
4116+
search_indicators_obj_search_after.search_indicators_by_version()
4117+
4118+
assert search_indicators_obj_search_after._search_after_param == 5
4119+
assert search_indicators_obj_search_after._page == 0

‎Packs/Base/Scripts/GetIndicatorsByQuery/GetIndicatorsByQuery.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,14 +113,14 @@ def find_indicators_with_limit_loop(indicator_query: str, limit: int, total_fetc
113113
Finds indicators using while loop with demisto.searchIndicators, and returns result and last page
114114
"""
115115
iocs: List[dict] = []
116+
search_indicators = IndicatorsSearcher(page=next_page)
116117
if not last_found_len:
117118
last_found_len = total_fetched
118119
while last_found_len == PAGE_SIZE and limit and total_fetched < limit:
119-
fetched_iocs = demisto.searchIndicators(query=indicator_query, page=next_page, size=PAGE_SIZE).get('iocs')
120+
fetched_iocs = search_indicators.search_indicators_by_version(query=indicator_query, size=PAGE_SIZE).get('iocs')
120121
iocs.extend(fetched_iocs)
121122
last_found_len = len(fetched_iocs)
122123
total_fetched += last_found_len
123-
next_page += 1
124124
return list(map(lambda x: parse_ioc(x), iocs)), next_page
125125

126126

‎Packs/Base/Scripts/GetIndicatorsByQuery/GetIndicatorsByQuery.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ tags:
6060
- ml
6161
timeout: '0'
6262
type: python
63-
dockerimage: demisto/python3:3.8.5.10845
63+
dockerimage: demisto/python3:3.9.4.18682
6464
runas: DBotWeakRole
6565
runonce: false
6666
fromversion: 5.5.0

‎Packs/Base/Scripts/GetIndicatorsByQuery/get_indicators_by_query_test.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ def get_args_with_unpopulate():
5656
return args
5757

5858

59-
def search_indicators(query, page, size):
59+
def search_indicators(query, page, size, fromDate, toDate, value):
6060
return {'iocs': [ioc1, ioc2]}
6161

6262

@@ -95,7 +95,6 @@ def test_main_populate(mocker):
9595
def test_main_unpopulate(mocker):
9696
mocker.patch.object(demisto, 'args', side_effect=get_args_with_unpopulate)
9797
mocker.patch.object(demisto, 'searchIndicators', side_effect=search_indicators)
98-
9998
entry = main()
10099
indicators = entry['Contents']
101100
assert len(indicators) == 2

‎Packs/Base/pack_metadata.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"name": "Base",
33
"description": "The base pack for Cortex XSOAR.",
44
"support": "xsoar",
5-
"currentVersion": "1.8.9",
5+
"currentVersion": "1.8.10",
66
"author": "Cortex XSOAR",
77
"serverMinVersion": "6.0.0",
88
"url": "https://www.paloaltonetworks.com/cortex",

‎Packs/EDL/Integrations/EDL/EDL.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -214,13 +214,14 @@ def find_indicators_to_limit_loop(indicator_query: str, limit: int, total_fetche
214214
(tuple): The iocs and the last page
215215
"""
216216
iocs: List[dict] = []
217+
search_indicators = IndicatorsSearcher(page=next_page)
217218
if last_found_len is None:
218219
last_found_len = PAGE_SIZE
219220
if not last_found_len:
220221
last_found_len = total_fetched
221222
# last_found_len should be PAGE_SIZE (or PAGE_SIZE - 1, as observed for some users) for full pages
222223
while last_found_len in (PAGE_SIZE, PAGE_SIZE - 1) and limit and total_fetched < limit:
223-
fetched_iocs = demisto.searchIndicators(query=indicator_query, page=next_page, size=PAGE_SIZE).get('iocs')
224+
fetched_iocs = search_indicators.search_indicators_by_version(query=indicator_query, size=PAGE_SIZE).get('iocs')
224225
# In case the result from searchIndicators includes the key `iocs` but it's value is None
225226
fetched_iocs = fetched_iocs or []
226227

@@ -229,8 +230,7 @@ def find_indicators_to_limit_loop(indicator_query: str, limit: int, total_fetche
229230
for ioc in fetched_iocs)
230231
last_found_len = len(fetched_iocs)
231232
total_fetched += last_found_len
232-
next_page += 1
233-
return iocs, next_page
233+
return iocs, search_indicators.page
234234

235235

236236
def ip_groups_to_cidrs(ip_range_groups: list):

‎Packs/EDL/ReleaseNotes/1_0_14.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
2+
#### Integrations
3+
##### Palo Alto Networks PAN-OS EDL Service
4+
- Fixed an issue where searching more than 10K indicators failed when using ElasticSearch.

‎Packs/EDL/pack_metadata.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"name": "Palo Alto Networks PAN-OS EDL Service",
33
"description": "This integration provides External Dynamic List (EDL) as a service for the system indicators (Outbound feed).",
44
"support": "xsoar",
5-
"currentVersion": "1.0.13",
5+
"currentVersion": "1.0.14",
66
"author": "Cortex XSOAR",
77
"url": "https://www.paloaltonetworks.com/cortex",
88
"email": "",

0 commit comments

Comments
 (0)