From a25b9303c46d5bfe17782c705b011f95249923bb Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Fri, 29 Apr 2022 09:16:55 +0200 Subject: [PATCH 1/2] perf: use less memory for apply --- docarray/array/mixins/parallel.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docarray/array/mixins/parallel.py b/docarray/array/mixins/parallel.py index acc588bde13..e1ab11cb10b 100644 --- a/docarray/array/mixins/parallel.py +++ b/docarray/array/mixins/parallel.py @@ -60,11 +60,8 @@ def apply(self: 'T', *args, **kwargs) -> 'T': # noqa: DAR201 :return: a new :class:`DocumentArray` """ - from ... import DocumentArray - - new_da = DocumentArray(self.map(*args, **kwargs)) - self.clear() - self.extend(new_da) + for doc in self.map(*args, **kwargs): + self[doc.id] = doc return self def map( From e3b484919d3d0e3de7c5c4289b0b52fb303139ab Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Fri, 29 Apr 2022 15:40:26 +0200 Subject: [PATCH 2/2] perf: optimize memory usage for apply batch --- docarray/array/mixins/parallel.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docarray/array/mixins/parallel.py b/docarray/array/mixins/parallel.py index e1ab11cb10b..4b215d33617 100644 --- a/docarray/array/mixins/parallel.py +++ b/docarray/array/mixins/parallel.py @@ -154,13 +154,8 @@ def apply_batch(self: 'T', *args, **kwargs) -> 'T': # noqa: DAR201 :return: a new :class:`DocumentArray` """ - from ... import DocumentArray - - new_da = DocumentArray() for _b in self.map_batch(*args, **kwargs): - new_da.extend(_b) - self.clear() - self.extend(new_da) + self[[doc.id for doc in _b]] = _b return self def map_batch(