From d2bfe2401839bce08aac94a2332a6edf134a6391 Mon Sep 17 00:00:00 2001 From: Johannes Messner Date: Fri, 2 Dec 2022 11:12:02 +0100 Subject: [PATCH 1/4] feat: native len for milvus Signed-off-by: Johannes Messner --- docarray/array/storage/milvus/seqlike.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docarray/array/storage/milvus/seqlike.py b/docarray/array/storage/milvus/seqlike.py index 1711c5b8080..d1ce651c0c6 100644 --- a/docarray/array/storage/milvus/seqlike.py +++ b/docarray/array/storage/milvus/seqlike.py @@ -1,6 +1,6 @@ from typing import Iterable, Iterator, Union, TYPE_CHECKING from docarray.array.storage.base.seqlike import BaseSequenceLikeMixin -from docarray.array.storage.milvus.backend import _batch_list +from docarray.array.storage.milvus.backend import _batch_list, _always_true_expr from docarray import Document @@ -56,3 +56,11 @@ def _extend(self, values: Iterable['Document'], **kwargs): payload = self._docs_to_milvus_payload(docs_batch) self._collection.insert(payload, **kwargs) self._offset2ids.extend([doc.id for doc in docs_batch]) + + def __len__(self): + with self.loaded_collection(): + res = self._collection.query( + expr=_always_true_expr('document_id'), + output_fields=['document_id'], + ) + return len(res) From e8422c8a2f8990fcf2689d6506bcb220a457e3a6 Mon Sep 17 00:00:00 2001 From: Johannes Messner Date: Fri, 2 Dec 2022 11:15:32 +0100 Subject: [PATCH 2/4] fix: make implementing len non-optional Signed-off-by: Johannes Messner --- docarray/array/storage/base/seqlike.py | 2 +- docs/advanced/document-store/extend.md | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docarray/array/storage/base/seqlike.py b/docarray/array/storage/base/seqlike.py index 5e46cafe607..ce89b82a3bf 100644 --- a/docarray/array/storage/base/seqlike.py +++ b/docarray/array/storage/base/seqlike.py @@ -50,7 +50,7 @@ def __eq__(self, other): ... def __len__(self): - return len(self._offset2ids) + ... def __iter__(self) -> Iterator['Document']: for _id in self._offset2ids: diff --git a/docs/advanced/document-store/extend.md b/docs/advanced/document-store/extend.md index a65d5ac32bb..591d2ce8832 100644 --- a/docs/advanced/document-store/extend.md +++ b/docs/advanced/document-store/extend.md @@ -145,6 +145,9 @@ class SequenceLikeMixin(BaseSequenceLikeMixin): def __add__(self, other: Union['Document', Iterable['Document']]): ... + def __len__(self): + ... + def insert(self, index: int, value: 'Document'): # Optional. By default, this will add a new item and update offset2id # if you want to customize this, make sure to handle offset2id @@ -158,10 +161,6 @@ class SequenceLikeMixin(BaseSequenceLikeMixin): # Optional. Override this if you have better implementation than appending one by one ... - def __len__(self): - # Optional. By default, this will rely on offset2id to get the length - ... - def __iter__(self) -> Iterator['Document']: # Optional. By default, this will rely on offset2id to iterate ... From 814cf6951e11ad0d0c05319112cd16f98ba7587f Mon Sep 17 00:00:00 2001 From: Johannes Messner Date: Thu, 15 Dec 2022 11:27:05 +0100 Subject: [PATCH 3/4] fix: raise pydantic version to fix to_dict bug Signed-off-by: Johannes Messner --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 63ce1795748..c1fc9e3662f 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ 'matplotlib', 'Pillow', 'fastapi', + 'pydantic>=1.9.0', 'uvicorn', ], 'full': [ @@ -63,6 +64,7 @@ 'scipy', 'av', 'fastapi', + 'pydantic>=1.9.0', 'uvicorn', 'strawberry-graphql', ], From 0e5e6771e5d99dada4f600d5ea179ebd7f35f86d Mon Sep 17 00:00:00 2001 From: Johannes Messner Date: Fri, 16 Dec 2022 11:26:05 +0100 Subject: [PATCH 4/4] test: add test Signed-off-by: Johannes Messner --- tests/unit/test_pydantic.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/unit/test_pydantic.py b/tests/unit/test_pydantic.py index 9ff6c8b052a..da67954efe5 100644 --- a/tests/unit/test_pydantic.py +++ b/tests/unit/test_pydantic.py @@ -159,6 +159,22 @@ def test_tags_int_float_str_bool(tag_type, tag_value, protocol): assert isinstance(dd, tag_type) +@pytest.mark.parametrize('protocol', ['protobuf', 'jsonschema']) +def test_infinity_no_coercion(protocol): + # Test for issue #948: https://github.com/docarray/docarray/issues/948 + d = Document() + d.tags['title'] = 'Infinity' + + d_pydantic = d.to_pydantic_model() + d_pydantic.tags['title'] = 'Infinity' + + d_json = d.to_json(protocol=protocol) + assert '"title": "Infinity"' in d_json + + d_dict = d.to_dict(protocol=protocol) + assert d_dict['tags']['title'] == 'Infinity' + + @pytest.mark.parametrize( 'blob', [None, b'123', bytes(Document()), bytes(bytearray(os.urandom(512 * 4)))] )