From d3fcbac54e8d5a65299aba087af6a5b301d045ae Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 11:41:23 +0200 Subject: [PATCH 1/3] wip Signed-off-by: samsja --- docarray/store/file.py | 4 ++-- docarray/store/jac.py | 6 +++--- docarray/store/s3.py | 4 ++-- docarray/utils/__init__.py | 5 +++++ docarray/utils/cache.py | 2 +- docarray/utils/progress_bar.py | 4 ++-- tests/documentation/test_docstring.py | 0 tests/integrations/store/test_file.py | 4 ++-- 8 files changed, 17 insertions(+), 12 deletions(-) create mode 100644 tests/documentation/test_docstring.py diff --git a/docarray/store/file.py b/docarray/store/file.py index aeb355e2e17..e88e87b23f8 100644 --- a/docarray/store/file.py +++ b/docarray/store/file.py @@ -7,7 +7,7 @@ from docarray.store.abstract_doc_store import AbstractDocStore from docarray.store.exceptions import ConcurrentPushException from docarray.store.helpers import _from_binary_stream, _to_binary_stream -from docarray.utils.cache import get_cache_path +from docarray.utils.cache import _get_cache_path if TYPE_CHECKING: from docarray import BaseDoc, DocArray @@ -23,7 +23,7 @@ def _abs_filepath(name: str) -> Path: If it is a path, it is resolved to an absolute path. """ if not (name.startswith('/') or name.startswith('~') or name.startswith('.')): - name = str(get_cache_path() / name) + name = str(_get_cache_path() / name) if name.startswith('~'): name = str(Path.home() / name[2:]) return Path(name).resolve() diff --git a/docarray/store/jac.py b/docarray/store/jac.py index 75685e61cb4..969df0a6233 100644 --- a/docarray/store/jac.py +++ b/docarray/store/jac.py @@ -24,7 +24,7 @@ get_version_info, raise_req_error, ) -from docarray.utils.cache import get_cache_path +from docarray.utils.cache import _get_cache_path if TYPE_CHECKING: # pragma: no cover import io @@ -331,7 +331,7 @@ def pull_stream( _BufferedCachingRequestReader, io.BufferedReader ] = _BufferedCachingRequestReader(r, tmp_cache_file) - cache_file = get_cache_path() / f'{save_name}.da' + cache_file = _get_cache_path() / f'{save_name}.da' if local_cache and cache_file.exists(): _cache_len = cache_file.stat().st_size if _cache_len == int(r.headers['Content-length']): @@ -354,7 +354,7 @@ def pull_stream( if local_cache: if isinstance(_source, _BufferedCachingRequestReader): - Path(get_cache_path()).mkdir(parents=True, exist_ok=True) + Path(_get_cache_path()).mkdir(parents=True, exist_ok=True) tmp_cache_file.rename(cache_file) else: _source.close() diff --git a/docarray/store/s3.py b/docarray/store/s3.py index b038e7c4b41..8c5604b6d35 100644 --- a/docarray/store/s3.py +++ b/docarray/store/s3.py @@ -10,7 +10,7 @@ from docarray.store.abstract_doc_store import AbstractDocStore from docarray.store.helpers import _from_binary_stream, _to_binary_stream -from docarray.utils.cache import get_cache_path +from docarray.utils.cache import _get_cache_path if TYPE_CHECKING: # pragma: no cover from docarray import BaseDoc, DocArray @@ -212,7 +212,7 @@ def pull_stream( bucket, name = name.split('/', 1) save_name = name.replace('/', '_') - cache_path = get_cache_path() / f'{save_name}.da' + cache_path = _get_cache_path() / f'{save_name}.da' source = _BufferedCachingReader( open(f"s3://{bucket}/{name}.da", 'rb', compression='.gz'), diff --git a/docarray/utils/__init__.py b/docarray/utils/__init__.py index e69de29bb2d..a1a68cf806e 100644 --- a/docarray/utils/__init__.py +++ b/docarray/utils/__init__.py @@ -0,0 +1,5 @@ +from docarray.utils.filter import filter_docs +from docarray.utils.find import find, find_batched +from docarray.utils.map import map_docs, map_docs_batch + +__all__ = ['filter_docs', 'find', 'find_batched', 'map_docs', 'map_docs_batch'] diff --git a/docarray/utils/cache.py b/docarray/utils/cache.py index 4df305414b3..249c4f9d179 100644 --- a/docarray/utils/cache.py +++ b/docarray/utils/cache.py @@ -4,7 +4,7 @@ @lru_cache(maxsize=None) -def get_cache_path() -> Path: +def _get_cache_path() -> Path: """ Get the path to the cache directory. diff --git a/docarray/utils/progress_bar.py b/docarray/utils/progress_bar.py index ad7bb3dcd5d..4750c509a1a 100644 --- a/docarray/utils/progress_bar.py +++ b/docarray/utils/progress_bar.py @@ -12,7 +12,7 @@ ) -class QPSColumn(TextColumn): +class _QPSColumn(TextColumn): def render(self, task) -> Text: if task.speed: _text = f'{task.speed:.0f} QPS' @@ -34,7 +34,7 @@ def _get_pbar(disable: bool, total: Optional[int] = None): BarColumn(), MofNCompleteColumn(), '•', - QPSColumn('{task.speed} QPS', justify='right', style='progress.data.speed'), + _QPSColumn('{task.speed} QPS', justify='right', style='progress.data.speed'), '•', TimeRemainingColumn() if total else TimeElapsedColumn(), '•', diff --git a/tests/documentation/test_docstring.py b/tests/documentation/test_docstring.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integrations/store/test_file.py b/tests/integrations/store/test_file.py index 43205260abd..0019f6970c0 100644 --- a/tests/integrations/store/test_file.py +++ b/tests/integrations/store/test_file.py @@ -6,7 +6,7 @@ from docarray import DocArray from docarray.documents import TextDoc from docarray.store.file import ConcurrentPushException, FileDocStore -from docarray.utils.cache import get_cache_path +from docarray.utils.cache import _get_cache_path from tests.integrations.store import gen_text_docs, get_test_da, profile_memory DA_LEN: int = 2**10 @@ -14,7 +14,7 @@ def test_path_resolution(): - assert FileDocStore._abs_filepath('meow') == get_cache_path() / 'meow' + assert FileDocStore._abs_filepath('meow') == _get_cache_path() / 'meow' assert FileDocStore._abs_filepath('/meow') == Path('/meow') assert FileDocStore._abs_filepath('~/meow') == Path.home() / 'meow' assert FileDocStore._abs_filepath('./meow') == Path.cwd() / 'meow' From df3ac0223d089d4c93249c4e4b6b2ff869e0ac89 Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 12:26:05 +0200 Subject: [PATCH 2/3] fix: cleanup namespace utils Signed-off-by: samsja --- docarray/array/abstract_array.py | 2 +- docarray/array/array/io.py | 8 ++++---- docarray/array/array/sequence_indexing_mixin.py | 2 +- docarray/array/stacked/array_stacked.py | 4 ++-- docarray/base_doc/mixins/io.py | 4 ++-- docarray/data/torch_dataset.py | 2 +- docarray/documents/audio.py | 2 +- docarray/documents/image.py | 2 +- docarray/documents/point_cloud/point_cloud_3d.py | 2 +- docarray/documents/point_cloud/points_and_colors.py | 2 +- docarray/documents/video.py | 2 +- docarray/index/abstract.py | 4 ++-- docarray/index/backends/hnswlib.py | 2 +- docarray/store/file.py | 2 +- docarray/store/helpers.py | 2 +- docarray/store/jac.py | 2 +- docarray/store/s3.py | 2 +- docarray/typing/__init__.py | 2 +- docarray/typing/tensor/__init__.py | 2 +- docarray/typing/tensor/audio/__init__.py | 2 +- docarray/typing/tensor/audio/abstract_audio_tensor.py | 2 +- docarray/typing/tensor/audio/audio_tensor.py | 2 +- docarray/typing/tensor/embedding/__init__.py | 2 +- docarray/typing/tensor/embedding/embedding.py | 2 +- docarray/typing/tensor/image/__init__.py | 2 +- docarray/typing/tensor/image/abstract_image_tensor.py | 2 +- docarray/typing/tensor/image/image_tensor.py | 2 +- docarray/typing/tensor/tensor.py | 2 +- docarray/typing/tensor/video/__init__.py | 2 +- docarray/typing/tensor/video/video_tensor.py | 2 +- docarray/typing/tensor/video/video_tensor_mixin.py | 2 +- docarray/typing/url/audio_url.py | 2 +- docarray/typing/url/image_url.py | 2 +- docarray/typing/url/video_url.py | 2 +- docarray/utils/__init__.py | 5 ----- docarray/utils/{query_language => _internal}/__init__.py | 0 docarray/utils/{ => _internal}/_typing.py | 0 docarray/utils/{ => _internal}/cache.py | 0 docarray/utils/{ => _internal}/compress.py | 2 +- docarray/utils/{ => _internal}/misc.py | 0 docarray/utils/{ => _internal}/progress_bar.py | 0 docarray/utils/_internal/query_language/__init__.py | 0 docarray/utils/{ => _internal}/query_language/lookup.py | 0 .../utils/{ => _internal}/query_language/query_parser.py | 2 +- docarray/utils/filter.py | 2 +- tests/integrations/document/test_proto.py | 2 +- tests/integrations/predefined_document/test_audio.py | 2 +- tests/integrations/predefined_document/test_image.py | 2 +- .../integrations/predefined_document/test_point_cloud.py | 2 +- tests/integrations/predefined_document/test_video.py | 2 +- tests/integrations/store/test_file.py | 2 +- tests/integrations/typing/test_tensor.py | 2 +- tests/integrations/typing/test_tensorflow_tensor.py | 2 +- tests/integrations/typing/test_tensors_interop.py | 2 +- tests/units/array/stack/test_array_stacked_tf.py | 2 +- tests/units/array/test_array.py | 2 +- .../tensorflow_backend/test_basics.py | 2 +- .../tensorflow_backend/test_metrics.py | 2 +- .../tensorflow_backend/test_retrieval.py | 2 +- tests/units/document/proto/test_document_proto.py | 2 +- tests/units/typing/tensor/test_audio_tensor.py | 2 +- tests/units/typing/tensor/test_tensor_flow_tensor.py | 2 +- tests/units/typing/tensor/test_video_tensor.py | 2 +- tests/units/typing/url/test_audio_url.py | 2 +- tests/units/typing/url/test_video_url.py | 2 +- tests/units/util/query_language/test_lookup.py | 2 +- tests/units/util/test_typing.py | 4 ++-- 67 files changed, 66 insertions(+), 71 deletions(-) rename docarray/utils/{query_language => _internal}/__init__.py (100%) rename docarray/utils/{ => _internal}/_typing.py (100%) rename docarray/utils/{ => _internal}/cache.py (100%) rename docarray/utils/{ => _internal}/compress.py (97%) rename docarray/utils/{ => _internal}/misc.py (100%) rename docarray/utils/{ => _internal}/progress_bar.py (100%) create mode 100644 docarray/utils/_internal/query_language/__init__.py rename docarray/utils/{ => _internal}/query_language/lookup.py (100%) rename docarray/utils/{ => _internal}/query_language/query_parser.py (98%) diff --git a/docarray/array/abstract_array.py b/docarray/array/abstract_array.py index 762f37cfb42..0669a927451 100644 --- a/docarray/array/abstract_array.py +++ b/docarray/array/abstract_array.py @@ -22,7 +22,7 @@ from docarray.base_doc import BaseDoc from docarray.display.document_array_summary import DocArraySummary from docarray.typing.abstract_type import AbstractType -from docarray.utils._typing import change_cls_name +from docarray.utils._internal._typing import change_cls_name if TYPE_CHECKING: from docarray.proto import DocArrayProto, NodeProto diff --git a/docarray/array/array/io.py b/docarray/array/array/io.py index 8659af08588..1d8de498638 100644 --- a/docarray/array/array/io.py +++ b/docarray/array/array/io.py @@ -31,7 +31,7 @@ _all_access_paths_valid, _dict_to_access_paths, ) -from docarray.utils.compress import _decompress_bytes, _get_compress_ctx +from docarray.utils._internal.compress import _decompress_bytes, _get_compress_ctx if TYPE_CHECKING: import pandas as pd @@ -201,7 +201,7 @@ def to_binary_stream( from rich import filesize if show_progress: - from docarray.utils.progress_bar import _get_progressbar + from docarray.utils._internal.progress_bar import _get_progressbar pbar, t = _get_progressbar( 'Serializing', disable=not show_progress, total=len(self) @@ -564,7 +564,7 @@ def _load_binary_all( else: from rich import filesize - from docarray.utils.progress_bar import _get_progressbar + from docarray.utils._internal.progress_bar import _get_progressbar # 1 byte (uint8) # 8 bytes (uint64) @@ -629,7 +629,7 @@ def _load_binary_stream( num_docs = int.from_bytes(version_numdocs_lendoc0[1:9], 'big', signed=False) if show_progress: - from docarray.utils.progress_bar import _get_progressbar + from docarray.utils._internal.progress_bar import _get_progressbar pbar, t = _get_progressbar( 'Deserializing', disable=not show_progress, total=num_docs diff --git a/docarray/array/array/sequence_indexing_mixin.py b/docarray/array/array/sequence_indexing_mixin.py index cc12b6a9a9f..01db74c0218 100644 --- a/docarray/array/array/sequence_indexing_mixin.py +++ b/docarray/array/array/sequence_indexing_mixin.py @@ -14,7 +14,7 @@ import numpy as np -from docarray.utils.misc import is_torch_available +from docarray.utils._internal.misc import is_torch_available T_item = TypeVar('T_item') T = TypeVar('T', bound='IndexingSequenceMixin') diff --git a/docarray/array/stacked/array_stacked.py b/docarray/array/stacked/array_stacked.py index d2630a4b655..1445b97379e 100644 --- a/docarray/array/stacked/array_stacked.py +++ b/docarray/array/stacked/array_stacked.py @@ -26,8 +26,8 @@ from docarray.base_doc.mixins.io import _type_to_protobuf from docarray.typing import NdArray from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._typing import is_tensor_union -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal._typing import is_tensor_union +from docarray.utils._internal.misc import is_tf_available, is_torch_available if TYPE_CHECKING: from pydantic.fields import ModelField diff --git a/docarray/base_doc/mixins/io.py b/docarray/base_doc/mixins/io.py index 13f723df3a3..e0fde4b5d4f 100644 --- a/docarray/base_doc/mixins/io.py +++ b/docarray/base_doc/mixins/io.py @@ -20,8 +20,8 @@ from docarray.base_doc.base_node import BaseNode from docarray.typing import NdArray from docarray.typing.proto_register import _PROTO_TYPE_NAME_TO_CLASS -from docarray.utils.compress import _compress_bytes, _decompress_bytes -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.compress import _compress_bytes, _decompress_bytes +from docarray.utils._internal.misc import is_tf_available, is_torch_available tf_available = is_tf_available() if tf_available: diff --git a/docarray/data/torch_dataset.py b/docarray/data/torch_dataset.py index 3b5cb0f87a9..dd58035cd33 100644 --- a/docarray/data/torch_dataset.py +++ b/docarray/data/torch_dataset.py @@ -4,7 +4,7 @@ from docarray import BaseDoc, DocArray, DocArrayStacked from docarray.typing import TorchTensor -from docarray.utils._typing import change_cls_name +from docarray.utils._internal._typing import change_cls_name T_doc = TypeVar('T_doc', bound=BaseDoc) diff --git a/docarray/documents/audio.py b/docarray/documents/audio.py index ab0bbe00b34..e27abcb5408 100644 --- a/docarray/documents/audio.py +++ b/docarray/documents/audio.py @@ -7,7 +7,7 @@ from docarray.typing.bytes.audio_bytes import AudioBytes from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.audio.audio_tensor import AudioTensor -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/documents/image.py b/docarray/documents/image.py index f2375637de6..bf281f83d8f 100644 --- a/docarray/documents/image.py +++ b/docarray/documents/image.py @@ -6,7 +6,7 @@ from docarray.typing import AnyEmbedding, ImageBytes, ImageUrl from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.image.image_tensor import ImageTensor -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available T = TypeVar('T', bound='ImageDoc') diff --git a/docarray/documents/point_cloud/point_cloud_3d.py b/docarray/documents/point_cloud/point_cloud_3d.py index 3cee613f596..25e0e6437d1 100644 --- a/docarray/documents/point_cloud/point_cloud_3d.py +++ b/docarray/documents/point_cloud/point_cloud_3d.py @@ -6,7 +6,7 @@ from docarray.documents.point_cloud.points_and_colors import PointsAndColors from docarray.typing import AnyEmbedding, PointCloud3DUrl from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/documents/point_cloud/points_and_colors.py b/docarray/documents/point_cloud/points_and_colors.py index af4917bf5a7..82ce82f9ba1 100644 --- a/docarray/documents/point_cloud/points_and_colors.py +++ b/docarray/documents/point_cloud/points_and_colors.py @@ -5,7 +5,7 @@ from docarray.base_doc import BaseDoc from docarray.typing import AnyTensor from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/documents/video.py b/docarray/documents/video.py index 3770ee245fc..679f10ea8ba 100644 --- a/docarray/documents/video.py +++ b/docarray/documents/video.py @@ -8,7 +8,7 @@ from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.video.video_tensor import VideoTensor from docarray.typing.url.video_url import VideoUrl -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/index/abstract.py b/docarray/index/abstract.py index 3eac3d7f869..c587846c8c4 100644 --- a/docarray/index/abstract.py +++ b/docarray/index/abstract.py @@ -26,9 +26,9 @@ from docarray import BaseDoc, DocArray from docarray.array.abstract_array import AnyDocArray from docarray.typing import AnyTensor -from docarray.utils._typing import unwrap_optional_type +from docarray.utils._internal._typing import unwrap_optional_type +from docarray.utils._internal.misc import is_tf_available, torch_imported from docarray.utils.find import FindResult, _FindResult -from docarray.utils.misc import is_tf_available, torch_imported if TYPE_CHECKING: from pydantic.fields import ModelField diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 38b827539ab..ab8aa6e56c1 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -29,9 +29,9 @@ _raise_not_supported, ) from docarray.proto import DocumentProto +from docarray.utils._internal.misc import is_np_int, is_tf_available, is_torch_available from docarray.utils.filter import filter_docs from docarray.utils.find import _FindResult -from docarray.utils.misc import is_np_int, is_tf_available, is_torch_available TSchema = TypeVar('TSchema', bound=BaseDoc) T = TypeVar('T', bound='HnswDocumentIndex') diff --git a/docarray/store/file.py b/docarray/store/file.py index e88e87b23f8..bb79162109b 100644 --- a/docarray/store/file.py +++ b/docarray/store/file.py @@ -7,7 +7,7 @@ from docarray.store.abstract_doc_store import AbstractDocStore from docarray.store.exceptions import ConcurrentPushException from docarray.store.helpers import _from_binary_stream, _to_binary_stream -from docarray.utils.cache import _get_cache_path +from docarray.utils._internal.cache import _get_cache_path if TYPE_CHECKING: from docarray import BaseDoc, DocArray diff --git a/docarray/store/helpers.py b/docarray/store/helpers.py index 25e40991e68..457d160d41c 100644 --- a/docarray/store/helpers.py +++ b/docarray/store/helpers.py @@ -6,7 +6,7 @@ from rich import filesize from typing_extensions import TYPE_CHECKING, Protocol -from docarray.utils.progress_bar import _get_progressbar +from docarray.utils._internal.progress_bar import _get_progressbar if TYPE_CHECKING: from pathlib import Path diff --git a/docarray/store/jac.py b/docarray/store/jac.py index 969df0a6233..c9350fae18a 100644 --- a/docarray/store/jac.py +++ b/docarray/store/jac.py @@ -24,7 +24,7 @@ get_version_info, raise_req_error, ) -from docarray.utils.cache import _get_cache_path +from docarray.utils._internal.cache import _get_cache_path if TYPE_CHECKING: # pragma: no cover import io diff --git a/docarray/store/s3.py b/docarray/store/s3.py index 8c5604b6d35..7711432900c 100644 --- a/docarray/store/s3.py +++ b/docarray/store/s3.py @@ -10,7 +10,7 @@ from docarray.store.abstract_doc_store import AbstractDocStore from docarray.store.helpers import _from_binary_stream, _to_binary_stream -from docarray.utils.cache import _get_cache_path +from docarray.utils._internal.cache import _get_cache_path if TYPE_CHECKING: # pragma: no cover from docarray import BaseDoc, DocArray diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index cd48ff7c26b..daf89af9fe8 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -38,7 +38,7 @@ 'ImageNdArray', ] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/__init__.py b/docarray/typing/tensor/__init__.py index 07a27292aee..f2be5ecc0ac 100644 --- a/docarray/typing/tensor/__init__.py +++ b/docarray/typing/tensor/__init__.py @@ -13,7 +13,7 @@ 'TensorFlowTensor', ] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/audio/__init__.py b/docarray/typing/tensor/audio/__init__.py index 4b1b7edc34d..4f730451603 100644 --- a/docarray/typing/tensor/audio/__init__.py +++ b/docarray/typing/tensor/audio/__init__.py @@ -2,7 +2,7 @@ __all__ = ['AudioNdArray'] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/audio/abstract_audio_tensor.py b/docarray/typing/tensor/audio/abstract_audio_tensor.py index 7e677bcf952..e84b4ebc13e 100644 --- a/docarray/typing/tensor/audio/abstract_audio_tensor.py +++ b/docarray/typing/tensor/audio/abstract_audio_tensor.py @@ -3,7 +3,7 @@ from typing import Any, BinaryIO, Dict, TypeVar, Union from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook T = TypeVar('T', bound='AbstractAudioTensor') diff --git a/docarray/typing/tensor/audio/audio_tensor.py b/docarray/typing/tensor/audio/audio_tensor.py index 188ccba26a5..7d1f4a9e315 100644 --- a/docarray/typing/tensor/audio/audio_tensor.py +++ b/docarray/typing/tensor/audio/audio_tensor.py @@ -1,7 +1,7 @@ from typing import Union from docarray.typing.tensor.audio.audio_ndarray import AudioNdArray -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/embedding/__init__.py b/docarray/typing/tensor/embedding/__init__.py index 6cc8b1d6008..f844b7abaa9 100644 --- a/docarray/typing/tensor/embedding/__init__.py +++ b/docarray/typing/tensor/embedding/__init__.py @@ -3,7 +3,7 @@ __all__ = ['NdArrayEmbedding', 'AnyEmbedding'] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/embedding/embedding.py b/docarray/typing/tensor/embedding/embedding.py index a8bb78a33d2..2eb1fa7e842 100644 --- a/docarray/typing/tensor/embedding/embedding.py +++ b/docarray/typing/tensor/embedding/embedding.py @@ -1,7 +1,7 @@ from typing import Union from docarray.typing.tensor.embedding.ndarray import NdArrayEmbedding -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/image/__init__.py b/docarray/typing/tensor/image/__init__.py index 03bd92a5342..2fb32d452ab 100644 --- a/docarray/typing/tensor/image/__init__.py +++ b/docarray/typing/tensor/image/__init__.py @@ -3,7 +3,7 @@ __all__ = ['ImageNdArray', 'ImageTensor'] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/image/abstract_image_tensor.py b/docarray/typing/tensor/image/abstract_image_tensor.py index 97b8fd8e23a..b02d1452f63 100644 --- a/docarray/typing/tensor/image/abstract_image_tensor.py +++ b/docarray/typing/tensor/image/abstract_image_tensor.py @@ -3,7 +3,7 @@ from abc import ABC from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook class AbstractImageTensor(AbstractTensor, ABC): diff --git a/docarray/typing/tensor/image/image_tensor.py b/docarray/typing/tensor/image/image_tensor.py index 747b080b455..af439ce607e 100644 --- a/docarray/typing/tensor/image/image_tensor.py +++ b/docarray/typing/tensor/image/image_tensor.py @@ -1,7 +1,7 @@ from typing import Union from docarray.typing.tensor.image.image_ndarray import ImageNdArray -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/tensor.py b/docarray/typing/tensor/tensor.py index 6cff0b99e26..19a89cbed3f 100644 --- a/docarray/typing/tensor/tensor.py +++ b/docarray/typing/tensor/tensor.py @@ -1,7 +1,7 @@ from typing import Union from docarray.typing.tensor.ndarray import NdArray -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/video/__init__.py b/docarray/typing/tensor/video/__init__.py index add0c958d86..424758cfc91 100644 --- a/docarray/typing/tensor/video/__init__.py +++ b/docarray/typing/tensor/video/__init__.py @@ -2,7 +2,7 @@ __all__ = ['VideoNdArray'] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/video/video_tensor.py b/docarray/typing/tensor/video/video_tensor.py index 974f409fcb3..fa4fbb47d6b 100644 --- a/docarray/typing/tensor/video/video_tensor.py +++ b/docarray/typing/tensor/video/video_tensor.py @@ -1,7 +1,7 @@ from typing import Union from docarray.typing.tensor.video.video_ndarray import VideoNdArray -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/video/video_tensor_mixin.py b/docarray/typing/tensor/video/video_tensor_mixin.py index cfa5a46ea70..5dd11fb2da1 100644 --- a/docarray/typing/tensor/video/video_tensor_mixin.py +++ b/docarray/typing/tensor/video/video_tensor_mixin.py @@ -7,7 +7,7 @@ from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.audio.audio_tensor import AudioTensor -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook T = TypeVar('T', bound='VideoTensorMixin') diff --git a/docarray/typing/url/audio_url.py b/docarray/typing/url/audio_url.py index c971f6d364f..dbfdd84029b 100644 --- a/docarray/typing/url/audio_url.py +++ b/docarray/typing/url/audio_url.py @@ -7,7 +7,7 @@ from docarray.typing.proto_register import _register_proto from docarray.typing.url.any_url import AnyUrl from docarray.typing.url.filetypes import AUDIO_FILE_FORMATS -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook if TYPE_CHECKING: from pydantic import BaseConfig diff --git a/docarray/typing/url/image_url.py b/docarray/typing/url/image_url.py index b5c40b71e2d..25f07ed4707 100644 --- a/docarray/typing/url/image_url.py +++ b/docarray/typing/url/image_url.py @@ -5,7 +5,7 @@ from docarray.typing.proto_register import _register_proto from docarray.typing.url.any_url import AnyUrl -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook if TYPE_CHECKING: from pydantic import BaseConfig diff --git a/docarray/typing/url/video_url.py b/docarray/typing/url/video_url.py index 44c2d33f9b2..1ff5e96c9c4 100644 --- a/docarray/typing/url/video_url.py +++ b/docarray/typing/url/video_url.py @@ -4,7 +4,7 @@ from docarray.typing.bytes.video_bytes import VideoLoadResult from docarray.typing.proto_register import _register_proto from docarray.typing.url.any_url import AnyUrl -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook if TYPE_CHECKING: from pydantic import BaseConfig diff --git a/docarray/utils/__init__.py b/docarray/utils/__init__.py index a1a68cf806e..e69de29bb2d 100644 --- a/docarray/utils/__init__.py +++ b/docarray/utils/__init__.py @@ -1,5 +0,0 @@ -from docarray.utils.filter import filter_docs -from docarray.utils.find import find, find_batched -from docarray.utils.map import map_docs, map_docs_batch - -__all__ = ['filter_docs', 'find', 'find_batched', 'map_docs', 'map_docs_batch'] diff --git a/docarray/utils/query_language/__init__.py b/docarray/utils/_internal/__init__.py similarity index 100% rename from docarray/utils/query_language/__init__.py rename to docarray/utils/_internal/__init__.py diff --git a/docarray/utils/_typing.py b/docarray/utils/_internal/_typing.py similarity index 100% rename from docarray/utils/_typing.py rename to docarray/utils/_internal/_typing.py diff --git a/docarray/utils/cache.py b/docarray/utils/_internal/cache.py similarity index 100% rename from docarray/utils/cache.py rename to docarray/utils/_internal/cache.py diff --git a/docarray/utils/compress.py b/docarray/utils/_internal/compress.py similarity index 97% rename from docarray/utils/compress.py rename to docarray/utils/_internal/compress.py index d427e998c89..0ff00f8eecd 100644 --- a/docarray/utils/compress.py +++ b/docarray/utils/_internal/compress.py @@ -1,4 +1,4 @@ -from typing import Optional, Callable, IO +from typing import IO, Callable, Optional def _compress_bytes(data: bytes, algorithm: Optional[str] = None) -> bytes: diff --git a/docarray/utils/misc.py b/docarray/utils/_internal/misc.py similarity index 100% rename from docarray/utils/misc.py rename to docarray/utils/_internal/misc.py diff --git a/docarray/utils/progress_bar.py b/docarray/utils/_internal/progress_bar.py similarity index 100% rename from docarray/utils/progress_bar.py rename to docarray/utils/_internal/progress_bar.py diff --git a/docarray/utils/_internal/query_language/__init__.py b/docarray/utils/_internal/query_language/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docarray/utils/query_language/lookup.py b/docarray/utils/_internal/query_language/lookup.py similarity index 100% rename from docarray/utils/query_language/lookup.py rename to docarray/utils/_internal/query_language/lookup.py diff --git a/docarray/utils/query_language/query_parser.py b/docarray/utils/_internal/query_language/query_parser.py similarity index 98% rename from docarray/utils/query_language/query_parser.py rename to docarray/utils/_internal/query_language/query_parser.py index 8bce97f6e0d..b8381584a94 100644 --- a/docarray/utils/query_language/query_parser.py +++ b/docarray/utils/_internal/query_language/query_parser.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Optional, Union -from docarray.utils.query_language.lookup import ( +from docarray.utils._internal.query_language.lookup import ( LookupLeaf, LookupNode, LookupTreeElem, diff --git a/docarray/utils/filter.py b/docarray/utils/filter.py index 7a9887364fc..a68641a7b13 100644 --- a/docarray/utils/filter.py +++ b/docarray/utils/filter.py @@ -63,7 +63,7 @@ class MyDocument(BaseDoc): :return: A DocArray containing the Documents in `docs` that fulfill the filter conditions in the `query` """ - from docarray.utils.query_language.query_parser import QueryParser + from docarray.utils._internal.query_language.query_parser import QueryParser if query: query = query if not isinstance(query, str) else json.loads(query) diff --git a/tests/integrations/document/test_proto.py b/tests/integrations/document/test_proto.py index c1f38d05b41..02b9aae3f00 100644 --- a/tests/integrations/document/test_proto.py +++ b/tests/integrations/document/test_proto.py @@ -18,7 +18,7 @@ TorchTensor, ) from docarray.typing.tensor import NdArrayEmbedding -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/integrations/predefined_document/test_audio.py b/tests/integrations/predefined_document/test_audio.py index c25aab9b1ab..2ba207245f7 100644 --- a/tests/integrations/predefined_document/test_audio.py +++ b/tests/integrations/predefined_document/test_audio.py @@ -10,7 +10,7 @@ from docarray.documents import AudioDoc from docarray.typing import AudioUrl from docarray.typing.tensor.audio import AudioNdArray, AudioTorchTensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available from tests import TOYDATA_DIR tf_available = is_tf_available() diff --git a/tests/integrations/predefined_document/test_image.py b/tests/integrations/predefined_document/test_image.py index 92e19d09a14..f77e945d714 100644 --- a/tests/integrations/predefined_document/test_image.py +++ b/tests/integrations/predefined_document/test_image.py @@ -5,7 +5,7 @@ from docarray import BaseDoc from docarray.documents import ImageDoc -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/integrations/predefined_document/test_point_cloud.py b/tests/integrations/predefined_document/test_point_cloud.py index 76b7e0236d8..b8a75914f26 100644 --- a/tests/integrations/predefined_document/test_point_cloud.py +++ b/tests/integrations/predefined_document/test_point_cloud.py @@ -5,7 +5,7 @@ from docarray import BaseDoc from docarray.documents import PointCloud3D -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available from tests import TOYDATA_DIR tf_available = is_tf_available() diff --git a/tests/integrations/predefined_document/test_video.py b/tests/integrations/predefined_document/test_video.py index e208e3890c9..606dad6a6c1 100644 --- a/tests/integrations/predefined_document/test_video.py +++ b/tests/integrations/predefined_document/test_video.py @@ -6,7 +6,7 @@ from docarray import BaseDoc from docarray.documents import VideoDoc from docarray.typing import AudioNdArray, NdArray, VideoNdArray -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available from tests import TOYDATA_DIR tf_available = is_tf_available() diff --git a/tests/integrations/store/test_file.py b/tests/integrations/store/test_file.py index 0019f6970c0..4b6a72c5b62 100644 --- a/tests/integrations/store/test_file.py +++ b/tests/integrations/store/test_file.py @@ -6,7 +6,7 @@ from docarray import DocArray from docarray.documents import TextDoc from docarray.store.file import ConcurrentPushException, FileDocStore -from docarray.utils.cache import _get_cache_path +from docarray.utils._internal.cache import _get_cache_path from tests.integrations.store import gen_text_docs, get_test_da, profile_memory DA_LEN: int = 2**10 diff --git a/tests/integrations/typing/test_tensor.py b/tests/integrations/typing/test_tensor.py index ba15e2d5c94..6b32432d971 100644 --- a/tests/integrations/typing/test_tensor.py +++ b/tests/integrations/typing/test_tensor.py @@ -4,7 +4,7 @@ from docarray import BaseDoc from docarray.typing import AnyTensor, NdArray, TorchTensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/integrations/typing/test_tensorflow_tensor.py b/tests/integrations/typing/test_tensorflow_tensor.py index 84505968090..edd64e3dcbb 100644 --- a/tests/integrations/typing/test_tensorflow_tensor.py +++ b/tests/integrations/typing/test_tensorflow_tensor.py @@ -1,7 +1,7 @@ import pytest from docarray import BaseDoc -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/integrations/typing/test_tensors_interop.py b/tests/integrations/typing/test_tensors_interop.py index d59751f21af..47023dca96a 100644 --- a/tests/integrations/typing/test_tensors_interop.py +++ b/tests/integrations/typing/test_tensors_interop.py @@ -8,7 +8,7 @@ NdArrayEmbedding, TorchEmbedding, ) -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/array/stack/test_array_stacked_tf.py b/tests/units/array/stack/test_array_stacked_tf.py index e82bfc7716a..0ec91268575 100644 --- a/tests/units/array/stack/test_array_stacked_tf.py +++ b/tests/units/array/stack/test_array_stacked_tf.py @@ -5,7 +5,7 @@ from docarray import BaseDoc, DocArray from docarray.array import DocArrayStacked from docarray.typing import AnyTensor, NdArray -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/array/test_array.py b/tests/units/array/test_array.py index bab9ccd1313..be4fa6fa505 100644 --- a/tests/units/array/test_array.py +++ b/tests/units/array/test_array.py @@ -6,7 +6,7 @@ from docarray import BaseDoc, DocArray from docarray.typing import ImageUrl, NdArray, TorchTensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/computation_backends/tensorflow_backend/test_basics.py b/tests/units/computation_backends/tensorflow_backend/test_basics.py index 6eed1a58cf0..ae5f9b44264 100644 --- a/tests/units/computation_backends/tensorflow_backend/test_basics.py +++ b/tests/units/computation_backends/tensorflow_backend/test_basics.py @@ -1,7 +1,7 @@ import numpy as np import pytest -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/computation_backends/tensorflow_backend/test_metrics.py b/tests/units/computation_backends/tensorflow_backend/test_metrics.py index 863dd18fe4c..196297adf0b 100644 --- a/tests/units/computation_backends/tensorflow_backend/test_metrics.py +++ b/tests/units/computation_backends/tensorflow_backend/test_metrics.py @@ -1,6 +1,6 @@ import pytest -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/computation_backends/tensorflow_backend/test_retrieval.py b/tests/units/computation_backends/tensorflow_backend/test_retrieval.py index feac992d760..283d7f8b44a 100644 --- a/tests/units/computation_backends/tensorflow_backend/test_retrieval.py +++ b/tests/units/computation_backends/tensorflow_backend/test_retrieval.py @@ -1,6 +1,6 @@ import pytest -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/document/proto/test_document_proto.py b/tests/units/document/proto/test_document_proto.py index e6e8a58fa99..1642c17631d 100644 --- a/tests/units/document/proto/test_document_proto.py +++ b/tests/units/document/proto/test_document_proto.py @@ -7,7 +7,7 @@ from docarray import DocArray from docarray.base_doc import BaseDoc from docarray.typing import NdArray, TorchTensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available if is_tf_available(): import tensorflow as tf diff --git a/tests/units/typing/tensor/test_audio_tensor.py b/tests/units/typing/tensor/test_audio_tensor.py index 3bc20a1c7f6..4a145f3c4b7 100644 --- a/tests/units/typing/tensor/test_audio_tensor.py +++ b/tests/units/typing/tensor/test_audio_tensor.py @@ -8,7 +8,7 @@ from docarray import BaseDoc from docarray.typing.tensor.audio.audio_ndarray import AudioNdArray from docarray.typing.tensor.audio.audio_torch_tensor import AudioTorchTensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/typing/tensor/test_tensor_flow_tensor.py b/tests/units/typing/tensor/test_tensor_flow_tensor.py index 5c7d942a02d..3a51f5a95aa 100644 --- a/tests/units/typing/tensor/test_tensor_flow_tensor.py +++ b/tests/units/typing/tensor/test_tensor_flow_tensor.py @@ -4,7 +4,7 @@ from pydantic.tools import parse_obj_as from docarray.base_doc.io.json import orjson_dumps -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/typing/tensor/test_video_tensor.py b/tests/units/typing/tensor/test_video_tensor.py index f9981dedbc9..23ae8115e64 100644 --- a/tests/units/typing/tensor/test_video_tensor.py +++ b/tests/units/typing/tensor/test_video_tensor.py @@ -12,7 +12,7 @@ VideoNdArray, VideoTorchTensor, ) -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/typing/url/test_audio_url.py b/tests/units/typing/url/test_audio_url.py index 6b876d81de7..9168302aada 100644 --- a/tests/units/typing/url/test_audio_url.py +++ b/tests/units/typing/url/test_audio_url.py @@ -8,7 +8,7 @@ from docarray import BaseDoc from docarray.base_doc.io.json import orjson_dumps from docarray.typing import AudioTorchTensor, AudioUrl -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available from tests import TOYDATA_DIR tf_available = is_tf_available() diff --git a/tests/units/typing/url/test_video_url.py b/tests/units/typing/url/test_video_url.py index a1401bde896..e0dba53b6fe 100644 --- a/tests/units/typing/url/test_video_url.py +++ b/tests/units/typing/url/test_video_url.py @@ -14,7 +14,7 @@ VideoTorchTensor, VideoUrl, ) -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available from tests import TOYDATA_DIR tf_available = is_tf_available() diff --git a/tests/units/util/query_language/test_lookup.py b/tests/units/util/query_language/test_lookup.py index 37b3c2a1a4c..dd30f51c8f0 100644 --- a/tests/units/util/query_language/test_lookup.py +++ b/tests/units/util/query_language/test_lookup.py @@ -1,6 +1,6 @@ import pytest -from docarray.utils.query_language.lookup import dunder_get, lookup +from docarray.utils._internal.query_language.lookup import dunder_get, lookup class A: diff --git a/tests/units/util/test_typing.py b/tests/units/util/test_typing.py index 2db742522f5..5446cf3ce04 100644 --- a/tests/units/util/test_typing.py +++ b/tests/units/util/test_typing.py @@ -4,8 +4,8 @@ from docarray.typing import NdArray, TorchTensor from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._typing import is_tensor_union, is_type_tensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal._typing import is_tensor_union, is_type_tensor +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: From 090091f978ca7e7da3e6ee1ad242ab4adcbc3895 Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 13:08:47 +0200 Subject: [PATCH 3/3] feat: add docstring test Signed-off-by: samsja --- docarray/typing/__init__.py | 3 +- tests/documentation/test_docstring.py | 48 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index daf89af9fe8..3deefe96f01 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -31,7 +31,6 @@ 'AnyUrl', 'ID', 'AnyTensor', - 'TensorFlowTensor', 'NdArrayEmbedding', 'ImageBytes', 'ImageTensor', @@ -59,7 +58,7 @@ tf_available = is_tf_available() if tf_available: - from docarray.typing.tensor import TensorFlowTensor + from docarray.typing.tensor import TensorFlowTensor # noqa: F401 from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 diff --git a/tests/documentation/test_docstring.py b/tests/documentation/test_docstring.py index e69de29bb2d..c6a8e3d9a40 100644 --- a/tests/documentation/test_docstring.py +++ b/tests/documentation/test_docstring.py @@ -0,0 +1,48 @@ +""" +this test check the docstring of all of our public API. It does it +by checking the `__all__` of each of our namespace. + +to add a new namespace you need to +* import it +* add it to the `SUB_MODULE_TO_CHECK` list +""" + +import pytest +from mktestdocs import check_docstring, get_codeblock_members + +import docarray.data +import docarray.documents +import docarray.index +import docarray.store +import docarray.typing + +SUB_MODULE_TO_CHECK = [ + docarray, + docarray.index, + docarray.data, + docarray.documents, + docarray.store, + docarray.typing, +] + + +def get_obj_to_check(lib): + obj_to_check = [] + for obj in lib.__all__: + obj_to_check.append(getattr(lib, obj)) + return obj_to_check + + +obj_to_check = [] + +for lib in SUB_MODULE_TO_CHECK: + obj_to_check.extend(get_obj_to_check(lib)) + +members = [] +for obj in obj_to_check: + members.extend(get_codeblock_members(obj)) + + +@pytest.mark.parametrize("obj", members, ids=lambda d: d.__qualname__) +def test_member(obj): + check_docstring(obj)