diff --git a/docarray/array/abstract_array.py b/docarray/array/abstract_array.py index 762f37cfb42..0669a927451 100644 --- a/docarray/array/abstract_array.py +++ b/docarray/array/abstract_array.py @@ -22,7 +22,7 @@ from docarray.base_doc import BaseDoc from docarray.display.document_array_summary import DocArraySummary from docarray.typing.abstract_type import AbstractType -from docarray.utils._typing import change_cls_name +from docarray.utils._internal._typing import change_cls_name if TYPE_CHECKING: from docarray.proto import DocArrayProto, NodeProto diff --git a/docarray/array/array/io.py b/docarray/array/array/io.py index 8659af08588..1d8de498638 100644 --- a/docarray/array/array/io.py +++ b/docarray/array/array/io.py @@ -31,7 +31,7 @@ _all_access_paths_valid, _dict_to_access_paths, ) -from docarray.utils.compress import _decompress_bytes, _get_compress_ctx +from docarray.utils._internal.compress import _decompress_bytes, _get_compress_ctx if TYPE_CHECKING: import pandas as pd @@ -201,7 +201,7 @@ def to_binary_stream( from rich import filesize if show_progress: - from docarray.utils.progress_bar import _get_progressbar + from docarray.utils._internal.progress_bar import _get_progressbar pbar, t = _get_progressbar( 'Serializing', disable=not show_progress, total=len(self) @@ -564,7 +564,7 @@ def _load_binary_all( else: from rich import filesize - from docarray.utils.progress_bar import _get_progressbar + from docarray.utils._internal.progress_bar import _get_progressbar # 1 byte (uint8) # 8 bytes (uint64) @@ -629,7 +629,7 @@ def _load_binary_stream( num_docs = int.from_bytes(version_numdocs_lendoc0[1:9], 'big', signed=False) if show_progress: - from docarray.utils.progress_bar import _get_progressbar + from docarray.utils._internal.progress_bar import _get_progressbar pbar, t = _get_progressbar( 'Deserializing', disable=not show_progress, total=num_docs diff --git a/docarray/array/array/sequence_indexing_mixin.py b/docarray/array/array/sequence_indexing_mixin.py index cc12b6a9a9f..01db74c0218 100644 --- a/docarray/array/array/sequence_indexing_mixin.py +++ b/docarray/array/array/sequence_indexing_mixin.py @@ -14,7 +14,7 @@ import numpy as np -from docarray.utils.misc import is_torch_available +from docarray.utils._internal.misc import is_torch_available T_item = TypeVar('T_item') T = TypeVar('T', bound='IndexingSequenceMixin') diff --git a/docarray/array/stacked/array_stacked.py b/docarray/array/stacked/array_stacked.py index d2630a4b655..1445b97379e 100644 --- a/docarray/array/stacked/array_stacked.py +++ b/docarray/array/stacked/array_stacked.py @@ -26,8 +26,8 @@ from docarray.base_doc.mixins.io import _type_to_protobuf from docarray.typing import NdArray from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._typing import is_tensor_union -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal._typing import is_tensor_union +from docarray.utils._internal.misc import is_tf_available, is_torch_available if TYPE_CHECKING: from pydantic.fields import ModelField diff --git a/docarray/base_doc/mixins/io.py b/docarray/base_doc/mixins/io.py index 13f723df3a3..e0fde4b5d4f 100644 --- a/docarray/base_doc/mixins/io.py +++ b/docarray/base_doc/mixins/io.py @@ -20,8 +20,8 @@ from docarray.base_doc.base_node import BaseNode from docarray.typing import NdArray from docarray.typing.proto_register import _PROTO_TYPE_NAME_TO_CLASS -from docarray.utils.compress import _compress_bytes, _decompress_bytes -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.compress import _compress_bytes, _decompress_bytes +from docarray.utils._internal.misc import is_tf_available, is_torch_available tf_available = is_tf_available() if tf_available: diff --git a/docarray/data/torch_dataset.py b/docarray/data/torch_dataset.py index 3b5cb0f87a9..dd58035cd33 100644 --- a/docarray/data/torch_dataset.py +++ b/docarray/data/torch_dataset.py @@ -4,7 +4,7 @@ from docarray import BaseDoc, DocArray, DocArrayStacked from docarray.typing import TorchTensor -from docarray.utils._typing import change_cls_name +from docarray.utils._internal._typing import change_cls_name T_doc = TypeVar('T_doc', bound=BaseDoc) diff --git a/docarray/documents/audio.py b/docarray/documents/audio.py index ab0bbe00b34..e27abcb5408 100644 --- a/docarray/documents/audio.py +++ b/docarray/documents/audio.py @@ -7,7 +7,7 @@ from docarray.typing.bytes.audio_bytes import AudioBytes from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.audio.audio_tensor import AudioTensor -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/documents/image.py b/docarray/documents/image.py index f2375637de6..bf281f83d8f 100644 --- a/docarray/documents/image.py +++ b/docarray/documents/image.py @@ -6,7 +6,7 @@ from docarray.typing import AnyEmbedding, ImageBytes, ImageUrl from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.image.image_tensor import ImageTensor -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available T = TypeVar('T', bound='ImageDoc') diff --git a/docarray/documents/point_cloud/point_cloud_3d.py b/docarray/documents/point_cloud/point_cloud_3d.py index 3cee613f596..25e0e6437d1 100644 --- a/docarray/documents/point_cloud/point_cloud_3d.py +++ b/docarray/documents/point_cloud/point_cloud_3d.py @@ -6,7 +6,7 @@ from docarray.documents.point_cloud.points_and_colors import PointsAndColors from docarray.typing import AnyEmbedding, PointCloud3DUrl from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/documents/point_cloud/points_and_colors.py b/docarray/documents/point_cloud/points_and_colors.py index af4917bf5a7..82ce82f9ba1 100644 --- a/docarray/documents/point_cloud/points_and_colors.py +++ b/docarray/documents/point_cloud/points_and_colors.py @@ -5,7 +5,7 @@ from docarray.base_doc import BaseDoc from docarray.typing import AnyTensor from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/documents/video.py b/docarray/documents/video.py index 3770ee245fc..679f10ea8ba 100644 --- a/docarray/documents/video.py +++ b/docarray/documents/video.py @@ -8,7 +8,7 @@ from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.video.video_tensor import VideoTensor from docarray.typing.url.video_url import VideoUrl -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/index/abstract.py b/docarray/index/abstract.py index 3eac3d7f869..c587846c8c4 100644 --- a/docarray/index/abstract.py +++ b/docarray/index/abstract.py @@ -26,9 +26,9 @@ from docarray import BaseDoc, DocArray from docarray.array.abstract_array import AnyDocArray from docarray.typing import AnyTensor -from docarray.utils._typing import unwrap_optional_type +from docarray.utils._internal._typing import unwrap_optional_type +from docarray.utils._internal.misc import is_tf_available, torch_imported from docarray.utils.find import FindResult, _FindResult -from docarray.utils.misc import is_tf_available, torch_imported if TYPE_CHECKING: from pydantic.fields import ModelField diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 38b827539ab..ab8aa6e56c1 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -29,9 +29,9 @@ _raise_not_supported, ) from docarray.proto import DocumentProto +from docarray.utils._internal.misc import is_np_int, is_tf_available, is_torch_available from docarray.utils.filter import filter_docs from docarray.utils.find import _FindResult -from docarray.utils.misc import is_np_int, is_tf_available, is_torch_available TSchema = TypeVar('TSchema', bound=BaseDoc) T = TypeVar('T', bound='HnswDocumentIndex') diff --git a/docarray/store/file.py b/docarray/store/file.py index aeb355e2e17..bb79162109b 100644 --- a/docarray/store/file.py +++ b/docarray/store/file.py @@ -7,7 +7,7 @@ from docarray.store.abstract_doc_store import AbstractDocStore from docarray.store.exceptions import ConcurrentPushException from docarray.store.helpers import _from_binary_stream, _to_binary_stream -from docarray.utils.cache import get_cache_path +from docarray.utils._internal.cache import _get_cache_path if TYPE_CHECKING: from docarray import BaseDoc, DocArray @@ -23,7 +23,7 @@ def _abs_filepath(name: str) -> Path: If it is a path, it is resolved to an absolute path. """ if not (name.startswith('/') or name.startswith('~') or name.startswith('.')): - name = str(get_cache_path() / name) + name = str(_get_cache_path() / name) if name.startswith('~'): name = str(Path.home() / name[2:]) return Path(name).resolve() diff --git a/docarray/store/helpers.py b/docarray/store/helpers.py index 25e40991e68..457d160d41c 100644 --- a/docarray/store/helpers.py +++ b/docarray/store/helpers.py @@ -6,7 +6,7 @@ from rich import filesize from typing_extensions import TYPE_CHECKING, Protocol -from docarray.utils.progress_bar import _get_progressbar +from docarray.utils._internal.progress_bar import _get_progressbar if TYPE_CHECKING: from pathlib import Path diff --git a/docarray/store/jac.py b/docarray/store/jac.py index 75685e61cb4..c9350fae18a 100644 --- a/docarray/store/jac.py +++ b/docarray/store/jac.py @@ -24,7 +24,7 @@ get_version_info, raise_req_error, ) -from docarray.utils.cache import get_cache_path +from docarray.utils._internal.cache import _get_cache_path if TYPE_CHECKING: # pragma: no cover import io @@ -331,7 +331,7 @@ def pull_stream( _BufferedCachingRequestReader, io.BufferedReader ] = _BufferedCachingRequestReader(r, tmp_cache_file) - cache_file = get_cache_path() / f'{save_name}.da' + cache_file = _get_cache_path() / f'{save_name}.da' if local_cache and cache_file.exists(): _cache_len = cache_file.stat().st_size if _cache_len == int(r.headers['Content-length']): @@ -354,7 +354,7 @@ def pull_stream( if local_cache: if isinstance(_source, _BufferedCachingRequestReader): - Path(get_cache_path()).mkdir(parents=True, exist_ok=True) + Path(_get_cache_path()).mkdir(parents=True, exist_ok=True) tmp_cache_file.rename(cache_file) else: _source.close() diff --git a/docarray/store/s3.py b/docarray/store/s3.py index b038e7c4b41..7711432900c 100644 --- a/docarray/store/s3.py +++ b/docarray/store/s3.py @@ -10,7 +10,7 @@ from docarray.store.abstract_doc_store import AbstractDocStore from docarray.store.helpers import _from_binary_stream, _to_binary_stream -from docarray.utils.cache import get_cache_path +from docarray.utils._internal.cache import _get_cache_path if TYPE_CHECKING: # pragma: no cover from docarray import BaseDoc, DocArray @@ -212,7 +212,7 @@ def pull_stream( bucket, name = name.split('/', 1) save_name = name.replace('/', '_') - cache_path = get_cache_path() / f'{save_name}.da' + cache_path = _get_cache_path() / f'{save_name}.da' source = _BufferedCachingReader( open(f"s3://{bucket}/{name}.da", 'rb', compression='.gz'), diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index cd48ff7c26b..c9dceb600c9 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -1,4 +1,4 @@ -from docarray.typing.bytes import ImageBytes +from docarray.typing.bytes import AudioBytes, ImageBytes, VideoBytes from docarray.typing.id import ID from docarray.typing.tensor import ImageNdArray, ImageTensor from docarray.typing.tensor.audio import AudioNdArray @@ -31,14 +31,16 @@ 'AnyUrl', 'ID', 'AnyTensor', - 'TensorFlowTensor', 'NdArrayEmbedding', 'ImageBytes', 'ImageTensor', 'ImageNdArray', + 'ImageBytes', + 'VideoBytes', + 'AudioBytes', ] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: @@ -59,7 +61,7 @@ tf_available = is_tf_available() if tf_available: - from docarray.typing.tensor import TensorFlowTensor + from docarray.typing.tensor import TensorFlowTensor # noqa: F401 from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 diff --git a/docarray/typing/bytes/__init__.py b/docarray/typing/bytes/__init__.py index 9b622669919..2cf8524bcc0 100644 --- a/docarray/typing/bytes/__init__.py +++ b/docarray/typing/bytes/__init__.py @@ -1,3 +1,5 @@ +from docarray.typing.bytes.audio_bytes import AudioBytes from docarray.typing.bytes.image_bytes import ImageBytes +from docarray.typing.bytes.video_bytes import VideoBytes -__all__ = ['ImageBytes'] +__all__ = ['ImageBytes', 'VideoBytes', 'AudioBytes'] diff --git a/docarray/typing/bytes/audio_bytes.py b/docarray/typing/bytes/audio_bytes.py index 6a6c4e14bfd..649d06bac03 100644 --- a/docarray/typing/bytes/audio_bytes.py +++ b/docarray/typing/bytes/audio_bytes.py @@ -46,32 +46,35 @@ def load(self) -> Tuple[np.ndarray, int]: """ Load the Audio from the bytes into a numpy.ndarray Audio tensor - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional + from docarray import BaseDoc + from docarray.typing import AudioUrl, NdArray, AudioBytes - from docarray import BaseDoc - import numpy as np + import numpy as np - from docarray.typing import AudioUrl + class MyAudio(BaseDoc): + url: AudioUrl + tensor: Optional[NdArray] + bytes: Optional[AudioBytes] + frame_rate: Optional[float] - class MyAudio(Document): - url: AudioUrl - tensor: Optional[NdArray] - bytes: Optional[bytes] + doc = MyAudio(url='https://www.kozco.com/tech/piano2.wav') + doc.bytes = doc.url.load_bytes() + doc.tensor, doc.frame_rate = doc.bytes.load() - doc = MyAudio(url="toydata/hello.wav") - doc.bytes = doc.url.load_bytes() - doc.tensor, doc.frame_rate = doc.bytes.load() + # Note this is equivalent to do - # Note this is equivalent to do + doc.tensor, doc.frame_rate = doc.url.load() - doc.tensor, doc.frame_rate = doc.url.load() - - assert isinstance(doc.audio_tensor, np.ndarray) + assert isinstance(doc.tensor, np.ndarray) + ``` + --- :return: np.ndarray representing the Audio as RGB values """ diff --git a/docarray/typing/bytes/image_bytes.py b/docarray/typing/bytes/image_bytes.py index 2888f60fadd..f8e68ba03f0 100644 --- a/docarray/typing/bytes/image_bytes.py +++ b/docarray/typing/bytes/image_bytes.py @@ -51,34 +51,35 @@ def load( """ Load the image from the bytes into a numpy.ndarray image tensor - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from docarray import BaseDoc + from docarray.typing import ImageUrl + import numpy as np - from docarray import BaseDoc - from docarray.typing import ImageUrl - import numpy as np + class MyDoc(BaseDoc): + img_url: ImageUrl - class MyDoc(BaseDoc): - img_url: ImageUrl + doc = MyDoc( + img_url="https://upload.wikimedia.org/wikipedia/commons/8/80/" + "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg" + ) - doc = MyDoc( - img_url="https://upload.wikimedia.org/wikipedia/commons/8/80/" - "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg" - ) + img_tensor = doc.img_url.load() + assert isinstance(img_tensor, np.ndarray) - img_tensor = doc.img_url.load() - assert isinstance(img_tensor, np.ndarray) + img_tensor = doc.img_url.load(height=224, width=224) + assert img_tensor.shape == (224, 224, 3) - img_tensor = doc.img_url.load(height=224, width=224) - assert img_tensor.shape == (224, 224, 3) - - layout = ('C', 'W', 'H') - img_tensor = doc.img_url.load(height=100, width=200, axis_layout=layout) - assert img_tensor.shape == (3, 200, 100) + layout = ('C', 'W', 'H') + img_tensor = doc.img_url.load(height=100, width=200, axis_layout=layout) + assert img_tensor.shape == (3, 200, 100) + ``` + --- :param width: width of the image tensor. :param height: height of the image tensor. diff --git a/docarray/typing/bytes/video_bytes.py b/docarray/typing/bytes/video_bytes.py index fb45a36063f..a48face2b6b 100644 --- a/docarray/typing/bytes/video_bytes.py +++ b/docarray/typing/bytes/video_bytes.py @@ -5,9 +5,9 @@ from pydantic import parse_obj_as from pydantic.validators import bytes_validator -from docarray.typing import AudioNdArray, NdArray, VideoNdArray from docarray.typing.abstract_type import AbstractType from docarray.typing.proto_register import _register_proto +from docarray.typing.tensor import AudioNdArray, NdArray, VideoNdArray if TYPE_CHECKING: from pydantic.fields import BaseConfig, ModelField @@ -55,29 +55,33 @@ def load(self, **kwargs) -> VideoLoadResult: (`VideoLoadResult.audio`) and an NdArray containing the key frame indices (`VideoLoadResult.key_frame_indices`). - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from docarray import BaseDoc + from docarray.typing import VideoUrl + import numpy as np - from docarray import BaseDoc - from docarray.typing import VideoUrl - import numpy as np + class MyDoc(BaseDoc): + video_url: VideoUrl - class MyDoc(BaseDoc): - video_url: VideoUrl + doc = MyDoc( + video_url='https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true' + ) - doc = MyDoc(video_url="toydata/mp_.mp4") + video, audio, key_frame_indices = doc.video_url.load() + assert isinstance(video, np.ndarray) + assert isinstance(audio, np.ndarray) + assert isinstance(key_frame_indices, np.ndarray) + ``` + + --- - video, audio, key_frame_indices = doc.video_url.load() - assert isinstance(video, np.ndarray) - assert isinstance(audio, np.ndarray) - assert isinstance(key_frame_indices, np.ndarray) :param kwargs: supports all keyword arguments that are being supported by - av.open() as described in: - https://pyav.org/docs/stable/api/_globals.html?highlight=open#av.open + av.open() as described [here](https://pyav.org/docs/stable/api/_globals.html?highlight=open#av.open) :return: a VideoLoadResult instance with video, audio and keyframe indices """ import av diff --git a/docarray/typing/tensor/__init__.py b/docarray/typing/tensor/__init__.py index 07a27292aee..4185f23c22d 100644 --- a/docarray/typing/tensor/__init__.py +++ b/docarray/typing/tensor/__init__.py @@ -1,7 +1,9 @@ +from docarray.typing.tensor.audio import AudioNdArray from docarray.typing.tensor.embedding import AnyEmbedding, NdArrayEmbedding from docarray.typing.tensor.image import ImageNdArray, ImageTensor from docarray.typing.tensor.ndarray import NdArray from docarray.typing.tensor.tensor import AnyTensor +from docarray.typing.tensor.video import VideoNdArray __all__ = [ 'NdArray', @@ -10,26 +12,47 @@ 'NdArrayEmbedding', 'ImageNdArray', 'ImageTensor', - 'TensorFlowTensor', + 'AudioNdArray', + 'VideoNdArray', ] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: + from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F401 from docarray.typing.tensor.embedding import TorchEmbedding # noqa: F401 from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 from docarray.typing.tensor.torch_tensor import TorchTensor # noqa: F401 - - __all__.extend(['TorchEmbedding', 'TorchTensor', 'ImageTorchTensor']) + from docarray.typing.tensor.video import VideoTorchTensor # noqa: F401 + + __all__.extend( + [ + 'TorchEmbedding', + 'TorchTensor', + 'ImageTorchTensor', + 'AudioTorchTensor', + 'VideoTorchTensor', + ] + ) torch_available = is_torch_available() tf_available = is_tf_available() if tf_available: + from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 from docarray.typing.tensor.tensorflow_tensor import TensorFlowTensor # noqa: F401 - - __all__.extend(['TensorFlowEmbedding', 'TensorFlowTensor', 'ImageTensorFlowTensor']) + from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa: F401 + + __all__.extend( + [ + 'TensorFlowEmbedding', + 'TensorFlowTensor', + 'ImageTensorFlowTensor', + 'AudioTensorFlowTensor', + 'VideoTensorFlowTensor', + ] + ) diff --git a/docarray/typing/tensor/audio/__init__.py b/docarray/typing/tensor/audio/__init__.py index 4b1b7edc34d..4f730451603 100644 --- a/docarray/typing/tensor/audio/__init__.py +++ b/docarray/typing/tensor/audio/__init__.py @@ -2,7 +2,7 @@ __all__ = ['AudioNdArray'] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/audio/abstract_audio_tensor.py b/docarray/typing/tensor/audio/abstract_audio_tensor.py index 7e677bcf952..e84b4ebc13e 100644 --- a/docarray/typing/tensor/audio/abstract_audio_tensor.py +++ b/docarray/typing/tensor/audio/abstract_audio_tensor.py @@ -3,7 +3,7 @@ from typing import Any, BinaryIO, Dict, TypeVar, Union from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook T = TypeVar('T', bound='AbstractAudioTensor') diff --git a/docarray/typing/tensor/audio/audio_ndarray.py b/docarray/typing/tensor/audio/audio_ndarray.py index bae64ed36c2..f47f40b9d8a 100644 --- a/docarray/typing/tensor/audio/audio_ndarray.py +++ b/docarray/typing/tensor/audio/audio_ndarray.py @@ -10,43 +10,44 @@ class AudioNdArray(AbstractAudioTensor, NdArray): Adds audio-specific features to the tensor. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + from docarray import BaseDoc + from docarray.typing import AudioNdArray, AudioUrl + import numpy as np - from docarray import BaseDoc - from docarray.typing import AudioNdArray, AudioUrl - import numpy as np + class MyAudioDoc(BaseDoc): + title: str + audio_tensor: Optional[AudioNdArray] + url: Optional[AudioUrl] + bytes_: Optional[bytes] - class MyAudioDoc(BaseDoc): - title: str - audio_tensor: Optional[AudioNdArray] - url: Optional[AudioUrl] - bytes_: Optional[bytes] + # from tensor + doc_1 = MyAudioDoc( + title='my_first_audio_doc', + audio_tensor=np.random.rand(1000, 2), + ) - # from tensor - doc_1 = MyAudioDoc( - title='my_first_audio_doc', - audio_tensor=np.random.rand(1000, 2), - ) + doc_1.audio_tensor.save(file_path='/tmp/file_1.wav') + doc_1.bytes_ = doc_1.audio_tensor.to_bytes() - doc_1.audio_tensor.save(file_path='path/to/file_1.wav') - doc_1.bytes_ = doc_1.audio_tensor.to_bytes() + # from url + doc_2 = MyAudioDoc( + title='my_second_audio_doc', + url='https://www.kozco.com/tech/piano2.wav', + ) - # from url - doc_2 = MyAudioDoc( - title='my_second_audio_doc', - url='https://www.kozco.com/tech/piano2.wav', - ) - - doc_2.audio_tensor = doc_2.url.load() - doc_2.audio_tensor.save(file_path='path/to/file_2.wav') - doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + doc_2.audio_tensor, _ = doc_2.url.load() + doc_2.audio_tensor.save(file_path='/tmp/file_2.wav') + doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + ``` + --- """ ... diff --git a/docarray/typing/tensor/audio/audio_tensor.py b/docarray/typing/tensor/audio/audio_tensor.py index 188ccba26a5..7d1f4a9e315 100644 --- a/docarray/typing/tensor/audio/audio_tensor.py +++ b/docarray/typing/tensor/audio/audio_tensor.py @@ -1,7 +1,7 @@ from typing import Union from docarray.typing.tensor.audio.audio_ndarray import AudioNdArray -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/audio/audio_tensorflow_tensor.py b/docarray/typing/tensor/audio/audio_tensorflow_tensor.py index a91b5b3183a..497a57f9464 100644 --- a/docarray/typing/tensor/audio/audio_tensorflow_tensor.py +++ b/docarray/typing/tensor/audio/audio_tensorflow_tensor.py @@ -16,44 +16,47 @@ class AudioTensorFlowTensor( Adds audio-specific features to the tensor. - EXAMPLE USAGE - .. code-block:: python + ``` - from typing import Optional + --- + from typing import Optional - import tensorflow as tf - from pydantic import parse_obj_as + import tensorflow as tf + from pydantic import parse_obj_as - from docarray import BaseDoc - from docarray.typing import AudioTensorFlowTensor, AudioUrl + from docarray import BaseDoc + from docarray.typing import AudioTensorFlowTensor, AudioUrl - class MyAudioDoc(BaseDoc): - title: str - audio_tensor: Optional[AudioTensorFlowTensor] - url: Optional[AudioUrl] - bytes_: Optional[bytes] + class MyAudioDoc(BaseDoc): + title: str + audio_tensor: Optional[AudioTensorFlowTensor] + url: Optional[AudioUrl] + bytes_: Optional[bytes] - doc_1 = MyAudioDoc( - title='my_first_audio_doc', - audio_tensor=tf.random.normal((1000, 2)), - ) + doc_1 = MyAudioDoc( + title='my_first_audio_doc', + audio_tensor=tf.random.normal((1000, 2)), + ) - doc_1.audio_tensor.save(file_path='path/to/file_1.wav') - doc_1.bytes_ = doc_1.audio_tensor.to_bytes() + doc_1.audio_tensor.save(file_path='path/to/file_1.wav') + doc_1.bytes_ = doc_1.audio_tensor.to_bytes() - doc_2 = MyAudioDoc( - title='my_second_audio_doc', - url='https://www.kozco.com/tech/piano2.wav', - ) + doc_2 = MyAudioDoc( + title='my_second_audio_doc', + url='https://www.kozco.com/tech/piano2.wav', + ) - doc_2.audio_tensor = doc_2.url.load() - doc_2.audio_tensor.save(file_path='path/to/file_2.wav') - doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + doc_2.audio_tensor = doc_2.url.load() + doc_2.audio_tensor.save(file_path='path/to/file_2.wav') + doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + ``` + + --- """ ... diff --git a/docarray/typing/tensor/audio/audio_torch_tensor.py b/docarray/typing/tensor/audio/audio_torch_tensor.py index 86d09e2212f..532d97a7f8e 100644 --- a/docarray/typing/tensor/audio/audio_torch_tensor.py +++ b/docarray/typing/tensor/audio/audio_torch_tensor.py @@ -10,41 +10,43 @@ class AudioTorchTensor(AbstractAudioTensor, TorchTensor, metaclass=metaTorchAndN Adds audio-specific features to the tensor. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + import torch - import torch + from docarray import BaseDoc + from docarray.typing import AudioTorchTensor, AudioUrl - from docarray import BaseDoc - from docarray.typing import AudioTorchTensor, AudioUrl + class MyAudioDoc(BaseDoc): + title: str + audio_tensor: Optional[AudioTorchTensor] + url: Optional[AudioUrl] + bytes_: Optional[bytes] - class MyAudioDoc(BaseDoc): - title: str - audio_tensor: Optional[AudioTorchTensor] - url: Optional[AudioUrl] - bytes_: Optional[bytes] + doc_1 = MyAudioDoc( + title='my_first_audio_doc', + audio_tensor=torch.zeros(1000, 2), + ) - doc_1 = MyAudioDoc( - title='my_first_audio_doc', - audio_tensor=torch.randn(size=(1000, 2)), - ) + doc_1.audio_tensor.save(file_path='/tmp/file_1.wav') + doc_1.bytes_ = doc_1.audio_tensor.to_bytes() - doc_1.audio_tensor.save(file_path='path/to/file_1.wav') - doc_1.bytes_ = doc_1.audio_tensor.to_bytes() + doc_2 = MyAudioDoc( + title='my_second_audio_doc', + url='https://www.kozco.com/tech/piano2.wav', + ) - doc_2 = MyAudioDoc( - title='my_second_audio_doc', - url='https://www.kozco.com/tech/piano2.wav', - ) + doc_2.audio_tensor, _ = doc_2.url.load() + doc_2.audio_tensor.save(file_path='/tmp/file_2.wav') + doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + ``` - doc_2.audio_tensor = doc_2.url.load() - doc_2.audio_tensor.save(file_path='path/to/file_2.wav') - doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + --- """ ... diff --git a/docarray/typing/tensor/embedding/__init__.py b/docarray/typing/tensor/embedding/__init__.py index 6cc8b1d6008..f844b7abaa9 100644 --- a/docarray/typing/tensor/embedding/__init__.py +++ b/docarray/typing/tensor/embedding/__init__.py @@ -3,7 +3,7 @@ __all__ = ['NdArrayEmbedding', 'AnyEmbedding'] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/embedding/embedding.py b/docarray/typing/tensor/embedding/embedding.py index a8bb78a33d2..2eb1fa7e842 100644 --- a/docarray/typing/tensor/embedding/embedding.py +++ b/docarray/typing/tensor/embedding/embedding.py @@ -1,7 +1,7 @@ from typing import Union from docarray.typing.tensor.embedding.ndarray import NdArrayEmbedding -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/image/__init__.py b/docarray/typing/tensor/image/__init__.py index 03bd92a5342..2fb32d452ab 100644 --- a/docarray/typing/tensor/image/__init__.py +++ b/docarray/typing/tensor/image/__init__.py @@ -3,7 +3,7 @@ __all__ = ['ImageNdArray', 'ImageTensor'] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/image/abstract_image_tensor.py b/docarray/typing/tensor/image/abstract_image_tensor.py index 97b8fd8e23a..b02d1452f63 100644 --- a/docarray/typing/tensor/image/abstract_image_tensor.py +++ b/docarray/typing/tensor/image/abstract_image_tensor.py @@ -3,7 +3,7 @@ from abc import ABC from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook class AbstractImageTensor(AbstractTensor, ABC): diff --git a/docarray/typing/tensor/image/image_ndarray.py b/docarray/typing/tensor/image/image_ndarray.py index df304cd1d21..4ac022b300b 100644 --- a/docarray/typing/tensor/image/image_ndarray.py +++ b/docarray/typing/tensor/image/image_ndarray.py @@ -14,34 +14,35 @@ class ImageNdArray(AbstractImageTensor, NdArray): optimized to send over the wire - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + from docarray import BaseDoc + from docarray.typing import ImageNdArray, ImageUrl - from docarray import BaseDoc - from docarray.typing import ImageNdArray, ImageUrl + class MyImageDoc(BaseDoc): + title: str + tensor: Optional[ImageNdArray] + url: Optional[ImageUrl] + bytes: Optional[bytes] - class MyImageDoc(BaseDoc): - title: str - tensor: Optional[ImageNdArray] - url: Optional[ImageUrl] - bytes: Optional[bytes] + # from url + doc = MyImageDoc( + title='my_second_audio_doc', + url="https://upload.wikimedia.org/wikipedia/commons/8/80/" + "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", + ) - # from url - doc = MyImageDoc( - title='my_second_audio_doc', - url="https://upload.wikimedia.org/wikipedia/commons/8/80/" - "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", - ) + doc.tensor = doc.url.load() - doc.tensor = doc.url.load() - - doc.bytes = doc.tensor.to_bytes() + doc.bytes = doc.tensor.to_bytes() + ``` + --- """ ... diff --git a/docarray/typing/tensor/image/image_tensor.py b/docarray/typing/tensor/image/image_tensor.py index 747b080b455..af439ce607e 100644 --- a/docarray/typing/tensor/image/image_tensor.py +++ b/docarray/typing/tensor/image/image_tensor.py @@ -1,7 +1,7 @@ from typing import Union from docarray.typing.tensor.image.image_ndarray import ImageNdArray -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/image/image_tensorflow_tensor.py b/docarray/typing/tensor/image/image_tensorflow_tensor.py index 7afcbb38086..95aba9d5337 100644 --- a/docarray/typing/tensor/image/image_tensorflow_tensor.py +++ b/docarray/typing/tensor/image/image_tensorflow_tensor.py @@ -18,31 +18,32 @@ class ImageTensorFlowTensor( optimized to send over the wire - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + from docarray import BaseDoc + from docarray.typing import ImageTensorFlowTensor, ImageUrl - from docarray import BaseDoc - from docarray.typing import ImageTensorFlowTensor, ImageUrl + class MyImageDoc(BaseDoc): + title: str + tensor: Optional[ImageTensorFlowTensor] + url: Optional[ImageUrl] + bytes: Optional[bytes] - class MyImageDoc(BaseDoc): - title: str - tensor: Optional[ImageTensorFlowTensor] - url: Optional[ImageUrl] - bytes: Optional[bytes] + doc = MyImageDoc( + title='my_second_image_doc', + url="https://upload.wikimedia.org/wikipedia/commons/8/80/" + "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", + ) + doc.tensor = doc.url.load() + doc.bytes = doc.tensor.to_bytes() + ``` - doc = MyImageDoc( - title='my_second_image_doc', - url="https://upload.wikimedia.org/wikipedia/commons/8/80/" - "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", - ) - doc.tensor = doc.url.load() - doc.bytes = doc.tensor.to_bytes() - + --- """ ... diff --git a/docarray/typing/tensor/image/image_torch_tensor.py b/docarray/typing/tensor/image/image_torch_tensor.py index 90ecac13750..bc2a8dbc7db 100644 --- a/docarray/typing/tensor/image/image_torch_tensor.py +++ b/docarray/typing/tensor/image/image_torch_tensor.py @@ -16,31 +16,33 @@ class ImageTorchTensor(AbstractImageTensor, TorchTensor, metaclass=metaTorchAndN optimized to send over the wire - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + from docarray import BaseDoc + from docarray.typing import ImageTorchTensor, ImageUrl - from docarray import BaseDoc - from docarray.typing import ImageTorchTensor, ImageUrl + class MyImageDoc(BaseDoc): + title: str + tensor: Optional[ImageTorchTensor] + url: Optional[ImageUrl] + bytes: Optional[bytes] - class MyImageDoc(BaseDoc): - title: str - tensor: Optional[ImageTorchTensor] - url: Optional[ImageUrl] - bytes: Optional[bytes] + doc = MyImageDoc( + title='my_second_image_doc', + url="https://upload.wikimedia.org/wikipedia/commons/8/80/" + "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", + ) - doc = MyImageDoc( - title='my_second_image_doc', - url="https://upload.wikimedia.org/wikipedia/commons/8/80/" - "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", - ) + doc.tensor = doc.url.load() + doc.bytes = doc.tensor.to_bytes() + ``` - doc.tensor = doc.url.load() - doc.bytes = doc.tensor.to_bytes() + --- """ ... diff --git a/docarray/typing/tensor/ndarray.py b/docarray/typing/tensor/ndarray.py index 0fc41a0c184..49b23ea8e9c 100644 --- a/docarray/typing/tensor/ndarray.py +++ b/docarray/typing/tensor/ndarray.py @@ -46,47 +46,54 @@ class NdArray(np.ndarray, AbstractTensor, Generic[ShapeT]): This type can also be used in a parametrized way, specifying the shape of the array. - EXAMPLE USAGE - - .. code-block:: python - - from docarray import BaseDoc - from docarray.typing import NdArray - import numpy as np - - - class MyDoc(BaseDoc): - arr: NdArray - image_arr: NdArray[3, 224, 224] - square_crop: NdArray[3, 'x', 'x'] - random_image: NdArray[3, ...] # first dimension is fixed, can have arbitrary shape - - - # create a document with tensors - doc = MyDoc( - arr=np.zeros((128,)), - image_arr=np.zeros((3, 224, 224)), - square_crop=np.zeros((3, 64, 64)), - random_image=np.zeros(3, 128, 256), - ) - assert doc.image_arr.shape == (3, 224, 224) - - # automatic shape conversion - doc = MyDoc( - arr=np.zeros((128,)), - image_arr=np.zeros((224, 224, 3)), # will reshape to (3, 224, 224) - square_crop=np.zeros((3, 128, 128)), - random_image=np.zeros(3, 64, 128), - ) - assert doc.image_arr.shape == (3, 224, 224) - - # !! The following will raise an error due to shape mismatch !! + --- + + ```python + from docarray import BaseDoc + from docarray.typing import NdArray + import numpy as np + + + class MyDoc(BaseDoc): + arr: NdArray + image_arr: NdArray[3, 224, 224] + square_crop: NdArray[3, 'x', 'x'] + random_image: NdArray[3, ...] # first dimension is fixed, can have arbitrary shape + + + # create a document with tensors + doc = MyDoc( + arr=np.zeros((128,)), + image_arr=np.zeros((3, 224, 224)), + square_crop=np.zeros((3, 64, 64)), + random_image=np.zeros((3, 128, 256)), + ) + assert doc.image_arr.shape == (3, 224, 224) + + # automatic shape conversion + doc = MyDoc( + arr=np.zeros((128,)), + image_arr=np.zeros((224, 224, 3)), # will reshape to (3, 224, 224) + square_crop=np.zeros((3, 128, 128)), + random_image=np.zeros((3, 64, 128)), + ) + assert doc.image_arr.shape == (3, 224, 224) + + # !! The following will raise an error due to shape mismatch !! + from pydantic import ValidationError + + try: doc = MyDoc( arr=np.zeros((128,)), image_arr=np.zeros((224, 224)), # this will fail validation square_crop=np.zeros((3, 128, 64)), # this will also fail validation - random_image=np.zeros(4, 64, 128), # this will also fail validation + random_image=np.zeros((4, 64, 128)), # this will also fail validation ) + except ValidationError as e: + pass + ``` + + --- """ __parametrized_meta__ = metaNumpy diff --git a/docarray/typing/tensor/tensor.py b/docarray/typing/tensor/tensor.py index 6cff0b99e26..19a89cbed3f 100644 --- a/docarray/typing/tensor/tensor.py +++ b/docarray/typing/tensor/tensor.py @@ -1,7 +1,7 @@ from typing import Union from docarray.typing.tensor.ndarray import NdArray -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/torch_tensor.py b/docarray/typing/tensor/torch_tensor.py index 7ca9cf70a89..011208b02bc 100644 --- a/docarray/typing/tensor/torch_tensor.py +++ b/docarray/typing/tensor/torch_tensor.py @@ -46,47 +46,54 @@ class TorchTensor( This type can also be used in a parametrized way, specifying the shape of the tensor. - EXAMPLE USAGE - - .. code-block:: python - - from docarray import BaseDoc - from docarray.typing import TorchTensor - import torch - - - class MyDoc(BaseDoc): - tensor: TorchTensor - image_tensor: TorchTensor[3, 224, 224] - square_crop: TorchTensor[3, 'x', 'x'] - random_image: TorchTensor[ - 3, ... - ] # first dimension is fixed, can have arbitrary shape - - - # create a document with tensors - doc = MyDoc( - tensor=torch.zeros(128), - image_tensor=torch.zeros(3, 224, 224), - square_crop=torch.zeros(3, 64, 64), - random_image=torch.zeros(3, 128, 256), - ) - - # automatic shape conversion - doc = MyDoc( - tensor=torch.zeros(128), - image_tensor=torch.zeros(224, 224, 3), # will reshape to (3, 224, 224) - square_crop=torch.zeros(3, 128, 128), - random_image=torch.zeros(3, 64, 128), - ) - - # !! The following will raise an error due to shape mismatch !! + --- + + ```python + from docarray import BaseDoc + from docarray.typing import TorchTensor + import torch + + + class MyDoc(BaseDoc): + tensor: TorchTensor + image_tensor: TorchTensor[3, 224, 224] + square_crop: TorchTensor[3, 'x', 'x'] + random_image: TorchTensor[ + 3, ... + ] # first dimension is fixed, can have arbitrary shape + + + # create a document with tensors + doc = MyDoc( + tensor=torch.zeros(128), + image_tensor=torch.zeros(3, 224, 224), + square_crop=torch.zeros(3, 64, 64), + random_image=torch.zeros(3, 128, 256), + ) + + # automatic shape conversion + doc = MyDoc( + tensor=torch.zeros(128), + image_tensor=torch.zeros(224, 224, 3), # will reshape to (3, 224, 224) + square_crop=torch.zeros(3, 128, 128), + random_image=torch.zeros(3, 64, 128), + ) + + # !! The following will raise an error due to shape mismatch !! + from pydantic import ValidationError + + try: doc = MyDoc( tensor=torch.zeros(128), image_tensor=torch.zeros(224, 224), # this will fail validation square_crop=torch.zeros(3, 128, 64), # this will also fail validation random_image=torch.zeros(4, 64, 128), # this will also fail validation ) + except ValidationError as e: + pass + ``` + + --- """ __parametrized_meta__ = metaTorchAndNode diff --git a/docarray/typing/tensor/video/__init__.py b/docarray/typing/tensor/video/__init__.py index add0c958d86..424758cfc91 100644 --- a/docarray/typing/tensor/video/__init__.py +++ b/docarray/typing/tensor/video/__init__.py @@ -2,7 +2,7 @@ __all__ = ['VideoNdArray'] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/video/video_ndarray.py b/docarray/typing/tensor/video/video_ndarray.py index d52e9a1ae28..f6b3cd8d3b1 100644 --- a/docarray/typing/tensor/video/video_ndarray.py +++ b/docarray/typing/tensor/video/video_ndarray.py @@ -19,41 +19,41 @@ class VideoNdArray(NdArray, VideoTensorMixin): Subclass of NdArray, to represent a video tensor. Adds video-specific features to the tensor. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + import numpy as np + from pydantic import parse_obj_as - import numpy as np - from pydantic import parse_obj_as + from docarray import BaseDoc + from docarray.typing import VideoNdArray, VideoUrl - from docarray import BaseDoc - from docarray.typing import VideoNdArray, VideoUrl + class MyVideoDoc(BaseDoc): + title: str + url: Optional[VideoUrl] + video_tensor: Optional[VideoNdArray] - class MyVideoDoc(BaseDoc): - title: str - url: Optional[VideoUrl] - video_tensor: Optional[VideoNdArray] + doc_1 = MyVideoDoc( + title='my_first_video_doc', + video_tensor=np.random.random((100, 224, 224, 3)), + ) - doc_1 = MyVideoDoc( - title='my_first_video_doc', - video_tensor=np.random.random((100, 224, 224, 3)), - ) + doc_1.video_tensor.save(file_path='file_1.mp4') - doc_1.video_tensor.save(file_path='file_1.mp4') + doc_2 = MyVideoDoc( + title='my_second_video_doc', + url='file_1.mp4', + ) + doc_2.video_tensor = parse_obj_as(VideoNdArray, doc_2.url.load().video) + doc_2.video_tensor.save(file_path='file_2.mp4') + ``` - doc_2 = MyVideoDoc( - title='my_second_video_doc', - url='https://www.kozco.com/tech/piano2.wav', - ) - - doc_2.video_tensor = parse_obj_as(VideoNdArray, doc_2.url.load().video) - doc_2.video_tensor.save(file_path='file_2.mp4') - + --- """ @classmethod diff --git a/docarray/typing/tensor/video/video_tensor.py b/docarray/typing/tensor/video/video_tensor.py index 974f409fcb3..fa4fbb47d6b 100644 --- a/docarray/typing/tensor/video/video_tensor.py +++ b/docarray/typing/tensor/video/video_tensor.py @@ -1,7 +1,7 @@ from typing import Union from docarray.typing.tensor.video.video_ndarray import VideoNdArray -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/video/video_tensor_mixin.py b/docarray/typing/tensor/video/video_tensor_mixin.py index cfa5a46ea70..3d19e499bf4 100644 --- a/docarray/typing/tensor/video/video_tensor_mixin.py +++ b/docarray/typing/tensor/video/video_tensor_mixin.py @@ -7,7 +7,7 @@ from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.audio.audio_tensor import AudioTensor -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook T = TypeVar('T', bound='VideoTensorMixin') @@ -38,6 +38,34 @@ def save( """ Save video tensor to a .mp4 file. + --- + + ```python + import numpy as np + + from docarray import BaseDoc + from docarray.typing.tensor.audio.audio_tensor import AudioTensor + from docarray.typing.tensor.video.video_tensor import VideoTensor + + + class MyDoc(BaseDoc): + video_tensor: VideoTensor + audio_tensor: AudioTensor + + + doc = MyDoc( + video_tensor=np.random.randint(low=0, high=256, size=(10, 200, 300, 3)), + audio_tensor=np.random.randn(100, 1, 1024).astype("float32"), + ) + + doc.video_tensor.save( + file_path="/tmp/mp_.mp4", + audio_tensor=doc.audio_tensor, + audio_format="flt", + ) + ``` + + --- :param file_path: path to a .mp4 file. If file is a string, open the file by that name, otherwise treat it as a file-like object. :param audio_tensor: AudioTensor containing the video's soundtrack. @@ -47,33 +75,6 @@ def save( :param audio_codec: the name of an audio decoder/encoder. :param audio_format: the name of one of the audio formats supported by PyAV, such as 'flt', 'fltp', 's16' or 's16p'. - - EXAMPLE USAGE - - .. code-block:: python - import numpy as np - - from docarray import BaseDoc - from docarray.typing.tensor.audio.audio_tensor import AudioTensor - from docarray.typing.tensor.video.video_tensor import VideoTensor - - - class MyDoc(BaseDoc): - video_tensor: VideoTensor - audio_tensor: AudioTensor - - - doc = MyDoc( - video_tensor=np.random.randint(low=0, high=256, size=(10, 200, 300, 3)), - audio_tensor=np.random.randn(100, 1, 1024).astype("float32"), - ) - - doc.video_tensor.save( - file_path="toydata/mp_.mp4", - audio_tensor=doc.audio_tensor, - audio_format="flt", - ) - """ import av diff --git a/docarray/typing/url/audio_url.py b/docarray/typing/url/audio_url.py index c971f6d364f..bab24b518cf 100644 --- a/docarray/typing/url/audio_url.py +++ b/docarray/typing/url/audio_url.py @@ -7,7 +7,7 @@ from docarray.typing.proto_register import _register_proto from docarray.typing.url.any_url import AnyUrl from docarray.typing.url.filetypes import AUDIO_FILE_FORMATS -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook if TYPE_CHECKING: from pydantic import BaseConfig @@ -48,27 +48,30 @@ def load(self: T) -> Tuple[np.ndarray, int]: """ Load the data from the url into an AudioNdArray. - :return: AudioNdArray representing the audio file content. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional + from docarray import BaseDoc + import numpy as np - from docarray import BaseDoc - import numpy as np + from docarray.typing import AudioUrl, AudioNdArray - from docarray.typing import AudioUrl + class MyDoc(BaseDoc): + audio_url: AudioUrl + audio_tensor: Optional[AudioNdArray] - class MyDoc(Document): - audio_url: AudioUrl - audio_tensor: AudioNdArray + doc = MyDoc(audio_url='https://www.kozco.com/tech/piano2.wav') + doc.audio_tensor, _ = doc.audio_url.load() + assert isinstance(doc.audio_tensor, np.ndarray) + ``` - doc = MyDoc(audio_url="toydata/hello.wav") - doc.audio_tensor, doc.frame_rate = doc.audio_url.load() - assert isinstance(doc.audio_tensor, np.ndarray) + --- + :return: AudioNdArray representing the audio file content. """ bytes_ = AudioBytes(self.load_bytes()) return bytes_.load() diff --git a/docarray/typing/url/data/05978.jpg b/docarray/typing/url/data/05978.jpg deleted file mode 100644 index 3f0bf32e01d..00000000000 Binary files a/docarray/typing/url/data/05978.jpg and /dev/null differ diff --git a/docarray/typing/url/image_url.py b/docarray/typing/url/image_url.py index b5c40b71e2d..4c25e68094e 100644 --- a/docarray/typing/url/image_url.py +++ b/docarray/typing/url/image_url.py @@ -5,7 +5,7 @@ from docarray.typing.proto_register import _register_proto from docarray.typing.url.any_url import AnyUrl -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook if TYPE_CHECKING: from pydantic import BaseConfig @@ -49,34 +49,35 @@ def load( """ Load the data from the url into a numpy.ndarray image tensor - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from docarray import BaseDoc + from docarray.typing import ImageUrl + import numpy as np - from docarray import BaseDoc - from docarray.typing import ImageUrl - import numpy as np + class MyDoc(BaseDoc): + img_url: ImageUrl - class MyDoc(BaseDoc): - img_url: ImageUrl + doc = MyDoc( + img_url="https://upload.wikimedia.org/wikipedia/commons/8/80/" + "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg" + ) - doc = MyDoc( - img_url="https://upload.wikimedia.org/wikipedia/commons/8/80/" - "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg" - ) - - img_tensor = doc.img_url.load() - assert isinstance(img_tensor, np.ndarray) + img_tensor = doc.img_url.load() + assert isinstance(img_tensor, np.ndarray) - img_tensor = doc.img_url.load(height=224, width=224) - assert img_tensor.shape == (224, 224, 3) + img_tensor = doc.img_url.load(height=224, width=224) + assert img_tensor.shape == (224, 224, 3) - layout = ('C', 'W', 'H') - img_tensor = doc.img_url.load(height=100, width=200, axis_layout=layout) - assert img_tensor.shape == (3, 200, 100) + layout = ('C', 'W', 'H') + img_tensor = doc.img_url.load(height=100, width=200, axis_layout=layout) + assert img_tensor.shape == (3, 200, 100) + ``` + --- :param width: width of the image tensor. :param height: height of the image tensor. diff --git a/docarray/typing/url/text_url.py b/docarray/typing/url/text_url.py index 049de511d1f..1ebfad1c511 100644 --- a/docarray/typing/url/text_url.py +++ b/docarray/typing/url/text_url.py @@ -43,31 +43,26 @@ def load(self, charset: str = 'utf-8', timeout: Optional[float] = None) -> str: """ Load the text file into a string. - EXAMPLE USAGE - .. code-block:: python + --- - from docarray import BaseDoc - from docarray.typing import TextUrl + ```python + from docarray import BaseDoc + from docarray.typing import TextUrl - class MyDoc(BaseDoc): - remote_url: TextUrl - local_url: TextUrl + class MyDoc(BaseDoc): + remote_url: TextUrl - doc = MyDoc( - remote_url='https://de.wikipedia.org/wiki/Brixen', - local_url='home/username/my_file.txt', - ) + doc = MyDoc( + remote_url='https://de.wikipedia.org/wiki/Brixen', + ) - remote_txt = doc.remote_url.load() - print(remote_txt) - # prints: ```\n ...``` + remote_txt = doc.remote_url.load() + ``` - local_txt = doc.local_url.load() - print(local_txt) - # prints content of my_file.txt + --- :param timeout: timeout (sec) for urlopen network request. diff --git a/docarray/typing/url/url_3d/mesh_url.py b/docarray/typing/url/url_3d/mesh_url.py index e820ba8a4d5..073f4a006a1 100644 --- a/docarray/typing/url/url_3d/mesh_url.py +++ b/docarray/typing/url/url_3d/mesh_url.py @@ -26,33 +26,34 @@ def load( trimesh_args: Optional[Dict[str, Any]] = None, ) -> 'VerticesAndFaces': """ - Load the data from the url into a VerticesAndFaces object containing - vertices and faces information. + Load the data from the url into a VerticesAndFaces object containing + vertices and faces information. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from docarray import BaseDoc + import numpy as np - from docarray import BaseDoc - import numpy as np + from docarray.typing import Mesh3DUrl, NdArray - from docarray.typing import Mesh3DUrl, NdArray + class MyDoc(BaseDoc): + mesh_url: Mesh3DUrl - class MyDoc(BaseDoc): - mesh_url: Mesh3DUrl + doc = MyDoc(mesh_url="toydata/tetrahedron.obj") - doc = MyDoc(mesh_url="toydata/tetrahedron.obj") + tensors = doc.mesh_url.load() + assert isinstance(tensors.vertices, NdArray) + assert isinstance(tensors.faces, NdArray) + ``` - tensors = doc.mesh_url.load() - assert isinstance(tensors.vertices, NdArray) - assert isinstance(tensors.faces, NdArray) - - :param skip_materials: Skip materials if True, else skip. - :param trimesh_args: dictionary of additional arguments for `trimesh.load()` - or `trimesh.load_remote()`. - :return: VerticesAndFaces object containing vertices and faces information. + --- + :param skip_materials: Skip materials if True, else skip. + :param trimesh_args: dictionary of additional arguments for `trimesh.load()` + or `trimesh.load_remote()`. + :return: VerticesAndFaces object containing vertices and faces information. """ from docarray.documents.mesh.vertices_and_faces import VerticesAndFaces diff --git a/docarray/typing/url/url_3d/point_cloud_url.py b/docarray/typing/url/url_3d/point_cloud_url.py index 502a29c740d..eba73f5da6f 100644 --- a/docarray/typing/url/url_3d/point_cloud_url.py +++ b/docarray/typing/url/url_3d/point_cloud_url.py @@ -31,26 +31,29 @@ def load( """ Load the data from the url into an NdArray containing point cloud information. - EXAMPLE USAGE - .. code-block:: python + --- - import numpy as np - from docarray import BaseDoc + ```python + import numpy as np + from docarray import BaseDoc + + from docarray.typing import PointCloud3DUrl - from docarray.typing import PointCloud3DUrl + class MyDoc(BaseDoc): + point_cloud_url: PointCloud3DUrl - class MyDoc(BaseDoc): - point_cloud_url: PointCloud3DUrl + doc = MyDoc(point_cloud_url="toydata/tetrahedron.obj") - doc = MyDoc(point_cloud_url="toydata/tetrahedron.obj") + # point_cloud = doc.point_cloud_url.load(samples=100) - point_cloud = doc.point_cloud_url.load(samples=100) + # assert isinstance(point_cloud, np.ndarray) + # assert point_cloud.shape == (100, 3) + ``` - assert isinstance(point_cloud, np.ndarray) - assert point_cloud.shape == (100, 3) + --- :param samples: number of points to sample from the mesh :param multiple_geometries: if False, store point cloud in 2D np.ndarray. diff --git a/docarray/typing/url/video_url.py b/docarray/typing/url/video_url.py index 44c2d33f9b2..e6b35642f5e 100644 --- a/docarray/typing/url/video_url.py +++ b/docarray/typing/url/video_url.py @@ -4,7 +4,7 @@ from docarray.typing.bytes.video_bytes import VideoLoadResult from docarray.typing.proto_register import _register_proto from docarray.typing.url.any_url import AnyUrl -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook if TYPE_CHECKING: from pydantic import BaseConfig @@ -43,57 +43,61 @@ def load(self: T, **kwargs) -> VideoLoadResult: Load the data from the url into a named Tuple of VideoNdArray, AudioNdArray and NdArray. - :param kwargs: supports all keyword arguments that are being supported by - av.open() as described in: - https://pyav.org/docs/stable/api/_globals.html?highlight=open#av.open + --- - :return: AudioNdArray representing the audio content, VideoNdArray representing - the images of the video, NdArray of the key frame indices. + ```python + from typing import Optional + from docarray import BaseDoc - EXAMPLE USAGE + from docarray.typing import VideoUrl, VideoNdArray, AudioNdArray, NdArray - .. code-block:: python - from typing import Optional + class MyDoc(BaseDoc): + video_url: VideoUrl + video: Optional[VideoNdArray] + audio: Optional[AudioNdArray] + key_frame_indices: Optional[NdArray] - from docarray import BaseDoc - from docarray.typing import VideoUrl, VideoNdArray, AudioNdArray, NdArray + doc = MyDoc( + video_url='https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true' + ) + doc.video, doc.audio, doc.key_frame_indices = doc.video_url.load() + assert isinstance(doc.video, VideoNdArray) + assert isinstance(doc.audio, AudioNdArray) + assert isinstance(doc.key_frame_indices, NdArray) + ``` - class MyDoc(BaseDoc): - video_url: VideoUrl - video: Optional[VideoNdArray] - audio: Optional[AudioNdArray] - key_frame_indices: Optional[NdArray] + --- + You can load only the key frames (or video, audio respectively): - doc = MyDoc( - video_url='https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true' - ) - doc.video, doc.audio, doc.key_frame_indices = doc.video_url.load() + --- - assert isinstance(doc.video, VideoNdArray) - assert isinstance(doc.audio, AudioNdArray) - assert isinstance(doc.key_frame_indices, NdArray) + ```python + from pydantic import parse_obj_as - You can load only the key frames (or video, audio respectively): + from docarray.typing import NdArray, VideoUrl - .. code-block:: python - from pydantic import parse_obj_as + url = parse_obj_as( + VideoUrl, + 'https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true', + ) + key_frame_indices = url.load().key_frame_indices + assert isinstance(key_frame_indices, NdArray) + ``` - from docarray.typing import NdArray, VideoUrl + --- + :param kwargs: supports all keyword arguments that are being supported by + av.open() as described in: + https://pyav.org/docs/stable/api/_globals.html?highlight=open#av.open - url = parse_obj_as( - VideoUrl, - 'https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true', - ) - key_frame_indices = url.load().key_frame_indices - assert isinstance(key_frame_indices, NdArray) - + :return: AudioNdArray representing the audio content, VideoNdArray representing + the images of the video, NdArray of the key frame indices. """ from docarray.typing.bytes.video_bytes import VideoBytes diff --git a/docarray/utils/query_language/__init__.py b/docarray/utils/_internal/__init__.py similarity index 100% rename from docarray/utils/query_language/__init__.py rename to docarray/utils/_internal/__init__.py diff --git a/docarray/utils/_typing.py b/docarray/utils/_internal/_typing.py similarity index 100% rename from docarray/utils/_typing.py rename to docarray/utils/_internal/_typing.py diff --git a/docarray/utils/cache.py b/docarray/utils/_internal/cache.py similarity index 93% rename from docarray/utils/cache.py rename to docarray/utils/_internal/cache.py index 4df305414b3..249c4f9d179 100644 --- a/docarray/utils/cache.py +++ b/docarray/utils/_internal/cache.py @@ -4,7 +4,7 @@ @lru_cache(maxsize=None) -def get_cache_path() -> Path: +def _get_cache_path() -> Path: """ Get the path to the cache directory. diff --git a/docarray/utils/compress.py b/docarray/utils/_internal/compress.py similarity index 97% rename from docarray/utils/compress.py rename to docarray/utils/_internal/compress.py index d427e998c89..0ff00f8eecd 100644 --- a/docarray/utils/compress.py +++ b/docarray/utils/_internal/compress.py @@ -1,4 +1,4 @@ -from typing import Optional, Callable, IO +from typing import IO, Callable, Optional def _compress_bytes(data: bytes, algorithm: Optional[str] = None) -> bytes: diff --git a/docarray/utils/misc.py b/docarray/utils/_internal/misc.py similarity index 100% rename from docarray/utils/misc.py rename to docarray/utils/_internal/misc.py diff --git a/docarray/utils/progress_bar.py b/docarray/utils/_internal/progress_bar.py similarity index 92% rename from docarray/utils/progress_bar.py rename to docarray/utils/_internal/progress_bar.py index ad7bb3dcd5d..4750c509a1a 100644 --- a/docarray/utils/progress_bar.py +++ b/docarray/utils/_internal/progress_bar.py @@ -12,7 +12,7 @@ ) -class QPSColumn(TextColumn): +class _QPSColumn(TextColumn): def render(self, task) -> Text: if task.speed: _text = f'{task.speed:.0f} QPS' @@ -34,7 +34,7 @@ def _get_pbar(disable: bool, total: Optional[int] = None): BarColumn(), MofNCompleteColumn(), '•', - QPSColumn('{task.speed} QPS', justify='right', style='progress.data.speed'), + _QPSColumn('{task.speed} QPS', justify='right', style='progress.data.speed'), '•', TimeRemainingColumn() if total else TimeElapsedColumn(), '•', diff --git a/docarray/utils/_internal/query_language/__init__.py b/docarray/utils/_internal/query_language/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docarray/utils/query_language/lookup.py b/docarray/utils/_internal/query_language/lookup.py similarity index 100% rename from docarray/utils/query_language/lookup.py rename to docarray/utils/_internal/query_language/lookup.py diff --git a/docarray/utils/query_language/query_parser.py b/docarray/utils/_internal/query_language/query_parser.py similarity index 98% rename from docarray/utils/query_language/query_parser.py rename to docarray/utils/_internal/query_language/query_parser.py index 8bce97f6e0d..b8381584a94 100644 --- a/docarray/utils/query_language/query_parser.py +++ b/docarray/utils/_internal/query_language/query_parser.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Optional, Union -from docarray.utils.query_language.lookup import ( +from docarray.utils._internal.query_language.lookup import ( LookupLeaf, LookupNode, LookupTreeElem, diff --git a/docarray/utils/filter.py b/docarray/utils/filter.py index 7a9887364fc..a68641a7b13 100644 --- a/docarray/utils/filter.py +++ b/docarray/utils/filter.py @@ -63,7 +63,7 @@ class MyDocument(BaseDoc): :return: A DocArray containing the Documents in `docs` that fulfill the filter conditions in the `query` """ - from docarray.utils.query_language.query_parser import QueryParser + from docarray.utils._internal.query_language.query_parser import QueryParser if query: query = query if not isinstance(query, str) else json.loads(query) diff --git a/docs/api_references/typing/bytes.md b/docs/api_references/typing/bytes.md new file mode 100644 index 00000000000..a263581af0d --- /dev/null +++ b/docs/api_references/typing/bytes.md @@ -0,0 +1,6 @@ +# Bytes + +::: docarray.typing.bytes + + + diff --git a/docs/api_references/typing/id.md b/docs/api_references/typing/id.md new file mode 100644 index 00000000000..6dbfbd39dea --- /dev/null +++ b/docs/api_references/typing/id.md @@ -0,0 +1,6 @@ +# Id + +::: docarray.typing.id + + + diff --git a/docs/api_references/typing/tensor.md b/docs/api_references/typing/tensor.md new file mode 100644 index 00000000000..f8040705f32 --- /dev/null +++ b/docs/api_references/typing/tensor.md @@ -0,0 +1,3 @@ +# Tensor + +::: docarray.typing.tensor diff --git a/docs/api_references/typing/typing.md b/docs/api_references/typing/typing.md deleted file mode 100644 index fa29d35750c..00000000000 --- a/docs/api_references/typing/typing.md +++ /dev/null @@ -1,4 +0,0 @@ -# Typing - -::: docarray.typing - diff --git a/docs/api_references/typing/url.md b/docs/api_references/typing/url.md new file mode 100644 index 00000000000..f58c3b6c722 --- /dev/null +++ b/docs/api_references/typing/url.md @@ -0,0 +1,6 @@ +# Url + +::: docarray.typing.url + + + diff --git a/file_1.mp4 b/file_1.mp4 new file mode 100644 index 00000000000..15b58e871af Binary files /dev/null and b/file_1.mp4 differ diff --git a/file_2.mp4 b/file_2.mp4 new file mode 100644 index 00000000000..15b58e871af Binary files /dev/null and b/file_2.mp4 differ diff --git a/tests/documentation/test_docstring.py b/tests/documentation/test_docstring.py new file mode 100644 index 00000000000..c6a8e3d9a40 --- /dev/null +++ b/tests/documentation/test_docstring.py @@ -0,0 +1,48 @@ +""" +this test check the docstring of all of our public API. It does it +by checking the `__all__` of each of our namespace. + +to add a new namespace you need to +* import it +* add it to the `SUB_MODULE_TO_CHECK` list +""" + +import pytest +from mktestdocs import check_docstring, get_codeblock_members + +import docarray.data +import docarray.documents +import docarray.index +import docarray.store +import docarray.typing + +SUB_MODULE_TO_CHECK = [ + docarray, + docarray.index, + docarray.data, + docarray.documents, + docarray.store, + docarray.typing, +] + + +def get_obj_to_check(lib): + obj_to_check = [] + for obj in lib.__all__: + obj_to_check.append(getattr(lib, obj)) + return obj_to_check + + +obj_to_check = [] + +for lib in SUB_MODULE_TO_CHECK: + obj_to_check.extend(get_obj_to_check(lib)) + +members = [] +for obj in obj_to_check: + members.extend(get_codeblock_members(obj)) + + +@pytest.mark.parametrize("obj", members, ids=lambda d: d.__qualname__) +def test_member(obj): + check_docstring(obj) diff --git a/tests/integrations/document/test_proto.py b/tests/integrations/document/test_proto.py index c1f38d05b41..02b9aae3f00 100644 --- a/tests/integrations/document/test_proto.py +++ b/tests/integrations/document/test_proto.py @@ -18,7 +18,7 @@ TorchTensor, ) from docarray.typing.tensor import NdArrayEmbedding -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/integrations/predefined_document/test_audio.py b/tests/integrations/predefined_document/test_audio.py index c25aab9b1ab..2ba207245f7 100644 --- a/tests/integrations/predefined_document/test_audio.py +++ b/tests/integrations/predefined_document/test_audio.py @@ -10,7 +10,7 @@ from docarray.documents import AudioDoc from docarray.typing import AudioUrl from docarray.typing.tensor.audio import AudioNdArray, AudioTorchTensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available from tests import TOYDATA_DIR tf_available = is_tf_available() diff --git a/tests/integrations/predefined_document/test_image.py b/tests/integrations/predefined_document/test_image.py index 92e19d09a14..f77e945d714 100644 --- a/tests/integrations/predefined_document/test_image.py +++ b/tests/integrations/predefined_document/test_image.py @@ -5,7 +5,7 @@ from docarray import BaseDoc from docarray.documents import ImageDoc -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/integrations/predefined_document/test_point_cloud.py b/tests/integrations/predefined_document/test_point_cloud.py index 76b7e0236d8..b8a75914f26 100644 --- a/tests/integrations/predefined_document/test_point_cloud.py +++ b/tests/integrations/predefined_document/test_point_cloud.py @@ -5,7 +5,7 @@ from docarray import BaseDoc from docarray.documents import PointCloud3D -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available from tests import TOYDATA_DIR tf_available = is_tf_available() diff --git a/tests/integrations/predefined_document/test_video.py b/tests/integrations/predefined_document/test_video.py index e208e3890c9..606dad6a6c1 100644 --- a/tests/integrations/predefined_document/test_video.py +++ b/tests/integrations/predefined_document/test_video.py @@ -6,7 +6,7 @@ from docarray import BaseDoc from docarray.documents import VideoDoc from docarray.typing import AudioNdArray, NdArray, VideoNdArray -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available from tests import TOYDATA_DIR tf_available = is_tf_available() diff --git a/tests/integrations/store/test_file.py b/tests/integrations/store/test_file.py index 43205260abd..4b6a72c5b62 100644 --- a/tests/integrations/store/test_file.py +++ b/tests/integrations/store/test_file.py @@ -6,7 +6,7 @@ from docarray import DocArray from docarray.documents import TextDoc from docarray.store.file import ConcurrentPushException, FileDocStore -from docarray.utils.cache import get_cache_path +from docarray.utils._internal.cache import _get_cache_path from tests.integrations.store import gen_text_docs, get_test_da, profile_memory DA_LEN: int = 2**10 @@ -14,7 +14,7 @@ def test_path_resolution(): - assert FileDocStore._abs_filepath('meow') == get_cache_path() / 'meow' + assert FileDocStore._abs_filepath('meow') == _get_cache_path() / 'meow' assert FileDocStore._abs_filepath('/meow') == Path('/meow') assert FileDocStore._abs_filepath('~/meow') == Path.home() / 'meow' assert FileDocStore._abs_filepath('./meow') == Path.cwd() / 'meow' diff --git a/tests/integrations/typing/test_tensor.py b/tests/integrations/typing/test_tensor.py index ba15e2d5c94..6b32432d971 100644 --- a/tests/integrations/typing/test_tensor.py +++ b/tests/integrations/typing/test_tensor.py @@ -4,7 +4,7 @@ from docarray import BaseDoc from docarray.typing import AnyTensor, NdArray, TorchTensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/integrations/typing/test_tensorflow_tensor.py b/tests/integrations/typing/test_tensorflow_tensor.py index 84505968090..edd64e3dcbb 100644 --- a/tests/integrations/typing/test_tensorflow_tensor.py +++ b/tests/integrations/typing/test_tensorflow_tensor.py @@ -1,7 +1,7 @@ import pytest from docarray import BaseDoc -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/integrations/typing/test_tensors_interop.py b/tests/integrations/typing/test_tensors_interop.py index d59751f21af..47023dca96a 100644 --- a/tests/integrations/typing/test_tensors_interop.py +++ b/tests/integrations/typing/test_tensors_interop.py @@ -8,7 +8,7 @@ NdArrayEmbedding, TorchEmbedding, ) -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/array/stack/test_array_stacked_tf.py b/tests/units/array/stack/test_array_stacked_tf.py index e82bfc7716a..0ec91268575 100644 --- a/tests/units/array/stack/test_array_stacked_tf.py +++ b/tests/units/array/stack/test_array_stacked_tf.py @@ -5,7 +5,7 @@ from docarray import BaseDoc, DocArray from docarray.array import DocArrayStacked from docarray.typing import AnyTensor, NdArray -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/array/test_array.py b/tests/units/array/test_array.py index bab9ccd1313..be4fa6fa505 100644 --- a/tests/units/array/test_array.py +++ b/tests/units/array/test_array.py @@ -6,7 +6,7 @@ from docarray import BaseDoc, DocArray from docarray.typing import ImageUrl, NdArray, TorchTensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/computation_backends/tensorflow_backend/test_basics.py b/tests/units/computation_backends/tensorflow_backend/test_basics.py index 6eed1a58cf0..ae5f9b44264 100644 --- a/tests/units/computation_backends/tensorflow_backend/test_basics.py +++ b/tests/units/computation_backends/tensorflow_backend/test_basics.py @@ -1,7 +1,7 @@ import numpy as np import pytest -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/computation_backends/tensorflow_backend/test_metrics.py b/tests/units/computation_backends/tensorflow_backend/test_metrics.py index 863dd18fe4c..196297adf0b 100644 --- a/tests/units/computation_backends/tensorflow_backend/test_metrics.py +++ b/tests/units/computation_backends/tensorflow_backend/test_metrics.py @@ -1,6 +1,6 @@ import pytest -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/computation_backends/tensorflow_backend/test_retrieval.py b/tests/units/computation_backends/tensorflow_backend/test_retrieval.py index feac992d760..283d7f8b44a 100644 --- a/tests/units/computation_backends/tensorflow_backend/test_retrieval.py +++ b/tests/units/computation_backends/tensorflow_backend/test_retrieval.py @@ -1,6 +1,6 @@ import pytest -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/document/proto/test_document_proto.py b/tests/units/document/proto/test_document_proto.py index e6e8a58fa99..1642c17631d 100644 --- a/tests/units/document/proto/test_document_proto.py +++ b/tests/units/document/proto/test_document_proto.py @@ -7,7 +7,7 @@ from docarray import DocArray from docarray.base_doc import BaseDoc from docarray.typing import NdArray, TorchTensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available if is_tf_available(): import tensorflow as tf diff --git a/tests/units/typing/tensor/test_audio_tensor.py b/tests/units/typing/tensor/test_audio_tensor.py index 3bc20a1c7f6..4a145f3c4b7 100644 --- a/tests/units/typing/tensor/test_audio_tensor.py +++ b/tests/units/typing/tensor/test_audio_tensor.py @@ -8,7 +8,7 @@ from docarray import BaseDoc from docarray.typing.tensor.audio.audio_ndarray import AudioNdArray from docarray.typing.tensor.audio.audio_torch_tensor import AudioTorchTensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/typing/tensor/test_tensor_flow_tensor.py b/tests/units/typing/tensor/test_tensor_flow_tensor.py index 5c7d942a02d..3a51f5a95aa 100644 --- a/tests/units/typing/tensor/test_tensor_flow_tensor.py +++ b/tests/units/typing/tensor/test_tensor_flow_tensor.py @@ -4,7 +4,7 @@ from pydantic.tools import parse_obj_as from docarray.base_doc.io.json import orjson_dumps -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/typing/tensor/test_video_tensor.py b/tests/units/typing/tensor/test_video_tensor.py index f9981dedbc9..23ae8115e64 100644 --- a/tests/units/typing/tensor/test_video_tensor.py +++ b/tests/units/typing/tensor/test_video_tensor.py @@ -12,7 +12,7 @@ VideoNdArray, VideoTorchTensor, ) -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/typing/url/test_audio_url.py b/tests/units/typing/url/test_audio_url.py index 6b876d81de7..9168302aada 100644 --- a/tests/units/typing/url/test_audio_url.py +++ b/tests/units/typing/url/test_audio_url.py @@ -8,7 +8,7 @@ from docarray import BaseDoc from docarray.base_doc.io.json import orjson_dumps from docarray.typing import AudioTorchTensor, AudioUrl -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available from tests import TOYDATA_DIR tf_available = is_tf_available() diff --git a/tests/units/typing/url/test_video_url.py b/tests/units/typing/url/test_video_url.py index a1401bde896..e0dba53b6fe 100644 --- a/tests/units/typing/url/test_video_url.py +++ b/tests/units/typing/url/test_video_url.py @@ -14,7 +14,7 @@ VideoTorchTensor, VideoUrl, ) -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available from tests import TOYDATA_DIR tf_available = is_tf_available() diff --git a/tests/units/util/query_language/test_lookup.py b/tests/units/util/query_language/test_lookup.py index 37b3c2a1a4c..dd30f51c8f0 100644 --- a/tests/units/util/query_language/test_lookup.py +++ b/tests/units/util/query_language/test_lookup.py @@ -1,6 +1,6 @@ import pytest -from docarray.utils.query_language.lookup import dunder_get, lookup +from docarray.utils._internal.query_language.lookup import dunder_get, lookup class A: diff --git a/tests/units/util/test_typing.py b/tests/units/util/test_typing.py index 2db742522f5..5446cf3ce04 100644 --- a/tests/units/util/test_typing.py +++ b/tests/units/util/test_typing.py @@ -4,8 +4,8 @@ from docarray.typing import NdArray, TorchTensor from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._typing import is_tensor_union, is_type_tensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal._typing import is_tensor_union, is_type_tensor +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: