From 4a0e617fc0dfac68bf88a0097591fb99e7c9361b Mon Sep 17 00:00:00 2001 From: Joan Fontanals Martinez Date: Thu, 26 May 2022 11:14:54 +0200 Subject: [PATCH 01/20] feat: bump protobuf --- docarray/proto/docarray_pb2.py | 626 ++------------------------------- 1 file changed, 29 insertions(+), 597 deletions(-) diff --git a/docarray/proto/docarray_pb2.py b/docarray/proto/docarray_pb2.py index 2beb100c43e..7c87802c3c0 100644 --- a/docarray/proto/docarray_pb2.py +++ b/docarray/proto/docarray_pb2.py @@ -2,9 +2,9 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: docarray.proto """Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) @@ -14,599 +14,31 @@ from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 -DESCRIPTOR = _descriptor.FileDescriptor( - name='docarray.proto', - package='docarray', - syntax='proto3', - serialized_options=None, - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\x0e\x64ocarray.proto\x12\x08\x64ocarray\x1a\x1cgoogle/protobuf/struct.proto\"A\n\x11\x44\x65nseNdArrayProto\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\x0c\x12\r\n\x05shape\x18\x02 \x03(\r\x12\r\n\x05\x64type\x18\x03 \x01(\t\"\xb6\x01\n\x0cNdArrayProto\x12,\n\x05\x64\x65nse\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProtoH\x00\x12.\n\x06sparse\x18\x02 \x01(\x0b\x32\x1c.docarray.SparseNdArrayProtoH\x00\x12\x10\n\x08\x63ls_name\x18\x03 \x01(\t\x12+\n\nparameters\x18\x04 \x01(\x0b\x32\x17.google.protobuf.StructB\t\n\x07\x63ontent\"~\n\x12SparseNdArrayProto\x12,\n\x07indices\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12+\n\x06values\x18\x02 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12\r\n\x05shape\x18\x03 \x03(\r\"V\n\x0fNamedScoreProto\x12\r\n\x05value\x18\x01 \x01(\x02\x12\x0f\n\x07op_name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0e\n\x06ref_id\x18\x04 \x01(\t\"\xed\x05\n\rDocumentProto\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0e\n\x04\x62lob\x18\x02 \x01(\x0cH\x00\x12(\n\x06tensor\x18\x03 \x01(\x0b\x32\x16.docarray.NdArrayProtoH\x00\x12\x0e\n\x04text\x18\x04 \x01(\tH\x00\x12\x13\n\x0bgranularity\x18\x05 \x01(\r\x12\x11\n\tadjacency\x18\x06 \x01(\r\x12\x11\n\tparent_id\x18\x07 \x01(\t\x12\x0e\n\x06weight\x18\x08 \x01(\x02\x12\x0b\n\x03uri\x18\t \x01(\t\x12\x10\n\x08modality\x18\n \x01(\t\x12\x11\n\tmime_type\x18\x0b \x01(\t\x12\x0e\n\x06offset\x18\x0c \x01(\x02\x12\x10\n\x08location\x18\r \x03(\x02\x12\'\n\x06\x63hunks\x18\x0e \x03(\x0b\x32\x17.docarray.DocumentProto\x12(\n\x07matches\x18\x0f \x03(\x0b\x32\x17.docarray.DocumentProto\x12)\n\tembedding\x18\x10 \x01(\x0b\x32\x16.docarray.NdArrayProto\x12%\n\x04tags\x18\x11 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x33\n\x06scores\x18\x12 \x03(\x0b\x32#.docarray.DocumentProto.ScoresEntry\x12=\n\x0b\x65valuations\x18\x13 \x03(\x0b\x32(.docarray.DocumentProto.EvaluationsEntry\x12*\n\t_metadata\x18\x14 \x01(\x0b\x32\x17.google.protobuf.Struct\x1aH\n\x0bScoresEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.docarray.NamedScoreProto:\x02\x38\x01\x1aM\n\x10\x45valuationsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.docarray.NamedScoreProto:\x02\x38\x01\x42\t\n\x07\x63ontent\";\n\x12\x44ocumentArrayProto\x12%\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x17.docarray.DocumentProtob\x06proto3' - , - dependencies=[google_dot_protobuf_dot_struct__pb2.DESCRIPTOR,]) - - - - -_DENSENDARRAYPROTO = _descriptor.Descriptor( - name='DenseNdArrayProto', - full_name='docarray.DenseNdArrayProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='buffer', full_name='docarray.DenseNdArrayProto.buffer', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='shape', full_name='docarray.DenseNdArrayProto.shape', index=1, - number=2, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='dtype', full_name='docarray.DenseNdArrayProto.dtype', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=58, - serialized_end=123, -) - - -_NDARRAYPROTO = _descriptor.Descriptor( - name='NdArrayProto', - full_name='docarray.NdArrayProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='dense', full_name='docarray.NdArrayProto.dense', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='sparse', full_name='docarray.NdArrayProto.sparse', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='cls_name', full_name='docarray.NdArrayProto.cls_name', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='parameters', full_name='docarray.NdArrayProto.parameters', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='content', full_name='docarray.NdArrayProto.content', - index=0, containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[]), - ], - serialized_start=126, - serialized_end=308, -) - - -_SPARSENDARRAYPROTO = _descriptor.Descriptor( - name='SparseNdArrayProto', - full_name='docarray.SparseNdArrayProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='indices', full_name='docarray.SparseNdArrayProto.indices', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='values', full_name='docarray.SparseNdArrayProto.values', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='shape', full_name='docarray.SparseNdArrayProto.shape', index=2, - number=3, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=310, - serialized_end=436, -) - - -_NAMEDSCOREPROTO = _descriptor.Descriptor( - name='NamedScoreProto', - full_name='docarray.NamedScoreProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='value', full_name='docarray.NamedScoreProto.value', index=0, - number=1, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='op_name', full_name='docarray.NamedScoreProto.op_name', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='description', full_name='docarray.NamedScoreProto.description', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='ref_id', full_name='docarray.NamedScoreProto.ref_id', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=438, - serialized_end=524, -) - - -_DOCUMENTPROTO_SCORESENTRY = _descriptor.Descriptor( - name='ScoresEntry', - full_name='docarray.DocumentProto.ScoresEntry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='docarray.DocumentProto.ScoresEntry.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='value', full_name='docarray.DocumentProto.ScoresEntry.value', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=b'8\001', - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1114, - serialized_end=1186, -) - -_DOCUMENTPROTO_EVALUATIONSENTRY = _descriptor.Descriptor( - name='EvaluationsEntry', - full_name='docarray.DocumentProto.EvaluationsEntry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='docarray.DocumentProto.EvaluationsEntry.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='value', full_name='docarray.DocumentProto.EvaluationsEntry.value', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=b'8\001', - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1188, - serialized_end=1265, -) - -_DOCUMENTPROTO = _descriptor.Descriptor( - name='DocumentProto', - full_name='docarray.DocumentProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='id', full_name='docarray.DocumentProto.id', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='blob', full_name='docarray.DocumentProto.blob', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='tensor', full_name='docarray.DocumentProto.tensor', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='text', full_name='docarray.DocumentProto.text', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='granularity', full_name='docarray.DocumentProto.granularity', index=4, - number=5, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='adjacency', full_name='docarray.DocumentProto.adjacency', index=5, - number=6, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='parent_id', full_name='docarray.DocumentProto.parent_id', index=6, - number=7, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='weight', full_name='docarray.DocumentProto.weight', index=7, - number=8, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='uri', full_name='docarray.DocumentProto.uri', index=8, - number=9, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='modality', full_name='docarray.DocumentProto.modality', index=9, - number=10, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='mime_type', full_name='docarray.DocumentProto.mime_type', index=10, - number=11, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='offset', full_name='docarray.DocumentProto.offset', index=11, - number=12, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='location', full_name='docarray.DocumentProto.location', index=12, - number=13, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='chunks', full_name='docarray.DocumentProto.chunks', index=13, - number=14, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='matches', full_name='docarray.DocumentProto.matches', index=14, - number=15, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='embedding', full_name='docarray.DocumentProto.embedding', index=15, - number=16, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='tags', full_name='docarray.DocumentProto.tags', index=16, - number=17, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='scores', full_name='docarray.DocumentProto.scores', index=17, - number=18, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='evaluations', full_name='docarray.DocumentProto.evaluations', index=18, - number=19, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='_metadata', full_name='docarray.DocumentProto._metadata', index=19, - number=20, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[_DOCUMENTPROTO_SCORESENTRY, _DOCUMENTPROTO_EVALUATIONSENTRY, ], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='content', full_name='docarray.DocumentProto.content', - index=0, containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[]), - ], - serialized_start=527, - serialized_end=1276, -) - - -_DOCUMENTARRAYPROTO = _descriptor.Descriptor( - name='DocumentArrayProto', - full_name='docarray.DocumentArrayProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='docs', full_name='docarray.DocumentArrayProto.docs', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1278, - serialized_end=1337, -) - -_NDARRAYPROTO.fields_by_name['dense'].message_type = _DENSENDARRAYPROTO -_NDARRAYPROTO.fields_by_name['sparse'].message_type = _SPARSENDARRAYPROTO -_NDARRAYPROTO.fields_by_name['parameters'].message_type = google_dot_protobuf_dot_struct__pb2._STRUCT -_NDARRAYPROTO.oneofs_by_name['content'].fields.append( - _NDARRAYPROTO.fields_by_name['dense']) -_NDARRAYPROTO.fields_by_name['dense'].containing_oneof = _NDARRAYPROTO.oneofs_by_name['content'] -_NDARRAYPROTO.oneofs_by_name['content'].fields.append( - _NDARRAYPROTO.fields_by_name['sparse']) -_NDARRAYPROTO.fields_by_name['sparse'].containing_oneof = _NDARRAYPROTO.oneofs_by_name['content'] -_SPARSENDARRAYPROTO.fields_by_name['indices'].message_type = _DENSENDARRAYPROTO -_SPARSENDARRAYPROTO.fields_by_name['values'].message_type = _DENSENDARRAYPROTO -_DOCUMENTPROTO_SCORESENTRY.fields_by_name['value'].message_type = _NAMEDSCOREPROTO -_DOCUMENTPROTO_SCORESENTRY.containing_type = _DOCUMENTPROTO -_DOCUMENTPROTO_EVALUATIONSENTRY.fields_by_name['value'].message_type = _NAMEDSCOREPROTO -_DOCUMENTPROTO_EVALUATIONSENTRY.containing_type = _DOCUMENTPROTO -_DOCUMENTPROTO.fields_by_name['tensor'].message_type = _NDARRAYPROTO -_DOCUMENTPROTO.fields_by_name['chunks'].message_type = _DOCUMENTPROTO -_DOCUMENTPROTO.fields_by_name['matches'].message_type = _DOCUMENTPROTO -_DOCUMENTPROTO.fields_by_name['embedding'].message_type = _NDARRAYPROTO -_DOCUMENTPROTO.fields_by_name['tags'].message_type = google_dot_protobuf_dot_struct__pb2._STRUCT -_DOCUMENTPROTO.fields_by_name['scores'].message_type = _DOCUMENTPROTO_SCORESENTRY -_DOCUMENTPROTO.fields_by_name['evaluations'].message_type = _DOCUMENTPROTO_EVALUATIONSENTRY -_DOCUMENTPROTO.fields_by_name['_metadata'].message_type = google_dot_protobuf_dot_struct__pb2._STRUCT -_DOCUMENTPROTO.oneofs_by_name['content'].fields.append( - _DOCUMENTPROTO.fields_by_name['blob']) -_DOCUMENTPROTO.fields_by_name['blob'].containing_oneof = _DOCUMENTPROTO.oneofs_by_name['content'] -_DOCUMENTPROTO.oneofs_by_name['content'].fields.append( - _DOCUMENTPROTO.fields_by_name['tensor']) -_DOCUMENTPROTO.fields_by_name['tensor'].containing_oneof = _DOCUMENTPROTO.oneofs_by_name['content'] -_DOCUMENTPROTO.oneofs_by_name['content'].fields.append( - _DOCUMENTPROTO.fields_by_name['text']) -_DOCUMENTPROTO.fields_by_name['text'].containing_oneof = _DOCUMENTPROTO.oneofs_by_name['content'] -_DOCUMENTARRAYPROTO.fields_by_name['docs'].message_type = _DOCUMENTPROTO -DESCRIPTOR.message_types_by_name['DenseNdArrayProto'] = _DENSENDARRAYPROTO -DESCRIPTOR.message_types_by_name['NdArrayProto'] = _NDARRAYPROTO -DESCRIPTOR.message_types_by_name['SparseNdArrayProto'] = _SPARSENDARRAYPROTO -DESCRIPTOR.message_types_by_name['NamedScoreProto'] = _NAMEDSCOREPROTO -DESCRIPTOR.message_types_by_name['DocumentProto'] = _DOCUMENTPROTO -DESCRIPTOR.message_types_by_name['DocumentArrayProto'] = _DOCUMENTARRAYPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -DenseNdArrayProto = _reflection.GeneratedProtocolMessageType('DenseNdArrayProto', (_message.Message,), { - 'DESCRIPTOR' : _DENSENDARRAYPROTO, - '__module__' : 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.DenseNdArrayProto) - }) -_sym_db.RegisterMessage(DenseNdArrayProto) - -NdArrayProto = _reflection.GeneratedProtocolMessageType('NdArrayProto', (_message.Message,), { - 'DESCRIPTOR' : _NDARRAYPROTO, - '__module__' : 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.NdArrayProto) - }) -_sym_db.RegisterMessage(NdArrayProto) - -SparseNdArrayProto = _reflection.GeneratedProtocolMessageType('SparseNdArrayProto', (_message.Message,), { - 'DESCRIPTOR' : _SPARSENDARRAYPROTO, - '__module__' : 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.SparseNdArrayProto) - }) -_sym_db.RegisterMessage(SparseNdArrayProto) - -NamedScoreProto = _reflection.GeneratedProtocolMessageType('NamedScoreProto', (_message.Message,), { - 'DESCRIPTOR' : _NAMEDSCOREPROTO, - '__module__' : 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.NamedScoreProto) - }) -_sym_db.RegisterMessage(NamedScoreProto) - -DocumentProto = _reflection.GeneratedProtocolMessageType('DocumentProto', (_message.Message,), { - - 'ScoresEntry' : _reflection.GeneratedProtocolMessageType('ScoresEntry', (_message.Message,), { - 'DESCRIPTOR' : _DOCUMENTPROTO_SCORESENTRY, - '__module__' : 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.DocumentProto.ScoresEntry) - }) - , - - 'EvaluationsEntry' : _reflection.GeneratedProtocolMessageType('EvaluationsEntry', (_message.Message,), { - 'DESCRIPTOR' : _DOCUMENTPROTO_EVALUATIONSENTRY, - '__module__' : 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.DocumentProto.EvaluationsEntry) - }) - , - 'DESCRIPTOR' : _DOCUMENTPROTO, - '__module__' : 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.DocumentProto) - }) -_sym_db.RegisterMessage(DocumentProto) -_sym_db.RegisterMessage(DocumentProto.ScoresEntry) -_sym_db.RegisterMessage(DocumentProto.EvaluationsEntry) - -DocumentArrayProto = _reflection.GeneratedProtocolMessageType('DocumentArrayProto', (_message.Message,), { - 'DESCRIPTOR' : _DOCUMENTARRAYPROTO, - '__module__' : 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.DocumentArrayProto) - }) -_sym_db.RegisterMessage(DocumentArrayProto) - - -_DOCUMENTPROTO_SCORESENTRY._options = None -_DOCUMENTPROTO_EVALUATIONSENTRY._options = None +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0e\x64ocarray.proto\x12\x08\x64ocarray\x1a\x1cgoogle/protobuf/struct.proto\"A\n\x11\x44\x65nseNdArrayProto\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\x0c\x12\r\n\x05shape\x18\x02 \x03(\r\x12\r\n\x05\x64type\x18\x03 \x01(\t\"\xb6\x01\n\x0cNdArrayProto\x12,\n\x05\x64\x65nse\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProtoH\x00\x12.\n\x06sparse\x18\x02 \x01(\x0b\x32\x1c.docarray.SparseNdArrayProtoH\x00\x12\x10\n\x08\x63ls_name\x18\x03 \x01(\t\x12+\n\nparameters\x18\x04 \x01(\x0b\x32\x17.google.protobuf.StructB\t\n\x07\x63ontent\"~\n\x12SparseNdArrayProto\x12,\n\x07indices\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12+\n\x06values\x18\x02 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12\r\n\x05shape\x18\x03 \x03(\r\"V\n\x0fNamedScoreProto\x12\r\n\x05value\x18\x01 \x01(\x02\x12\x0f\n\x07op_name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0e\n\x06ref_id\x18\x04 \x01(\t\"\xed\x05\n\rDocumentProto\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0e\n\x04\x62lob\x18\x02 \x01(\x0cH\x00\x12(\n\x06tensor\x18\x03 \x01(\x0b\x32\x16.docarray.NdArrayProtoH\x00\x12\x0e\n\x04text\x18\x04 \x01(\tH\x00\x12\x13\n\x0bgranularity\x18\x05 \x01(\r\x12\x11\n\tadjacency\x18\x06 \x01(\r\x12\x11\n\tparent_id\x18\x07 \x01(\t\x12\x0e\n\x06weight\x18\x08 \x01(\x02\x12\x0b\n\x03uri\x18\t \x01(\t\x12\x10\n\x08modality\x18\n \x01(\t\x12\x11\n\tmime_type\x18\x0b \x01(\t\x12\x0e\n\x06offset\x18\x0c \x01(\x02\x12\x10\n\x08location\x18\r \x03(\x02\x12\'\n\x06\x63hunks\x18\x0e \x03(\x0b\x32\x17.docarray.DocumentProto\x12(\n\x07matches\x18\x0f \x03(\x0b\x32\x17.docarray.DocumentProto\x12)\n\tembedding\x18\x10 \x01(\x0b\x32\x16.docarray.NdArrayProto\x12%\n\x04tags\x18\x11 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x33\n\x06scores\x18\x12 \x03(\x0b\x32#.docarray.DocumentProto.ScoresEntry\x12=\n\x0b\x65valuations\x18\x13 \x03(\x0b\x32(.docarray.DocumentProto.EvaluationsEntry\x12*\n\t_metadata\x18\x14 \x01(\x0b\x32\x17.google.protobuf.Struct\x1aH\n\x0bScoresEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.docarray.NamedScoreProto:\x02\x38\x01\x1aM\n\x10\x45valuationsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.docarray.NamedScoreProto:\x02\x38\x01\x42\t\n\x07\x63ontent\";\n\x12\x44ocumentArrayProto\x12%\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x17.docarray.DocumentProtob\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'docarray_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _DOCUMENTPROTO_SCORESENTRY._options = None + _DOCUMENTPROTO_SCORESENTRY._serialized_options = b'8\001' + _DOCUMENTPROTO_EVALUATIONSENTRY._options = None + _DOCUMENTPROTO_EVALUATIONSENTRY._serialized_options = b'8\001' + _DENSENDARRAYPROTO._serialized_start=58 + _DENSENDARRAYPROTO._serialized_end=123 + _NDARRAYPROTO._serialized_start=126 + _NDARRAYPROTO._serialized_end=308 + _SPARSENDARRAYPROTO._serialized_start=310 + _SPARSENDARRAYPROTO._serialized_end=436 + _NAMEDSCOREPROTO._serialized_start=438 + _NAMEDSCOREPROTO._serialized_end=524 + _DOCUMENTPROTO._serialized_start=527 + _DOCUMENTPROTO._serialized_end=1276 + _DOCUMENTPROTO_SCORESENTRY._serialized_start=1114 + _DOCUMENTPROTO_SCORESENTRY._serialized_end=1186 + _DOCUMENTPROTO_EVALUATIONSENTRY._serialized_start=1188 + _DOCUMENTPROTO_EVALUATIONSENTRY._serialized_end=1265 + _DOCUMENTARRAYPROTO._serialized_start=1278 + _DOCUMENTARRAYPROTO._serialized_end=1337 # @@protoc_insertion_point(module_scope) From e625741489e86b3e5e3ce72c24eaa8d72db9c765 Mon Sep 17 00:00:00 2001 From: Joan Fontanals Martinez Date: Thu, 26 May 2022 12:06:58 +0200 Subject: [PATCH 02/20] fix: fix flake8 --- .github/workflows/ci.yml | 4 ++-- docarray/helper.py | 2 +- setup.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9a15dfb96d..db3d1a861db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,9 +60,9 @@ jobs: run: | pip install flake8 # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/resources/ + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,docarray/proto/docarray_pb2.py # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/resources/ + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,docarray/proto/docarray_pb2.py check-black: runs-on: ubuntu-latest diff --git a/docarray/helper.py b/docarray/helper.py index 865f19ff80d..94cffbda2b8 100644 --- a/docarray/helper.py +++ b/docarray/helper.py @@ -176,7 +176,7 @@ def get_full_version() -> Dict: return { 'docarray': __version__, 'protobuf': google.protobuf.__version__, - 'proto-backend': api_implementation._default_implementation_type, + 'proto-backend': api_implementation.Type(), 'python': platform.python_version(), 'platform': platform.system(), 'platform-release': platform.release(), diff --git a/setup.py b/setup.py index d6ef424b5fa..2f3b8b20d98 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ extras_require={ # req usage, please see https://docarray.jina.ai/#install 'common': [ - 'protobuf>=3.13.0,<=3.20.1', + 'protobuf>=3.21.0', 'lz4', 'requests', 'matplotlib', @@ -52,7 +52,7 @@ 'uvicorn', ], 'full': [ - 'protobuf>=3.13.0,<=3.20.1', + 'protobuf>=3.21.0', 'lz4', 'requests', 'matplotlib', From 9698e75f784804c20d181e12f2bb07a2a0a93f60 Mon Sep 17 00:00:00 2001 From: Han Xiao Date: Thu, 26 May 2022 17:44:37 +0200 Subject: [PATCH 03/20] fix(protobuf): set python parsing for tf and paddle --- tests/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index f358cb479bc..786939e6efb 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,7 @@ import numpy as np +import os + +os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python' from docarray import DocumentArray, Document From 72f4a0547a4878a5e1b8013f7ec8914992bc3429 Mon Sep 17 00:00:00 2001 From: Joan Fontanals Martinez Date: Tue, 5 Jul 2022 10:58:14 +0200 Subject: [PATCH 04/20] test: do not use special flag --- tests/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 786939e6efb..54fcb5da974 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,8 +1,4 @@ import numpy as np -import os - -os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python' - from docarray import DocumentArray, Document From 3496cf43ce806081db2662230b0b7967b54d6e03 Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Tue, 23 Aug 2022 12:33:25 +0100 Subject: [PATCH 05/20] feat: flexible build proto script --- docarray/proto/build-proto.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docarray/proto/build-proto.sh b/docarray/proto/build-proto.sh index cfd7552bf3a..b8aefe15287 100755 --- a/docarray/proto/build-proto.sh +++ b/docarray/proto/build-proto.sh @@ -2,16 +2,24 @@ set -ex # Do NOT use this directly, use jinaai/protogen image +# use jinaai/protogen:3.21 in order to use compiler version == 21 (creates docarray_pb.py) +# and use jinaai/protogen:latest to use compiler version <= 20 (creates docarray_pb2.py) +# make sure to use jinaai/protogen:3.21 to avoid overriting the module # -# current dir: docarray root (the one with README.md) +# current dir: docarray/docarray # run the following in bash: # docker run -v $(pwd)/proto:/jina/proto jinaai/protogen SRC_DIR=./ -SRC_NAME=docarray.proto +MODULE=docarray +SRC_NAME="${MODULE}.proto" +COMP_OUT_NAME="${MODULE}_pb2.py" + +OUT_FOLDER=${2:-pb2} + VER_FILE=../__init__.py -if [ "$#" -ne 1 ]; then +if [ "$#" -ne 1 ] && [ "$#" -ne 2 ]; then echo "Error: Please specify the [PATH_TO_GRPC_PYTHON_PLUGIN], refer more details at " \ "https://docarray.jina.ai/" printf "\n" @@ -26,3 +34,4 @@ PLUGIN_PATH=${1} # /Volumes/TOSHIBA-4T/Documents/grpc/bins/opt/grpc_python_plug printf "\e[1;33mgenerating protobuf and grpc python interface\e[0m\n" protoc -I ${SRC_DIR} --python_out=${SRC_DIR} ${SRC_DIR}${SRC_NAME} +mv ${COMP_OUT_NAME} "${OUT_FOLDER}/${COMP_OUT_NAME}" From 4de9687a671c38e3ebea4f66ba58cf8c668dc1db Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Tue, 23 Aug 2022 12:42:41 +0100 Subject: [PATCH 06/20] feat: flexible import of proto modules --- docarray/proto/__init__.py | 8 + docarray/proto/docarray_pb2.py | 612 ----------------------------- docarray/proto/pb/__init__.py | 0 docarray/proto/pb/docarray_pb2.py | 47 +++ docarray/proto/pb2/__init__.py | 0 docarray/proto/pb2/docarray_pb2.py | 143 +++++++ 6 files changed, 198 insertions(+), 612 deletions(-) delete mode 100644 docarray/proto/docarray_pb2.py create mode 100644 docarray/proto/pb/__init__.py create mode 100644 docarray/proto/pb/docarray_pb2.py create mode 100644 docarray/proto/pb2/__init__.py create mode 100644 docarray/proto/pb2/docarray_pb2.py diff --git a/docarray/proto/__init__.py b/docarray/proto/__init__.py index e69de29bb2d..48c7d755c3b 100644 --- a/docarray/proto/__init__.py +++ b/docarray/proto/__init__.py @@ -0,0 +1,8 @@ +from google.protobuf import __version__ as __pb__version__ + +if __pb__version__.startswith('4'): + print('importing pb4') + from .pb.docarray_pb2 import DocumentProto, DocumentArrayProto, NdArrayProto +else: + print('importing pb3') + from .pb2.docarray_pb2 import DocumentProto, DocumentArrayProto, NdArrayProto diff --git a/docarray/proto/docarray_pb2.py b/docarray/proto/docarray_pb2.py deleted file mode 100644 index 2beb100c43e..00000000000 --- a/docarray/proto/docarray_pb2.py +++ /dev/null @@ -1,612 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: docarray.proto -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='docarray.proto', - package='docarray', - syntax='proto3', - serialized_options=None, - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\x0e\x64ocarray.proto\x12\x08\x64ocarray\x1a\x1cgoogle/protobuf/struct.proto\"A\n\x11\x44\x65nseNdArrayProto\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\x0c\x12\r\n\x05shape\x18\x02 \x03(\r\x12\r\n\x05\x64type\x18\x03 \x01(\t\"\xb6\x01\n\x0cNdArrayProto\x12,\n\x05\x64\x65nse\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProtoH\x00\x12.\n\x06sparse\x18\x02 \x01(\x0b\x32\x1c.docarray.SparseNdArrayProtoH\x00\x12\x10\n\x08\x63ls_name\x18\x03 \x01(\t\x12+\n\nparameters\x18\x04 \x01(\x0b\x32\x17.google.protobuf.StructB\t\n\x07\x63ontent\"~\n\x12SparseNdArrayProto\x12,\n\x07indices\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12+\n\x06values\x18\x02 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12\r\n\x05shape\x18\x03 \x03(\r\"V\n\x0fNamedScoreProto\x12\r\n\x05value\x18\x01 \x01(\x02\x12\x0f\n\x07op_name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0e\n\x06ref_id\x18\x04 \x01(\t\"\xed\x05\n\rDocumentProto\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0e\n\x04\x62lob\x18\x02 \x01(\x0cH\x00\x12(\n\x06tensor\x18\x03 \x01(\x0b\x32\x16.docarray.NdArrayProtoH\x00\x12\x0e\n\x04text\x18\x04 \x01(\tH\x00\x12\x13\n\x0bgranularity\x18\x05 \x01(\r\x12\x11\n\tadjacency\x18\x06 \x01(\r\x12\x11\n\tparent_id\x18\x07 \x01(\t\x12\x0e\n\x06weight\x18\x08 \x01(\x02\x12\x0b\n\x03uri\x18\t \x01(\t\x12\x10\n\x08modality\x18\n \x01(\t\x12\x11\n\tmime_type\x18\x0b \x01(\t\x12\x0e\n\x06offset\x18\x0c \x01(\x02\x12\x10\n\x08location\x18\r \x03(\x02\x12\'\n\x06\x63hunks\x18\x0e \x03(\x0b\x32\x17.docarray.DocumentProto\x12(\n\x07matches\x18\x0f \x03(\x0b\x32\x17.docarray.DocumentProto\x12)\n\tembedding\x18\x10 \x01(\x0b\x32\x16.docarray.NdArrayProto\x12%\n\x04tags\x18\x11 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x33\n\x06scores\x18\x12 \x03(\x0b\x32#.docarray.DocumentProto.ScoresEntry\x12=\n\x0b\x65valuations\x18\x13 \x03(\x0b\x32(.docarray.DocumentProto.EvaluationsEntry\x12*\n\t_metadata\x18\x14 \x01(\x0b\x32\x17.google.protobuf.Struct\x1aH\n\x0bScoresEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.docarray.NamedScoreProto:\x02\x38\x01\x1aM\n\x10\x45valuationsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.docarray.NamedScoreProto:\x02\x38\x01\x42\t\n\x07\x63ontent\";\n\x12\x44ocumentArrayProto\x12%\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x17.docarray.DocumentProtob\x06proto3' - , - dependencies=[google_dot_protobuf_dot_struct__pb2.DESCRIPTOR,]) - - - - -_DENSENDARRAYPROTO = _descriptor.Descriptor( - name='DenseNdArrayProto', - full_name='docarray.DenseNdArrayProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='buffer', full_name='docarray.DenseNdArrayProto.buffer', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='shape', full_name='docarray.DenseNdArrayProto.shape', index=1, - number=2, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='dtype', full_name='docarray.DenseNdArrayProto.dtype', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=58, - serialized_end=123, -) - - -_NDARRAYPROTO = _descriptor.Descriptor( - name='NdArrayProto', - full_name='docarray.NdArrayProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='dense', full_name='docarray.NdArrayProto.dense', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='sparse', full_name='docarray.NdArrayProto.sparse', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='cls_name', full_name='docarray.NdArrayProto.cls_name', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='parameters', full_name='docarray.NdArrayProto.parameters', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='content', full_name='docarray.NdArrayProto.content', - index=0, containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[]), - ], - serialized_start=126, - serialized_end=308, -) - - -_SPARSENDARRAYPROTO = _descriptor.Descriptor( - name='SparseNdArrayProto', - full_name='docarray.SparseNdArrayProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='indices', full_name='docarray.SparseNdArrayProto.indices', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='values', full_name='docarray.SparseNdArrayProto.values', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='shape', full_name='docarray.SparseNdArrayProto.shape', index=2, - number=3, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=310, - serialized_end=436, -) - - -_NAMEDSCOREPROTO = _descriptor.Descriptor( - name='NamedScoreProto', - full_name='docarray.NamedScoreProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='value', full_name='docarray.NamedScoreProto.value', index=0, - number=1, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='op_name', full_name='docarray.NamedScoreProto.op_name', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='description', full_name='docarray.NamedScoreProto.description', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='ref_id', full_name='docarray.NamedScoreProto.ref_id', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=438, - serialized_end=524, -) - - -_DOCUMENTPROTO_SCORESENTRY = _descriptor.Descriptor( - name='ScoresEntry', - full_name='docarray.DocumentProto.ScoresEntry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='docarray.DocumentProto.ScoresEntry.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='value', full_name='docarray.DocumentProto.ScoresEntry.value', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=b'8\001', - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1114, - serialized_end=1186, -) - -_DOCUMENTPROTO_EVALUATIONSENTRY = _descriptor.Descriptor( - name='EvaluationsEntry', - full_name='docarray.DocumentProto.EvaluationsEntry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='docarray.DocumentProto.EvaluationsEntry.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='value', full_name='docarray.DocumentProto.EvaluationsEntry.value', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=b'8\001', - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1188, - serialized_end=1265, -) - -_DOCUMENTPROTO = _descriptor.Descriptor( - name='DocumentProto', - full_name='docarray.DocumentProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='id', full_name='docarray.DocumentProto.id', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='blob', full_name='docarray.DocumentProto.blob', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='tensor', full_name='docarray.DocumentProto.tensor', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='text', full_name='docarray.DocumentProto.text', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='granularity', full_name='docarray.DocumentProto.granularity', index=4, - number=5, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='adjacency', full_name='docarray.DocumentProto.adjacency', index=5, - number=6, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='parent_id', full_name='docarray.DocumentProto.parent_id', index=6, - number=7, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='weight', full_name='docarray.DocumentProto.weight', index=7, - number=8, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='uri', full_name='docarray.DocumentProto.uri', index=8, - number=9, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='modality', full_name='docarray.DocumentProto.modality', index=9, - number=10, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='mime_type', full_name='docarray.DocumentProto.mime_type', index=10, - number=11, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='offset', full_name='docarray.DocumentProto.offset', index=11, - number=12, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='location', full_name='docarray.DocumentProto.location', index=12, - number=13, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='chunks', full_name='docarray.DocumentProto.chunks', index=13, - number=14, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='matches', full_name='docarray.DocumentProto.matches', index=14, - number=15, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='embedding', full_name='docarray.DocumentProto.embedding', index=15, - number=16, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='tags', full_name='docarray.DocumentProto.tags', index=16, - number=17, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='scores', full_name='docarray.DocumentProto.scores', index=17, - number=18, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='evaluations', full_name='docarray.DocumentProto.evaluations', index=18, - number=19, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='_metadata', full_name='docarray.DocumentProto._metadata', index=19, - number=20, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[_DOCUMENTPROTO_SCORESENTRY, _DOCUMENTPROTO_EVALUATIONSENTRY, ], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='content', full_name='docarray.DocumentProto.content', - index=0, containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[]), - ], - serialized_start=527, - serialized_end=1276, -) - - -_DOCUMENTARRAYPROTO = _descriptor.Descriptor( - name='DocumentArrayProto', - full_name='docarray.DocumentArrayProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='docs', full_name='docarray.DocumentArrayProto.docs', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1278, - serialized_end=1337, -) - -_NDARRAYPROTO.fields_by_name['dense'].message_type = _DENSENDARRAYPROTO -_NDARRAYPROTO.fields_by_name['sparse'].message_type = _SPARSENDARRAYPROTO -_NDARRAYPROTO.fields_by_name['parameters'].message_type = google_dot_protobuf_dot_struct__pb2._STRUCT -_NDARRAYPROTO.oneofs_by_name['content'].fields.append( - _NDARRAYPROTO.fields_by_name['dense']) -_NDARRAYPROTO.fields_by_name['dense'].containing_oneof = _NDARRAYPROTO.oneofs_by_name['content'] -_NDARRAYPROTO.oneofs_by_name['content'].fields.append( - _NDARRAYPROTO.fields_by_name['sparse']) -_NDARRAYPROTO.fields_by_name['sparse'].containing_oneof = _NDARRAYPROTO.oneofs_by_name['content'] -_SPARSENDARRAYPROTO.fields_by_name['indices'].message_type = _DENSENDARRAYPROTO -_SPARSENDARRAYPROTO.fields_by_name['values'].message_type = _DENSENDARRAYPROTO -_DOCUMENTPROTO_SCORESENTRY.fields_by_name['value'].message_type = _NAMEDSCOREPROTO -_DOCUMENTPROTO_SCORESENTRY.containing_type = _DOCUMENTPROTO -_DOCUMENTPROTO_EVALUATIONSENTRY.fields_by_name['value'].message_type = _NAMEDSCOREPROTO -_DOCUMENTPROTO_EVALUATIONSENTRY.containing_type = _DOCUMENTPROTO -_DOCUMENTPROTO.fields_by_name['tensor'].message_type = _NDARRAYPROTO -_DOCUMENTPROTO.fields_by_name['chunks'].message_type = _DOCUMENTPROTO -_DOCUMENTPROTO.fields_by_name['matches'].message_type = _DOCUMENTPROTO -_DOCUMENTPROTO.fields_by_name['embedding'].message_type = _NDARRAYPROTO -_DOCUMENTPROTO.fields_by_name['tags'].message_type = google_dot_protobuf_dot_struct__pb2._STRUCT -_DOCUMENTPROTO.fields_by_name['scores'].message_type = _DOCUMENTPROTO_SCORESENTRY -_DOCUMENTPROTO.fields_by_name['evaluations'].message_type = _DOCUMENTPROTO_EVALUATIONSENTRY -_DOCUMENTPROTO.fields_by_name['_metadata'].message_type = google_dot_protobuf_dot_struct__pb2._STRUCT -_DOCUMENTPROTO.oneofs_by_name['content'].fields.append( - _DOCUMENTPROTO.fields_by_name['blob']) -_DOCUMENTPROTO.fields_by_name['blob'].containing_oneof = _DOCUMENTPROTO.oneofs_by_name['content'] -_DOCUMENTPROTO.oneofs_by_name['content'].fields.append( - _DOCUMENTPROTO.fields_by_name['tensor']) -_DOCUMENTPROTO.fields_by_name['tensor'].containing_oneof = _DOCUMENTPROTO.oneofs_by_name['content'] -_DOCUMENTPROTO.oneofs_by_name['content'].fields.append( - _DOCUMENTPROTO.fields_by_name['text']) -_DOCUMENTPROTO.fields_by_name['text'].containing_oneof = _DOCUMENTPROTO.oneofs_by_name['content'] -_DOCUMENTARRAYPROTO.fields_by_name['docs'].message_type = _DOCUMENTPROTO -DESCRIPTOR.message_types_by_name['DenseNdArrayProto'] = _DENSENDARRAYPROTO -DESCRIPTOR.message_types_by_name['NdArrayProto'] = _NDARRAYPROTO -DESCRIPTOR.message_types_by_name['SparseNdArrayProto'] = _SPARSENDARRAYPROTO -DESCRIPTOR.message_types_by_name['NamedScoreProto'] = _NAMEDSCOREPROTO -DESCRIPTOR.message_types_by_name['DocumentProto'] = _DOCUMENTPROTO -DESCRIPTOR.message_types_by_name['DocumentArrayProto'] = _DOCUMENTARRAYPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -DenseNdArrayProto = _reflection.GeneratedProtocolMessageType('DenseNdArrayProto', (_message.Message,), { - 'DESCRIPTOR' : _DENSENDARRAYPROTO, - '__module__' : 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.DenseNdArrayProto) - }) -_sym_db.RegisterMessage(DenseNdArrayProto) - -NdArrayProto = _reflection.GeneratedProtocolMessageType('NdArrayProto', (_message.Message,), { - 'DESCRIPTOR' : _NDARRAYPROTO, - '__module__' : 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.NdArrayProto) - }) -_sym_db.RegisterMessage(NdArrayProto) - -SparseNdArrayProto = _reflection.GeneratedProtocolMessageType('SparseNdArrayProto', (_message.Message,), { - 'DESCRIPTOR' : _SPARSENDARRAYPROTO, - '__module__' : 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.SparseNdArrayProto) - }) -_sym_db.RegisterMessage(SparseNdArrayProto) - -NamedScoreProto = _reflection.GeneratedProtocolMessageType('NamedScoreProto', (_message.Message,), { - 'DESCRIPTOR' : _NAMEDSCOREPROTO, - '__module__' : 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.NamedScoreProto) - }) -_sym_db.RegisterMessage(NamedScoreProto) - -DocumentProto = _reflection.GeneratedProtocolMessageType('DocumentProto', (_message.Message,), { - - 'ScoresEntry' : _reflection.GeneratedProtocolMessageType('ScoresEntry', (_message.Message,), { - 'DESCRIPTOR' : _DOCUMENTPROTO_SCORESENTRY, - '__module__' : 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.DocumentProto.ScoresEntry) - }) - , - - 'EvaluationsEntry' : _reflection.GeneratedProtocolMessageType('EvaluationsEntry', (_message.Message,), { - 'DESCRIPTOR' : _DOCUMENTPROTO_EVALUATIONSENTRY, - '__module__' : 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.DocumentProto.EvaluationsEntry) - }) - , - 'DESCRIPTOR' : _DOCUMENTPROTO, - '__module__' : 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.DocumentProto) - }) -_sym_db.RegisterMessage(DocumentProto) -_sym_db.RegisterMessage(DocumentProto.ScoresEntry) -_sym_db.RegisterMessage(DocumentProto.EvaluationsEntry) - -DocumentArrayProto = _reflection.GeneratedProtocolMessageType('DocumentArrayProto', (_message.Message,), { - 'DESCRIPTOR' : _DOCUMENTARRAYPROTO, - '__module__' : 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.DocumentArrayProto) - }) -_sym_db.RegisterMessage(DocumentArrayProto) - - -_DOCUMENTPROTO_SCORESENTRY._options = None -_DOCUMENTPROTO_EVALUATIONSENTRY._options = None -# @@protoc_insertion_point(module_scope) diff --git a/docarray/proto/pb/__init__.py b/docarray/proto/pb/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docarray/proto/pb/docarray_pb2.py b/docarray/proto/pb/docarray_pb2.py new file mode 100644 index 00000000000..ee7260dd27e --- /dev/null +++ b/docarray/proto/pb/docarray_pb2.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: docarray.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x0e\x64ocarray.proto\x12\x08\x64ocarray\x1a\x1cgoogle/protobuf/struct.proto\"A\n\x11\x44\x65nseNdArrayProto\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\x0c\x12\r\n\x05shape\x18\x02 \x03(\r\x12\r\n\x05\x64type\x18\x03 \x01(\t\"\xb6\x01\n\x0cNdArrayProto\x12,\n\x05\x64\x65nse\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProtoH\x00\x12.\n\x06sparse\x18\x02 \x01(\x0b\x32\x1c.docarray.SparseNdArrayProtoH\x00\x12\x10\n\x08\x63ls_name\x18\x03 \x01(\t\x12+\n\nparameters\x18\x04 \x01(\x0b\x32\x17.google.protobuf.StructB\t\n\x07\x63ontent\"~\n\x12SparseNdArrayProto\x12,\n\x07indices\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12+\n\x06values\x18\x02 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12\r\n\x05shape\x18\x03 \x03(\r\"V\n\x0fNamedScoreProto\x12\r\n\x05value\x18\x01 \x01(\x02\x12\x0f\n\x07op_name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0e\n\x06ref_id\x18\x04 \x01(\t\"\xed\x05\n\rDocumentProto\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0e\n\x04\x62lob\x18\x02 \x01(\x0cH\x00\x12(\n\x06tensor\x18\x03 \x01(\x0b\x32\x16.docarray.NdArrayProtoH\x00\x12\x0e\n\x04text\x18\x04 \x01(\tH\x00\x12\x13\n\x0bgranularity\x18\x05 \x01(\r\x12\x11\n\tadjacency\x18\x06 \x01(\r\x12\x11\n\tparent_id\x18\x07 \x01(\t\x12\x0e\n\x06weight\x18\x08 \x01(\x02\x12\x0b\n\x03uri\x18\t \x01(\t\x12\x10\n\x08modality\x18\n \x01(\t\x12\x11\n\tmime_type\x18\x0b \x01(\t\x12\x0e\n\x06offset\x18\x0c \x01(\x02\x12\x10\n\x08location\x18\r \x03(\x02\x12\'\n\x06\x63hunks\x18\x0e \x03(\x0b\x32\x17.docarray.DocumentProto\x12(\n\x07matches\x18\x0f \x03(\x0b\x32\x17.docarray.DocumentProto\x12)\n\tembedding\x18\x10 \x01(\x0b\x32\x16.docarray.NdArrayProto\x12%\n\x04tags\x18\x11 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x33\n\x06scores\x18\x12 \x03(\x0b\x32#.docarray.DocumentProto.ScoresEntry\x12=\n\x0b\x65valuations\x18\x13 \x03(\x0b\x32(.docarray.DocumentProto.EvaluationsEntry\x12*\n\t_metadata\x18\x14 \x01(\x0b\x32\x17.google.protobuf.Struct\x1aH\n\x0bScoresEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.docarray.NamedScoreProto:\x02\x38\x01\x1aM\n\x10\x45valuationsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.docarray.NamedScoreProto:\x02\x38\x01\x42\t\n\x07\x63ontent\";\n\x12\x44ocumentArrayProto\x12%\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x17.docarray.DocumentProtob\x06proto3' +) + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'docarray_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _DOCUMENTPROTO_SCORESENTRY._options = None + _DOCUMENTPROTO_SCORESENTRY._serialized_options = b'8\001' + _DOCUMENTPROTO_EVALUATIONSENTRY._options = None + _DOCUMENTPROTO_EVALUATIONSENTRY._serialized_options = b'8\001' + _DENSENDARRAYPROTO._serialized_start = 58 + _DENSENDARRAYPROTO._serialized_end = 123 + _NDARRAYPROTO._serialized_start = 126 + _NDARRAYPROTO._serialized_end = 308 + _SPARSENDARRAYPROTO._serialized_start = 310 + _SPARSENDARRAYPROTO._serialized_end = 436 + _NAMEDSCOREPROTO._serialized_start = 438 + _NAMEDSCOREPROTO._serialized_end = 524 + _DOCUMENTPROTO._serialized_start = 527 + _DOCUMENTPROTO._serialized_end = 1276 + _DOCUMENTPROTO_SCORESENTRY._serialized_start = 1114 + _DOCUMENTPROTO_SCORESENTRY._serialized_end = 1186 + _DOCUMENTPROTO_EVALUATIONSENTRY._serialized_start = 1188 + _DOCUMENTPROTO_EVALUATIONSENTRY._serialized_end = 1265 + _DOCUMENTARRAYPROTO._serialized_start = 1278 + _DOCUMENTARRAYPROTO._serialized_end = 1337 +# @@protoc_insertion_point(module_scope) diff --git a/docarray/proto/pb2/__init__.py b/docarray/proto/pb2/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docarray/proto/pb2/docarray_pb2.py b/docarray/proto/pb2/docarray_pb2.py new file mode 100644 index 00000000000..16e4a387abc --- /dev/null +++ b/docarray/proto/pb2/docarray_pb2.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: docarray.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x0e\x64ocarray.proto\x12\x08\x64ocarray\x1a\x1cgoogle/protobuf/struct.proto\"A\n\x11\x44\x65nseNdArrayProto\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\x0c\x12\r\n\x05shape\x18\x02 \x03(\r\x12\r\n\x05\x64type\x18\x03 \x01(\t\"\xb6\x01\n\x0cNdArrayProto\x12,\n\x05\x64\x65nse\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProtoH\x00\x12.\n\x06sparse\x18\x02 \x01(\x0b\x32\x1c.docarray.SparseNdArrayProtoH\x00\x12\x10\n\x08\x63ls_name\x18\x03 \x01(\t\x12+\n\nparameters\x18\x04 \x01(\x0b\x32\x17.google.protobuf.StructB\t\n\x07\x63ontent\"~\n\x12SparseNdArrayProto\x12,\n\x07indices\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12+\n\x06values\x18\x02 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12\r\n\x05shape\x18\x03 \x03(\r\"V\n\x0fNamedScoreProto\x12\r\n\x05value\x18\x01 \x01(\x02\x12\x0f\n\x07op_name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x0e\n\x06ref_id\x18\x04 \x01(\t\"\xed\x05\n\rDocumentProto\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0e\n\x04\x62lob\x18\x02 \x01(\x0cH\x00\x12(\n\x06tensor\x18\x03 \x01(\x0b\x32\x16.docarray.NdArrayProtoH\x00\x12\x0e\n\x04text\x18\x04 \x01(\tH\x00\x12\x13\n\x0bgranularity\x18\x05 \x01(\r\x12\x11\n\tadjacency\x18\x06 \x01(\r\x12\x11\n\tparent_id\x18\x07 \x01(\t\x12\x0e\n\x06weight\x18\x08 \x01(\x02\x12\x0b\n\x03uri\x18\t \x01(\t\x12\x10\n\x08modality\x18\n \x01(\t\x12\x11\n\tmime_type\x18\x0b \x01(\t\x12\x0e\n\x06offset\x18\x0c \x01(\x02\x12\x10\n\x08location\x18\r \x03(\x02\x12\'\n\x06\x63hunks\x18\x0e \x03(\x0b\x32\x17.docarray.DocumentProto\x12(\n\x07matches\x18\x0f \x03(\x0b\x32\x17.docarray.DocumentProto\x12)\n\tembedding\x18\x10 \x01(\x0b\x32\x16.docarray.NdArrayProto\x12%\n\x04tags\x18\x11 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x33\n\x06scores\x18\x12 \x03(\x0b\x32#.docarray.DocumentProto.ScoresEntry\x12=\n\x0b\x65valuations\x18\x13 \x03(\x0b\x32(.docarray.DocumentProto.EvaluationsEntry\x12*\n\t_metadata\x18\x14 \x01(\x0b\x32\x17.google.protobuf.Struct\x1aH\n\x0bScoresEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.docarray.NamedScoreProto:\x02\x38\x01\x1aM\n\x10\x45valuationsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.docarray.NamedScoreProto:\x02\x38\x01\x42\t\n\x07\x63ontent\";\n\x12\x44ocumentArrayProto\x12%\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x17.docarray.DocumentProtob\x06proto3' +) + + +_DENSENDARRAYPROTO = DESCRIPTOR.message_types_by_name['DenseNdArrayProto'] +_NDARRAYPROTO = DESCRIPTOR.message_types_by_name['NdArrayProto'] +_SPARSENDARRAYPROTO = DESCRIPTOR.message_types_by_name['SparseNdArrayProto'] +_NAMEDSCOREPROTO = DESCRIPTOR.message_types_by_name['NamedScoreProto'] +_DOCUMENTPROTO = DESCRIPTOR.message_types_by_name['DocumentProto'] +_DOCUMENTPROTO_SCORESENTRY = _DOCUMENTPROTO.nested_types_by_name['ScoresEntry'] +_DOCUMENTPROTO_EVALUATIONSENTRY = _DOCUMENTPROTO.nested_types_by_name[ + 'EvaluationsEntry' +] +_DOCUMENTARRAYPROTO = DESCRIPTOR.message_types_by_name['DocumentArrayProto'] +DenseNdArrayProto = _reflection.GeneratedProtocolMessageType( + 'DenseNdArrayProto', + (_message.Message,), + { + 'DESCRIPTOR': _DENSENDARRAYPROTO, + '__module__': 'docarray_pb2' + # @@protoc_insertion_point(class_scope:docarray.DenseNdArrayProto) + }, +) +_sym_db.RegisterMessage(DenseNdArrayProto) + +NdArrayProto = _reflection.GeneratedProtocolMessageType( + 'NdArrayProto', + (_message.Message,), + { + 'DESCRIPTOR': _NDARRAYPROTO, + '__module__': 'docarray_pb2' + # @@protoc_insertion_point(class_scope:docarray.NdArrayProto) + }, +) +_sym_db.RegisterMessage(NdArrayProto) + +SparseNdArrayProto = _reflection.GeneratedProtocolMessageType( + 'SparseNdArrayProto', + (_message.Message,), + { + 'DESCRIPTOR': _SPARSENDARRAYPROTO, + '__module__': 'docarray_pb2' + # @@protoc_insertion_point(class_scope:docarray.SparseNdArrayProto) + }, +) +_sym_db.RegisterMessage(SparseNdArrayProto) + +NamedScoreProto = _reflection.GeneratedProtocolMessageType( + 'NamedScoreProto', + (_message.Message,), + { + 'DESCRIPTOR': _NAMEDSCOREPROTO, + '__module__': 'docarray_pb2' + # @@protoc_insertion_point(class_scope:docarray.NamedScoreProto) + }, +) +_sym_db.RegisterMessage(NamedScoreProto) + +DocumentProto = _reflection.GeneratedProtocolMessageType( + 'DocumentProto', + (_message.Message,), + { + 'ScoresEntry': _reflection.GeneratedProtocolMessageType( + 'ScoresEntry', + (_message.Message,), + { + 'DESCRIPTOR': _DOCUMENTPROTO_SCORESENTRY, + '__module__': 'docarray_pb2' + # @@protoc_insertion_point(class_scope:docarray.DocumentProto.ScoresEntry) + }, + ), + 'EvaluationsEntry': _reflection.GeneratedProtocolMessageType( + 'EvaluationsEntry', + (_message.Message,), + { + 'DESCRIPTOR': _DOCUMENTPROTO_EVALUATIONSENTRY, + '__module__': 'docarray_pb2' + # @@protoc_insertion_point(class_scope:docarray.DocumentProto.EvaluationsEntry) + }, + ), + 'DESCRIPTOR': _DOCUMENTPROTO, + '__module__': 'docarray_pb2' + # @@protoc_insertion_point(class_scope:docarray.DocumentProto) + }, +) +_sym_db.RegisterMessage(DocumentProto) +_sym_db.RegisterMessage(DocumentProto.ScoresEntry) +_sym_db.RegisterMessage(DocumentProto.EvaluationsEntry) + +DocumentArrayProto = _reflection.GeneratedProtocolMessageType( + 'DocumentArrayProto', + (_message.Message,), + { + 'DESCRIPTOR': _DOCUMENTARRAYPROTO, + '__module__': 'docarray_pb2' + # @@protoc_insertion_point(class_scope:docarray.DocumentArrayProto) + }, +) +_sym_db.RegisterMessage(DocumentArrayProto) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _DOCUMENTPROTO_SCORESENTRY._options = None + _DOCUMENTPROTO_SCORESENTRY._serialized_options = b'8\001' + _DOCUMENTPROTO_EVALUATIONSENTRY._options = None + _DOCUMENTPROTO_EVALUATIONSENTRY._serialized_options = b'8\001' + _DENSENDARRAYPROTO._serialized_start = 58 + _DENSENDARRAYPROTO._serialized_end = 123 + _NDARRAYPROTO._serialized_start = 126 + _NDARRAYPROTO._serialized_end = 308 + _SPARSENDARRAYPROTO._serialized_start = 310 + _SPARSENDARRAYPROTO._serialized_end = 436 + _NAMEDSCOREPROTO._serialized_start = 438 + _NAMEDSCOREPROTO._serialized_end = 524 + _DOCUMENTPROTO._serialized_start = 527 + _DOCUMENTPROTO._serialized_end = 1276 + _DOCUMENTPROTO_SCORESENTRY._serialized_start = 1114 + _DOCUMENTPROTO_SCORESENTRY._serialized_end = 1186 + _DOCUMENTPROTO_EVALUATIONSENTRY._serialized_start = 1188 + _DOCUMENTPROTO_EVALUATIONSENTRY._serialized_end = 1265 + _DOCUMENTARRAYPROTO._serialized_start = 1278 + _DOCUMENTARRAYPROTO._serialized_end = 1337 +# @@protoc_insertion_point(module_scope) From d8a17caed2f1cff865243767c5685126ba22e754 Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Tue, 23 Aug 2022 12:43:08 +0100 Subject: [PATCH 07/20] chore: update proto compiling instructions --- docarray/proto/build-proto.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docarray/proto/build-proto.sh b/docarray/proto/build-proto.sh index b8aefe15287..66482a70974 100755 --- a/docarray/proto/build-proto.sh +++ b/docarray/proto/build-proto.sh @@ -2,13 +2,14 @@ set -ex # Do NOT use this directly, use jinaai/protogen image -# use jinaai/protogen:3.21 in order to use compiler version == 21 (creates docarray_pb.py) -# and use jinaai/protogen:latest to use compiler version <= 20 (creates docarray_pb2.py) +# use jinaai/protogen:3.21 in order to use compiler version == 21 (creates pb/docarray_pb2.py) +# and use jinaai/protogen:latest to use compiler version <= 20 (creates pb2/docarray_pb2.py) # make sure to use jinaai/protogen:3.21 to avoid overriting the module # # current dir: docarray/docarray # run the following in bash: # docker run -v $(pwd)/proto:/jina/proto jinaai/protogen +# finally, set back owner of the generated files using: sudo chown -R $(id -u ${USER}):$(id -g ${USER}) . SRC_DIR=./ MODULE=docarray From 420acb5642343cd55209350b9ddc8f5a8bbdfa3d Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Tue, 23 Aug 2022 12:48:41 +0100 Subject: [PATCH 08/20] chore: update usages of proto --- docarray/array/mixins/io/binary.py | 6 +++--- docarray/proto/io/__init__.py | 2 +- docarray/proto/io/ndarray.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docarray/array/mixins/io/binary.py b/docarray/array/mixins/io/binary.py index 7f4118898cf..1364532718f 100644 --- a/docarray/array/mixins/io/binary.py +++ b/docarray/array/mixins/io/binary.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from docarray.typing import T - from docarray.proto.docarray_pb2 import DocumentArrayProto + from docarray.proto import DocumentArrayProto from docarray import Document, DocumentArray @@ -161,7 +161,7 @@ def _load_binary_all( compress = None if protocol == 'protobuf-array': - from docarray.proto.docarray_pb2 import DocumentArrayProto + from docarray.proto import DocumentArrayProto dap = DocumentArrayProto() dap.ParseFromString(d) @@ -347,7 +347,7 @@ def to_protobuf(self, ndarray_type: Optional[str] = None) -> 'DocumentArrayProto Documents to ``List`` or ``numpy.ndarray``. :return: the protobuf message """ - from docarray.proto.docarray_pb2 import DocumentArrayProto + from docarray.proto import DocumentArrayProto dap = DocumentArrayProto() for d in self: diff --git a/docarray/proto/io/__init__.py b/docarray/proto/io/__init__.py index 3dd2802cae3..bb443d77763 100644 --- a/docarray/proto/io/__init__.py +++ b/docarray/proto/io/__init__.py @@ -5,7 +5,7 @@ from google.protobuf.struct_pb2 import Struct from docarray.proto.io.ndarray import flush_ndarray, read_ndarray -from docarray.proto.docarray_pb2 import NdArrayProto, DocumentProto +from docarray.proto import NdArrayProto, DocumentProto if TYPE_CHECKING: from docarray import Document diff --git a/docarray/proto/io/ndarray.py b/docarray/proto/io/ndarray.py index 627dc0cc48a..7d1d3a2f714 100644 --- a/docarray/proto/io/ndarray.py +++ b/docarray/proto/io/ndarray.py @@ -6,7 +6,7 @@ if TYPE_CHECKING: from docarray.typing import ArrayType - from docarray.proto.docarray_pb2 import NdArrayProto + from docarray.proto import NdArrayProto def read_ndarray(pb_msg: 'NdArrayProto') -> 'ArrayType': From 386342cd2adda544022ebaf268c05c809795bf97 Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Tue, 23 Aug 2022 12:52:24 +0100 Subject: [PATCH 09/20] chore: remove print --- docarray/proto/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/docarray/proto/__init__.py b/docarray/proto/__init__.py index 48c7d755c3b..be38aa820b8 100644 --- a/docarray/proto/__init__.py +++ b/docarray/proto/__init__.py @@ -1,8 +1,6 @@ from google.protobuf import __version__ as __pb__version__ if __pb__version__.startswith('4'): - print('importing pb4') from .pb.docarray_pb2 import DocumentProto, DocumentArrayProto, NdArrayProto else: - print('importing pb3') from .pb2.docarray_pb2 import DocumentProto, DocumentArrayProto, NdArrayProto From 850d6e085b099a1ea5a0a06c06642b4e1f6f7b2c Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Tue, 23 Aug 2022 12:53:03 +0100 Subject: [PATCH 10/20] chore: free protobuf version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 21ddf977266..46713ca5c3e 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ extras_require={ # req usage, please see https://docarray.jina.ai/#install 'common': [ - 'protobuf>=3.13.0,<=3.20.1', + 'protobuf>=3.13.0', 'lz4', 'requests', 'matplotlib', From f36312eb3585726ba4a056a7dc269ba4b7699719 Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Tue, 23 Aug 2022 12:59:28 +0100 Subject: [PATCH 11/20] chore: update config files --- .github/workflows/ci.yml | 4 ++-- .pre-commit-config.yaml | 2 +- scripts/black.sh | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db3d1a861db..78f72461d1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,9 +60,9 @@ jobs: run: | pip install flake8 # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,docarray/proto/docarray_pb2.py + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,docarray/proto/pb/docarray_pb2.py,docarray/proto/pb2/docarray_pb2.py # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,docarray/proto/docarray_pb2.py + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,docarray/proto/pb/docarray_pb2.py,docarray/proto/pb2/docarray_pb2.py check-black: runs-on: ubuntu-latest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bc68ed3bb29..bf37fe88c71 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: black types: [python] - exclude: ^(docarray/proto/docarray_pb2.py|docs/|docarray/resources/) + exclude: ^(docarray/proto/pb/docarray_pb2.py|docarray/proto/pb2/docarray_pb2.py|docs/|docarray/resources/) args: - -S - repo: https://github.com/asottile/blacken-docs diff --git a/scripts/black.sh b/scripts/black.sh index 2e3c3662e77..b1221374c09 100755 --- a/scripts/black.sh +++ b/scripts/black.sh @@ -3,7 +3,8 @@ pip install black==22.3.0 arrVar=() echo we ignore non-*.py files and files generated from protobuf excluded_files=( - docarray/proto/docarray_pb2.py + docarray/proto/pb/docarray_pb2.py + docarray/proto/pb2/docarray_pb2.py docs/conf.py ) for changed_file in $CHANGED_FILES; do From 2bc9e75dbdeaae80a8ada05b6eec8ba32e02f6e3 Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Tue, 23 Aug 2022 15:12:29 +0100 Subject: [PATCH 12/20] ci: install test tag before full --- .github/workflows/cd.yml | 3 ++- .github/workflows/ci.yml | 3 ++- setup.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 8a07429c661..60ced0f000b 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -40,7 +40,8 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install wheel - pip install --no-cache-dir ".[full,test]" + pip install --no-cache-dir ".[test]" + pip install --no-cache-dir ".[full]" sudo apt-get install libsndfile1 - name: Test id: test diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78f72461d1e..c3bd91b7ef5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,7 +129,8 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install wheel - pip install --no-cache-dir ".[full,test]" + pip install --no-cache-dir ".[test]" + pip install --no-cache-dir ".[full]" sudo apt-get install libsndfile1 - name: Test id: test diff --git a/setup.py b/setup.py index c78e271bc62..1f2bab7d6bc 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ 'jina-hubble-sdk>=0.11.0', ], 'full': [ - 'protobuf>=3.21.0', + 'protobuf>=3.13.0', 'lz4', 'requests', 'matplotlib', From 1a9569f99297607b70df404e134864882d0298fe Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Tue, 23 Aug 2022 15:22:41 +0100 Subject: [PATCH 13/20] fix: change proto usage --- docarray/document/mixins/porting.py | 6 +++--- docarray/document/mixins/protobuf.py | 2 +- tests/unit/math/test_ndarray.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docarray/document/mixins/porting.py b/docarray/document/mixins/porting.py index fde058f3faf..39be173787f 100644 --- a/docarray/document/mixins/porting.py +++ b/docarray/document/mixins/porting.py @@ -27,7 +27,7 @@ def from_dict( return cls.from_pydantic_model(PydanticDocument.parse_obj(obj, **kwargs)) elif protocol == 'protobuf': from google.protobuf import json_format - from docarray.proto.docarray_pb2 import DocumentProto + from docarray.proto import DocumentProto pb_msg = DocumentProto() json_format.ParseDict(obj, pb_msg, **kwargs) @@ -55,7 +55,7 @@ def from_json( return cls.from_pydantic_model(PydanticDocument.parse_raw(obj, **kwargs)) elif protocol == 'protobuf': from google.protobuf import json_format - from docarray.proto.docarray_pb2 import DocumentProto + from docarray.proto import DocumentProto pb_msg = DocumentProto() json_format.Parse(obj, pb_msg, **kwargs) @@ -113,7 +113,7 @@ def from_bytes( if protocol == 'pickle': return pickle.loads(bstr) elif protocol == 'protobuf': - from docarray.proto.docarray_pb2 import DocumentProto + from docarray.proto import DocumentProto pb_msg = DocumentProto() pb_msg.ParseFromString(bstr) diff --git a/docarray/document/mixins/protobuf.py b/docarray/document/mixins/protobuf.py index faa454d2274..875c2f46021 100644 --- a/docarray/document/mixins/protobuf.py +++ b/docarray/document/mixins/protobuf.py @@ -2,7 +2,7 @@ if TYPE_CHECKING: from docarray.typing import T - from docarray.proto.docarray_pb2 import DocumentProto + from docarray.proto import DocumentProto class ProtobufMixin: diff --git a/tests/unit/math/test_ndarray.py b/tests/unit/math/test_ndarray.py index 562ae0f98f1..546af78ab1c 100644 --- a/tests/unit/math/test_ndarray.py +++ b/tests/unit/math/test_ndarray.py @@ -6,7 +6,7 @@ from scipy.sparse import csr_matrix, coo_matrix, bsr_matrix, csc_matrix, issparse from docarray.math.ndarray import get_array_rows, check_arraylike_equality -from docarray.proto.docarray_pb2 import NdArrayProto +from docarray.proto import NdArrayProto from docarray.proto.io import flush_ndarray, read_ndarray From 963b6b08505300915eb81eac905c4ddf0b3313a6 Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Tue, 23 Aug 2022 17:01:44 +0100 Subject: [PATCH 14/20] chore: expose all pb2 module for dependents --- docarray/proto/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docarray/proto/__init__.py b/docarray/proto/__init__.py index be38aa820b8..8ee0c2b6b32 100644 --- a/docarray/proto/__init__.py +++ b/docarray/proto/__init__.py @@ -2,5 +2,11 @@ if __pb__version__.startswith('4'): from .pb.docarray_pb2 import DocumentProto, DocumentArrayProto, NdArrayProto + + # compatibility with any dependent that imports docarray_pb2 (for example jina) + from .pb import docarray_pb2 else: from .pb2.docarray_pb2 import DocumentProto, DocumentArrayProto, NdArrayProto + + # compatibility with any dependent that imports docarray_pb2 (for example jina) + from .pb2 import docarray_pb2 From 4a376d79c3e9d93a66dee60aae283771e93c5ecb Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Wed, 24 Aug 2022 10:26:20 +0100 Subject: [PATCH 15/20] chore: add comment to explain --- .github/workflows/cd.yml | 1 + .github/workflows/ci.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 60ced0f000b..f91045b6c5c 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -40,6 +40,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install wheel + # pip does not properly resolve dependency versions with syntax pip install --no-cache-dir ".[test,full]" pip install --no-cache-dir ".[test]" pip install --no-cache-dir ".[full]" sudo apt-get install libsndfile1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3bd91b7ef5..d703c990683 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,6 +129,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install wheel + # pip does not properly resolve dependency versions with syntax pip install --no-cache-dir ".[test,full]" pip install --no-cache-dir ".[test]" pip install --no-cache-dir ".[full]" sudo apt-get install libsndfile1 From 1a1e65af2652616aba4b6e90e6bfb381514bc714 Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Wed, 24 Aug 2022 14:30:00 +0100 Subject: [PATCH 16/20] chore: increase version --- docarray/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docarray/__init__.py b/docarray/__init__.py index 8af94036068..2b56fb52726 100644 --- a/docarray/__init__.py +++ b/docarray/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.15.4' +__version__ = '0.16.0' import os From ac66a5d7840d3942fab6ceefc21f9b1c9c2f0754 Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Wed, 24 Aug 2022 14:30:14 +0100 Subject: [PATCH 17/20] ci: install modified jina in CI --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d703c990683..4d174ca6f3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,6 +129,8 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install wheel + # test install jina with new proto version + pip install git+https://github.com/jina-ai/jina.git@feat-bump-protobuf # pip does not properly resolve dependency versions with syntax pip install --no-cache-dir ".[test,full]" pip install --no-cache-dir ".[test]" pip install --no-cache-dir ".[full]" From 27c4b1bcf7592bb007b657348df1d7eff3cb8d62 Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Wed, 24 Aug 2022 16:50:46 +0100 Subject: [PATCH 18/20] fix: preserver file structure for compatibility --- docarray/proto/__init__.py | 12 ------------ docarray/proto/build-proto.sh | 5 ++--- docarray/proto/docarray_pb2.py | 6 ++++++ 3 files changed, 8 insertions(+), 15 deletions(-) create mode 100644 docarray/proto/docarray_pb2.py diff --git a/docarray/proto/__init__.py b/docarray/proto/__init__.py index 8ee0c2b6b32..e69de29bb2d 100644 --- a/docarray/proto/__init__.py +++ b/docarray/proto/__init__.py @@ -1,12 +0,0 @@ -from google.protobuf import __version__ as __pb__version__ - -if __pb__version__.startswith('4'): - from .pb.docarray_pb2 import DocumentProto, DocumentArrayProto, NdArrayProto - - # compatibility with any dependent that imports docarray_pb2 (for example jina) - from .pb import docarray_pb2 -else: - from .pb2.docarray_pb2 import DocumentProto, DocumentArrayProto, NdArrayProto - - # compatibility with any dependent that imports docarray_pb2 (for example jina) - from .pb2 import docarray_pb2 diff --git a/docarray/proto/build-proto.sh b/docarray/proto/build-proto.sh index 66482a70974..dff28ff2220 100755 --- a/docarray/proto/build-proto.sh +++ b/docarray/proto/build-proto.sh @@ -16,7 +16,7 @@ MODULE=docarray SRC_NAME="${MODULE}.proto" COMP_OUT_NAME="${MODULE}_pb2.py" -OUT_FOLDER=${2:-pb2} +OUT_FOLDER="${2:-pb2}/" VER_FILE=../__init__.py @@ -34,5 +34,4 @@ PLUGIN_PATH=${1} # /Volumes/TOSHIBA-4T/Documents/grpc/bins/opt/grpc_python_plug printf "\e[1;33mgenerating protobuf and grpc python interface\e[0m\n" -protoc -I ${SRC_DIR} --python_out=${SRC_DIR} ${SRC_DIR}${SRC_NAME} -mv ${COMP_OUT_NAME} "${OUT_FOLDER}/${COMP_OUT_NAME}" +protoc -I ${SRC_DIR} --python_out="${SRC_DIR}${OUT_FOLDER}" ${SRC_DIR}${SRC_NAME} diff --git a/docarray/proto/docarray_pb2.py b/docarray/proto/docarray_pb2.py new file mode 100644 index 00000000000..2ac63e16f6c --- /dev/null +++ b/docarray/proto/docarray_pb2.py @@ -0,0 +1,6 @@ +from google.protobuf import __version__ as __pb__version__ + +if __pb__version__.startswith('4'): + from .pb.docarray_pb2 import * +else: + from .pb2.docarray_pb2 import * From 70e3e2fee9680e6f175eb7ce5a45eddeb476d121 Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Thu, 25 Aug 2022 08:58:42 +0100 Subject: [PATCH 19/20] Revert "chore: update usages of proto" This reverts commit 420acb5642343cd55209350b9ddc8f5a8bbdfa3d. --- docarray/array/mixins/io/binary.py | 6 +++--- docarray/proto/io/__init__.py | 2 +- docarray/proto/io/ndarray.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docarray/array/mixins/io/binary.py b/docarray/array/mixins/io/binary.py index 1364532718f..7f4118898cf 100644 --- a/docarray/array/mixins/io/binary.py +++ b/docarray/array/mixins/io/binary.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from docarray.typing import T - from docarray.proto import DocumentArrayProto + from docarray.proto.docarray_pb2 import DocumentArrayProto from docarray import Document, DocumentArray @@ -161,7 +161,7 @@ def _load_binary_all( compress = None if protocol == 'protobuf-array': - from docarray.proto import DocumentArrayProto + from docarray.proto.docarray_pb2 import DocumentArrayProto dap = DocumentArrayProto() dap.ParseFromString(d) @@ -347,7 +347,7 @@ def to_protobuf(self, ndarray_type: Optional[str] = None) -> 'DocumentArrayProto Documents to ``List`` or ``numpy.ndarray``. :return: the protobuf message """ - from docarray.proto import DocumentArrayProto + from docarray.proto.docarray_pb2 import DocumentArrayProto dap = DocumentArrayProto() for d in self: diff --git a/docarray/proto/io/__init__.py b/docarray/proto/io/__init__.py index bb443d77763..3dd2802cae3 100644 --- a/docarray/proto/io/__init__.py +++ b/docarray/proto/io/__init__.py @@ -5,7 +5,7 @@ from google.protobuf.struct_pb2 import Struct from docarray.proto.io.ndarray import flush_ndarray, read_ndarray -from docarray.proto import NdArrayProto, DocumentProto +from docarray.proto.docarray_pb2 import NdArrayProto, DocumentProto if TYPE_CHECKING: from docarray import Document diff --git a/docarray/proto/io/ndarray.py b/docarray/proto/io/ndarray.py index 7d1d3a2f714..627dc0cc48a 100644 --- a/docarray/proto/io/ndarray.py +++ b/docarray/proto/io/ndarray.py @@ -6,7 +6,7 @@ if TYPE_CHECKING: from docarray.typing import ArrayType - from docarray.proto import NdArrayProto + from docarray.proto.docarray_pb2 import NdArrayProto def read_ndarray(pb_msg: 'NdArrayProto') -> 'ArrayType': From 9b115b7e2aa89c108e718bc00f3d1302d4470eaf Mon Sep 17 00:00:00 2001 From: Alaeddine Abdessalem Date: Thu, 25 Aug 2022 08:59:20 +0100 Subject: [PATCH 20/20] Revert "fix: change proto usage" This reverts commit 1a9569f99297607b70df404e134864882d0298fe. --- docarray/document/mixins/porting.py | 6 +++--- docarray/document/mixins/protobuf.py | 2 +- tests/unit/math/test_ndarray.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docarray/document/mixins/porting.py b/docarray/document/mixins/porting.py index 39be173787f..fde058f3faf 100644 --- a/docarray/document/mixins/porting.py +++ b/docarray/document/mixins/porting.py @@ -27,7 +27,7 @@ def from_dict( return cls.from_pydantic_model(PydanticDocument.parse_obj(obj, **kwargs)) elif protocol == 'protobuf': from google.protobuf import json_format - from docarray.proto import DocumentProto + from docarray.proto.docarray_pb2 import DocumentProto pb_msg = DocumentProto() json_format.ParseDict(obj, pb_msg, **kwargs) @@ -55,7 +55,7 @@ def from_json( return cls.from_pydantic_model(PydanticDocument.parse_raw(obj, **kwargs)) elif protocol == 'protobuf': from google.protobuf import json_format - from docarray.proto import DocumentProto + from docarray.proto.docarray_pb2 import DocumentProto pb_msg = DocumentProto() json_format.Parse(obj, pb_msg, **kwargs) @@ -113,7 +113,7 @@ def from_bytes( if protocol == 'pickle': return pickle.loads(bstr) elif protocol == 'protobuf': - from docarray.proto import DocumentProto + from docarray.proto.docarray_pb2 import DocumentProto pb_msg = DocumentProto() pb_msg.ParseFromString(bstr) diff --git a/docarray/document/mixins/protobuf.py b/docarray/document/mixins/protobuf.py index 875c2f46021..faa454d2274 100644 --- a/docarray/document/mixins/protobuf.py +++ b/docarray/document/mixins/protobuf.py @@ -2,7 +2,7 @@ if TYPE_CHECKING: from docarray.typing import T - from docarray.proto import DocumentProto + from docarray.proto.docarray_pb2 import DocumentProto class ProtobufMixin: diff --git a/tests/unit/math/test_ndarray.py b/tests/unit/math/test_ndarray.py index 546af78ab1c..562ae0f98f1 100644 --- a/tests/unit/math/test_ndarray.py +++ b/tests/unit/math/test_ndarray.py @@ -6,7 +6,7 @@ from scipy.sparse import csr_matrix, coo_matrix, bsr_matrix, csc_matrix, issparse from docarray.math.ndarray import get_array_rows, check_arraylike_equality -from docarray.proto import NdArrayProto +from docarray.proto.docarray_pb2 import NdArrayProto from docarray.proto.io import flush_ndarray, read_ndarray