Fix structured arrays that contain objects #806

Attila Bergou · Attila Bergou · commit 635d5acd93c7 · 2021-08-20T11:49:30.000-04:00
* Ensures that the fill value of structured arrays that contain objects
  is encoded using object_codec.
diff --git a/zarr/meta.py b/zarr/meta.py
@@ -40,7 +40,14 @@ def decode_array_metadata(s: Union[MappingType, str]) -> MappingType[str, Any]:
     # extract array metadata fields
     try:
         dtype = decode_dtype(meta['dtype'])
-        fill_value = decode_fill_value(meta['fill_value'], dtype)
+
+        if dtype.hasobject:
+            import numcodecs
+            object_codec = numcodecs.get_codec(meta['filters'][0])
+        else:
+            object_codec = None
+
+        fill_value = decode_fill_value(meta['fill_value'], dtype, object_codec)
         meta = dict(
             zarr_format=meta['zarr_format'],
             shape=tuple(meta['shape']),
@@ -66,14 +73,18 @@ def encode_array_metadata(meta: MappingType[str, Any]) -> bytes:
         dtype, sdshape = dtype.subdtype
 
     dimension_separator = meta.get('dimension_separator')
-
+    if dtype.hasobject:
+        import numcodecs
+        object_codec = numcodecs.get_codec(meta['filters'][0])
+    else:
+        object_codec = None
     meta = dict(
         zarr_format=ZARR_FORMAT,
         shape=meta['shape'] + sdshape,
         chunks=meta['chunks'],
         dtype=encode_dtype(dtype),
         compressor=meta['compressor'],
-        fill_value=encode_fill_value(meta['fill_value'], dtype),
+        fill_value=encode_fill_value(meta['fill_value'], dtype, object_codec),
         order=meta['order'],
         filters=meta['filters'],
     )
@@ -132,11 +143,16 @@ def encode_group_metadata(meta=None) -> bytes:
 }
 
 
-def decode_fill_value(v, dtype):
+def decode_fill_value(v, dtype, object_codec=None):
     # early out
     if v is None:
         return v
-    if dtype.kind == 'f':
+    if dtype.hasobject:
+        v = base64.standard_b64decode(v)
+        v = object_codec.decode(v)
+        v = np.array(v, dtype=dtype)[()]
+        return v
+    elif dtype.kind == 'f':
         if v == 'NaN':
             return np.nan
         elif v == 'Infinity':
@@ -171,10 +187,14 @@ def decode_fill_value(v, dtype):
         return np.array(v, dtype=dtype)[()]
 
 
-def encode_fill_value(v: Any, dtype: np.dtype) -> Any:
+def encode_fill_value(v: Any, dtype: np.dtype, object_codec: Any = None) -> Any:
     # early out
     if v is None:
         return v
+    if dtype.hasobject:
+        v = object_codec.encode(v)
+        v = str(base64.standard_b64encode(v), 'ascii')
+        return v
     if dtype.kind == 'f':
         if np.isnan(v):
             return 'NaN'
@@ -190,8 +210,8 @@ def encode_fill_value(v: Any, dtype: np.dtype) -> Any:
         return bool(v)
     elif dtype.kind in 'c':
         c = cast(np.complex128, np.dtype(complex).type())
-        v = (encode_fill_value(v.real, c.real.dtype),
-             encode_fill_value(v.imag, c.imag.dtype))
+        v = (encode_fill_value(v.real, c.real.dtype, object_codec),
+             encode_fill_value(v.imag, c.imag.dtype, object_codec))
         return v
     elif dtype.kind in 'SV':
         v = str(base64.standard_b64encode(v), 'ascii')
diff --git a/zarr/storage.py b/zarr/storage.py
@@ -423,7 +423,7 @@ def _init_array_metadata(
         filters_config = []
 
     # deal with object encoding
-    if dtype == object:
+    if dtype.hasobject:
         if object_codec is None:
             if not filters:
                 # there are no filters so we can be sure there is no object codec
diff --git a/zarr/util.py b/zarr/util.py
@@ -253,10 +253,9 @@ def normalize_dimension_separator(sep: Optional[str]) -> Optional[str]:
 
 def normalize_fill_value(fill_value, dtype: np.dtype):
 
-    if fill_value is None:
+    if fill_value is None or dtype.hasobject:
         # no fill value
         pass
-
     elif fill_value == 0:
         # this should be compatible across numpy versions for any array type, including
         # structured arrays
