Enum ArrayBytes 

pub enum ArrayBytes<'a> {
    Fixed(Cow<'a, [u8]>),
    Variable(ArrayBytesVariableLength<'a>),
    Optional(ArrayBytesOptional<'a>),
}

Fixed or variable length array bytes.

Variants

Fixed(Cow<'a, [u8]>)

Bytes for a fixed length array.

These represent elements in C-contiguous order (i.e. row-major order) where the last dimension varies the fastest.

Variable(ArrayBytesVariableLength<'a>)

Bytes and element byte offsets for a variable length array.

Optional(ArrayBytesOptional<'a>)

Bytes for an optional array (data with a validity mask).

The data can be either Fixed or Variable length. The mask is validated at construction to have 1 byte per element.

Implementations

impl<'a> ArrayBytes<'a>

pub fn new_flen(bytes: impl Into<Cow<'a, [u8]>>) -> ArrayBytes<'a>

Create a new fixed length array bytes from bytes.

bytes must be C-contiguous.

pub fn new_vlen( bytes: impl Into<Cow<'a, [u8]>>, offsets: ArrayBytesOffsets<'a>, ) -> Result<ArrayBytes<'a>, ArrayBytesRawOffsetsOutOfBoundsError>

Create a new variable length array bytes from bytes and offsets.

Errors

Returns a ArrayBytesRawOffsetsOutOfBoundsError if the last offset is out of bounds of the bytes.

pub unsafe fn new_vlen_unchecked( bytes: impl Into<Cow<'a, [u8]>>, offsets: ArrayBytesOffsets<'a>, ) -> ArrayBytes<'a>

Create a new variable length array bytes from bytes and offsets without checking the offsets.

Safety

The last offset must be less than or equal to the length of the bytes.

Examples found in repository

examples/custom_data_type_variable_size.rs (line 57)

    fn to_array_bytes<'a>(
        data_type: &DataType,
        elements: &'a [Self],
    ) -> Result<zarrs::array::ArrayBytes<'a>, ElementError> {
        Self::validate_data_type(data_type)?;
        let mut bytes = Vec::new();
        let mut offsets = Vec::with_capacity(elements.len() + 1);

        for element in elements {
            offsets.push(bytes.len());
            if let Some(value) = element.0 {
                bytes.extend_from_slice(&value.to_le_bytes());
            }
        }
        offsets.push(bytes.len());
        let offsets = unsafe {
            // SAFETY: Constructed correctly above
            ArrayBytesOffsets::new_unchecked(offsets)
        };
        unsafe { Ok(ArrayBytes::new_vlen_unchecked(bytes, offsets)) }
    }

pub fn with_optional_mask( self, mask: impl Into<Cow<'a, [u8]>>, ) -> ArrayBytes<'a>

Wrap the array bytes with an optional validity mask.

This creates an Optional variant that contains the current array bytes and the provided mask.

pub fn new_fill_value( data_type: &DataType, num_elements: u64, fill_value: &FillValue, ) -> Result<ArrayBytes<'a>, DataTypeFillValueError>

Create a new ArrayBytes with num_elements composed entirely of the fill_value.

Errors

Returns DataTypeFillValueError if the fill value is incompatible with the data type.

Panics

Panics if num_elements exceeds usize::MAX.

pub fn into_fixed(self) -> Result<Cow<'a, [u8]>, ExpectedFixedLengthBytesError>

Convert the array bytes into fixed length bytes.

Errors

Returns an ExpectedFixedLengthBytesError if the bytes are not fixed.

Examples found in repository

examples/custom_data_type_float8_e3m4.rs (line 192)

    fn from_array_bytes(
        data_type: &DataType,
        bytes: ArrayBytes<'_>,
    ) -> Result<Vec<Self>, ElementError> {
        Self::validate_data_type(data_type)?;
        let bytes = bytes.into_fixed()?;
        let bytes_len = bytes.len();
        let mut elements = Vec::with_capacity(bytes_len);
        // NOTE: Could memcpy here
        for byte in bytes.iter() {
            elements.push(CustomDataTypeFloat8e3m4Element(*byte))
        }
        Ok(elements)
    }

examples/custom_data_type_uint4.rs (line 184)

    fn from_array_bytes(
        data_type: &DataType,
        bytes: ArrayBytes<'_>,
    ) -> Result<Vec<Self>, ElementError> {
        Self::validate_data_type(data_type)?;
        let bytes = bytes.into_fixed()?;
        let bytes_len = bytes.len();
        let mut elements = Vec::with_capacity(bytes_len / size_of::<CustomDataTypeUInt4Element>());
        for byte in bytes.iter() {
            elements.push(CustomDataTypeUInt4Element(*byte))
        }
        Ok(elements)
    }

examples/custom_data_type_uint12.rs (line 182)

    fn from_array_bytes(
        data_type: &DataType,
        bytes: ArrayBytes<'_>,
    ) -> Result<Vec<Self>, ElementError> {
        Self::validate_data_type(data_type)?;
        let bytes = bytes.into_fixed()?;
        let bytes_len = bytes.len();
        let mut elements = Vec::with_capacity(bytes_len / size_of::<CustomDataTypeUInt12Element>());
        for chunk in bytes.as_chunks::<2>().0 {
            elements.push(CustomDataTypeUInt12Element::from_le_bytes(*chunk))
        }
        Ok(elements)
    }

examples/custom_data_type_fixed_size.rs (line 140)

    fn from_array_bytes(
        data_type: &DataType,
        bytes: ArrayBytes<'_>,
    ) -> Result<Vec<Self>, ElementError> {
        Self::validate_data_type(data_type)?;
        let bytes = bytes.into_fixed()?;
        let bytes_len = bytes.len();
        let mut elements =
            Vec::with_capacity(bytes_len / size_of::<CustomDataTypeFixedSizeBytes>());
        for bytes in bytes
            .as_chunks::<{ size_of::<CustomDataTypeFixedSizeBytes>() }>()
            .0
        {
            elements.push(CustomDataTypeFixedSizeElement::from_ne_bytes(bytes))
        }
        Ok(elements)
    }

examples/sharded_array_write_read.rs (line 138)

fn sharded_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
    use std::sync::Arc;

    use rayon::prelude::{IntoParallelIterator, ParallelIterator};
    use zarrs::array::{ArraySubset, codec, data_type};
    use zarrs::node::Node;
    use zarrs::storage::store;

    // Create a store
    // let path = tempfile::TempDir::new()?;
    // let mut store: ReadableWritableListableStorage =
    //     Arc::new(zarrs::filesystem::FilesystemStore::new(path.path())?);
    // let mut store: ReadableWritableListableStorage = Arc::new(
    //     zarrs::filesystem::FilesystemStore::new("zarrs/tests/data/sharded_array_write_read.zarr")?,
    // );
    let mut store: ReadableWritableListableStorage = Arc::new(store::MemoryStore::new());
    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1)
        && arg1 == "--usage-log"
    {
        let log_writer = Arc::new(std::sync::Mutex::new(
            // std::io::BufWriter::new(
            std::io::stdout(),
            //    )
        ));
        store = Arc::new(UsageLogStorageAdapter::new(store, log_writer, || {
            chrono::Utc::now().format("[%T%.3f] ").to_string()
        }));
    }

    // Create the root group
    zarrs::group::GroupBuilder::new()
        .build(store.clone(), "/")?
        .store_metadata()?;

    // Create a group with attributes
    let group_path = "/group";
    let mut group = zarrs::group::GroupBuilder::new().build(store.clone(), group_path)?;
    group
        .attributes_mut()
        .insert("foo".into(), serde_json::Value::String("bar".into()));
    group.store_metadata()?;

    // Create an array
    let array_path = "/group/array";
    let subchunk_shape = vec![4, 4];
    let array = zarrs::array::ArrayBuilder::new(
        vec![8, 8], // array shape
        vec![4, 8], // chunk (shard) shape
        data_type::uint16(),
        0u16,
    )
    .subchunk_shape(subchunk_shape.clone())
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(codec::GzipCodec::new(5)?),
    ])
    .dimension_names(["y", "x"].into())
    // .storage_transformers(vec![].into())
    .build(store.clone(), array_path)?;

    // Write array metadata to store
    array.store_metadata()?;

    // The array metadata is
    println!(
        "The array metadata is:\n{}\n",
        array.metadata().to_string_pretty()
    );

    // Use default codec options (concurrency etc)
    let options = CodecOptions::default();

    // Write some shards (in parallel)
    (0..2).into_par_iter().try_for_each(|s| {
        let chunk_grid = array.chunk_grid();
        let chunk_indices = vec![s, 0];
        if let Some(chunk_shape) = chunk_grid.chunk_shape(&chunk_indices)? {
            let chunk_array = ndarray::ArrayD::<u16>::from_shape_fn(
                chunk_shape
                    .iter()
                    .map(|u| u.get() as usize)
                    .collect::<Vec<_>>(),
                |ij| {
                    (s * chunk_shape[0].get() * chunk_shape[1].get()
                        + ij[0] as u64 * chunk_shape[1].get()
                        + ij[1] as u64) as u16
                },
            );
            array.store_chunk(&chunk_indices, chunk_array)
        } else {
            Err(zarrs::array::ArrayError::InvalidChunkGridIndicesError(
                chunk_indices.to_vec(),
            ))
        }
    })?;

    // Read the whole array
    let data_all: ArrayD<u16> = array.retrieve_array_subset(&array.subset_all())?;
    println!("The whole array is:\n{data_all}\n");

    // Read a shard back from the store
    let shard_indices = vec![1, 0];
    let data_shard: ArrayD<u16> = array.retrieve_chunk(&shard_indices)?;
    println!("Shard [1,0] is:\n{data_shard}\n");

    // Read a subchunk from the store
    let subset_chunk_1_0 = ArraySubset::new_with_ranges(&[4..8, 0..4]);
    let data_chunk: ArrayD<u16> = array.retrieve_array_subset(&subset_chunk_1_0)?;
    println!("Chunk [1,0] is:\n{data_chunk}\n");

    // Read the central 4x2 subset of the array
    let subset_4x2 = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region
    let data_4x2: ArrayD<u16> = array.retrieve_array_subset(&subset_4x2)?;
    println!("The middle 4x2 subset is:\n{data_4x2}\n");

    // Decode subchunks
    // In some cases, it might be preferable to decode subchunks in a shard directly.
    // If using the partial decoder, then the shard index will only be read once from the store.
    let partial_decoder = array.partial_decoder(&[0, 0])?;
    println!("Decoded subchunks:");
    for subchunk_subset in [
        ArraySubset::new_with_start_shape(vec![0, 0], subchunk_shape.clone())?,
        ArraySubset::new_with_start_shape(vec![0, 4], subchunk_shape.clone())?,
    ] {
        println!("{subchunk_subset}");
        let decoded_subchunk_bytes = partial_decoder.partial_decode(&subchunk_subset, &options)?;
        let ndarray = bytes_to_ndarray::<u16>(
            &subchunk_shape,
            decoded_subchunk_bytes.into_fixed()?.into_owned(),
        )?;
        println!("{ndarray}\n");
    }

    // Show the hierarchy
    let node = Node::open(&store, "/").unwrap();
    let tree = node.hierarchy_tree();
    println!("The Zarr hierarchy tree is:\n{}", tree);

    println!(
        "The keys in the store are:\n[{}]",
        store.list().unwrap_or_default().iter().format(", ")
    );

    Ok(())
}

pub fn into_variable( self, ) -> Result<ArrayBytesVariableLength<'a>, ExpectedVariableLengthBytesError>

Convert the array bytes into variable length bytes and element byte offsets.

Errors

Returns an ExpectedVariableLengthBytesError if the bytes are not variable.

Examples found in repository

examples/custom_data_type_variable_size.rs (line 74)

    fn from_array_bytes(
        data_type: &DataType,
        bytes: ArrayBytes<'_>,
    ) -> Result<Vec<Self>, ElementError> {
        Self::validate_data_type(data_type)?;
        let (bytes, offsets) = bytes.into_variable()?.into_parts();

        let mut elements = Vec::with_capacity(offsets.len().saturating_sub(1));
        for (curr, next) in offsets.iter().tuple_windows() {
            let bytes = &bytes[*curr..*next];
            if let Ok(bytes) = <[u8; 4]>::try_from(bytes) {
                let value = f32::from_le_bytes(bytes);
                elements.push(CustomDataTypeVariableSizeElement(Some(value)));
            } else if bytes.is_empty() {
                elements.push(CustomDataTypeVariableSizeElement(None));
            } else {
                panic!()
            }
        }

        Ok(elements)
    }

examples/array_write_read_string.rs (line 96)

fn array_write_read() -> Result<(), Box<dyn std::error::Error>> {
    use std::sync::Arc;

    use zarrs::array::{ArrayBytes, data_type};
    use zarrs::storage::store;

    // Create a store
    // let path = tempfile::TempDir::new()?;
    // let mut store: ReadableWritableListableStorage =
    //     Arc::new(zarrs::filesystem::FilesystemStore::new(path.path())?);
    // let mut store: ReadableWritableListableStorage = Arc::new(
    //     zarrs::filesystem::FilesystemStore::new("zarrs/tests/data/array_write_read.zarr")?,
    // );
    let mut store: ReadableWritableListableStorage = Arc::new(store::MemoryStore::new());
    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1)
        && arg1 == "--usage-log"
    {
        let log_writer = Arc::new(std::sync::Mutex::new(
            // std::io::BufWriter::new(
            std::io::stdout(),
            //    )
        ));
        store = Arc::new(UsageLogStorageAdapter::new(store, log_writer, || {
            chrono::Utc::now().format("[%T%.3f] ").to_string()
        }));
    }

    // Create the root group
    zarrs::group::GroupBuilder::new()
        .build(store.clone(), "/")?
        .store_metadata()?;

    // Create a group with attributes
    let group_path = "/group";
    let mut group = zarrs::group::GroupBuilder::new().build(store.clone(), group_path)?;
    group
        .attributes_mut()
        .insert("foo".into(), serde_json::Value::String("bar".into()));
    group.store_metadata()?;

    println!(
        "The group metadata is:\n{}\n",
        group.metadata().to_string_pretty()
    );

    // Create an array
    let array_path = "/group/array";
    let array = zarrs::array::ArrayBuilder::new(
        vec![4, 4], // array shape
        vec![2, 2], // regular chunk shape
        data_type::string(),
        "_",
    )
    // .bytes_to_bytes_codecs(vec![]) // uncompressed
    .dimension_names(["y", "x"].into())
    // .storage_transformers(vec![].into())
    .build(store.clone(), array_path)?;

    // Write array metadata to store
    array.store_metadata()?;

    println!(
        "The array metadata is:\n{}\n",
        array.metadata().to_string_pretty()
    );

    // Write some chunks
    array.store_chunk(
        &[0, 0],
        ArrayD::<&str>::from_shape_vec(vec![2, 2], vec!["a", "bb", "ccc", "dddd"]).unwrap(),
    )?;
    array.store_chunk(
        &[0, 1],
        ArrayD::<&str>::from_shape_vec(vec![2, 2], vec!["4444", "333", "22", "1"]).unwrap(),
    )?;
    let subset_all = array.subset_all();
    let data_all: ArrayD<String> = array.retrieve_array_subset(&subset_all)?;
    println!("store_chunk [0, 0] and [0, 1]:\n{data_all}\n");

    // Write a subset spanning multiple chunks, including updating chunks already written
    let ndarray_subset: Array2<&str> = array![["!", "@@"], ["###", "$$$$"]];
    array.store_array_subset(&[1..3, 1..3], ndarray_subset)?;
    let data_all: ArrayD<String> = array.retrieve_array_subset(&subset_all)?;
    println!("store_array_subset [1..3, 1..3]:\nndarray::ArrayD<String>\n{data_all}");

    // Retrieve bytes directly, convert into a single string allocation, create a &str ndarray
    // TODO: Add a convenience function for this?
    let data_all: ArrayBytes = array.retrieve_array_subset(&subset_all)?;
    let (bytes, offsets) = data_all.into_variable()?.into_parts();
    let string = String::from_utf8(bytes.into_owned())?;
    let elements = offsets
        .iter()
        .tuple_windows()
        .map(|(&curr, &next)| &string[curr..next])
        .collect::<Vec<&str>>();
    let ndarray = ArrayD::<&str>::from_shape_vec(subset_all.shape_usize(), elements)?;
    println!("ndarray::ArrayD<&str>:\n{ndarray}");

    Ok(())
}

pub fn into_optional( self, ) -> Result<ArrayBytesOptional<'a>, ExpectedOptionalBytesError>

Convert the array bytes into optional data and validity mask.

Errors

Returns an ExpectedOptionalBytesError if the bytes are not optional.

pub fn size(&self) -> usize

Returns the size (in bytes) of the underlying element bytes.

This only considers the size of the element bytes, and does not include the element offsets for a variable sized array or the mask for optional arrays.

pub fn offsets(&self) -> Option<&ArrayBytesOffsets<'a>>

Return the byte offsets for variable sized bytes. Returns None for fixed size bytes.

pub fn into_owned(self) -> ArrayBytes<'static>

Convert into owned ArrayBytes<'static>.

Examples found in repository

examples/custom_data_type_fixed_size.rs (line 129)

    fn into_array_bytes(
        data_type: &DataType,
        elements: Vec<Self>,
    ) -> Result<zarrs::array::ArrayBytes<'static>, ElementError> {
        Ok(Self::to_array_bytes(data_type, &elements)?.into_owned())
    }

examples/custom_data_type_float8_e3m4.rs (line 181)

    fn into_array_bytes(
        data_type: &DataType,
        elements: Vec<Self>,
    ) -> Result<zarrs::array::ArrayBytes<'static>, ElementError> {
        Ok(Self::to_array_bytes(data_type, &elements)?.into_owned())
    }

examples/custom_data_type_uint12.rs (line 171)

    fn into_array_bytes(
        data_type: &DataType,
        elements: Vec<Self>,
    ) -> Result<zarrs::array::ArrayBytes<'static>, ElementError> {
        Ok(Self::to_array_bytes(data_type, &elements)?.into_owned())
    }

examples/custom_data_type_uint4.rs (line 173)

    fn into_array_bytes(
        data_type: &DataType,
        elements: Vec<Self>,
    ) -> Result<zarrs::array::ArrayBytes<'static>, ElementError> {
        Ok(Self::to_array_bytes(data_type, &elements)?.into_owned())
    }

examples/custom_data_type_variable_size.rs (line 64)

    fn into_array_bytes(
        data_type: &DataType,
        elements: Vec<Self>,
    ) -> Result<zarrs::array::ArrayBytes<'static>, ElementError> {
        Ok(Self::to_array_bytes(data_type, &elements)?.into_owned())
    }

pub fn validate( &self, num_elements: u64, data_type: &DataType, ) -> Result<(), CodecError>

Validate that the array has a valid encoding.

For a fixed-length array, check it matches the expected size. For a variable-length array, check that the offsets are monotonically increasing and the largest offset is equal to the array length.

Errors

Returns an error if the array is not valid.

pub fn is_fill_value(&self, fill_value: &FillValue) -> bool

Returns true if the array is empty for the given fill value.

pub fn extract_array_subset( &self, indexer: &dyn Indexer, array_shape: &[u64], data_type: &DataType, ) -> Result<ArrayBytes<'_>, CodecError>

Extract a subset of the array bytes.

Errors

Returns a CodecError::IncompatibleIndexer if the indexer is incompatible with subset.

Panics

Panics if indices in the subset exceed usize::MAX.

Trait Implementations

impl<'a> Clone for ArrayBytes<'a>

fn clone(&self) -> ArrayBytes<'a>

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<'a> Debug for ArrayBytes<'a>

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl<'a> From<&'a [u8]> for ArrayBytes<'a>

fn from(bytes: &'a [u8]) -> ArrayBytes<'a>

Converts to this type from the input type.

impl<'a, const N: usize> From<&'a [u8; N]> for ArrayBytes<'a>

fn from(bytes: &'a [u8; N]) -> ArrayBytes<'a>

Converts to this type from the input type.

impl<'a> From<ArrayBytesOptional<'a>> for ArrayBytes<'a>

fn from(value: ArrayBytesOptional<'a>) -> ArrayBytes<'a>

Converts to this type from the input type.

impl<'a> From<ArrayBytesVariableLength<'a>> for ArrayBytes<'a>

fn from(value: ArrayBytesVariableLength<'a>) -> ArrayBytes<'a>

Converts to this type from the input type.

impl<'a> From<Cow<'a, [u8]>> for ArrayBytes<'a>

fn from(value: Cow<'a, [u8]>) -> ArrayBytes<'a>

Converts to this type from the input type.

impl From<Vec<u8>> for ArrayBytes<'_>

fn from(bytes: Vec<u8>) -> ArrayBytes<'_>

Converts to this type from the input type.

impl FromArrayBytes for ArrayBytes<'static>

fn from_array_bytes( bytes: ArrayBytes<'static>, _shape: &[u64], _data_type: &DataType, ) -> Result<Self, ArrayError>

Convert ArrayBytes into Self.

fn from_array_bytes_arc( bytes: Arc<ArrayBytes<'static>>, shape: &[u64], data_type: &DataType, ) -> Result<Self, ArrayError>

Convert an Arc<ArrayBytes> into Self.

impl<'a> IntoArrayBytes<'a> for ArrayBytes<'a>

fn into_array_bytes( self, _data_type: &DataType, ) -> Result<ArrayBytes<'a>, ElementError>

Convert self into ArrayBytes.

impl<'a> PartialEq for ArrayBytes<'a>

fn eq(&self, other: &ArrayBytes<'a>) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl<'a> Eq for ArrayBytes<'a>

impl<'a> StructuralPartialEq for ArrayBytes<'a>