in safetensors/src/tensor.rs [919:966]
fn arbitrary_metadata() -> impl Strategy<Value = Metadata> {
// We generate at least one tensor.
(1..MAX_TENSORS)
.prop_flat_map(|size| {
// Returns a strategy generating `size` data types and shapes.
(
prop::collection::vec(arbitrary_dtype(), size),
prop::collection::vec(arbitrary_shape(), size),
)
})
.prop_filter_map("Misaligned slices", |(dtypes, shapes)| {
// Returns a valid metadata object for a random (length, dtypes, shapes) triple.
let mut start = 0;
let tensors: Vec<TensorInfo> = dtypes
.iter()
.zip(shapes)
.flat_map(|(dtype, shape)| {
// This cannot overflow because the size of
// the vector and elements are so small.
let bitlength: usize = shape.iter().product::<usize>() * dtype.bitsize();
if bitlength % 8 != 0 {
return None;
}
let length = bitlength.div_ceil(8);
let end = start + length;
let tensor = TensorInfo {
dtype: *dtype,
shape,
data_offsets: (start, end),
};
start = end;
Some(tensor)
})
.collect();
let index_map = (0..tensors.len())
.map(|index| (format!("t.{index}"), index))
.collect();
if tensors.is_empty() {
None
} else {
Some(Metadata {
metadata: None,
tensors,
index_map,
})
}
})
}