in datafusion/substrait/src/logical_plan/producer.rs [1794:2050]
fn to_substrait_type(dt: &DataType, nullable: bool) -> Result<substrait::proto::Type> {
let nullability = if nullable {
r#type::Nullability::Nullable as i32
} else {
r#type::Nullability::Required as i32
};
match dt {
DataType::Null => internal_err!("Null cast is not valid"),
DataType::Boolean => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Bool(r#type::Boolean {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::Int8 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::I8(r#type::I8 {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::UInt8 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::I8(r#type::I8 {
type_variation_reference: UNSIGNED_INTEGER_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::Int16 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::I16(r#type::I16 {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::UInt16 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::I16(r#type::I16 {
type_variation_reference: UNSIGNED_INTEGER_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::Int32 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::I32(r#type::I32 {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::UInt32 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::I32(r#type::I32 {
type_variation_reference: UNSIGNED_INTEGER_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::Int64 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::I64(r#type::I64 {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::UInt64 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::I64(r#type::I64 {
type_variation_reference: UNSIGNED_INTEGER_TYPE_VARIATION_REF,
nullability,
})),
}),
// Float16 is not supported in Substrait
DataType::Float32 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Fp32(r#type::Fp32 {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::Float64 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Fp64(r#type::Fp64 {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::Timestamp(unit, tz) => {
let precision = match unit {
TimeUnit::Second => 0,
TimeUnit::Millisecond => 3,
TimeUnit::Microsecond => 6,
TimeUnit::Nanosecond => 9,
};
let kind = match tz {
None => r#type::Kind::PrecisionTimestamp(r#type::PrecisionTimestamp {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability,
precision,
}),
Some(_) => {
// If timezone is present, no matter what the actual tz value is, it indicates the
// value of the timestamp is tied to UTC epoch. That's all that Substrait cares about.
// As the timezone is lost, this conversion may be lossy for downstream use of the value.
r#type::Kind::PrecisionTimestampTz(r#type::PrecisionTimestampTz {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability,
precision,
})
}
};
Ok(substrait::proto::Type { kind: Some(kind) })
}
DataType::Date32 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Date(r#type::Date {
type_variation_reference: DATE_32_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::Date64 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Date(r#type::Date {
type_variation_reference: DATE_64_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::Interval(interval_unit) => {
match interval_unit {
IntervalUnit::YearMonth => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::IntervalYear(r#type::IntervalYear {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability,
})),
}),
IntervalUnit::DayTime => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::IntervalDay(r#type::IntervalDay {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability,
precision: Some(3), // DayTime precision is always milliseconds
})),
}),
IntervalUnit::MonthDayNano => {
Ok(substrait::proto::Type {
kind: Some(r#type::Kind::IntervalCompound(
r#type::IntervalCompound {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability,
precision: 9, // nanos
},
)),
})
}
}
}
DataType::Binary => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Binary(r#type::Binary {
type_variation_reference: DEFAULT_CONTAINER_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::FixedSizeBinary(length) => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::FixedBinary(r#type::FixedBinary {
length: *length,
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::LargeBinary => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Binary(r#type::Binary {
type_variation_reference: LARGE_CONTAINER_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::BinaryView => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Binary(r#type::Binary {
type_variation_reference: VIEW_CONTAINER_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::Utf8 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::String(r#type::String {
type_variation_reference: DEFAULT_CONTAINER_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::LargeUtf8 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::String(r#type::String {
type_variation_reference: LARGE_CONTAINER_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::Utf8View => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::String(r#type::String {
type_variation_reference: VIEW_CONTAINER_TYPE_VARIATION_REF,
nullability,
})),
}),
DataType::List(inner) => {
let inner_type = to_substrait_type(inner.data_type(), inner.is_nullable())?;
Ok(substrait::proto::Type {
kind: Some(r#type::Kind::List(Box::new(r#type::List {
r#type: Some(Box::new(inner_type)),
type_variation_reference: DEFAULT_CONTAINER_TYPE_VARIATION_REF,
nullability,
}))),
})
}
DataType::LargeList(inner) => {
let inner_type = to_substrait_type(inner.data_type(), inner.is_nullable())?;
Ok(substrait::proto::Type {
kind: Some(r#type::Kind::List(Box::new(r#type::List {
r#type: Some(Box::new(inner_type)),
type_variation_reference: LARGE_CONTAINER_TYPE_VARIATION_REF,
nullability,
}))),
})
}
DataType::Map(inner, _) => match inner.data_type() {
DataType::Struct(key_and_value) if key_and_value.len() == 2 => {
let key_type = to_substrait_type(
key_and_value[0].data_type(),
key_and_value[0].is_nullable(),
)?;
let value_type = to_substrait_type(
key_and_value[1].data_type(),
key_and_value[1].is_nullable(),
)?;
Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Map(Box::new(r#type::Map {
key: Some(Box::new(key_type)),
value: Some(Box::new(value_type)),
type_variation_reference: DEFAULT_CONTAINER_TYPE_VARIATION_REF,
nullability,
}))),
})
}
_ => plan_err!("Map fields must contain a Struct with exactly 2 fields"),
},
DataType::Struct(fields) => {
let field_types = fields
.iter()
.map(|field| to_substrait_type(field.data_type(), field.is_nullable()))
.collect::<Result<Vec<_>>>()?;
Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Struct(r#type::Struct {
types: field_types,
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability,
})),
})
}
DataType::Decimal128(p, s) => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Decimal(r#type::Decimal {
type_variation_reference: DECIMAL_128_TYPE_VARIATION_REF,
nullability,
scale: *s as i32,
precision: *p as i32,
})),
}),
DataType::Decimal256(p, s) => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Decimal(r#type::Decimal {
type_variation_reference: DECIMAL_256_TYPE_VARIATION_REF,
nullability,
scale: *s as i32,
precision: *p as i32,
})),
}),
_ => not_impl_err!("Unsupported cast type: {dt:?}"),
}
}