in arrow/internal/arrjson/arrjson.go [967:1384]
func arrayFromJSON(mem memory.Allocator, dt arrow.DataType, arr Array) arrow.ArrayData {
switch dt := dt.(type) {
case *arrow.NullType:
return array.NewNull(arr.Count).Data()
case *arrow.BooleanType:
bldr := array.NewBooleanBuilder(mem)
defer bldr.Release()
data := boolsFromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Int8Type:
bldr := array.NewInt8Builder(mem)
defer bldr.Release()
data := i8FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Int16Type:
bldr := array.NewInt16Builder(mem)
defer bldr.Release()
data := i16FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Int32Type:
bldr := array.NewInt32Builder(mem)
defer bldr.Release()
data := i32FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Int64Type:
bldr := array.NewInt64Builder(mem)
defer bldr.Release()
data := i64FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Uint8Type:
bldr := array.NewUint8Builder(mem)
defer bldr.Release()
data := u8FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Uint16Type:
bldr := array.NewUint16Builder(mem)
defer bldr.Release()
data := u16FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Uint32Type:
bldr := array.NewUint32Builder(mem)
defer bldr.Release()
data := u32FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Uint64Type:
bldr := array.NewUint64Builder(mem)
defer bldr.Release()
data := u64FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Float16Type:
bldr := array.NewFloat16Builder(mem)
defer bldr.Release()
data := f16FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Float32Type:
bldr := array.NewFloat32Builder(mem)
defer bldr.Release()
data := f32FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Float64Type:
bldr := array.NewFloat64Builder(mem)
defer bldr.Release()
data := f64FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.StringType:
bldr := array.NewStringBuilder(mem)
defer bldr.Release()
data := strFromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.LargeStringType:
bldr := array.NewLargeStringBuilder(mem)
defer bldr.Release()
data := strFromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.LargeBinaryType:
bldr := array.NewBinaryBuilder(mem, dt)
defer bldr.Release()
data := bytesFromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.BinaryType:
bldr := array.NewBinaryBuilder(mem, dt)
defer bldr.Release()
data := bytesFromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case arrow.BinaryViewDataType:
valids := validsToBitmap(validsFromJSON(arr.Valids), mem)
nulls := arr.Count - bitutil.CountSetBits(valids.Bytes(), 0, arr.Count)
headers := stringHeadersFromJSON(mem, !dt.IsUtf8(), arr.Views)
extraBufs := variadicBuffersFromJSON(arr.Variadic)
defer valids.Release()
defer headers.Release()
return array.NewData(dt, arr.Count,
append([]*memory.Buffer{valids, headers}, extraBufs...),
nil, nulls, 0)
case *arrow.ListType:
valids := validsFromJSON(arr.Valids)
elems := arrayFromJSON(mem, dt.Elem(), arr.Children[0])
defer elems.Release()
bitmap := validsToBitmap(valids, mem)
defer bitmap.Release()
nulls := arr.Count - bitutil.CountSetBits(bitmap.Bytes(), 0, arr.Count)
return array.NewData(dt, arr.Count, []*memory.Buffer{bitmap,
memory.NewBufferBytes(arrow.Int32Traits.CastToBytes(arr.Offset.([]int32)))},
[]arrow.ArrayData{elems}, nulls, 0)
case *arrow.LargeListType:
valids := validsFromJSON(arr.Valids)
elems := arrayFromJSON(mem, dt.Elem(), arr.Children[0])
defer elems.Release()
bitmap := validsToBitmap(valids, mem)
defer bitmap.Release()
nulls := arr.Count - bitutil.CountSetBits(bitmap.Bytes(), 0, arr.Count)
return array.NewData(dt, arr.Count, []*memory.Buffer{bitmap,
memory.NewBufferBytes(arrow.Int64Traits.CastToBytes(arr.Offset.([]int64)))},
[]arrow.ArrayData{elems}, nulls, 0)
case *arrow.ListViewType:
valids := validsFromJSON(arr.Valids)
elems := arrayFromJSON(mem, dt.Elem(), arr.Children[0])
defer elems.Release()
bitmap := validsToBitmap(valids, mem)
defer bitmap.Release()
nulls := arr.Count - bitutil.CountSetBits(bitmap.Bytes(), 0, arr.Count)
var offsets, sizes *memory.Buffer
if arr.Count == 0 {
emptyBuffer := memory.NewBufferBytes(nil)
offsets, sizes = emptyBuffer, emptyBuffer
} else {
offsets = memory.NewBufferBytes(arrow.Int32Traits.CastToBytes(arr.Offset.([]int32)))
sizes = memory.NewBufferBytes(arrow.Int32Traits.CastToBytes(arr.Size.([]int32)))
}
return array.NewData(dt, arr.Count, []*memory.Buffer{bitmap, offsets, sizes}, []arrow.ArrayData{elems}, nulls, 0)
case *arrow.LargeListViewType:
valids := validsFromJSON(arr.Valids)
elems := arrayFromJSON(mem, dt.Elem(), arr.Children[0])
defer elems.Release()
bitmap := validsToBitmap(valids, mem)
defer bitmap.Release()
nulls := arr.Count - bitutil.CountSetBits(bitmap.Bytes(), 0, arr.Count)
var offsets, sizes *memory.Buffer
if arr.Count == 0 {
emptyBuffer := memory.NewBufferBytes(nil)
offsets, sizes = emptyBuffer, emptyBuffer
} else {
offsets = memory.NewBufferBytes(arrow.Int64Traits.CastToBytes(arr.Offset.([]int64)))
sizes = memory.NewBufferBytes(arrow.Int64Traits.CastToBytes(arr.Size.([]int64)))
}
return array.NewData(dt, arr.Count, []*memory.Buffer{bitmap, offsets, sizes}, []arrow.ArrayData{elems}, nulls, 0)
case *arrow.FixedSizeListType:
valids := validsFromJSON(arr.Valids)
elems := arrayFromJSON(mem, dt.Elem(), arr.Children[0])
defer elems.Release()
bitmap := validsToBitmap(valids, mem)
defer bitmap.Release()
nulls := arr.Count - bitutil.CountSetBits(bitmap.Bytes(), 0, arr.Count)
return array.NewData(dt, arr.Count, []*memory.Buffer{bitmap}, []arrow.ArrayData{elems}, nulls, 0)
case *arrow.StructType:
valids := validsFromJSON(arr.Valids)
bitmap := validsToBitmap(valids, mem)
defer bitmap.Release()
nulls := arr.Count - bitutil.CountSetBits(bitmap.Bytes(), 0, arr.Count)
fields := make([]arrow.ArrayData, dt.NumFields())
for i := range fields {
child := arrayFromJSON(mem, dt.Field(i).Type, arr.Children[i])
defer child.Release()
fields[i] = child
}
return array.NewData(dt, arr.Count, []*memory.Buffer{bitmap}, fields, nulls, 0)
case *arrow.FixedSizeBinaryType:
bldr := array.NewFixedSizeBinaryBuilder(mem, dt)
defer bldr.Release()
strdata := strFromJSON(arr.Data)
data := make([][]byte, len(strdata))
for i, v := range strdata {
if len(v) != 2*dt.ByteWidth {
panic(fmt.Errorf("arrjson: invalid hex-string length (got=%d, want=%d)", len(v), 2*dt.ByteWidth))
}
vv, err := hex.DecodeString(v)
if err != nil {
panic(err)
}
data[i] = vv
}
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.MapType:
valids := validsFromJSON(arr.Valids)
elems := arrayFromJSON(mem, dt.Elem(), arr.Children[0])
defer elems.Release()
bitmap := validsToBitmap(valids, mem)
defer bitmap.Release()
nulls := arr.Count - bitutil.CountSetBits(bitmap.Bytes(), 0, arr.Count)
return array.NewData(dt, arr.Count, []*memory.Buffer{bitmap,
memory.NewBufferBytes(arrow.Int32Traits.CastToBytes(arr.Offset.([]int32)))},
[]arrow.ArrayData{elems}, nulls, 0)
case *arrow.Date32Type:
bldr := array.NewDate32Builder(mem)
defer bldr.Release()
data := date32FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Date64Type:
bldr := array.NewDate64Builder(mem)
defer bldr.Release()
data := date64FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Time32Type:
bldr := array.NewTime32Builder(mem, dt)
defer bldr.Release()
data := time32FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Time64Type:
bldr := array.NewTime64Builder(mem, dt)
defer bldr.Release()
data := time64FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.TimestampType:
bldr := array.NewTimestampBuilder(mem, dt)
defer bldr.Release()
data := timestampFromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.MonthIntervalType:
bldr := array.NewMonthIntervalBuilder(mem)
defer bldr.Release()
data := monthintervalFromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.DayTimeIntervalType:
bldr := array.NewDayTimeIntervalBuilder(mem)
defer bldr.Release()
data := daytimeintervalFromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.MonthDayNanoIntervalType:
bldr := array.NewMonthDayNanoIntervalBuilder(mem)
defer bldr.Release()
data := monthDayNanointervalFromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.DurationType:
bldr := array.NewDurationBuilder(mem, dt)
defer bldr.Release()
data := durationFromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Decimal32Type:
bldr := array.NewDecimal32Builder(mem, dt)
defer bldr.Release()
data := decimal32FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Decimal64Type:
bldr := array.NewDecimal64Builder(mem, dt)
defer bldr.Release()
data := decimal64FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Decimal128Type:
bldr := array.NewDecimal128Builder(mem, dt)
defer bldr.Release()
data := decimal128FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case *arrow.Decimal256Type:
bldr := array.NewDecimal256Builder(mem, dt)
defer bldr.Release()
data := decimal256FromJSON(arr.Data)
valids := validsFromJSON(arr.Valids)
bldr.AppendValues(data, valids)
return returnNewArrayData(bldr)
case arrow.ExtensionType:
storage := arrayFromJSON(mem, dt.StorageType(), arr)
defer storage.Release()
return array.NewData(dt, storage.Len(), storage.Buffers(), storage.Children(), storage.NullN(), storage.Offset())
case *arrow.DictionaryType:
indices := arrayFromJSON(mem, dt.IndexType, arr)
defer indices.Release()
return array.NewData(dt, indices.Len(), indices.Buffers(), indices.Children(), indices.NullN(), indices.Offset())
case *arrow.RunEndEncodedType:
runEnds := arrayFromJSON(mem, dt.RunEnds(), arr.Children[0])
defer runEnds.Release()
values := arrayFromJSON(mem, dt.Encoded(), arr.Children[1])
defer values.Release()
return array.NewData(dt, arr.Count, []*memory.Buffer{nil}, []arrow.ArrayData{runEnds, values}, 0, 0)
case arrow.UnionType:
fields := make([]arrow.ArrayData, dt.NumFields())
for i, f := range dt.Fields() {
child := arrayFromJSON(mem, f.Type, arr.Children[i])
defer child.Release()
fields[i] = child
}
typeIdBuf := memory.NewBufferBytes(arrow.Int8Traits.CastToBytes(arr.TypeID))
defer typeIdBuf.Release()
buffers := []*memory.Buffer{nil, typeIdBuf}
if dt.Mode() == arrow.DenseMode {
var offsets []byte
if arr.Offset == nil {
offsets = []byte{}
} else {
offsets = arrow.Int32Traits.CastToBytes(arr.Offset.([]int32))
}
offsetBuf := memory.NewBufferBytes(offsets)
defer offsetBuf.Release()
buffers = append(buffers, offsetBuf)
}
return array.NewData(dt, arr.Count, buffers, fields, 0, 0)
default:
panic(fmt.Errorf("unknown data type %v %T", dt, dt))
}
}