in src/Custom/Embeddings/OpenAIEmbedding.cs [108:148]
private static ReadOnlyMemory<float> ConvertToVectorOfFloats(BinaryData binaryData)
{
ReadOnlySpan<byte> base64 = binaryData.ToMemory().Span;
// Remove quotes around base64 string.
if (base64.Length < 2 || base64[0] != (byte)'"' || base64[base64.Length - 1] != (byte)'"')
{
ThrowInvalidData();
}
base64 = base64.Slice(1, base64.Length - 2);
// Decode base64 string to bytes.
byte[] bytes = ArrayPool<byte>.Shared.Rent(Base64.GetMaxDecodedFromUtf8Length(base64.Length));
OperationStatus status = Base64.DecodeFromUtf8(base64, bytes.AsSpan(), out int bytesConsumed, out int bytesWritten);
if (status != OperationStatus.Done || bytesWritten % sizeof(float) != 0)
{
ThrowInvalidData();
}
// Interpret bytes as floats
float[] vector = new float[bytesWritten / sizeof(float)];
bytes.AsSpan(0, bytesWritten).CopyTo(MemoryMarshal.AsBytes(vector.AsSpan()));
if (!BitConverter.IsLittleEndian)
{
Span<int> ints = MemoryMarshal.Cast<float, int>(vector.AsSpan());
#if NET8_0_OR_GREATER
BinaryPrimitives.ReverseEndianness(ints, ints);
#else
for (int i = 0; i < ints.Length; i++)
{
ints[i] = BinaryPrimitives.ReverseEndianness(ints[i]);
}
#endif
}
ArrayPool<byte>.Shared.Return(bytes);
return new ReadOnlyMemory<float>(vector);
static void ThrowInvalidData() =>
throw new FormatException("The input is not a valid Base64 string of encoded floats.");
}