in native/src/seal/util/ztools.cpp [505:666]
unsigned zstd_deflate_array_inplace(DynArray<seal_byte> &in, MemoryPoolHandle pool)
{
if (!pool)
{
throw invalid_argument("pool is uninitialized");
}
PointerStorage ptr_storage(pool);
// Set up the custom allocator
ZSTD_customMem mem;
mem.customAlloc = zstd_alloc_impl;
mem.customFree = zstd_free_impl;
mem.opaque = &ptr_storage;
ZSTD_CCtx *cctx = ZSTD_createCCtx_advanced(mem);
if (!cctx)
{
// Failed to set up the context; there is something wrong with the allocator
return ZSTD_error_GENERIC;
}
// How much data was finally produced
size_t bytes_written_to_in = 0;
size_t bytes_read_from_in = 0;
// Allocate a temporary buffer for output
auto temp_out = DynArray<seal_byte>(buffer_size, pool);
// Where we are writing output now; start writing to the temporary buffer
seal_byte *out_head = temp_out.begin();
// How much of input do we have left to process
size_t in_size = in.size();
// Size of the current output buffer
size_t out_size = buffer_size;
// Are we overwriting in at this time?
bool out_is_in = false;
// Holds the return value of the stream compression call. This is either the amount of data that remains
// to be flushed from internal buffers, or an error code.
size_t pending = 0;
do
{
// The number of bytes we can read at a time is capped by zstd_process_bytes_in_max
size_t process_bytes_in = min<size_t>(in_size, zstd_process_bytes_in_max);
ZSTD_inBuffer input = { in.cbegin() + bytes_read_from_in, process_bytes_in, 0 };
size_t prev_pos = 0;
// Number of bytes left after this round; if we are at the end set flush accordingly
in_size -= process_bytes_in;
ZSTD_EndDirective flush = in_size ? ZSTD_e_continue : ZSTD_e_end;
// Loop while we have input left
do
{
// First ensure we have output space
while (!out_size)
{
// We are out of output buffer
if (!out_is_in)
{
// If we have been writing to the temporary buffer, then see if we can copy to in
size_t temp_out_size = temp_out.size();
if (bytes_read_from_in >= bytes_written_to_in + temp_out_size)
{
// All is good; we can copy over the buffer to in
out_head = in.begin() + bytes_written_to_in;
memcpy(out_head, temp_out.cbegin(), temp_out_size);
out_head += temp_out_size;
bytes_written_to_in += temp_out_size;
// For next writes, try to write to in
out_is_in = true;
// Reset out_size
out_size = bytes_read_from_in - bytes_written_to_in;
// Reset temp_out to have size buffer_size
temp_out.resize(buffer_size, false);
}
else
{
// We don't have enough room to copy to in; instead, resize temp_out and continue
// using it, hoping that the situation will change
out_size = temp_out_size + buffer_size;
temp_out.resize(out_size, false);
out_size = buffer_size;
out_head = temp_out.begin() + temp_out_size;
}
}
else
{
// We are writing to in but are out of space; switch to temp_out for the moment
out_is_in = false;
// Set size and pointer
out_size = temp_out.size();
out_head = temp_out.begin();
}
}
// Cap the out size to zstd_process_bytes_out_max
size_t process_bytes_out = min<size_t>(out_size, zstd_process_bytes_out_max);
ZSTD_outBuffer output = { out_head, process_bytes_out, 0 };
// Call the stream compression; the return value indicates remaining data in internal buffers,
// or an error code, which we need to check.
pending = ZSTD_compressStream2(cctx, &output, &input, flush);
if (ZSTD_isError(pending))
{
// Something went wrong; return the error code
return static_cast<unsigned>(pending);
}
// True number of bytes written
process_bytes_out = output.pos;
out_size -= process_bytes_out;
out_head += process_bytes_out;
// Number of bytes read
bytes_read_from_in += input.pos - prev_pos;
prev_pos = input.pos;
if (out_is_in)
{
// Update number of bytes written to in
bytes_written_to_in += process_bytes_out;
}
// Continue while not all input has been read, or while there is data pending in the internal
// buffers
} while (pending || (input.pos != input.size));
} while (in_size);
if (!out_is_in)
{
// We are done but the last writes were to temp_out
size_t bytes_in_temp_out = temp_out.size() - out_size;
// Resize in to fit the remaining data
in.resize(bytes_written_to_in + bytes_in_temp_out);
// Copy over the buffer to in
out_head = in.begin() + bytes_written_to_in;
memcpy(out_head, temp_out.cbegin(), bytes_in_temp_out);
bytes_written_to_in += bytes_in_temp_out;
}
else
{
// Just resize in to the right size
in.resize(bytes_written_to_in);
}
ZSTD_freeCCtx(cctx);
return ZSTD_error_no_error;
}