in rally-custom/custom_tracks/elasticsearch/openai_vector/_tools/parse_queries.py [0:0]
def output_queries(input_filename: str, queries_file: typing.TextIO):
with pa.memory_map(input_filename, "rb") as source:
query_table = pa.ipc.open_stream(source).read_all()
for record_batch in query_table.to_batches(max_chunksize=BATCH_SIZE):
query_list = record_batch.column(QUERY_COLUMN)
for query in query_list:
queries_file.write(json.dumps(query.as_py()))
queries_file.write("\n")