private unsafe void LoadDataFromFile()

in cs/benchmark/TestLoader.cs [147:298]
124 lines of code
24 McCabe index (conditional complexity)

        private unsafe void LoadDataFromFile<TKey, TKeySetter>(string filePath, string distribution, TKey[] init_keys, TKey[] txn_keys, TKeySetter keySetter)
            where TKeySetter : IKeySetter<TKey>
        {
            string init_filename = filePath + "/load_" + distribution + "_250M_raw.dat";
            string txn_filename = filePath + "/run_" + distribution + "_250M_1000M_raw.dat";

            var sw = Stopwatch.StartNew();

            if (this.Options.UseSmallData)
            {
                Console.WriteLine($"loading subset of keys and txns from {txn_filename} into memory...");
                using FileStream stream = File.Open(txn_filename, FileMode.Open, FileAccess.Read, FileShare.Read);
                byte[] chunk = new byte[YcsbConstants.kFileChunkSize];
                GCHandle chunk_handle = GCHandle.Alloc(chunk, GCHandleType.Pinned);
                byte* chunk_ptr = (byte*)chunk_handle.AddrOfPinnedObject();

                var initValueSet = new HashSet<long>(init_keys.Length);

                long init_count = 0;
                long txn_count = 0;

                long offset = 0;

                while (true)
                {
                    stream.Position = offset;
                    int size = stream.Read(chunk, 0, YcsbConstants.kFileChunkSize);
                    for (int idx = 0; idx < size && txn_count < txn_keys.Length; idx += 8)
                    {
                        var value = *(long*)(chunk_ptr + idx);
                        if (!initValueSet.Contains(value))
                        {
                            if (init_count >= init_keys.Length)
                            {
                                if (distribution == YcsbConstants.ZipfDist)
                                    continue;

                                // Uniform distribution at current small-data counts is about a 1% hit rate, which is too slow here, so just modulo.
                                value %= init_keys.Length;
                            }
                            else
                            {
                                initValueSet.Add(value);
                                keySetter.Set(init_keys, init_count, value);
                                ++init_count;
                            }
                        }
                        keySetter.Set(txn_keys, txn_count, value);
                        ++txn_count;
                    }
                    if (size == YcsbConstants.kFileChunkSize)
                        offset += YcsbConstants.kFileChunkSize;
                    else
                        break;

                    if (txn_count == txn_keys.Length)
                        break;
                }

                sw.Stop();
                chunk_handle.Free();

                if (init_count != init_keys.Length)
                    throw new InvalidDataException($"Init file subset load fail! Expected {init_keys.Length} keys; found {init_count}");
                if (txn_count != txn_keys.Length)
                    throw new InvalidDataException($"Txn file subset load fail! Expected {txn_keys.Length} keys; found {txn_count}");

                Console.WriteLine($"loaded {init_keys.Length:N0} keys and {txn_keys.Length:N0} txns in {(double)sw.ElapsedMilliseconds / 1000:N3} seconds");
                return;
            }

            Console.WriteLine($"loading all keys from {init_filename} into memory...");
            long count = 0;

            using (FileStream stream = File.Open(init_filename, FileMode.Open, FileAccess.Read, FileShare.Read))
            {
                byte[] chunk = new byte[YcsbConstants.kFileChunkSize];
                GCHandle chunk_handle = GCHandle.Alloc(chunk, GCHandleType.Pinned);
                byte* chunk_ptr = (byte*)chunk_handle.AddrOfPinnedObject();

                long offset = 0;

                while (true)
                {
                    stream.Position = offset;
                    int size = stream.Read(chunk, 0, YcsbConstants.kFileChunkSize);
                    for (int idx = 0; idx < size; idx += 8)
                    {
                        keySetter.Set(init_keys, count, *(long*)(chunk_ptr + idx));
                        ++count;
                        if (count == init_keys.Length)
                            break;
                    }
                    if (size == YcsbConstants.kFileChunkSize)
                        offset += YcsbConstants.kFileChunkSize;
                    else
                        break;

                    if (count == init_keys.Length)
                        break;
                }

                chunk_handle.Free();

                if (count != init_keys.Length)
                    throw new InvalidDataException($"Init file load fail! Expected {init_keys.Length} keys; found {count}");
            }

            sw.Stop();
            Console.WriteLine($"loaded {init_keys.Length:N0} keys in {(double)sw.ElapsedMilliseconds / 1000:N3} seconds");

            Console.WriteLine($"loading all txns from {txn_filename} into memory...");
            sw.Restart();

            using (FileStream stream = File.Open(txn_filename, FileMode.Open, FileAccess.Read, FileShare.Read))
            {
                byte[] chunk = new byte[YcsbConstants.kFileChunkSize];
                GCHandle chunk_handle = GCHandle.Alloc(chunk, GCHandleType.Pinned);
                byte* chunk_ptr = (byte*)chunk_handle.AddrOfPinnedObject();

                count = 0;
                long offset = 0;

                while (true)
                {
                    stream.Position = offset;
                    int size = stream.Read(chunk, 0, YcsbConstants.kFileChunkSize);
                    for (int idx = 0; idx < size; idx += 8)
                    {
                        keySetter.Set(txn_keys, count, *(long*)(chunk_ptr + idx));
                        ++count;
                        if (count == txn_keys.Length)
                            break;
                    }
                    if (size == YcsbConstants.kFileChunkSize)
                        offset += YcsbConstants.kFileChunkSize;
                    else
                        break;

                    if (count == txn_keys.Length)
                        break;
                }

                chunk_handle.Free();

                if (count != txn_keys.Length)
                    throw new InvalidDataException($"Txn file load fail! Expected {txn_keys.Length} keys; found {count}");
            }

            sw.Stop();
            Console.WriteLine($"loaded {txn_keys.Length:N0} txns in {(double)sw.ElapsedMilliseconds / 1000:N3} seconds");
        }