public long process()

in ytdb/src/main/java/com/youtrackdb/ldbc/ytdb/loader/CsvProcessor.java [26:63]

29 lines of code
5 McCabe index (conditional complexity)


    public long process(Path csvFile,
                       Function<String[], T> parser,
                       Consumer<List<T>> batchConsumer) throws IOException {

        log.debug("Processing CSV file: {}", csvFile.getFileName());

        try (Stream<String> lines = Files.lines(csvFile)) {
            var batch = new ArrayList<T>(batchSize);
            long count = 0;

            // Skip header and process lines
            var iterator = lines.skip(1).iterator();

            while (iterator.hasNext()) {
                String line = iterator.next();
                String[] fields = line.split(DELIMITER, -1);

                try {
                    T record = parser.apply(fields);
                    batch.add(record);
                    count++;

                    if (batch.size() >= batchSize) {
                        batchConsumer.accept(List.copyOf(batch));
                        batch.clear();
                    }
                } catch (Exception e) {
                    log.warn("Failed to parse line in {}: {}", csvFile.getFileName(), e.getMessage());
                }
            }

            if (!batch.isEmpty()) {
                batchConsumer.accept(List.copyOf(batch));
            }

            return count;
        }
    }