in src/mlio/csv_record_tokenizer.cc [24:119]
bool Csv_record_tokenizer::next()
{
value_.clear();
truncated_ = false;
if (finished_) {
eof_ = true;
return false;
}
char chr{};
// Start of a new field.
Parser_state state = Parser_state::new_field;
if (!try_get_next_char(chr)) {
goto end_line; // NOLINT
}
if (chr == delimiter_) {
goto end_field; // NOLINT
}
else if (chr == quote_char_) {
goto in_quoted_field; // NOLINT
}
else {
push_char(chr);
goto in_field; // NOLINT
}
in_field:
state = Parser_state::in_field;
if (!try_get_next_char(chr)) {
goto end_line; // NOLINT
}
if (chr == delimiter_) {
goto end_field; // NOLINT
}
else {
push_char(chr);
goto in_field; // NOLINT
}
in_quoted_field:
state = Parser_state::in_quoted_field;
if (!try_get_next_char(chr)) {
goto end_line; // NOLINT
}
if (chr == quote_char_) {
goto quote_in_quoted_field; // NOLINT
}
else {
push_char(chr);
goto in_quoted_field; // NOLINT
}
quote_in_quoted_field:
state = Parser_state::quote_in_quoted_field;
if (!try_get_next_char(chr)) {
goto end_line; // NOLINT
}
if (chr == delimiter_) {
goto end_field; // NOLINT
}
else if (chr == quote_char_) {
push_char(chr);
goto in_quoted_field; // NOLINT
}
else {
push_char(chr);
goto in_field; // NOLINT
}
end_line:
switch (state) {
case Parser_state::new_field:
case Parser_state::in_field:
case Parser_state::quote_in_quoted_field:
finished_ = true;
break;
case Parser_state::in_quoted_field:
throw Corrupt_record_error{"EOF reached inside a quoted field."};
}
end_field:
return true;
}