in storage/ndb/tools/NdbImportCsv.cpp [1657:2315]
void NdbImportCsv::Eval::eval_field(Row *row, Line *line, Field *field,
const uint attr_id) {
const Opt &opt = m_util.c_opt;
const CHARSET_INFO *cs = opt.m_charset;
const Table &table = m_input.m_table;
const Attrs &attrs = table.m_attrs;
Buf &buf = m_input.m_buf;
uchar *bufdata = &buf.m_data[buf.m_start];
char *bufdatac = (char *)bufdata;
// internal counts file lines and fields from 0
const uint64 lineno = m_input.m_startlineno + line->m_lineno;
const uint fieldno = field->m_fieldno;
// user wants the counts from 1
const uint64 linenr = 1 + lineno;
const uint fieldnr = 1 + fieldno;
const Attr &attr = attrs[attr_id];
uint pos = field->m_pack_pos;
uint end = field->m_pack_end;
uint length = end - pos;
uchar *data = &bufdata[pos];
char *datac = &bufdatac[pos];
/*
* A field is followed by non-empty separator or terminator.
* We null-terminate the field and restore it at end.
*/
uchar saveterm = data[length];
data[length] = 0;
Error error; // local error
/*
* Lots of repeated code here but it is not worth changing
* before it moves to some datatypes library.
*/
switch (attr.m_type) {
case NdbDictionary::Column::Tinyint: {
int err = 0;
const char *endptr = nullptr;
int val = cs->cset->strntol(cs, datac, length, 10, &endptr, &err);
if (err != 0) {
m_util.set_error_data(error, __LINE__, err,
"line %" PRIu64 " field %u: eval %s failed",
linenr, fieldnr, attr.m_sqltype);
break;
}
if (uint(endptr - datac) != length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: bad format",
linenr, fieldnr, attr.m_sqltype);
break;
}
const int minval = -128;
const int maxval = +127;
if (val < minval || val > maxval) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: "
"value %d out of range",
linenr, fieldnr, attr.m_sqltype, val);
break;
}
int8 byteval = val;
attr.set_value(row, &byteval, 1);
} break;
case NdbDictionary::Column::Smallint: {
int err = 0;
const char *endptr = nullptr;
int val = cs->cset->strntol(cs, datac, length, 10, &endptr, &err);
if (err != 0) {
m_util.set_error_data(error, __LINE__, err,
"line %" PRIu64 " field %u: eval %s failed",
linenr, fieldnr, attr.m_sqltype);
break;
}
if (uint(endptr - datac) != length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: bad format",
linenr, fieldnr, attr.m_sqltype);
break;
}
const int minval = -32768;
const int maxval = +32767;
if (val < minval || val > maxval) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: "
"value %d out of range",
linenr, fieldnr, attr.m_sqltype, val);
break;
}
int16 shortval = val;
attr.set_value(row, &shortval, 2);
} break;
case NdbDictionary::Column::Mediumint: {
int err = 0;
const char *endptr = nullptr;
int val = cs->cset->strntol(cs, datac, length, 10, &endptr, &err);
if (err != 0) {
m_util.set_error_data(error, __LINE__, err,
"line %" PRIu64 " field %u: eval %s failed",
linenr, fieldnr, attr.m_sqltype);
break;
}
if (uint(endptr - datac) != length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: bad format",
linenr, fieldnr, attr.m_sqltype);
break;
}
const int minval = -8388608;
const int maxval = +8388607;
if (val < minval || val > maxval) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: "
"value %d out of range",
linenr, fieldnr, attr.m_sqltype, val);
break;
}
uchar val3[3];
int3store(val3, (uint)val);
attr.set_value(row, val3, 3);
} break;
case NdbDictionary::Column::Int: {
int err = 0;
const char *endptr = nullptr;
int32 val = cs->cset->strntol(cs, datac, length, 10, &endptr, &err);
if (err != 0) {
m_util.set_error_data(error, __LINE__, err,
"line %" PRIu64 " field %u: eval %s failed",
linenr, fieldnr, attr.m_sqltype);
break;
}
if (uint(endptr - datac) != length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: bad format",
linenr, fieldnr, attr.m_sqltype);
break;
}
attr.set_value(row, &val, 4);
} break;
case NdbDictionary::Column::Bigint: {
int err = 0;
const char *endptr = nullptr;
int64 val = cs->cset->strntoll(cs, datac, length, 10, &endptr, &err);
if (err != 0) {
m_util.set_error_data(error, __LINE__, err,
"line %" PRIu64 " field %u: eval %s failed",
linenr, fieldnr, attr.m_sqltype);
break;
}
if (uint(endptr - datac) != length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: bad format",
linenr, fieldnr, attr.m_sqltype);
break;
}
attr.set_value(row, &val, 8);
} break;
case NdbDictionary::Column::Tinyunsigned: {
int err = 0;
const char *endptr = nullptr;
uint val = cs->cset->strntoul(cs, datac, length, 10, &endptr, &err);
if (err != 0) {
m_util.set_error_data(error, __LINE__, err,
"line %" PRIu64 " field %u: eval %s failed",
linenr, fieldnr, attr.m_sqltype);
break;
}
if (uint(endptr - datac) != length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: bad format",
linenr, fieldnr, attr.m_sqltype);
break;
}
const uint maxval = 255;
if (val > maxval) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: "
"value %u out of range",
linenr, fieldnr, attr.m_sqltype, val);
break;
}
uint8 byteval = val;
attr.set_value(row, &byteval, 1);
} break;
case NdbDictionary::Column::Smallunsigned: {
int err = 0;
const char *endptr = nullptr;
uint val = cs->cset->strntoul(cs, datac, length, 10, &endptr, &err);
if (err != 0) {
m_util.set_error_data(error, __LINE__, err,
"line %" PRIu64 " field %u: eval %s failed",
linenr, fieldnr, attr.m_sqltype);
break;
}
if (uint(endptr - datac) != length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: bad format",
linenr, fieldnr, attr.m_sqltype);
break;
}
const uint maxval = 65535;
if (val > maxval) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: "
"value %u out of range",
linenr, fieldnr, attr.m_sqltype, val);
break;
}
uint16 shortval = val;
attr.set_value(row, &shortval, 2);
} break;
case NdbDictionary::Column::Mediumunsigned: {
int err = 0;
const char *endptr = nullptr;
uint val = cs->cset->strntoul(cs, datac, length, 10, &endptr, &err);
if (err != 0) {
m_util.set_error_data(error, __LINE__, err,
"line %" PRIu64 " field %u: eval %s failed",
linenr, fieldnr, attr.m_sqltype);
break;
}
if (uint(endptr - datac) != length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: bad format",
linenr, fieldnr, attr.m_sqltype);
break;
}
const uint maxval = 16777215;
if (val > maxval) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: "
"value %u out of range",
linenr, fieldnr, attr.m_sqltype, val);
break;
}
uchar val3[3];
int3store(val3, val);
attr.set_value(row, val3, 3);
} break;
case NdbDictionary::Column::Unsigned: {
int err = 0;
const char *endptr = 0;
uint32 val = (uint32)cs->cset->strntoull10rnd(cs, datac, length, true,
&endptr, &err);
if (err == MY_ERRNO_ERANGE) log_debug(1, "Value out of range.");
if (err != 0) {
m_util.set_error_data(error, __LINE__, err,
"line %" PRIu64 " field %u: eval %s failed",
linenr, fieldnr, attr.m_sqltype);
break;
}
if (uint(endptr - datac) != length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: bad format",
linenr, fieldnr, attr.m_sqltype);
break;
}
attr.set_value(row, &val, 4);
} break;
case NdbDictionary::Column::Bigunsigned: {
int err = 0;
const char *endptr = 0;
uint64 val = (uint64)cs->cset->strntoull10rnd(cs, datac, length, true,
&endptr, &err);
if (err == MY_ERRNO_ERANGE) log_debug(1, "Value out of range.");
if (err != 0) {
m_util.set_error_data(error, __LINE__, err,
"line %" PRIu64 " field %u: eval %s failed",
linenr, fieldnr, attr.m_sqltype);
break;
}
if (uint(endptr - datac) != length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: bad format",
linenr, fieldnr, attr.m_sqltype);
break;
}
attr.set_value(row, &val, 8);
} break;
case NdbDictionary::Column::Decimal: {
uchar val[200];
Ndb_import_csv_error csv_error;
if (!ndb_import_csv_parse_decimal(attr, false, datac, length, val,
sizeof(val), csv_error)) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: %s at %d",
linenr, fieldnr, attr.m_sqltype,
csv_error.error_text, csv_error.error_line);
break;
}
attr.set_value(row, val, attr.m_size);
} break;
case NdbDictionary::Column::Decimalunsigned: {
uchar val[200];
Ndb_import_csv_error csv_error;
if (!ndb_import_csv_parse_decimal(attr, true, datac, length, val,
sizeof(val), csv_error)) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: %s at %d",
linenr, fieldnr, attr.m_sqltype,
csv_error.error_text, csv_error.error_line);
break;
}
attr.set_value(row, val, attr.m_size);
} break;
/*
* Float and Double. We use same methods as LOAD DATA but for
* some reason there are occasional infinitesimal diffs on "el6".
* Fix by using ::strtod if charset allows (it does).
*/
case NdbDictionary::Column::Float: {
uint data_length;
double val = 0.0;
bool use_os_strtod =
#ifndef _WIN32
(opt.m_charset == &my_charset_bin);
#else
false;
#endif
if (use_os_strtod) {
errno = 0;
char *endptr = nullptr;
val = ::strtod(datac, &endptr);
data_length = endptr - datac;
if (errno != 0) {
m_util.set_error_data(error, __LINE__, errno,
"line %" PRIu64 " field %u: eval %s failed",
linenr, fieldnr, attr.m_sqltype);
break;
}
} else {
int err = 0;
const char *endptr = nullptr;
val = cs->cset->strntod(cs, datac, length, &endptr, &err);
data_length = endptr - datac;
if (err != 0) {
m_util.set_error_data(error, __LINE__, err,
"line %" PRIu64 " field %u: eval %s failed",
linenr, fieldnr, attr.m_sqltype);
break;
}
}
if (data_length != length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: bad format",
linenr, fieldnr, attr.m_sqltype);
break;
}
if (std::isnan(val)) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: invalid value",
linenr, fieldnr, attr.m_sqltype);
break;
}
const double max_val = FLT_MAX;
if (val < -max_val || val > max_val) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: value out of range",
linenr, fieldnr, attr.m_sqltype);
break;
}
float valf = (float)val;
attr.set_value(row, &valf, 4);
} break;
case NdbDictionary::Column::Double: {
int err = 0;
uint data_length;
double val = 0.0;
bool use_os_strtod =
#ifndef _WIN32
(opt.m_charset == &my_charset_bin);
#else
false;
#endif
if (use_os_strtod) {
errno = 0;
char *endptr = nullptr;
val = ::strtod(datac, &endptr);
data_length = endptr - datac;
if (errno != 0) {
m_util.set_error_data(error, __LINE__, errno,
"line %" PRIu64 " field %u: eval %s failed",
linenr, fieldnr, attr.m_sqltype);
break;
}
} else {
const char *endptr = nullptr;
val = cs->cset->strntod(cs, datac, length, &endptr, &err);
data_length = endptr - datac;
if (err != 0) {
m_util.set_error_data(error, __LINE__, err,
"line %" PRIu64 " field %u: eval %s failed",
linenr, fieldnr, attr.m_sqltype);
break;
}
}
if (data_length != length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: bad format",
linenr, fieldnr, attr.m_sqltype);
break;
}
if (std::isnan(val)) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: invalid value",
linenr, fieldnr, attr.m_sqltype);
break;
}
const double max_val = DBL_MAX;
if (val < -max_val || val > max_val) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: value out of range",
linenr, fieldnr, attr.m_sqltype);
break;
}
attr.set_value(row, &val, 8);
} break;
case NdbDictionary::Column::Char: {
const char *val = datac;
if (length > attr.m_length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: "
"byte length too long (%u > %u)",
linenr, fieldnr, attr.m_sqltype, length,
attr.m_length);
break;
}
attr.set_value(row, val, length);
} break;
case NdbDictionary::Column::Varchar: {
const char *val = datac;
if (length > attr.m_length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: "
"byte length too long (%u > %u)",
linenr, fieldnr, attr.m_sqltype, length,
attr.m_length);
break;
}
attr.set_value(row, val, length);
} break;
case NdbDictionary::Column::Longvarchar: {
const char *val = datac;
if (length > attr.m_length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: "
"byte length too long (%u > %u)",
linenr, fieldnr, attr.m_sqltype, length,
attr.m_length);
break;
}
attr.set_value(row, val, length);
} break;
case NdbDictionary::Column::Binary: {
const char *val = datac;
if (length > attr.m_length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: "
"length too long (%u > %u)",
linenr, fieldnr, attr.m_sqltype, length,
attr.m_length);
break;
}
attr.set_value(row, val, length);
} break;
case NdbDictionary::Column::Varbinary: {
const char *val = datac;
if (length > attr.m_length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: "
"length too long (%u > %u)",
linenr, fieldnr, attr.m_sqltype, length,
attr.m_length);
break;
}
attr.set_value(row, val, length);
} break;
case NdbDictionary::Column::Longvarbinary: {
const char *val = datac;
if (length > attr.m_length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: "
"length too long (%u > %u)",
linenr, fieldnr, attr.m_sqltype, length,
attr.m_length);
break;
}
attr.set_value(row, val, length);
} break;
case NdbDictionary::Column::Bit: {
require(attr.m_length <= 64);
uint bytelength = (attr.m_length + 7) / 8;
require(bytelength <= 8);
uchar val[8];
memset(val, 0, sizeof(val));
uint i = 0;
uint j = Inval_uint; // highest non-zero byte
while (i < length) {
uchar b = data[length - 1 - i];
if (b != 0) j = i;
if (i < bytelength) val[i] = b;
i++;
}
if (j != Inval_uint) {
uint k = 8; // highest bit at j
while (k != 0) {
k--;
if ((data[length - 1 - j] & (1 << k)) != 0) break;
}
uint hibit = 8 * (length - 1 - j) + k;
if (hibit >= attr.m_length) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: "
"highest set bit %u out of range",
linenr, fieldnr, attr.m_sqltype, hibit);
break;
}
}
#if defined(WORDS_BIGENDIAN)
std::swap(val[0], val[3]);
std::swap(val[1], val[2]);
std::swap(val[4], val[7]);
std::swap(val[5], val[6]);
#endif
attr.set_value(row, val, attr.m_size);
} break;
case NdbDictionary::Column::Year: {
NdbSqlUtil::Year s;
Ndb_import_csv_error csv_error;
if (!ndb_import_csv_parse_year(attr, datac, s, csv_error)) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: %s at %d",
linenr, fieldnr, attr.m_sqltype,
csv_error.error_text, csv_error.error_line);
break;
}
uchar val[1];
NdbSqlUtil::pack_year(s, val);
attr.set_value(row, val, 1);
} break;
case NdbDictionary::Column::Date: {
NdbSqlUtil::Date s;
Ndb_import_csv_error csv_error;
if (!ndb_import_csv_parse_date(attr, datac, s, csv_error)) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: %s at %d",
linenr, fieldnr, attr.m_sqltype,
csv_error.error_text, csv_error.error_line);
break;
}
uchar val[3];
NdbSqlUtil::pack_date(s, val);
attr.set_value(row, val, 3);
} break;
case NdbDictionary::Column::Time2: {
NdbSqlUtil::Time2 s;
Ndb_import_csv_error csv_error;
if (!ndb_import_csv_parse_time2(attr, datac, s, csv_error)) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: %s at %d",
linenr, fieldnr, attr.m_sqltype,
csv_error.error_text, csv_error.error_line);
break;
}
uint prec = attr.m_precision;
require(prec <= 6);
uint flen = (1 + prec) / 2;
uint len = 3 + flen;
require(len <= 6);
uchar val[6];
NdbSqlUtil::pack_time2(s, val, prec);
attr.set_value(row, val, len);
} break;
case NdbDictionary::Column::Datetime2: {
NdbSqlUtil::Datetime2 s;
Ndb_import_csv_error csv_error;
if (!ndb_import_csv_parse_datetime2(attr, datac, s, csv_error)) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: %s at %d",
linenr, fieldnr, attr.m_sqltype,
csv_error.error_text, csv_error.error_line);
break;
}
uint prec = attr.m_precision;
require(prec <= 6);
uint flen = (1 + prec) / 2;
uint len = 5 + flen;
require(len <= 8);
uchar val[8];
NdbSqlUtil::pack_datetime2(s, val, prec);
attr.set_value(row, val, len);
} break;
case NdbDictionary::Column::Timestamp2: {
NdbSqlUtil::Timestamp2 s;
Ndb_import_csv_error csv_error;
if (!ndb_import_csv_parse_timestamp2(attr, datac, s, csv_error)) {
m_util.set_error_data(error, __LINE__, 0,
"line %" PRIu64
" field %u: eval %s failed: %s at %d",
linenr, fieldnr, attr.m_sqltype,
csv_error.error_text, csv_error.error_line);
break;
}
uint prec = attr.m_precision;
require(prec <= 6);
uint flen = (1 + prec) / 2;
uint len = 4 + flen;
require(len <= 7);
uchar val[7];
NdbSqlUtil::pack_timestamp2(s, val, prec);
attr.set_value(row, val, len);
} break;
case NdbDictionary::Column::Blob:
case NdbDictionary::Column::Text: {
const char *val = datac;
attr.set_blob(row, val, length);
} break;
default:
require(false);
break;
}
data[length] = saveterm;
if (m_util.has_error(error)) {
m_input.reject_line(line, field, error);
line->m_reject = true;
}
}