in unittest/shell_script_tester.cc [563:824]
bool Shell_script_tester::validate(const std::string &context,
const std::string &chunk_id, bool optional) {
std::string original_std_out = output_handler.std_out;
std::string original_std_err = output_handler.std_err;
size_t out_position = 0;
size_t err_position = 0;
std::string validation_id = _chunks[chunk_id].def->validation_id;
if (_chunk_validations.find(validation_id) != _chunk_validations.end()) {
bool expect_failures = false;
// Identifies the validations to be done based on the context
std::vector<std::shared_ptr<Validation>> validations;
for (const auto &val : _chunk_validations[validation_id]) {
bool enabled = false;
try {
enabled = context_enabled(val->def->context);
if (val->def->stream == "PROTOCOL" && !_options->trace_protocol) {
ADD_FAILURE_AT("validation file", val->def->linenum)
<< "ERROR TESTING PROTOCOL: Protocol tracing is disabled."
<< "\n"
<< "\tCHUNK: " << val->def->line << "\n";
}
} catch (const std::invalid_argument &e) {
ADD_FAILURE_AT("validation file", val->def->linenum)
<< "ERROR EVALUATING VALIDATION CONTEXT: " << e.what() << "\n"
<< "\tCHUNK: " << val->def->line << "\n";
}
if (enabled) {
validations.push_back(val);
if (!val->expected_error.empty()) expect_failures = true;
} else {
if (output_handler.internal_std_err.empty()) {
SKIP_VALIDATION(val->def->line);
} else {
SKIP_VALIDATION(val->def->line + ": " +
output_handler.internal_std_err);
}
}
}
std::string full_statement;
// The validations will be performed ONLY if the context is enabled
for (size_t valindex = 0; valindex < validations.size(); valindex++) {
// Validation goes against validation code
if (!validations[valindex]->code.empty()) {
// Before cleaning up, prints any error found on the script execution
if (valindex == 0 && !original_std_err.empty()) {
ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
_chunks[chunk_id].code[0].first)
<< makered("\tUnexpected Error: " + original_std_err) << "\n";
output_handler.wipe_all();
return false;
}
output_handler.wipe_all();
_cout.str("");
_cout.clear();
std::string backup = _custom_context;
full_statement.append(validations[valindex]->code);
_custom_context += "[" + full_statement + "]";
execute(validations[valindex]->code);
_custom_context = backup;
if (_interactive_shell->input_state() == shcore::Input_state::Ok)
full_statement.clear();
else
full_statement.append("\n");
original_std_err = output_handler.std_err;
original_std_out = output_handler.std_out;
out_position = 0;
err_position = 0;
output_handler.wipe_all();
_cout.str("");
_cout.clear();
}
// Validates unexpected error
if (!expect_failures && !original_std_err.empty()) {
ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
_chunks[chunk_id].code[0].first)
<< "while executing chunk: " + _chunks[chunk_id].def->line << "\n"
<< makered("\tUnexpected Error: ") << original_std_err << "\n";
output_handler.wipe_all();
return false;
}
// Validates expected output if any
if (!validations[valindex]->expected_output.empty()) {
std::string out = validations[valindex]->expected_output;
out = resolve_string(out);
if (out != "*") {
if (validations[valindex]->def->validation ==
ValidationType::Simple) {
auto matched =
multi_value_compare(out, original_std_out, false, out_position,
nullptr, &out_position);
if (!matched) {
if (out_position == 0) {
ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
_chunks[chunk_id].code[0].first)
<< "while executing chunk: " + _chunks[chunk_id].def->line
<< "\nwith validation at "
<< validations[valindex]->def->linenum << "\n"
<< makeyellow("\tSTDOUT missing: ") << out << "\n"
<< makeyellow("\tSTDOUT actual: ") + original_std_out
<< "\n";
} else {
ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
_chunks[chunk_id].code[0].first)
<< "while executing chunk: " + _chunks[chunk_id].def->line
<< "\nwith validation at "
<< validations[valindex]->def->linenum << "\n"
<< makeyellow("\tSTDOUT missing: ") << out << "\n"
<< makeyellow("\tSTDOUT actual: ") +
original_std_out.substr(out_position)
<< "\n"
<< makeyellow("\tSTDOUT original: ") + original_std_out
<< "\n";
}
output_handler.wipe_all();
return false;
}
} else {
SCOPED_TRACE(_chunks[chunk_id].source);
if (!validate_line_by_line(
context, chunk_id, "STDOUT", out,
validations[valindex]->def->stream == "PROTOCOL"
? _cout.str()
: original_std_out,
_chunks[chunk_id].code[0].first,
validations[valindex]->def->linenum)) {
output_handler.wipe_all();
return false;
}
}
}
}
// Validates unexpected output if any
if (!validations[valindex]->unexpected_output.empty()) {
std::string out = validations[valindex]->unexpected_output;
out = resolve_string(out);
bool matched = false;
if (validations[valindex]->def->stream == "PROTOCOL")
matched = _cout.str().find(out) != std::string::npos;
else
matched = multi_value_compare(out, original_std_out, false);
if (matched) {
ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
_chunks[chunk_id].code[0].first)
<< "while executing chunk: " + _chunks[chunk_id].def->line << "\n"
<< "with validation at " << validations[valindex]->def->linenum
<< "\n"
<< makeyellow("\tSTDOUT unexpected: ") << out << "\n"
<< makeyellow("\tSTDOUT actual: ") << original_std_out << "\n";
output_handler.wipe_all();
return false;
}
}
// Validates expected error if any
if (!validations[valindex]->expected_error.empty()) {
std::string error = validations[valindex]->expected_error;
error = resolve_string(error);
if (error != "*") {
if (validations[valindex]->def->validation ==
ValidationType::Simple) {
bool matched =
multi_value_compare(error, original_std_err, false,
err_position, nullptr, &err_position);
if (!matched) {
if (err_position == 0) {
ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
_chunks[chunk_id].code[0].first)
<< "while executing chunk: " + _chunks[chunk_id].def->line
<< "\n"
<< "with validation at "
<< validations[valindex]->def->linenum << "\n"
<< makeyellow("\tSTDERR missing: ") + error << "\n"
<< makeyellow("\tSTDERR actual: ") + original_std_err
<< "\n";
} else {
ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
_chunks[chunk_id].code[0].first)
<< "while executing chunk: " + _chunks[chunk_id].def->line
<< "\n"
<< "with validation at "
<< validations[valindex]->def->linenum << "\n"
<< makeyellow("\tSTDERR missing: ") + error << "\n"
<< makeyellow("\tSTDERR actual: ") +
original_std_err.substr(err_position)
<< "\n"
<< makeyellow("\tSTDERR original: ") + original_std_err
<< "\n";
}
output_handler.wipe_all();
return false;
}
} else {
SCOPED_TRACE(_chunks[chunk_id].source);
if (!validate_line_by_line(context, chunk_id, "STDERR", error,
original_std_err,
_chunks[chunk_id].code[0].first,
validations[valindex]->def->linenum)) {
output_handler.wipe_all();
return false;
}
}
}
}
}
if (!optional && validations.empty()) {
ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
_chunks[chunk_id].code[0].first)
<< makered("MISSING VALIDATIONS FOR CHUNK ")
<< _chunks[chunk_id].def->line << "\n"
<< makeyellow("\tSTDOUT: ") << original_std_out << "\n"
<< makeyellow("\tSTDERR: ") << original_std_err << "\n";
return false;
}
output_handler.wipe_all();
_cout.str("");
_cout.clear();
} else {
// There were errors
if (!original_std_err.empty()) {
ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
_chunks[chunk_id].code[0].first)
<< "while executing chunk: " + _chunks[chunk_id].def->line << "\n"
<< makered("\tUnexpected Error: ") + original_std_err << "\n";
} else if (!optional && _chunks.find(chunk_id) != _chunks.end()) {
// The error is that there are no validations
ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
_chunks[chunk_id].code[0].first)
<< makered("MISSING VALIDATIONS FOR CHUNK ")
<< _chunks[chunk_id].def->line << "\n"
<< makeyellow("\tSTDOUT: ") << original_std_out << "\n"
<< makeyellow("\tSTDERR: ") << original_std_err << "\n";
}
output_handler.wipe_all();
_cout.str("");
_cout.clear();
}
return true;
}