in src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java [332:609]
public static VariableCPInstruction parseInstruction ( String str ) {
String[] parts = InstructionUtils.getInstructionPartsWithValueType ( str );
String opcode = parts[0];
VariableOperationCode voc = getVariableOperationCode(opcode);
if ( voc == VariableOperationCode.CreateVariable ){
if ( parts.length < 5 ) //&& parts.length != 10 )
throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
}
else if ( voc == VariableOperationCode.MoveVariable) {
// mvvar tempA A; or mvvar mvar5 "data/out.mtx" "binary"
if ( parts.length !=3 && parts.length != 4)
throw new DMLRuntimeException("Invalid number of operands in mvvar instruction: " + str);
}
else if ( voc == VariableOperationCode.Write ) {
// All write instructions have 3 parameters, except in case of delimited/csv/libsvm file.
// Write instructions for csv files also include three additional parameters (hasHeader, delimiter, sparse)
// Write instructions for libsvm files also include one additional parameters (sparse)
// TODO - replace hardcoded numbers with more sophisticated code
if ( parts.length != 6 && parts.length != 7 && parts.length != 9 )
throw new DMLRuntimeException("Invalid number of operands in write instruction: " + str);
}
else if(voc == VariableOperationCode.CastAsFrameVariable){
InstructionUtils.checkNumFields(parts, 3, 4, 5);
}
else {
try{
if( voc != VariableOperationCode.RemoveVariable )
InstructionUtils.checkNumFields ( parts, getArity(voc) ); // no output
}
catch(Exception e){
throw new DMLRuntimeException("Invalid number of fields with operation code: " + voc, e);
}
}
CPOperand in1=null, in2=null, in3=null, in4=null, out=null;
int k = 1;
switch (voc) {
case CreateVariable:
// variable name
DataType dt = DataType.valueOf(parts[4]);
//TODO choose correct value type for tensor
ValueType vt = dt==DataType.MATRIX ? ValueType.FP64 : ValueType.STRING;
int extSchema = (dt==DataType.FRAME && parts.length>=12) ? 1 : 0;
in1 = new CPOperand(parts[1], vt, dt);
// file name
in2 = new CPOperand(parts[2], ValueType.STRING, DataType.SCALAR);
// file name override flag (always literal)
in3 = new CPOperand(parts[3], ValueType.BOOLEAN, DataType.SCALAR);
// format
String fmt = parts[5];
if ( fmt.equalsIgnoreCase("csv") ) {
// Cretevar instructions for CSV format either has 13 or 14 inputs.
// 13 inputs: createvar corresponding to WRITE -- includes properties hasHeader, delim, and sparse
// 14 inputs: createvar corresponding to READ -- includes properties hasHeader, delim, fill, and fillValue
if ( parts.length < 14+extSchema || parts.length > 16+extSchema )
throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
}
else if(fmt.equalsIgnoreCase("libsvm")) {
// 13 inputs: createvar corresponding to WRITE -- includes properties delim, index delim, and sparse
// 12 inputs: createvar corresponding to READ -- includes properties delim, index delim, and sparse
if(parts.length < 12 + extSchema)
throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
}
else if(fmt.equalsIgnoreCase("hdf5")) {
// 11 inputs: createvar corresponding to WRITE/READ -- includes properties dataset name
if(parts.length < 11 + extSchema)
throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
}
else {
if ( parts.length != 6 && parts.length != 11+extSchema )
throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
}
MetaDataFormat iimd = null;
if (dt == DataType.MATRIX || dt == DataType.FRAME || dt == DataType.LIST) {
DataCharacteristics mc = new MatrixCharacteristics();
if (parts.length == 6) {
// do nothing
}
else if (parts.length >= 10) {
// matrix characteristics
mc.setDimension(Long.parseLong(parts[6]), Long.parseLong(parts[7]));
mc.setBlocksize(Integer.parseInt(parts[8]));
mc.setNonZeros(Long.parseLong(parts[9]));
}
else {
throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
}
iimd = new MetaDataFormat(mc, FileFormat.safeValueOf(fmt));
}
else if (dt == DataType.TENSOR) {
TensorCharacteristics tc = new TensorCharacteristics(new long[]{1, 1}, 0);
if (parts.length == 6) {
// do nothing
}
else if (parts.length >= 10) {
// TODO correct sizes
tc.setDim(0, Long.parseLong(parts[6]));
tc.setDim(1, Long.parseLong(parts[7]));
tc.setBlocksize(Integer.parseInt(parts[8]));
}
else {
throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
}
iimd = new MetaDataFormat(tc, FileFormat.safeValueOf(fmt));
}
UpdateType updateType = UpdateType.COPY;
if ( parts.length >= 11 )
updateType = UpdateType.valueOf(parts[10].toUpperCase());
//handle frame schema
String schema = (dt==DataType.FRAME && parts.length>=12) ? parts[parts.length-1] : null;
if ( fmt.equalsIgnoreCase("csv") ) {
// Cretevar instructions for CSV format either has 13 or 14 inputs.
// 13 inputs: createvar corresponding to WRITE -- includes properties hasHeader, delim, and sparse
// 14 inputs: createvar corresponding to READ -- includes properties hasHeader, delim, fill, and fillValue
FileFormatProperties fmtProperties = null;
int curPos = 11;
if ( parts.length == 14+extSchema ) {
boolean hasHeader = Boolean.parseBoolean(parts[curPos]);
String delim = parts[curPos+1];
boolean sparse = Boolean.parseBoolean(parts[curPos+2]);
fmtProperties = new FileFormatPropertiesCSV(hasHeader, delim, sparse) ;
}
else {
boolean hasHeader = Boolean.parseBoolean(parts[curPos]);
String delim = parts[curPos+1];
boolean fill = Boolean.parseBoolean(parts[curPos+2]);
double fillValue = Double.parseDouble(parts[curPos+3]);
String naStrings = null;
if ( parts.length == 16+extSchema )
naStrings = parts[curPos+4];
fmtProperties = new FileFormatPropertiesCSV(hasHeader, delim, fill, fillValue, naStrings) ;
}
return new VariableCPInstruction(VariableOperationCode.CreateVariable,
in1, in2, in3, iimd, updateType, fmtProperties, schema, opcode, str);
}
else if(fmt.equalsIgnoreCase("libsvm")) {
// Cretevar instructions for LIBSVM format has 13.
// 13 inputs: createvar corresponding to WRITE -- includes properties delim, index delim and sparse
// 12 inputs: createvar corresponding to READ -- includes properties delim, index delim, and sparse
FileFormatProperties fmtProperties = null;
int curPos = 11;
if(parts.length == 12 + extSchema) {
String delim = parts[curPos];
String indexDelim = parts[curPos + 1];
fmtProperties = new FileFormatPropertiesLIBSVM(delim, indexDelim);
}
else {
String delim = parts[curPos];
String indexDelim = parts[curPos + 1];
boolean sparse = Boolean.parseBoolean(parts[curPos + 2]);
fmtProperties = new FileFormatPropertiesLIBSVM(delim, indexDelim, sparse);
}
return new VariableCPInstruction(VariableOperationCode.CreateVariable,
in1, in2, in3, iimd, updateType, fmtProperties, schema, opcode, str);
}
else if(fmt.equalsIgnoreCase("hdf5")) {
// Cretevar instructions for HDF5 format has 13.
// 11 inputs: createvar corresponding to WRITE/READ -- includes properties dataset name
int curPos = 11;
String datasetName = parts[curPos];
FileFormatProperties fmtProperties = new FileFormatPropertiesHDF5(datasetName);
return new VariableCPInstruction(VariableOperationCode.CreateVariable,
in1, in2, in3, iimd, updateType, fmtProperties, schema, opcode, str);
}
else {
return new VariableCPInstruction(VariableOperationCode.CreateVariable, in1, in2, in3, iimd, updateType, schema, opcode, str);
}
case AssignVariable:
in1 = new CPOperand(parts[1]);
in2 = new CPOperand(parts[2]);
break;
case CopyVariable:
// Value types are not given here
boolean withTypes = parts[1].split(VALUETYPE_PREFIX).length > 2 && parts[2].split(VALUETYPE_PREFIX).length > 2;
in1 = withTypes ? new CPOperand(parts[1]) : new CPOperand(parts[1], ValueType.UNKNOWN, DataType.UNKNOWN);
in2 = withTypes ? new CPOperand(parts[2]) : new CPOperand(parts[2], ValueType.UNKNOWN, DataType.UNKNOWN);
break;
case MoveVariable:
in1 = new CPOperand(parts[1], ValueType.UNKNOWN, DataType.UNKNOWN);
in2 = new CPOperand(parts[2], ValueType.UNKNOWN, DataType.UNKNOWN);
if(parts.length > 3)
in3 = new CPOperand(parts[3], ValueType.UNKNOWN, DataType.UNKNOWN);
break;
case RemoveVariable:
VariableCPInstruction rminst = new VariableCPInstruction(
getVariableOperationCode(opcode), null, null, null, out, opcode, str);
for( int i=1; i<parts.length; i++ )
rminst.addInput(new CPOperand(parts[i], ValueType.UNKNOWN, DataType.SCALAR));
return rminst;
case RemoveVariableAndFile:
in1 = new CPOperand(parts[1]);
in2 = new CPOperand(parts[2]);
// second argument must be a boolean
if ( in2.getValueType() != ValueType.BOOLEAN)
throw new DMLRuntimeException("Unexpected value type for second argument in: " + str);
break;
case CastAsFrameVariable:
if(parts.length==5){
in1 = new CPOperand(parts[1]); // input to cast
in2 = new CPOperand(parts[2]); // list of column names
out = new CPOperand(parts[3]); // output
k = Integer.parseInt(parts[4]);
break;
}
case CastAsScalarVariable:
case CastAsMatrixVariable:
case CastAsListVariable:
case CastAsDoubleVariable:
case CastAsIntegerVariable:
case CastAsBooleanVariable:
in1 = new CPOperand(parts[1]); // first operand is a variable name => string value type
out = new CPOperand(parts[2]); // output variable name
k = Integer.parseInt(parts[3]); // thread count
break;
case Write:
in1 = new CPOperand(parts[1]);
in2 = new CPOperand(parts[2]);
in3 = new CPOperand(parts[3]);
FileFormatProperties fprops = null;
if ( in3.getName().equalsIgnoreCase("csv") ) {
boolean hasHeader = Boolean.parseBoolean(parts[4]);
String delim = parts[5];
boolean sparse = Boolean.parseBoolean(parts[6]);
fprops = new FileFormatPropertiesCSV(hasHeader, delim, sparse);
in4 = new CPOperand(parts[7]); // description
}
else if ( in3.getName().equalsIgnoreCase("libsvm") ) {
String delim = parts[4];
String indexDelim = parts[5];
boolean sparse = Boolean.parseBoolean(parts[6]);
fprops = new FileFormatPropertiesLIBSVM(delim, indexDelim, sparse);
}
else if(in3.getName().equalsIgnoreCase("hdf5") ){
String datasetName = parts[4];
fprops = new FileFormatPropertiesHDF5(datasetName);
}
else {
fprops = new FileFormatProperties();
in4 = new CPOperand(parts[5]); // blocksize in empty description
}
VariableCPInstruction inst = new VariableCPInstruction(
getVariableOperationCode(opcode), in1, in2, in3, out, null, fprops, null, null, opcode, str);
inst.addInput(in4);
return inst;
case Read:
in1 = new CPOperand(parts[1]);
in2 = new CPOperand(parts[2]);
break;
case SetFileName:
in1 = new CPOperand(parts[1]); // variable name
in2 = new CPOperand(parts[2], ValueType.UNKNOWN, DataType.UNKNOWN); // file name
in3 = new CPOperand(parts[3], ValueType.UNKNOWN, DataType.UNKNOWN); // option: remote or local
break;
}
return new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, out, opcode, str, k);
}