in src/main/java/org/apache/sysds/hops/recompile/Recompiler.java [1321:1491]
public static void rUpdateStatistics( Hop hop, LocalVariableMap vars )
{
if( hop.isVisited() )
return;
//recursively process children
if( hop.getInput() != null )
for( Hop c : hop.getInput() )
rUpdateStatistics(c, vars);
//update statistics for transient reads according to current statistics
//(with awareness not to override persistent reads to an existing name)
if( HopRewriteUtils.isData(hop, OpOpData.TRANSIENTREAD) ) {
DataOp d = (DataOp) hop;
String varName = d.getName();
if( vars.keySet().contains( varName ) ) {
Data dat = vars.get(varName);
if( dat instanceof MatrixObject ) {
MatrixObject mo = (MatrixObject) dat;
d.setDim1(mo.getNumRows());
d.setDim2(mo.getNumColumns());
d.setNnz(mo.getNnz());
if(mo.isCompressed()){
d.setCompressedSize(mo.getCompressedSize());
d.setCompressedOutput(true);
}
}
else if( dat instanceof FrameObject ) {
FrameObject fo = (FrameObject) dat;
d.setDim1(fo.getNumRows());
d.setDim2(fo.getNumColumns());
}
else if( dat instanceof ListObject ) {
ListObject lo = (ListObject) dat;
d.setDim1(lo.getLength());
d.setDim2(1);
}
else if( dat instanceof TensorObject) {
TensorObject to = (TensorObject) dat;
// TODO: correct dimensions
d.setDim1(to.getNumRows());
d.setDim2(to.getNumColumns());
d.setNnz(to.getNnz());
}
if( dat instanceof CacheableData<?> ) {
CacheableData<?> cd = (CacheableData<?>) dat;
d.setOnlyRDD(!cd.isCached(true) &&cd.getRDDHandle()!=null);
}
}
}
//special case for persistent reads with unknown size (read-after-write)
else if( HopRewriteUtils.isData(hop, OpOpData.PERSISTENTREAD)
&& !hop.dimsKnown() && ((DataOp)hop).getFileFormat()!=FileFormat.CSV
&& !ConfigurationManager.getCompilerConfigFlag(ConfigType.IGNORE_READ_WRITE_METADATA) )
{
//update hop with read meta data
DataOp dop = (DataOp) hop;
tryReadMetaDataFileDataCharacteristics(dop);
}
//update size expression for rand/seq according to symbol table entries
else if ( hop instanceof DataGenOp )
{
DataGenOp d = (DataGenOp) hop;
Map<String,Integer> params = d.getParamIndexMap();
if ( d.getOp() == OpOpDG.RAND || d.getOp()==OpOpDG.SINIT
|| d.getOp() == OpOpDG.SAMPLE || d.getOp() == OpOpDG.FRAMEINIT )
{
boolean initUnknown = !d.dimsKnown();
// TODO refresh tensor size information
if (params.containsKey(DataExpression.RAND_ROWS) && params.containsKey(DataExpression.RAND_COLS)) {
int ix1 = params.get(DataExpression.RAND_ROWS);
int ix2 = params.get(DataExpression.RAND_COLS);
//update rows/cols by evaluating simple expression of literals, nrow, ncol, scalars, binaryops
Map<Long, Long> memo = new HashMap<>();
d.refreshRowsParameterInformation(d.getInput().get(ix1), vars, memo);
d.refreshColsParameterInformation(d.getInput().get(ix2), vars, memo);
if( !(initUnknown & d.dimsKnown()) )
d.refreshSizeInformation();
}
}
else if ( d.getOp() == OpOpDG.SEQ )
{
boolean initUnknown = !d.dimsKnown();
int ix1 = params.get(Statement.SEQ_FROM);
int ix2 = params.get(Statement.SEQ_TO);
int ix3 = params.get(Statement.SEQ_INCR);
Map<Long, Double> memo = new HashMap<>();
double from = Hop.computeBoundsInformation(d.getInput().get(ix1), vars, memo);
double to = Hop.computeBoundsInformation(d.getInput().get(ix2), vars, memo);
double incr = Hop.computeBoundsInformation(d.getInput().get(ix3), vars, memo);
//special case increment
if ( from!=Double.MAX_VALUE && to!=Double.MAX_VALUE ) {
incr *= ((from > to && incr > 0) || (from < to && incr < 0)) ? -1.0 : 1.0;
}
if ( from!=Double.MAX_VALUE && to!=Double.MAX_VALUE && incr!=Double.MAX_VALUE ) {
d.setDim1( UtilFunctions.getSeqLength(from, to, incr) );
d.setDim2( 1 );
d.setIncrementValue( incr );
}
if( !(initUnknown & d.dimsKnown()) )
d.refreshSizeInformation();
}
else if (d.getOp() == OpOpDG.TIME) {
d.refreshSizeInformation();
}
else {
throw new DMLRuntimeException("Unexpected data generation method: " + d.getOp());
}
}
//update size expression for reshape according to symbol table entries
else if( HopRewriteUtils.isReorg(hop, ReOrgOp.RESHAPE) ) {
if (hop.getDataType() != DataType.TENSOR) {
hop.refreshSizeInformation(); //update incl reset
if (!hop.dimsKnown()) {
Map<Long, Long> memo = new HashMap<>();
hop.refreshRowsParameterInformation(hop.getInput().get(1), vars, memo);
hop.refreshColsParameterInformation(hop.getInput().get(2), vars, memo);
}
} else {
//TODO tensor rewrite
}
}
//update size expression for indexing according to symbol table entries
else if( hop instanceof IndexingOp ) {
hop.refreshSizeInformation(); //update, incl reset
if( !hop.dimsKnown() ) {
if( hop.getDataType().isList()
&& hop.getInput().get(1).getValueType() == ValueType.STRING ) {
hop.setDim1(1);
hop.setDim2(1);
}
else {
Map<Long, Double> memo = new HashMap<>();
double rl = Hop.computeBoundsInformation(hop.getInput().get(1), vars, memo);
double ru = Hop.computeBoundsInformation(hop.getInput().get(2), vars, memo);
double cl = Hop.computeBoundsInformation(hop.getInput().get(3), vars, memo);
double cu = Hop.computeBoundsInformation(hop.getInput().get(4), vars, memo);
if( rl!=Double.MAX_VALUE && ru!=Double.MAX_VALUE )
hop.setDim1( (long)(ru-rl+1) );
if( cl!=Double.MAX_VALUE && cu!=Double.MAX_VALUE )
hop.setDim2( (long)(cu-cl+1) );
}
}
}
else if(HopRewriteUtils.isUnary(hop, OpOp1.CAST_AS_MATRIX)
&& hop.getInput(0) instanceof IndexingOp && hop.getInput(0).getDataType().isList()
&& HopRewriteUtils.isData(hop.getInput(0).getInput(0), OpOpData.TRANSIENTREAD) ) {
Data ldat = vars.get(hop.getInput(0).getInput(0).getName()); //list, or matrix during IPA
Hop rix = hop.getInput(0);
if( ldat != null && ldat instanceof ListObject
&& rix.getInput(1) instanceof LiteralOp
&& rix.getInput(2) instanceof LiteralOp
&& HopRewriteUtils.isEqualValue(rix.getInput(1), rix.getInput(2))) {
ListObject list = (ListObject) ldat;
MatrixObject mo = (MatrixObject) ((rix.getInput(1).getValueType() == ValueType.STRING) ?
list.getData(((LiteralOp)rix.getInput(1)).getStringValue()) :
list.getData((int)HopRewriteUtils.getIntValueSafe(rix.getInput(1))-1));
hop.setDim1(mo.getNumRows());
hop.setDim2(mo.getNumColumns());
}
}
else {
//propagate statistics along inner nodes of DAG,
//without overwriting inferred size expressions
hop.refreshSizeInformation();
}
hop.setVisited();
}