in src/main/java/org/apache/sysds/runtime/compress/workload/WorkloadAnalyzer.java [382:660]
private void createOp(Hop hop, AWTreeNode parent) {
if(hop.getDataType().isMatrix()) {
Op o = null;
if(HopRewriteUtils.isData(hop, OpOpData.PERSISTENTREAD, OpOpData.TRANSIENTREAD))
return;
else if(HopRewriteUtils.isData(hop, OpOpData.TRANSIENTWRITE, OpOpData.PERSISTENTWRITE)) {
transientCompressed.put(hop.getName(), hop.getInput(0).getHopID());
compressed.add(hop.getHopID());
o = new OpMetadata(hop, hop.getInput(0));
if(isOverlapping(hop.getInput(0)))
o.setOverlapping();
}
else if(hop instanceof ReorgOp && ((ReorgOp) hop).getOp() == ReOrgOp.TRANS) {
transposed.add(hop.getHopID());
compressed.add(hop.getHopID());
// hack add to transient compressed since the decompression is marking the parents.
transientCompressed.put(hop.getName(), hop.getHopID());
// transientCompressed.add(hop.getName());
o = new OpMetadata(hop, hop.getInput(0));
if(isOverlapping(hop.getInput(0)))
o.setOverlapping();
}
else if(hop instanceof AggUnaryOp) {
if((isOverlapping(hop.getInput().get(0)) && !HopRewriteUtils.isAggUnaryOp(hop, AggOp.SUM, AggOp.MEAN)) ||
HopRewriteUtils.isAggUnaryOp(hop, AggOp.TRACE)) {
setDecompressionOnAllInputs(hop, parent);
return;
}
else {
boolean compressedOut = false;
Hop parentHop = hop.getInput(0);
if(HopRewriteUtils.isBinary(parentHop, OpOp2.EQUAL, OpOp2.NOTEQUAL, OpOp2.LESS,
OpOp2.LESSEQUAL, OpOp2.GREATER, OpOp2.GREATEREQUAL)){
Hop leftIn = parentHop.getInput(0);
Hop rightIn = parentHop.getInput(1);
// input ops might be not in the current statement block -> check for transient reads
if(HopRewriteUtils.isAggUnaryOp(leftIn, AggOp.MIN, AggOp.MAX) ||
HopRewriteUtils.isAggUnaryOp(rightIn, AggOp.MIN, AggOp.MAX) ||
checkTransientRead(hop, leftIn) ||
checkTransientRead(hop, rightIn)
)
compressedOut = true;
}
o = new OpNormal(hop, compressedOut);
}
}
else if(hop instanceof UnaryOp) {
if(!HopRewriteUtils.isUnary(hop, OpOp1.MULT2, OpOp1.MINUS1_MULT, OpOp1.MINUS_RIGHT, OpOp1.CAST_AS_MATRIX)) {
if(isOverlapping(hop.getInput(0))) {
treeLookup.get(hop.getInput(0).getHopID()).setDecompressing();
return;
}
}
else if(HopRewriteUtils.isUnary(hop, OpOp1.DETECTSCHEMA)) {
o = new OpNormal(hop, false);
}
}
else if(hop instanceof AggBinaryOp) {
AggBinaryOp agbhop = (AggBinaryOp) hop;
List<Hop> in = agbhop.getInput();
boolean transposedLeft = transposed.contains(in.get(0).getHopID());
boolean transposedRight = transposed.contains(in.get(1).getHopID());
boolean left = compressed.contains(in.get(0).getHopID()) ||
transientCompressed.containsKey(in.get(0).getName());
boolean right = compressed.contains(in.get(1).getHopID()) ||
transientCompressed.containsKey(in.get(1).getName());
OpSided ret = new OpSided(hop, left, right, transposedLeft, transposedRight);
if(ret.isRightMM()) {
overlapping.add(hop.getHopID());
ret.setOverlapping();
if(!ret.isCompressedOutput())
ret.setDecompressing();
}
o = ret;
}
else if(hop instanceof BinaryOp) {
if(HopRewriteUtils.isBinary(hop, OpOp2.CBIND)) {
List<Hop> in = hop.getInput();
o = new OpNormal(hop, true);
if(isOverlapping(in.get(0)) || isOverlapping(in.get(1))) {
overlapping.add(hop.getHopID());
o.setOverlapping();
}
// assume that CBind have to decompress, but only such that it also have the compressed version
// available. Therefore add a new OpNormal, set to decompressing.
o.setDecompressing();
}
else if(HopRewriteUtils.isBinary(hop, OpOp2.RBIND)) {
setDecompressionOnAllInputs(hop, parent);
return;
}
else if(HopRewriteUtils.isBinary(hop, OpOp2.APPLY_SCHEMA)) {
o = new OpNormal(hop, true);
}
else {
List<Hop> in = hop.getInput();
final boolean ol0 = isOverlapping(in.get(0));
final boolean ol1 = isOverlapping(in.get(1));
final boolean ol = ol0 || ol1;
// shortcut instead of comparing to MatrixScalar or RowVector.
if(in.get(1).getDim1() == 1 || in.get(1).isScalar() || in.get(0).isScalar()) {
if(ol && HopRewriteUtils.isBinary(hop, OpOp2.PLUS, OpOp2.MULT, OpOp2.DIV, OpOp2.MINUS)) {
overlapping.add(hop.getHopID());
o = new OpNormal(hop, true);
o.setOverlapping();
}
else if(ol) {
if(in.get(0) != null) {
Op oo = treeLookup.get(in.get(0).getHopID());
if(oo != null)
oo.setDecompressing();
}
return;
}
else {
o = new OpNormal(hop, true);
}
if(!HopRewriteUtils.isBinarySparseSafe(hop))
o.setDensifying();
} else if(HopRewriteUtils.isBinaryMatrixColVectorOperation(hop) ) {
Hop leftIn = hop.getInput(0);
Hop rightIn = hop.getInput(1);
if(HopRewriteUtils.isBinary(hop, OpOp2.DIV) && rightIn instanceof AggUnaryOp && leftIn == rightIn.getInput(0)){
o = new OpNormal(hop, true);
} else {
setDecompressionOnAllInputs(hop, parent);
return;
}
}
else if(HopRewriteUtils.isBinaryMatrixMatrixOperation(hop) ||
HopRewriteUtils.isBinaryMatrixMatrixOperationWithSharedInput(hop)) {
setDecompressionOnAllInputs(hop, parent);
return;
}
else if(ol0 || ol1) {
setDecompressionOnAllInputs(hop, parent);
return;
}
else {
String ex = "Setting decompressed because input Binary Op dimensions is unknown:\n"
+ Explain.explain(hop);
LOG.warn(ex);
setDecompressionOnAllInputs(hop, parent);
return;
}
}
}
else if(hop instanceof IndexingOp) {
final boolean isOverlapping = isOverlapping(hop.getInput(0));
o = new OpNormal(hop, true);
if(isOverlapping) {
overlapping.add(hop.getHopID());
o.setOverlapping();
}
}
else if(HopRewriteUtils.isTernary(hop, OpOp3.MINUS_MULT, OpOp3.PLUS_MULT, OpOp3.QUANTILE, OpOp3.CTABLE)) {
setDecompressionOnAllInputs(hop, parent);
return;
}
else if(HopRewriteUtils.isTernary(hop, OpOp3.IFELSE)) {
final Hop o1 = hop.getInput(1);
final Hop o2 = hop.getInput(2);
if(isCompressed(o1) && isCompressed(o2)) {
o = new OpMetadata(hop, o1);
if(isOverlapping(o1) || isOverlapping(o2))
o.setOverlapping();
}
else if(isCompressed(o1)) {
o = new OpMetadata(hop, o1);
if(isOverlapping(o1))
o.setOverlapping();
}
else if(isCompressed(o2)) {
o = new OpMetadata(hop, o2);
if(isOverlapping(o2))
o.setOverlapping();
}
else {
setDecompressionOnAllInputs(hop, parent);
}
}
else if(hop instanceof ParameterizedBuiltinOp) {
if(HopRewriteUtils.isParameterizedBuiltinOp(hop, ParamBuiltinOp.REPLACE, ParamBuiltinOp.TRANSFORMAPPLY)) {
o = new OpNormal(hop, true);
}
else {
LOG.warn("Unknown ParameterizedBuiltinOp Hop:" + hop.getClass().getSimpleName() + "\n" + Explain.explain(hop));
setDecompressionOnAllInputs(hop, parent);
return;
}
}
else if(hop instanceof NaryOp) {
setDecompressionOnAllInputs(hop, parent);
return;
}
else if(hop instanceof ReorgOp){
setDecompressionOnAllInputs(hop, parent);
return;
}
else {
LOG.warn("Unknown Matrix Hop:" + hop.getClass().getSimpleName() + "\n" + Explain.explain(hop));
setDecompressionOnAllInputs(hop, parent);
return;
}
o = o != null ? o : new OpNormal(hop, RewriteCompressedReblock.satisfiesSizeConstraintsForCompression(hop));
treeLookup.put(hop.getHopID(), o);
parent.addOp(o);
if(o.isCompressedOutput())
compressed.add(hop.getHopID());
}
else if(hop.getDataType().isFrame()) {
Op o = null;
if(HopRewriteUtils.isData(hop, OpOpData.PERSISTENTREAD, OpOpData.TRANSIENTREAD))
return;
else if(HopRewriteUtils.isData(hop, OpOpData.TRANSIENTWRITE, OpOpData.PERSISTENTWRITE)) {
transientCompressed.put(hop.getName(), hop.getInput(0).getHopID());
compressed.add(hop.getHopID());
o = new OpMetadata(hop, hop.getInput(0));
if(isOverlapping(hop.getInput(0)))
o.setOverlapping();
}
else if(HopRewriteUtils.isUnary(hop, OpOp1.DETECTSCHEMA)) {
o = new OpNormal(hop, false);
}
else if(HopRewriteUtils.isBinary(hop, OpOp2.APPLY_SCHEMA)) {
o = new OpNormal(hop, true);
}
else if(hop instanceof AggUnaryOp) {
o = new OpNormal(hop, false);
}
else {
LOG.warn("Unknown Frame Hop:" + hop.getClass().getSimpleName() + "\n" + Explain.explain(hop));
setDecompressionOnAllInputs(hop, parent);
return;
}
o = o != null ? o : new OpNormal(hop, RewriteCompressedReblock.satisfiesSizeConstraintsForCompression(hop));
treeLookup.put(hop.getHopID(), o);
parent.addOp(o);
if(o.isCompressedOutput())
compressed.add(hop.getHopID());
}
else if(HopRewriteUtils.isTransformEncode(hop)) {
Hop matrix = ((FunctionOp) hop).getOutputs().get(0);
compressed.add(matrix.getHopID());
transientCompressed.put(matrix.getName(), matrix.getHopID());
parent.addOp(new OpNormal(hop, true));
}
else if(hop instanceof FunctionOp && ((FunctionOp) hop).getFunctionNamespace().equals(".builtinNS")) {
parent.addOp(new OpNormal(hop, false));
}
else if(hop instanceof AggUnaryOp) {
if((isOverlapping(hop.getInput().get(0)) && !HopRewriteUtils.isAggUnaryOp(hop, AggOp.SUM, AggOp.MEAN)) ||
HopRewriteUtils.isAggUnaryOp(hop, AggOp.TRACE)) {
setDecompressionOnAllInputs(hop, parent);
return;
}
else {
Op o = new OpNormal(hop, false);
treeLookup.put(hop.getHopID(), o);
parent.addOp(o);
}
}
else {
LOG.warn(
"Unknown Matrix or Frame Hop:" + hop.getClass().getSimpleName() + "\n" + hop.getDataType() + "\n" + Explain.explain(hop));
parent.addOp(new OpNormal(hop, false));
}
}