src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java (29 lines): - line 117: // TODO make specialized version that ignore rl if rl == 0. - line 120: // TODO make specialized version that ignore ru if ru == nRows. - line 158: // TODO make specialized version that ignore rl if rl == 0. - line 161: // TODO make specialized version that ignore ru if ru == nRows. - line 200: // TODO make specialized version that ignore rl if rl == 0. - line 203: // TODO make specialized version that ignore ru if ru == nRows. - line 236: // TODO make specialized version that ignore rl if rl == 0. - line 239: // TODO make specialized version that ignore ru if ru == nRows. - line 274: // TODO: add support for FORRLE if applicable case. - line 287: // TODO: add support for FORRLE if applicable case. - line 322: // TODO create rle from boolean list. - line 343: // TODO add cache blocking - line 358: // TODO make specialized version that ignore rl if rl == 0. - line 361: // TODO make specialized version that ignore ru if ru == nRows. - line 406: // TODO add cache blocking - line 417: // TODO make specialized version that ignore rl if rl == 0. - line 420: // TODO make specialized version that ignore ru if ru == nRows. - line 447: // TODO add cache blocking - line 458: // TODO make specialized version that ignore rl if rl == 0. - line 461: // TODO make specialized version that ignore ru if ru == nRows. - line 489: // TODO add cache blocking - line 500: // TODO make specialized version that ignore rl if rl == 0. - line 503: // TODO make specialized version that ignore ru if ru == nRows. - line 530: // TODO add cache blocking - line 541: // TODO make specialized version that ignore rl if rl == 0. - line 544: // TODO make specialized version that ignore ru if ru == nRows. - line 687: for(int r = rl; r < ru; r++) { // TODO move rl and ru to innermost loop. - line 699: // TODO make specialized version that ignore cu if cu == nRows. - line 771: // TODO make specialized version that ignore cu if cu == nRows. src/main/java/org/apache/sysds/runtime/lineage/LineageCache.java (22 lines): - line 115: //TODO need to also move execution of compensation plan out of here - line 183: else { //TODO handle locks on gpu objects - line 250: //TODO: handling of recursive calls - line 386: //TODO: reuse only those UDFs which are part of reusable instructions - line 393: //TODO: support multi-return UDFs - line 395: //TODO: trace all UDFs - line 455: //TODO: dedicated stats for federated reuse - line 530: //TODO problematic as after probe the matrix might be kicked out of cache - line 595: //TODO why do we need both of these public put methods - line 640: // TODO: gpu multi-return instructions - line 780: // TODO: Cache sparse pointers - line 918: //FIXME: if boundLI is for a MultiReturnBuiltin instruction - line 942: //TODO: trace all UDFs - line 977: //TODO: maintain statistics, lineage estimate - line 1078: //FIXME: different o/p datatypes for MultiReturnBuiltins. - line 1165: // TODO: Handling of func/SB cache entries for Spark and GPU - line 1272: //TODO: All other relevant instruction types. - line 1354: //TODO: cm, variance - line 1356: //TODO: support other PBuiltin ops - line 1393: nflops = 0; //FIXME: this method will be deprecated - line 1412: //TODO: Replace with generic type - line 1415: //FIXME: Replace getLineageItem with get/getOrCreate to avoid creating a new LI object src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java (16 lines): - line 128: //TODO unify memory management for CP, par RDDs, and potentially broadcasts - line 271: //TODO if spark context passed in from outside (mlcontext), we need to clean this up at the end - line 478: // TODO: Is that a bug because it triggers extra file reading? rdd is needed only for the first case - line 533: // TODO implement hadoop read write for tensor - line 548: // TODO use fromFile instead of false and true - line 558: // TODO implement hadoop read write for tensor - line 564: // TODO: TensorMarket? - line 566: // TODO support other Input formats - line 580: * FIXME: currently this implementation assumes matrix representations but frame signature - line 672: //TODO: synchronize - line 978: // TODO rows and columns for matrix characteristics - line 1309: // TODO special case single block - line 1311: // TODO asynchronous allocation - line 1338: // TODO sparse copy - line 1340: // TODO keep track of nnz - line 1343: // TODO post-processing output tensor (nnz, sparsity) src/main/java/org/apache/sysds/runtime/lineage/LineageRewriteReuse.java (15 lines): - line 84: // TODO extend recompiler to allow use without explain output - line 128: DMLScript.EXPLAIN = et; //TODO can't change this here - line 150: // TODO: For now assumption is that a single column is being appended in a loop - line 236: //TODO: support for block of rows - line 416: //TODO: support for block of rows - line 462: //TODO: support for block of rows - line 543: //TODO: support for block of rows - line 600: //TODO: support for block of rows - line 657: //TODO: support for block of rows - line 711: * TODO: Generalize for all cases and move to compiler - line 913: //TODO: support nary cbind - line 944: //TODO: support nary cbind - line 1156: * TODO: generalized but robust applicability function - line 1200: //TODO: delta with more than one column - line 1239: // TODO extend recompiler to allow use without explain output src/main/java/org/apache/sysds/runtime/data/LibTensorAgg.java (13 lines): - line 51: // TODO more conditions depending on operation - line 64: // TODO filter empty input blocks (incl special handling for sparse-unsafe operations) - line 97: // TODO change to sparse if worth it - line 108: // TODO implement for other optypes - line 134: // TODO support indexfn that are not reduce all - line 135: // TODO support for all shapes - line 199: // TODO handle different index functions - line 202: // TODO other aggregations - line 213: //TODO special handling for mean where the final aggregate operator - line 224: // TODO: SparseBlock - line 230: //TODO switch to no-op nnz meta data once available - line 271: // TODO maybe merge this, and other parts, with `LibMatrixAgg` - line 298: //TODO recompute non-zeros of partial result src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCUDA.java (11 lines): - line 503: // For sparse TSMM, invoke matmult (TODO: possible performance improvement) - line 1163: // TODO: - line 1178: // TODO: Potential to optimize - line 1204: // TODO: Performance optimization: Call cublasDaxpy if(in.getNumRows() == 1 || in.getNumColumns() == 1) - line 1305: Pointer A = getDensePointer(gCtx, in, instName); // TODO: FIXME: Implement sparse binCellSparseScalarOp kernel - line 1392: Pointer A = getDensePointer(gCtx, in1, instName); // TODO: FIXME: Implement sparse binCellSparseOp kernel - line 1393: Pointer B = getDensePointer(gCtx, in2, instName); // TODO: FIXME: Implement sparse binCellSparseOp kernel - line 1496: Pointer srcPtr = getDensePointer(gCtx, src, instName); // TODO: FIXME: Implement sparse kernel - line 1612: // TODO: Implement sparse-dense matrix cublasDgeam kernel - line 1638: // TODO: to implement the transposed + dgeam for sparse matrices, they need to be converted to csc, which is effectively a tranpose - line 2680: // TODO : Find a way to assign bTobj directly to the output and set the correct flags so as to not crash src/main/java/org/apache/sysds/runtime/meta/TensorCharacteristics.java (11 lines): - line 31: // TODO move block size to `ConfigurationManager` - line 222: // TODO Auto-generated method stub - line 228: // TODO Auto-generated method stub - line 234: // TODO Auto-generated method stub - line 240: // TODO Auto-generated method stub - line 246: // TODO Auto-generated method stub - line 252: // TODO Auto-generated method stub - line 258: // TODO Auto-generated method stub - line 264: // TODO Auto-generated method stub - line 270: // TODO Auto-generated method stub - line 276: // TODO Auto-generated method stub src/main/java/org/apache/sysds/runtime/data/TensorBlock.java (11 lines): - line 230: //TODO perf, do not fill, instead save schema - line 261: // TODO Auto-generated method stub - line 267: // TODO Auto-generated method stub - line 273: // TODO Auto-generated method stub - line 279: // TODO Auto-generated method stub - line 284: // TODO Auto-generated method stub - line 517: // TODO perf - line 573: // TODO perf - line 660: //TODO perf bits instead of bytes - line 813: // FIXME readUTF is not supported for CacheDataInput - line 859: // TODO reconsider with operation types src/main/java/org/apache/sysds/runtime/data/BasicTensorBlock.java (10 lines): - line 149: //TODO simplify estimated non-zeros - line 204: // TODO: use _denseBlock.reset instead, since LDRB need to check dimensions for actually available space - line 302: // TODO: Implement sparse - line 329: // TODO: Implement sparse - line 464: // TODO implement sparse set instead of converting to dense - line 484: else // TODO copy sparse to dense, dense to dense or sparse to dense - line 522: // TODO consider sparse - line 545: // TODO Auto-generated method stub - line 559: // TODO allow to aggregate along a dimension? - line 560: // TODO performance src/main/java/org/apache/sysds/runtime/instructions/spark/AggregateUnarySPInstruction.java (10 lines): - line 117: //TODO: Make memory for the future matrix block - line 174: // TODO support DataTensor - line 178: // TODO: filter input blocks for trace - line 186: // TODO filter non empty blocks if sparse safe - line 192: // TODO generalize to drop depending on location of correction - line 193: // TODO support DataTensor - line 206: // TODO MULTI_BLOCK - line 306: // TODO support DataTensor - line 355: // TODO support DataTensor - line 362: // TODO generalize to drop depending on location of correction src/main/java/org/apache/sysds/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java (10 lines): - line 130: * TODO fuse also result merge into fused data partitioning and execute - line 136: * TODO take remote memory into account in data/result partitioning rewrites (smaller/larger) - line 137: * TODO memory estimates with shared reads - line 138: * TODO memory estimates of result merge into plan tree - line 139: * TODO blockwise partitioning - line 466: * TODO consolidate mem estimation with Indexing Hop - line 844: //TODO get a proper time estimate based to capture compute-intensive scenarios - line 1162: //FIXME rework for nested parfor parallelism and body w/o gpu ops - line 1408: * TODO modify 'set exec strategy' and related rewrites for conditional data partitioning. - line 2125: //TODO rework memory handling for spark src/main/java/org/apache/sysds/runtime/transform/encode/MultiColumnEncoder.java (8 lines): - line 362: // FIXME: estimate nnz for multiple encoders including dummycode - line 387: // TODO smart checks - line 409: // TODO set NNZ explicit count them in the encoders - line 550: // TODO: If still don't fit, serialize the column encoders - line 587: // TODO: Reduce only the ones with large maps - line 618: // TODO: Reduce the large encoder types, similar to getNumBuildBlksMemorySafe - line 1072: // TODO cache results for faster access - line 1563: // TODO: experiment if it makes sense to parallize here (for frames with many rows) src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java (7 lines): - line 114: * TODO: reduction variables (operations: +=, -=, /=, *=, min, max) - line 115: * TODO: papply(A,1:2,FUN) language construct (compiled to ParFOR) via DML function repository => modules OK, but second-order functions required - line 687: //TODO reset of hop parallelism constraint (e.g., ba+*) - line 874: // TODO: remove duplicate lineage items in ec.getLineage() - line 966: || OptimizerUtils.isSparkExecutionMode() ) //TODO support for binarycell - line 1130: //TODO needs as precondition a systematic treatment of persistent read information. - line 1445: * TODO The optimizer should explicitly decide about parallel result merge and its degree of parallelism. src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java (7 lines): - line 350: // TODO - replace hardcoded numbers with more sophisticated code - line 375: //TODO choose correct value type for tensor - line 433: // TODO correct sizes - line 896: //TODO handling of cleanup status, potentially new object - line 933: //TODO handling of cleanup status, potentially new object - line 1083: // TODO write tensor - line 1210: //FIXME why is this writer never used? src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java (7 lines): - line 220: //TODO: LSTM on GPU has different INPUT/OUTPUT than LSTM on CPU - line 1125: //output.setDataType(id.getDataType()); //TODO whenever we support multiple matrix value types, currently noop. - line 1134: //output.setDataType(id.getDataType()); //TODO whenever we support multiple matrix value types, currently noop. - line 1143: //output.setDataType(id.getDataType()); //TODO whenever we support multiple matrix value types, currently noop. - line 1230: // TODO: remove this when ppred has been removed from DML - line 1610: //TODO set output type to boolean when supported - line 2287: // TODO get supported Operations form builtins src/main/java/org/apache/sysds/runtime/instructions/spark/RandSPInstruction.java (7 lines): - line 536: //step 2: TODO potential in-memory rand operations if applicable - line 541: // TODO calculate totalSize - line 542: // TODO use real blocksize given by instruction (once correct) - line 607: // TODO getDimLengthPerBlock accurate for each dimension - line 988: //TODO fix hard-coded seed - line 1104: // TODO: accurate block size computation - line 1114: // TODO implement sparse support src/main/python/systemds/scuro/modality/joined.py (7 lines): - line 60: chunked_execution # TODO: maybe move this into parent class - line 85: ): # TODO compute correct alignment timestamps/spatial params - line 100: # TODO: need to extract the shape of the data from the metadata - line 140: ] # TODO: check correct loading for all data layouts, this is similar to missing data, add a different operation for this - line 170: ): # TODO: use the filed name to extract data entries from modalities - line 206: ): # TODO: only if the layout is list_of_lists_of_numpy_array - line 222: else: # TODO: refactor this approach (it is changing the way the modalities are joined) src/main/java/org/apache/sysds/runtime/lineage/LineageRecomputeUtils.java (6 lines): - line 98: // Disable GPU execution. TODO: Support GPU - line 135: //FIXME: any lineage-based restore in the bufferpool flushes the lineage cache - line 409: //TODO handling of tsmm type left and right -> placement transpose - line 496: //FIXME: the keys of operands can be replaced inside rConstructHops - line 513: // TODO: multi-return function - line 802: //TODO handling of tsmm type left and right -> placement transpose src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederationMap.java (6 lines): - line 277: //TODO support operations with fully broadcast objects - line 520: // TODO handling of file path, but no danger as never written - line 538: // TODO handling of file path, but no danger as never written - line 728: // TODO perf - line 729: // TODO: add a check if the map is sorted based on indexes before reversing. - line 730: // TODO: add a setup such that on construction the federated map is already sorted. src/main/java/org/apache/sysds/parser/DataExpression.java (6 lines): - line 446: // TODO use byRow parameter - line 646: // TODO convert double Matrix to String Frame - line 673: // TODO convert double Matrix to long Matrix - line 1657: // TODO set correct dimensions - line 2062: // TODO get size - line 2140: // TODO value type for federated object src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/FederatedPSControlThread.java (5 lines): - line 105: _weighting = weighting && (!use_homomorphic_encryption); // FIXME: this disables weighting in favor of homomorphic encryption - line 319: // TODO: set other CKKS parameters - line 320: // TODO generate partial public key - line 668: // FIXME ensure that with modelAvg we always update the model - line 688: // TODO double check cleanup gradients and models src/main/java/org/apache/sysds/hops/AggBinaryOp.java (5 lines): - line 141: //return false; // TODO: Disabling any fused transa optimization in 1.0 release. - line 1136: //TODO relax requirement of valid CP dimensions once we support broadcast creation from files/RDDs - line 1184: //TODO currently we reuse the mr estimates, these need to be fine-tune for our spark operators - line 1200: // TODO: we must factor in the "sparsity" - line 1227: // TODO: we must factor in the "sparsity" src/main/java/org/apache/sysds/runtime/instructions/fed/InitFEDInstruction.java (5 lines): - line 136: // TODO relax requirement by using different execution contexts per federated data? - line 325: // TODO: Construct new protocol name for Federated communication - line 331: // TODO: Support IPV6 address for Federated communication - line 356: // TODO: Construct new protocol name for Federated communication - line 362: // TODO: Support IPV6 address for Federated communication src/main/java/org/apache/sysds/runtime/lineage/LineageSparkCacheEviction.java (5 lines): - line 45: //TODO: Cleanup the RDDs and BC variables - line 69: // FIXME: avoid when called from partial reuse methods - line 128: // TODO: Reduce to avoid disk spilling. 80% of storage. - line 150: // FIXME: Actual memory usage is often way more than the estimated, - line 188: // TODO: Cleanup the child RDDs of the persisted RDDs src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixAgg.java (5 lines): - line 94: * TODO next opcode extensions: a+, colindexmax - line 703: * TODO: this should not be part of block operations but of the MR instruction. - line 3659: //TODO: currently impossible case, since Dedup reshape is not supported yet, once it is, this method needs - line 3679: //TODO: currently impossible case, since Dedup reshape is not supported yet, once it is, this method needs - line 3699: //TODO: currently impossible case, since Dedup reshape is not supported yet, once it is, this method needs src/main/java/org/apache/sysds/runtime/iogen/FormatIdentifyer.java (5 lines): - line 290: // TODO: extend sequential scattered format algorithm for heterogeneous structures - line 310: // TODO: decrease the number of row/col indexes want to check(3 or 5) - line 1063: // TODO Removed an unlikely argument it should not be a problem. - line 1114: // TODO find out if used: - line 1353: // TODO: we have to apply tokenizer src/main/java/org/apache/sysds/lops/compile/linearization/LinearizerCostBased.java (5 lines): - line 44: // TODO: Partition the DAG if connected by a single node. Optimize separately - line 54: // TODO: Handle distributed and GPU operators (0 compute cost, memory overhead on collect) - line 55: // TODO: Asynchronous operators (max of compute costs, total operation memory overhead) - line 57: // TODO: Select the order with minimum compute cost and buffer pool evictions - line 207: // TODO: Add IO time to compute cost for evictions src/main/java/org/apache/sysds/hops/OptimizerUtils.java (5 lines): - line 225: //TODO enabling it by default requires modifications in lineage-based reuse - line 485: //TODO this is a workaround for MLContext to avoid a major refactoring before the release; this method - line 650: // TODO estimate exact size - line 852: // TODO estimate partitioned size exact for tensor - line 977: // TODO accurate estimation src/main/java/org/apache/sysds/runtime/instructions/spark/MatrixReshapeSPInstruction.java (4 lines): - line 73: //TODO handle dims for tensors parts[4] - line 120: // TODO Tensor reshape - line 125: // TODO merge by key - line 203: // // TODO create iterator src/main/java/org/apache/sysds/parser/dml/DmlSyntacticValidator.java (4 lines): - line 525: // TODO: remove this when castAsScalar has been removed from DML/PYDML - line 1073: // TODO Auto-generated method stub - line 1460: // TODO: extract more meaningful info from this exception. - line 1561: // TODO: extract more meaningful info from this exception. src/main/java/org/apache/sysds/common/Builtins.java (4 lines): - line 260: OUTLIER("outlier", true, false), //TODO parameterize opposite - line 315: SLICEFINDER("slicefinder", true), //TODO remove - line 351: WINSORIZE("winsorize", true, false), //TODO parameterize w/ prob, min/max val - line 352: WINSORIZEAPPLY("winsorizeApply", true, false), //TODO parameterize w/ prob, min/max val src/main/java/org/apache/sysds/hops/fedplanner/FederatedPlanCostEnumerator.java (4 lines): - line 73: // TODO: Just for debug, remove later - line 183: // TODO: Do not descend for visited functions (use a hash set for functions using their names) - line 245: // TODO: Should we check all statement-level root hops to see if they are not referenced? - line 248: // TODO: Just for debug, remove later src/main/java/org/apache/sysds/runtime/controlprogram/caching/TensorObject.java (4 lines): - line 149: // TODO correct blocksize; - line 150: // TODO read from RDD - line 179: //TODO check correct blocking - line 199: //TODO rdd write src/main/java/org/apache/sysds/runtime/lineage/LineageItemUtils.java (4 lines): - line 202: //TODO: trace all UDFs - line 317: //TODO: include all the other hops - line 372: //TODO should use the same placeholder meta data as codegen - line 547: //TODO: add 'read' in this list src/main/java/org/apache/sysds/hops/Hop.java (4 lines): - line 389: //FIXME: Compute cost. Handle multiple Lops from one Hop case - line 486: //TODO generalize threads - line 725: * TODO remove memo table and, on constructor refresh, inference in refresh, single compute mem, - line 1191: //TODO: set the flag in the HOP via a rewrite src/main/java/org/apache/sysds/lops/rewrite/RewriteAddPrefetchLop.java (4 lines): - line 77: //FIXME: Rewire _inputParams when needed (e.g. GroupedAggregate) - line 116: // FIXME: Rewire _inputParams when needed (e.g. GroupedAggregate) - line 121: // TODO: support non-matrix outputs - line 136: // FIXME: Rewire _inputParams when needed (e.g. Replace) scripts/staging/onnx/onnx_systemds/onnx_helper.py (4 lines): - line 158: # TODO: these type translations are not correct double -> float - line 184: # TODO: add support for other data types - line 202: # TODO: shapes with no value but instead name -> support? - line 208: # TODO: not sure this is the solution for every instance of this problem src/main/java/org/apache/sysds/runtime/lineage/LineageCacheEviction.java (4 lines): - line 90: // FIXME: avoid when called from partial reuse methods - line 245: //TODO: Graceful handling of status. - line 335: // TODO: exponential smoothing with arbitrary smoothing factor - line 427: // TODO: set cache status as RELOADED for this entry src/main/java/org/apache/sysds/hops/DataGenOp.java (4 lines): - line 122: //TODO size information for tensor - line 252: // TODO size information for tensors - line 341: // TODO size information for tensor - line 361: //TODO fix parser exception handling and enable check by default src/main/java/org/apache/sysds/runtime/lineage/LineageItem.java (4 lines): - line 43: private final BooleanArray32 _specialValueBits; // TODO: Move this to a new subclass - line 46: //TODO replace with thread local concurrent hashmap per worker - line 343: // FIXME: Two dedup nodes can have matching patches but different #inputs - line 377: continue; //FIXME: fill phMap src/main/python/systemds/context/systemds_context.py (4 lines): - line 254: # TODO: refine the choise of configuration file - line 461: :param shape: shape (rows and cols) of the matrix TODO tensor - line 645: # TODO Support tensors. - line 673: # TODO Support tensors. src/main/java/org/apache/sysds/runtime/data/SparseBlockDCSR.java (4 lines): - line 53: //TODO: This allocates too much space (we care about number of non-empty rows) - line 389: // TODO: performance - line 435: // TODO performance - line 607: // TODO: performance src/main/java/org/apache/sysds/runtime/io/ReaderCOG.java (3 lines): - line 76: // TODO: Currently only reads the first image which is the full resolution image - line 79: // TODO: Is the metadata (e.g. the coordinates) necessary in SystemDS? Currently not possible as we only return a MatrixBlock - line 164: // TODO: Currently this doesn't seem standardized properly, there are still open GitHub issues about that src/main/java/org/apache/sysds/runtime/lineage/LineageCacheEntry.java (3 lines): - line 358: // TODO: Multiple eviction policies. - line 366: // TODO: timestamp >> DAg_height. Normalize timestamp and DAG height. - line 453: // TODO: Support sparse pointer caching scripts/staging/onnx/onnx_systemds/operator_gen.py (3 lines): - line 259: batch_size = x_shape[0] # TODO: currently not used - line 357: batch_size = x_shape[0] # TODO: Batch size unused? - line 365: C_group = W_shape[1] # TODO Channels/group unused? src/main/java/org/apache/sysds/runtime/instructions/spark/BinarySPInstruction.java (3 lines): - line 210: // TODO blocking scheme for matrices with mismatching number of dimensions - line 295: // TODO less dims broadcast variable - line 417: // TODO print dimensions src/main/java/org/apache/sysds/hops/rewrite/RewriteElementwiseMultChainOptimization.java (3 lines): - line 255: //TODO replace by ComparableHop wrapper around hop that implements equals and compareTo - line 335: // TODO proper handling of DAGs (avoid collecting the same leaf multiple times) - line 336: // TODO exclude hops with unknown dimensions and move rewrites to dynamic rewrites src/main/java/org/apache/sysds/runtime/transform/encode/ColumnEncoderComposite.java (3 lines): - line 46: // TODO assert each type of encoder can only be present once - line 173: // TODO: is that still needed? currently there is no CompositeEncoder with 2 encoders with build phase - line 266: // TODO maybe assert that the _encoders never have the same type of encoder twice or more src/main/java/org/apache/sysds/runtime/instructions/fed/MultiReturnParameterizedBuiltinFEDInstruction.java (3 lines): - line 356: //encoder.build(fb, OptimizerUtils.getTransformNumThreads()); // FIXME skip equi-height sorting - line 357: // FIXME: Enabling multithreading intermittently hangs - line 392: // FIXME: Enabling multithreading intermittently hangs src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java (3 lines): - line 394: if(shouldTryAsyncCompress()) // TODO: replace the reused object - line 593: //TODO handling of spark instructions should be under control of federated site not coordinator - line 619: // TODO move this thread naming to Netty thread creation! src/main/java/org/apache/sysds/runtime/controlprogram/caching/UnifiedMemoryManager.java (3 lines): - line 152: // FIXME: Parfor splits this 70% into smaller limits - line 163: // TODO: Track preserved output memory to protect from concurrent threads - line 410: //TODO implement src/main/java/org/apache/sysds/lops/rewrite/RewriteAddChkpointInLoop.java (3 lines): - line 50: // TODO: support If-Else block inside loop. Consumers inside branches. - line 85: // TODO: A rewrite pass to remove less effective checkpoints - line 97: // TODO: Check if this lop leads to one of those variables src/main/java/org/apache/sysds/hops/TernaryOp.java (3 lines): - line 551: //TODO double check reset (dimsInputPresent?) - line 579: // TODO qx1 - line 654: // TODO: to rewrite is not currently not triggered if outdim are given --> getInput().size()>=3 src/main/java/org/apache/sysds/runtime/instructions/fed/CumulativeOffsetFEDInstruction.java (3 lines): - line 174: //TODO perf simple rbind, as the first row (init) is anyway not transferred - line 262: //TODO perf no need to execute here, we can piggyback the requests onto the final cumagg - line 272: //TODO perf improvement (currently this creates a sliced broadcast in the size of the original matrix src/main/java/org/apache/sysds/hops/rewrite/MarkForLineageReuse.java (3 lines): - line 84: //TODO: nested loops. - line 150: //TODO: extend all the hops to propagate till variablecp output - line 152: // TODO: logic to separate out partially reusable cases (e.g cbind-tsmm) src/main/java/org/apache/sysds/parser/StatementBlock.java (3 lines): - line 497: //TODO handle predicates - line 528: //TODO handle predicates - line 1422: // FIXME: Type is not the best key as many Lops may have the same types src/main/java/org/apache/sysds/runtime/instructions/gpu/context/GPUMemoryManager.java (3 lines): - line 284: // TODO: Copy conditionally (if score > theta) - line 292: // TODO: Handle live (dirty) objects separately. Copy them back to the host - line 348: // FIXME: performance improvement. Slow due to looping and holes. src/main/java/org/apache/sysds/runtime/frame/data/columns/BitSetArray.java (3 lines): - line 258: // TODO add an vectorized setNz. - line 403: // TODO: fix inefficient transfer 8 x bigger. - line 568: // TODO vectorize src/main/java/org/apache/sysds/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java (3 lines): - line 199: hi = reorderMinusMatrixMult(hop, hi, i); //e.g., (-t(X))%*%y->-(t(X)%*%y), TODO size - line 830: // TODO implement for tensor - line 951: else //diagm2v TODO support tensor operation src/main/java/org/apache/sysds/runtime/controlprogram/parfor/opt/OptTreeConverter.java (3 lines): - line 261: //TODO task size - line 274: //TODO remove this workaround once this information can be obtained from hops/lops compiler - line 324: // TODO: create execution mode for parfor loop src/main/java/org/apache/sysds/lops/rewrite/RewriteAddBroadcastLop.java (3 lines): - line 53: //FIXME: Wire Broadcast only with the necessary outputs - line 59: //FIXME: Rewire _inputParams when needed (e.g. GroupedAggregate) - line 79: //TODO: Early broadcast objects that are bigger than a single block src/main/java/org/apache/sysds/runtime/instructions/gpu/context/GPUMemoryEviction.java (3 lines): - line 78: // TODO: First remove the gobj chains that don't contain any live and dirty objects. - line 106: //FIXME: adding to rmVar cache causes multiple failures due to concurrent - line 141: if (DMLScript.STATISTICS) //TODO: dedicated statistics for lineage src/main/java/org/apache/sysds/runtime/instructions/spark/DnnSPInstruction.java (3 lines): - line 220: // TODO: Inject checkpoint to avoid doing this repeated for validation set - line 244: // TODO: Handle large filters > 2G - line 280: long nnz = -1; // TODO: Handle nnz src/main/java/org/apache/sysds/runtime/instructions/gpu/context/GPUObject.java (3 lines): - line 336: // TODO: What if mat.getNnz() is -1 ? - line 665: // TODO: Future optimization: - line 851: // TODO - should we do this on the GPU using cusparsecoo2csr() ? src/main/java/org/apache/sysds/hops/DnnOp.java (3 lines): - line 255: //TODO move these custom rewrites to the general hop rewrites - line 335: // TODO double check that optionalMaxPoolOutput adheres to proper - line 578: // TODO: Fix this after adding remaining spark instructions src/main/java/org/apache/sysds/runtime/instructions/fed/UnaryMatrixFEDInstruction.java (3 lines): - line 166: //TODO perf simple rbind, as the first row (init) is anyway not transferred - line 254: //TODO perf no need to execute here, we can piggyback the requests onto the final cumagg - line 264: //TODO perf improvement (currently this creates a sliced broadcast in the size of the original matrix src/main/java/org/apache/sysds/api/DMLScript.java (3 lines): - line 244: //TODO fix null default options - line 251: //TODO fix null default options - line 624: // TODO fix and replace localhost identifyer with hostname in federated instructions SYSTEMDS-3440 src/main/java/org/apache/sysds/hops/recompile/Recompiler.java (3 lines): - line 1360: // TODO: correct dimensions - line 1389: // TODO refresh tensor size information - line 1442: //TODO tensor rewrite src/main/java/org/apache/sysds/runtime/instructions/cp/ReshapeCPInstruction.java (3 lines): - line 75: // TODO support DataTensor - line 84: // TODO metadata operation - line 89: // TODO support frame and list. Before we implement list it might be good to implement heterogeneous tensors src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java (3 lines): - line 567: //TODO perf: reconsider partition sort to avoid unnecessary barriers - line 1165: //TODO more robust (memory efficient) implementation w/o tmp block - line 2372: // TODO use optimized mod inverse operation. src/main/java/org/apache/sysds/runtime/util/DataConverter.java (2 lines): - line 1007: // TODO use sparse iterator for sparse block - line 1272: // TODO warning if double value is cast to long? src/main/java/org/apache/sysds/hops/cost/CostEstimatorStaticRuntime.java (2 lines): - line 398: //TODO add cost functions for commons math builtins: inverse, cholesky - line 486: xf = 32; // TODO - assuming worst case for now src/main/java/org/apache/sysds/hops/fedplanner/FederatedPlanCostEstimator.java (2 lines): - line 61: // TODO: Q. Shouldn't the child's forwarding cost follow the parent's weight, regardless of loops or if-else statements? - line 91: // Therefore, the weight for forwarding follows the parent's weight (TODO: Q. Is the parent also receiving forwarding once?) src/main/java/org/apache/sysds/lops/rewrite/RewriteAddGPUEvictLop.java (2 lines): - line 47: // TODO: Move this as a Statement block rewrite - line 72: // TODO: Add another input for the backend (GPU/CPU/Spark) src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java (2 lines): - line 54: //TODO perf (exploit sparse representation for better asymptotic behavior) - line 62: //TODO perf (exploit sparse representation for better asymptotic behavior) src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java (2 lines): - line 63: //TODO: Reuse everything. - line 427: // TODO: Automatic tuning of weights. src/main/java/org/apache/sysds/utils/NativeHelper.java (2 lines): - line 358: // TODO: Add pmm, wsloss, mmchain, etc. - line 373: // TODO: case not handled: sparse filters (which will only be executed in Java). Since filters are relatively smaller, src/main/java/org/apache/sysds/hops/codegen/opt/PlanSelectionFuseCostBased.java (2 lines): - line 367: //TODO branch and bound pruning, right now we use exhaustive enum for early experiments - line 661: case SINH: costs = 93; break; // TODO: src/main/java/org/apache/sysds/runtime/instructions/spark/utils/RDDAggregateUtils.java (2 lines): - line 679: // TODO remove once KahanPlus is completely replaced by plus - line 685: // TODO support DataTensor src/main/java/org/apache/sysds/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java (2 lines): - line 273: // TODO check in debbuger if correct - line 358: else //TODO: support equi-height src/main/java/org/apache/sysds/lops/rewrite/RewriteUpdateGPUPlacements.java (2 lines): - line 50: // TODO: Iterate multiple times to propagate the updates - line 88: // TODO: Propagate GPU execution types to DataOps (in hop level or lop level). src/main/java/org/apache/sysds/runtime/io/ReaderCOGParallel.java (2 lines): - line 87: // TODO: Currently only reads the first image which is the full resolution image - line 192: // TODO: If the tile is compressed, decompress the currentTileData here src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/homomorphicEncryption/SEALServer.java (2 lines): - line 39: // TODO take params here, like slot_count etc. - line 104: // TODO: extract an interface for this and use it here src/main/java/org/apache/sysds/hops/UnaryOp.java (2 lines): - line 162: // TODO additional physical operation if offsets fit in memory - line 578: // TODO handle overlapping state, since some operations would not lead to compressed output. src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedStatistics.java (2 lines): - line 229: //TODO merge with existing - line 686: // TODO fix this class to use shallow pointers. src/main/java/org/apache/sysds/runtime/lineage/LineageDedupUtils.java (2 lines): - line 109: // TODO: find the inputs from the ProgramBlock instead of StatementBlock - line 204: // FIXME: This logic removes only the live variables from lineage cache. src/main/cpp/he/he.cpp (2 lines): - line 33: // FIXME: this copies string data once. maybe implement a custom stream - line 42: string data = stream.str(); // FIXME: this copies string content. maybe implement custom ostream src/main/java/org/apache/sysds/runtime/lineage/LineageCacheStatistics.java (2 lines): - line 190: // TODO: Handle overflow - line 196: // TODO: Handle overflow src/main/java/org/apache/sysds/parser/dml/CustomErrorListener.java (2 lines): - line 73: //TODO MB: we should not redundantly log errors here - line 295: //TODO MB: This classification is misleading as it only refers to variations of parsing issues. src/main/java/org/apache/sysds/runtime/transform/encode/ColumnEncoderUDF.java (2 lines): - line 46: //TODO pass execution context through encoder factory for arbitrary functions not just builtin - line 47: //TODO integration into IPA to ensure existence of unoptimized functions src/main/java/org/apache/sysds/hops/fedplanner/FederatedPlannerFedAll.java (2 lines): - line 81: //TODO currently this rewrite assumes consistent decisions in conditional control flow - line 122: //TODO handle function calls src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederationUtils.java (2 lines): - line 110: //TODO remove rmFedOutFlag, once all federated instructions have this flag, then unconditionally remove - line 239: // TODO handle non-contiguous cases src/main/java/org/apache/sysds/runtime/instructions/fed/BinaryFEDInstruction.java (2 lines): - line 142: // TODO remove - line 160: // TODO different binary instructions src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixBincell.java (2 lines): - line 215: // TODO fix all variants to be feasible for multi-threading - line 1238: // TODO change this to a fill operation. src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java (2 lines): - line 378: // TODO: add specialization for sparse/dense matrix blocks - line 467: // TODO: implement src/main/java/org/apache/sysds/hops/rewrite/RewriteAlgebraicSimplificationStatic.java (2 lines): - line 226: //TODO move to dynamic rewrites (since size dependent to account for mv binary cell and outer operations) - line 2306: //TODO add remaining cases of comparison operators src/main/java/org/apache/sysds/runtime/controlprogram/paramserv/SparkParamservUtils.java (2 lines): - line 52: //TODO Add an additional physical operator which broadcasts the labels directly (broadcast join with features) if certain memory budgets are satisfied - line 57: //TODO could use join and aggregation to avoid unnecessary shuffle introduced by reduceByKey src/main/java/org/apache/sysds/runtime/compress/lib/CLALibScalar.java (2 lines): - line 222: else // TODO : cleanup to also colapse groups in this case. - line 247: else // TODO : cleanup to also colapse groups in this case. src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixMult.java (2 lines): - line 221: //TODO: currently impossible case, since Dedup reshape is not supported yet, once it is, this method needs - line 1184: //TODO: currently impossible case, since Dedup reshape is not supported yet, once it is, this method needs src/main/java/org/apache/sysds/runtime/instructions/cp/DataGenCPInstruction.java (2 lines): - line 335: // TODO handle runtime seed - line 353: // TODO fix hard-coded seed, consistently with sparse frame init src/main/java/org/apache/sysds/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java (2 lines): - line 647: //FIXME: Taking only spec file name as a literal leads to wrong reuse - line 648: //TODO: Add Embedding to the lineage item src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java (2 lines): - line 423: // TODO: Add append N function. - line 1306: // TODO remove sync locks on array types where they are not needed. src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNN.java (2 lines): - line 74: // TODO: experiment how often does dense matrix is empty where recomputing nnz before calling CuDNN will help - line 789: Pointer srcData = getDensePointerForCuDNN(gCtx, in, instName); // TODO: FIXME: Add sparse kernel support for relu src/main/java/org/apache/sysds/lops/Sql.java (2 lines): - line 60: // TODO spark - line 87: // TODO Sql.toString() lop src/main/java/org/apache/sysds/runtime/instructions/fed/ReshapeFEDInstruction.java (2 lines): - line 67: // TODO: add dims argument (for tensors) to MatrixReshapeSPInstruction - line 150: // TODO support tensor out, frame and list src/main/java/org/apache/sysds/runtime/instructions/fed/IndexingFEDInstruction.java (2 lines): - line 296: // TODO shallow copy, add more advanced update in place for federated - line 321: //TODO remove explicit put (unnecessary in CP, only spark which is about to be cleaned up) src/main/java/org/apache/sysds/lops/rewrite/RewriteAddChkpointLop.java (2 lines): - line 61: // TODO: A rewrite pass to remove less effective checkpoints - line 101: // TODO: Call recompiler on the loops src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCountDistinct.java (2 lines): - line 60: * TODO: If the MatrixBlock type is CompressedMatrix, simply read the values from the ColGroups. - line 198: // TODO Explore row-wise and column-wise methods for SparseBlockIterator src/main/java/org/apache/sysds/lops/rewrite/LopRewriter.java (2 lines): - line 49: // TODO: A rewrite pass to remove less effective chkpoints - line 118: //TODO: parfor statement blocks src/main/java/org/apache/sysds/runtime/data/LibTensorBincell.java (2 lines): - line 45: // TODO separate implementations for matching dims and broadcasting - line 46: // TODO perf (empty, sparse safe, etc.) src/main/java/org/apache/sysds/runtime/instructions/fed/SpoofFEDInstruction.java (2 lines): - line 241: //TODO fix check by num rows/cols - line 518: // TODO: use the `SpoofOperator` instead of the `Class`, as it is more robust to inheritance hierarchy change src/main/java/org/apache/sysds/runtime/instructions/InstructionUtils.java (2 lines): - line 720: // TODO add Multithreaded threads to Scalar operations. - line 1092: //TODO use appropriate function objects for min/max (see sum) src/main/java/org/apache/sysds/runtime/controlprogram/parfor/RemoteDPParForSpark.java (2 lines): - line 60: * TODO heavy hitter maintenance - line 61: * TODO data partitioning with binarycell src/main/java/org/apache/sysds/runtime/transform/tokenize/Tokenizer.java (2 lines): - line 126: // TODO further optimisation of task graph to reduce memory usage! - line 127: // TODO add cache awareness src/main/java/org/apache/sysds/runtime/data/LibTensorReorg.java (2 lines): - line 63: else // TODO deep copy - line 68: // TODO eval sparse output src/main/java/org/apache/sysds/runtime/lineage/LineageGPUCacheEviction.java (2 lines): - line 109: // TODO: Scale score after every cache hit if not LRU - line 227: // TODO: move to the shadow buffer. Convert to double precision only when reused. src/main/java/org/apache/sysds/hops/codegen/SpoofCompiler.java (2 lines): - line 226: // TODO: Code is unreachable here - line 288: //FIXME completely remove or load via resource stream (see builtin functions) src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64.java (2 lines): - line 169: //TODO generalize to non-double dense blocks - line 170: //TODO should we specify that the length HAS TO BE (ru - rl) * (ou - ol) == db.size()? or allow partial sets src/main/java/org/apache/sysds/runtime/instructions/cp/FunctionCallCPInstruction.java (2 lines): - line 253: //FIXME: interferes with reuse. Removes broadcasts before materialization - line 281: //FIXME: send _boundOutputNames instead of fpb.getOutputParams as src/main/python/systemds/scuro/representations/window.py (2 lines): - line 25: # TODO: move this into the aggregation class and add an aggregate() and a window(window_size) function there so they can use the same functionality. - line 45: ) # TODO: add actual aggregation function here src/main/python/systemds/operator/algorithm/builtin/garch.py (2 lines): - line 46: TODO: use BFGS or BHHH if it is available (this are go to methods) - line 47: TODO: (only then) extend to garch(p,q); otherwise the search space is way too big for the current method src/main/java/org/apache/sysds/runtime/instructions/cp/ParamservBuiltinCPInstruction.java (2 lines): - line 121: // FIXME: does not work if features are federated, but labels are not - line 233: // TODO: accumulate public keys with SEAL src/main/java/org/apache/sysds/runtime/instructions/fed/VariableFEDInstruction.java (2 lines): - line 48: // TODO: merge with `CastFEDInstruction` - line 99: // TODO Add write command to the federated site if the matrix has been modified src/main/java/org/apache/sysds/runtime/transform/encode/ColumnEncoderBin.java (2 lines): - line 317: // FIXME current NaN handling introduces 0s and thus - line 663: // TODO: Derive bin boundaries from partial aggregates, avoiding src/main/cuda/kernels/SystemDS.cu (2 lines): - line 265: * TODO: Alternative approach: use dynamic parallelism. We are skipping this - line 411: // TODO: Potential to reduce the number of threads by half src/main/java/org/apache/sysds/runtime/io/FrameWriterParquet.java (2 lines): - line 90: // TODO:Experiment with different batch sizes? - line 94: // Write data using ParquetWriter //FIXME replace example writer? src/main/java/org/apache/sysds/runtime/data/DenseBlockInt32.java (2 lines): - line 156: //TODO investigate potential deadlocks if already in parallel setting w/ commonPool - line 163: //TODO avoid array allocation src/main/java/org/apache/sysds/runtime/instructions/spark/ComputationSPInstruction.java (2 lines): - line 97: // TODO the dcOut dims will not be accurate here, because set output dimensions currently do only support - line 105: // TODO remove this once dcOut dims are accurate if known src/main/java/org/apache/sysds/hops/BinaryOp.java (2 lines): - line 984: // TODO size information for tensor - line 1041: //TODO quantile src/main/java/org/apache/sysds/hops/FunctionOp.java (2 lines): - line 227: // TODO: To allow for initial version to always run on the GPU - line 307: // TODO: To allow for initial version to always run on the GPU src/main/java/org/apache/sysds/runtime/lineage/LineageEstimator.java (2 lines): - line 72: // TODO: handling of parfor, statementblock reuse - line 73: // TODO: collect lineage tracing and probing overhead (computation) src/main/java/org/apache/sysds/runtime/instructions/gpu/DnnGPUInstruction.java (2 lines): - line 619: // TODO: find out if memset is necessary - line 825: // TODO: For now always copy the device data to host src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java (2 lines): - line 187: // TODO decompression Condition - line 357: // TODO handle of functions in a more fine-grained manner src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java (2 lines): - line 43: * TODO: extensions for faster incremental construction (e.g., max row) - line 44: * TODO more efficient fused setIndexRange impl to avoid repeated copies and updates src/main/java/org/apache/sysds/common/Types.java (2 lines): - line 79: //TODO remove from Data Type -> generic object - line 558: //low-level operators //TODO used? src/main/java/org/apache/sysds/hops/ParameterizedBuiltinOp.java (2 lines): - line 303: //TODO this special case could be taken into account for memory estimates in order - line 1084: private boolean isRemoveEmptyBcSP() // TODO find if 2 x size needed. src/main/java/org/apache/sysds/runtime/instructions/cp/SqlCPInstruction.java (2 lines): - line 85: // TODO: perf - line 114: //TODO reconsider exact value mapping numeric/decimal src/main/java/org/apache/sysds/parser/ParForStatementBlock.java (1 line): - line 258: * (TODO: in order to remove the last restriction, dependencies must be checked again after src/main/java/org/apache/sysds/runtime/instructions/fed/QuaternaryFEDInstruction.java (1 line): - line 196: // TODO: don't perform replacement over the whole instruction string, possibly changing string literals, src/main/java/org/apache/sysds/runtime/instructions/spark/utils/RDDConverterUtilsExt.java (1 line): - line 110: // TODO: Double-check if synchronization is required here. src/main/java/org/apache/sysds/runtime/iogen/template/FrameGenerateReaderParallel.java (1 line): - line 204: // TODO: implement parallel reader for input stream src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixNative.java (1 line): - line 176: //TODO flip upper triangular matrix down for consistent src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java (1 line): - line 212: // TODO make sparse decompression where the iterator is known in argument src/main/python/systemds/scuro/aligner/alignment_strategy.py (1 line): - line 40: # TODO: Add additional alignment methods src/main/java/org/apache/sysds/parser/FunctionStatementBlock.java (1 line): - line 38: * TODO: DRB: This needs to be changed to reflect: src/main/java/org/apache/sysds/hops/LeftIndexingOp.java (1 line): - line 197: //TODO also use worstcase estimate for output src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java (1 line): - line 524: // TODO sparse optimization src/main/java/org/apache/sysds/api/mlcontext/MLContextConversionUtil.java (1 line): - line 375: // TODO extend frame schema by column names (right now dropped) src/main/python/systemds/script_building/dag.py (1 line): - line 61: # TODO why do we not want to store the results? The execution script will should stay the same src/main/java/org/apache/sysds/runtime/instructions/spark/functions/CopyTensorBlockPairFunction.java (1 line): - line 64: // TODO: always create deep copies in more memory-efficient CSR representation src/main/java/org/apache/sysds/runtime/io/FrameReaderJSONL.java (1 line): - line 118: // TODO Needs Optimisation! "split" is inefficient src/main/java/org/apache/sysds/runtime/data/DenseBlockBool.java (1 line): - line 185: //TODO perf computed indexes src/main/java/org/apache/sysds/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java (1 line): - line 41: * TODO extend impl for binarycell and textcell src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDNN.java (1 line): - line 339: // TODO: Handle sparse image and dout cases: src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryTensorTensorCPInstruction.java (1 line): - line 47: // TODO Ensure right dense/sparse output representation (guarded by released input memory) src/main/java/org/apache/sysds/runtime/instructions/spark/CheckpointSPInstruction.java (1 line): - line 84: // TODO: Synchronize. Avoid double execution src/main/java/org/apache/sysds/runtime/io/TensorWriterTextCellParallel.java (1 line): - line 43: // TODO accurate estimation src/main/java/org/apache/sysds/runtime/transform/encode/EncoderOmit.java (1 line): - line 162: // TODO perf evaluate if column-wise scan more efficient src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkloadAnalyzer.java (1 line): - line 70: // TODO: Count transitive closure via lineage src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinCompress.java (1 line): - line 49: // FIXME Optional: recompress, else can be removed (lines 54-55) once fixed compression src/main/java/org/apache/sysds/runtime/instructions/spark/WriteSPInstruction.java (1 line): - line 331: // TODO: implement for libsvm src/main/java/org/apache/sysds/lops/Checkpoint.java (1 line): - line 52: * TODO change string parameter storage.level to StorageLevel as soon as we can assume src/main/java/org/apache/sysds/lops/UAggOuterChain.java (1 line): - line 34: * TODO Currently this lop only support the right hand side in distributed cache. This src/main/java/org/apache/sysds/runtime/compress/plan/CompressionPlanFactory.java (1 line): - line 64: // TODO check for overlapping src/main/java/org/apache/sysds/runtime/instructions/cp/AggregateUnaryCPInstruction.java (1 line): - line 259: // TODO support DataTensor src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSC.java (1 line): - line 35: * TODO implement row interface of sparse blocks (can be slow but must be correct; src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java (1 line): - line 110: // TODO add support for changing to SDCSINGLE. src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/ConstScheme.java (1 line): - line 83: // TODO specialize for sparse data. But would only be used in rare cases src/main/java/org/apache/sysds/runtime/io/ListWriter.java (1 line): - line 67: //TODO additional parallelization over objects (in addition to parallel writers) src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java (1 line): - line 783: //TODO specialize for COO, but but equivalent to existing sparse ops src/main/java/org/apache/sysds/runtime/functionobjects/ParameterizedBuiltin.java (1 line): - line 322: // TODO: more accurate distribution-specific computation of upper tail probabilities src/main/java/org/apache/sysds/hops/cost/CostEstimationWrapper.java (1 line): - line 49: //TODO config parameter? src/main/java/org/apache/sysds/runtime/instructions/cp/KahanObject.java (1 line): - line 100: // TODO Auto-generated method stub src/main/java/org/apache/sysds/hops/rewrite/RewriteRemoveReadAfterWrite.java (1 line): - line 65: || writes.get(rfname).getEndLine() t(t(y) %*% x) src/main/java/org/apache/sysds/runtime/io/cog/COGByteReader.java (1 line): - line 79: // TODO: When properly implementing BigTIFF, this could be a problem when not being able to skip bytes src/main/java/org/apache/sysds/runtime/matrix/data/sketch/countdistinctapprox/KMVSketch.java (1 line): - line 43: * TODO: Add multi-threaded version src/main/java/org/apache/sysds/runtime/instructions/cp/MatrixAppendCPInstruction.java (1 line): - line 74: // TODO: break append to cbind and rbind for full compilation chain src/main/java/org/apache/sysds/lops/DataGen.java (1 line): - line 90: // TODO size for tensor src/main/java/org/apache/sysds/api/mlcontext/MLContextUtil.java (1 line): - line 882: str = object.toString(); // TODO: Deal with OOM for other src/main/java/org/apache/sysds/runtime/lineage/LineageCodegenItem.java (1 line): - line 38: //TODO: test with parfor (synchronization, placeholders, etc) src/main/java/org/apache/sysds/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java (1 line): - line 44: * TODO (2) implement injection for multiple consumers (local and global). src/main/java/org/apache/sysds/runtime/io/WriterTextCSV.java (1 line): - line 254: * TODO: Remove this roundabout way! src/main/java/org/apache/sysds/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java (1 line): - line 97: // TODO: Assign #threads in compiler and pass via the instruction string src/main/java/org/apache/sysds/runtime/instructions/cp/SpoofCPInstruction.java (1 line): - line 91: //FIXME fused codegen operators already support compressed main inputs src/main/python/systemds/scuro/modality/joined_transformed.py (1 line): - line 65: ): # TODO: only if the layout is list_of_lists_of_numpy_array src/main/java/org/apache/sysds/runtime/instructions/cp/PrefetchCPInstruction.java (1 line): - line 46: // TODO: handle non-matrix objects src/main/java/org/apache/sysds/runtime/io/TensorReaderBinaryBlock.java (1 line): - line 61: // TODO reuse blocks src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java (1 line): - line 941: // TODO add sparse optmization src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java (1 line): - line 598: // TODO add previous command that would allow us to simplify this loop. src/main/java/org/apache/sysds/runtime/instructions/spark/FrameAppendRSPInstruction.java (1 line): - line 74: //TODO preserve info if already aligned, and only align if necessary src/main/java/org/apache/sysds/runtime/functionobjects/CTable.java (1 line): - line 156: * TODO: Support other cases src/main/java/org/apache/sysds/runtime/iogen/codegen/CodeGenTrie.java (1 line): - line 62: // TODO: same pattern for all columns but the ValueTypes are different- fix it ! src/main/java/org/apache/sysds/resource/cost/IOCostUtils.java (1 line): - line 527: // TODO: ensure the object related to stats is read in memory already ot add logic to account for its read time