tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java (18 lines): - line 128: // TODO Ensure MAPREDUCE-4457 is factored in. Also MAPREDUCE-4068. - line 129: // TODO Consider TAL registration in the TaskAttempt instead of the container. - line 243: // TODO TEZ-2003 (post) TEZ-2667 We may need some additional state management for STATUS_UPDATES, FAILED, KILLED coming in before - line 493: // TODO Maybe consider not failing REDUCE tasks in this case. Also, - line 1011: // TODO: JobHistory - line 1012: // TODO Change to return a JobHistoryEvent. - line 1036: // TODO Incorporate MAPREDUCE-4838 - line 1096: // (int) cpuCounter.getValue()); // long to int? TODO: FIX. Same below - line 1154: // FIXME how do we store information regd completion events - line 1188: // FIXME how do we store information regd completion events - line 1310: // TODO Post TEZ-2003. Allow for a policy in the VMPlugin to define locality for different attempts. - line 1344: // TODO Jira post TEZ-2003 getVertex implementation is very inefficient. This should be via references, instead of locked table lookups. - line 1443: // TODO Resolve to host / IP in case of a local address. - line 1444: InetSocketAddress nodeHttpInetAddr = NetUtils.createSocketAddr(container.getNodeHttpAddress()); // TODO: Costly? - line 1457: // TODO Remove after HDFS-5098 - line 1677: // TODO maybe. For reuse ... Stacking pulls for a reduce task, even if the - line 1764: // TODO - TEZ-834. This assumes that the outputs were on that node - line 1884: // TODO at some point. Nodes may be interested in FetchFailure info. tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java (16 lines): - line 150: otherInfo.put(ATSConstants.CONFIG, new JSONObject()); // TODO: config from proto? - line 174: // TODO decide whether this goes into different events, - line 209: // TODO decide whether this goes into different events, - line 241: // TODO decide whether this goes into different events, - line 250: // TODO add other container info here? or assume AHS will have this? - line 251: // TODO container logs? - line 277: // TODO decide whether this goes into different events, - line 286: // TODO add other container info here? or assume AHS will have this? - line 287: // TODO container logs? - line 305: // TODO decide whether this goes into different events, - line 372: // TODO decide whether this goes into different events, - line 426: // TODO decide whether this goes into different events, - line 599: // TODO fix schedule/launch time to be events - line 676: // TODO add more on all other updated information - line 703: // TODO fix requested times to be events - line 742: // TODO fix requested times to be events tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java (15 lines): - line 209: // TODO decide whether this goes into different events, - line 250: // TODO decide whether this goes into different events, - line 284: // TODO decide whether this goes into different events, - line 294: // TODO add other container info here? or assume AHS will have this? - line 295: // TODO container logs? - line 323: // TODO decide whether this goes into different events, - line 333: // TODO add other container info here? or assume AHS will have this? - line 334: // TODO container logs? - line 353: // TODO decide whether this goes into different events, - line 428: // TODO decide whether this goes into different events, - line 501: // TODO decide whether this goes into different events, - line 687: // TODO fix schedule/launch time to be events - line 768: // TODO add more on all other updated information - line 795: // TODO fix requested times to be events - line 835: // TODO fix requested times to be events tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java (13 lines): - line 136: // TODO Metrics - line 210: // TODO Task should go back to SCHEDULED state TEZ-495 - line 323: // TODO: Recovery - line 643: //TODO Verify behaviour is Task is killed (no finished attempt) - line 651: // FIXME shouldn't this not count attempts killed after an attempt succeeds - line 661: // TODO Metrics - line 815: // TODO: Recovery - line 930: // FIXME need to handle getting finish time as this function - line 1003: // TODO (TEZ-2938) - line 1211: if (isRejection) { // TODO: remove as part of TEZ-3881. - line 1433: // TODO Metrics - line 1455: // TODO TEZ-1577. - line 1517: // TODO Metrics tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java (13 lines): - line 233: // TODO Metrics - line 1324: // FIXME a better lightweight approach for counters is needed - line 1708: // TODO TEZ-2246 hints size must match num tasks - line 1953: // TODO TEZ-937 - add e mechanism to query vertex managers, or for VMs to indicate readines - line 2134: // TODO Metrics - line 3085: // TODO move before to handle NEW state - line 3447: // TODO: Metrics - line 3516: // TODO this is broken after rerun - line 3622: // TODO: Metrics - line 3772: // TODO Metrics - line 3781: // TODO Metrics - line 3787: // TODO Metrics - line 4651: // TODO DAGAM This should be based on the edge type. tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java (12 lines): - line 171: // TODO Recovery - line 193: // TODO Metrics - line 638: // TODO maybe removed after TEZ-74 - line 748: // FIXME a better lightweight approach for counters is needed - line 1617: // TODO TEZ-2003 (post) TEZ-2624 Ideally, this should be per source. - line 1876: // TODO Metrics - line 1893: // TODO Metrics - line 1922: // TODO Metrics - line 2032: // TODO Metrics - line 2272: // TODO: Metrics - line 2282: // TODO: Metrics - line 2425: // TODO TEZ-2250 go to TERMINATING to wait for all vertices and commits completed tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java (10 lines): - line 433: //TODO these bytes should be versioned - line 447: //TODO this should be returning a class not just an int - line 463: //TODO these bytes should be versioned - line 487: // TODO these bytes should be versioned - line 490: // TODO: Once SHuffle is out of NM, this can use MR APIs - line 495: // TODO add API to AuxiliaryServices to report failures - line 507: // TODO add API to AuxiliaryServices to report failures - line 558: // TODO change AbstractService to throw InterruptedException - line 656: // TODO add API to AuxiliaryServices to report failures - line 1618: // TODO Change this over to just store local dir indices, instead of the tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java (10 lines): - line 127: // TODO how to update progress - line 160: // TODO Post MRR - line 192: // TODO Can this be avoided all together. Have the MRTezOutputCommitter use - line 194: // TODO This could be fetched from the env if YARN is setting it for all - line 216: // TODO NEWTEZ Is most of this functionality required ? - line 253: // TODO NEWTEZ Is this required ? - line 359: // TODO change this to use the new context - line 360: // TODO TEZ Interaciton between Commit and OutputReady. Merge ? - line 365: // TODO EVENTUALLY - Commit is not required for map tasks. - line 379: // TODO call progress update here if not being called within Map/Reduce tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java (9 lines): - line 46: // TODO EVENTUALLY A description for each property. - line 126: // TODO Use the default value - line 191: * TODO TEZ-3303 Given ShuffleVertexManager doesn't consume precise stats - line 500: // TODO Move this key to MapReduce - line 543: * Speculative execution needs to be turned off when using this parameter. //TODO: TEZ-2132 - line 552: * Speculative execution needs to be turned off when disabling this parameter. //TODO: TEZ-2132 - line 592: // TODO TEZ-1233 - allow this property to be set per vertex - line 593: // TODO TEZ-1231 - move these properties out since they are not relevant for Inputs / Outputs - line 690: // TODO TEZ-1232 Filter out parameters from TezConfiguration, and Task specific confs tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManager.java (8 lines): - line 156: // TODO TEZ-2118 Start using taskCommunicator indices properly - line 161: // TODO Why is init tied to serviceStart - line 245: // TODO TEZ-2003 (post) TEZ-2666. An exception back is likely a better approach than sending a shouldDie = true, - line 387: // TODO TEZ-2003 (post) TEZ-2671 Maybe consider un-registering here itself, since the task is not active anymore, - line 400: // TODO TEZ-2003 (post) TEZ-2671 Maybe consider un-registering here itself, since the task is not active anymore, - line 453: // TODO TEZ-2335. Cleanup TaskHeartbeat handler structures. - line 454: // TODO TEZ-2345. Also cleanup attemptInfo map, so that any tasks which heartbeat are told to die. - line 458: // TODO TEZ-2336. Send a signal to containers indicating DAG completion. tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java (7 lines): - line 127: // TODO Consider registering with the TAL, instead of the TAL pulling. - line 130: // TODO What should be done with pendingAttempts. Nullify when handled ? - line 133: // TODO Create a generic ERROR state. Container tries informing relevant components in this case. - line 279: // TODO This transition is wrong. Should be a noop / error. - line 447: // TODO Can't set state to COMPLETED. Add a default error state. - line 489: // TODO TEZ-2625 This should ideally be handled inside of user code. Will change once - line 1065: // TODO CREUSE CRITICAL: This is completely incorrect. COMPLETED comes tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java (7 lines): - line 223: // TODO More counters - FetchErrors, speed? - line 513: //TODO: TEZ-2132 for error handling. As of now, fail fast if there is a different attempt - line 747: // TODO NEWTEZ Maybe inform the fetcher about this. For now, this is used during the initial fetch list construction. - line 844: // TODO NEWTEZ Maybe inform fetchers, in case they have an alternate attempt of the same task in their queue. - line 865: // TODO Should eventually be controlled by Inputs which are processing the data. - line 950: // TODO NEWTEZ. Implement logic to report fetch failures after a threshold. - line 1012: //TODO: need to cleanup all FetchedInput (DiskFetchedInput, LocalDisFetchedInput), lockFile tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java (7 lines): - line 77: /* TODO not yet updating cluster nodes on every allocate response - line 281: // TODO Post TEZ-2003. Make all of these final fields. - line 447: // TODO later we may want to check if exit code matched expectation - line 907: // TODO this will not handle dynamic changes in resources - line 966: // TODO check for nulls etc - line 967: // TODO extra memory allocation - line 1263: // TODO this is subject to error wrt RM resource normalization tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java (7 lines): - line 375: // TODO what happens to the task that was connected to this container? - line 392: // TODO does this container need to be stopped via C_STOP_REQUEST - line 431: // TODO either ways send the necessary events - line 767: // TODO - perhaps check if the task still needs this container - line 900: // TODO Handle this in TEZ-2124. Need a way to know which scheduler is calling in. - line 986: // TODO Why is this making a call back into the scheduler, when the call is originating from there. - line 1082: // FIXME: we should not need this check in the first place. tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientServiceDelegate.java (6 lines): - line 56: // FIXME needs counters support from DAG - line 65: // FIXME seems like there is support in client to query task failure - line 74: // FIXME need support to query task diagnostics? - line 92: // FIXME need support to kill a task attempt? - line 98: // FIXME need support to kill a dag? - line 107: // FIXME logs for an attempt? tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java (6 lines): - line 311: // FIXME fix pattern support - line 323: // TODO gross hack - line 337: // FIXME isn't this a nice mess of a client? - line 472: // TODO Add localResources and Environment - line 568: // FIXME set up job resources - line 572: // FIXME createDAG should take the tezConf as a parameter, instead of using tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java (6 lines): - line 422: @ConfigurationScope(Scope.AM) // TODO DAG/Vertex level - line 434: @ConfigurationScope(Scope.AM) // TODO DAG/Vertex level - line 620: @ConfigurationScope(Scope.VERTEX) // TODO Verify the vertex speculation, TEZ-1788 - line 991: @ConfigurationScope(Scope.DAG) // TODO vertex level - line 1000: @ConfigurationScope(Scope.DAG) // TODO vertex level - line 2063: // TODO only validate property here, value can also be validated if necessary tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java (5 lines): - line 161: //TODO Clean this and other maps at some point - line 698: // TODO NEWTEZ Should this be releasing the output, if not committed ? Possible memory leak in case of speculation. - line 703: //TODO: TEZ-2132 for error handling. As of now, fail fast if there is a different attempt - line 833: // TODO TEZ-922 hostFailures isn't really used for anything apart from - line 1223: //TODO: need to handle deterministic spills later. tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java (5 lines): - line 133: // TODO Set additional countesr - total bytes written, spills etc. - line 359: //TODO: fix per item being passed. - line 543: //TODO: honor cache limits - line 634: //TODO: honor cache limits - line 1430: // TODO: a better threshold check than 1 key repeating tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskAttemptContextImpl.java (5 lines): - line 74: // FIXME we need to use DAG Id but we are using App Id - line 78: // TODO NEWTEZ Can the jt Identifier string be taskContext.getUniqueId ? - line 83: //FIXME we need to use DAG Id but we are using App Id - line 86: // TODO NEWTEZ Can the jt Identifier string be taskContext.getUniqueId ? - line 98: // TODO NEWTEZ This is broken. Mainly set after all records are processed. Not set for Inputs/Outputs tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezContainerLauncherImpl.java (5 lines): - line 72: // TODO See what part of this lifecycle and state management can be simplified. - line 75: // TODO Review this entire code and clean it up. - line 82: // TODO Ensure the same thread is used to launch / stop the same container. Or - ensure event ordering. - line 334: // TODO: Group launching of multiple containers to a single - line 399: // TODO: Do it only once per NodeManager. tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java (5 lines): - line 276: //TODO: Cap it to MAX_VALUE until MapOutput starts supporting > 2 GB - line 801: // TODO Is this doing any combination ? - line 874: // TODO Maybe track serialized vs deserialized bytes. - line 909: // TODO Counters for Combine - line 1041: // TODO Maybe differentiate between data written because of Merges and tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java (4 lines): - line 73: // TODO Auto-generated method stub - line 180: // TODO use new method in MRInput to get required info - line 210: // TODO use new method in MRInput to get required info - line 321: // TODO broken tez-dag/src/main/java/org/apache/tez/dag/app/web/AMWebController.java (4 lines): - line 539: // TODO: add an error message instead for counter key - line 615: // TODO: add an error message instead for counter key - line 775: // TODO: add an error message instead for counter key - line 873: // TODO: add an error message instead for counter key tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java (3 lines): - line 350: // JDK7 - TODO: use Files implementation to speed up this process - line 414: // JDK7 - TODO: use AsynchronousFileChannel instead of RandomAccessFile - line 950: // TODO TEZ-957. handle IOException here when Broadcast has better error checking tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java (3 lines): - line 129: //TODO: Do we need to support sending payloads via events? - line 186: // TODO Maybe add helper methods to extract keys - line 187: // TODO Maybe add constants or an Enum to access the keys tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java (3 lines): - line 80: * TODO: TEZ-2722 - line 104: * TODO: TEZ-2722 - line 129: * TODO: TEZ-2722 tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/DAGJobStatus.java (3 lines): - line 222: // TODO Auto-generated method stub - line 277: // FIXME handle retired jobs? - line 283: // FIXME handle history in status tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java (3 lines): - line 312: // TODO May not need to be a bidi map - line 993: // TODO:currently just wait for some time so clients can know the - line 1050: // TODO Does this move to the client in case of work-preserving recovery. tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java (3 lines): - line 60: // TODO NEWTEZ Figure out how this can be improved. i.e. access from context instead of setting in conf ? - line 110: int numSplits = WritableUtils.readVInt(in); // TODO: check for insane values - line 146: final int numSplits = WritableUtils.readVInt(in); // TODO: check for insane values tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java (3 lines): - line 568: // Environment variables used by Pipes. (TODO: these - line 631: // TODO Fix this. Not accessible in JobClient - line 648: // TODO Move this over to Tez tez-dag/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicator.java (3 lines): - line 30: // TODO TEZ-2003 (post) TEZ-2665. Move to the tez-api module - line 31: // TODO TEZ-2003 (post) TEZ-2664. Ideally, don't expose YARN containerId; instead expose a Tez specific construct. - line 47: // TODO TEZ-2003 (post) TEZ-2666 Enhancements to interface tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java (3 lines): - line 35: // FIXME hardcoded assumption that one app is one dag - line 61: // FIXME hardcoded assumption that one app is one dag - line 68: // FIXME hack alert converting objects with hard coded id tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java (3 lines): - line 200: //TODO: might not be respected when fetcher is in progress / server is busy. TEZ-711 - line 456: // TODO Review: Multiple header reads in case of status WAIT ? - line 771: //TODO: Refactor following to make use of methods from TezTaskOutputFiles to be consistent. tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java (3 lines): - line 122: public void setEventReceivedTime(long eventReceivedTime) { // TODO save - line 159: // TODO NEWTEZ convert to PB - line 259: // TODO NEWTEZ convert to PB tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SkewAnalyzer.java (3 lines): - line 135: //TODO: Tez counters always ends up adding fgroups and groups, due to which we end up - line 194: //TODO: Tez counters always ends up adding fgroups and groups, due to which we end up - line 253: //TODO: Tez counters always ends up adding fgroups and groups, due to which we end up tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java (3 lines): - line 176: // TODO Remove jobToken from here post TEZ-421 - line 371: // TODO Maybe close initialized inputs / outputs in case of failure to - line 760: // TODO: Don't catch Throwables. tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/readers/UnorderedKVReader.java (2 lines): - line 74: // TODO Remove this once per I/O counters are separated properly. Relying on - line 107: // TODO NEWTEZ Maybe add an interface to check whether next will block. tez-dag/src/main/java/org/apache/tez/dag/app/dag/RootInputInitializerManager.java (2 lines): - line 189: // TODO Rework the flow as per the first comment on TEZ-1076 - line 440: // TODO TEZ-1577. Get rid of this. tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java (2 lines): - line 614: * TODO: When better stats are available, this condition can be avoided. - line 865: // TODO Would it ever make sense to make this an in-memory writer ? tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezTaskContextImpl.java (2 lines): - line 99: // TODO Maybe change this to be task id specific at some point. For now - line 105: // TODO NEWTEZ at some point dag attempt should not map to app attempt tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java (2 lines): - line 73: // TODO NEWTEZ Progress reporting to Tez framework. (making progress vs %complete) - line 1287: // TODO Change event generation to say there is no data rather than generating a dummy file tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java (2 lines): - line 372: // TODO add later after TEZ-1714 @Override - line 444: // TODO TEZ-2066 tracks moving this async. tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java (2 lines): - line 390: // TODO Remove this method once we move to the Protobuf RPC engine - line 396: // TODO Remove this method once we move to the Protobuf RPC engine tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java (2 lines): - line 413: // TODO Maybe add helper methods to extract keys - line 414: // TODO Maybe add constants or an Enum to access the keys tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java (2 lines): - line 57: // TODO Default value handling. - line 98: // TODO TEZAM4 Sometime, make sure this gets loaded by default. Instead of the current initialization in MRAppMaster, TezChild. tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java (2 lines): - line 272: //TODO: Make use of TezSharedExecutor later - line 1112: // TODO Inefficient. If spills are not compressed, a direct copy should be possible tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java (2 lines): - line 61: // TODO Move defaults over to a constants file. - line 114: // TODO Fix name tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java (2 lines): - line 324: // TODO TEZ-1798. Handle contents of Tez archives for duplicate LocalResource checks - line 498: // TODO these bytes should be versioned tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRCounters.java (2 lines): - line 218: // TODO Auto-generated method stub - line 286: // TODO Auto-generated method stub tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowestVertexAnalyzer.java (2 lines): - line 104: //TODO: Tez counters always ends up adding fgroups and groups, due to which we end up - line 125: //ignore. TODO: hack for taskcounter issue tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java (2 lines): - line 283: // TODO Maybe add helper methods to extract keys - line 284: // TODO Maybe add constants or an Enum to access the keys tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskSchedulerContext.java (2 lines): - line 69: // TODO TEZ-2003 (post) TEZ-2664. Remove references to YARN constructs like Container, ContainerStatus, NodeReport - line 70: // TODO TEZ-2003 (post) TEZ-2668 Enhancements to TaskScheduler interfaces tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRReporter.java (2 lines): - line 45: //TODO NEWTEZ - line 90: // TODO NEWTEZ This is likely broken. Only set on task complete in Map/ReduceProcessor tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java (2 lines): - line 47: // FIXME Add support for service meta data comparisons - line 126: // TODO Once LRs are handled via YARN, remove this check - and ensure tez-dag/src/main/java/org/apache/tez/serviceplugins/api/TaskHeartbeatRequest.java (2 lines): - line 22: // TODO TEZ-2003 (post) TEZ-2665. Move to the tez-api module - line 25: // TODO TEZ-2003 (post) TEZ-2666 Ideally containerIdentifier should not be part of the request. tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java (2 lines): - line 85: //TODO: Sorter classes should override this method later. - line 357: //TODO: What if when same volume rename happens (have to rely on job completion cleanup) tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/Constants.java (2 lines): - line 25: // TODO NEWTEZ Check which of these constants are expecting specific pieces of information which are being removed - like taskAttemptId - line 39: // TODO NEWTEZ Remove this constant once the old code is removed. tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/DiskFetchedInput.java (2 lines): - line 102: // TODO NEWTEZ Maybe defer this to container cleanup - line 116: // TODO NEWTEZ Maybe defer this to container cleanup tez-dag/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorContext.java (2 lines): - line 32: // TODO TEZ-2003 (post) TEZ-2665. Move to the tez-api module - line 35: // TODO TEZ-2003 (post) TEZ-2666 Enhancements to API tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner2.java (2 lines): - line 74: // TODO It may be easier to model this as a state machine. - line 571: // TODO Ideally differentiate between FAILED/KILLED tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java (2 lines): - line 263: // TODO Maybe add helper methods to extract keys - line 264: // TODO Maybe add constants or an Enum to access the keys tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherContext.java (2 lines): - line 27: // TODO TEZ-2003 (post) TEZ-2664 Tez abstraction for ContainerId, NodeId, other YARN constructs - line 61: // TODO TEZ-2003 (post). TEZ-2676 TaskAttemptEndReason does not belong here, and is an unnecessary leak. tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java (2 lines): - line 201: // TODO Handle the case where a node is created due to the RM reporting it's - line 246: // TODO Should this fail the job ? tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/TezRuntimeUtils.java (2 lines): - line 69: // TODO Maybe include a dag name in this. - line 264: //TODO: support other async packages? httpclient-async? tez-runtime-library/src/main/java/org/apache/tez/runtime/library/resources/WeightedScalingMemoryDistributor.java (2 lines): - line 154: // TODO Later - If requestedSize is used, the difference (allocated - - line 203: // TODO Later - If requestedSize is used, the difference could be allocated to others. tez-runtime-internals/src/main/java/org/apache/tez/common/ContainerContext.java (2 lines): - line 28: // TODO EVENTUALLY move this over to PB. Fix package/module. - line 29: // TODO EVENTUALLY unit tests for functionality. tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java (2 lines): - line 162: // TODO Maybe add helper methods to extract keys - line 163: // TODO Maybe add constants or an Enum to access the keys tez-api/src/main/java/org/apache/tez/common/counters/TaskCounter.java (2 lines): - line 27: // TODO Eventually, rename counters to be non-MR specific and map them to MR equivalent. - line 106: * //TODO: As of now supporting broadcast data only. tez-common/src/main/java/org/apache/tez/runtime/common/resources/InitialMemoryRequestContext.java (1 line): - line 34: // TODO Replace this with the entire descriptor at some point. ComponentType tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java (1 line): - line 77: // TODO Auto-generated method stub tez-runtime-library/src/main/java/org/apache/tez/runtime/library/hadoop/compat/NullProgressable.java (1 line): - line 28: // TODO Auto-generated constructor stub tez-dag/src/main/java/org/apache/tez/serviceplugins/api/TaskHeartbeatResponse.java (1 line): - line 21: // TODO TEZ-2003 (post) TEZ-2665. Move to the tez-api module tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java (1 line): - line 86: // TODO Until TEZ-1013 we cannot handle custom input formats tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrderControlled.java (1 line): - line 74: // TODO Does ordering matter - it currently depends on the order returned by vertex.getOutput* tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerAllocator.java (1 line): - line 23: // TODO XXX Rename to AMScheduler. tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java (1 line): - line 36: // TODO Remove unused methods tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ValuesIterator.java (1 line): - line 114: // TODO NEWTEZ Maybe add another method which returns an iterator instead of iterable tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEventType.java (1 line): - line 21: // TODO - Re-use the events in ContainerLauncher.. tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherOperationBase.java (1 line): - line 27: // TODO TEZ-2702 (TEZ-2003 post) tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/InputSpec.java (1 line): - line 63: // TODO TEZ-305 convert this to PB tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherManager.java (1 line): - line 155: // TODO Post TEZ-2003. LocalContainerLauncher is special cased, since it makes use of tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java (1 line): - line 50: private int targetIndex; // TODO Likely to be multiple at a later point. tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetchedInputAllocatorOrderedGrouped.java (1 line): - line 25: // TODO TEZ-912 Consolidate this with FetchedInputAllocator. tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java (1 line): - line 53: // TODO TEZ-2003 (post) TEZ-2669 Propagate errors back to the AM with proper error reporting tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java (1 line): - line 237: // TODO MR Compat: Check against max block locations per split. tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java (1 line): - line 62: //TODO: unused. Consider removing later? tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java (1 line): - line 97: // TODO: TEZ-4607: replace with CommonConfigurationKeysPublic enum values tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskScheduler.java (1 line): - line 42: // TODO TEZ-2003 (post) TEZ-2668 tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java (1 line): - line 41: * should be invoked only once per task. TODO Eventually - change this so that tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java (1 line): - line 40: // TODO MRR FIXME based on conf format. tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java (1 line): - line 203: //TODO: support for multiple dags per ATS file later. tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowTaskIdentifier.java (1 line): - line 41: * //TODO: We do not get counters for killed task attempts yet. tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTaskReporter.java (1 line): - line 73: // TODO FIXME NEWTEZ - will MROutput's reporter use this api? tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/Partitioner.java (1 line): - line 31: * TODO NEWTEZ Change construction to first check for a Constructor with a bytep[] payload tez-dag/src/main/java/org/apache/tez/dag/utils/TaskSpecificLaunchCmdOption.java (1 line): - line 200: * TODO: this is horrible way to check the ranges. tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventType.java (1 line): - line 40: //TODO ZZZ CREUSE: Consider introducing a new event C_NODE_BLACKLISTED -> container can take a call on what to do if this event comes in. tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java (1 line): - line 138: //TODO: Not getting populated correctly for lots of jobs. Verify tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEventHandler.java (1 line): - line 150: // TODO at some point we should look at removing this once tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java (1 line): - line 362: // TODO must be done later after TEZ-1714 tez-api/src/main/java/org/apache/tez/common/ATSConstants.java (1 line): - line 26: // TODO remove once YARN exposes proper constants tez-ui/src/main/webapp/app/models/abstract.js (1 line): - line 31: //TODO - Find a better alternative to detect property change in a model tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java (1 line): - line 181: // TODO remove this once RootInputVertexManager is fixed to not use tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfo.java (1 line): - line 30: // TODO Fix this to be more usable. Interface is broken since half the methods apply to only a specific type. tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java (1 line): - line 63: // TODO Maybe validate that the correct value is being set. tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java (1 line): - line 272: // TODO Post TEZ-2003. Consolidate these states, and mappings. tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImpl.java (1 line): - line 184: // TODO Is this correct for recovery ? tez-dag/src/main/java/org/apache/tez/dag/app/dag/StateChangeNotifier.java (1 line): - line 113: // TODO send user code exception - TEZ-2332 tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java (1 line): - line 126: // TODO Post TEZ-2003. Most of this information is dynamic and only available after the AM tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/SimpleExponentialTaskRuntimeEstimator.java (1 line): - line 172: //TODO What is the best way to measure variance in runtime tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/SimpleFetchedInputAllocator.java (1 line): - line 111: //TODO: cap it to MAX_VALUE until MemoryFetchedInput can handle > 2 GB tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFileInputStream.java (1 line): - line 267: //TODO: add checksumSize to currentOffset. tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/BaseUnorderedPartitionedKVWriter.java (1 line): - line 178: //TODO: UnorderedPartitionedKVWriter should override this method later. tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseInfo.java (1 line): - line 115: //TODO: FS, TaskCounters are directly getting added as TezCounters always pass those. Need a tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFileOutputStream.java (1 line): - line 105: // FIXME if needed re-enable this in debug mode tez-api/src/main/java/org/apache/tez/dag/api/EntityDescriptor.java (1 line): - line 116: // TODO: TEZ-305 - using protobuf serde instead of Writable serde. tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java (1 line): - line 64: //TODO: let this be a map of values in protobuf 3.x tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLaunchRequest.java (1 line): - line 50: // TODO Post TEZ-2003. TEZ-2625. ContainerLaunchContext needs to be built here instead of being passed in. tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java (1 line): - line 26: // TODO These two parameters really aren't required in this event. tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java (1 line): - line 29: // TODO Get rid of remoteTask from here. Can be forgotten after it has been assigned. tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java (1 line): - line 227: // TODO what if derived class has already called this tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java (1 line): - line 407: // TODO clean up with better usage and error handling tez-ui/src/main/webapp/app/serializers/loader.js (1 line): - line 25: // TODO - Move to more js tez-dag/src/main/java/org/apache/tez/dag/utils/TezRuntimeChildJVM.java (1 line): - line 100: // TODO Is this StringBuilder really required ? YARN already accepts a list of commands. tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/OutputSpec.java (1 line): - line 59: // TODO TEZ-305 convert this to PB tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskSpec.java (1 line): - line 206: // TODO TEZ-305 convert this to PB tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java (1 line): - line 332: payloadBuilder.setRunDuration(0); //TODO: who is dependent on this? tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacySpeculator.java (1 line): - line 474: // TODO track the tasks that are potentially worth looking at tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java (1 line): - line 49: // TODO Add support for counters - merged with the Vertex counters. tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ResourceMgrDelegate.java (1 line): - line 98: // TODO: Implement getBlacklistedTrackers tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java (1 line): - line 36: // FIXME we need to use DAG Id but we are using App Id tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexManager.java (1 line): - line 104: // TODO TEZ-1714 for locking issues getContext().vertexManagerDone(); tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java (1 line): - line 157: // FIXME we are using ApplicationId as DAG id tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/PerSourceNodeTracker.java (1 line): - line 199: // TODO TEZ-2337 Maybe reset failures from previous DAGs tez-dag/src/main/java/org/apache/tez/dag/app/launcher/DeletionTrackerImpl.java (1 line): - line 63: //TODO: add check for healthy node tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java (1 line): - line 31: // TODO Maybe have tht TAL pull the remoteTask from the TaskAttempt itself ? tez-runtime-library/findbugs-exclude.xml (1 line): - line 88: tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryWriter.java (1 line): - line 38: // TODO Verify and fix counters if required. tez-dag/src/main/java/org/apache/tez/dag/app/rm/DagAwareYarnTaskScheduler.java (1 line): - line 739: // TODO this will not handle dynamic changes in resources tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopFailed.java (1 line): - line 24: // TODO XXX Not being used for anything. May be useful if we rely less on tez-api/src/main/java/org/apache/tez/common/Preconditions.java (1 line): - line 25: //TODO remove this and make Preconditions class in common module available everywhere tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java (1 line): - line 125: // TODO Change this to use YARN utilities once YARN-1664 is fixed. tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java (1 line): - line 49: // container. TODO: Document the case. tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/Shuffle.java (1 line): - line 134: // TODO TEZ Get rid of Map / Reduce references. tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java (1 line): - line 183: // TODO Deprecation reason does not seem to reflect in the config ? tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInput.java (1 line): - line 33: WAIT, // TODO NEWTEZ Implement this, only if required. tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java (1 line): - line 506: // TODO TEZ-1233. This needs to be moved over the wire rather than localizing the file tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleVertexManager.java (1 line): - line 200: // TODO TEZ-3500 tez-dag/src/main/java/org/apache/tez/dag/app/ContainerHeartbeatHandler.java (1 line): - line 38: // TODO Maybe define separate timeouts for Containers and tasks. tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientCache.java (1 line): - line 40: //TODO: evict from the cache on some threshold tez-runtime-library/src/main/java/org/apache/tez/http/async/netty/AsyncHttpConnection.java (1 line): - line 90: * TODO : following settings need fine tuning.