in client-spark/extension/src/main/scala/org/apache/spark/ui/ShufflePage.scala [80:290]
override def render(request: HttpServletRequest): Seq[Node] = {
val originWriteMetric = runtimeStatusStore.aggregatedShuffleWriteMetrics()
val originReadMetric = runtimeStatusStore.aggregatedShuffleReadMetrics()
// render header
val aggTaskInfo = runtimeStatusStore.aggregatedTaskInfo
val taskInfo =
if (aggTaskInfo == null)
AggregatedTaskInfoUIData(0, 0, 0, 0)
else
aggTaskInfo
val percent =
if (taskInfo.cpuTimeMillis == 0)
0
else
(taskInfo.shuffleWriteMillis + taskInfo.shuffleReadMillis).toDouble / taskInfo.cpuTimeMillis
// render build info
val buildInfo = runtimeStatusStore.buildInfo()
val buildInfoTableUI = UIUtils.listingTable(
propertyHeader,
propertyRow,
buildInfo.info,
fixedWidth = true
)
// render uniffle configs
val rssConf = runtimeStatusStore.uniffleProperties()
val rssConfTableUI = UIUtils.listingTable(
propertyHeader,
propertyRow,
rssConf.info,
fixedWidth = true
)
// render shuffle-servers write+read statistics
val shuffleWriteMetrics = shuffleSpeedStatistics(originWriteMetric.metrics.asScala.toSeq)
val shuffleReadMetrics = shuffleSpeedStatistics(originReadMetric.metrics.asScala.toSeq)
val shuffleHeader = Seq("Avg", "Min", "P25", "P50", "P75", "Max")
val shuffleMetricsRows = createShuffleMetricsRows(shuffleWriteMetrics, shuffleReadMetrics)
val shuffleMetricsTableUI =
<table class="table table-bordered table-sm table-striped">
<thead>
<tr>
{("Metric" +: shuffleHeader).map(header => <th>
{header}
</th>)}
</tr>
</thead>
<tbody>
{shuffleMetricsRows}
</tbody>
</table>
// render all assigned shuffle-servers
val allServers = unionByServerId(
originWriteMetric.metrics,
originReadMetric.metrics
)
val allServersTableUI = UIUtils.listingTable(
Seq("Shuffle Server ID", "Write Bytes", "Write Duration", "Write Speed (MB/sec)", "Read Bytes", "Read Duration", "Read Speed (MB/sec)"),
allServerRow,
allServers,
fixedWidth = true
)
// render reading hybrid storage statistics
val readMetrics = originReadMetric.metrics
val aggregatedByStorage = readMetrics.asScala.values
.flatMap { metric =>
Seq(
("MEMORY", metric.memoryByteSize, metric.memoryDurationMills),
("LOCALFILE", metric.localfileByteSize, metric.localfileDurationMillis),
("HADOOP", metric.hadoopByteSize, metric.hadoopDurationMillis)
)
}
.groupBy(_._1)
.mapValues { values =>
val totalBytes = values.map(_._2).sum
val totalTime = values.map(_._3).sum
val speed = if (totalTime != 0) totalBytes.toDouble / totalTime / 1000 else 0L
(totalBytes, totalTime, speed)
}
.toSeq
val readTableUI = UIUtils.listingTable(
Seq("Storage Type", "Read Bytes", "Read Time", "Read Speed (MB/sec)"),
{ row: (String, Long, Long, Double) =>
<tr>
<td>{row._1}</td>
<td>{Utils.bytesToString(row._2)}</td>
<td>{UIUtils.formatDuration(row._3)}</td>
<td>{roundToTwoDecimals(row._4)}</td>
</tr>
},
aggregatedByStorage.map { case (storageType, (bytes, time, speed)) =>
(storageType, bytes, time, speed)
},
fixedWidth = true
)
// render assignment info
val assignmentInfos = runtimeStatusStore.assignmentInfos
val assignmentTableUI = UIUtils.listingTable(
Seq("Shuffle ID", "Assigned Server Number"),
propertyRow,
assignmentInfos.map(x => (x.shuffleId.toString, x.shuffleServerIdList.size().toString)),
fixedWidth = true
)
val summary: NodeSeq = {
<div>
<div>
<ul class="list-unstyled">
<li id="completed-summary" data-relingo-block="true">
<a>
<strong>Total shuffle bytes:</strong>
</a>
{Utils.bytesToString(taskInfo.shuffleBytes)}
</li><li data-relingo-block="true">
<a>
<strong>Shuffle Duration (write+read) / Task Duration:</strong>
</a>
{UIUtils.formatDuration(taskInfo.shuffleWriteMillis + taskInfo.shuffleReadMillis)}
({UIUtils.formatDuration(taskInfo.shuffleWriteMillis)}+{UIUtils.formatDuration(taskInfo.shuffleReadMillis)})
/ {UIUtils.formatDuration(taskInfo.cpuTimeMillis)} = {roundToTwoDecimals(percent)}
</li>
</ul>
</div>
<div>
<span class="collapse-build-info-properties collapse-table"
onClick="collapseTable('collapse-build-info-properties', 'build-info-table')">
<h4>
<span class="collapse-table-arrow arrow-closed"></span>
<a>Uniffle Build Information</a>
</h4>
</span>
<div class="build-info-table collapsible-table collapsed">
{buildInfoTableUI}
</div>
</div>
<div>
<span class="collapse-uniffle-config-properties collapse-table"
onClick="collapseTable('collapse-uniffle-config-properties', 'uniffle-config-table')">
<h4>
<span class="collapse-table-arrow arrow-closed"></span>
<a>Uniffle Properties</a>
</h4>
</span>
<div class="uniffle-config-table collapsible-table collapsed">
{rssConfTableUI}
</div>
</div>
<div>
<span class="collapse-throughput-properties collapse-table"
onClick="collapseTable('collapse-throughput-properties', 'statistics-table')">
<h4>
<span class="collapse-table-arrow arrow-closed"></span>
<a>Shuffle Throughput Statistics</a>
</h4>
</span>
<div class="statistics-table collapsible-table collapsed">
{shuffleMetricsTableUI}
</div>
</div>
<div>
<span class="collapse-read-throughput-properties collapse-table"
onClick="collapseTable('collapse-read-throughput-properties', 'read-statistics-table')">
<h4>
<span class="collapse-table-arrow arrow-closed"></span>
<a>Hybrid Storage Read Statistics</a>
</h4>
</span>
<div class="read-statistics-table collapsible-table collapsed">
{readTableUI}
</div>
</div>
<div>
<span class="collapse-server-properties collapse-table"
onClick="collapseTable('collapse-server-properties', 'all-servers-table')">
<h4>
<span class="collapse-table-arrow arrow-closed"></span>
<a>Shuffle Server ({allServers.length})</a>
</h4>
</span>
<div class="all-servers-table collapsible-table collapsed">
{allServersTableUI}
</div>
</div>
<div>
<span class="collapse-assignment-properties collapse-table"
onClick="collapseTable('collapse-assignment-properties', 'assignment-table')">
<h4>
<span class="collapse-table-arrow arrow-closed"></span>
<a>Assignment ({assignmentInfos.length})</a>
</h4>
</span>
<div class="assignment-table collapsible-table collapsed">
{assignmentTableUI}
</div>
</div>
</div>
}
UIUtils.headerSparkPage(request, "Uniffle", summary, parent)
}