apache / spark - Unit Size

Intro

Unit size measurements show the distribution of size of units of code (methods, functions...).
Units are classified in four categories based on their size (lines of code): 1-20 (small units), 20-50 (medium size units), 51-100 (long units), 101+ (very long units).
You should aim at keeping units small (< 20 lines). Long units may become "bloaters", code that have increased to such gargantuan proportions that they are hard to work with.

Learn more...

Unit Size Overall

There are 41,097 units with 443,389 lines of code in units (69.6% of code).

193 very long units (30,835 lines of code)
796 long units (53,411 lines of code)
4,000 medium size units (120,742 lines of code)
6,952 small units (100,315 lines of code)
29,156 very small units (138,086 lines of code)

Legend:

101+

51-100

21-50

11-20

1-10

Unit Size per Extension

101+

51-100

21-50

11-20

1-10

Unit Size per Logical Component

primary logical decomposition

101+

51-100

21-50

11-20

1-10

Alternative Visuals

3D view of all units

Longest Units

Top 50 longest units

Unit	# lines	McCabe index	# params
private def createToScalaConverter() in sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala	878	24	2
def checkAnalysis0() in sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala	544	225	1
def read_udtf() in python/pyspark/worker.py	533	139	3
private[deploy] def prepareSubmitEnvironment() in core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala	515	156	2
override def apply() in sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala	389	113	1
override def apply() in sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala	381	116	1
def read_udfs() in python/pyspark/worker.py	361	72	3
def apply() in sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/DecorrelateInnerQuery.scala	343	64	3
private[scheduler] def handleTaskCompletion() in core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala	336	81	1
private[arrow] def deserializerFor() in sql/connect/common/src/main/scala/org/apache/spark/sql/connect/client/arrow/ArrowDeserializer.scala	301	49	3
def unwrapperFor() in sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala	288	93	1
def render() in core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala	287	25	1
def checkForStreaming() in sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala	275	133	2
def user() in sql/api/src/main/scala/org/apache/spark/sql/functions.scala	260	3	0
private[arrow] def serializerFor[E]() in sql/connect/common/src/main/scala/org/apache/spark/sql/connect/client/arrow/ArrowSerializer.scala	255	64	2
def taskSummary() in core/src/main/scala/org/apache/spark/status/AppStatusStore.scala	253	9	3
private def generateExpression() in sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala	252	137	2
override def run() in core/src/main/scala/org/apache/spark/executor/Executor.scala	240	36	0
override def createEvaluator() in sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinEvaluatorFactory.scala	237	32	0
def deserialize() in sql/core/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala	234	77	1
def microsToDuration() in sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkIntervalUtils.scala	228	68	1
def prepareLocalResources() in resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala	212	30	2
def schema_of_xml() in sql/api/src/main/scala/org/apache/spark/sql/functions.scala	210	1	1
def handleMlCommand() in sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLHandler.scala	208	45	2
override def next() in core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala	206	42	0
private def codegenFullOuter() in sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala	202	26	1
private[sql] def transformWithState[U: Encoder]() in sql/connect/common/src/main/scala/org/apache/spark/sql/connect/KeyValueGroupedDataset.scala	202	1	5
private def newConverter() in sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala	201	67	3
protected override def doConsumeWithKeys() in sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala	197	15	2
private def newWriter() in connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/ProtobufDeserializer.scala	197	56	4
private def newConverter() in connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/ProtobufSerializer.scala	196	55	4
def apply() in sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala	195	70	1
def compute() in python/pyspark/pandas/correlation.py	194	9	3
def makeConverter() in sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala	192	86	1
def boost() in mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala	192	18	7
def apply() in sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala	191	81	1
override def doGenCode() in sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala	191	45	2
def fit() in mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala	189	40	2
private def resolveWithCTE() in sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveWithCTE.scala	185	27	3
protected def newWriter() in core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala	183	32	5
private def createDeserializer() in sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala	182	41	3
private def toSqlTypeHelper() in sql/core/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala	179	45	6
def resourceOffers() in core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala	179	32	2
private[tree] def findBestSplits() in mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala	178	21	12
def __setitem__() in python/pyspark/pandas/indexing.py	177	50	3
def isDeleted() in resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala	177	38	1
override def configurePod() in resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala	176	17	1
override def buildReaderWithPartitionValues() in sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala	170	18	8
protected override def doProduceWithKeys() in sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala	166	17	1
def maxBroadcastTableSizeInBytes: Long = getConf() in sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala	165	6	1