site/error-message-guidelines.html (545 lines of code) (raw):
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>
Error message guidelines | Apache Spark
</title>
<link href="/css/bootstrap.min.css" rel="stylesheet">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=DM+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&Courier+Prime:wght@400;700&display=swap" rel="stylesheet">
<link href="/css/custom.css" rel="stylesheet">
<!-- Code highlighter CSS -->
<link href="/css/pygments-default.css" rel="stylesheet">
<link rel="icon" href="/favicon.ico" type="image/x-icon">
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
_paq.push(["disableCookies"]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '40']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
</head>
<body class="global">
<nav class="navbar navbar-expand-lg navbar-dark p-0 px-4" style="background: #1D6890;">
<a class="navbar-brand" href="/">
<img src="/images/spark-logo-rev.svg" alt="" width="141" height="72">
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarContent"
aria-controls="navbarContent" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse col-md-12 col-lg-auto pt-4" id="navbarContent">
<ul class="navbar-nav me-auto">
<li class="nav-item">
<a class="nav-link active" aria-current="page" href="/downloads.html">Download</a>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="libraries" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Libraries
</a>
<ul class="dropdown-menu" aria-labelledby="libraries">
<li><a class="dropdown-item" href="/sql/">SQL and DataFrames</a></li>
<li><a class="dropdown-item" href="/spark-connect/">Spark Connect</a></li>
<li><a class="dropdown-item" href="/streaming/">Spark Streaming</a></li>
<li><a class="dropdown-item" href="/pandas-on-spark/">pandas on Spark</a></li>
<li><a class="dropdown-item" href="/mllib/">MLlib (machine learning)</a></li>
<li><a class="dropdown-item" href="/graphx/">GraphX (graph)</a></li>
<li>
<hr class="dropdown-divider">
</li>
<li><a class="dropdown-item" href="/third-party-projects.html">Third-Party Projects</a></li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="documentation" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Documentation
</a>
<ul class="dropdown-menu" aria-labelledby="documentation">
<li><a class="dropdown-item" href="/docs/latest/">Latest Release</a></li>
<li><a class="dropdown-item" href="/documentation.html">Older Versions and Other Resources</a></li>
<li><a class="dropdown-item" href="/faq.html">Frequently Asked Questions</a></li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link active" aria-current="page" href="/examples.html">Examples</a>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="community" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Community
</a>
<ul class="dropdown-menu" aria-labelledby="community">
<li><a class="dropdown-item" href="/community.html">Mailing Lists & Resources</a></li>
<li><a class="dropdown-item" href="/contributing.html">Contributing to Spark</a></li>
<li><a class="dropdown-item" href="/improvement-proposals.html">Improvement Proposals (SPIP)</a>
</li>
<li><a class="dropdown-item" href="https://issues.apache.org/jira/browse/SPARK">Issue Tracker</a>
</li>
<li><a class="dropdown-item" href="/powered-by.html">Powered By</a></li>
<li><a class="dropdown-item" href="/committers.html">Project Committers</a></li>
<li><a class="dropdown-item" href="/history.html">Project History</a></li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="developers" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Developers
</a>
<ul class="dropdown-menu" aria-labelledby="developers">
<li><a class="dropdown-item" href="/developer-tools.html">Useful Developer Tools</a></li>
<li><a class="dropdown-item" href="/versioning-policy.html">Versioning Policy</a></li>
<li><a class="dropdown-item" href="/release-process.html">Release Process</a></li>
<li><a class="dropdown-item" href="/security.html">Security</a></li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="github" role="button" data-bs-toggle="dropdown" aria-expanded="false">
GitHub
</a>
<ul class="dropdown-menu" aria-labelledby="github">
<li><a class="dropdown-item" href="https://github.com/apache/spark">spark</a></li>
<li><a class="dropdown-item" href="https://github.com/apache/spark-connect-go">spark-connect-go</a></li>
<li><a class="dropdown-item" href="https://github.com/apache/spark-connect-swift">spark-connect-swift</a></li>
<li><a class="dropdown-item" href="https://github.com/apache/spark-docker">spark-docker</a></li>
<li><a class="dropdown-item" href="https://github.com/apache/spark-kubernetes-operator">spark-kubernetes-operator</a></li>
<li><a class="dropdown-item" href="https://github.com/apache/spark-website">spark-website</a></li>
</ul>
</li>
</ul>
<ul class="navbar-nav ml-auto">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="apacheFoundation" role="button"
data-bs-toggle="dropdown" aria-expanded="false">
Apache Software Foundation
</a>
<ul class="dropdown-menu" aria-labelledby="apacheFoundation">
<li><a class="dropdown-item" href="https://www.apache.org/">Apache Homepage</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/licenses/">License</a></li>
<li><a class="dropdown-item"
href="https://www.apache.org/foundation/sponsorship.html">Sponsorship</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/foundation/thanks.html">Thanks</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/events/current-event">Event</a></li>
</ul>
</li>
</ul>
</div>
</nav>
<div class="container">
<div class="row mt-4">
<div class="col-12 col-md-9">
<h2 id="error-message-guidelines">Error Message Guidelines</h2>
<p>This guide is a reference for composing standardized and actionable
error messages in Apache Spark.</p>
<h3 id="include-what-why-and-how">Include What, Why, and How</h3>
<p>Exceptions thrown from Spark should answer the Five W’s and How:</p>
<ul>
<li><strong>Who</strong> encountered the problem?</li>
<li><strong>What</strong> was the problem?</li>
<li><strong>When</strong> did the problem happen?</li>
<li><strong>Where</strong> did the problem happen?</li>
<li><strong>Why</strong> did the problem happen?</li>
<li><strong>How</strong> can the problem be solved?</li>
</ul>
<p>The context provided by exceptions can help answer <strong>who</strong> (usually the
user), <strong>when</strong> (usually included in the log via <code>log4j</code>), and <strong>where</strong>
(usually included in the stack trace). However, these answers alone are
often insufficient for the user to solve the problem. An error message
that answers the remaining questions
— <strong>what</strong>, <strong>why</strong>, and <strong>how</strong> — minimizes user frustration.</p>
<h4 id="explicitly-answer-what-why-and-how">Explicitly answer What, Why and How</h4>
<p>In many cases, the error message should explicitly answer <strong>what</strong>,
<strong>why</strong>, and <strong>how</strong>.</p>
<h5 id="example-1">Example 1</h5>
<p><code style="white-space:normal">
<a href="https://github.com/apache/spark/blob/569fb133d09e24e4ed56ed7efff641512d98b01b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala#L160">
Unable to generate an encoder for inner class {} without access to the
scope that this class was defined in. Try moving this class out of its
parent class.
</a>
</code></p>
<ul>
<li><strong>What:</strong> Unable to generate encoder inner class.</li>
<li><strong>Why:</strong> Did not have access to the scope that the class was defined in.</li>
<li><strong>How:</strong> Try moving this class out of its parent class.</li>
</ul>
<h5 id="example-2">Example 2</h5>
<p>If the proposed fix (<strong>how</strong>) feels arbitrary, providing an explanation
for <strong>why</strong> the error occurred can reduce user frustration.</p>
<p><strong>Before</strong></p>
<p><code style="white-space:normal">
<a href="https://github.com/apache/spark/blob/03dd33cc982ebb3de4354274ac49da31521b8195/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala#L498">
<span style="background-color: #ffcccc">Unsupported</span> function name {}.
</a>
</code></p>
<ul>
<li><strong>What:</strong> Unsupported function name.</li>
<li><strong>Why:</strong> Unclear.</li>
<li><strong>How:</strong> Unclear.</li>
</ul>
<p><strong>After</strong></p>
<p><em>Function name {} <span style="background-color: #ccffcc">is invalid. Temporary functions cannot belong to a
catalog. Specify a function name with one or two parts.</span></em></p>
<ul>
<li><strong>What:</strong> Invalid function name.</li>
<li><strong>Why:</strong> Temporary functions cannot belong to a catalog.</li>
<li><strong>How:</strong> Specify a function name with one or two parts.</li>
</ul>
<h4 id="implicitly-answer-how">Implicitly answer How</h4>
<p>Not all error messages should be this verbose. Sometimes, explicitly
explaining <strong>how</strong> to resolve the problem would be redundant;
you may skip an explicit explanation in this case.</p>
<h5 id="example-1-1">Example 1</h5>
<p><code style="white-space:normal">
<a href="https://github.com/apache/spark/blob/e5d972e84e973d9a2e62312dc471df30c35269bc/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala#L63">
Invalid pivot column {}. Pivot columns must be comparable.
</a>
</code></p>
<ul>
<li><strong>What:</strong> Invalid pivot column.</li>
<li><strong>Why:</strong> Pivot columns must be comparable.</li>
<li><strong>How (</strong><strong><em>implied by Why</em></strong><strong>):</strong> Use comparable pivot columns.</li>
</ul>
<h5 id="example-2-1">Example 2</h5>
<p><strong>Before</strong></p>
<p><code style="white-space:normal">
<a href="https://github.com/apache/spark/blob/9809a2f1c5187205c81542dbdc84b71db535f6e1/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala#L325">
Cannot specify window frame for {} function
</a>
</code></p>
<ul>
<li><strong>What:</strong> Cannot specify window frame for the function.</li>
<li><strong>Why</strong>: Unclear.</li>
<li><strong>How:</strong> Unclear.</li>
</ul>
<p><strong>After</strong></p>
<p><code style="white-space:normal">
Cannot specify frame for window expression {}. <span style="background-color: #ccffcc">Window expression
contains mismatch between function frame {} and specification frame {}.</span>
</code></p>
<ul>
<li><strong>What:</strong> Cannot specify the frame for the window expression.</li>
<li><strong>Why:</strong> Window expression contains mismatch between function frame
and specification frame.</li>
<li><strong>How (</strong><strong><em>implied by Why</em></strong><strong>):</strong> Match the function frame and
specification frame.</li>
</ul>
<h5 id="example-3">Example 3</h5>
<p><strong>Before</strong></p>
<p><code style="white-space:normal">
<a href="https://github.com/apache/spark/blob/aff6c0febb40d9713895ba00d8c77ba00f04bd16/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala#L93">
Cannot parse any decimal.
</a>
</code></p>
<ul>
<li><strong>What:</strong> Cannot parse decimal.</li>
<li><strong>Why</strong>: Unclear.</li>
<li><strong>How:</strong> Unclear.</li>
</ul>
<p><strong>After</strong></p>
<p><code style="white-space:normal">
Invalid decimal {}; <span style="background-color: #ccffcc">encountered error while parsing at position {}.</span>
</code></p>
<ul>
<li><strong>What:</strong> Invalid decimal.</li>
<li><strong>Why</strong>: The decimal parser encountered an error at the specified position.</li>
<li><strong>How (</strong><strong><em>implied by Why</em></strong><strong>):</strong> Fix the error at the specified position.</li>
</ul>
<h4 id="implicitly-answer-why-and-how">Implicitly answer Why and How</h4>
<p>Sometimes, even explicitly explaining <strong>why</strong> the problem happened would
be redundant; you may skip an explicit explanation in this case.</p>
<p><code style="white-space:normal">
<a href="https://github.com/apache/spark/blob/569fb133d09e24e4ed56ed7efff641512d98b01b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala#L770">
Path does not exist: {}
</a>
</code></p>
<ul>
<li><strong>What:</strong> Path does not exist.</li>
<li><strong>Why (</strong><strong><em>implied by What</em></strong><strong>):</strong> User specified an invalid path.</li>
<li><strong>How (</strong><strong><em>implied by What</em></strong><strong>):</strong> Use a different path.</li>
</ul>
<h3 id="use-clear-language">Use clear language</h3>
<h4 id="diction-guide">Diction guide</h4>
<table class="table table-hover">
<thead>
<tr>
<th scope="col">Phrases</th>
<th scope="col">When to use</th>
<th scope="col">Example</th>
</tr>
</thead>
<tbody>
<tr>
<th scope="row">Unsupported</th>
<td>
The user may reasonably assume that the operation is supported, but it is not. This error may go away in the future if developers add support for the operation.
</td>
<td>
<code style="white-space:normal">
Data type {} is <span style="background-color: #ccffff">unsupported</span>.
</code>
</td>
</tr>
<tr>
<th rowspan="3" scope="rowgroup">
Invalid / Not allowed / Unexpected
</th>
<td rowspan="3">
The user made a mistake when specifying an operation. The message should inform the user of how to resolve the error.
</td>
<td>
<code style="white-space:normal">
Array has size {}, index {} is <span style="background-color: #ccffff">invalid</span>.
</code>
</td>
</tr>
<tr>
<td>
<code style="white-space:normal">
Found {} generators for the clause {}. Only one generator is <span style="background-color: #ccffff">allowed</span>.
</code>
</td>
</tr>
<tr>
<td>
<code style="white-space:normal">
Found an <span style="background-color: #ccffff">unexpected</span> state format version {}. Expected versions 1 or 2.
</code>
</td>
</tr>
<tr>
<th scope="row">Failed to</th>
<td>
The system encountered an unexpected error that cannot be reasonably attributed to user error.
</td>
<td>
<code style="white-space:normal">
<span style="background-color: #ccffff">Failed to</span> compile {}.
</code>
</td>
</tr>
<tr>
<th scope="row">Cannot</th>
<td>
Any time, preferably only if one of the above alternatives does not apply.
</td>
<td>
<code style="white-space:normal">
<span style="background-color: #ccffff">Cannot</span> generate code for unsupported type {}.
</code>
</td>
</tr>
</tbody>
</table>
<h4 id="wording-guide">Wording guide</h4>
<table class="table table-hover">
<thead>
<tr>
<th scope="col">Best practice</th>
<th scope="col">Before</th>
<th scope="col">After</th>
</tr>
</thead>
<tbody>
<tr>
<th scope="row">
Use active voice
</th>
<td>
<code style="white-space:normal">
<a href="https://github.com/apache/spark/blob/73857cdd87757d2888bd92f6b7c2fad709701484/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala#L704">
DataType {} is <span style="background-color: #ffcccc">not supported by</span> {}.
</a>
</code>
</td>
<td>
<code style="white-space:normal">
{} <span style="background-color: #ccffcc">does not</span> support datatype {}.
</code>
</td>
</tr>
<tr>
<th rowspan="2" scope="rowgroup">
Avoid time-based statements, such as promises of future support
</th>
<td>
<code style="white-space:normal">
<a href="https://github.com/apache/spark/blob/27bec91bc971b393bd91f2ec8c6483b33f844f12/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala#L185">
Pandas UDF aggregate expressions are <span style="background-color: #ffcccc">currently</span> not supported in pivot.
</a>
</code>
</td>
<td>
<code style="white-space:normal">
Pivot <span style="background-color: #ccffcc">does not</span> support Pandas UDF aggregate expressions.
</code>
</td>
</tr>
<tr>
<td>
<code style="white-space:normal">
<a href="https://github.com/apache/spark/blob/569fb133d09e24e4ed56ed7efff641512d98b01b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala#L1076">
Parquet type not <span style="background-color: #ffcccc">yet</span> supported: {}.
</a>
</code>
</td>
<td>
<code style="white-space:normal">
{} <span style="background-color: #ccffcc">does not</span> support Parquet type.
</code>
</td>
</tr>
<tr>
<th rowspan="2" scope="rowgroup">Use the present tense to describe the error and provide suggestions</th>
<td>
<code style="white-space:normal">
<a href="https://github.com/apache/spark/blob/9809a2f1c5187205c81542dbdc84b71db535f6e1/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala#L166">
<span style="background-color: #ffcccc">Couldn't</span> find the reference column for {} at {}.
</a>
</code>
</td>
<td>
<code style="white-space:normal"><span style="background-color: #ccffcc">Cannot</span> find the reference column for {} at {}.</code>
</td>
</tr>
<tr>
<td>
<code style="white-space:normal">
<a href="https://github.com/apache/spark/blob/9809a2f1c5187205c81542dbdc84b71db535f6e1/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala#L409">
Join strategy hint parameter <span style="background-color: #ffcccc">should be</span> an identifier or string but was {}.
</a>
</code>
</td>
<td>
<code style="white-space:normal">
Cannot use join strategy hint parameter {}. <span style="background-color: #ccffcc">Use</span> a table name or identifier to specify the parameter.
</code>
</td>
</tr>
<tr>
<th scope="row">Provide concrete examples if the resolution is unclear</th>
<td>
<code style="white-space:normal">
<a href="https://github.com/apache/spark/blob/569fb133d09e24e4ed56ed7efff641512d98b01b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala#L422">
{} Hint expects a partition number as a parameter.
</a>
</code>
</td>
<td>
<code style="white-space:normal">
{} Hint expects a partition number as a parameter. <span style="background-color: #ccffcc">For example, specify 3 partitions with {}(3)</span>.
</code>
</td>
</tr>
<tr>
<th scope="row">Avoid sounding accusatory, judgmental, or insulting</th>
<td>
<code style="white-space:normal">
<a href="https://github.com/apache/spark/blob/569fb133d09e24e4ed56ed7efff641512d98b01b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala#L143">
<span style="background-color: #ffcccc">You must</span> specify an amount for {}.
</a>
</code>
</td>
<td>
<code style="white-space:normal">
{} cannot be empty. Specify an amount for {}.
</code>
</td>
</tr>
<tr>
<th scope="row">Be direct</th>
<td>
<code style="white-space:normal">
<a href="https://github.com/apache/spark/blob/4b5fc1da752ec008468ef80a5717c8beab468387/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala#L119">
LEGACY store assignment policy is disallowed in Spark data source V2. <span style="background-color: #ffcccc">Please</span> set the configuration spark.sql.storeAssignmentPolicy to <span style="background-color: #ffcccc">other values</span>.
</a>
</code>
</td>
<td>
<code style="white-space:normal">
Spark data source V2 does not allow the LEGACY store assignment policy. Set the configuration spark.sql.storeAssignment to ANSI or STRICT.
</code>
</td>
</tr>
<tr>
<th scope="row">Do not use programming jargon in user-facing errors</th>
<td>
<code style="white-space:normal">
<a href="https://github.com/apache/spark/blob/4b5fc1da752ec008468ef80a5717c8beab468387/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala#L583">
RENAME TABLE source and destination databases do not match: '{}' <span style="background-color: #ffcccc">!=</span> '{}'.
</a>
</code>
</td>
<td>
<code style="white-space:normal">
RENAME TABLE source and destination databases do not match. The source database is {}, but the destination database is {}.
</code>
</td>
</tr>
</tbody>
</table>
</div>
<div class="col-12 col-md-3">
<div class="news" style="margin-bottom: 20px;">
<h5>Latest News</h5>
<ul class="list-unstyled">
<li><a href="/news/spark-3-5-5-released.html">Spark 3.5.5 released</a>
<span class="small">(Feb 27, 2025)</span></li>
<li><a href="/news/spark-3-5-4-released.html">Spark 3.5.4 released</a>
<span class="small">(Dec 20, 2024)</span></li>
<li><a href="/news/spark-3-4-4-released.html">Spark 3.4.4 released</a>
<span class="small">(Oct 27, 2024)</span></li>
<li><a href="/news/spark-4.0.0-preview2.html">Preview release of Spark 4.0</a>
<span class="small">(Sep 26, 2024)</span></li>
</ul>
<p class="small" style="text-align: right;"><a href="/news/index.html">Archive</a></p>
</div>
<div style="text-align:center; margin-bottom: 20px;">
<a href="https://www.apache.org/events/current-event.html">
<img src="https://www.apache.org/events/current-event-234x60.png" style="max-width: 100%;"/>
</a>
</div>
<div class="hidden-xs hidden-sm">
<a href="/downloads.html" class="btn btn-cta btn-lg d-grid" style="margin-bottom: 30px;">
Download Spark
</a>
<p style="font-size: 16px; font-weight: 500; color: #555;">
Built-in Libraries:
</p>
<ul class="list-none">
<li><a href="/sql/">SQL and DataFrames</a></li>
<li><a href="/streaming/">Spark Streaming</a></li>
<li><a href="/mllib/">MLlib (machine learning)</a></li>
<li><a href="/graphx/">GraphX (graph)</a></li>
</ul>
<a href="/third-party-projects.html">Third-Party Projects</a>
</div>
</div>
</div>
<footer class="small">
<hr>
Apache Spark, Spark, Apache, the Apache feather logo, and the Apache Spark project logo are either registered
trademarks or trademarks of The Apache Software Foundation in the United States and other countries.
See guidance on use of Apache Spark <a href="/trademarks.html">trademarks</a>.
All other marks mentioned may be trademarks or registered trademarks of their respective owners.
Copyright © 2018 The Apache Software Foundation, Licensed under the
<a href="https://www.apache.org/licenses/">Apache License, Version 2.0</a>.
</footer>
</div>
<script src="/js/jquery.js"></script>
<script src="/js/bootstrap.bundle.min.js"></script>
<script src="/js/lang-tabs.js"></script>
<script src="/js/downloads.js"></script>
</body>
</html>