site/releases/spark-release-3-4-4.html (265 lines of code) (raw):
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>
Spark Release 3.4.4 | Apache Spark
</title>
<link href="/css/bootstrap.min.css" rel="stylesheet">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=DM+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&Courier+Prime:wght@400;700&display=swap" rel="stylesheet">
<link href="/css/custom.css" rel="stylesheet">
<!-- Code highlighter CSS -->
<link href="/css/pygments-default.css" rel="stylesheet">
<link rel="icon" href="/favicon.ico" type="image/x-icon">
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
_paq.push(["disableCookies"]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '40']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
</head>
<body class="global">
<nav class="navbar navbar-expand-lg navbar-dark p-0 px-4" style="background: #1D6890;">
<a class="navbar-brand" href="/">
<img src="/images/spark-logo-rev.svg" alt="" width="141" height="72">
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarContent"
aria-controls="navbarContent" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse col-md-12 col-lg-auto pt-4" id="navbarContent">
<ul class="navbar-nav me-auto">
<li class="nav-item">
<a class="nav-link active" aria-current="page" href="/downloads.html">Download</a>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="libraries" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Libraries
</a>
<ul class="dropdown-menu" aria-labelledby="libraries">
<li><a class="dropdown-item" href="/sql/">SQL and DataFrames</a></li>
<li><a class="dropdown-item" href="/spark-connect/">Spark Connect</a></li>
<li><a class="dropdown-item" href="/streaming/">Spark Streaming</a></li>
<li><a class="dropdown-item" href="/pandas-on-spark/">pandas on Spark</a></li>
<li><a class="dropdown-item" href="/mllib/">MLlib (machine learning)</a></li>
<li><a class="dropdown-item" href="/graphx/">GraphX (graph)</a></li>
<li>
<hr class="dropdown-divider">
</li>
<li><a class="dropdown-item" href="/third-party-projects.html">Third-Party Projects</a></li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="documentation" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Documentation
</a>
<ul class="dropdown-menu" aria-labelledby="documentation">
<li><a class="dropdown-item" href="/docs/latest/">Latest Release</a></li>
<li><a class="dropdown-item" href="/documentation.html">Older Versions and Other Resources</a></li>
<li><a class="dropdown-item" href="/faq.html">Frequently Asked Questions</a></li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link active" aria-current="page" href="/examples.html">Examples</a>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="community" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Community
</a>
<ul class="dropdown-menu" aria-labelledby="community">
<li><a class="dropdown-item" href="/community.html">Mailing Lists & Resources</a></li>
<li><a class="dropdown-item" href="/contributing.html">Contributing to Spark</a></li>
<li><a class="dropdown-item" href="/improvement-proposals.html">Improvement Proposals (SPIP)</a>
</li>
<li><a class="dropdown-item" href="https://issues.apache.org/jira/browse/SPARK">Issue Tracker</a>
</li>
<li><a class="dropdown-item" href="/powered-by.html">Powered By</a></li>
<li><a class="dropdown-item" href="/committers.html">Project Committers</a></li>
<li><a class="dropdown-item" href="/history.html">Project History</a></li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="developers" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Developers
</a>
<ul class="dropdown-menu" aria-labelledby="developers">
<li><a class="dropdown-item" href="/developer-tools.html">Useful Developer Tools</a></li>
<li><a class="dropdown-item" href="/versioning-policy.html">Versioning Policy</a></li>
<li><a class="dropdown-item" href="/release-process.html">Release Process</a></li>
<li><a class="dropdown-item" href="/security.html">Security</a></li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="github" role="button" data-bs-toggle="dropdown" aria-expanded="false">
GitHub
</a>
<ul class="dropdown-menu" aria-labelledby="github">
<li><a class="dropdown-item" href="https://github.com/apache/spark">spark</a></li>
<li><a class="dropdown-item" href="https://github.com/apache/spark-connect-go">spark-connect-go</a></li>
<li><a class="dropdown-item" href="https://github.com/apache/spark-connect-swift">spark-connect-swift</a></li>
<li><a class="dropdown-item" href="https://github.com/apache/spark-docker">spark-docker</a></li>
<li><a class="dropdown-item" href="https://github.com/apache/spark-kubernetes-operator">spark-kubernetes-operator</a></li>
<li><a class="dropdown-item" href="https://github.com/apache/spark-website">spark-website</a></li>
</ul>
</li>
</ul>
<ul class="navbar-nav ml-auto">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="apacheFoundation" role="button"
data-bs-toggle="dropdown" aria-expanded="false">
Apache Software Foundation
</a>
<ul class="dropdown-menu" aria-labelledby="apacheFoundation">
<li><a class="dropdown-item" href="https://www.apache.org/">Apache Homepage</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/licenses/">License</a></li>
<li><a class="dropdown-item"
href="https://www.apache.org/foundation/sponsorship.html">Sponsorship</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/foundation/thanks.html">Thanks</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/events/current-event">Event</a></li>
</ul>
</li>
</ul>
</div>
</nav>
<div class="container">
<div class="row mt-4">
<div class="col-12 col-md-9">
<h2>Spark Release 3.4.4</h2>
<p>Spark 3.4.4 is the last maintenance release containing security and correctness fixes. This release is based on the branch-3.4 maintenance branch of Spark. We strongly recommend all 3.4 users to upgrade to this stable release.</p>
<h3 id="notable-changes">Notable changes</h3>
<ul>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43242">[SPARK-43242]</a>: Fix throw ‘Unexpected type of BlockId’ in shuffle corruption diagnose</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45988">[SPARK-45988]</a>: Fix typehints to handle <code class="language-plaintext highlighter-rouge">list</code> GenericAlias in Python 3.11+</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46535">[SPARK-46535]</a>: Fix NPE when describe extended a column without col stats</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46957">[SPARK-46957]</a>: Decommission migrated shuffle files should be able to cleanup from executor</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-47129">[SPARK-47129]</a>: Make ResolveRelations cache connect plan properly</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-47172">[SPARK-47172]</a>: Add support for AES-GCM for RPC encryption</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-47828">[SPARK-47828]</a>: DataFrameWriterV2.overwrite fails with invalid plan</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-47895">[SPARK-47895]</a>: group by all should be idempotent</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-47897">[SPARK-47897]</a>: Fix ExpressionSet performance regression in scala 2.12</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-47927">[SPARK-47927]</a>: Fix nullability attribute in UDF decoder</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48016">[SPARK-48016]</a>: Fix a bug in try_divide function when with decimals</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48019">[SPARK-48019]</a>: Fix incorrect behavior in ColumnVector/ColumnarArray with dictionary and nulls</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48037">[SPARK-48037]</a>: Fix SortShuffleWriter lacks shuffle write related metrics resulting in potentially inaccurate data</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48081">[SPARK-48081]</a>: Fix ClassCastException in NTile.checkInputDataTypes() when argument is non-foldable or of wrong type</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48105">[SPARK-48105]</a>: Fix the race condition between state store unloading and snapshotting</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48128">[SPARK-48128]</a>: For BitwiseCount / bit_count expression, fix codegen syntax error for boolean type inputs</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48155">[SPARK-48155]</a>: AQEPropagateEmptyRelation for join should check if remain child is just BroadcastQueryStageExec</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48172">[SPARK-48172]</a>: Fix escaping issues in JDBCDialects</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48248">[SPARK-48248]</a>: Fix nested array to respect legacy conf of inferArrayTypeFromFirstElement</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48292">[SPARK-48292]</a>: Revert SPARK-39195 Spark OutputCommitCoordinator should abort stage when committed file not consistent with task status</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48484">[SPARK-48484]</a>: Fix: V2Write use the same TaskAttemptId for different task attempts</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48642">[SPARK-48642]</a>: False SparkOutOfMemoryError caused by killing task on spilling</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48710">[SPARK-48710]</a>: Limit NumPy version to supported range (>=1.15,<2)</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48759">[SPARK-48759]</a>: Add migration doc for CREATE TABLE AS SELECT behavior change behavior change since Spark 3.4</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48791">[SPARK-48791]</a>: Fix perf regression caused by the accumulators registration overhead using CopyOnWriteArrayList</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48930">[SPARK-48930]</a>: Redact <code class="language-plaintext highlighter-rouge">awsAccessKeyId</code> by including <code class="language-plaintext highlighter-rouge">accesskey</code> pattern</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48934">[SPARK-48934]</a>: Python datetime types converted incorrectly for setting timeout in applyInPandasWithState</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48965">[SPARK-48965]</a>: Use the correct schema in <code class="language-plaintext highlighter-rouge">Dataset#toJSON</code></li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-48991">[SPARK-48991]</a>: Move path initialization into try-catch block in FileStreamSink.hasMetadata</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49000">[SPARK-49000]</a>: Fix “select count(distinct 1) from t” where t is empty table by expanding RewriteDistinctAggregates</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49005">[SPARK-49005]</a>: Use <code class="language-plaintext highlighter-rouge">17-jammy</code> tag instead of <code class="language-plaintext highlighter-rouge">17-jre</code> to prevent Python 3.12</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49038">[SPARK-49038]</a>: SQLMetric should report the raw value in the accumulator update event</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49039">[SPARK-49039]</a>: Reset checkbox when executor metrics are loaded in the Stages tab</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49094">[SPARK-49094]</a>: Fix ignoreCorruptFiles non-functioning for hive orc impl with mergeSchema off</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49176">[SPARK-49176]</a>: Fix <code class="language-plaintext highlighter-rouge">spark.ui.custom.executor.log.url</code> docs by adding K8s</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49179">[SPARK-49179]</a>: Fix v2 multi bucketed inner joins throw AssertionError</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49182">[SPARK-49182]</a>: Stop publish site/docs/{version}/api/python/_sources dir</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49193">[SPARK-49193]</a>: Improve the performance of RowSetUtils.toColumnBasedSet</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49197">[SPARK-49197]</a>: Redact <code class="language-plaintext highlighter-rouge">Spark Command</code> output in <code class="language-plaintext highlighter-rouge">launcher</code> module</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49261">[SPARK-49261]</a>: Don’t replace literals in aggregate expressions with group-by expressions</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49352">[SPARK-49352]</a>: Avoid redundant array transform for identical expression</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49385">[SPARK-49385]</a>: Fix <code class="language-plaintext highlighter-rouge">getReusablePVCs</code> to use <code class="language-plaintext highlighter-rouge">podCreationTimeout</code> instead of <code class="language-plaintext highlighter-rouge">podAllocationDelay</code></li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49408">[SPARK-49408]</a>: Use IndexedSeq in ProjectingInternalRow</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49595">[SPARK-49595]</a>: Fix <code class="language-plaintext highlighter-rouge">DataFrame.unpivot/melt</code> in Spark Connect Scala Client</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49628">[SPARK-49628]</a>: ConstantFolding should copy stateful expression before evaluating</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49750">[SPARK-49750]</a>: Mention delegation token support in K8s mode</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49760">[SPARK-49760]</a>: Correct handling of <code class="language-plaintext highlighter-rouge">SPARK_USER</code> env variable override in app master</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49804">[SPARK-49804]</a>: Fix to use the exit code of executor container always</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49836">[SPARK-49836]</a>: Fix possibly broken query when window is provided to window/session_window fn</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49843">[SPARK-49843]</a>: Fix change comment on char/varchar columns</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-49959">[SPARK-49959]</a>: Fix ColumnarArray.copy() to read nulls from the correct offset</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-50021">[SPARK-50021]</a>: Fix <code class="language-plaintext highlighter-rouge">ApplicationPage</code> to hide App UI links when UI is disabled</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-50022">[SPARK-50022]</a>: Fix <code class="language-plaintext highlighter-rouge">MasterPage</code> to hide App UI links when UI is disabled</li>
</ul>
<h3 id="dependency-changes">Dependency Changes</h3>
<p>While being a maintenance release we did still upgrade some dependencies in this release they are:</p>
<ul>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43394">[SPARK-43394]</a>: Upgrade maven to 3.8.8</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45590">[SPARK-45590]</a>: Upgrade okio to 1.17.6 from 1.15.0</li>
</ul>
<p>You can consult JIRA for the <a href="https://s.apache.org/spark-3.4.4">detailed changes</a>.</p>
<p>We would like to acknowledge all community members for contributing patches to this release.</p>
<p>
<br/>
<a href="/news/">Spark News Archive</a>
</p>
</div>
<div class="col-12 col-md-3">
<div class="news" style="margin-bottom: 20px;">
<h5>Latest News</h5>
<ul class="list-unstyled">
<li><a href="/news/spark-3-5-5-released.html">Spark 3.5.5 released</a>
<span class="small">(Feb 27, 2025)</span></li>
<li><a href="/news/spark-3-5-4-released.html">Spark 3.5.4 released</a>
<span class="small">(Dec 20, 2024)</span></li>
<li><a href="/news/spark-3-4-4-released.html">Spark 3.4.4 released</a>
<span class="small">(Oct 27, 2024)</span></li>
<li><a href="/news/spark-4.0.0-preview2.html">Preview release of Spark 4.0</a>
<span class="small">(Sep 26, 2024)</span></li>
</ul>
<p class="small" style="text-align: right;"><a href="/news/index.html">Archive</a></p>
</div>
<div style="text-align:center; margin-bottom: 20px;">
<a href="https://www.apache.org/events/current-event.html">
<img src="https://www.apache.org/events/current-event-234x60.png" style="max-width: 100%;"/>
</a>
</div>
<div class="hidden-xs hidden-sm">
<a href="/downloads.html" class="btn btn-cta btn-lg d-grid" style="margin-bottom: 30px;">
Download Spark
</a>
<p style="font-size: 16px; font-weight: 500; color: #555;">
Built-in Libraries:
</p>
<ul class="list-none">
<li><a href="/sql/">SQL and DataFrames</a></li>
<li><a href="/streaming/">Spark Streaming</a></li>
<li><a href="/mllib/">MLlib (machine learning)</a></li>
<li><a href="/graphx/">GraphX (graph)</a></li>
</ul>
<a href="/third-party-projects.html">Third-Party Projects</a>
</div>
</div>
</div>
<footer class="small">
<hr>
Apache Spark, Spark, Apache, the Apache feather logo, and the Apache Spark project logo are either registered
trademarks or trademarks of The Apache Software Foundation in the United States and other countries.
See guidance on use of Apache Spark <a href="/trademarks.html">trademarks</a>.
All other marks mentioned may be trademarks or registered trademarks of their respective owners.
Copyright © 2018 The Apache Software Foundation, Licensed under the
<a href="https://www.apache.org/licenses/">Apache License, Version 2.0</a>.
</footer>
</div>
<script src="/js/jquery.js"></script>
<script src="/js/bootstrap.bundle.min.js"></script>
<script src="/js/lang-tabs.js"></script>
<script src="/js/downloads.js"></script>
</body>
</html>