content/2024/05/17/apache-flink-cdc-3.1.0-release-announcement/index.html (401 lines of code) (raw):
<!DOCTYPE html>
<html lang="en" dir=ZgotmplZ>
<head>
<link rel="stylesheet" href="/bootstrap/css/bootstrap.min.css">
<script src="/bootstrap/js/bootstrap.bundle.min.js"></script>
<link rel="stylesheet" type="text/css" href="/font-awesome/css/font-awesome.min.css">
<script src="/js/anchor.min.js"></script>
<script src="/js/flink.js"></script>
<link rel="canonical" href="https://flink.apache.org/2024/05/17/apache-flink-cdc-3.1.0-release-announcement/">
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="The Apache Flink community is excited to announce the release of Flink CDC 3.1.0! This is the first release after the community accepted the donation of Flink CDC as a sub-project of Apache Flink, with exciting new features such as transform and table merging. The eco-system of Flink CDC keeps expanding, including new Kafka and Paimon pipeline sinks and enhancement to existing connectors.
We’d like to invite you to check out Flink CDC documentation and have a try on the quickstart tutorial to explore the world of Flink CDC.">
<meta name="theme-color" content="#FFFFFF"><meta property="og:title" content="Apache Flink CDC 3.1.0 Release Announcement" />
<meta property="og:description" content="The Apache Flink community is excited to announce the release of Flink CDC 3.1.0! This is the first release after the community accepted the donation of Flink CDC as a sub-project of Apache Flink, with exciting new features such as transform and table merging. The eco-system of Flink CDC keeps expanding, including new Kafka and Paimon pipeline sinks and enhancement to existing connectors.
We’d like to invite you to check out Flink CDC documentation and have a try on the quickstart tutorial to explore the world of Flink CDC." />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://flink.apache.org/2024/05/17/apache-flink-cdc-3.1.0-release-announcement/" /><meta property="article:section" content="posts" />
<meta property="article:published_time" content="2024-05-17T08:00:00+00:00" />
<meta property="article:modified_time" content="2024-05-17T08:00:00+00:00" />
<title>Apache Flink CDC 3.1.0 Release Announcement | Apache Flink</title>
<link rel="manifest" href="/manifest.json">
<link rel="icon" href="/favicon.png" type="image/x-icon">
<link rel="stylesheet" href="/book.min.22eceb4d17baa9cdc0f57345edd6f215a40474022dfee39b63befb5fb3c596b5.css" integrity="sha256-IuzrTRe6qc3A9XNF7dbyFaQEdAIt/uObY777X7PFlrU=">
<script defer src="/en.search.min.6950414fdb865fa2a65062656ee52f8a1d13a0578a8866521a5e3501cdc3b475.js" integrity="sha256-aVBBT9uGX6KmUGJlbuUvih0ToFeKiGZSGl41Ac3DtHU="></script>
<!--
Made with Book Theme
https://github.com/alex-shpak/hugo-book
-->
<meta name="generator" content="Hugo 0.124.1">
<script>
var _paq = window._paq = window._paq || [];
_paq.push(['disableCookies']);
_paq.push(["setDomains", ["*.flink.apache.org","*.nightlies.apache.org/flink"]]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="//analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '1']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
</head>
<body dir=ZgotmplZ>
<header>
<nav class="navbar navbar-expand-xl">
<div class="container-fluid">
<a class="navbar-brand" href="/">
<img src="/img/logo/png/100/flink_squirrel_100_color.png" alt="Apache Flink" height="47" width="47" class="d-inline-block align-text-middle">
<span>Apache Flink</span>
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
<i class="fa fa-bars navbar-toggler-icon"></i>
</button>
<div class="collapse navbar-collapse" id="navbarSupportedContent">
<ul class="navbar-nav">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">About</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/what-is-flink/flink-architecture/">Architecture</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-applications/">Applications</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-operations/">Operations</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/use-cases/">Use Cases</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/powered-by/">Powered By</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/roadmap/">Roadmap</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/community/">Community & Project Info</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/security/">Security</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/special-thanks/">Special Thanks</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Getting Started</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/try-flink/local_installation/">With Flink<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/docs/try-flink-kubernetes-operator/quick-start/">With Flink Kubernetes Operator<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable/docs/get-started/introduction/">With Flink CDC<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/docs/try-flink-ml/quick-start/">With Flink ML<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/getting-started/project-setup.html">With Flink Stateful Functions<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/learn-flink/overview/">Training Course<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Documentation</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/">Flink 2.0 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-lts/">Flink 1.20 (LTS)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-master/">Flink Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/">Kubernetes Operator 1.11 (latest)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main">Kubernetes Operator Main (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable">CDC 3.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-master">CDC Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/">ML 2.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-master">ML Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/">Stateful Functions 3.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-master">Stateful Functions Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">How to Contribute</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/how-to-contribute/overview/">Overview</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-code/">Contribute Code</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/reviewing-prs/">Review Pull Requests</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/code-style-and-quality-preamble/">Code Style and Quality Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-documentation/">Contribute Documentation</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/documentation-style-guide/">Documentation Style Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/improve-website/">Contribute to the Website</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/getting-help/">Getting Help</a>
</li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link" href="/posts/">Flink Blog</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/downloads/">Downloads</a>
</li>
</ul>
<div class="book-search">
<div class="book-search-spinner hidden">
<i class="fa fa-refresh fa-spin"></i>
</div>
<form class="search-bar d-flex" onsubmit="return false;"su>
<input type="text" id="book-search-input" placeholder="Search" aria-label="Search" maxlength="64" data-hotkeys="s/">
<i class="fa fa-search search"></i>
<i class="fa fa-circle-o-notch fa-spin spinner"></i>
</form>
<div class="book-search-spinner hidden"></div>
<ul id="book-search-results"></ul>
</div>
</div>
</div>
</nav>
<div class="navbar-clearfix"></div>
</header>
<main class="flex">
<section class="container book-page">
<article class="markdown">
<h1>
<a href="/2024/05/17/apache-flink-cdc-3.1.0-release-announcement/">Apache Flink CDC 3.1.0 Release Announcement</a>
</h1>
May 17, 2024 -
Qingsheng Ren
<a href="https://twitter.com/renqstuite">(@renqstuite)</a>
<p><p>The Apache Flink community is excited to announce the release of Flink CDC 3.1.0! This is the first release after the community accepted the donation of Flink CDC as a sub-project of Apache Flink, with exciting new features such as transform and table merging. The eco-system of Flink CDC keeps expanding, including new Kafka and Paimon pipeline sinks and enhancement to existing connectors.</p>
<p>We’d like to invite you to check out <a href="https://nightlies.apache.org/flink/flink-cdc-docs-release-3.1">Flink CDC documentation</a> and have a try on <a href="https://nightlies.apache.org/flink/flink-cdc-docs-release-3.1/docs/get-started/introduction">the quickstart tutorial</a> to explore the world of Flink CDC. Also we encourage you to <a href="https://flink.apache.org/downloads.html#flink-cdc">download the release</a> and share your feedback with the community through the Flink <a href="https://flink.apache.org/community.html#mailing-lists">mailing lists</a> or <a href="https://issues.apache.org/jira/browse/flink">JIRA</a>! We hope you like the new release and we’d be eager to learn about your experience with it.</p>
<h2 id="highlights">
Highlights
<a class="anchor" href="#highlights">#</a>
</h2>
<h3 id="transformation-support-in-pipeline">
Transformation Support in Pipeline
<a class="anchor" href="#transformation-support-in-pipeline">#</a>
</h3>
<p>Flink CDC 3.1.0 introduces the ability of making transformations in the CDC pipeline. By incorporating a <code>transform</code> section within the YAML pipeline definitions, users can now easily apply a variety of transformations to data change event from source, including projections, calculations, and addition of constant columns, enhancing the effectiveness of data integration pipelines. Leveraging an SQL-like syntax for defining these transformations, the new feature ensures that users can quickly adapt to and utilize it.</p>
<p>You can find examples of using transformations in the <a href="https://nightlies.apache.org/flink/flink-cdc-docs-release-3.1/docs/core-concept/transform/#example">Flink CDC documentation</a>.</p>
<h3 id="table-merging-support">
Table Merging Support
<a class="anchor" href="#table-merging-support">#</a>
</h3>
<p>Flink CDC 3.1.0 now supports merging multiple tables into one by configuring <code>route</code> in the YAML pipeline definition. It is a prevalent occurrence where business data is partitioned across tables even databases due to the substantial volume. By configuring <code>route</code>s that map multiple tables into one, data change events will be merged into the same destination table. Moreover, schema changes on source tables will also be applied to the destination.</p>
<p>You can find examples of using routes in the <a href="https://nightlies.apache.org/flink/flink-cdc-docs-release-3.1/docs/core-concept/route/#example">Flink CDC documentation</a>.</p>
<h3 id="connectors">
Connectors
<a class="anchor" href="#connectors">#</a>
</h3>
<h4 id="distributions-of-mysql--oracle--oceanbase--db2-connectors">
Distributions of MySQL / Oracle / OceanBase / Db2 connectors
<a class="anchor" href="#distributions-of-mysql--oracle--oceanbase--db2-connectors">#</a>
</h4>
<p>Unfortunately due to the license incompatibility, we cannot ship JDBC drivers of the following connectors together with our binary release:</p>
<ul>
<li>Db2</li>
<li>MySQL</li>
<li>Oracle</li>
<li>OceanBase</li>
</ul>
<p>Please manually download the corresponding JDBC driver into <code>$FLINK_HOME/lib</code> of your Flink cluster, or specify the path of driver when submitting YAML pipelines with <code>--jar</code>. Please make sure they are under the classpath if you are using Flink SQL.</p>
<h4 id="sinkfunction-support">
SinkFunction Support
<a class="anchor" href="#sinkfunction-support">#</a>
</h4>
<p>Although <code>SinkFunction</code> has been marked as deprecated in Flink, considering some connectors are still using the API, we also support <code>SinkFunction</code> API for CDC pipeline sinks to help expand the ecosystem of Flink CDC.</p>
<h4 id="new-pipeline-connectors">
New Pipeline Connectors
<a class="anchor" href="#new-pipeline-connectors">#</a>
</h4>
<p>Flink CDC 3.1.0 introduces 2 new pipeline connectors:</p>
<ul>
<li>Apache Kafka sink (depends on Kafka 3.2.3)</li>
<li>Apache Paimon sink (depends on Paimon 0.7.0)</li>
</ul>
<h4 id="mysql">
MySQL
<a class="anchor" href="#mysql">#</a>
</h4>
<p>In this release, MySQL pipeline source introduces a new option <code>tables.exclude</code> to exclude unnecessary tables from capturing with an easier expression. MySQL CDC source is now shipped with a custom converter <code>MysqlDebeziumTimeConverter</code> for converting temporal type columns to a more human-readable and serialize-friendly string.</p>
<h4 id="oceanbase">
OceanBase
<a class="anchor" href="#oceanbase">#</a>
</h4>
<p>OceanBase CDC source now supports specifying the general <code>DebeziumDeserializationSchema</code> for reusing existing Debezium deserializers.</p>
<h4 id="db2">
Db2
<a class="anchor" href="#db2">#</a>
</h4>
<p>Db2 CDC source is now migrated to the unified incremental snapshot framework.</p>
<h3 id="cli">
CLI
<a class="anchor" href="#cli">#</a>
</h3>
<p>Flink CDC pipeline submission CLI now supports recovering a pipeline execution from a specific savepoint file by using command line argument <code>--from-savepoint</code></p>
<h2 id="list-of-contributors">
List of Contributors
<a class="anchor" href="#list-of-contributors">#</a>
</h2>
<p>Check Null, FocusComputing, GOODBOY008, Hang Ruan, He Wang, Hongshun Wang, Jiabao Sun, Kunni, L, Laffery, Leonard Xu, Muhammet Orazov, Paul Lin, PengFei Li, Qingsheng Ren, Qishang Zhong, Shawn Huang, Thorne, TorinJie, Xianxun Ye, Xin Gong, Yaroslav Tkachenko, e-mhui, gongzhongqiang, joyCurry30, kunni, lzshlzsh, qwding, shikai93, sky, skylines, wenmo, wudi, xleoken, xuzifu666, yanghuaiGit, yux, yuxiqian, 张田</p>
</p>
</article>
<div class="edit-this-page">
<p>
<a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Translation+Specifications">Want to contribute translation?</a>
</p>
<p>
<a href="//github.com/apache/flink-web/edit/asf-site/docs/content/posts/2024-05-17-release-cdc-3.1.0.md">
Edit This Page<i class="fa fa-edit fa-fw"></i>
</a>
</p>
</div>
</section>
<aside class="book-toc">
<nav id="TableOfContents"><h3>On This Page <a href="javascript:void(0)" class="toc" onclick="collapseToc()"><i class="fa fa-times" aria-hidden="true"></i></a></h3>
<ul>
<li>
<ul>
<li><a href="#highlights">Highlights</a>
<ul>
<li><a href="#transformation-support-in-pipeline">Transformation Support in Pipeline</a></li>
<li><a href="#table-merging-support">Table Merging Support</a></li>
<li><a href="#connectors">Connectors</a></li>
<li><a href="#cli">CLI</a></li>
</ul>
</li>
<li><a href="#list-of-contributors">List of Contributors</a></li>
</ul>
</li>
</ul>
</nav>
</aside>
<aside class="expand-toc hidden">
<a class="toc" onclick="expandToc()" href="javascript:void(0)">
<i class="fa fa-bars" aria-hidden="true"></i>
</a>
</aside>
</main>
<footer>
<div class="separator"></div>
<div class="panels">
<div class="wrapper">
<div class="panel">
<ul>
<li>
<a href="https://flink-packages.org/">flink-packages.org</a>
</li>
<li>
<a href="https://www.apache.org/">Apache Software Foundation</a>
</li>
<li>
<a href="https://www.apache.org/licenses/">License</a>
</li>
<li>
<a href="/zh/">
<i class="fa fa-globe" aria-hidden="true"></i> 中文版
</a>
</li>
</ul>
</div>
<div class="panel">
<ul>
<li>
<a href="/what-is-flink/security">Security</a-->
</li>
<li>
<a href="https://www.apache.org/foundation/sponsorship.html">Donate</a>
</li>
<li>
<a href="https://www.apache.org/foundation/thanks.html">Thanks</a>
</li>
</ul>
</div>
<div class="panel icons">
<div>
<a href="/posts">
<div class="icon flink-blog-icon"></div>
<span>Flink blog</span>
</a>
</div>
<div>
<a href="https://github.com/apache/flink">
<div class="icon flink-github-icon"></div>
<span>Github</span>
</a>
</div>
<div>
<a href="https://twitter.com/apacheflink">
<div class="icon flink-twitter-icon"></div>
<span>Twitter</span>
</a>
</div>
</div>
</div>
</div>
<hr/>
<div class="container disclaimer">
<p>The contents of this website are © 2024 Apache Software Foundation under the terms of the Apache License v2. Apache Flink, Flink, and the Flink logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
</div>
</footer>
</body>
</html>