in packages/constructs/L3/dataops/dataops-nifi-l3-construct/lib/cdk8s/nifi-cluster-chart.ts [260:635]
private createNifiStatefulSet(nifiService: k8s.KubeService, nifiSecretName: string): k8s.KubeStatefulSet {
// nosemgrep
const nifiInitScriptsConfigMapData = Object.fromEntries(
fs.readdirSync(`${__dirname}/../../scripts/nifi`).map(fileName => {
// nosemgrep
return [fileName, fs.readFileSync(`${__dirname}/../../scripts/nifi/${fileName}`, 'utf-8')];
}),
);
const nifiInitConfigMap = new k8s.KubeConfigMap(this, 'nifi-init-configmap', {
metadata: {
name: 'nifi-init-scripts',
},
data: nifiInitScriptsConfigMapData,
});
let volId = 0;
const pvs = this.props.efsPersistentVolumes.map(efsPvProps => {
const pv = new k8s.KubePersistentVolume(this, `nifi-persistent-volume-${volId}`, {
metadata: {
name: `nifi-vol-${efsPvProps.efsFsId}-${efsPvProps.efsApId}`,
labels: {
app: 'nifi',
},
},
spec: {
volumeMode: 'Filesystem',
capacity: {
storage: k8s.Quantity.fromString('60Gi'),
},
accessModes: ['ReadWriteOnce'],
persistentVolumeReclaimPolicy: 'Retain',
storageClassName: this.props.efsStorageClassName,
csi: {
driver: 'efs.csi.aws.com',
volumeHandle: `${efsPvProps.efsFsId}::${efsPvProps.efsApId}`,
},
claimRef: {
name: `nifi-data-nifi-${volId}`,
namespace: this.namespace,
},
},
});
volId = volId + 1;
return pv;
});
const sslBasePath = '/opt/nifi/ssl';
const nifiDataDir = '/opt/nifi/data';
const nifiInitBaseDir = '/opt/nifi/init';
const nifiConfigMap = this.createNifiConfigMap(sslBasePath, nifiDataDir);
const sslSecretVolumes: [k8s.Volume, k8s.VolumeMount][] = this.nodeList.map(hostname => {
const volume: k8s.Volume = {
name: `${hostname}-ssl`,
secret: {
secretName: `${hostname}-ssl`,
},
};
const mount: k8s.VolumeMount = {
mountPath: `${sslBasePath}/${hostname}`,
name: volume.name,
readOnly: true,
};
return [volume, mount];
});
const serviceAccount = new k8s.KubeServiceAccount(this, 'nifi-service-account', {
metadata: {
name: 'nifi',
annotations: {
'eks.amazonaws.com/role-arn': this.props.nifiServiceRoleArn,
},
},
});
const nifiStsProps: k8s.KubeStatefulSetProps = {
metadata: {
name: 'nifi',
},
spec: {
podManagementPolicy: 'Parallel',
serviceName: nifiService.name,
replicas: this.props.nodeCount,
selector: {
matchLabels: {
app: 'nifi',
},
},
persistentVolumeClaimRetentionPolicy: {
whenDeleted: 'Retain',
whenScaled: 'Delete',
},
volumeClaimTemplates: [
{
metadata: {
name: 'nifi-data',
},
spec: {
selector: {
matchLabels: {
app: 'nifi',
},
},
storageClassName: this.props.efsStorageClassName,
accessModes: ['ReadWriteOnce'],
resources: {
requests: {
storage: k8s.Quantity.fromString('5Gi'),
},
},
},
},
],
template: {
metadata: {
labels: {
app: 'nifi',
},
},
spec: {
serviceAccountName: serviceAccount.name,
tolerations: [
{
key: 'eks.amazonaws.com/compute-type',
value: 'fargate',
},
],
dnsConfig: {
searches: [`${nifiService.name}.${this.namespace}.svc.cluster.local`],
},
securityContext: {
runAsUser: 1000,
runAsGroup: 1000,
fsGroup: 1000,
},
volumes: [
{
name: 'nifi-init-scripts',
configMap: {
name: nifiInitConfigMap.name,
defaultMode: 0o755,
},
},
{
name: 'nifi-config',
configMap: {
name: nifiConfigMap.name,
defaultMode: 0o755,
},
},
{
name: 'aws-creds',
emptyDir: {},
},
{
name: 'pip-local',
emptyDir: {},
},
{
name: `manager-ssl`,
secret: {
secretName: `manager-ssl`,
},
},
...sslSecretVolumes.map(x => x[0]),
],
shareProcessNamespace: true,
containers: [
{
name: 'nifi-manager',
image: this.props.nifiManagerImageUri,
command: ['sh', `/opt/nifi/scripts/nifi_manager.sh`],
resources: {
requests: {
memory: k8s.Quantity.fromString('0.5Gi'),
cpu: k8s.Quantity.fromString('250m'),
},
limits: {
memory: k8s.Quantity.fromString('0.5Gi'),
cpu: k8s.Quantity.fromString('250m'),
},
},
env: [
{
name: 'NIFI_APP',
value: 'nifi',
},
{
name: 'MANAGER_CONFIG',
value: `${nifiInitBaseDir}/conf/nifi_manager.json`,
},
{
name: 'NIFI_INIT_DIR',
value: nifiInitBaseDir,
},
{
name: 'NIFI_DATA_DIR',
value: nifiDataDir,
},
{
name: 'NIFI_SENSITIVE_PROPS_KEY',
valueFrom: {
secretKeyRef: {
name: nifiSecretName,
key: 'sensitive-props-key',
optional: false,
},
},
},
{
name: 'NIFI_SSL_BASE_PATH',
value: sslBasePath,
},
{
name: 'NIFI_ZOOKEEPER_CONNECT_STRING',
value: this.props.zkConnectString,
},
{
name: 'NIFI_KEYSTORE_PASSWORD',
valueFrom: {
secretKeyRef: {
name: nifiSecretName,
key: 'keystore-password',
optional: false,
},
},
},
{
name: 'NIFI_TRUSTSTORE_PASSWORD',
valueFrom: {
secretKeyRef: {
name: nifiSecretName,
key: 'keystore-password',
optional: false,
},
},
},
{
name: 'PYTHONUNBUFFERED',
value: '1',
},
{
name: 'NIFI_NODES',
value: this.nodeList.map(x => `CN=${x}.${this.namespace}.${this.props.hostedZoneName}`).join(','),
},
],
volumeMounts: [
{
name: 'nifi-config',
mountPath: `${nifiInitBaseDir}/conf`,
},
{
name: 'aws-creds',
mountPath: `/home/nifi/.aws`,
},
{
name: 'pip-local',
mountPath: `/.local`,
},
{
name: 'nifi-init-scripts',
mountPath: `${nifiInitBaseDir}/scripts`,
},
{
name: 'nifi-data',
mountPath: `${nifiDataDir}`,
},
{
mountPath: `${sslBasePath}/manager`,
name: 'manager-ssl',
readOnly: true,
},
...sslSecretVolumes.map(x => x[1]),
],
},
{
name: 'nifi',
image: `apache/nifi:${this.props.nifiImageTag ?? NifiClusterChart.DEFAULT_NIFI_IMAGE_TAG}`,
ports: [{ containerPort: this.props.httpsPort }],
command: ['bash', '-c', `${nifiInitBaseDir}/scripts/nifi_start.sh`],
resources: {
requests: {
memory: k8s.Quantity.fromString(this.props.nodeMemory),
cpu: k8s.Quantity.fromString(this.props.nodeCpu),
},
limits: {
memory: k8s.Quantity.fromString(this.props.nodeMemory),
cpu: k8s.Quantity.fromString(this.props.nodeCpu),
},
},
env: [
{
name: 'NIFI_INIT_DIR',
value: nifiInitBaseDir,
},
{
name: 'NIFI_DATA_DIR',
value: nifiDataDir,
},
{
name: 'NIFI_HOME',
value: '/opt/nifi/nifi-current',
},
{
name: 'NIFI_SENSITIVE_PROPS_KEY',
valueFrom: {
secretKeyRef: {
name: nifiSecretName,
key: 'sensitive-props-key',
optional: false,
},
},
},
{
name: 'NIFI_SSL_BASE_PATH',
value: sslBasePath,
},
{
name: 'NIFI_ZOOKEEPER_CONNECT_STRING',
value: this.props.zkConnectString,
},
{
name: 'NIFI_KEYSTORE_PASSWORD',
valueFrom: {
secretKeyRef: {
name: nifiSecretName,
key: 'keystore-password',
optional: false,
},
},
},
{
name: 'NIFI_TRUSTSTORE_PASSWORD',
valueFrom: {
secretKeyRef: {
name: nifiSecretName,
key: 'keystore-password',
optional: false,
},
},
},
],
volumeMounts: [
{
name: 'nifi-config',
mountPath: `${nifiInitBaseDir}/conf`,
},
{
name: 'nifi-init-scripts',
mountPath: `${nifiInitBaseDir}/scripts`,
},
{
name: 'nifi-data',
mountPath: `${nifiDataDir}`,
},
{
name: 'aws-creds',
mountPath: `/home/nifi/.aws`,
},
...sslSecretVolumes.map(x => x[1]),
],
},
],
},
},
},
};
const sts = new k8s.KubeStatefulSet(this, 'nifi-sts', nifiStsProps);
pvs.forEach(pv => {
sts.addDependency(pv);
});
return sts;
}