Configure Celeborn for Spark on Kubernetes using CLI

sites:
  core:
    fs.defaultFS: hdfs://adh
    hadoop.security.authentication: simple
    dfs.client.failover.proxy.provider.adh: org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider
    dfs.ha.namenodes.adh: nn_tsn-adh-k8s-1,nn_tsn-adh-k8s-3
    dfs.namenode.rpc-address.adh.nn_tsn-adh-k8s-1: tsn-adh-k8s-1.ru-central1.internal:8020
    dfs.namenode.rpc-address.adh.nn_tsn-adh-k8s-3: tsn-adh-k8s-3.ru-central1.internal:8020
    dfs.nameservices: adh
  hdfs:
    dfs.client.read.shortcircuit: false
  ozone:
    ozone.om.address.adh.om_tsn-adh-k8s-1: tsn-adh-k8s-1.ru-central1.internal:9862
    ozone.om.address.adh.om_tsn-adh-k8s-2: tsn-adh-k8s-2.ru-central1.internal:9862
    ozone.om.address.adh.om_tsn-adh-k8s-3: tsn-adh-k8s-3.ru-central1.internal:9862
    ozone.om.nodes.adh: om_tsn-adh-k8s-1,om_tsn-adh-k8s-2,om_tsn-adh-k8s-3
    ozone.om.service.ids: adhom
  hive:
    hive.metastore.sasl.enabled: false
    hive.metastore.uris: thrift://tsn-adh-k8s-1.ru-central1.internal:9083
    metastore.use.SSL: false

apiVersion: adc.arenadata.io/v1alpha1
kind: Celeborn
metadata:
  name: celeborn
  namespace: celeborn (1)
spec:
  image: hub.arenadata.io/adc-enterprise/celeborn:<tag> (2)

  # Whether the CLI creates the product namespace.
  # The namespace name is set in metadata.namespace.
  namespace:
    create: true

  ## Image pull secret for a private registry.
  ## Set 'externalSecretName' to reference an existing Secret,
  ## or set 'credentials' and optionally 'secretName' to let the CLI create one.
  #imagePullSecret:
  #
  #  # Use a Secret managed outside ADC.
  #  externalSecretName: existing-registry-secret
  #
  #  ## Or let ADC create the Secret.
  #  #secretName: custom-registry-secret
  #  #credentials:
  #  #  registry: registry.example.com
  #  #  username: user
  #  #  password: pass
  master:
    replicas: 1
    #resources:
    #  limits:
    #    cpu: "2"
    #    memory: 2Gi
    #  requests:
    #    cpu: "1"
    #    memory: 1Gi

    # Environment variables passed to the component container.
    env:
      - name: CELEBORN_MASTER_MEMORY
        value: 512m
      - name: CELEBORN_LOG_DIR
        value: /tmp/logs
      - name: CELEBORN_NO_DAEMONIZE
        value: "true"
      - name: CELEBORN_MASTER_JAVA_OPTS
        value: -Dio.netty.leakDetectionLevel=advanced -Djava.net.preferIPv4Stack=true
  worker:
    replicas: 2
    #resources:
    #  limits:
    #    cpu: "2"
    #    memory: 6Gi
    #  requests:
    #    cpu: "1"
    #    memory: 3Gi

    # Environment variables passed to the component container.
    env:
      - name: CELEBORN_WORKER_MEMORY
        value: 512m
      - name: CELEBORN_LOG_DIR
        value: /tmp/logs
      - name: CELEBORN_NO_DAEMONIZE
        value: "true"
      - name: CELEBORN_WORKER_JAVA_OPTS
        value: -Dio.netty.leakDetectionLevel=advanced -Djava.net.preferIPv4Stack=true
      - name: CELEBORN_WORKER_OFFHEAP_MEMORY
        value: 12g

  ## Monitoring configuration.
  #monitoring:
  #  # Enables Prometheus metrics export from this product's pods.
  #  exportMetrics: true
  #
  #  ## Renders a namespace-scoped vmagent that sends metrics to an external ADM.
  #  #vmagent:
  #  #  remoteWrite:
  #  #    url: http://vminsert.example.com:8480/insert/0/prometheus/api/v1/write
  #  #  scrapeInterval: 15s
  #  #  image: hub.arenadata.io/adm-enterprise/vmagent:1.136.0-adm-5.0.0-x86_64
  #  #
  #  #  ## HTTPS settings used by vmagent when scraping product metrics.
  #  #  #tls:
  #  #  #  ## CA certificate source used to verify the metrics endpoint.
  #  #  #  #ca:
  #  #  #  #  # Use a Secret managed outside ADC.
  #  #  #  #  externalSecretName: existing-product-metrics-ca
  #  #  #  #
  #  #  #  #  ## Or let ADC create the Secret.
  #  #  #  #  #secretName: product-metrics-ca
  #  #  #  #
  #  #  #  #  # Key containing the CA certificate in the referenced Secret.
  #  #  #  #  certificateKey: ca.crt
  #  #  #  #
  #  #  #  #  ## Local CA certificate read by ADC to create the configured Secret.
  #  #  #  #  #files:
  #  #  #  #  #  certificatePath: /path/to/ca.crt
  #  #  #
  #  #  #  ## Server name used to verify the metrics endpoint certificate hostname.
  #  #  #  #serverName: metrics.example.com
  #  #  #
  #  #  #  # Skip verification of the metrics endpoint certificate. Do not use together with ca.
  #  #  #  insecureSkipVerify: true

  # Shuffle data storage, declared as a list of tiers (storage.tiers).
  storage:
    # Storage tiers, one per backend; the order does not matter. type: SSD, HDD, S3, HDFS, MEMORY.
    # Local SSD/HDD may repeat (one per disk; optional limit, plus volumeClaimTemplate or hostPath). Remote
    # S3/HDFS are single; dir is the URI (s3a:// for S3; hdfs:// or ofs:// for Ozone on HDFS).
    # MEMORY is a cache tier (no dir) needing a durable tier; HDFS Kerberos/NameNode go in 'hadoop:'. Example:
    #   - dir: /mnt/disk2
    #     limit: 200Gi
    #     type: SSD
    #     volumeClaimTemplate:
    #       accessModes: [ReadWriteOnce]
    #       resources: { requests: { storage: 200Gi } }
    #       storageClassName: fast-ssd
    #   - dir: s3a://bucket/celeborn
    #     s3:
    #       accessKey: ACCESSKEY     # with secretKey; omit both if they come from core-site.xml
    #       endpoint: https://s3:9878
    #       pathStyleAccess: true    # default; required by the Ozone S3 Gateway
    #       region: us-east-1
    #       secretKey: SECRETKEY
    #     type: S3
    #   - dir: hdfs://nn/celeborn   # or ofs://om/volume/bucket/celeborn for Ozone
    #     type: HDFS
    #   - type: MEMORY   # no dir/limit; size via celeborn.worker.directMemoryRatioForMemoryFileStorage
    tiers: (3)
      - type: HDD (4)
        dir: /data
        limit: 50Gi
      - type: HDFS (5)
        dir: hdfs:///tmp/celeborn
      #- type: S3 (6)
      #  dir: s3a://nia-demo/celeborn
      #  s3:
      #    accessKey: <access-key>
      #    endpoint: <endpoint>
      #    pathStyleAccess: true
      #    region: <region>
      #    secretKey: <secret-key>
      - type: MEMORY (7)

  ## Extra celeborn.conf properties (key: value), applied last. Overrides the CLI-managed defaults; use with care.
  #config:
  #  celeborn.worker.flusher.threads: "256"

  ## Hadoop client configuration merged into core-site.xml.
  ## Optional maps: core, hdfs, ozone.
  hadoop:
    core:
      dfs.client.failover.proxy.provider.adh: org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider
      dfs.ha.namenodes.adh: nn_tsn-adh-k8s-1,nn_tsn-adh-k8s-3
      dfs.namenode.rpc-address.adh.nn_tsn-adh-k8s-1: tsn-adh-k8s-1.ru-central1.internal:8020
      dfs.namenode.rpc-address.adh.nn_tsn-adh-k8s-3: tsn-adh-k8s-3.ru-central1.internal:8020
      dfs.nameservices: adh
      fs.defaultFS: hdfs://adh
      hadoop.security.authentication: simple
    hdfs:
      dfs.client.read.shortcircuit: "false"
    hive:
      hive.metastore.sasl.enabled: "false"
      hive.metastore.uris: thrift://tsn-adh-k8s-1.ru-central1.internal:9083
      metastore.use.SSL: "false"
    ozone:
      ozone.om.address.adh.om_tsn-adh-k8s-1: tsn-adh-k8s-1.ru-central1.internal:9862
      ozone.om.address.adh.om_tsn-adh-k8s-2: tsn-adh-k8s-2.ru-central1.internal:9862
      ozone.om.address.adh.om_tsn-adh-k8s-3: tsn-adh-k8s-3.ru-central1.internal:9862
      ozone.om.nodes.adh: om_tsn-adh-k8s-1,om_tsn-adh-k8s-2,om_tsn-adh-k8s-3
      ozone.om.service.ids: adhom

  ## Use an external Secret containing a complete celeborn-defaults.conf,
  ## or let ADC render and manage this Secret.
  #configsSecret:
  #  # Use a Secret managed outside ADC.
  #  externalSecretName: existing-celeborn-defaults
  #
  #  ## Or let ADC create the Secret.
  #  #secretName: custom-celeborn-defaults

  ## Java KeyStore/TrustStore certificate configuration.
  ## Set externalSecretName to reference an existing Secret,
  ## or set files and optional secretName to have ADC create it.
  #ssl:
  #
  #  ## Name of the Secret containing Java keystores.
  #  #secretName: custom-ssl-secret
  #  externalSecretName: existing-ssl-secret
  #
  #  # Key in the Secret containing the truststore file.
  #  trustStoreKey: truststore.jks
  #
  #  ## Password for the truststore (optional).
  #  #trustStorePassword: bigdata
  #
  #  ## Key in the Secret containing the keystore file (optional).
  #  #keyStoreKey: keystore.jks
  #
  #  ## Password for the keystore (optional).
  #  #keyStorePassword: bigdata
  #
  #  ## Local files 'adc apply' puts into the Secret named by ssl.secretName.
  #  ## Relative paths are resolved against the config file.
  #  #files:
  #  #  trustStorePath: /path/to/truststore.jks
  #
  #  # Enable TLS on the Celeborn RPC service (rpc_service). When true the CLI renders
  #  # the celeborn.ssl.rpc_service.* properties
  #  # into celeborn-defaults.conf and requires ssl.keyStoreKey and ssl.trustStoreKey.
  #  rpcEncryption: false
  #
  #  # Enable TLS on Celeborn's data-plane modules (push, fetch, replicate). When
  #  # true the CLI renders celeborn.ssl.push.*
  #  # celeborn.ssl.fetch.* and celeborn.ssl.replicate.* properties into celeborn-defaults.conf
  #  # and requires ssl.keyStoreKey and ssl.trustStoreKey.
  #  dataEncryption: false

  ## Kerberos configuration for authentication.
  #kerberos:
  #  realm: EXAMPLE.COM
  #
  #  # Service name in the Kerberos principal. Defaults to the product name.
  #  service: celeborn
  #
  #  # Hostname in the Kerberos principal.
  #  # Required for a fixed service principal; leave it empty only to derive one principal per pod from the cluster domain.
  #  hostname: kerberos.example.com
  #  keytab:
  #    # true - kerberos-operator creates the keytab Secret.
  #    # false (default) - reference an existing keytab Secret with name keytab.secretName.
  #    create: false
  #
  #    # Name of the keytab Secret.
  #    # Optional when create: true - names the generated Secret (default: <name>-keytab).
  #    # Required when create: false - must reference an existing Secret.
  #    secretName: celeborn-keytab
  #
  #    # Label selector for the Pod that generates the keytab.
  #    # Required when create: true; ignored when create: false.
  #    labelSelector:
  #      env: prod
  #    rotation:
  #      interval: 24h
  #      checkInterval: 1h

  ## Controls whether Secret/ConfigMap changes restart pods.
  ## Set enabled: false to update referenced Secrets without restarting
  ## the workload; pods keep running the previous configuration until
  ## the policy is re-enabled. Defaults to enabled.
  #configurationRollout:
  #  enabled: false

---
apiVersion: v1
kind: Namespace
metadata:
  name: celeborn
spec: {}
status: {}
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  labels:
    app.kubernetes.io/component: master
    app.kubernetes.io/instance: celeborn
    app.kubernetes.io/managed-by: adc-cli
    app.kubernetes.io/name: celeborn
  name: celeborn-celeborn-master
  namespace: celeborn
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/component: master
      app.kubernetes.io/instance: celeborn
      app.kubernetes.io/managed-by: adc-cli
      app.kubernetes.io/name: celeborn
  serviceName: celeborn-celeborn-master-svc
  template:
    metadata:
      labels:
        app.kubernetes.io/component: master
        app.kubernetes.io/instance: celeborn
        app.kubernetes.io/managed-by: adc-cli
        app.kubernetes.io/name: celeborn
    spec:
      containers:
      - args:
        - /bin/sh
        - -c
        - until nslookup celeborn-celeborn-master-0.celeborn-celeborn-master-svc.celeborn.svc.cluster.local
          && true; do echo 'waiting for master'; sleep 2; done && exec /opt/celeborn/sbin/start-master.sh
        command:
        - /usr/bin/tini
        - --
        env:
        - name: CELEBORN_MASTER_MEMORY
          value: 512m
        - name: CELEBORN_LOG_DIR
          value: /tmp/logs
        - name: CELEBORN_NO_DAEMONIZE
          value: "true"
        - name: CELEBORN_MASTER_JAVA_OPTS
          value: -Dio.netty.leakDetectionLevel=advanced -Djava.net.preferIPv4Stack=true
        - name: HADOOP_CONF_DIR
          value: /opt/celeborn/conf
        image: hub.arenadata.io/adc-enterprise/celeborn:<tag>
        imagePullPolicy: Always
        name: app
        ports:
        - containerPort: 9097
          name: rpc
        - containerPort: 9098
          name: internal
        resources:
          limits:
            cpu: "2"
            memory: 2Gi
          requests:
            cpu: "1"
            memory: 1Gi
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          privileged: false
          readOnlyRootFilesystem: true
          runAsGroup: 10006
          runAsNonRoot: true
          runAsUser: 10006
        volumeMounts:
        - mountPath: /opt/celeborn/conf/
          name: celeborn-conf
        - mountPath: /tmp
          name: tmp
      securityContext:
        fsGroup: 10006
        fsGroupChangePolicy: OnRootMismatch
        runAsGroup: 10006
        runAsNonRoot: true
        runAsUser: 10006
        supplementalGroups:
        - 10006
      volumes:
      - name: celeborn-conf
        secret:
          secretName: celeborn-celeborn-defaults
      - emptyDir: {}
        name: tmp
  updateStrategy: {}
status:
  availableReplicas: 0
  replicas: 0
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  labels:
    app.kubernetes.io/component: worker
    app.kubernetes.io/instance: celeborn
    app.kubernetes.io/managed-by: adc-cli
    app.kubernetes.io/name: celeborn
  name: celeborn-celeborn-worker
  namespace: celeborn
spec:
  replicas: 2
  selector:
    matchLabels:
      app.kubernetes.io/component: worker
      app.kubernetes.io/instance: celeborn
      app.kubernetes.io/managed-by: adc-cli
      app.kubernetes.io/name: celeborn
  serviceName: celeborn-celeborn-worker-svc
  template:
    metadata:
      labels:
        app.kubernetes.io/component: worker
        app.kubernetes.io/instance: celeborn
        app.kubernetes.io/managed-by: adc-cli
        app.kubernetes.io/name: celeborn
    spec:
      containers:
      - args:
        - /bin/sh
        - -c
        - until nslookup celeborn-celeborn-master-0.celeborn-celeborn-master-svc.celeborn.svc.cluster.local
          && true; do echo 'waiting for master'; sleep 2; done && exec /opt/celeborn/sbin/start-worker.sh
        command:
        - /usr/bin/tini
        - --
        env:
        - name: CELEBORN_WORKER_MEMORY
          value: 512m
        - name: CELEBORN_LOG_DIR
          value: /tmp/logs
        - name: CELEBORN_NO_DAEMONIZE
          value: "true"
        - name: CELEBORN_WORKER_JAVA_OPTS
          value: -Dio.netty.leakDetectionLevel=advanced -Djava.net.preferIPv4Stack=true
        - name: CELEBORN_WORKER_OFFHEAP_MEMORY
          value: 12g
        - name: HADOOP_CONF_DIR
          value: /opt/celeborn/conf
        image: hub.arenadata.io/adc-enterprise/celeborn:<tag>
        imagePullPolicy: Always
        name: app
        resources:
          limits:
            cpu: "2"
            memory: 6Gi
          requests:
            cpu: "1"
            memory: 3Gi
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          privileged: false
          readOnlyRootFilesystem: true
          runAsGroup: 10006
          runAsNonRoot: true
          runAsUser: 10006
        volumeMounts:
        - mountPath: /opt/celeborn/conf/
          name: celeborn-conf
        - mountPath: /tmp
          name: tmp
        - mountPath: /data
          name: data-0
      securityContext:
        fsGroup: 10006
        fsGroupChangePolicy: OnRootMismatch
        runAsGroup: 10006
        runAsNonRoot: true
        runAsUser: 10006
        supplementalGroups:
        - 10006
      volumes:
      - name: celeborn-conf
        secret:
          secretName: celeborn-celeborn-defaults
      - emptyDir: {}
        name: tmp
      - emptyDir: {}
        name: data-0
  updateStrategy: {}
status:
  availableReplicas: 0
  replicas: 0
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app.kubernetes.io/component: master
    app.kubernetes.io/instance: celeborn
    app.kubernetes.io/managed-by: adc-cli
    app.kubernetes.io/name: celeborn
  name: celeborn-celeborn-master-svc
  namespace: celeborn
spec:
  clusterIP: None
  ports:
  - name: rpc
    port: 9097
    protocol: TCP
    targetPort: 9097
  selector:
    app.kubernetes.io/component: master
    app.kubernetes.io/instance: celeborn
    app.kubernetes.io/managed-by: adc-cli
    app.kubernetes.io/name: celeborn
status:
  loadBalancer: {}
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app.kubernetes.io/component: worker
    app.kubernetes.io/instance: celeborn
    app.kubernetes.io/managed-by: adc-cli
    app.kubernetes.io/name: celeborn
  name: celeborn-celeborn-worker-svc
  namespace: celeborn
spec:
  clusterIP: None
  ports:
  - name: rpc
    port: 9096
    protocol: TCP
    targetPort: 9096
  selector:
    app.kubernetes.io/component: worker
    app.kubernetes.io/instance: celeborn
    app.kubernetes.io/managed-by: adc-cli
    app.kubernetes.io/name: celeborn
status:
  loadBalancer: {}
---
apiVersion: v1
kind: Secret
metadata:
  name: celeborn-celeborn-defaults
  namespace: celeborn
stringData:
  celeborn-defaults.conf: |
    celeborn.network.bind.preferIpAddress = false
    celeborn.worker.sortPartition.threads = 64
    celeborn.worker.commitFiles.timeout = 240s
    celeborn.master.slot.assign.policy = roundrobin
    celeborn.rpc.askTimeout = 240s
    celeborn.worker.storage.dirs = /data:disktype=HDD:capacity=53687091200
    celeborn.worker.commitFiles.threads = 128
    celeborn.worker.replicate.fastFail.duration = 240s
    celeborn.master.endpoints = celeborn-celeborn-master-0.celeborn-celeborn-master-svc.celeborn.svc.cluster.local:9097
    celeborn.storage.availableTypes = HDD,HDFS,MEMORY
    celeborn.storage.hdfs.dir = hdfs:///tmp/celeborn
    celeborn.hadoop.fs.hdfs.impl.disable.cache = true
    celeborn.worker.flusher.hdfs.buffer.size = 4m
    celeborn.worker.directMemoryRatioForMemoryFileStorage = 0.1
  core-site.xml: |-
    <configuration>
      <property>
        <name>dfs.client.failover.proxy.provider.adh</name>
        <value>org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider</value>
      </property>
      <property>
        <name>dfs.client.read.shortcircuit</name>
        <value>false</value>
      </property>
      <property>
        <name>dfs.ha.namenodes.adh</name>
        <value>nn_tsn-adh-k8s-1,nn_tsn-adh-k8s-3</value>
      </property>
      <property>
        <name>dfs.namenode.rpc-address.adh.nn_tsn-adh-k8s-1</name>
        <value>tsn-adh-k8s-1.ru-central1.internal:8020</value>
      </property>
      <property>
        <name>dfs.namenode.rpc-address.adh.nn_tsn-adh-k8s-3</name>
        <value>tsn-adh-k8s-3.ru-central1.internal:8020</value>
      </property>
      <property>
        <name>dfs.nameservices</name>
        <value>adh</value>
      </property>
      <property>
        <name>fs.defaultFS</name>
        <value>hdfs://adh</value>
      </property>
      <property>
        <name>hadoop.security.authentication</name>
        <value>simple</value>
      </property>
      <property>
        <name>ozone.om.address.adh.om_tsn-adh-k8s-1</name>
        <value>tsn-adh-k8s-1.ru-central1.internal:9862</value>
      </property>
      <property>
        <name>ozone.om.address.adh.om_tsn-adh-k8s-2</name>
        <value>tsn-adh-k8s-2.ru-central1.internal:9862</value>
      </property>
      <property>
        <name>ozone.om.address.adh.om_tsn-adh-k8s-3</name>
        <value>tsn-adh-k8s-3.ru-central1.internal:9862</value>
      </property>
      <property>
        <name>ozone.om.nodes.adh</name>
        <value>om_tsn-adh-k8s-1,om_tsn-adh-k8s-2,om_tsn-adh-k8s-3</value>
      </property>
      <property>
        <name>ozone.om.service.ids</name>
        <value>adhom</value>
      </property>
    </configuration>
  hive-site.xml: |-
    <configuration>
      <property>
        <name>hive.metastore.sasl.enabled</name>
        <value>false</value>
      </property>
      <property>
        <name>hive.metastore.uris</name>
        <value>thrift://tsn-adh-k8s-1.ru-central1.internal:9083</value>
      </property>
      <property>
        <name>metastore.use.SSL</name>
        <value>false</value>
      </property>
    </configuration>
type: Opaque

sites:
  core:
    fs.defaultFS: hdfs://adh
    hadoop.security.authentication: simple
    dfs.client.failover.proxy.provider.adh: org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider
    dfs.ha.namenodes.adh: nn_tsn-adh-k8s-1,nn_tsn-adh-k8s-3
    dfs.namenode.rpc-address.adh.nn_tsn-adh-k8s-1: tsn-adh-k8s-1.ru-central1.internal:8020
    dfs.namenode.rpc-address.adh.nn_tsn-adh-k8s-3: tsn-adh-k8s-3.ru-central1.internal:8020
    dfs.nameservices: adh
  hdfs:
    dfs.client.read.shortcircuit: false
  ozone:
    ozone.om.address.adh.om_tsn-adh-k8s-1: tsn-adh-k8s-1.ru-central1.internal:9862
    ozone.om.address.adh.om_tsn-adh-k8s-2: tsn-adh-k8s-2.ru-central1.internal:9862
    ozone.om.address.adh.om_tsn-adh-k8s-3: tsn-adh-k8s-3.ru-central1.internal:9862
    ozone.om.nodes.adh: om_tsn-adh-k8s-1,om_tsn-adh-k8s-2,om_tsn-adh-k8s-3
    ozone.om.service.ids: adhom
  hive:
    hive.metastore.sasl.enabled: false
    hive.metastore.uris: thrift://tsn-adh-k8s-1.ru-central1.internal:9083
    metastore.use.SSL: false

apiVersion: adc.arenadata.io/v1alpha1
kind: SparkApplication
metadata:
  name: spark-application
  namespace: spark-applications (1)
spec:
  image: hub.arenadata.io/adc-enterprise/spark3:<tag> (2)

  ## Image pull secret for a private registry.
  ## Set 'externalSecretName' to reference an existing Secret,
  ## or set 'credentials' and optionally 'secretName' to let the CLI create one.
  #imagePullSecret:
  #  # Use a Secret managed outside ADC.
  #  externalSecretName: existing-registry-secret
  #
  #  ## Or let ADC create the Secret.
  #  #secretName: custom-registry-secret
  #
  #  #credentials:
  #  #  registry: registry.example.com
  #  #  username: user
  #  #  password: pass

  hadoop: (3)
    core:
      dfs.client.failover.proxy.provider.adh: org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider
      dfs.ha.namenodes.adh: nn_tsn-adh-k8s-1,nn_tsn-adh-k8s-3
      dfs.namenode.rpc-address.adh.nn_tsn-adh-k8s-1: tsn-adh-k8s-1.ru-central1.internal:8020
      dfs.namenode.rpc-address.adh.nn_tsn-adh-k8s-3: tsn-adh-k8s-3.ru-central1.internal:8020
      dfs.nameservices: adh
      fs.defaultFS: hdfs://adh
      hadoop.security.authentication: simple
    hdfs:
      dfs.client.read.shortcircuit: "false"
    hive:
      hive.metastore.sasl.enabled: "false"
      hive.metastore.uris: thrift://tsn-adh-k8s-1.ru-central1.internal:9083
      metastore.use.SSL: "false"
    ozone:
      ozone.om.address.adh.om_tsn-adh-k8s-1: tsn-adh-k8s-1.ru-central1.internal:9862
      ozone.om.address.adh.om_tsn-adh-k8s-2: tsn-adh-k8s-2.ru-central1.internal:9862
      ozone.om.address.adh.om_tsn-adh-k8s-3: tsn-adh-k8s-3.ru-central1.internal:9862
      ozone.om.nodes.adh: om_tsn-adh-k8s-1,om_tsn-adh-k8s-2,om_tsn-adh-k8s-3
      ozone.om.service.ids: adhom

  ## Kerberos configuration for authentication.
  #kerberos:
  #  principal: user@EXAMPLE.COM
  #
  #  # CLI reads the local files and creates the kerberos-ccache Secret on 'adc apply'.
  #  # Alternative - keytab mode: replace this block with:
  #  #   keytab:
  #  #     secretName: <name-of-existing-keytab-secret>
  #  ticketCache:
  #    #secretName: custom-ticket-cache
  #    externalSecretName: existing-ticket-cache
  #    #ticketPath: /tmp/krb5cc_1000
  #    #krb5ConfPath: /etc/krb5.conf

  ## Ranger plugin configuration.
  ## Uncomment and fill the lines below. adc apply derives the rest.
  #ranger:
  #  # fill ranger.plugin.spark.policy.rest.url below with Ranger endpoint, e.g. https://adps-adc.ru-central1.internal:6182
  #  # fill ranger.plugin.spark.service.name below with Ranger service name you want to use for product, e.g. adc_spark_id_1
  #  security:
  #    ranger.plugin.spark.policy.rest.url: ""
  #    ranger.plugin.spark.service.name: ""
  #
  #  # fill xasecure.audit.destination.solr.zookeepers below with Zookeepers endpoints to resolve solr service, e.g. adps-adc.ru-central1.internal:2181/Arenadata.Hadoop-2.solr.server
  #  audit:
  #    xasecure.audit.destination.solr.zookeepers: ""
  #
  #  # Local Ranger files 'adc apply' writes into the configs Secret.
  #  # Relative paths are resolved against the config file.
  #  files:
  #    jceksStorePath: /path/to/ranger.jceks

  ## Java KeyStore/TrustStore certificate configuration.
  ## Set externalSecretName to reference an existing Secret,
  ## or set files and optional secretName to have ADC create it.
  #ssl:
  #  ## Name of the Secret containing Java keystores.
  #  #secretName: custom-ssl-secret
  #  externalSecretName: existing-ssl-secret
  #
  #  # Key in the Secret containing the truststore file.
  #  trustStoreKey: truststore.jks
  #
  #  ## Password for the truststore (optional).
  #  #trustStorePassword: bigdata
  #
  #  ## Key in the Secret containing the keystore file (optional).
  #  #keyStoreKey: keystore.jks
  #
  #  ## Password for the keystore (optional).
  #  #keyStorePassword: bigdata
  #
  #  ## Local files 'adc apply' puts into the Secret named by ssl.secretName.
  #  ## Relative paths are resolved against the config file.
  #  #files:
  #  #  trustStorePath: /path/to/truststore.jks
  #  #  #keyStorePath: /path/to/keystore.jks

  ## Use an external Hadoop configs Secret instead of the one rendered by ADC.
  #hadoopConfigsSecret:
  #  # Use a Secret managed outside ADC.
  #  externalSecretName: existing-spark-hadoop-configs
  #
  #  ## Or let ADC create the Secret.
  #  #secretName: custom-spark-hadoop-configs

  ## Use an external Ranger configs Secret instead of the one rendered by ADC.
  #rangerConfigsSecret:
  #  # Use a Secret managed outside ADC.
  #  externalSecretName: existing-spark-ranger-configs
  #
  #  ## Or let ADC create the Secret.
  #  #secretName: custom-spark-ranger-configs

  # Spark application main resource (e.g. local:///opt/spark/examples/jars/spark-examples.jar).
  mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.13-3.5.4.4-4.3.0-2.jar" (4)

  ## HDFS or local directory for the Spark event log.
  ## When set, the CLI adds spark.eventLog.enabled=true, spark.eventLog.dir,
  ## spark.eventLog.rolling.enabled=true and spark.eventLog.rolling.interval=30s to sparkConf.
  #eventLogDir: ""

  ## Fully-qualified main class name. Required for Java/Scala applications.
  mainClass: "org.apache.spark.examples.sql.SparkSQLExample" (5)

  # ServiceAccount used by the Spark driver, also injected into
  # spark.kubernetes.authenticate.driver.serviceAccountName. The CLI creates it, plus a Role
  # and RoleBinding for Spark pods, by default (create: true). Set create: false to skip that
  # and only reference a ServiceAccount managed elsewhere.
  # The Role rules are managed by the CLI and cannot be customized.
  serviceAccount: (6)
    create: true
    name: spark-application
  job: (7)
    ## true (default) deletes the spark-submit Job pod after it finishes; set false to keep it for debugging.
    deleteOnTermination: false

    #resources:
    #  limits:
    #    cpu: "1"
    #    memory: 512Mi
    #  requests:
    #    cpu: 500m
    #    memory: 64Mi

    ## Component arguments. Key-value pairs passed to the component configuration.
    #args:
    #  executor-memory: 1g
    #  num-executors: "2"

  ## Application arguments appended after mainApplicationFile.
  #args:
  #  - "100"

  # Spark configuration entries (spark.*).
  sparkConf: (8)
    spark.artifactory.dir.path: /tmp/artifacts
    spark.jars.ivy: /tmp/ivy
    spark.local.dir: /tmp/data
    spark.sql.catalog.spark_catalog: org.apache.iceberg.spark.SparkSessionCatalog
    spark.sql.extensions: org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
    spark.sql.security.confblacklist: spark.sql.extensions

  ## Seconds after the application finishes (Succeeded, or Failed with no
  ## retries left) before the SparkApplication is deleted. Omit to keep it
  ## until explicit deletion.
  #ttlSecondsAfterFinished: 3600 (9)

  ## Celeborn remote shuffle service for Spark.
  ## tiers mirror the cluster's storage.tiers: local (SSD/HDD) and MEMORY advertise their name only,
  ## remote (S3/HDFS) also set dir (and, for S3, credentials). Ozone is an HDFS tier with an ofs:// dir.
  celeborn:
  #  # Storage tiers mirroring the cluster's storage.tiers so the client advertises the same layers.
  #  # type: SSD, HDD, S3, HDFS, MEMORY. SSD/HDD and MEMORY are advertise-only here (no dir; worker-only
  #  # fields ignored); remote S3/HDFS set dir (s3a:// for S3; hdfs:// or ofs:// for Ozone on HDFS). Example:
  #  #   - type: MEMORY   # cache tier; pair with a durable tier below, no dir
  #  #   - type: SSD      # advertise-only on the client, no dir needed
  #  #   - dir: s3a://bucket/celeborn
  #  #     s3: { endpoint: https://s3:9878, region: us-east-1 }
  #  #     type: S3
  #  #   - dir: hdfs://nn/celeborn   # or ofs://om/volume/bucket/celeborn for Ozone
  #  #     type: HDFS
    tiers: (10)
      - type: HDD
      - type: HDFS
        dir: hdfs:///tmp/celeborn
      - type: MEMORY
    masterEndpoint: "celeborn-celeborn-master-0.celeborn-celeborn-master-svc.celeborn.svc.cluster.local:9097" (11)
    extraSparkConf:
      spark.sql.adaptive.enabled: "true"
  #
  #  # Enable TLS on the client's RPC connection to the Celeborn cluster.
  #  # When true the CLI renders spark.celeborn.ssl.* into the Spark conf
  #  # and requires the ssl section with trustStoreKey.
  #  rpcEncryption: false
  #
  #  # Enable TLS on the client's data module, which carries shuffle push/fetch
  #  # traffic between executors and workers. Requires the ssl section with trustStoreKey.
  #  dataEncryption: false

  ## YuniKorn scheduler configuration for queue selection and Gang scheduling.
  ## Uncomment the block to route the Spark job into a YuniKorn queue; the CLI renders the
  ## scheduler name, queue labels and gang annotations into sparkConf.
  #yunikorn:
  #  queue: root.analytics
  #  taskGroups:
  #    - minMember: 1
  #      minResource:
  #        cpu: "1"
  #        memory: 1433Mi
  #      name: spark-driver
  #    - minMember: 2
  #      minResource:
  #        cpu: "1"
  #        memory: 1433Mi
  #      name: spark-executor

  ## Monitoring configuration.
  #monitoring:
  #  # Enables Prometheus metrics export from this product's pods.
  #  exportMetrics: true

---
apiVersion: v1
kind: Secret
metadata:
  name: spark-application-configs
  namespace: spark-applications
stringData:
  core-site.xml: |-
    <configuration>
      <property>
        <name>dfs.client.failover.proxy.provider.adh</name>
        <value>org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider</value>
      </property>
      <property>
        <name>dfs.client.read.shortcircuit</name>
        <value>false</value>
      </property>
      <property>
        <name>dfs.ha.namenodes.adh</name>
        <value>nn_tsn-adh-k8s-1,nn_tsn-adh-k8s-3</value>
      </property>
      <property>
        <name>dfs.namenode.rpc-address.adh.nn_tsn-adh-k8s-1</name>
        <value>tsn-adh-k8s-1.ru-central1.internal:8020</value>
      </property>
      <property>
        <name>dfs.namenode.rpc-address.adh.nn_tsn-adh-k8s-3</name>
        <value>tsn-adh-k8s-3.ru-central1.internal:8020</value>
      </property>
      <property>
        <name>dfs.nameservices</name>
        <value>adh</value>
      </property>
      <property>
        <name>fs.defaultFS</name>
        <value>hdfs://adh</value>
      </property>
      <property>
        <name>hadoop.security.authentication</name>
        <value>simple</value>
      </property>
      <property>
        <name>ozone.om.address.adh.om_tsn-adh-k8s-1</name>
        <value>tsn-adh-k8s-1.ru-central1.internal:9862</value>
      </property>
      <property>
        <name>ozone.om.address.adh.om_tsn-adh-k8s-2</name>
        <value>tsn-adh-k8s-2.ru-central1.internal:9862</value>
      </property>
      <property>
        <name>ozone.om.address.adh.om_tsn-adh-k8s-3</name>
        <value>tsn-adh-k8s-3.ru-central1.internal:9862</value>
      </property>
      <property>
        <name>ozone.om.nodes.adh</name>
        <value>om_tsn-adh-k8s-1,om_tsn-adh-k8s-2,om_tsn-adh-k8s-3</value>
      </property>
      <property>
        <name>ozone.om.service.ids</name>
        <value>adhom</value>
      </property>
    </configuration>
  hive-site.xml: |-
    <configuration>
      <property>
        <name>hive.metastore.sasl.enabled</name>
        <value>false</value>
      </property>
      <property>
        <name>hive.metastore.uris</name>
        <value>thrift://tsn-adh-k8s-1.ru-central1.internal:9083</value>
      </property>
      <property>
        <name>metastore.use.SSL</name>
        <value>false</value>
      </property>
    </configuration>
type: Opaque
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: spark-application
  namespace: spark-applications
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: spark-application
  namespace: spark-applications
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - configmaps
  - persistentvolumeclaims
  - services
  - secrets
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
  - deletecollection
- apiGroups:
  - networking.k8s.io
  resources:
  - networkpolicies
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
- apiGroups:
  - events.k8s.io
  resources:
  - events
  verbs:
  - create
  - patch
  - update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: spark-application
  namespace: spark-applications
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: spark-application
subjects:
- kind: ServiceAccount
  name: spark-application
  namespace: spark-applications
---
apiVersion: spark.arenadata.io/v1alpha1
kind: SparkApplication
metadata:
  name: spark-application
  namespace: spark-applications
spec:
  args:
  - "100"
  driver:
    metadata: {}
    spec:
      image: hub.adsw.io/adc-enterprise/spark3:3.5.4.4-adc-1.4.0-rc2
      imagePullPolicy: Always
  executor:
    metadata: {}
    spec:
      image: hub.adsw.io/adc-enterprise/spark3:3.5.4.4-adc-1.4.0-rc2
      imagePullPolicy: Always
  hadoopConfigsSecretName: spark-application-configs
  job:
    deleteOnTermination: false
    metadata: {}
    spec:
      image: hub.arenadata.io/adh-enterprise/spark3:<tag>
      imagePullPolicy: Always
  mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.13-3.5.4.4-4.3.0-2.jar
  mainClass: org.apache.spark.examples.sql.SparkSQLExample
  serviceAccountName: spark-application
  sparkConf:
    spark.artifactory.dir.path: /tmp/artifacts
    spark.celeborn.client.push.replicate.enabled: "false"
    spark.celeborn.client.shuffle.compression.codec: zstd
    spark.celeborn.client.shuffle.compression.zstd.level: "12"
    spark.celeborn.client.spark.shuffle.writer: hash
    spark.celeborn.master.endpoints: celeborn-celeborn-master-0.celeborn-celeborn-master-svc.celeborn.svc.cluster.local:9097
    spark.celeborn.storage.availableTypes: HDD,HDFS,MEMORY
    spark.celeborn.storage.hdfs.dir: hdfs:///tmp/celeborn
    spark.dynamicAllocation.shuffleTracking.enabled: "false"
    spark.jars.ivy: /tmp/ivy
    spark.kubernetes.authenticate.driver.serviceAccountName: spark-application
    spark.kubernetes.namespace: spark-applications
    spark.local.dir: /tmp/data
    spark.serializer: org.apache.spark.serializer.KryoSerializer
    spark.shuffle.manager: org.apache.spark.shuffle.celeborn.SparkShuffleManager
    spark.shuffle.sort.io.plugin.class: org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO
    spark.sql.adaptive.enabled: "true"
    spark.sql.adaptive.localShuffleReader.enabled: "false"
    spark.sql.adaptive.skewJoin.enabled: "true"
    spark.sql.catalog.spark_catalog: org.apache.iceberg.spark.SparkSessionCatalog
    spark.sql.extensions: org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
    spark.sql.security.confblacklist: spark.sql.extensions
status: {}