Run Spark jobs on Kubernetes from any host using CLI

apiVersion: adc.arenadata.io/v1alpha1
kind: SparkOperator
metadata:
  name: spark-operator
  namespace: spark-operator (1)
spec:
  image: hub.arenadata.io/adc-enterprise/spark-operator:<tag> (2)

  # Number of replicas
  # replicas: 1

  resources:
    limits:
      cpu: 500m
      memory: 256Mi

  # Operator ServiceAccount. create: true (default) also creates the manager and per-payload-namespace Role/RoleBinding bound to it; create: false skips all three - name then refers to a ServiceAccount (and RBAC) managed entirely outside the CLI.
  serviceAccount: (3)
    create: true
    name: "spark-operator"

  # Whether the CLI creates the product namespace.
  # The namespace name is set in metadata.namespace.
  namespace:
    create: true

  # Create namespaces to run the payload.
  createPayloadNamespaces: true

  # List of namespaces to run the payload in.
  payloadNamespaces: (4)
    - spark-applications

  ## Image pull secret for a private registry.
  ## Set 'externalSecretName' to reference an existing Secret,
  ## or set 'credentials' and optionally 'secretName' to let the CLI create one.
  #imagePullSecret:
  #  # Use a Secret managed outside ADC.
  #  externalSecretName: existing-registry-secret
  #
  #  ## Or let ADC create the Secret.
  #  #secretName: custom-registry-secret
  #
  #  #credentials:
  #  #  registry: registry.example.com
  #  #  username: user
  #  #  password: pass

  ## Operator monitoring configuration. Supports product-specific metrics export and optional vmagent delivery to an external ADM.
  #monitoring:
  #  # Renders a namespace-scoped vmagent that sends metrics to an external ADM.
  #  vmagent:
  #    remoteWrite:
  #      url: http://vminsert.example.com/insert/0/prometheus/api/v1/write
  #    scrapeInterval: 15s
  #    image: hub.arenadata.io/adm-enterprise/vmagent:1.136.0-adm-5.0.0-x86_64
  #
  #    ## HTTPS settings used by vmagent when scraping product metrics.
  #    #tls:
  #    #  ## CA certificate source used to verify the metrics endpoint.
  #    #  #ca:
  #    #  #  # Use a Secret managed outside ADC.
  #    #  #  externalSecretName: existing-product-metrics-ca
  #    #  #
  #    #  #  ## Or let ADC create the Secret.
  #    #  #  #secretName: product-metrics-ca
  #    #  #
  #    #  #  # Key containing the CA certificate in the referenced Secret.
  #    #  #  certificateKey: ca.crt
  #    #  #
  #    #  #  ## Local CA certificate read by ADC to create the configured Secret.
  #    #  #  #files:
  #    #  #  #  certificatePath: /path/to/ca.crt
  #    #
  #    #  ## Server name used to verify the metrics endpoint certificate hostname.
  #    #  #serverName: metrics.example.com
  #    #
  #    #  # Skip verification of the metrics endpoint certificate. Do not use together with ca.
  #    #  insecureSkipVerify: true

---
apiVersion: v1
kind: Namespace
metadata:
  name: spark-operator
spec: {}
status: {}
---
apiVersion: v1
kind: Namespace
metadata:
  name: spark-applications
spec: {}
status: {}
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: spark-operator
  namespace: spark-operator
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: spark-operator-spark-operator-manager
  namespace: spark-operator
rules:
- apiGroups:
  - events.k8s.io
  resources:
  - events
  verbs:
  - create
  - patch
- apiGroups:
  - coordination.k8s.io
  resources:
  - leases
  verbs:
  - create
  - delete
  - get
  - list
  - patch
  - update
  - watch
- apiGroups:
  - spark.arenadata.io
  resources:
  - sparkapplications
  verbs:
  - get
  - list
  - patch
  - update
  - watch
- apiGroups:
  - spark.arenadata.io
  resources:
  - sparkapplications/status
  verbs:
  - get
  - patch
  - update
- apiGroups:
  - spark.arenadata.io
  resources:
  - sparkapplications/finalizers
  verbs:
  - update
- apiGroups:
  - spark.arenadata.io
  resources:
  - sparkhistoryservers
  verbs:
  - get
  - list
  - patch
  - update
  - watch
- apiGroups:
  - spark.arenadata.io
  resources:
  - sparkhistoryservers/status
  verbs:
  - get
  - patch
  - update
- apiGroups:
  - spark.arenadata.io
  resources:
  - sparkhistoryservers/finalizers
  verbs:
  - update
- apiGroups:
  - spark.arenadata.io
  resources:
  - sparkapplications
  verbs:
  - delete
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: spark-operator-spark-operator-manager
  namespace: spark-operator
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: spark-operator-spark-operator-manager
subjects:
- kind: ServiceAccount
  name: spark-operator
  namespace: spark-operator
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: spark-operator-spark-operator-payload
  namespace: spark-applications
rules:
- apiGroups:
  - events.k8s.io
  resources:
  - events
  verbs:
  - create
  - patch
- apiGroups:
  - ""
  resources:
  - configmaps
  - services
  - pods
  - secrets
  verbs:
  - create
  - delete
  - get
  - list
  - patch
  - update
  - watch
- apiGroups:
  - apps
  resources:
  - statefulsets
  verbs:
  - create
  - delete
  - get
  - list
  - patch
  - update
  - watch
- apiGroups:
  - batch
  resources:
  - jobs
  verbs:
  - create
  - delete
  - get
  - list
  - patch
  - update
  - watch
- apiGroups:
  - spark.arenadata.io
  resources:
  - sparkapplications
  verbs:
  - get
  - list
  - patch
  - update
  - watch
- apiGroups:
  - spark.arenadata.io
  resources:
  - sparkapplications/status
  verbs:
  - get
  - patch
  - update
- apiGroups:
  - spark.arenadata.io
  resources:
  - sparkapplications/finalizers
  verbs:
  - update
- apiGroups:
  - spark.arenadata.io
  resources:
  - sparkhistoryservers
  verbs:
  - get
  - list
  - patch
  - update
  - watch
- apiGroups:
  - spark.arenadata.io
  resources:
  - sparkhistoryservers/status
  verbs:
  - get
  - patch
  - update
- apiGroups:
  - spark.arenadata.io
  resources:
  - sparkhistoryservers/finalizers
  verbs:
  - update
- apiGroups:
  - spark.arenadata.io
  resources:
  - sparkapplications
  verbs:
  - delete
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: spark-operator-spark-operator-payload
  namespace: spark-applications
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: spark-operator-spark-operator-payload
subjects:
- kind: ServiceAccount
  name: spark-operator
  namespace: spark-operator
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app.kubernetes.io/component: operator
    app.kubernetes.io/managed-by: adc-cli
    arenadata.io/operator-type: spark
  name: spark-operator-spark-operator
  namespace: spark-operator
spec:
  selector:
    matchLabels:
      app.kubernetes.io/component: operator
      app.kubernetes.io/managed-by: adc-cli
      app.kubernetes.io/name: spark-operator-spark-operator
  strategy: {}
  template:
    metadata:
      labels:
        app.kubernetes.io/component: operator
        app.kubernetes.io/managed-by: adc-cli
        app.kubernetes.io/name: spark-operator-spark-operator
    spec:
      containers:
      - args:
        - -ns=spark-applications
        image: hub.arenadata.io/adc-enterprise/spark-operator:<tag>
        imagePullPolicy: Always
        livenessProbe:
          httpGet:
            path: /healthz
            port: 8081
          initialDelaySeconds: 5
          periodSeconds: 10
        name: app
        readinessProbe:
          httpGet:
            path: /readyz
            port: 8081
          initialDelaySeconds: 5
          periodSeconds: 10
        resources:
          limits:
            cpu: 500m
            memory: 256Mi
          requests:
            cpu: 500m
            memory: 256Mi
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          readOnlyRootFilesystem: true
          runAsGroup: 10001
          runAsNonRoot: true
          runAsUser: 10001
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
      serviceAccountName: spark-operator
      terminationGracePeriodSeconds: 10
status: {}

sites:
    core:
      fs.defaultFS: ""
    hdfs:
      dfs.encrypt.data.transfer.cipher.suites: AES/CTR/NoPadding
    hive:
      hive.metastore.uris: ""

apiVersion: adc.arenadata.io/v1alpha1
kind: SparkApplication
metadata:
  name: spark-application
  namespace: spark-applications (1)
spec:
  image: hub.arenadata.io/adc-enterprise/spark3:<tag> (2)

  ## Image pull secret for a private registry.
  ## Set 'externalSecretName' to reference an existing Secret,
  ## or set 'credentials' and optionally 'secretName' to let the CLI create one.
  #imagePullSecret:
  #  # Use a Secret managed outside ADC.
  #  externalSecretName: existing-registry-secret
  #
  #  ## Or let ADC create the Secret.
  #  #secretName: custom-registry-secret
  #
  #  #credentials:
  #  #  registry: registry.example.com
  #  #  username: user
  #  #  password: pass

  hadoop: (3)
    core:
      fs.defaultFS: ""
    hdfs:
      dfs.encrypt.data.transfer.cipher.suites: AES/CTR/NoPadding
    hive:
      hive.metastore.uris: ""

  ## Kerberos configuration for authentication.
  #kerberos:
  #  principal: user@EXAMPLE.COM
  #
  #  # CLI reads the local files and creates the kerberos-ccache Secret on 'adc apply'.
  #  # Alternative - keytab mode: replace this block with:
  #  #   keytab:
  #  #     secretName: <name-of-existing-keytab-secret>
  #  ticketCache:
  #    #secretName: custom-ticket-cache
  #    externalSecretName: existing-ticket-cache
  #    #ticketPath: /tmp/krb5cc_1000
  #    #krb5ConfPath: /etc/krb5.conf

  ## Ranger plugin configuration.
  ## Uncomment and fill the lines below. adc apply derives the rest.
  #ranger:
  #  # fill ranger.plugin.spark.policy.rest.url below with Ranger endpoint, e.g. https://adps-adc.ru-central1.internal:6182
  #  # fill ranger.plugin.spark.service.name below with Ranger service name you want to use for product, e.g. adc_spark_id_1
  #  security:
  #    ranger.plugin.spark.policy.rest.url: ""
  #    ranger.plugin.spark.service.name: ""
  #
  #  # fill xasecure.audit.destination.solr.zookeepers below with Zookeepers endpoints to resolve solr service, e.g. adps-adc.ru-central1.internal:2181/Arenadata.Hadoop-2.solr.server
  #  audit:
  #    xasecure.audit.destination.solr.zookeepers: ""
  #
  #  # Local Ranger files 'adc apply' writes into the configs Secret.
  #  # Relative paths are resolved against the config file.
  #  files:
  #    jceksStorePath: /path/to/ranger.jceks

  ## Java KeyStore/TrustStore certificate configuration.
  ## Set externalSecretName to reference an existing Secret,
  ## or set files and optional secretName to have ADC create it.
  #ssl:
  #  ## Name of the Secret containing Java keystores.
  #  #secretName: custom-ssl-secret
  #  externalSecretName: existing-ssl-secret
  #
  #  # Key in the Secret containing the truststore file.
  #  trustStoreKey: truststore.jks
  #
  #  ## Password for the truststore (optional).
  #  #trustStorePassword: bigdata
  #
  #  ## Key in the Secret containing the keystore file (optional).
  #  #keyStoreKey: keystore.jks
  #
  #  ## Password for the keystore (optional).
  #  #keyStorePassword: bigdata
  #
  #  ## Local files 'adc apply' puts into the Secret named by ssl.secretName.
  #  ## Relative paths are resolved against the config file.
  #  #files:
  #  #  trustStorePath: /path/to/truststore.jks
  #  #  #keyStorePath: /path/to/keystore.jks

  ## Use an external Hadoop configs Secret instead of the one rendered by ADC.
  #hadoopConfigsSecret:
  #  # Use a Secret managed outside ADC.
  #  externalSecretName: existing-spark-hadoop-configs
  #
  #  ## Or let ADC create the Secret.
  #  #secretName: custom-spark-hadoop-configs

  ## Use an external Ranger configs Secret instead of the one rendered by ADC.
  #rangerConfigsSecret:
  #  # Use a Secret managed outside ADC.
  #  externalSecretName: existing-spark-ranger-configs
  #
  #  ## Or let ADC create the Secret.
  #  #secretName: custom-spark-ranger-configs

  # Spark application main resource (e.g. local:///opt/spark/examples/jars/spark-examples.jar).
  mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.13-3.5.4.4-4.3.0-2.jar" (4)

  ## HDFS or local directory for the Spark event log.
  ## When set, the CLI adds spark.eventLog.enabled=true, spark.eventLog.dir,
  ## spark.eventLog.rolling.enabled=true and spark.eventLog.rolling.interval=30s to sparkConf.
  #eventLogDir: ""

  ## Fully-qualified main class name. Required for Java/Scala applications.
  mainClass: "org.apache.spark.examples.SparkPi" (5)

  # ServiceAccount used by the Spark driver, also injected into
  # spark.kubernetes.authenticate.driver.serviceAccountName. The CLI creates it, plus a Role
  # and RoleBinding for Spark pods, by default (create: true). Set create: false to skip that
  # and only reference a ServiceAccount managed elsewhere.
  # The Role rules are managed by the CLI and cannot be customized.
  serviceAccount: (6)
    create: true
    name: spark-application
  job: (7)
    ## true (default) deletes the spark-submit Job pod after it finishes; set false to keep it for debugging.
    deleteOnTermination: false

    #resources:
    #  limits:
    #    cpu: "1"
    #    memory: 512Mi
    #  requests:
    #    cpu: 500m
    #    memory: 64Mi

    ## Component arguments. Key-value pairs passed to the component configuration.
    #args:
    #  executor-memory: 1g
    #  num-executors: "2"

  ## Application arguments appended after mainApplicationFile.
  args:
    - "100"

  # Spark configuration entries (spark.*).
  sparkConf: (8)
    spark.artifactory.dir.path: /tmp/artifacts
    spark.jars.ivy: /tmp/ivy
    spark.local.dir: /tmp/data
    spark.sql.catalog.spark_catalog: org.apache.iceberg.spark.SparkSessionCatalog
    spark.sql.extensions: org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
    spark.sql.security.confblacklist: spark.sql.extensions

  ## Seconds after the application finishes (Succeeded, or Failed with no
  ## retries left) before the SparkApplication is deleted. Omit to keep it
  ## until explicit deletion.
  #ttlSecondsAfterFinished: 3600 (9)

  ## Celeborn remote shuffle service for Spark.
  ## tiers mirror the cluster's storage.tiers: local (SSD/HDD) and MEMORY advertise their name only,
  ## remote (S3/HDFS) also set dir (and, for S3, credentials). Ozone is an HDFS tier with an ofs:// dir.
  #celeborn:
  #  # Storage tiers mirroring the cluster's storage.tiers so the client advertises the same layers.
  #  # type: SSD, HDD, S3, HDFS, MEMORY. SSD/HDD and MEMORY are advertise-only here (no dir; worker-only
  #  # fields ignored); remote S3/HDFS set dir (s3a:// for S3; hdfs:// or ofs:// for Ozone on HDFS). Example:
  #  #   - type: MEMORY   # cache tier; pair with a durable tier below, no dir
  #  #   - type: SSD      # advertise-only on the client, no dir needed
  #  #   - dir: s3a://bucket/celeborn
  #  #     s3: { endpoint: https://s3:9878, region: us-east-1 }
  #  #     type: S3
  #  #   - dir: hdfs://nn/celeborn   # or ofs://om/volume/bucket/celeborn for Ozone
  #  #     type: HDFS
  #  tiers:
  #    - dir: s3a://shuffle/my-cluster
  #      s3:
  #        accessKey: <access-key>
  #        endpoint: https://s3.endpoint:443
  #        pathStyleAccess: true
  #        region: <region>
  #        secretKey: <secret-key>
  #      type: S3
  #  masterEndpoint: ""
  #  extraSparkConf:
  #    spark.sql.adaptive.enabled: "true"
  #
  #  # Enable TLS on the client's RPC connection to the Celeborn cluster.
  #  # When true the CLI renders spark.celeborn.ssl.* into the Spark conf
  #  # and requires the ssl section with trustStoreKey.
  #  rpcEncryption: false
  #
  #  # Enable TLS on the client's data module, which carries shuffle push/fetch
  #  # traffic between executors and workers. Requires the ssl section with trustStoreKey.
  #  dataEncryption: false

  ## YuniKorn scheduler configuration for queue selection and Gang scheduling.
  ## Uncomment the block to route the Spark job into a YuniKorn queue; the CLI renders the
  ## scheduler name, queue labels and gang annotations into sparkConf.
  #yunikorn:
  #  queue: root.analytics
  #  taskGroups:
  #    - minMember: 1
  #      minResource:
  #        cpu: "1"
  #        memory: 1433Mi
  #      name: spark-driver
  #    - minMember: 2
  #      minResource:
  #        cpu: "1"
  #        memory: 1433Mi
  #      name: spark-executor

  ## Monitoring configuration.
  #monitoring:
  #  # Enables Prometheus metrics export from this product's pods.
  #  exportMetrics: true

---
apiVersion: v1
kind: Secret
metadata:
  name: spark-application-configs
  namespace: spark-applications
stringData:
  core-site.xml: |-
    <configuration>
      <property>
        <name>dfs.encrypt.data.transfer.cipher.suites</name>
        <value>AES/CTR/NoPadding</value>
      </property>
      <property>
        <name>fs.defaultFS</name>
        <value></value>
      </property>
    </configuration>
  hive-site.xml: |-
    <configuration>
      <property>
        <name>hive.metastore.uris</name>
        <value></value>
      </property>
    </configuration>
type: Opaque
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: spark-application
  namespace: spark-applications
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: spark-application
  namespace: spark-applications
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - configmaps
  - persistentvolumeclaims
  - services
  - secrets
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
  - deletecollection
- apiGroups:
  - networking.k8s.io
  resources:
  - networkpolicies
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
- apiGroups:
  - events.k8s.io
  resources:
  - events
  verbs:
  - create
  - patch
  - update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: spark-application
  namespace: spark-applications
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: spark-application
subjects:
- kind: ServiceAccount
  name: spark-application
  namespace: spark-applications
---
apiVersion: spark.arenadata.io/v1alpha1
kind: SparkApplication
metadata:
  name: spark-application
  namespace: spark-applications
spec:
  args:
  - "100"
  driver:
    metadata: {}
    spec:
      image: hub.arenadata.io/adc-enterprise/spark3:<tag>
      imagePullPolicy: Always
  executor:
    metadata: {}
    spec:
      image: hub.arenadata.io/adc-enterprise/spark3:<tag>
      imagePullPolicy: Always
  hadoopConfigsSecretName: spark-application-configs
  job:
    deleteOnTermination: false
    metadata: {}
    spec:
      image: hub.arenadata.io/adc-enterprise/spark3:<tag>
      imagePullPolicy: Always
  mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.13-3.5.4.4-4.3.0-2.jar
  mainClass: org.apache.spark.examples.SparkPi
  serviceAccountName: spark-application
  sparkConf:
    spark.artifactory.dir.path: /tmp/artifacts
    spark.jars.ivy: /tmp/ivy
    spark.kubernetes.authenticate.driver.serviceAccountName: spark-application
    spark.kubernetes.namespace: spark-applications
    spark.local.dir: /tmp/data
    spark.sql.catalog.spark_catalog: org.apache.iceberg.spark.SparkSessionCatalog
    spark.sql.extensions: org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
    spark.sql.security.confblacklist: spark.sql.extensions
status: {}