Run Spark jobs on Kubernetes from any host using Helm

# Default values for spark-operator.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
replicas: 1
payloadNamespaces: (1)
  # Managed namespaces for Spark payload resources.
  names:
    - spark-application-min
  # Explicit opt-in for cluster-wide RBAC when payloadNamespaces.names is empty.
  # When false, chart rendering fails until namespaces are specified.
  allowClusterRole: false
  deleteProtection: false
  avoidCreation: false

# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
image:
  registry: "<registry>" (2)
  repository: "<image>" (3)
  # This sets the pull policy for images.
  pullPolicy: Always
  # Overrides the image tag whose default is the chart appVersion.
  tag: "<version>" (4)
  # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
  pullSecret: (5)
    name: ""
    ## List of secrets to create for image pulling in all product namespaces
    credentials: {}
#      registry: private-docker-registry
#      username: user
#      password: pass

# This is to override the chart name.
nameOverride: ""
fullnameOverride: ""

# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
serviceAccount:
  # Automatically mount a ServiceAccount's API credentials?
  automount: true
  # Annotations to add to the service account
  annotations: {}
  # The name of the service account to use.
  # If not set and create is true, a name is generated using the fullname template
  name: ""

# This is for setting Kubernetes Annotations to a Pod.
# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
podAnnotations: {}
# This is for setting Kubernetes Labels to a Pod.
# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
podLabels: {}

podSecurityContext: {}
  # fsGroup: 2000

securityContext:
  readOnlyRootFilesystem: true
  privileged: false
  allowPrivilegeEscalation: false
  runAsNonRoot: true
  runAsUser: 65532
  capabilities:
    drop:
      - ALL
  seccompProfile:
    type: RuntimeDefault

# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/
service:
  # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types
  type: ClusterIP
  # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports
  port: 8443

resources: {}
  # We usually recommend not to specify default resources and to leave this as a conscious
  # choice for the user. This also increases chances charts run on environments with little
  # resources, such as Minikube. If you do want to specify resources, uncomment the following
  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
  # limits:
  #   cpu: 100m
  #   memory: 128Mi
  # requests:
  #   cpu: 100m
  #   memory: 128Mi

nodeSelector: {}

tolerations: []

affinity: {}

terminationGracePeriodSeconds: 10

metrics:
  # Enable to protect the metrics endpoint with authn/authz. Requires ClusterRole.
  # See https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.23.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
  auth: false

image:
  registry: "<registry>" (1)
  repository: "<image>" (2)
  tag: "<tag>"
  ## Specify a pullPolicy
  ## ref: https://kubernetes.io/docs/concepts/containers/images/#pre-pulled-images
  ##
  pullPolicy: "Always"
  ## Existing secret or secret to create to use for image pulling, they must exist in all product namespaces
  ##
  pullSecret: (3)
    name: ""
    credentials: {}
#      registry: private-docker-registry
#      username: user
#      password: pass

#ServiceAccount name for Spark driver/executor pods
serviceAccountName: "spark-app"

#Application
#mainApplicationFile: path to the main app file (hdfs://, local://, etc.)
mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.13-3.5.4.4-4.3.0-2.jar" (4)

#Restart policy on failure: Never or OnFailure
#When set to OnFailure, maxRetries must be specified
restartPolicy: ""

#Maximum number of Job restart attempts on failure
#Required when restartPolicy is OnFailure
#maxRetries:

#mainClass: set only for JVM apps (e.g. SparkConnectServer)
mainClass: "org.apache.spark.examples.SparkPi"


#Hadoop configurations
hadoopConfigsSecretName: ""

#Spark configurations. When set, this client-managed Secret is the complete
#Spark configuration source. If Ranger is enabled, include the ranger-spark-*.xml
#files in this Secret; the chart does not create a separate Ranger config Secret.
sparkConfigsSecretName: ""

#Kerberos (keytab mode), disabled by default.
#The referenced Secret must already exist and carry two keys: keytab + krb5.conf.
#Ticket-cache mode is CLI-only and is not exposed here.
#Extra confs (e.g. spark.kerberos.access.hadoopFileSystems) go under sparkConf.
kerberos: {}
#  principal: user@RU-CENTRAL1.INTERNAL
#  keytab:
#    secretName: spark-keytab

# Arguments passed to the main application class after the main file
args:
  - "10"

#Spark configuration
#Key/value map rendered into spec.sparkConf
sparkConf: {}

#SSL / truststore settings. The referenced Secret must already exist;
#the operator mounts the stores at /etc/ssl.
#Enabled implicitly when secretName is set.
ssl: {}
#  secretName: "ca-store"
#  trustStoreKey: "truststore.jks"
#  keyStoreKey: "keystore.jks"

#Ranger authorization (Kyuubi Spark Authz plugin), disabled by default.
#When enabled without sparkConfigsSecretName, the chart renders the
#ranger-spark-*.xml config Secret. The spark-apps.sparkConf helper always appends
#the Ranger extension to spark.sql.extensions.
#The audit JAAS block is written only when the kerberos block is set (keytab mode);
#the policymgr-ssl truststore is rendered only when ssl.enabled.
ranger:
  enabled: false
  # ranger.plugin.spark.policy.rest.url
  policyRestURL: ""
  # ranger.plugin.spark.service.name
  serviceName: ""
  # xasecure.audit.destination.solr.zookeepers
  solrZookeepers: ""

job:
  ## @param replicas set number of job replicas
  ##
  replicas: 1

  ## When false, the spark-submit Job pod is not automatically deleted after completion (useful for log inspection)
  ##
  #deleteOnTermination: false

  ## Additional configuration that you want to be added to job-config
  args: {}
  #  "task.max-worker-threads": 8

  ## Annotations for job pods
  annotations: {}

  ## Set container requests and limits for resource like CPU or memory (essential for production workloads)
  ##
  resources: {}
    #limits:
    #  cpu: "2"
    #  memory: "8Gi"
    #requests:
    #  cpu: "2"
    #  memory: "8Gi"

  ## Request additional PVC for pod
  ##
  persistentVolume: {}
  #  mountPath: "/data/spark"
  #  volumeClaimTemplates:
  #    - metadata:
  #        name: data
  #      spec:
  #        accessModes: ["ReadWriteOnce"]
  #        resources:
  #          requests:
  #            storage: 10Gi
  #        storageClassName: default

  ## nodeAffinity: Object defining constraints to place pods on a specific set of Nodes
  ##
  nodeSelector: {}

  topologySpreadConstraints: []

  ## Allow a Pod to be scheduled onto nodes that have taints.
  ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
  ##
  tolerations: []
   # - key: "example-key"
   #   operator: "Exists"
   #   effect: "NoSchedule"

  ## affinity: Object defining soft rules to place pods on a specific set of Nodes or didn't place pod to nodes due to some conditions
  ##
  affinity: {}
  #  nodeAffinity:
  #      requiredDuringSchedulingIgnoredDuringExecution:
  #        nodeSelectorTerms:
  #        - matchExpressions:
  #          - key: topology.kubernetes.io/zone
  #            operator: In
  #            values:
  #            - antarctica-east1
  #            - antarctica-west1
  #      preferredDuringSchedulingIgnoredDuringExecution:
  #      - weight: 1
  #        preference:
  #          matchExpressions:
  #          - key: another-node-label-key
  #            operator: In
  #            values:
  #            - another-node-label-value

  startupProbe: {}
  #  type: httpGet
  #  port: 8080
  #  path: /v1/info
  #  scheme: HTTP
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  livenessProbe: {}
  #  type: httpGet
  #  port: 8080
  #  path: /v1/info
  #  scheme: HTTP
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  readinessProbe: {}
  #  type: exec
  #  command: test -f /opt/spark/etc/truststore/custom-truststore.jks
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  ## Mount additional secrets into the pod
  ##
  mountSecrets: []
  #  - secretName: my-secret
  #    mountPath: /etc/spark/my-secret

driver:
  ## @param replicas set number of driver replicas
  ##
  replicas: 1

  ## Additional configuration that you want to be added to driver-config
  args: {}
  #  "task.max-worker-threads": 8

  ## Annotations for driver pods
  annotations: {}

  ## Set container requests and limits for resource like CPU or memory (essential for production workloads)
  ##
  resources: {}
    #limits:
    #  cpu: "2"
    #  memory: "8Gi"
    #requests:
    #  cpu: "2"
    #  memory: "8Gi"

  ## Request additional PVC for pod
  ##
  persistentVolume: {}
  #  mountPath: "/data/spark"
  #  volumeClaimTemplates:
  #    - metadata:
  #        name: data
  #      spec:
  #        accessModes: ["ReadWriteOnce"]
  #        resources:
  #          requests:
  #            storage: 10Gi
  #        storageClassName: default

  ## nodeAffinity: Object defining constraints to place pods on a specific set of Nodes
  ##
  nodeSelector: {}

  topologySpreadConstraints: []

  ## Allow a Pod to be scheduled onto nodes that have taints.
  ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
  ##
  tolerations: []
   # - key: "example-key"
   #   operator: "Exists"
   #   effect: "NoSchedule"

  ## affinity: Object defining soft rules to place pods on a specific set of Nodes or didn't place pod to nodes due to some conditions
  ##
  affinity: {}
  #  nodeAffinity:
  #      requiredDuringSchedulingIgnoredDuringExecution:
  #        nodeSelectorTerms:
  #        - matchExpressions:
  #          - key: topology.kubernetes.io/zone
  #            operator: In
  #            values:
  #            - antarctica-east1
  #            - antarctica-west1
  #      preferredDuringSchedulingIgnoredDuringExecution:
  #      - weight: 1
  #        preference:
  #          matchExpressions:
  #          - key: another-node-label-key
  #            operator: In
  #            values:
  #            - another-node-label-value

  startupProbe: {}
  #  type: httpGet
  #  port: 8080
  #  path: /v1/info
  #  scheme: HTTP
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  livenessProbe: {}
  #  type: httpGet
  #  port: 8080
  #  path: /v1/info
  #  scheme: HTTP
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  readinessProbe: {}
  #  type: exec
  #  command: test -f /opt/spark/etc/truststore/custom-truststore.jks
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  ## Mount additional secrets into the pod
  ##
  mountSecrets: []
  #  - secretName: my-secret
  #    mountPath: /etc/spark/my-secret

executor:
  ## @param replicas set number of executor replicas
  ##
  replicas: 1

  ## Additional configuration that you want to be added to executor-config
  args: {}
  #  "task.max-worker-threads": 8

  ## Annotations for executor pods
  annotations: {}

  ## Set container requests and limits for resource like CPU or memory (essential for production workloads)
  ##
  resources: {}
    #limits:
    #  cpu: "2"
    #  memory: "8Gi"
    #requests:
    #  cpu: "2"
    #  memory: "8Gi"

  ## Request additional PVC for pod
  ##
  persistentVolume: {}
  #  mountPath: "/data/spark"
  #  volumeClaimTemplates:
  #    - metadata:
  #        name: data
  #      spec:
  #        accessModes: ["ReadWriteOnce"]
  #        resources:
  #          requests:
  #            storage: 10Gi
  #        storageClassName: default

  ## nodeAffinity: Object defining constraints to place pods on a specific set of Nodes
  ##
  nodeSelector: {}

  topologySpreadConstraints: []

  ## Allow a Pod to be scheduled onto nodes that have taints.
  ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
  ##
  tolerations: []
   # - key: "example-key"
   #   operator: "Exists"
   #   effect: "NoSchedule"

  ## affinity: Object defining soft rules to place pods on a specific set of Nodes or didn't place pod to nodes due to some conditions
  ##
  affinity: {}
  #  nodeAffinity:
  #      requiredDuringSchedulingIgnoredDuringExecution:
  #        nodeSelectorTerms:
  #        - matchExpressions:
  #          - key: topology.kubernetes.io/zone
  #            operator: In
  #            values:
  #            - antarctica-east1
  #            - antarctica-west1
  #      preferredDuringSchedulingIgnoredDuringExecution:
  #      - weight: 1
  #        preference:
  #          matchExpressions:
  #          - key: another-node-label-key
  #            operator: In
  #            values:
  #            - another-node-label-value

  startupProbe: {}
  #  type: httpGet
  #  port: 8080
  #  path: /v1/info
  #  scheme: HTTP
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  livenessProbe: {}
  #  type: httpGet
  #  port: 8080
  #  path: /v1/info
  #  scheme: HTTP
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  readinessProbe: {}
  #  type: exec
  #  command: test -f /opt/spark/etc/truststore/custom-truststore.jks
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  ## Mount additional secrets into the pod
  ##
  mountSecrets: []
  #  - secretName: my-secret
  #    mountPath: /etc/spark/my-secret


#Optional: inline Secret for properties-file pattern
propertiesFile:
  enabled: false
  # Raw content rendered into the Secret's stringData
  content: ""

#RBAC
rbac:
  # Set to false if the ServiceAccount/Role/RoleBinding already exist
  create: true
  rules:
    - apiGroups:
        - ""
      resources:
        - pods
        - configmaps
        - persistentvolumeclaims
        - services
        - secrets
      verbs:
        - get
        - list
        - watch
        - create
        - update
        - patch
        - delete
        - deletecollection
    - apiGroups:
        - networking.k8s.io
      verbs:
        - get
        - list
        - watch
        - create
        - update
        - patch
        - delete
      resources:
        - networkpolicies

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: spark-minimal
  namespace: spark-kbctl
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: spark-minimal-role
  namespace: spark-kbctl
rules:
  - apiGroups:
      - ""
    resources:
      - pods
      - configmaps
      - persistentvolumeclaims
      - services
      - secrets
    verbs:
      - get
      - list
      - watch
      - create
      - update
      - patch
      - delete
      - deletecollection
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: spark-minimal-role-binding
  namespace: spark-kbctl
subjects:
  - kind: ServiceAccount
    name: spark-minimal
    namespace: spark-kbctl
roleRef:
  kind: Role
  name: spark-minimal-role
  apiGroup: rbac.authorization.k8s.io

apiVersion: spark.arenadata.io/v1alpha1
kind: SparkApplication
metadata:
  labels:
    app.kubernetes.io/name: spark-operator
    app.kubernetes.io/managed-by: kustomize
  name: spark-pi-java
  namespace: spark-kbctl
spec:
  mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.13-3.5.4.4-4.3.0-2.jar"
  mainClass: "org.apache.spark.examples.SparkPi"
  args:
    - "1000"
  serviceAccountName: spark-minimal
  sparkConf:
    "spark.kubernetes.authenticate.driver.serviceAccountName": "spark-minimal"
    "spark.kubernetes.namespace": "spark-kbctl"
  job:
    spec:
      image: &image hub.adsw.io/adh-enterprise/spark3-docker:3.5.4.4-adh-4.3.0-x86_64
  driver:
    spec:
      image: *image
  executor:
    replicas: 2
    spec:
      image: *image