Настройка Kerberos и SSL для Spark в Kubernetes с помощью Helm

# Default values for spark-operator.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
replicas: 1
payloadNamespaces: (1)
  # Managed namespaces for Spark payload resources.
  names:
    - spark-helm
  # Explicit opt-in for cluster-wide RBAC when payloadNamespaces.names is empty.
  # When false, chart rendering fails until namespaces are specified.
  allowClusterRole: false
  deleteProtection: false
  avoidCreation: false

# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
image:
  registry: "<registry>" (2)
  repository: "<image>" (3)
  # This sets the pull policy for images.
  pullPolicy: Always
  # Overrides the image tag whose default is the chart appVersion.
  tag: "<tag>" (4)
  # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
  pullSecret:
    name: ""
    ## List of secrets to create for image pulling in all product namespaces
    credentials: {} (5)
#      registry: private-docker-registry
#      username: user
#      password: pass

# This is to override the chart name.
nameOverride: ""
fullnameOverride: ""

# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
serviceAccount:
  # Automatically mount a ServiceAccount's API credentials?
  automount: true
  # Annotations to add to the service account
  annotations: {}
  # The name of the service account to use.
  # If not set and create is true, a name is generated using the fullname template
  name: ""

# This is for setting Kubernetes Annotations to a Pod.
# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
podAnnotations: {}
# This is for setting Kubernetes Labels to a Pod.
# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
podLabels: {}

podSecurityContext: {}
  # fsGroup: 2000

securityContext:
  readOnlyRootFilesystem: true
  privileged: false
  allowPrivilegeEscalation: false
  runAsNonRoot: true
  runAsUser: 65532
  capabilities:
    drop:
      - ALL
  seccompProfile:
    type: RuntimeDefault

# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/
service:
  # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types
  type: ClusterIP
  # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports
  port: 8443

resources: {}
  # We usually recommend not to specify default resources and to leave this as a conscious
  # choice for the user. This also increases chances charts run on environments with little
  # resources, such as Minikube. If you do want to specify resources, uncomment the following
  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
  # limits:
  #   cpu: 100m
  #   memory: 128Mi
  # requests:
  #   cpu: 100m
  #   memory: 128Mi

nodeSelector: {}

tolerations: []

affinity: {}

terminationGracePeriodSeconds: 10

metrics:
  # Enable to protect the metrics endpoint with authn/authz. Requires ClusterRole.
  # See https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.23.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
  auth: false

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: spark-helm
  namespace: spark-helm
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: spark-helm
  namespace: spark-helm
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - configmaps
  - persistentvolumeclaims
  - services
  - secrets
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
  - deletecollection
- apiGroups:
  - networking.k8s.io
  resources:
  - networkpolicies
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
- apiGroups:
  - events.k8s.io
  resources:
  - events
  verbs:
  - create
  - patch
  - update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: spark-helm
  namespace: spark-helm
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: spark-helm
subjects:
- kind: ServiceAccount
  name: spark-helm
  namespace: spark-helm

<configuration>
  <property>
    <name>dfs.client.failover.proxy.provider.adh</name>
    <value>org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider</value>
  </property>
  <property>
    <name>dfs.ha.namenodes.adh</name>
    <value>nn_ka-adh-1,nn_ka-adh-2</value>
  </property>
  <property>
    <name>dfs.namenode.rpc-address.adh.nn_ka-adh-1</name>
    <value>ka-adh-1.ru-central1.internal:8020</value>
  </property>
  <property>
    <name>dfs.namenode.rpc-address.adh.nn_ka-adh-2</name>
    <value>ka-adh-2.ru-central1.internal:8020</value>
  </property>
  <property>
    <name>dfs.nameservices</name>
    <value>adh</value>
  </property>
  <property>
    <name>fs.defaultFS</name>
    <value>ofs://adho</value>
  </property>
  <property>
    <name>hadoop.security.authentication</name>
    <value>kerberos</value>
  </property>
  <property>
    <name>ozone.om.address.adho.om_ka-adh-1</name>
    <value>ka-adh-1.ru-central1.internal:9862</value>
  </property>
  <property>
    <name>ozone.om.address.adho.om_ka-adh-2</name>
    <value>ka-adh-2.ru-central1.internal:9862</value>
  </property>
  <property>
    <name>ozone.om.address.adho.om_ka-adh-3</name>
    <value>ka-adh-3.ru-central1.internal:9862</value>
  </property>
  <property>
    <name>ozone.om.kerberos.principal</name>
    <value>om/_HOST@RU-CENTRAL1.INTERNAL</value>
  </property>
  <property>
    <name>ozone.om.nodes.demo</name>
    <value>om_ka-adh-1,om_ka-adh-2,om_ka-adh-3</value>
  </property>
  <property>
    <name>ozone.om.service.ids</name>
    <value>adho</value>
  </property>
  <property>
    <name>ozone.security.enabled</name>
    <value>true</value>
  </property>
</configuration>

<configuration>
  <property>
    <name>hive.metastore.kerberos.principal</name>
    <value>hive/_HOST@RU-CENTRAL1.INTERNAL</value>
  </property>
  <property>
    <name>hive.metastore.sasl.enabled</name>
    <value>true</value>
  </property>
  <property>
    <name>hive.metastore.uris</name>
    <value>thrift://ka-adh-2.ru-central1.internal:9083</value>
  </property>
  <property>
    <name>hive.metastore.warehouse.dir</name>
    <value>ofs://adho/apps/hive/warehouse</value>
  </property>
  <property>
    <name>metastore.truststore.password</name>
    <value>bigdata</value>
  </property>
  <property>
    <name>metastore.truststore.path</name>
    <value>/etc/ssl/truststore.jks</value>
  </property>
  <property>
    <name>metastore.use.SSL</name>
    <value>true</value>
  </property>
</configuration>

image:
  registry: "<registry>" (1)
  repository: "<image>" (2)
  tag: "<tag>" (3)
  ## Specify a pullPolicy
  ## ref: https://kubernetes.io/docs/concepts/containers/images/#pre-pulled-images
  ##
  pullPolicy: "Always"
  ## Existing secret or secret to create to use for image pulling, they must exist in all product namespaces
  ##
  pullSecret:
    name: ""
    credentials: {}
#      registry: private-docker-registry
#      username: user
#      password: pass

#ServiceAccount name for Spark driver/executor pods
serviceAccountName: "spark-helm"

#Application
#mainApplicationFile: path to the main app file (hdfs://, local://, etc.)
mainApplicationFile: "ofs://demo/ozone/demo/word_count/demo-load.py" (4)

#Restart policy on failure: Never or OnFailure
#When set to OnFailure, maxRetries must be specified
restartPolicy: ""

#Maximum number of Job restart attempts on failure
#Required when restartPolicy is OnFailure
#maxRetries:

#Seconds to keep a finished (Succeeded or Failed) SparkApplication CR around
#before it is automatically deleted. A Failed application with restart
#attempts remaining does not count as finished. Omit to keep finished CRs
#until explicit deletion; 0 deletes as soon as the terminal status is recorded.
#ttlSecondsAfterFinished: 3600

#mainClass: set only for JVM apps (e.g. SparkConnectServer)
mainClass: ""


#Hadoop configurations
hadoopConfigsSecretName: "hadoop-conf" (5)

#Spark configurations. When set, this client-managed Secret is the complete
#Spark configuration source. If Ranger is enabled, include the ranger-spark-*.xml
#files in this Secret; the chart does not create a separate Ranger config Secret.
sparkConfigsSecretName: ""

#Kerberos (keytab mode), disabled by default.
#The referenced Secret must already exist and carry two keys: keytab + krb5.conf.
#Ticket-cache mode is CLI-only and is not exposed here.
#Extra confs (e.g. spark.kerberos.access.hadoopFileSystems) go under sparkConf.
kerberos: (6)
  principal: spark/spark-ozone.ru-central1.internal
  keytab:
    secretName: keytab-secret

# Arguments passed to the main application class after the main file
args: []
#  - "1000"

#Spark configuration
#Key/value map rendered into spec.sparkConf
sparkConf: (7)
  spark.artifactory.dir.path: /tmp/artifacts
  spark.jars.ivy: /tmp/ivy
  spark.local.dir: /tmp/data
  spark.sql.catalog.spark_catalog: org.apache.iceberg.spark.SparkSessionCatalog
  spark.sql.extensions: org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
  spark.sql.security.confblacklist: spark.sql.extensions
  spark.driver.extraJavaOptions: -Djavax.net.ssl.trustStore=/etc/ssl/truststore.jks -Djavax.net.ssl.trustStorePassword=bigdata (8)

#SSL / truststore settings. The referenced Secret must already exist;
#the operator mounts the stores at /etc/ssl.
#Enabled implicitly when secretName is set.
ssl: (9)
  secretName: "custom-ssl-secret"
  trustStoreKey: "truststore.jks"
#  keyStoreKey: "keystore.jks"

#Ranger authorization (Kyuubi Spark Authz plugin), disabled by default.
#When enabled without sparkConfigsSecretName, the chart renders the
#ranger-spark-*.xml config Secret. The spark-apps.sparkConf helper always appends
#the Ranger extension to spark.sql.extensions.
#The audit JAAS block is written only when the kerberos block is set (keytab mode);
#the policymgr-ssl truststore is rendered only when ssl.enabled.
ranger:
  enabled: false
  # ranger.plugin.spark.policy.rest.url
  policyRestURL: ""
  # ranger.plugin.spark.service.name
  serviceName: ""
  # xasecure.audit.destination.solr.zookeepers
  solrZookeepers: ""

job:
  ## @param replicas set number of job replicas
  ##
  replicas: 1

  ## When false, the spark-submit Job pod is not automatically deleted after completion (useful for log inspection)
  ##
  #deleteOnTermination: false

  ## Additional configuration that you want to be added to job-config
  args: {}
  #  task.max-worker-threads: 8

  ## Annotations for job pods
  annotations: {}

  ## Set container requests and limits for resource like CPU or memory (essential for production workloads)
  ##
  resources: {}
    #limits:
    #  cpu: "2"
    #  memory: "8Gi"
    #requests:
    #  cpu: "2"
    #  memory: "8Gi"

  ## Request additional PVC for pod
  ##
  persistentVolume: {}
  #  mountPath: "/data/spark"
  #  volumeClaimTemplates:
  #    - metadata:
  #        name: data
  #      spec:
  #        accessModes: ["ReadWriteOnce"]
  #        resources:
  #          requests:
  #            storage: 10Gi
  #        storageClassName: default

  ## nodeAffinity: Object defining constraints to place pods on a specific set of Nodes
  ##
  nodeSelector: {}

  topologySpreadConstraints: []

  ## Allow a Pod to be scheduled onto nodes that have taints.
  ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
  ##
  tolerations: []
   # - key: "example-key"
   #   operator: "Exists"
   #   effect: "NoSchedule"

  ## affinity: Object defining soft rules to place pods on a specific set of Nodes or didn't place pod to nodes due to some conditions
  ##
  affinity: {}
  #  nodeAffinity:
  #      requiredDuringSchedulingIgnoredDuringExecution:
  #        nodeSelectorTerms:
  #        - matchExpressions:
  #          - key: topology.kubernetes.io/zone
  #            operator: In
  #            values:
  #            - antarctica-east1
  #            - antarctica-west1
  #      preferredDuringSchedulingIgnoredDuringExecution:
  #      - weight: 1
  #        preference:
  #          matchExpressions:
  #          - key: another-node-label-key
  #            operator: In
  #            values:
  #            - another-node-label-value

  startupProbe: {}
  #  type: httpGet
  #  port: 8080
  #  path: /v1/info
  #  scheme: HTTP
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  livenessProbe: {}
  #  type: httpGet
  #  port: 8080
  #  path: /v1/info
  #  scheme: HTTP
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  readinessProbe: {}
  #  type: exec
  #  command: test -f /opt/spark/etc/truststore/custom-truststore.jks
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  ## Mount additional secrets into the pod
  ##
  mountSecrets: []
  #  - secretName: my-secret
  #    mountPath: /etc/spark/my-secret

driver:
  ## @param replicas set number of driver replicas
  ##
  replicas: 1

  ## Additional configuration that you want to be added to driver-config
  args: {}
  #  task.max-worker-threads: 8

  ## Annotations for driver pods
  annotations: {}

  ## Set container requests and limits for resource like CPU or memory (essential for production workloads)
  ##
  resources: {}
    #limits:
    #  cpu: "2"
    #  memory: "8Gi"
    #requests:
    #  cpu: "2"
    #  memory: "8Gi"

  ## Request additional PVC for pod
  ##
  persistentVolume: {}
  #  mountPath: "/data/spark"
  #  volumeClaimTemplates:
  #    - metadata:
  #        name: data
  #      spec:
  #        accessModes: ["ReadWriteOnce"]
  #        resources:
  #          requests:
  #            storage: 10Gi
  #        storageClassName: default

  ## nodeAffinity: Object defining constraints to place pods on a specific set of Nodes
  ##
  nodeSelector: {}

  topologySpreadConstraints: []

  ## Allow a Pod to be scheduled onto nodes that have taints.
  ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
  ##
  tolerations: []
   # - key: "example-key"
   #   operator: "Exists"
   #   effect: "NoSchedule"

  ## affinity: Object defining soft rules to place pods on a specific set of Nodes or didn't place pod to nodes due to some conditions
  ##
  affinity: {}
  #  nodeAffinity:
  #      requiredDuringSchedulingIgnoredDuringExecution:
  #        nodeSelectorTerms:
  #        - matchExpressions:
  #          - key: topology.kubernetes.io/zone
  #            operator: In
  #            values:
  #            - antarctica-east1
  #            - antarctica-west1
  #      preferredDuringSchedulingIgnoredDuringExecution:
  #      - weight: 1
  #        preference:
  #          matchExpressions:
  #          - key: another-node-label-key
  #            operator: In
  #            values:
  #            - another-node-label-value

  startupProbe: {}
  #  type: httpGet
  #  port: 8080
  #  path: /v1/info
  #  scheme: HTTP
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  livenessProbe: {}
  #  type: httpGet
  #  port: 8080
  #  path: /v1/info
  #  scheme: HTTP
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  readinessProbe: {}
  #  type: exec
  #  command: test -f /opt/spark/etc/truststore/custom-truststore.jks
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  ## Mount additional secrets into the pod
  ##
  mountSecrets: []
  #  - secretName: my-secret
  #    mountPath: /etc/spark/my-secret

executor:
  ## @param replicas set number of executor replicas
  ##
  replicas: 1

  ## Additional configuration that you want to be added to executor-config
  args: {}
  #  task.max-worker-threads: 8

  ## Annotations for executor pods
  annotations: {}

  ## Set container requests and limits for resource like CPU or memory (essential for production workloads)
  ##
  resources: {}
    #limits:
    #  cpu: "2"
    #  memory: "8Gi"
    #requests:
    #  cpu: "2"
    #  memory: "8Gi"

  ## Request additional PVC for pod
  ##
  persistentVolume: {}
  #  mountPath: "/data/spark"
  #  volumeClaimTemplates:
  #    - metadata:
  #        name: data
  #      spec:
  #        accessModes: ["ReadWriteOnce"]
  #        resources:
  #          requests:
  #            storage: 10Gi
  #        storageClassName: default

  ## nodeAffinity: Object defining constraints to place pods on a specific set of Nodes
  ##
  nodeSelector: {}

  topologySpreadConstraints: []

  ## Allow a Pod to be scheduled onto nodes that have taints.
  ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
  ##
  tolerations: []
   # - key: "example-key"
   #   operator: "Exists"
   #   effect: "NoSchedule"

  ## affinity: Object defining soft rules to place pods on a specific set of Nodes or didn't place pod to nodes due to some conditions
  ##
  affinity: {}
  #  nodeAffinity:
  #      requiredDuringSchedulingIgnoredDuringExecution:
  #        nodeSelectorTerms:
  #        - matchExpressions:
  #          - key: topology.kubernetes.io/zone
  #            operator: In
  #            values:
  #            - antarctica-east1
  #            - antarctica-west1
  #      preferredDuringSchedulingIgnoredDuringExecution:
  #      - weight: 1
  #        preference:
  #          matchExpressions:
  #          - key: another-node-label-key
  #            operator: In
  #            values:
  #            - another-node-label-value

  startupProbe: {}
  #  type: httpGet
  #  port: 8080
  #  path: /v1/info
  #  scheme: HTTP
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  livenessProbe: {}
  #  type: httpGet
  #  port: 8080
  #  path: /v1/info
  #  scheme: HTTP
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  readinessProbe: {}
  #  type: exec
  #  command: test -f /opt/spark/etc/truststore/custom-truststore.jks
  #  initialDelaySeconds: 20
  #  periodSeconds: 5
  #  timeoutSeconds: 3
  #  successThreshold: 1
  #  failureThreshold: 2

  ## Mount additional secrets into the pod
  ##
  mountSecrets: []
  #  - secretName: my-secret
  #    mountPath: /etc/spark/my-secret


#Optional: inline Secret for properties-file pattern
propertiesFile:
  enabled: false
  # Raw content rendered into the Secret's stringData
  content: ""

#RBAC
rbac:
  # Set to false if the ServiceAccount/Role/RoleBinding already exist
  create: false
  rules:
    - apiGroups:
        - ""
      resources:
        - pods
        - configmaps
        - persistentvolumeclaims
        - services
        - secrets
      verbs:
        - get
        - list
        - watch
        - create
        - update
        - patch
        - delete
        - deletecollection
    - apiGroups:
        - networking.k8s.io
      verbs:
        - get
        - list
        - watch
        - create
        - update
        - patch
        - delete
      resources:
        - networkpolicies