sparkHistoryOpts: "-Dspark.history.fs.logDirectory=s3a://${s3_bucket_name}/${s3_bucket_prefix}"

# Update spark conf according to your needs
sparkConf: |-
  spark.hadoop.fs.s3a.aws.credentials.provider=com.amazonaws.auth.WebIdentityTokenCredentialsProvider
  spark.history.fs.eventLog.rolling.maxFilesToRetain=5
  spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem
  spark.eventLog.enabled=true
  spark.history.ui.port=18080

resources:
  limits:
    cpu: 200m
    memory: 2G
  requests:
    cpu: 100m
    memory: 1G