apiVersion: karpenter.sh/v1alpha5
kind: Provisioner
metadata:
  name: notebook-executor-{{az}}
spec:
  # References cloud provider-specific custom resource, see your cloud provider specific documentation
  providerRef:
    name: notebook-executor-nodes-{{az}}

  # Labels are arbitrary key-values that are applied to all nodes
  labels:
    role: notebook
    node-lifecycle: spot
    spark-role: executor



  # Requirements that constrain the parameters of provisioned nodes.
  # These requirements are combined with pod.spec.affinity.nodeAffinity rules.
  # Operators { In, NotIn } are supported to enable including or excluding values
  requirements:
    # Include general purpose instance families
    - key: "karpenter.k8s.aws/instance-family"
      operator: In
      values: ["m5", "m6i","r5","r6i","m5a","r5a"]
    - key: "kubernetes.io/arch"
      operator: In
      values: ["amd64"]
    # Exclude smaller instance sizes
    - key: "karpenter.k8s.aws/instance-size"
      operator: In
      values: [2xlarge,4xlarge,8xlarge]
    - key: "karpenter.sh/capacity-type" # If not included, the webhook for the AWS cloud provider will default to on-demand
      operator: In
      values: ["spot"]
    - key: "topology.kubernetes.io/zone"
      operator: In
      values: ["{{az}}"]
    - key: "karpenter.k8s.aws/instance-hypervisor"
      operator: In
      values: ["nitro"]

  
  #kubeletConfiguration:
  #  kubeReserved:
  #    ephemeral-storage: 100Gi

  # Resource limits constrain the total size of the cluster.
  # Limits prevent Karpenter from creating new instances once the limit is exceeded.
  limits:
    resources:
      cpu: "800"
      memory: 3200Gi

  # If omitted, the feature is disabled and nodes will never expire.  If set to less time than it requires for a node
  # to become ready, the node may expire before any pods successfully start.
  ttlSecondsUntilExpired: 2592000 # 30 Days = 60 * 60 * 24 * 30 Seconds;

  # If omitted, the feature is disabled, nodes will never scale down due to low utilization
  ttlSecondsAfterEmpty: 300

  # Priority given to the provisioner when the scheduler considers which provisioner
  # to select. Higher weights indicate higher priority when comparing provisioners.
  # Specifying no weight is equivalent to specifying a weight of 0.
  weight: 20
  
---
apiVersion: karpenter.k8s.aws/v1alpha1
kind: AWSNodeTemplate
metadata:
  name: notebook-executor-nodes-{{az}}
spec:
  amiFamily: AL2
  subnetSelector:
    aws-ids: {{subnet-id}}
  securityGroupSelector:
    kubernetes.io/cluster/{{cluster-name}}: owned
  tags:
    KarpenerProvisionerName: "notebook-executor"
  blockDeviceMappings:
    - deviceName: /dev/xvda
      ebs:
        volumeSize: 100Gi
        volumeType: gp2
  userData: |
    MIME-Version: 1.0
    Content-Type: multipart/mixed; boundary="BOUNDARY"

    --BOUNDARY
    Content-Type: text/x-shellscript; charset="us-ascii"

    #!/bin/bash
    yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm,
    systemctl enable amazon-ssm-agent,
    systemctl start amazon-ssm-agent,

    --BOUNDARY--