##### README #####
# If (nodes*pods_per_node*client_qps % (pods_per_second*namespaces) != 0), test will fail.
# This is a clusterloader2 restriction, so ensure your parameters meet this condition.
##### Constants; these shouldn't typically be configured #####
{{ $NAMESPACES := DefaultParam .NAMESPACES 10}}
# CLIENT_QPS from the clusterloader2 client to the Kubernetes API Server
# Keep this value reasonably low to avoid the case where your test is bottlenecked on client throughput.
{{ $CLIENT_QPS := DefaultParam .CLIENT_QPS 1 }}

# Pods per second should not exceed Kubernetes Control Plane limits.
{{ $PODS_PER_SECOND := DefaultParam .PODS_PER_SECOND 20 }}

# NODE_CPU is the amount of schedulable cpu on the nodes
# kubectl get node -o json | jq -r ".items[].status.allocatable.cpu"
{{ $NODE_CPU := DefaultParam .NODE_CPU 1 }}

# NODE_MEMORY_MB is the amount of schedulable memory on the nodes, represented in MB.
# kubectl get node -o json | jq -r ".items[].status.allocatable.memory"
{{ $NODE_MEMORY_MB := DefaultParam .NODE_MEMORY_MB 3840 }}

# PODS_PER_DEPLOYMENT is the number of pods in each deployment
# This value is used to drive PODS_PER_SECOND
{{ $PODS_PER_DEPLOYMENT := DivideFloat $PODS_PER_SECOND $CLIENT_QPS }}


{{ $POD_GROUP_LABEL := DefaultParam .POD_GROUP_LABEL "cluster-loader" }}

##### Test Parameters, these variables are used to drive our test cases #####

# NODES is the total number of nodes for the test. It is passed in by clusterloader2 command line arguments.
{{ $NODES := .Nodes }}

# PODS_PER_NODE determines the average pod density for nodes
{{ $PODS_PER_NODE := DefaultParam .PODS_PER_NODE 10 }}

##### Computed, these are not configurable and are only used in our template #####
# TOTAL_PODS to be created in the test.
{{ $TOTAL_PODS := MultiplyInt $NODES $PODS_PER_NODE }}

{{ $DEPLOYMENTS := DivideFloat $TOTAL_PODS $PODS_PER_DEPLOYMENT }}
{{ $DEPLOYMENTS_PER_NAMESPACE := DivideFloat $DEPLOYMENTS $NAMESPACES }}

# Pod Resources ensure that the scheduler correctly schedules PODS_PER_NODE.
{{ $POD_CPU_MILLIS := DivideFloat (MultiplyFloat $NODE_CPU .9) $PODS_PER_NODE }}
{{ $POD_MEMORY_MB := DivideFloat (MultiplyFloat $NODE_MEMORY_MB .9) $PODS_PER_NODE }}

name: simple-bring-up
namespace:
  number: {{ $NAMESPACES }}
tuningSets:
- name: uniform-qps
  qpsLoad:
    qps: {{ $CLIENT_QPS }}

steps:
- name: start-measurements
  measurements:
  - Identifier: ScaleUpTimer
    Method: Timer
    Params:
      action: start
      label: scale-up-time
  - Identifier: PodStartupLatency
    Method: PodStartupLatency
    Params:
      action: start
      labelSelector: group = {{$POD_GROUP_LABEL}}
      #Set to a high number because default is 5s. Will fail if p99 > 5s
      threshold: 1h
  - Identifier: SchedulingThroughput
    Method: SchedulingThroughput
    Params:
      action: start
      labelSelector: group = {{$POD_GROUP_LABEL}}

- name: create-pods
  phases:
    - namespaceRange:
        min: 1
        max: {{ $NAMESPACES }}
      replicasPerNamespace: {{ $DEPLOYMENTS_PER_NAMESPACE }}
      tuningSet: uniform-qps
      objectBundle:
        - basename: deployment
          objectTemplatePath: deployment.yaml
          templateFillMap:
            Replicas: {{ $PODS_PER_DEPLOYMENT }}
            Group: {{ $POD_GROUP_LABEL }}
            CpuRequest: {{ $POD_CPU_MILLIS }}m
            MemoryRequest: {{ $POD_MEMORY_MB }}Mi

- name: wait-for-scale-up-complete
  measurements:
    - Identifier: WaitingForPods
      Method: WaitForRunningPods
      Params:
        labelSelector: group = {{ $POD_GROUP_LABEL }}
        desiredPodCount: {{ $TOTAL_PODS }}
        timeout: 4h

- name: stop-timing-for-scale-up
  measurements:
    - Identifier: ScaleUpTimer
      Method: Timer
      Params:
        action: stop
        label: scale-up-time
    - Identifier: PodStartupLatency
      Method: PodStartupLatency
      Params:
        action: gather

- name: start-timing-for-scale-down
  measurements:
    - Identifier: ScaleDownTimer
      Method: Timer
      Params:
        action: start
        label: scale-down-time

- name: delete-pods
  phases:
    - namespaceRange:
        min: 1
        max: {{ $NAMESPACES }}
      replicasPerNamespace: 0
      tuningSet: uniform-qps
      objectBundle:
        - basename: deployment
          objectTemplatePath: deployment.yaml

- name: wait-for-scale-down-complete
  measurements:
    - Identifier: WaitingForPods
      Method: WaitForRunningPods
      Params:
        labelSelector: group = {{ $POD_GROUP_LABEL }}
        desiredPodCount: 0
        timeout: 4h

- name: stop-measurements
  measurements:
    - Identifier: ScaleDownTimer
      Method: Timer
      Params:
        action: stop
        label: scale-down-time
    - Identifier: SchedulingThroughput
      Method: SchedulingThroughput
      Params:
        action: gather
        labelSelector: group = {{ $POD_GROUP_LABEL }}