{
  "variables": {
    "region": "us-west-2",
    "flag": "ul18-base",
    "subnet_id": "subnet-4532e73d",
    "security_groupids": "sg-092c73af8b33e6bab",
    "build_ami": "ami-090717c950a5c34d3",
    "habanalabs-driver-version": "1.1.0-614"
  },
  "builders": [{
    "type": "amazon-ebs",
    "region": "{{user `region`}}",
    "source_ami": "{{user `build_ami`}}",
    "run_tags": {
        "Name": "packer-dl1-processor-{{user `flag`}}"
    },
    "subnet_id": "{{user `subnet_id`}}",
    "security_group_ids": "{{user `security_groupids`}}",
    "instance_type": "dl1.24xlarge",
    "ssh_username": "ubuntu",
    "ami_name": "ul18-habana-dl1-processor_{{user `flag`}}-{{timestamp}}",
    "launch_block_device_mappings":[{
      "delete_on_termination": true,
      "device_name": "/dev/sda1",
      "volume_size": 200,
      "throughput": 1000,
      "iops": 8000,
      "volume_type": "gp3"
    }]
  }],
  "provisioners": [{
    "type": "shell",
    "expect_disconnect": true,
    "inline": [
      "sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt -y -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' upgrade -y",
      "sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y linux-aws git libelf-dev libtool autoconf cmake nasm unzip python3 python3-distutils python3-wheel python3-dev python3-numpy pigz parallel nfs-common build-essential hwloc libjemalloc1 libnuma-dev numactl libjemalloc-dev preload ulatency ulatencyd htop iftop liblapack-dev libgfortran3 ipcalc wget curl dkms",
      "echo 'GRUB_CMDLINE_LINUX='intel_idle.max_cstate=1'' | sudo tee -a /etc/default/grub",
      "curl https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py && sudo python3 /tmp/get-pip.py",
      "wget -O - https://fsx-lustre-client-repo-public-keys.s3.amazonaws.com/fsx-ubuntu-public-key.asc | sudo apt-key add -",
      "echo 'deb https://fsx-lustre-client-repo.s3.amazonaws.com/ubuntu bionic main' | sudo tee /etc/apt/sources.list.d/fsxlustreclientrepo.list",
      "uname -r",
      "sudo apt update",
      "sudo systemctl disable unattended-upgrades.service",
      "sudo systemctl disable ondemand.service",
      "sudo ufw disable",
      "sudo shutdown -r now"]
    },
    {
    "type": "shell",
    "inline_shebang":"/bin/bash -xe",
    "inline": [
      "uname -r",
      "sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y lustre-client-modules-$(uname -r) lustre-client-modules-aws"]
    },
    {
    "type": "shell",
    "inline": [
      "uname -r",
      "wget -O /tmp/awscli2.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip",
      "cd /tmp && sudo unzip /tmp/awscli2.zip",
      "sudo /tmp/aws/install",
      "aws configure set default.s3.max_concurrent_requests 100",
      "aws configure set default.s3.max_queue_size 10000",
      "aws configure set default.s3.multipart_threshold 64MB",
      "aws configure set default.s3.multipart_chunksize 16MB"]
    },
    {
    "type": "shell",
    "inline": [
      "echo 'net.core.default_qdisc = fq' | sudo tee -a /etc/sysctl.conf",
      "echo 'net.ipv4.tcp_congestion_control = bbr' | sudo tee -a /etc/sysctl.conf",
      "echo 'net.ipv4.tcp_timestamps = 0' | sudo tee -a /etc/sysctl.conf",
      "echo 'net.core.rmem_max = 67108864' | sudo tee -a /etc/sysctl.conf",
      "echo 'net.core.wmem_max = 67108864' | sudo tee -a /etc/sysctl.conf",
      "echo 'net.ipv4.tcp_rmem = 4096 87380 67108864' | sudo tee -a /etc/sysctl.conf",
      "echo 'net.ipv4.tcp_wmem = 4096 65536 67108864' | sudo tee -a /etc/sysctl.conf",
      "echo 'net.core.netdev_max_backlog = 30000' | sudo tee -a /etc/sysctl.conf",
      "echo 'net.core.rmem_default = 67108864' | sudo tee -a /etc/sysctl.conf",
      "echo 'net.core.wmem_default = 67108864' | sudo tee -a /etc/sysctl.conf",
      "echo 'net.ipv4.tcp_mem = 67108864 67108864 67108864' | sudo tee -a /etc/sysctl.conf",
      "echo 'net.ipv4.route.flush = 1' | sudo tee -a /etc/sysctl.conf",
      "sudo sysctl -p"]
    },
        {
    "type": "shell",
    "inline_shebang": "/bin/bash -xe",
    "inline": [
      "sudo apt install linux-headers-$(uname -r) -y",
      "curl -X GET https://vault.habana.ai/artifactory/api/gpg/key/public | sudo apt-key add -",
      "echo 'deb https://vault.habana.ai/artifactory/debian bionic main' | sudo tee /etc/apt/sources.list.d/habanalabs.list",
      "sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install habanalabs-container-runtime={{user `habanalabs-driver-version`}} habanalabs-qual={{user `habanalabs-driver-version`}} habanalabs-aeon={{user `habanalabs-driver-version`}} habanalabs-dkms={{user `habanalabs-driver-version`}} habanalabs-thunk={{user `habanalabs-driver-version`}} habanalabs-firmware={{user `habanalabs-driver-version`}} habanalabs-firmware-tools={{user `habanalabs-driver-version`}} habanalabs-graph={{user `habanalabs-driver-version`}} -y",
      "echo 'vm.nr_hugepages=15000' | sudo tee -a /etc/sysctl.conf",
      "sudo sysctl -p"]
    },
    {
    "type": "shell",
    "inline_shebang": "/bin/bash -xe",
    "inline": [
      "echo '    StrictHostKeyChecking no' | sudo tee -a /etc/ssh/ssh_config",
      "echo '    HostbasedAuthentication no' | sudo tee -a /etc/ssh/ssh_config",
      "echo '    CheckHostIP no' | sudo tee -a /etc/ssh/ssh_config"]
    },
    {
    "type": "shell",
    "inline_shebang": "/bin/bash -xe",
    "inline": [
      "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -",
      "sudo add-apt-repository 'deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable'",
      "distribution=$(. /etc/os-release;echo $ID$VERSION_ID)",
      "sudo apt-get update && sudo apt-get install -y docker-ce docker-ce-cli containerd.io",
      "sudo systemctl enable docker",
      "echo -e '{\"default-ulimits\":{\"memlock\":{\"Name\":\"memlock\",\"Soft\":-1,\"Hard\":-1}},\"default-runtime\":\"habana\",\"runtimes\":{\"habana\":{\"path\":\"/usr/bin/habana-container-runtime\",\"runtimeArgs\":[]}}}' | sudo tee /etc/docker/daemon.json",
      "sudo systemctl restart docker",
      "sudo usermod -aG docker $USER"]
    }
  ]
}