apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: cluster.x-k8s.io/cluster-name: {{.clusterName}} name: {{.clusterName}} namespace: {{.eksaSystemNamespace}} spec: clusterNetwork: pods: cidrBlocks: {{.podCidrs}} services: cidrBlocks: {{.serviceCidrs}} controlPlaneEndpoint: host: {{.controlPlaneEndpointHost}} port: {{.controlPlaneEndpointPort}} controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane name: {{.clusterName}} infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta3 kind: CloudStackCluster name: {{.clusterName}} {{- if .externalEtcd }} managedExternalEtcdRef: apiVersion: etcdcluster.cluster.x-k8s.io/v1beta1 kind: EtcdadmCluster name: {{.clusterName}}-etcd {{- end }} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta3 kind: CloudStackCluster metadata: name: {{.clusterName}} namespace: {{.eksaSystemNamespace}} spec: controlPlaneEndpoint: host: {{.controlPlaneEndpointHost}} port: {{.controlPlaneEndpointPort}} failureDomains:{{ range $az := .cloudstackAvailabilityZones }} - name: {{ $az.Name }} zone: id: {{ $az.Zone.Id }} name: {{ $az.Zone.Name }} network: id: {{ $az.Zone.Network.Id }} name: {{ $az.Zone.Network.Name }} domain: {{ $az.Domain }} account: {{ $az.Account }} acsEndpoint: name: {{ $az.CredentialsRef }} namespace: eksa-system {{- end }} --- apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane metadata: name: {{.clusterName}} namespace: {{.eksaSystemNamespace}} spec: machineTemplate: infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta3 kind: CloudStackMachineTemplate name: {{.controlPlaneTemplateName}} kubeadmConfigSpec: clusterConfiguration: imageRepository: {{.kubernetesRepository}} etcd: {{- if .externalEtcd }} external: endpoints: [] caFile: "/etc/kubernetes/pki/etcd/ca.crt" certFile: "/etc/kubernetes/pki/apiserver-etcd-client.crt" keyFile: "/etc/kubernetes/pki/apiserver-etcd-client.key" {{- else }} local: imageRepository: {{.etcdRepository}} imageTag: {{.etcdImageTag}} {{- if .etcdExtraArgs }} extraArgs: {{ .etcdExtraArgs.ToYaml | indent 12 }} {{- end }} {{- end }} dns: imageRepository: {{.corednsRepository}} imageTag: {{.corednsVersion}} apiServer: extraArgs: cloud-provider: external audit-policy-file: /etc/kubernetes/audit-policy.yaml audit-log-path: /var/log/kubernetes/api-audit.log audit-log-maxage: "30" audit-log-maxbackup: "10" audit-log-maxsize: "512" profiling: "false" {{- if .apiserverExtraArgs }} {{ .apiserverExtraArgs.ToYaml | indent 10 }} {{- end }} extraVolumes: - hostPath: /etc/kubernetes/audit-policy.yaml mountPath: /etc/kubernetes/audit-policy.yaml name: audit-policy pathType: File readOnly: true - hostPath: /var/log/kubernetes mountPath: /var/log/kubernetes name: audit-log-dir pathType: DirectoryOrCreate readOnly: false {{- if .awsIamAuth}} - hostPath: /var/lib/kubeadm/aws-iam-authenticator/ mountPath: /etc/kubernetes/aws-iam-authenticator/ name: authconfig readOnly: false - hostPath: /var/lib/kubeadm/aws-iam-authenticator/pki/ mountPath: /var/aws-iam-authenticator/ name: awsiamcert readOnly: false {{- end}} controllerManager: extraArgs: cloud-provider: external profiling: "false" {{- if .controllermanagerExtraArgs }} {{ .controllermanagerExtraArgs.ToYaml | indent 10 }} {{- end }} scheduler: extraArgs: profiling: "false" {{- if .schedulerExtraArgs }} {{ .schedulerExtraArgs.ToYaml | indent 10 }} {{- end }} files: {{- if .cloudstackKubeVip}} - content: | apiVersion: v1 kind: Pod metadata: creationTimestamp: null name: kube-vip namespace: kube-system spec: containers: - args: - manager env: - name: vip_arp value: "true" - name: port value: "6443" - name: vip_cidr value: "32" - name: cp_enable value: "true" - name: cp_namespace value: kube-system - name: vip_ddns value: "false" - name: vip_leaderelection value: "true" - name: vip_leaseduration value: "15" - name: vip_renewdeadline value: "10" - name: vip_retryperiod value: "2" - name: address value: {{.controlPlaneEndpointHost}} image: {{.kubeVipImage}} imagePullPolicy: IfNotPresent name: kube-vip resources: {} securityContext: capabilities: add: - NET_ADMIN - NET_RAW volumeMounts: - mountPath: /etc/kubernetes/admin.conf name: kubeconfig hostNetwork: true volumes: - hostPath: path: /etc/kubernetes/admin.conf name: kubeconfig status: {} owner: root:root path: /etc/kubernetes/manifests/kube-vip.yaml {{- end}} - content: | {{ .auditPolicy | indent 8 }} owner: root:root path: /etc/kubernetes/audit-policy.yaml {{- if .proxyConfig }} - content: | [Service] Environment="HTTP_PROXY={{.httpProxy}}" Environment="HTTPS_PROXY={{.httpsProxy}}" Environment="NO_PROXY={{ stringsJoin .noProxy "," }}" owner: root:root path: /etc/systemd/system/containerd.service.d/http-proxy.conf {{- end }} {{- if .registryCACert }} - content: | {{ .registryCACert | indent 8 }} owner: root:root path: "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" {{- end }} {{- if .registryMirrorMap }} - content: | [plugins."io.containerd.grpc.v1.cri".registry.mirrors] {{- range $orig, $mirror := .registryMirrorMap }} [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ $orig }}"] endpoint = ["https://{{ $mirror }}"] {{- end }} {{- if or .registryCACert .insecureSkip }} [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".tls] {{- if .registryCACert }} ca_file = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" {{- end }} {{- if .insecureSkip }} insecure_skip_verify = {{.insecureSkip}} {{- end }} {{- end }} owner: root:root path: "/etc/containerd/config_append.toml" {{- end }} {{- if .awsIamAuth}} - content: | # clusters refers to the remote service. clusters: - name: aws-iam-authenticator cluster: certificate-authority: /var/aws-iam-authenticator/cert.pem server: https://localhost:21362/authenticate # users refers to the API Server's webhook configuration # (we don't need to authenticate the API server). users: - name: apiserver # kubeconfig files require a context. Provide one for the API Server. current-context: webhook contexts: - name: webhook context: cluster: aws-iam-authenticator user: apiserver permissions: "0640" owner: root:root path: /var/lib/kubeadm/aws-iam-authenticator/kubeconfig.yaml - contentFrom: secret: name: {{.clusterName}}-aws-iam-authenticator-ca key: cert.pem permissions: "0640" owner: root:root path: /var/lib/kubeadm/aws-iam-authenticator/pki/cert.pem - contentFrom: secret: name: {{.clusterName}}-aws-iam-authenticator-ca key: key.pem permissions: "0640" owner: root:root path: /var/lib/kubeadm/aws-iam-authenticator/pki/key.pem {{- end}} initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock kubeletExtraArgs: provider-id: cloudstack:///'{{`{{ ds.meta_data.instance_id }}`}}' read-only-port: "0" anonymous-auth: "false" {{- if .kubeletExtraArgs }} {{ .kubeletExtraArgs.ToYaml | indent 10 }} {{- end }} name: "{{`{{ ds.meta_data.hostname }}`}}" {{- if .controlPlaneTaints }} taints: {{- range .controlPlaneTaints}} - key: {{ .Key }} value: {{ .Value }} effect: {{ .Effect }} {{- if .TimeAdded }} timeAdded: {{ .TimeAdded }} {{- end }} {{- end }} {{- end }} joinConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock kubeletExtraArgs: provider-id: cloudstack:///'{{`{{ ds.meta_data.instance_id }}`}}' read-only-port: "0" anonymous-auth: "false" {{- if .kubeletExtraArgs }} {{ .kubeletExtraArgs.ToYaml | indent 10 }} {{- end }} name: "{{`{{ ds.meta_data.hostname }}`}}" {{- if .controlPlaneTaints }} taints: {{- range .controlPlaneTaints}} - key: {{ .Key }} value: {{ .Value }} effect: {{ .Effect }} {{- if .TimeAdded }} timeAdded: {{ .TimeAdded }} {{- end }} {{- end }} {{- end }} preKubeadmCommands: - swapoff -a {{- if.registryMirrorMap }} - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml {{- end }} {{- if or .proxyConfig .registryMirrorMap }} - sudo systemctl daemon-reload - sudo systemctl restart containerd {{- end }} - hostname "{{`{{ ds.meta_data.hostname }}`}}" - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts - echo "127.0.0.1 localhost" >>/etc/hosts - echo "127.0.0.1 {{`{{ ds.meta_data.hostname }}`}}" >>/etc/hosts - echo "{{`{{ ds.meta_data.hostname }}`}}" >/etc/hostname {{- range $dir, $target := .cloudstackControlPlaneSymlinks}} - >- if [ ! -L {{$dir}} ] ; then mv {{$dir}} {{$dir}}-$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10) ; mkdir -p {{$target}} && ln -s {{$target}} {{$dir}} ; else echo "{{$dir}} already symlnk"; fi {{- end}} {{- if .cloudstackControlPlaneDiskOfferingProvided }} diskSetup: filesystems: - device: {{ .cloudstackControlPlaneDiskOfferingDevice }}1 overwrite: false extraOpts: - -E - lazy_itable_init=1,lazy_journal_init=1 filesystem: {{ .cloudstackControlPlaneDiskOfferingFilesystem }} label: {{ .cloudstackControlPlaneDiskOfferingLabel }} partitions: - device: {{ .cloudstackControlPlaneDiskOfferingDevice }} layout: true overwrite: false tableType: gpt mounts: - - LABEL={{ .cloudstackControlPlaneDiskOfferingLabel }} - {{ .cloudstackControlPlaneDiskOfferingPath }} {{- end }} useExperimentalRetryJoin: true users: - name: {{.controlPlaneSshUsername}} sshAuthorizedKeys: - '{{.cloudstackControlPlaneSshAuthorizedKey}}' sudo: ALL=(ALL) NOPASSWD:ALL format: {{.format}} replicas: {{.controlPlaneReplicas}} {{- if .upgradeRolloutStrategy }} rolloutStrategy: rollingUpdate: maxSurge: {{.maxSurge}} {{- end }} version: {{.kubernetesVersion}} {{- if .externalEtcd }} --- kind: EtcdadmCluster apiVersion: etcdcluster.cluster.x-k8s.io/v1beta1 metadata: name: {{.clusterName}}-etcd namespace: {{.eksaSystemNamespace}} spec: replicas: {{.externalEtcdReplicas}} etcdadmConfigSpec: etcdadmBuiltin: false format: {{.format}} cloudInitConfig: version: {{.externalEtcdVersion}} installDir: "/usr/bin" etcdadmInstallCommands: - echo this line exists so that etcdadmInstallCommands is not empty - echo etcdadmInstallCommands can be removed once etcdadm bootstrap and controller fix the bug - echo that preEtcdadmCommands not run unless etcdadmBuiltin is false - echo https://github.com/mrajashree/etcdadm-bootstrap-provider/issues/13 preEtcdadmCommands: - swapoff -a - hostname "{{`{{ ds.meta_data.hostname }}`}}" - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts - echo "127.0.0.1 localhost" >>/etc/hosts - echo "127.0.0.1 {{`{{ ds.meta_data.hostname }}`}}" >>/etc/hosts - echo "{{`{{ ds.meta_data.hostname }}`}}" >/etc/hostname {{- if .cloudstackEtcdDiskOfferingProvided }} - >- echo "type=83" | sfdisk {{ .cloudstackEtcdDiskOfferingDevice }} && mkfs -t {{ .cloudstackEtcdDiskOfferingFilesystem }} {{ .cloudstackEtcdDiskOfferingDevice }}1 && mkdir -p {{ .cloudstackEtcdDiskOfferingPath }} && echo {{ .cloudstackEtcdDiskOfferingDevice }}1 {{ .cloudstackEtcdDiskOfferingPath }} {{ .cloudstackEtcdDiskOfferingFilesystem }} defaults 0 2 >> /etc/fstab && mount {{ .cloudstackEtcdDiskOfferingPath }} {{- end }} {{- range $dir, $target := .cloudstackEtcdSymlinks}} - >- if [ ! -L {{$dir}} ] ; then mv {{$dir}} {{$dir}}-$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10) ; mkdir -p {{$target}} && ln -s {{$target}} {{$dir}} ; else echo "{{$dir}} already symlnk" ; fi {{- end}} {{- if .etcdCipherSuites }} cipherSuites: {{.etcdCipherSuites}} {{- end }} users: - name: {{.etcdSshUsername}} sshAuthorizedKeys: - '{{.cloudstackEtcdSshAuthorizedKey}}' sudo: ALL=(ALL) NOPASSWD:ALL {{- if .proxyConfig }} proxy: httpProxy: {{ .httpProxy }} httpsProxy: {{ .httpsProxy }} noProxy: {{ range .noProxy }} - {{ . }} {{- end }} {{- end }} {{- if .registryMirrorMap }} registryMirror: endpoint: {{ .publicMirror }} {{- if .registryCACert }} caCert: | {{ .registryCACert | indent 8 }} {{- end }} {{- end }} infrastructureTemplate: apiVersion: infrastructure.cluster.x-k8s.io/v1beta3 kind: CloudStackMachineTemplate name: {{.etcdTemplateName}} {{- end }} ---