// package kubecost installs Kubecost OS helm chart. // ref. https://www.kubecost.com/ package kubecost import ( "context" "errors" "fmt" "io" "path" "reflect" "strings" "time" "github.com/aws/aws-k8s-tester/client" "github.com/aws/aws-k8s-tester/k8s-tester/helm" k8s_tester "github.com/aws/aws-k8s-tester/k8s-tester/tester" "github.com/aws/aws-k8s-tester/utils/rand" utils_time "github.com/aws/aws-k8s-tester/utils/time" "github.com/manifoldco/promptui" "go.uber.org/zap" "k8s.io/utils/exec" ) type Config struct { Enable bool `json:"enable"` Prompt bool `json:"-"` Stopc chan struct{} `json:"-"` Logger *zap.Logger `json:"-"` LogWriter io.Writer `json:"-"` Client client.Client `json:"-"` // MinimumNodes is the minimum number of Kubernetes nodes required for installing this addon. MinimumNodes int `json:"minimum_nodes"` // HelmChartRepoURL is the helm chart repo URL. HelmChartRepoURL string `json:"helm_chart_repo_url"` // Namespace to create test resources. Namespace string `json:"namespace"` } func (cfg *Config) ValidateAndSetDefaults() error { if cfg.MinimumNodes == 0 { cfg.MinimumNodes = DefaultMinimumNodes } if cfg.Namespace == "" { return errors.New("empty Namespace") } return nil } const chartName = "cost-analyzer" const ( DefaultMinimumNodes int = 1 // https://github.com/kubecost/cost-analyzer-helm-chart/releases/download/v1.83.2/ // https://github.com/kubecost/cost-analyzer-helm-chart/archive/refs/tags/v1.83.2.tar.gz DefaultHelmChartRepoURL = "https://kubecost.github.io/cost-analyzer/" ) func NewDefault() *Config { return &Config{ Enable: false, Prompt: false, MinimumNodes: DefaultMinimumNodes, HelmChartRepoURL: DefaultHelmChartRepoURL, Namespace: pkgName + "-" + rand.String(10) + "-" + utils_time.GetTS(10), } } func New(cfg *Config) k8s_tester.Tester { return &tester{ cfg: cfg, } } type tester struct { cfg *Config } var pkgName = path.Base(reflect.TypeOf(tester{}).PkgPath()) func Env() string { return "ADD_ON_" + strings.ToUpper(strings.Replace(pkgName, "-", "_", -1)) } func (ts *tester) Name() string { return pkgName } func (ts *tester) Enabled() bool { return ts.cfg.Enable } func (ts *tester) Apply() error { if ok := ts.runPrompt("apply"); !ok { return errors.New("cancelled") } if nodes, err := client.ListNodes(ts.cfg.Client.KubernetesClient()); len(nodes) < ts.cfg.MinimumNodes || err != nil { return fmt.Errorf("failed to validate minimum nodes requirement %d (nodes %v, error %v)", ts.cfg.MinimumNodes, len(nodes), err) } if err := client.CreateNamespace(ts.cfg.Logger, ts.cfg.Client.KubernetesClient(), ts.cfg.Namespace); err != nil { return err } if err := ts.createHelmKubecost(); err != nil { return err } return nil } func (ts *tester) Delete() error { if ok := ts.runPrompt("delete"); !ok { return errors.New("cancelled") } var errs []string if err := ts.deleteHelmKubecost(); err != nil { errs = append(errs, err.Error()) } if err := client.DeleteNamespaceAndWait( ts.cfg.Logger, ts.cfg.Client.KubernetesClient(), ts.cfg.Namespace, client.DefaultNamespaceDeletionInterval, client.DefaultNamespaceDeletionTimeout, client.WithForceDelete(true), ); err != nil { errs = append(errs, fmt.Sprintf("failed to delete namespace (%v)", err)) } if len(errs) > 0 { return errors.New(strings.Join(errs, ", ")) } return nil } func (ts *tester) runPrompt(action string) (ok bool) { if ts.cfg.Prompt { msg := fmt.Sprintf("Ready to %q resources, should we continue?", action) prompt := promptui.Select{ Label: msg, Items: []string{ "No, cancel it!", fmt.Sprintf("Yes, let's %q!", action), }, } idx, answer, err := prompt.Run() if err != nil { panic(err) } if idx != 1 { fmt.Printf("cancelled %q [index %d, answer %q]\n", action, idx, answer) return false } } return true } // --set persistentVolume.enabled="false" --set prometheus.server.persistentVolume.enabled=false // https://github.com/kubecost/cost-analyzer-helm-chart/blob/develop/cost-analyzer/values.yaml func (ts *tester) createHelmKubecost() error { values := map[string]interface{}{ "persistentVolume": map[string]interface{}{ "enabled": false, }, "prometheus": map[string]interface{}{ "server": map[string]interface{}{ "persistentVolume": map[string]interface{}{ "enabled": false, }, }, }, } getAllArgs := []string{ ts.cfg.Client.Config().KubectlPath, "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath, "--namespace=" + ts.cfg.Namespace, "get", "all", } getAllCmd := strings.Join(getAllArgs, " ") descArgsDs := []string{ ts.cfg.Client.Config().KubectlPath, "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath, "--namespace=" + ts.cfg.Namespace, "describe", "daemonset.apps/kubecost-prometheus-node-exporter", } descCmdDs := strings.Join(descArgsDs, " ") descArgsPods := []string{ ts.cfg.Client.Config().KubectlPath, "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath, "--namespace=" + ts.cfg.Namespace, "describe", "pods", "--selector=app=cost-analyzer", } descCmdPods := strings.Join(descArgsPods, " ") logArgs := []string{ ts.cfg.Client.Config().KubectlPath, "--kubeconfig=" + ts.cfg.Client.Config().KubeconfigPath, "--namespace=" + ts.cfg.Namespace, "logs", "--selector=app=cost-analyzer", "--all-containers=true", "--timestamps", } logsCmd := strings.Join(logArgs, " ") return helm.Install(helm.InstallConfig{ Logger: ts.cfg.Logger, LogWriter: ts.cfg.LogWriter, Stopc: ts.cfg.Stopc, Timeout: 10 * time.Minute, KubeconfigPath: ts.cfg.Client.Config().KubeconfigPath, Namespace: ts.cfg.Namespace, ChartRepoURL: ts.cfg.HelmChartRepoURL, ChartName: chartName, ReleaseName: chartName, Values: values, LogFunc: func(format string, v ...interface{}) { ts.cfg.Logger.Info(fmt.Sprintf("[install] "+format, v...)) }, QueryFunc: func() { ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) output, err := exec.New().CommandContext(ctx, getAllArgs[0], getAllArgs[1:]...).CombinedOutput() cancel() out := strings.TrimSpace(string(output)) if err != nil { ts.cfg.Logger.Warn("'kubectl get all' failed", zap.Error(err)) ts.cfg.Logger.Warn("Kubecost Tests::", zap.String("TEST", "FAILED")) } fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", getAllCmd, out) ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second) output, err = exec.New().CommandContext(ctx, descArgsDs[0], descArgsDs[1:]...).CombinedOutput() cancel() out = strings.TrimSpace(string(output)) if err != nil { ts.cfg.Logger.Warn("'kubectl describe daemonset' failed", zap.Error(err)) ts.cfg.Logger.Warn("Kubecost Tests::", zap.String("TEST", "FAILED")) } fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmdDs, out) ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second) output, err = exec.New().CommandContext(ctx, descArgsPods[0], descArgsPods[1:]...).CombinedOutput() cancel() out = strings.TrimSpace(string(output)) if err != nil { ts.cfg.Logger.Warn("'kubectl describe pods' failed", zap.Error(err)) ts.cfg.Logger.Warn("Kubecost Tests::", zap.String("TEST", "FAILED")) } fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", descCmdPods, out) ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second) output, err = exec.New().CommandContext(ctx, logArgs[0], logArgs[1:]...).CombinedOutput() cancel() out = strings.TrimSpace(string(output)) if err != nil { ts.cfg.Logger.Warn("'kubectl logs' failed", zap.Error(err)) } fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", logsCmd, out) }, QueryInterval: 30 * time.Second, }) } func (ts *tester) deleteHelmKubecost() error { return helm.Uninstall(helm.InstallConfig{ Logger: ts.cfg.Logger, LogWriter: ts.cfg.LogWriter, Timeout: 15 * time.Minute, KubeconfigPath: ts.cfg.Client.Config().KubeconfigPath, Namespace: ts.cfg.Namespace, ChartName: chartName, ReleaseName: chartName, }) }